From 50b997ac91d80c9f97aec9767a21c16fbebebb4a Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 30 Oct 2025 19:46:49 -0700 Subject: [PATCH 01/10] remove unused parameters --- pkg/gateway/model/mock_tg_builder.go | 3 +-- pkg/gateway/model/model_build_listener.go | 8 ++++---- pkg/gateway/model/model_build_listener_test.go | 2 +- pkg/gateway/model/model_build_target_group.go | 12 ++++++------ pkg/gateway/model/model_build_target_group_test.go | 4 ++-- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/pkg/gateway/model/mock_tg_builder.go b/pkg/gateway/model/mock_tg_builder.go index 08ec381a3..74b7e02ab 100644 --- a/pkg/gateway/model/mock_tg_builder.go +++ b/pkg/gateway/model/mock_tg_builder.go @@ -2,7 +2,6 @@ package model import ( "k8s.io/apimachinery/pkg/util/intstr" - elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" @@ -16,7 +15,7 @@ type mockTargetGroupBuilder struct { } func (m *mockTargetGroupBuilder) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { var tg *elbv2model.TargetGroup if len(m.tgs) > 0 { diff --git a/pkg/gateway/model/model_build_listener.go b/pkg/gateway/model/model_build_listener.go index 7ebe37e7e..af83c42c7 100644 --- a/pkg/gateway/model/model_build_listener.go +++ b/pkg/gateway/model/model_build_listener.go @@ -71,7 +71,7 @@ func (l listenerBuilderImpl) buildListeners(ctx context.Context, stack core.Stac // build rules only for L7 gateways if l.loadBalancerType == elbv2model.LoadBalancerTypeApplication { - secretKeys, err := l.buildListenerRules(ctx, stack, ls, lb.Spec.IPAddressType, gw, port, lbCfg, routes) + secretKeys, err := l.buildListenerRules(ctx, stack, ls, lb.Spec.IPAddressType, gw, port, routes) if err != nil { return nil, err } @@ -177,7 +177,7 @@ func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core return nil, nil } - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, lbCfg, lb.Spec.IPAddressType, routeDescriptor, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, lb.Spec.IPAddressType, routeDescriptor, backend) if tgErr != nil { return &elbv2model.ListenerSpec{}, tgErr } @@ -185,7 +185,7 @@ func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core return listenerSpec, nil } -func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core.Stack, ls *elbv2model.Listener, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, port int32, lbCfg elbv2gw.LoadBalancerConfiguration, routes map[int32][]routeutils.RouteDescriptor) ([]types.NamespacedName, error) { +func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core.Stack, ls *elbv2model.Listener, ipAddressType elbv2model.IPAddressType, gw *gwv1.Gateway, port int32, routes map[int32][]routeutils.RouteDescriptor) ([]types.NamespacedName, error) { // sort all rules based on precedence rulesWithPrecedenceOrder := routeutils.SortAllRulesByPrecedence(routes[port], port) secrets := make([]types.NamespacedName, 0) @@ -222,7 +222,7 @@ func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core. } targetGroupTuples := make([]elbv2model.TargetGroupTuple, 0, len(rule.GetBackends())) for _, backend := range rule.GetBackends() { - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, lbCfg, ipAddressType, route, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, ipAddressType, route, backend) if tgErr != nil { return nil, tgErr } diff --git a/pkg/gateway/model/model_build_listener_test.go b/pkg/gateway/model/model_build_listener_test.go index 61f81eb00..00d413843 100644 --- a/pkg/gateway/model/model_build_listener_test.go +++ b/pkg/gateway/model/model_build_listener_test.go @@ -1718,7 +1718,7 @@ func Test_BuildListenerRules(t *testing.T) { Spec: elbv2model.ListenerSpec{ Protocol: tc.listenerProtocol, }, - }, tc.ipAddressType, &gwv1.Gateway{}, tc.port, elbv2gw.LoadBalancerConfiguration{}, tc.routes) + }, tc.ipAddressType, &gwv1.Gateway{}, tc.port, tc.routes) assert.NoError(t, err) var resLRs []*elbv2model.ListenerRule diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go index a96cfc638..062287856 100644 --- a/pkg/gateway/model/model_build_target_group.go +++ b/pkg/gateway/model/model_build_target_group.go @@ -33,7 +33,7 @@ type buildTargetGroupOutput struct { type targetGroupBuilder interface { buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) + gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) } type targetGroupBuilderImpl struct { @@ -101,10 +101,10 @@ func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper } func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { if backend.ServiceBackend != nil { - tg, err := builder.buildTargetGroupFromService(stack, gw, lbConfig, lbIPType, routeDescriptor, *backend.ServiceBackend) + tg, err := builder.buildTargetGroupFromService(stack, gw, lbIPType, routeDescriptor, *backend.ServiceBackend) if err != nil { return nil, err } @@ -120,14 +120,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, } func (builder *targetGroupBuilderImpl) buildTargetGroupFromService(stack core.Stack, - gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.ServiceBackendConfig) (*elbv2model.TargetGroup, error) { + gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.ServiceBackendConfig) (*elbv2model.TargetGroup, error) { targetGroupProps := backendConfig.ELBV2TargetGroupProps tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backendConfig.Service), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.ServicePort.TargetPort) if tg, exists := builder.tgByResID[tgResID]; exists { return tg, nil } - tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbConfig, lbIPType, backendConfig, targetGroupProps) + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, backendConfig, targetGroupProps) if err != nil { return nil, err } @@ -216,7 +216,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate }, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, backendConfig routeutils.ServiceBackendConfig, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { +func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbIPType elbv2model.IPAddressType, backendConfig routeutils.ServiceBackendConfig, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { targetType := builder.buildTargetGroupTargetType(targetGroupProps) tgProtocol, err := builder.buildTargetGroupProtocol(targetGroupProps, route) if err != nil { diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index 387e13346..9c4afeedb 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -313,7 +313,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { builder := newTargetGroupBuilder("my-cluster", "vpc-xxx", tagger, tc.lbType, &mockTargetGroupBindingNetworkingBuilder{}, gateway.NewTargetGroupConfigConstructor(), tc.defaultTargetType, nil) - out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(tc.gateway, tc.route, elbv2gw.LoadBalancerConfiguration{}, elbv2model.IPAddressTypeIPV4, tc.backend, nil) + out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(tc.gateway, tc.route, elbv2model.IPAddressTypeIPV4, tc.backend, nil) if tc.expectErr { assert.Error(t, err) return @@ -1887,7 +1887,7 @@ func Test_buildTargetGroupTags(t *testing.T) { } } - tgSpec, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(gateway, route, elbv2gw.LoadBalancerConfiguration{}, elbv2model.IPAddressTypeIPV4, backend, tgProps) + tgSpec, err := builder.(*targetGroupBuilderImpl).buildTargetGroupSpec(gateway, route, elbv2model.IPAddressTypeIPV4, backend, tgProps) if tc.expectErr { assert.Error(t, err) From 0819b451702362d6e2772b8259f94cccdb5cb312 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Thu, 30 Oct 2025 23:19:21 -0700 Subject: [PATCH 02/10] refactor service interactions into its own struct --- controllers/gateway/utils.go | 4 +- .../model/model_build_listener_test.go | 17 +- pkg/gateway/model/model_build_target_group.go | 96 +---- .../model/model_build_target_group_test.go | 344 +++--------------- pkg/gateway/routeutils/backend.go | 32 +- pkg/gateway/routeutils/backend_literal.go | 6 + pkg/gateway/routeutils/backend_service.go | 108 ++++++ .../routeutils/backend_service_test.go | 245 +++++++++++++ pkg/gateway/routeutils/backend_test.go | 8 +- 9 files changed, 471 insertions(+), 389 deletions(-) create mode 100644 pkg/gateway/routeutils/backend_literal.go create mode 100644 pkg/gateway/routeutils/backend_service.go create mode 100644 pkg/gateway/routeutils/backend_service_test.go diff --git a/controllers/gateway/utils.go b/controllers/gateway/utils.go index 73ddab7ff..78a396c94 100644 --- a/controllers/gateway/utils.go +++ b/controllers/gateway/utils.go @@ -9,8 +9,6 @@ import ( "time" "unicode/utf8" - "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" - "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -184,7 +182,7 @@ func getServicesFromRoutes(listenerRouteMap map[int32][]routeutils.RouteDescript for _, rr := range route.GetAttachedRules() { for _, be := range rr.GetBackends() { if be.ServiceBackend != nil { - res.Insert(k8s.NamespacedName(be.ServiceBackend.Service)) + res.Insert(be.ServiceBackend.GetBackendNamespacedName()) } } } diff --git a/pkg/gateway/model/model_build_listener_test.go b/pkg/gateway/model/model_build_listener_test.go index 00d413843..e0738b4f5 100644 --- a/pkg/gateway/model/model_build_listener_test.go +++ b/pkg/gateway/model/model_build_listener_test.go @@ -7,7 +7,6 @@ import ( elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "reflect" @@ -1319,8 +1318,8 @@ func Test_BuildListenerRules(t *testing.T) { BackendRefs: []routeutils.Backend{ { ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, + //Service: &corev1.Service{}, + //ServicePort: &corev1.ServicePort{Name: "svcport"}, }, Weight: 1, }, @@ -1392,8 +1391,8 @@ func Test_BuildListenerRules(t *testing.T) { BackendRefs: []routeutils.Backend{ { ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, + //Service: &corev1.Service{}, + //ServicePort: &corev1.ServicePort{Name: "svcport"}, }, Weight: 1, }, @@ -1452,8 +1451,8 @@ func Test_BuildListenerRules(t *testing.T) { BackendRefs: []routeutils.Backend{ { ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, + //Service: &corev1.Service{}, + //ServicePort: &corev1.ServicePort{Name: "svcport"}, }, Weight: 1, }, @@ -1527,8 +1526,8 @@ func Test_BuildListenerRules(t *testing.T) { BackendRefs: []routeutils.Backend{ { ServiceBackend: &routeutils.ServiceBackendConfig{ - Service: &corev1.Service{}, - ServicePort: &corev1.ServicePort{Name: "svcport"}, + //Service: &corev1.Service{}, + //ServicePort: &corev1.ServicePort{Name: "svcport"}, }, Weight: 1, }, diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go index 062287856..65469d89b 100644 --- a/pkg/gateway/model/model_build_target_group.go +++ b/pkg/gateway/model/model_build_target_group.go @@ -20,7 +20,6 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" elbv2modelk8s "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2/k8s" - "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_utils" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "strconv" @@ -121,13 +120,13 @@ func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, func (builder *targetGroupBuilderImpl) buildTargetGroupFromService(stack core.Stack, gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.ServiceBackendConfig) (*elbv2model.TargetGroup, error) { - targetGroupProps := backendConfig.ELBV2TargetGroupProps - tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backendConfig.Service), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.ServicePort.TargetPort) + targetGroupProps := backendConfig.GetTargetGroupProps() + tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), backendConfig.GetBackendNamespacedName(), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.GetIdentifierPort()) if tg, exists := builder.tgByResID[tgResID]; exists { return tg, nil } - tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, backendConfig, targetGroupProps) + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, &backendConfig, targetGroupProps) if err != nil { return nil, err } @@ -162,9 +161,9 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupFromStaticName(cfg routeu func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gateway, tgProps *elbv2gw.TargetGroupProps, tgSpec elbv2model.TargetGroupSpec, nodeSelector *metav1.LabelSelector, backendConfig routeutils.ServiceBackendConfig) (elbv2modelk8s.TargetGroupBindingResourceSpec, error) { targetType := elbv2api.TargetType(tgSpec.TargetType) - targetPort := backendConfig.ServicePort.TargetPort + targetPort := backendConfig.GetServicePort().TargetPort if targetType == elbv2api.TargetTypeInstance { - targetPort = intstr.FromInt32(backendConfig.ServicePort.NodePort) + targetPort = intstr.FromInt32(backendConfig.GetServicePort().NodePort) } tgbNetworking, err := builder.tgbNetworkBuilder.buildTargetGroupBindingNetworking(tgSpec, targetPort) if err != nil { @@ -193,7 +192,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate return elbv2modelk8s.TargetGroupBindingResourceSpec{ Template: elbv2modelk8s.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ - Namespace: backendConfig.Service.Namespace, + Namespace: backendConfig.GetBackendNamespacedName().Namespace, Name: tgSpec.Name, Annotations: annotations, Labels: labels, @@ -202,8 +201,8 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate TargetGroupARN: nil, // This should get filled in later! TargetType: &targetType, ServiceRef: elbv2api.ServiceReference{ - Name: backendConfig.Service.Name, - Port: intstr.FromInt32(backendConfig.ServicePort.Port), + Name: backendConfig.GetBackendNamespacedName().Name, + Port: intstr.FromInt32(backendConfig.GetServicePort().Port), }, Networking: tgbNetworking, NodeSelector: nodeSelector, @@ -216,7 +215,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate }, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbIPType elbv2model.IPAddressType, backendConfig routeutils.ServiceBackendConfig, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { +func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbIPType elbv2model.IPAddressType, backendConfig routeutils.TargetGroupConfigurator, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { targetType := builder.buildTargetGroupTargetType(targetGroupProps) tgProtocol, err := builder.buildTargetGroupProtocol(targetGroupProps, route) if err != nil { @@ -229,7 +228,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro return elbv2model.TargetGroupSpec{}, err } tgAttributesMap := builder.buildTargetGroupAttributes(targetGroupProps) - ipAddressType, err := builder.buildTargetGroupIPAddressType(backendConfig.Service, lbIPType) + ipAddressType, err := builder.buildTargetGroupIPAddressType(backendConfig, lbIPType) if err != nil { return elbv2model.TargetGroupSpec{}, err } @@ -238,8 +237,8 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro if err != nil { return elbv2model.TargetGroupSpec{}, err } - tgPort := builder.buildTargetGroupPort(targetType, *backendConfig.ServicePort) - name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), route.GetRouteKind(), k8s.NamespacedName(backendConfig.Service), tgPort, targetType, tgProtocol, tgProtocolVersion) + tgPort := backendConfig.GetTargetGroupPort(targetType) + name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), route.GetRouteKind(), backendConfig.GetBackendNamespacedName(), tgPort, targetType, tgProtocol, tgProtocolVersion) if tgPort == 0 { if targetType == elbv2model.TargetTypeIP { @@ -301,36 +300,12 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupTargetType(targetGroupPro return elbv2model.TargetType(*targetGroupProps.TargetType) } -func (builder *targetGroupBuilderImpl) buildTargetGroupIPAddressType(svc *corev1.Service, loadBalancerIPAddressType elbv2model.IPAddressType) (elbv2model.TargetGroupIPAddressType, error) { - var ipv6Configured bool - for _, ipFamily := range svc.Spec.IPFamilies { - if ipFamily == corev1.IPv6Protocol { - ipv6Configured = true - break - } - } - if ipv6Configured { - if !isIPv6Supported(loadBalancerIPAddressType) { - return "", errors.New("unsupported IPv6 configuration, lb not dual-stack") - } - return elbv2model.TargetGroupIPAddressTypeIPv6, nil +func (builder *targetGroupBuilderImpl) buildTargetGroupIPAddressType(backendConfig routeutils.TargetGroupConfigurator, loadBalancerIPAddressType elbv2model.IPAddressType) (elbv2model.TargetGroupIPAddressType, error) { + addressType := backendConfig.GetIPAddressType() + if addressType == elbv2model.TargetGroupIPAddressTypeIPv6 && !isIPv6Supported(loadBalancerIPAddressType) { + return "", errors.New("unsupported IPv6 configuration, lb not dual-stack") } - return elbv2model.TargetGroupIPAddressTypeIPv4, nil -} - -// buildTargetGroupPort constructs the TargetGroup's port. -// Note: TargetGroup's port is not in the data path as we always register targets with port specified. -// so these settings don't really matter to our controller, -// and we do our best to use the most appropriate port as targetGroup's port to avoid UX confusing. -func (builder *targetGroupBuilderImpl) buildTargetGroupPort(targetType elbv2model.TargetType, svcPort corev1.ServicePort) int32 { - if targetType == elbv2model.TargetTypeInstance { - return svcPort.NodePort - } - if svcPort.TargetPort.Type == intstr.Int { - return int32(svcPort.TargetPort.IntValue()) - } - // If all else fails, just return 1 as alluded to above, this setting doesn't really matter. - return 1 + return addressType, nil } func (builder *targetGroupBuilderImpl) buildTargetGroupProtocol(targetGroupProps *elbv2gw.TargetGroupProps, route routeutils.RouteDescriptor) (elbv2model.Protocol, error) { @@ -418,15 +393,15 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupProtocolVersion(targetGro return &http1 } -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion, targetType elbv2model.TargetType, backendConfig routeutils.ServiceBackendConfig) (elbv2model.TargetGroupHealthCheckConfig, error) { +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion, targetType elbv2model.TargetType, backendConfig routeutils.TargetGroupConfigurator) (elbv2model.TargetGroupHealthCheckConfig, error) { // add ServiceExternalTrafficPolicyLocal support var isServiceExternalTrafficPolicyTypeLocal = false if targetType == elbv2model.TargetTypeInstance && - backendConfig.Service.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeLocal && + backendConfig.GetExternalTrafficPolicy() == corev1.ServiceExternalTrafficPolicyTypeLocal && builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { isServiceExternalTrafficPolicyTypeLocal = true } - healthCheckPort, err := builder.buildTargetGroupHealthCheckPort(targetGroupProps, targetType, backendConfig.Service, isServiceExternalTrafficPolicyTypeLocal) + healthCheckPort, err := backendConfig.GetHealthCheckPort(targetType, isServiceExternalTrafficPolicyTypeLocal) if err != nil { return elbv2model.TargetGroupHealthCheckConfig{}, err } @@ -452,37 +427,6 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetG return hcConfig, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckPort(targetGroupProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType, svc *corev1.Service, isServiceExternalTrafficPolicyTypeLocal bool) (intstr.IntOrString, error) { - - portConfigNotExist := targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckPort == nil - - if portConfigNotExist && isServiceExternalTrafficPolicyTypeLocal { - return intstr.FromInt32(svc.Spec.HealthCheckNodePort), nil - } - - if portConfigNotExist || *targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { - return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil - } - - healthCheckPort := intstr.Parse(*targetGroupProps.HealthCheckConfig.HealthCheckPort) - if healthCheckPort.Type == intstr.Int { - return healthCheckPort, nil - } - hcSvcPort, err := k8s.LookupServicePort(svc, healthCheckPort) - if err != nil { - return intstr.FromString(""), err - } - - if targetType == elbv2model.TargetTypeInstance { - return intstr.FromInt(int(hcSvcPort.NodePort)), nil - } - - if hcSvcPort.TargetPort.Type == intstr.Int { - return hcSvcPort.TargetPort, nil - } - return intstr.IntOrString{}, errors.New("cannot use named healthCheckPort for IP TargetType when service's targetPort is a named port") -} - func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckProtocol(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, isServiceExternalTrafficPolicyTypeLocal bool) elbv2model.Protocol { if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckProtocol == nil { diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index 9c4afeedb..e6ae14c60 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -31,7 +31,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { defaultTargetType string gateway *gwv1.Gateway route *routeutils.MockRoute - backend routeutils.ServiceBackendConfig + backend *routeutils.ServiceBackendConfig tagErr error expectErr bool expectedTgSpec elbv2model.TargetGroupSpec @@ -53,14 +53,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -68,8 +69,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, NodePort: 8080, - }, - }, + }), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-8d8111f6ac", TargetType: elbv2model.TargetTypeInstance, @@ -108,14 +108,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -123,8 +124,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, NodePort: 8080, - }, - }, + }), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-224f4b6ea6", TargetType: elbv2model.TargetTypeInstance, @@ -168,14 +168,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -183,8 +184,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, NodePort: 8080, - }, - }, + }), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-3bce8b0f70", TargetType: elbv2model.TargetTypeIP, @@ -223,14 +223,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -239,7 +240,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, @@ -283,14 +284,15 @@ func Test_buildTargetGroupSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -298,7 +300,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { Type: intstr.Int, }, }, - }, + ), expectErr: true, }, } @@ -337,7 +339,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { defaultTargetType string gateway *gwv1.Gateway route *routeutils.MockRoute - backend routeutils.ServiceBackendConfig + backend *routeutils.ServiceBackendConfig tagErr error expectErr bool expectedTgSpec elbv2model.TargetGroupSpec @@ -359,14 +361,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -375,7 +378,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-d02da2803b", TargetType: elbv2model.TargetTypeInstance, @@ -433,14 +436,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -449,7 +453,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-224f4b6ea6", TargetType: elbv2model.TargetTypeInstance, @@ -512,14 +516,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -528,7 +533,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-3bce8b0f70", TargetType: elbv2model.TargetTypeIP, @@ -586,14 +591,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -602,7 +608,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, @@ -675,14 +681,15 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Name: "my-route", Namespace: "my-route-ns", }, - backend: routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend: routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", Name: "my-svc", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -691,7 +698,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, NodePort: 8080, }, - }, + ), expectedTgSpec: elbv2model.TargetGroupSpec{ Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, @@ -754,7 +761,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { builder := newTargetGroupBuilder("my-cluster", "vpc-xxx", tagger, tc.lbType, &mockTargetGroupBindingNetworkingBuilder{}, gateway.NewTargetGroupConfigConstructor(), tc.defaultTargetType, nil) - out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupBindingSpec(tc.gateway, nil, tc.expectedTgSpec, nil, tc.backend) + out, err := builder.(*targetGroupBuilderImpl).buildTargetGroupBindingSpec(tc.gateway, nil, tc.expectedTgSpec, nil, *tc.backend) assert.Equal(t, tc.expectedTgBindingSpec, out) assert.NoError(t, err) @@ -901,7 +908,7 @@ func Test_buildTargetGroupIPAddressType(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { builder := targetGroupBuilderImpl{} - res, err := builder.buildTargetGroupIPAddressType(tc.svc, tc.loadBalancerIPAddressType) + res, err := builder.buildTargetGroupIPAddressType(routeutils.NewServiceBackendConfig(tc.svc, nil, nil), tc.loadBalancerIPAddressType) if tc.expectErr { assert.Error(t, err) return @@ -913,57 +920,6 @@ func Test_buildTargetGroupIPAddressType(t *testing.T) { } } -func Test_buildTargetGroupPort(t *testing.T) { - testCases := []struct { - name string - targetType elbv2model.TargetType - svcPort corev1.ServicePort - expected int32 - }{ - { - name: "instance", - svcPort: corev1.ServicePort{ - NodePort: 8080, - }, - targetType: elbv2model.TargetTypeInstance, - expected: 8080, - }, - { - name: "instance - no node port", - svcPort: corev1.ServicePort{}, - targetType: elbv2model.TargetTypeInstance, - expected: 0, - }, - { - name: "ip", - svcPort: corev1.ServicePort{ - NodePort: 8080, - TargetPort: intstr.FromInt32(80), - }, - targetType: elbv2model.TargetTypeIP, - expected: 80, - }, - { - name: "ip - str port", - svcPort: corev1.ServicePort{ - NodePort: 8080, - TargetPort: intstr.FromString("foo"), - }, - targetType: elbv2model.TargetTypeIP, - expected: 1, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - builder := targetGroupBuilderImpl{} - res := builder.buildTargetGroupPort(tc.targetType, tc.svcPort) - assert.Equal(t, tc.expected, res) - - }) - } -} - func Test_buildTargetGroupProtocol(t *testing.T) { testCases := []struct { name string @@ -1244,189 +1200,6 @@ func Test_buildTargetGroupProtocolVersion(t *testing.T) { } } -func Test_buildTargetGroupHealthCheckPort(t *testing.T) { - testCases := []struct { - name string - isServiceExternalTrafficPolicyTypeLocal bool - targetGroupProps *elbv2gw.TargetGroupProps - targetType elbv2model.TargetType - svc *corev1.Service - expected intstr.IntOrString - expectErr bool - }{ - { - name: "nil props", - isServiceExternalTrafficPolicyTypeLocal: false, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "nil hc props", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{}, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "nil hc port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{}, - }, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "explicit is use traffic port hc port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String(shared_constants.HealthCheckPortTrafficPort), - }, - }, - expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), - }, - { - name: "explicit port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("80"), - }, - }, - expected: intstr.FromInt32(80), - }, - { - name: "resolve str port", - isServiceExternalTrafficPolicyTypeLocal: false, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromInt32(80), - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expected: intstr.FromInt32(80), - }, - { - name: "resolve str port - instance", - isServiceExternalTrafficPolicyTypeLocal: false, - targetType: elbv2model.TargetTypeInstance, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromInt32(80), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expected: intstr.FromInt32(1000), - }, - { - name: "resolve str port - resolves to other str port (error)", - isServiceExternalTrafficPolicyTypeLocal: false, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromString("bar"), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expectErr: true, - }, - { - name: "resolve str port - resolves to other str port but instance mode", - isServiceExternalTrafficPolicyTypeLocal: false, - targetType: elbv2model.TargetTypeInstance, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "foo", - TargetPort: intstr.FromString("bar"), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expected: intstr.FromInt32(1000), - }, - { - name: "resolve str port - cant find configured port", - isServiceExternalTrafficPolicyTypeLocal: false, - targetType: elbv2model.TargetTypeInstance, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "baz", - TargetPort: intstr.FromString("bar"), - NodePort: 1000, - }, - }, - }, - }, - targetGroupProps: &elbv2gw.TargetGroupProps{ - HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ - HealthCheckPort: awssdk.String("foo"), - }, - }, - expectErr: true, - }, - { - name: "with ExternalTrafficPolicyTypeLocal and HealthCheckNodePort specified", - isServiceExternalTrafficPolicyTypeLocal: true, - svc: &corev1.Service{ - Spec: corev1.ServiceSpec{ - HealthCheckNodePort: 32000, - ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, - }, - }, - expected: intstr.FromInt32(32000), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - builder := targetGroupBuilderImpl{} - res, err := builder.buildTargetGroupHealthCheckPort(tc.targetGroupProps, tc.targetType, tc.svc, tc.isServiceExternalTrafficPolicyTypeLocal) - if tc.expectErr { - assert.Error(t, err, res) - return - } - assert.NoError(t, err) - assert.Equal(t, tc.expected, res) - }) - } -} - func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { testCases := []struct { name string @@ -1862,14 +1635,15 @@ func Test_buildTargetGroupTags(t *testing.T) { Namespace: "test-namespace", } - backend := routeutils.ServiceBackendConfig{ - Service: &corev1.Service{ + backend := routeutils.NewServiceBackendConfig( + &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-namespace", Name: "test-service", }, }, - ServicePort: &corev1.ServicePort{ + nil, + &corev1.ServicePort{ Protocol: corev1.ProtocolTCP, Port: 80, TargetPort: intstr.IntOrString{ @@ -1877,7 +1651,7 @@ func Test_buildTargetGroupTags(t *testing.T) { Type: intstr.Int, }, }, - } + ) // Create target group props with user tags if specified var tgProps *elbv2gw.TargetGroupProps diff --git a/pkg/gateway/routeutils/backend.go b/pkg/gateway/routeutils/backend.go index 6923db3de..50313622c 100644 --- a/pkg/gateway/routeutils/backend.go +++ b/pkg/gateway/routeutils/backend.go @@ -3,6 +3,8 @@ package routeutils import ( "context" "fmt" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "strings" "github.com/pkg/errors" @@ -27,15 +29,21 @@ var ( tgConfigConstructor = gateway.NewTargetGroupConfigConstructor() ) -type ServiceBackendConfig struct { - Service *corev1.Service - ELBV2TargetGroupProps *elbv2gw.TargetGroupProps - ServicePort *corev1.ServicePort -} - -type LiteralTargetGroupConfig struct { - // GW API limits names to 253 characters, while a TG ARN might be 256, so just using the name. - Name string +type TargetGroupConfigurator interface { + // GetTargetGroupProps returns the target group properties associated with this backend + GetTargetGroupProps() *elbv2gw.TargetGroupProps + // GetBackendNamespacedName returns the namespaced name associated with the underlying backend. + GetBackendNamespacedName() types.NamespacedName + // GetIdentifierPort returns the port used when constructing the resource ID for the resource stack. + GetIdentifierPort() intstr.IntOrString + // GetExternalTrafficPolicy returns the external traffic policy for this backend service, if not applicable returns "ServiceExternalTrafficPolicyCluster". + GetExternalTrafficPolicy() corev1.ServiceExternalTrafficPolicyType + // GetIPAddressType returns the Target Group IP address type + GetIPAddressType() elbv2model.TargetGroupIPAddressType + // GetTargetGroupPort returns the port to attach to the Target Group + GetTargetGroupPort(targetType elbv2model.TargetType) int32 + // GetHealthCheckPort returns the port to send health check traffic + GetHealthCheckPort(targetType elbv2model.TargetType, isServiceExternalTrafficPolicyTypeLocal bool) (intstr.IntOrString, error) } // Backend an abstraction on the Gateway Backend, meant to hide the underlying backend type from consumers (unless they really want to see it :)) @@ -271,9 +279,9 @@ func serviceLoader(ctx context.Context, k8sClient client.Client, routeIdentifier } return &ServiceBackendConfig{ - Service: svc, - ServicePort: servicePort, - ELBV2TargetGroupProps: tgProps, + service: svc, + servicePort: servicePort, + targetGroupProps: tgProps, }, nil, nil } diff --git a/pkg/gateway/routeutils/backend_literal.go b/pkg/gateway/routeutils/backend_literal.go new file mode 100644 index 000000000..3693bb6cf --- /dev/null +++ b/pkg/gateway/routeutils/backend_literal.go @@ -0,0 +1,6 @@ +package routeutils + +type LiteralTargetGroupConfig struct { + // GW API limits names to 253 characters, while a TG ARN might be 256, so just using the name. + Name string +} diff --git a/pkg/gateway/routeutils/backend_service.go b/pkg/gateway/routeutils/backend_service.go new file mode 100644 index 000000000..4ebe4b46e --- /dev/null +++ b/pkg/gateway/routeutils/backend_service.go @@ -0,0 +1,108 @@ +package routeutils + +import ( + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" +) + +type ServiceBackendConfig struct { + service *corev1.Service + targetGroupProps *elbv2gw.TargetGroupProps + servicePort *corev1.ServicePort +} + +func NewServiceBackendConfig(service *corev1.Service, targetGroupProps *elbv2gw.TargetGroupProps, servicePort *corev1.ServicePort) *ServiceBackendConfig { + return &ServiceBackendConfig{ + service: service, + targetGroupProps: targetGroupProps, + servicePort: servicePort, + } +} + +func (s *ServiceBackendConfig) GetHealthCheckPort(targetType elbv2model.TargetType, isServiceExternalTrafficPolicyTypeLocal bool) (intstr.IntOrString, error) { + portConfigNotExist := s.targetGroupProps == nil || s.targetGroupProps.HealthCheckConfig == nil || s.targetGroupProps.HealthCheckConfig.HealthCheckPort == nil + + if portConfigNotExist && isServiceExternalTrafficPolicyTypeLocal { + return intstr.FromInt32(s.service.Spec.HealthCheckNodePort), nil + } + + if portConfigNotExist || *s.targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil + } + + healthCheckPort := intstr.Parse(*s.targetGroupProps.HealthCheckConfig.HealthCheckPort) + if healthCheckPort.Type == intstr.Int { + return healthCheckPort, nil + } + hcSvcPort, err := k8s.LookupServicePort(s.service, healthCheckPort) + if err != nil { + return intstr.FromString(""), err + } + + if targetType == elbv2model.TargetTypeInstance { + return intstr.FromInt(int(hcSvcPort.NodePort)), nil + } + + if hcSvcPort.TargetPort.Type == intstr.Int { + return hcSvcPort.TargetPort, nil + } + return intstr.IntOrString{}, errors.New("cannot use named healthCheckPort for IP TargetType when service's targetPort is a named port") +} + +// GetTargetGroupPort constructs the TargetGroup's port. +// Note: TargetGroup's port is not in the data path as we always register targets with port specified. +// so these settings don't really matter to our controller, +// and we do our best to use the most appropriate port as targetGroup's port to avoid UX confusing. + +func (s *ServiceBackendConfig) GetTargetGroupPort(targetType elbv2model.TargetType) int32 { + if targetType == elbv2model.TargetTypeInstance { + return s.servicePort.NodePort + } + if s.servicePort.TargetPort.Type == intstr.Int { + return int32(s.servicePort.TargetPort.IntValue()) + } + // If all else fails, return 1 as alluded to above, this setting doesn't really matter. + return 1 +} + +func (s *ServiceBackendConfig) GetIPAddressType() elbv2model.TargetGroupIPAddressType { + var ipv6Configured bool + for _, ipFamily := range s.service.Spec.IPFamilies { + if ipFamily == corev1.IPv6Protocol { + ipv6Configured = true + break + } + } + if ipv6Configured { + return elbv2model.TargetGroupIPAddressTypeIPv6 + } + return elbv2model.TargetGroupIPAddressTypeIPv4 +} + +func (s *ServiceBackendConfig) GetExternalTrafficPolicy() corev1.ServiceExternalTrafficPolicyType { + return s.service.Spec.ExternalTrafficPolicy +} + +func (s *ServiceBackendConfig) GetServicePort() *corev1.ServicePort { + return s.servicePort +} + +func (s *ServiceBackendConfig) GetIdentifierPort() intstr.IntOrString { + return s.servicePort.TargetPort +} + +func (s *ServiceBackendConfig) GetBackendNamespacedName() types.NamespacedName { + return k8s.NamespacedName(s.service) +} + +func (s *ServiceBackendConfig) GetTargetGroupProps() *elbv2gw.TargetGroupProps { + return s.targetGroupProps +} + +var _ TargetGroupConfigurator = &ServiceBackendConfig{} diff --git a/pkg/gateway/routeutils/backend_service_test.go b/pkg/gateway/routeutils/backend_service_test.go new file mode 100644 index 000000000..41e89e209 --- /dev/null +++ b/pkg/gateway/routeutils/backend_service_test.go @@ -0,0 +1,245 @@ +package routeutils + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + "testing" +) + +func Test_buildTargetGroupPort(t *testing.T) { + testCases := []struct { + name string + targetType elbv2model.TargetType + svcPort *corev1.ServicePort + expected int32 + }{ + { + name: "instance", + svcPort: &corev1.ServicePort{ + NodePort: 8080, + }, + targetType: elbv2model.TargetTypeInstance, + expected: 8080, + }, + { + name: "instance - no node port", + svcPort: &corev1.ServicePort{}, + targetType: elbv2model.TargetTypeInstance, + expected: 0, + }, + { + name: "ip", + svcPort: &corev1.ServicePort{ + NodePort: 8080, + TargetPort: intstr.FromInt32(80), + }, + targetType: elbv2model.TargetTypeIP, + expected: 80, + }, + { + name: "ip - str port", + svcPort: &corev1.ServicePort{ + NodePort: 8080, + TargetPort: intstr.FromString("foo"), + }, + targetType: elbv2model.TargetTypeIP, + expected: 1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + svcBackend := NewServiceBackendConfig(nil, nil, tc.svcPort) + res := svcBackend.GetTargetGroupPort(tc.targetType) + assert.Equal(t, res, tc.expected) + }) + } +} + +func Test_buildTargetGroupHealthCheckPort(t *testing.T) { + testCases := []struct { + name string + isServiceExternalTrafficPolicyTypeLocal bool + targetGroupProps *elbv2gw.TargetGroupProps + targetType elbv2model.TargetType + svc *corev1.Service + expected intstr.IntOrString + expectErr bool + }{ + { + name: "nil props", + isServiceExternalTrafficPolicyTypeLocal: false, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "nil hc props", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{}, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "nil hc port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{}, + }, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "explicit is use traffic port hc port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String(shared_constants.HealthCheckPortTrafficPort), + }, + }, + expected: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + }, + { + name: "explicit port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("80"), + }, + }, + expected: intstr.FromInt32(80), + }, + { + name: "resolve str port", + isServiceExternalTrafficPolicyTypeLocal: false, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromInt32(80), + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(80), + }, + { + name: "resolve str port - instance", + isServiceExternalTrafficPolicyTypeLocal: false, + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromInt32(80), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(1000), + }, + { + name: "resolve str port - resolves to other str port (error)", + isServiceExternalTrafficPolicyTypeLocal: false, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expectErr: true, + }, + { + name: "resolve str port - resolves to other str port but instance mode", + isServiceExternalTrafficPolicyTypeLocal: false, + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "foo", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expected: intstr.FromInt32(1000), + }, + { + name: "resolve str port - cant find configured port", + isServiceExternalTrafficPolicyTypeLocal: false, + targetType: elbv2model.TargetTypeInstance, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "baz", + TargetPort: intstr.FromString("bar"), + NodePort: 1000, + }, + }, + }, + }, + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("foo"), + }, + }, + expectErr: true, + }, + { + name: "with ExternalTrafficPolicyTypeLocal and HealthCheckNodePort specified", + isServiceExternalTrafficPolicyTypeLocal: true, + svc: &corev1.Service{ + Spec: corev1.ServiceSpec{ + HealthCheckNodePort: 32000, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + }, + }, + expected: intstr.FromInt32(32000), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + svcBackend := NewServiceBackendConfig(tc.svc, tc.targetGroupProps, nil) + res, err := svcBackend.GetHealthCheckPort(tc.targetType, tc.isServiceExternalTrafficPolicyTypeLocal) + if tc.expectErr { + assert.Error(t, err, res) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, res) + }) + } +} diff --git a/pkg/gateway/routeutils/backend_test.go b/pkg/gateway/routeutils/backend_test.go index 889bdbf6d..8b2eae6bd 100644 --- a/pkg/gateway/routeutils/backend_test.go +++ b/pkg/gateway/routeutils/backend_test.go @@ -390,14 +390,14 @@ func TestCommonBackendLoader_Service(t *testing.T) { return } - assert.Equal(t, tc.storedService, result.ServiceBackend.Service) + assert.Equal(t, tc.storedService, result.ServiceBackend.service) assert.Equal(t, tc.weight, result.Weight) - assert.Equal(t, tc.servicePort, result.ServiceBackend.ServicePort.Port) + assert.Equal(t, tc.servicePort, result.ServiceBackend.servicePort.Port) if tc.expectedTargetGroup == nil { - assert.Nil(t, result.ServiceBackend.ELBV2TargetGroupProps) + assert.Nil(t, result.ServiceBackend.targetGroupProps) } else { - assert.Equal(t, tc.expectedTargetGroup, result.ServiceBackend.ELBV2TargetGroupProps) + assert.Equal(t, tc.expectedTargetGroup, result.ServiceBackend.targetGroupProps) } }) } From 8fb9a046ebf5113486998785eb3c2e605de9cda2 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Fri, 31 Oct 2025 16:16:21 -0700 Subject: [PATCH 03/10] refactor front nlb data passing to deployer --- controllers/gateway/gateway_controller.go | 4 +- controllers/ingress/group_controller.go | 7 ++-- controllers/service/service_controller.go | 4 +- .../elbv2/frontend_nlb_target_synthesizer.go | 4 +- pkg/deploy/stack_deployer.go | 26 ++++++++---- pkg/gateway/routeutils/backend_gateway.go | 1 + pkg/ingress/model_build_frontend_nlb.go | 8 +++- pkg/ingress/model_build_frontend_nlb_test.go | 15 +++---- pkg/ingress/model_builder.go | 33 ++++++++------- pkg/ingress/model_builder_test.go | 42 ++++++++++++++++++- pkg/model/core/frontend_nlb_target_group.go | 31 -------------- pkg/model/elbv2/frontend_nlb_target_group.go | 35 ++++++++++++++++ 12 files changed, 133 insertions(+), 77 deletions(-) create mode 100644 pkg/gateway/routeutils/backend_gateway.go delete mode 100644 pkg/model/core/frontend_nlb_target_group.go create mode 100644 pkg/model/elbv2/frontend_nlb_target_group.go diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 2e03ef2f5..f1f0ee9f7 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -81,7 +81,7 @@ func newGatewayReconciler(controllerName string, lbType elbv2model.LoadBalancerT modelBuilder := gatewaymodel.NewModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), lbType, trackingProvider, elbv2TaggingManager, controllerConfig, cloud.EC2(), cloud.ELBV2(), cloud.ACM(), k8sClient, controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, supportedAddons, logger) stackMarshaller := deploy.NewDefaultStackMarshaller() - stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName, true, targetGroupCollector) + stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName, true, targetGroupCollector, lbType == elbv2model.LoadBalancerTypeApplication) cfgResolver := newGatewayConfigResolver() @@ -351,7 +351,7 @@ func (r *gatewayReconciler) reconcileUpdate(ctx context.Context, gw *gwv1.Gatewa } func (r *gatewayReconciler) deployModel(ctx context.Context, gw *gwv1.Gateway, stack core.Stack, secrets []types.NamespacedName) error { - if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName, nil); err != nil { + if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, r.controllerName); err != nil { var requeueNeededAfter *ctrlerrors.RequeueNeededAfter if errors.As(err, &requeueNeededAfter) { return err diff --git a/controllers/ingress/group_controller.go b/controllers/ingress/group_controller.go index 47ec0cb64..9e776bb90 100644 --- a/controllers/ingress/group_controller.go +++ b/controllers/ingress/group_controller.go @@ -70,7 +70,7 @@ func NewGroupReconciler(cloud services.Cloud, k8sClient client.Client, eventReco controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), targetGroupNameToArnMapper, logger, metricsCollector) stackMarshaller := deploy.NewDefaultStackMarshaller() stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, - controllerConfig, ingressTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector) + controllerConfig, ingressTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector, true) classLoader := ingress.NewDefaultClassLoader(k8sClient, true) classAnnotationMatcher := ingress.NewDefaultClassAnnotationMatcher(controllerConfig.IngressConfig.IngressClass) manageIngressesWithoutIngressClass := controllerConfig.IngressConfig.IngressClass == "" @@ -206,10 +206,9 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr var secrets []types.NamespacedName var backendSGRequired bool var err error - var frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState var frontendNlb *elbv2model.LoadBalancer buildModelFn := func() { - stack, lb, secrets, backendSGRequired, frontendNlbTargetGroupDesiredState, frontendNlb, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector) + stack, lb, secrets, backendSGRequired, frontendNlb, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector) } r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "build_model", buildModelFn) if err != nil { @@ -224,7 +223,7 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr r.logger.Info("successfully built model", "model", stackJSON) deployModelFn := func() { - err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress", frontendNlbTargetGroupDesiredState) + err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress") } r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "deploy_model", deployModelFn) if err != nil { diff --git a/controllers/service/service_controller.go b/controllers/service/service_controller.go index d7ab802d0..2595c7d93 100644 --- a/controllers/service/service_controller.go +++ b/controllers/service/service_controller.go @@ -55,7 +55,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector, controllerConfig.FeatureGates.Enabled(config.EnableTCPUDPListenerType)) stackMarshaller := deploy.NewDefaultStackMarshaller() - stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector) + stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector, false) return &serviceReconciler{ k8sClient: k8sClient, eventRecorder: eventRecorder, @@ -154,7 +154,7 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) } func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error { - if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service", nil); err != nil { + if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service"); err != nil { var requeueNeededAfter *ctrlerrors.RequeueNeededAfter if errors.As(err, &requeueNeededAfter) { return err diff --git a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go index 7fd011e61..e2ee67eea 100644 --- a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go +++ b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go @@ -21,7 +21,7 @@ type TargetGroupsResult struct { Err error } -func NewFrontendNlbTargetSynthesizer(k8sClient client.Client, trackingProvider tracking.Provider, taggingManager TaggingManager, frontendNlbTargetsManager FrontendNlbTargetsManager, logger logr.Logger, featureGates config.FeatureGates, stack core.Stack, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState, findSDKTargetGroups func() TargetGroupsResult) *frontendNlbTargetSynthesizer { +func NewFrontendNlbTargetSynthesizer(k8sClient client.Client, trackingProvider tracking.Provider, taggingManager TaggingManager, frontendNlbTargetsManager FrontendNlbTargetsManager, logger logr.Logger, featureGates config.FeatureGates, stack core.Stack, frontendNlbTargetGroupDesiredState *elbv2model.FrontendNlbTargetGroupDesiredState, findSDKTargetGroups func() TargetGroupsResult) *frontendNlbTargetSynthesizer { return &frontendNlbTargetSynthesizer{ k8sClient: k8sClient, trackingProvider: trackingProvider, @@ -43,7 +43,7 @@ type frontendNlbTargetSynthesizer struct { featureGates config.FeatureGates logger logr.Logger stack core.Stack - frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState + frontendNlbTargetGroupDesiredState *elbv2model.FrontendNlbTargetGroupDesiredState findSDKTargetGroups func() TargetGroupsResult } diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index 05da16730..92adca5d8 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" awsmetrics "sigs.k8s.io/aws-load-balancer-controller/pkg/metrics/aws" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sync" "github.com/go-logr/logr" @@ -22,16 +23,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - ingressController = "ingress" -) - // Using elbv2.TargetGroupsResult instead of defining our own // StackDeployer will deploy a resource stack into AWS and K8S. type StackDeployer interface { // Deploy a resource stack. - Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) error + Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string) error } // NewDefaultStackDeployer constructs new defaultStackDeployer. @@ -39,7 +36,7 @@ func NewDefaultStackDeployer(cloud services.Cloud, k8sClient client.Client, networkingManager networking.NetworkingManager, networkingSGManager networking.SecurityGroupManager, networkingSGReconciler networking.SecurityGroupReconciler, elbv2TaggingManager elbv2.TaggingManager, config config.ControllerConfig, tagPrefix string, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, controllerName string, enhancedDefaultingPolicyEnabled bool, - targetGroupCollector awsmetrics.TargetGroupCollector) *defaultStackDeployer { + targetGroupCollector awsmetrics.TargetGroupCollector, enableFrontendNLB bool) *defaultStackDeployer { trackingProvider := tracking.NewDefaultProvider(tagPrefix, config.ClusterName) ec2TaggingManager := ec2.NewDefaultTaggingManager(cloud.EC2(), networkingSGManager, cloud.VpcID(), logger) @@ -67,6 +64,7 @@ func NewDefaultStackDeployer(cloud services.Cloud, k8sClient client.Client, logger: logger, metricsCollector: metricsCollector, controllerName: controllerName, + enableFrontendNLB: enableFrontendNLB, } } @@ -95,6 +93,7 @@ type defaultStackDeployer struct { vpcID string metricsCollector lbcmetrics.MetricCollector controllerName string + enableFrontendNLB bool logger logr.Logger } @@ -105,7 +104,7 @@ type ResourceSynthesizer interface { } // Deploy a resource stack. -func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) error { +func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, metricsCollector lbcmetrics.MetricCollector, controllerName string) error { synthesizers := []ResourceSynthesizer{ ec2.NewSecurityGroupSynthesizer(d.cloud.EC2(), d.trackingProvider, d.ec2TaggingManager, d.ec2SGManager, d.vpcID, d.logger, stack), } @@ -121,9 +120,18 @@ func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, met return elbv2.TargetGroupsResult{TargetGroups: tgs, Err: err} }) - if controllerName == ingressController { + if d.enableFrontendNLB { + var desiredFENLBState []*elbv2model.FrontendNlbTargetGroupDesiredState + err := stack.ListResources(&desiredFENLBState) + d.logger.Info(fmt.Sprintf("Got this result!!! %+v %+v", desiredFENLBState, err)) + + var frontendNLBState *elbv2model.FrontendNlbTargetGroupDesiredState + if len(desiredFENLBState) == 1 { + frontendNLBState = desiredFENLBState[0] + } + synthesizers = append(synthesizers, elbv2.NewFrontendNlbTargetSynthesizer( - d.k8sClient, d.trackingProvider, d.elbv2TaggingManager, d.elbv2FrontendNlbTargetsManager, d.logger, d.featureGates, stack, frontendNlbTargetGroupDesiredState, findSDKTargetGroups)) + d.k8sClient, d.trackingProvider, d.elbv2TaggingManager, d.elbv2FrontendNlbTargetsManager, d.logger, d.featureGates, stack, frontendNLBState, findSDKTargetGroups)) } synthesizers = append(synthesizers, diff --git a/pkg/gateway/routeutils/backend_gateway.go b/pkg/gateway/routeutils/backend_gateway.go new file mode 100644 index 000000000..90d5e5490 --- /dev/null +++ b/pkg/gateway/routeutils/backend_gateway.go @@ -0,0 +1 @@ +package routeutils diff --git a/pkg/ingress/model_build_frontend_nlb.go b/pkg/ingress/model_build_frontend_nlb.go index 08fc16f7f..1ef2a9aba 100644 --- a/pkg/ingress/model_build_frontend_nlb.go +++ b/pkg/ingress/model_build_frontend_nlb.go @@ -561,7 +561,13 @@ func (t *defaultModelBuildTask) buildFrontendNlbListenerSpec(ctx context.Context defaultActions := t.buildFrontendNlbListenerDefaultActions(ctx, targetGroup) - t.frontendNlbTargetGroupDesiredState.AddTargetGroup(targetGroup.Spec.Name, targetGroup.TargetGroupARN(), t.loadBalancer.LoadBalancerARN(), *targetGroup.Spec.Port, config.TargetPort) + t.localFrontendNlbData[targetGroup.Spec.Name] = &elbv2model.FrontendNlbTargetGroupState{ + Name: targetGroup.Spec.Name, + ARN: targetGroup.TargetGroupARN(), + Port: port, + TargetARN: t.loadBalancer.LoadBalancerARN(), + TargetPort: config.TargetPort, + } return elbv2model.ListenerSpec{ LoadBalancerARN: t.frontendNlb.LoadBalancerARN(), diff --git a/pkg/ingress/model_build_frontend_nlb_test.go b/pkg/ingress/model_build_frontend_nlb_test.go index 7fce2e7f9..c7bde65ae 100644 --- a/pkg/ingress/model_build_frontend_nlb_test.go +++ b/pkg/ingress/model_build_frontend_nlb_test.go @@ -1321,20 +1321,17 @@ func Test_defaultModelBuildTask_buildFrontendNlbListeners(t *testing.T) { t.Run(tt.name, func(t *testing.T) { stack := core.NewDefaultStack(core.StackID{Name: "awesome-stack"}) - desiredState := &core.FrontendNlbTargetGroupDesiredState{ - TargetGroups: make(map[string]*core.FrontendNlbTargetGroupState), - } mockLoadBalancer := elbv2model.NewLoadBalancer(stack, "FrontendNlb", elbv2model.LoadBalancerSpec{ IPAddressType: elbv2model.IPAddressTypeIPV4, }) task := &defaultModelBuildTask{ - ingGroup: tt.ingGroup, - annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), - loadBalancer: tt.loadBalancer, - frontendNlb: mockLoadBalancer, - stack: stack, - frontendNlbTargetGroupDesiredState: desiredState, + ingGroup: tt.ingGroup, + annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), + loadBalancer: tt.loadBalancer, + frontendNlb: mockLoadBalancer, + stack: stack, + localFrontendNlbData: make(map[string]*elbv2model.FrontendNlbTargetGroupState), } err := task.buildFrontendNlbListeners(context.Background(), tt.listenerPortConfigByIngress) diff --git a/pkg/ingress/model_builder.go b/pkg/ingress/model_builder.go index 265782bb5..f9ba29e31 100644 --- a/pkg/ingress/model_builder.go +++ b/pkg/ingress/model_builder.go @@ -46,7 +46,7 @@ const ( // ModelBuilder is responsible for build mode stack for a IngressGroup. type ModelBuilder interface { // build mode stack for a IngressGroup. - Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *core.FrontendNlbTargetGroupDesiredState, *elbv2model.LoadBalancer, error) + Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *elbv2model.LoadBalancer, error) } // NewDefaultModelBuilder constructs new defaultModelBuilder. @@ -135,9 +135,8 @@ type defaultModelBuilder struct { } // build mode stack for a IngressGroup. -func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *core.FrontendNlbTargetGroupDesiredState, *elbv2model.LoadBalancer, error) { +func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metricsCollector lbcmetrics.MetricCollector) (core.Stack, *elbv2model.LoadBalancer, []types.NamespacedName, bool, *elbv2model.LoadBalancer, error) { stack := core.NewDefaultStack(core.StackID(ingGroup.ID)) - frontendNlbTargetGroupDesiredState := core.NewFrontendNlbTargetGroupDesiredState() task := &defaultModelBuildTask{ k8sClient: b.k8sClient, @@ -165,9 +164,8 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metrics enableIPTargetType: b.enableIPTargetType, metricsCollector: b.metricsCollector, - ingGroup: ingGroup, - stack: stack, - frontendNlbTargetGroupDesiredState: frontendNlbTargetGroupDesiredState, + ingGroup: ingGroup, + stack: stack, defaultTags: b.defaultTags, externalManagedTags: b.externalManagedTags, @@ -192,11 +190,14 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group, metrics backendServices: make(map[types.NamespacedName]*corev1.Service), targetGroupNameToArnMapper: b.targetGroupNameToArnMapper, webACLNameToArnMapper: b.webACLNameToArnMapper, + localFrontendNlbData: make(map[string]*elbv2model.FrontendNlbTargetGroupState), } if err := task.run(ctx); err != nil { - return nil, nil, nil, false, nil, nil, err + return nil, nil, nil, false, nil, err } - return task.stack, task.loadBalancer, task.secretKeys, task.backendSGAllocated, frontendNlbTargetGroupDesiredState, task.frontendNlb, nil + + _ = elbv2model.NewFrontendNlbTargetGroupDesiredState(task.stack, task.localFrontendNlbData) + return task.stack, task.loadBalancer, task.secretKeys, task.backendSGAllocated, task.frontendNlb, nil } // the default model build task @@ -248,14 +249,14 @@ type defaultModelBuildTask struct { defaultHealthCheckMatcherHTTPCode string defaultHealthCheckMatcherGRPCCode string - loadBalancer *elbv2model.LoadBalancer - tgByResID map[string]*elbv2model.TargetGroup - backendServices map[types.NamespacedName]*corev1.Service - secretKeys []types.NamespacedName - frontendNlb *elbv2model.LoadBalancer - frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState - targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper - webACLNameToArnMapper *webACLNameToArnMapper + loadBalancer *elbv2model.LoadBalancer + tgByResID map[string]*elbv2model.TargetGroup + backendServices map[types.NamespacedName]*corev1.Service + secretKeys []types.NamespacedName + frontendNlb *elbv2model.LoadBalancer + localFrontendNlbData map[string]*elbv2model.FrontendNlbTargetGroupState + targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper + webACLNameToArnMapper *webACLNameToArnMapper metricsCollector lbcmetrics.MetricCollector } diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index 2f2c34bc4..a42be793d 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -299,6 +299,11 @@ const baseStackJSON = ` } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding":{ "ns-1/ing-1-svc-1:http":{ "spec":{ @@ -798,6 +803,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": { "spec": { @@ -1735,6 +1745,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { "ns-1/ing-1-svc-2:http": null, "ns-1/ing-1-svc-3:https": null }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:80": { "spec": { @@ -2603,6 +2618,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding":{ "ns-1/ing-1-svc-1:http":{ "spec":{ @@ -3166,6 +3186,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -3398,6 +3423,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -3668,6 +3698,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -3829,6 +3864,11 @@ func Test_defaultModelBuilder_Build(t *testing.T) { } } }, + "FrontendNLBTargetGroup": { + "FrontendNLBTargetGroup": { + "TargetGroups": {} + } + }, "K8S::ElasticLoadBalancingV2::TargetGroupBinding": { "ns-1/ing-1-svc-1:http": null, "ns-1/ing-1-svc-2:http": null, @@ -4793,7 +4833,7 @@ func Test_defaultModelBuilder_Build(t *testing.T) { b.enableIPTargetType = *tt.enableIPTargetType } - gotStack, _, _, _, _, _, err := b.Build(context.Background(), tt.args.ingGroup, b.metricsCollector) + gotStack, _, _, _, _, err := b.Build(context.Background(), tt.args.ingGroup, b.metricsCollector) if tt.wantErr != "" { assert.EqualError(t, err, tt.wantErr) } else { diff --git a/pkg/model/core/frontend_nlb_target_group.go b/pkg/model/core/frontend_nlb_target_group.go deleted file mode 100644 index f83ef844b..000000000 --- a/pkg/model/core/frontend_nlb_target_group.go +++ /dev/null @@ -1,31 +0,0 @@ -package core - -// FrontendNlbTargetGroupState represents the state of a single ALB Target Type target group with its ALB target -type FrontendNlbTargetGroupState struct { - Name string - ARN StringToken - Port int32 - TargetARN StringToken - TargetPort int32 -} - -// FrontendNlbTargetGroupDesiredState maintains a mapping of target groups targeting ALB -type FrontendNlbTargetGroupDesiredState struct { - TargetGroups map[string]*FrontendNlbTargetGroupState -} - -func NewFrontendNlbTargetGroupDesiredState() *FrontendNlbTargetGroupDesiredState { - return &FrontendNlbTargetGroupDesiredState{ - TargetGroups: make(map[string]*FrontendNlbTargetGroupState), - } -} - -func (m *FrontendNlbTargetGroupDesiredState) AddTargetGroup(targetGroupName string, targetGroupARN StringToken, targetARN StringToken, port int32, targetPort int32) { - m.TargetGroups[targetGroupName] = &FrontendNlbTargetGroupState{ - Name: targetGroupName, - ARN: targetGroupARN, - Port: port, - TargetARN: targetARN, - TargetPort: targetPort, - } -} diff --git a/pkg/model/elbv2/frontend_nlb_target_group.go b/pkg/model/elbv2/frontend_nlb_target_group.go new file mode 100644 index 000000000..667d225e6 --- /dev/null +++ b/pkg/model/elbv2/frontend_nlb_target_group.go @@ -0,0 +1,35 @@ +package elbv2 + +import "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" + +var _ core.Resource = &FrontendNlbTargetGroupDesiredState{} + +const ( + FrontNLBResourceId = "FrontendNLBTargetGroup" +) + +// FrontendNlbTargetGroupState represents the state of a single ALB Target Type target group with its ALB target +type FrontendNlbTargetGroupState struct { + Name string + ARN core.StringToken + Port int32 + TargetARN core.StringToken + TargetPort int32 +} + +// FrontendNlbTargetGroupDesiredState maintains a mapping of target groups targeting ALB +type FrontendNlbTargetGroupDesiredState struct { + core.ResourceMeta `json:"-"` + + // Maps target group name -> The FE NLB configuration. + TargetGroups map[string]*FrontendNlbTargetGroupState +} + +func NewFrontendNlbTargetGroupDesiredState(stack core.Stack, stateConfig map[string]*FrontendNlbTargetGroupState) *FrontendNlbTargetGroupDesiredState { + desiredState := &FrontendNlbTargetGroupDesiredState{ + ResourceMeta: core.NewResourceMeta(stack, FrontNLBResourceId, FrontNLBResourceId), + TargetGroups: stateConfig, + } + stack.AddResource(desiredState) + return desiredState +} From 9bd4c27e4507bb8f794a6147fd32a9e18453ad8f Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Sun, 2 Nov 2025 18:29:09 -0800 Subject: [PATCH 04/10] working alb as target of nlb in gateway api --- .../gateway/eventhandlers/tcp_route_events.go | 12 + controllers/gateway/gateway_controller.go | 2 +- .../elbv2/frontend_nlb_target_synthesizer.go | 1 + pkg/gateway/gatewayutils/gateway_utils.go | 43 ++++ pkg/gateway/model/base_model_builder.go | 2 + pkg/gateway/model/mock_tg_builder.go | 11 +- pkg/gateway/model/model_build_listener.go | 4 +- pkg/gateway/model/model_build_target_group.go | 65 ++++-- .../model/model_build_target_group_test.go | 165 +++++++++++--- pkg/gateway/routeutils/backend.go | 165 +++----------- pkg/gateway/routeutils/backend_gateway.go | 166 ++++++++++++++ .../routeutils/backend_gateway_test.go | 207 ++++++++++++++++++ pkg/gateway/routeutils/backend_service.go | 126 ++++++++++- .../routeutils/backend_service_test.go | 21 ++ pkg/gateway/routeutils/backend_test.go | 163 ++++++++++++-- pkg/gateway/routeutils/constants.go | 2 +- pkg/gateway/routeutils/utils.go | 3 + pkg/model/elbv2/frontend_nlb_target_group.go | 10 +- 18 files changed, 962 insertions(+), 206 deletions(-) create mode 100644 pkg/gateway/routeutils/backend_gateway_test.go diff --git a/controllers/gateway/eventhandlers/tcp_route_events.go b/controllers/gateway/eventhandlers/tcp_route_events.go index 8ec77e6be..d609d57b2 100644 --- a/controllers/gateway/eventhandlers/tcp_route_events.go +++ b/controllers/gateway/eventhandlers/tcp_route_events.go @@ -69,4 +69,16 @@ func (h *enqueueRequestsForTCPRouteEvent) enqueueImpactedGateways(ctx context.Co "gateway", gw) queue.Add(reconcile.Request{NamespacedName: gw}) } + + gateways, err = gatewayutils.GetImpactedGatewaysFromBackendRefs(ctx, h.k8sClient, route.Spec.Rules, route.Namespace, constants.NLBGatewayController) + if err != nil { + h.logger.V(1).Info("ignoring unknown gateways referred by", "tcproute", route.Name, "error", err) + } + for _, gw := range gateways { + h.logger.V(1).Info("enqueue gateway for tcproute event", + "tcproute", k8s.NamespacedName(route), + "gateway", gw) + queue.Add(reconcile.Request{NamespacedName: gw}) + } + } diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index f1f0ee9f7..13b4ef3e3 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -81,7 +81,7 @@ func newGatewayReconciler(controllerName string, lbType elbv2model.LoadBalancerT modelBuilder := gatewaymodel.NewModelBuilder(subnetResolver, vpcInfoProvider, cloud.VpcID(), lbType, trackingProvider, elbv2TaggingManager, controllerConfig, cloud.EC2(), cloud.ELBV2(), cloud.ACM(), k8sClient, controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, sets.New(controllerConfig.ExternalManagedTags...), controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, controllerConfig.IngressConfig.AllowedCertificateAuthorityARNs, supportedAddons, logger) stackMarshaller := deploy.NewDefaultStackMarshaller() - stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName, true, targetGroupCollector, lbType == elbv2model.LoadBalancerTypeApplication) + stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, gatewayTagPrefix, logger, metricsCollector, controllerName, true, targetGroupCollector, lbType == elbv2model.LoadBalancerTypeNetwork) cfgResolver := newGatewayConfigResolver() diff --git a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go index e2ee67eea..64e46632e 100644 --- a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go +++ b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go @@ -146,6 +146,7 @@ func (s *frontendNlbTargetSynthesizer) PostSynthesize(ctx context.Context) error }) if err != nil { + s.logger.Error(err, "Got this error!") requeueMsg := "Failed to register target, retrying after deplay for target group: " + resTG.Status.TargetGroupARN return ctrlerrors.NewRequeueNeededAfter(requeueMsg, 15*time.Second) } diff --git a/pkg/gateway/gatewayutils/gateway_utils.go b/pkg/gateway/gatewayutils/gateway_utils.go index e5197e2af..aa1edf8a9 100644 --- a/pkg/gateway/gatewayutils/gateway_utils.go +++ b/pkg/gateway/gatewayutils/gateway_utils.go @@ -9,6 +9,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // IsGatewayManagedByLBController checks if a Gateway is managed by the ALB/NLB Gateway Controller @@ -103,6 +104,48 @@ func GetImpactedGatewaysFromParentRefs(ctx context.Context, k8sClient client.Cli return impactedGateways, err } +func GetImpactedGatewaysFromBackendRefs(ctx context.Context, k8sClient client.Client, tcpRoutes []gwalpha2.TCPRouteRule, resourceNamespace string, gwController string) ([]types.NamespacedName, error) { + resultSet := sets.New[types.NamespacedName]() + processedSet := sets.New[types.NamespacedName]() + errorSet := sets.New[types.NamespacedName]() + for _, route := range tcpRoutes { + for _, ref := range route.BackendRefs { + if ref.Kind == nil || *ref.Kind != "Gateway" { + continue + } + + ns := resourceNamespace + if ref.Namespace != nil { + ns = string(*ref.Namespace) + } + nsn := types.NamespacedName{ + Namespace: ns, + Name: string(ref.Name), + } + if processedSet.Has(nsn) { + continue + } + + gw := &gwv1.Gateway{} + if err := k8sClient.Get(ctx, nsn, gw); err != nil { + // Ignore and continue processing other refs + errorSet.Insert(nsn) + continue + } + processedSet.Insert(nsn) + + if IsGatewayManagedByLBController(ctx, k8sClient, gw, gwController) { + resultSet.Insert(nsn) + } + } + } + var err error + if resultSet.Len() > 0 { + err = fmt.Errorf("failed to list gateways, %s", resultSet.UnsortedList()) + } + return resultSet.UnsortedList(), err +} + // GetImpactedGatewayClassesFromLbConfig identifies GatewayClasses affected by LoadBalancer configuration changes. // Returns GatewayClasses that reference the specified LoadBalancer configuration. func GetImpactedGatewayClassesFromLbConfig(ctx context.Context, k8sClient client.Client, lbconfig *elbv2gw.LoadBalancerConfiguration, gwControllers sets.Set[string]) (map[string]*gwv1.GatewayClass, error) { diff --git a/pkg/gateway/model/base_model_builder.go b/pkg/gateway/model/base_model_builder.go index 2143e1cb2..d909d3849 100644 --- a/pkg/gateway/model/base_model_builder.go +++ b/pkg/gateway/model/base_model_builder.go @@ -200,6 +200,8 @@ func (baseBuilder *baseModelBuilder) Build(ctx context.Context, gw *gwv1.Gateway psa.AddToStack(stack, lb.LoadBalancerARN()) } + _ = elbv2model.NewFrontendNlbTargetGroupDesiredState(stack, tgBuilder.getLocalFrontendNlbData()) + return stack, lb, newAddonConfig, securityGroups.backendSecurityGroupAllocated, secrets, nil } diff --git a/pkg/gateway/model/mock_tg_builder.go b/pkg/gateway/model/mock_tg_builder.go index 74b7e02ab..21b080bb4 100644 --- a/pkg/gateway/model/mock_tg_builder.go +++ b/pkg/gateway/model/mock_tg_builder.go @@ -10,12 +10,17 @@ import ( ) type mockTargetGroupBuilder struct { - tgs []*elbv2model.TargetGroup - buildErr error + tgs []*elbv2model.TargetGroup + localFrontendNlbData map[string]*elbv2model.FrontendNlbTargetGroupState + buildErr error +} + +func (m *mockTargetGroupBuilder) getLocalFrontendNlbData() map[string]*elbv2model.FrontendNlbTargetGroupState { + return m.localFrontendNlbData } func (m *mockTargetGroupBuilder) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { var tg *elbv2model.TargetGroup if len(m.tgs) > 0 { diff --git a/pkg/gateway/model/model_build_listener.go b/pkg/gateway/model/model_build_listener.go index af83c42c7..e1ebafabd 100644 --- a/pkg/gateway/model/model_build_listener.go +++ b/pkg/gateway/model/model_build_listener.go @@ -177,7 +177,7 @@ func (l listenerBuilderImpl) buildL4ListenerSpec(ctx context.Context, stack core return nil, nil } - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, lb.Spec.IPAddressType, routeDescriptor, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, lb.Spec.IPAddressType, routeDescriptor, backend) if tgErr != nil { return &elbv2model.ListenerSpec{}, tgErr } @@ -222,7 +222,7 @@ func (l listenerBuilderImpl) buildListenerRules(ctx context.Context, stack core. } targetGroupTuples := make([]elbv2model.TargetGroupTuple, 0, len(rule.GetBackends())) for _, backend := range rule.GetBackends() { - arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, ipAddressType, route, backend) + arn, tgErr := l.tgBuilder.buildTargetGroup(stack, gw, port, ipAddressType, route, backend) if tgErr != nil { return nil, tgErr } diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go index 65469d89b..ce826db72 100644 --- a/pkg/gateway/model/model_build_target_group.go +++ b/pkg/gateway/model/model_build_target_group.go @@ -32,7 +32,8 @@ type buildTargetGroupOutput struct { type targetGroupBuilder interface { buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) + getLocalFrontendNlbData() map[string]*elbv2model.FrontendNlbTargetGroupState } type targetGroupBuilderImpl struct { @@ -48,6 +49,8 @@ type targetGroupBuilderImpl struct { tgbNetworkBuilder targetGroupBindingNetworkBuilder targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper + localFrontendNlbData map[string]*elbv2model.FrontendNlbTargetGroupState + defaultTargetType elbv2model.TargetType defaultHealthCheckMatcherHTTPCode string @@ -70,6 +73,10 @@ type targetGroupBuilderImpl struct { defaultHealthCheckUnhealthyThresholdForInstanceModeLocal int32 } +func (builder *targetGroupBuilderImpl) getLocalFrontendNlbData() map[string]*elbv2model.FrontendNlbTargetGroupState { + return builder.localFrontendNlbData +} + func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper, loadBalancerType elbv2model.LoadBalancerType, tgbNetworkBuilder targetGroupBindingNetworkBuilder, tgPropertiesConstructor gateway.TargetGroupConfigConstructor, defaultTargetType string, targetGroupNameToArnMapper shared_utils.TargetGroupARNMapper) targetGroupBuilder { return &targetGroupBuilderImpl{ loadBalancerType: loadBalancerType, @@ -79,6 +86,7 @@ func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper tgPropertiesConstructor: tgPropertiesConstructor, targetGroupNameToArnMapper: targetGroupNameToArnMapper, tgByResID: make(map[string]*elbv2model.TargetGroup), + localFrontendNlbData: make(map[string]*elbv2model.FrontendNlbTargetGroupState), tagHelper: tagHelper, defaultTargetType: elbv2model.TargetType(defaultTargetType), defaultHealthCheckMatcherHTTPCode: "200-399", @@ -100,7 +108,7 @@ func newTargetGroupBuilder(clusterName string, vpcId string, tagHelper tagHelper } func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, - gw *gwv1.Gateway, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend) (core.StringToken, error) { if backend.ServiceBackend != nil { tg, err := builder.buildTargetGroupFromService(stack, gw, lbIPType, routeDescriptor, *backend.ServiceBackend) @@ -110,6 +118,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, return tg.TargetGroupARN(), nil } + if backend.GatewayBackend != nil { + tg, err := builder.buildTargetGroupFromGateway(stack, gw, listenerPort, lbIPType, routeDescriptor, *backend.GatewayBackend) + if err != nil { + return nil, err + } + return tg.TargetGroupARN(), nil + } + if backend.LiteralTargetGroup != nil { arn, err := builder.buildTargetGroupFromStaticName(*backend.LiteralTargetGroup) return arn, err @@ -148,6 +164,33 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupFromService(stack core.St return tg, nil } +func (builder *targetGroupBuilderImpl) buildTargetGroupFromGateway(stack core.Stack, + gw *gwv1.Gateway, listenerPort int32, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backendConfig routeutils.GatewayBackendConfig) (*elbv2model.TargetGroup, error) { + targetGroupProps := backendConfig.GetTargetGroupProps() + tgResID := builder.buildTargetGroupResourceID(k8s.NamespacedName(gw), backendConfig.GetBackendNamespacedName(), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backendConfig.GetIdentifierPort()) + if tg, exists := builder.tgByResID[tgResID]; exists { + return tg, nil + } + + tgSpec, err := builder.buildTargetGroupSpec(gw, routeDescriptor, lbIPType, &backendConfig, targetGroupProps) + if err != nil { + return nil, err + } + + tg := elbv2model.NewTargetGroup(stack, tgResID, tgSpec) + builder.tgByResID[tgResID] = tg + + builder.localFrontendNlbData[tgSpec.Name] = &elbv2model.FrontendNlbTargetGroupState{ + Name: tgSpec.Name, + ARN: tg.TargetGroupARN(), + Port: listenerPort, + TargetARN: core.LiteralStringToken(backendConfig.GetALBARN()), + TargetPort: *tg.Spec.Port, + } + + return tg, nil +} + func (builder *targetGroupBuilderImpl) buildTargetGroupFromStaticName(cfg routeutils.LiteralTargetGroupConfig) (core.StringToken, error) { tgArn, err := builder.targetGroupNameToArnMapper.GetArnByName(context.Background(), cfg.Name) @@ -216,7 +259,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate } func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, route routeutils.RouteDescriptor, lbIPType elbv2model.IPAddressType, backendConfig routeutils.TargetGroupConfigurator, targetGroupProps *elbv2gw.TargetGroupProps) (elbv2model.TargetGroupSpec, error) { - targetType := builder.buildTargetGroupTargetType(targetGroupProps) + targetType := backendConfig.GetTargetType(builder.defaultTargetType) tgProtocol, err := builder.buildTargetGroupProtocol(targetGroupProps, route) if err != nil { return elbv2model.TargetGroupSpec{}, err @@ -292,14 +335,6 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *el return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedName, uuid) } -func (builder *targetGroupBuilderImpl) buildTargetGroupTargetType(targetGroupProps *elbv2gw.TargetGroupProps) elbv2model.TargetType { - if targetGroupProps == nil || targetGroupProps.TargetType == nil { - return builder.defaultTargetType - } - - return elbv2model.TargetType(*targetGroupProps.TargetType) -} - func (builder *targetGroupBuilderImpl) buildTargetGroupIPAddressType(backendConfig routeutils.TargetGroupConfigurator, loadBalancerIPAddressType elbv2model.IPAddressType) (elbv2model.TargetGroupIPAddressType, error) { addressType := backendConfig.GetIPAddressType() if addressType == elbv2model.TargetGroupIPAddressTypeIPv6 && !isIPv6Supported(loadBalancerIPAddressType) { @@ -405,7 +440,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetG if err != nil { return elbv2model.TargetGroupHealthCheckConfig{}, err } - healthCheckProtocol := builder.buildTargetGroupHealthCheckProtocol(targetGroupProps, tgProtocol, isServiceExternalTrafficPolicyTypeLocal) // + healthCheckProtocol := builder.buildTargetGroupHealthCheckProtocol(targetGroupProps, targetType, tgProtocol, isServiceExternalTrafficPolicyTypeLocal) // healthCheckPath := builder.buildTargetGroupHealthCheckPath(targetGroupProps, tgProtocolVersion, healthCheckProtocol, isServiceExternalTrafficPolicyTypeLocal) // healthCheckMatcher := builder.buildTargetGroupHealthCheckMatcher(targetGroupProps, tgProtocolVersion, healthCheckProtocol) // @@ -427,13 +462,17 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckConfig(targetG return hcConfig, nil } -func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckProtocol(targetGroupProps *elbv2gw.TargetGroupProps, tgProtocol elbv2model.Protocol, isServiceExternalTrafficPolicyTypeLocal bool) elbv2model.Protocol { +func (builder *targetGroupBuilderImpl) buildTargetGroupHealthCheckProtocol(targetGroupProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType, tgProtocol elbv2model.Protocol, isServiceExternalTrafficPolicyTypeLocal bool) elbv2model.Protocol { if targetGroupProps == nil || targetGroupProps.HealthCheckConfig == nil || targetGroupProps.HealthCheckConfig.HealthCheckProtocol == nil { if builder.loadBalancerType == elbv2model.LoadBalancerTypeNetwork { if isServiceExternalTrafficPolicyTypeLocal { return builder.defaultHealthCheckProtocolForInstanceModeLocal } + // ALB targets only support HTTP / HTTPS health checks. + if targetType == elbv2model.TargetTypeALB { + return elbv2model.ProtocolHTTP + } return elbv2model.ProtocolTCP } return tgProtocol diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index e6ae14c60..5b7246ccb 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -13,6 +13,8 @@ import ( elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/routeutils" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" elbv2modelk8s "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2/k8s" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" @@ -823,24 +825,6 @@ func Test_buildTargetGroupName(t *testing.T) { } } -func Test_buildTargetGroupTargetType(t *testing.T) { - builder := targetGroupBuilderImpl{ - defaultTargetType: elbv2model.TargetTypeIP, - } - - res := builder.buildTargetGroupTargetType(nil) - assert.Equal(t, elbv2model.TargetTypeIP, res) - - res = builder.buildTargetGroupTargetType(&elbv2gw.TargetGroupProps{}) - assert.Equal(t, elbv2model.TargetTypeIP, res) - - inst := elbv2gw.TargetTypeInstance - res = builder.buildTargetGroupTargetType(&elbv2gw.TargetGroupProps{ - TargetType: &inst, - }) - assert.Equal(t, elbv2model.TargetTypeInstance, res) -} - func Test_buildTargetGroupIPAddressType(t *testing.T) { testCases := []struct { name string @@ -1203,6 +1187,7 @@ func Test_buildTargetGroupProtocolVersion(t *testing.T) { func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { testCases := []struct { name string + targetType elbv2model.TargetType lbType elbv2model.LoadBalancerType targetGroupProps *elbv2gw.TargetGroupProps tgProtocol elbv2model.Protocol @@ -1210,19 +1195,22 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { }{ { name: "nlb - default", + targetType: elbv2model.TargetTypeInstance, lbType: elbv2model.LoadBalancerTypeNetwork, tgProtocol: elbv2model.ProtocolUDP, expected: elbv2model.ProtocolTCP, }, { name: "alb - default", + targetType: elbv2model.TargetTypeInstance, lbType: elbv2model.LoadBalancerTypeApplication, tgProtocol: elbv2model.ProtocolHTTP, expected: elbv2model.ProtocolHTTP, }, { - name: "specified http", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "specified http", + targetType: elbv2model.TargetTypeInstance, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolHTTP))), @@ -1232,8 +1220,9 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTP, }, { - name: "specified https", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "specified https", + targetType: elbv2model.TargetTypeInstance, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolHTTPS))), @@ -1243,8 +1232,9 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { expected: elbv2model.ProtocolHTTPS, }, { - name: "specified tcp", - lbType: elbv2model.LoadBalancerTypeApplication, + name: "specified tcp", + targetType: elbv2model.TargetTypeInstance, + lbType: elbv2model.LoadBalancerTypeApplication, targetGroupProps: &elbv2gw.TargetGroupProps{ HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ HealthCheckProtocol: (*elbv2gw.TargetGroupHealthCheckProtocol)(awssdk.String(string(elbv2gw.ProtocolTCP))), @@ -1253,6 +1243,13 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { tgProtocol: elbv2model.ProtocolTCP, expected: elbv2model.ProtocolTCP, }, + { + name: "nlb with alb target default", + targetType: elbv2model.TargetTypeALB, + lbType: elbv2model.LoadBalancerTypeNetwork, + tgProtocol: elbv2model.ProtocolTCP, + expected: elbv2model.ProtocolHTTP, + }, } for _, tc := range testCases { @@ -1261,7 +1258,7 @@ func Test_buildTargetGroupHealthCheckProtocol(t *testing.T) { loadBalancerType: tc.lbType, } - res := builder.buildTargetGroupHealthCheckProtocol(tc.targetGroupProps, tc.tgProtocol, false) + res := builder.buildTargetGroupHealthCheckProtocol(tc.targetGroupProps, tc.targetType, tc.tgProtocol, false) assert.Equal(t, tc.expected, res) }) } @@ -1674,6 +1671,124 @@ func Test_buildTargetGroupTags(t *testing.T) { } } +func Test_buildTargetGroupFromGateway(t *testing.T) { + testCases := []struct { + name string + gateway *gwv1.Gateway + listenerPort int32 + lbIPType elbv2model.IPAddressType + route *routeutils.MockRoute + backendConfig *routeutils.GatewayBackendConfig + existingTG bool + expectedTGName string + expectedFrontendData bool + }{ + { + name: "new target group creation", + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-gw", + }, + }, + listenerPort: 80, + lbIPType: elbv2model.IPAddressTypeIPV4, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "test-route", + Namespace: "test-ns", + }, + backendConfig: routeutils.NewGatewayBackendConfig(&gwv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "backend-gw", Namespace: "backend-ns"}}, nil, "arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/test-alb/1234567890123456", 8080), + expectedTGName: "k8s-testns-testrout", + expectedFrontendData: true, + }, + { + name: "existing target group reuse", + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-gw", + }, + }, + listenerPort: 80, + lbIPType: elbv2model.IPAddressTypeIPV4, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "test-route", + Namespace: "test-ns", + }, + backendConfig: routeutils.NewGatewayBackendConfig(&gwv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "backend-gw", Namespace: "backend-ns"}}, nil, "arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/test-alb/1234567890123456", 8080), + existingTG: true, + expectedFrontendData: false, + }, + { + name: "with target group props", + gateway: &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-gw", + }, + }, + listenerPort: 443, + lbIPType: elbv2model.IPAddressTypeIPV4, + route: &routeutils.MockRoute{ + Kind: routeutils.HTTPRouteKind, + Name: "test-route", + Namespace: "test-ns", + }, + backendConfig: routeutils.NewGatewayBackendConfig(&gwv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "backend-gw", Namespace: "backend-ns"}}, &elbv2gw.TargetGroupProps{ + TargetGroupName: awssdk.String("custom-tg-name"), + }, "arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/test-alb/1234567890123456", 8443), + expectedTGName: "custom-tg-name", + expectedFrontendData: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tagger := &mockTagHelper{ + tags: make(map[string]string), + } + + builder := newTargetGroupBuilder("test-cluster", "vpc-xxx", tagger, elbv2model.LoadBalancerTypeApplication, &mockTargetGroupBindingNetworkingBuilder{}, gateway.NewTargetGroupConfigConstructor(), string(elbv2model.TargetTypeALB), nil) + impl := builder.(*targetGroupBuilderImpl) + + stack := core.NewDefaultStack(core.StackID{Namespace: "test", Name: "test"}) + + // Pre-populate existing target group if needed + if tc.existingTG { + tgResID := impl.buildTargetGroupResourceID(k8s.NamespacedName(tc.gateway), tc.backendConfig.GetBackendNamespacedName(), tc.route.GetRouteNamespacedName(), tc.route.GetRouteKind(), tc.backendConfig.GetIdentifierPort()) + existingTG := elbv2model.NewTargetGroup(stack, tgResID, elbv2model.TargetGroupSpec{ + Name: "existing-tg", + }) + impl.tgByResID[tgResID] = existingTG + } + + result, err := impl.buildTargetGroupFromGateway(stack, tc.gateway, tc.listenerPort, tc.lbIPType, tc.route, *tc.backendConfig) + + assert.NoError(t, err) + assert.NotNil(t, result) + + if tc.existingTG { + assert.Equal(t, "existing-tg", result.Spec.Name) + } else { + assert.Contains(t, result.Spec.Name, tc.expectedTGName) + assert.Equal(t, elbv2model.TargetTypeALB, result.Spec.TargetType) + } + + // Check frontend NLB data + if tc.expectedFrontendData { + frontendData, exists := impl.localFrontendNlbData[result.Spec.Name] + assert.True(t, exists) + assert.Equal(t, result.Spec.Name, frontendData.Name) + assert.Equal(t, tc.listenerPort, frontendData.Port) + //assert.Equal(t, tc.backendConfig.GetALBARN(), frontendData.TargetARN.Resolve(context.Background())) + assert.Equal(t, *result.Spec.Port, frontendData.TargetPort) + } + }) + } +} + func protocolPtr(protocol elbv2gw.Protocol) *elbv2gw.Protocol { return &protocol } diff --git a/pkg/gateway/routeutils/backend.go b/pkg/gateway/routeutils/backend.go index 50313622c..1d11f0724 100644 --- a/pkg/gateway/routeutils/backend.go +++ b/pkg/gateway/routeutils/backend.go @@ -3,17 +3,14 @@ package routeutils import ( "context" "fmt" - "k8s.io/apimachinery/pkg/util/intstr" - elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" - "strings" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway" "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" - "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -21,6 +18,7 @@ import ( const ( serviceKind = "Service" + gatewayKind = "Gateway" referenceGrantNotExists = "No explicit ReferenceGrant exists to allow the reference." maxWeight = 999 ) @@ -29,7 +27,10 @@ var ( tgConfigConstructor = gateway.NewTargetGroupConfigConstructor() ) +// TargetGroupConfigurator defines methods used to construct an ELB target group from a Kubernetes based backend. type TargetGroupConfigurator interface { + // GetTargetType returns the Target Type to associate with this target group. + GetTargetType(defaultTargetType elbv2model.TargetType) elbv2model.TargetType // GetTargetGroupProps returns the target group properties associated with this backend GetTargetGroupProps() *elbv2gw.TargetGroupProps // GetBackendNamespacedName returns the namespaced name associated with the underlying backend. @@ -50,6 +51,7 @@ type TargetGroupConfigurator interface { type Backend struct { ServiceBackend *ServiceBackendConfig LiteralTargetGroup *LiteralTargetGroupConfig + GatewayBackend *GatewayBackendConfig Weight int } @@ -125,20 +127,23 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, backendRe var serviceBackend *ServiceBackendConfig var literalTargetGroup *LiteralTargetGroupConfig + var gatewayBackend *GatewayBackendConfig var warn error var fatal error // We only support references of type service. - if backendRef.Kind == nil || *backendRef.Kind == "Service" { + if backendRef.Kind == nil || *backendRef.Kind == serviceKind { serviceBackend, warn, fatal = serviceLoader(ctx, k8sClient, routeIdentifier, routeKind, backendRef) - } else if string(*backendRef.Kind) == TargetGroupNameBackend { + } else if string(*backendRef.Kind) == targetGroupNameBackend { literalTargetGroup, warn, fatal = literalTargetGroupLoader(backendRef) + } else if string(*backendRef.Kind) == gatewayKind { + gatewayBackend, warn, fatal = gatewayLoader(ctx, k8sClient, routeIdentifier, routeKind, backendRef) } if warn != nil || fatal != nil { return nil, warn, fatal } - if serviceBackend == nil && literalTargetGroup == nil { + if serviceBackend == nil && literalTargetGroup == nil && gatewayBackend == nil { initialErrorMessage := "Unknown backend reference kind" wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonInvalidKind, &wrappedGatewayErrorMessage, nil), nil @@ -165,126 +170,12 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, backendRe } return &Backend{ ServiceBackend: serviceBackend, + GatewayBackend: gatewayBackend, LiteralTargetGroup: literalTargetGroup, Weight: weight, }, nil, nil } -func serviceLoader(ctx context.Context, k8sClient client.Client, routeIdentifier types.NamespacedName, routeKind RouteKind, backendRef gwv1.BackendRef) (*ServiceBackendConfig, error, error) { - if backendRef.Port == nil { - initialErrorMessage := "Port is required" - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedValue, &wrappedGatewayErrorMessage, nil), nil - } - - var svcNamespace string - if backendRef.Namespace == nil { - svcNamespace = routeIdentifier.Namespace - } else { - svcNamespace = string(*backendRef.Namespace) - } - - svcIdentifier := types.NamespacedName{ - Namespace: svcNamespace, - Name: string(backendRef.Name), - } - - // Check for reference grant when performing cross namespace gateway -> route attachment - if svcNamespace != routeIdentifier.Namespace { - allowed, err := referenceGrantCheck(ctx, k8sClient, svcIdentifier, routeIdentifier, routeKind) - if err != nil { - // Currently, this API only fails for a k8s related error message, hence no status update + make the error fatal. - return nil, nil, errors.Wrapf(err, "Unable to perform reference grant check") - } - - // We should not give any hints about the existence of this resource, therefore, we return nil. - // That way, users can't infer if the route is missing because of a misconfigured service reference - // or the sentence grant is not allowing the connection. - if !allowed { - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(referenceGrantNotExists, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", referenceGrantNotExists), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonRefNotPermitted, &wrappedGatewayErrorMessage, nil), nil - } - } - - svc := &corev1.Service{} - err := k8sClient.Get(ctx, svcIdentifier, svc) - if err != nil { - - convertToNotFoundError := client.IgnoreNotFound(err) - - if convertToNotFoundError == nil { - // Svc not found, post an updated status. - initialErrorMessage := fmt.Sprintf("Service (%s:%s) not found)", svcIdentifier.Namespace, svcIdentifier.Name) - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil - } - // Otherwise, general error. No need for status update. - return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier)) - } - - // TODO -- This should be updated, to handle UDP and TCP on the same service port. - // Currently, it will just arbitrarily take one. - - var servicePort *corev1.ServicePort - - for _, svcPort := range svc.Spec.Ports { - if svcPort.Port == int32(*backendRef.Port) { - servicePort = &svcPort - break - } - } - - if servicePort == nil { - initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port) - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil - } - - tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, k8s.NamespacedName(svc)) - - if err != nil { - // As of right now, this error can only be thrown because of a k8s api error hence no status update. - return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) - } - - var tgProps *elbv2gw.TargetGroupProps - - if tgConfig != nil { - tgProps = tgConfigConstructor.ConstructTargetGroupConfigForRoute(tgConfig, routeIdentifier.Name, routeIdentifier.Namespace, string(routeKind)) - } - - // validate if protocol version is compatible with appProtocol - if tgProps != nil && servicePort.AppProtocol != nil { - appProtocol := strings.ToLower(*servicePort.AppProtocol) - if tgProps.ProtocolVersion != nil { - isCompatible := true - switch *tgProps.ProtocolVersion { - case elbv2gw.ProtocolVersionGRPC: - if appProtocol == "http" { - isCompatible = false - } - case elbv2gw.ProtocolVersionHTTP1, elbv2gw.ProtocolVersionHTTP2: - if appProtocol == "grpc" { - isCompatible = false - } - } - if !isCompatible { - initialErrorMessage := fmt.Sprintf("Service port appProtocol %s is not compatible with target group protocolVersion %s", *servicePort.AppProtocol, *tgProps.ProtocolVersion) - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - - // This potentially could be fatal, but let's make the reconcile cycle as resilient as possible. - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedProtocol, &wrappedGatewayErrorMessage, nil), nil - } - } - } - - return &ServiceBackendConfig{ - service: svc, - servicePort: servicePort, - targetGroupProps: tgProps, - }, nil, nil -} - func literalTargetGroupLoader(backendRef gwv1.BackendRef) (*LiteralTargetGroupConfig, error, error) { return &LiteralTargetGroupConfig{ Name: string(backendRef.Name), @@ -293,22 +184,30 @@ func literalTargetGroupLoader(backendRef gwv1.BackendRef) (*LiteralTargetGroupCo // LookUpTargetGroupConfiguration given a service, lookup the target group configuration associated with the service. // recall that target group configuration always lives within the same namespace as the service. -func LookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client, serviceMetadata types.NamespacedName) (*elbv2gw.TargetGroupConfiguration, error) { +func LookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client, objectKind string, objectMetadata types.NamespacedName) (*elbv2gw.TargetGroupConfiguration, error) { tgConfigList := &elbv2gw.TargetGroupConfigurationList{} // TODO - Add index - if err := k8sClient.List(ctx, tgConfigList, client.InNamespace(serviceMetadata.Namespace)); err != nil { + if err := k8sClient.List(ctx, tgConfigList, client.InNamespace(objectMetadata.Namespace)); err != nil { return nil, err } for _, tgConfig := range tgConfigList.Items { - if tgConfig.Spec.TargetReference.Kind != nil && *tgConfig.Spec.TargetReference.Kind != serviceKind { + + var isEligible bool + // Special case, nil kind == Service. + if tgConfig.Spec.TargetReference.Kind == nil && objectKind == serviceKind { + isEligible = true + } else if tgConfig.Spec.TargetReference.Kind != nil && objectKind == *tgConfig.Spec.TargetReference.Kind { + isEligible = true + } + + if !isEligible { continue } - // TODO - Add a webhook to validate that only one target group config references this service. // TODO - Add an index for this - if tgConfig.Spec.TargetReference.Name == serviceMetadata.Name { + if tgConfig.Spec.TargetReference.Name == objectMetadata.Name { return &tgConfig, nil } } @@ -317,9 +216,9 @@ func LookUpTargetGroupConfiguration(ctx context.Context, k8sClient client.Client // Implements the reference grant API // https://gateway-api.sigs.k8s.io/api-types/referencegrant/ -func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdentifier types.NamespacedName, routeIdentifier types.NamespacedName, routeKind RouteKind) (bool, error) { +func referenceGrantCheck(ctx context.Context, k8sClient client.Client, objKind string, objIdentifier types.NamespacedName, routeIdentifier types.NamespacedName, routeKind RouteKind) (bool, error) { referenceGrantList := &gwbeta1.ReferenceGrantList{} - if err := k8sClient.List(ctx, referenceGrantList, client.InNamespace(svcIdentifier.Namespace)); err != nil { + if err := k8sClient.List(ctx, referenceGrantList, client.InNamespace(objIdentifier.Namespace)); err != nil { return false, err } @@ -336,13 +235,13 @@ func referenceGrantCheck(ctx context.Context, k8sClient client.Client, svcIdenti if routeAllowed { for _, to := range grant.Spec.To { - // As this is a backend reference, we only care about the "Service" Kind. - if to.Kind != serviceKind { + // Make sure the kind is correct for our query. + if string(to.Kind) != objKind { continue } // If name is specified, we need to ensure that svc name matches the "to" name. - if to.Name != nil && string(*to.Name) != svcIdentifier.Name { + if to.Name != nil && string(*to.Name) != objIdentifier.Name { continue } diff --git a/pkg/gateway/routeutils/backend_gateway.go b/pkg/gateway/routeutils/backend_gateway.go index 90d5e5490..b74123a16 100644 --- a/pkg/gateway/routeutils/backend_gateway.go +++ b/pkg/gateway/routeutils/backend_gateway.go @@ -1 +1,167 @@ package routeutils + +import ( + "context" + "fmt" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +var _ TargetGroupConfigurator = &GatewayBackendConfig{} + +type GatewayBackendConfig struct { + gateway *gwv1.Gateway + targetGroupProps *elbv2gw.TargetGroupProps + arn string + port int32 +} + +func NewGatewayBackendConfig(gateway *gwv1.Gateway, targetGroupProps *elbv2gw.TargetGroupProps, arn string, port int32) *GatewayBackendConfig { + return &GatewayBackendConfig{ + gateway: gateway, + targetGroupProps: targetGroupProps, + arn: arn, + port: port, + } +} + +func (g *GatewayBackendConfig) GetALBARN() string { + return g.arn +} + +func (g *GatewayBackendConfig) GetTargetType(_ elbv2model.TargetType) elbv2model.TargetType { + return elbv2model.TargetTypeALB +} + +func (g *GatewayBackendConfig) GetTargetGroupProps() *elbv2gw.TargetGroupProps { + return g.targetGroupProps +} + +func (g *GatewayBackendConfig) GetBackendNamespacedName() types.NamespacedName { + return k8s.NamespacedName(g.gateway) +} + +func (g *GatewayBackendConfig) GetIdentifierPort() intstr.IntOrString { + return intstr.FromInt32(g.port) +} + +// GetExternalTrafficPolicy doesn't really apply to this backend type, so we return the most permissive type. +func (g *GatewayBackendConfig) GetExternalTrafficPolicy() corev1.ServiceExternalTrafficPolicyType { + return corev1.ServiceExternalTrafficPolicyTypeCluster +} + +// GetIPAddressType Gateway based backends always communicate over IPv4. +func (g *GatewayBackendConfig) GetIPAddressType() elbv2model.TargetGroupIPAddressType { + return elbv2model.TargetGroupIPAddressTypeIPv4 +} + +// GetTargetGroupPort Gateway based backends always forward traffic to the Gateway listener. +func (g *GatewayBackendConfig) GetTargetGroupPort(_ elbv2model.TargetType) int32 { + return g.port +} + +func (g *GatewayBackendConfig) GetHealthCheckPort(_ elbv2model.TargetType, _ bool) (intstr.IntOrString, error) { + portConfigNotExist := g.targetGroupProps == nil || g.targetGroupProps.HealthCheckConfig == nil || g.targetGroupProps.HealthCheckConfig.HealthCheckPort == nil + + if portConfigNotExist || *g.targetGroupProps.HealthCheckConfig.HealthCheckPort == shared_constants.HealthCheckPortTrafficPort { + return intstr.FromString(shared_constants.HealthCheckPortTrafficPort), nil + } + + return intstr.FromInt32(g.port), nil +} + +func gatewayLoader(ctx context.Context, k8sClient client.Client, routeIdentifier types.NamespacedName, routeKind RouteKind, backendRef gwv1.BackendRef) (*GatewayBackendConfig, error, error) { + if backendRef.Port == nil { + initialErrorMessage := "Port is required" + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedValue, &wrappedGatewayErrorMessage, nil), nil + } + + var gwNamespace string + if backendRef.Namespace == nil { + gwNamespace = routeIdentifier.Namespace + } else { + gwNamespace = string(*backendRef.Namespace) + } + + gwIdentifier := types.NamespacedName{ + Namespace: gwNamespace, + Name: string(backendRef.Name), + } + + // Check for reference grant when performing cross namespace gateway -> route attachment + if gwIdentifier.Namespace != routeIdentifier.Namespace { + allowed, err := referenceGrantCheck(ctx, k8sClient, gatewayKind, gwIdentifier, routeIdentifier, routeKind) + if err != nil { + // Currently, this API only fails for a k8s related error message, hence no status update + make the error fatal. + return nil, nil, errors.Wrapf(err, "Unable to perform reference grant check") + } + + // We should not give any hints about the existence of this resource, therefore, we return nil. + // That way, users can't infer if the route is missing because of a misconfigured gateway reference + // or the sentence grant is not allowing the connection. + if !allowed { + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(referenceGrantNotExists, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", referenceGrantNotExists), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonRefNotPermitted, &wrappedGatewayErrorMessage, nil), nil + } + } + + gw := &gwv1.Gateway{} + err := k8sClient.Get(ctx, gwIdentifier, gw) + if err != nil { + + convertToNotFoundError := client.IgnoreNotFound(err) + + if convertToNotFoundError == nil { + // Svc not found, post an updated status. + initialErrorMessage := fmt.Sprintf("Gateway (%s:%s) not found)", gwIdentifier.Namespace, gwIdentifier.Name) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + // Otherwise, general error. No need for status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch gw object %+v", gw)) + } + + tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, gatewayKind, k8s.NamespacedName(gw)) + + if err != nil { + // As of right now, this error can only be thrown because of a k8s api error hence no status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) + } + + var tgProps *elbv2gw.TargetGroupProps + + if tgConfig != nil { + tgProps = tgConfigConstructor.ConstructTargetGroupConfigForRoute(tgConfig, routeIdentifier.Name, routeIdentifier.Namespace, string(routeKind)) + } + + var arn string + + // Find the ALB ARN within the Gateway Programmed Condition, the controller will always embed the ARN there. + for _, cond := range gw.Status.Conditions { + if cond.Type == string(gwv1.GatewayConditionProgrammed) { + if cond.Status == metav1.ConditionTrue { + arn = cond.Message + break + } + } + } + + if arn == "" { + // If the ARN is not available, then the backend is not yet usable. + initialErrorMessage := fmt.Sprintf("Gateway (%s:%s) is not usable yet, LB ARN is not provisioned)", gwIdentifier.Namespace, gwIdentifier.Name) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + + return NewGatewayBackendConfig(gw, tgProps, arn, int32(*backendRef.Port)), nil, nil +} diff --git a/pkg/gateway/routeutils/backend_gateway_test.go b/pkg/gateway/routeutils/backend_gateway_test.go new file mode 100644 index 000000000..ba496d964 --- /dev/null +++ b/pkg/gateway/routeutils/backend_gateway_test.go @@ -0,0 +1,207 @@ +package routeutils + +import ( + "testing" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" + "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func TestGatewayBackendConfig_GetTargetType(t *testing.T) { + config := &GatewayBackendConfig{} + + tests := []struct { + name string + defaultType elbv2model.TargetType + expectedType elbv2model.TargetType + }{ + { + name: "returns ALB regardless of default", + defaultType: elbv2model.TargetTypeInstance, + expectedType: elbv2model.TargetTypeALB, + }, + { + name: "returns ALB with IP default", + defaultType: elbv2model.TargetTypeIP, + expectedType: elbv2model.TargetTypeALB, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := config.GetTargetType(tt.defaultType) + assert.Equal(t, tt.expectedType, result) + }) + } +} + +func TestGatewayBackendConfig_GetTargetGroupProps(t *testing.T) { + props := &elbv2gw.TargetGroupProps{} + config := &GatewayBackendConfig{targetGroupProps: props} + + assert.Equal(t, props, config.GetTargetGroupProps()) +} + +func TestGatewayBackendConfig_GetBackendNamespacedName(t *testing.T) { + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gateway", + Namespace: "test-namespace", + }, + } + config := &GatewayBackendConfig{gateway: gateway} + + expected := types.NamespacedName{ + Name: "test-gateway", + Namespace: "test-namespace", + } + + assert.Equal(t, expected, config.GetBackendNamespacedName()) +} + +func TestGatewayBackendConfig_GetIdentifierPort(t *testing.T) { + config := &GatewayBackendConfig{port: 8080} + + expected := intstr.FromInt32(8080) + assert.Equal(t, expected, config.GetIdentifierPort()) +} + +func TestGatewayBackendConfig_GetExternalTrafficPolicy(t *testing.T) { + config := &GatewayBackendConfig{} + + result := config.GetExternalTrafficPolicy() + assert.Equal(t, corev1.ServiceExternalTrafficPolicyTypeCluster, result) +} + +func TestGatewayBackendConfig_GetIPAddressType(t *testing.T) { + config := &GatewayBackendConfig{} + + result := config.GetIPAddressType() + assert.Equal(t, elbv2model.TargetGroupIPAddressTypeIPv4, result) +} + +func TestGatewayBackendConfig_GetTargetGroupPort(t *testing.T) { + config := &GatewayBackendConfig{port: 9090} + + tests := []struct { + name string + targetType elbv2model.TargetType + expected int32 + }{ + { + name: "ALB target type", + targetType: elbv2model.TargetTypeALB, + expected: 9090, + }, + { + name: "Instance target type", + targetType: elbv2model.TargetTypeInstance, + expected: 9090, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := config.GetTargetGroupPort(tt.targetType) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestGatewayBackendConfig_GetHealthCheckPort(t *testing.T) { + tests := []struct { + name string + targetGroupProps *elbv2gw.TargetGroupProps + port int32 + targetType elbv2model.TargetType + useNodePort bool + expectedPort intstr.IntOrString + expectedError bool + }{ + { + name: "no target group props", + targetGroupProps: nil, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "no health check config", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: nil, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "no health check port config", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: nil, + }, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "health check port set to traffic-port", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String(shared_constants.HealthCheckPortTrafficPort), + }, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromString(shared_constants.HealthCheckPortTrafficPort), + expectedError: false, + }, + { + name: "health check port set to custom value", + targetGroupProps: &elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{ + HealthCheckPort: awssdk.String("9090"), + }, + }, + port: 8080, + targetType: elbv2model.TargetTypeALB, + useNodePort: false, + expectedPort: intstr.FromInt32(8080), + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := &GatewayBackendConfig{ + targetGroupProps: tt.targetGroupProps, + port: tt.port, + } + + result, err := config.GetHealthCheckPort(tt.targetType, tt.useNodePort) + + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedPort, result) + } + }) + } +} diff --git a/pkg/gateway/routeutils/backend_service.go b/pkg/gateway/routeutils/backend_service.go index 4ebe4b46e..295473bf7 100644 --- a/pkg/gateway/routeutils/backend_service.go +++ b/pkg/gateway/routeutils/backend_service.go @@ -1,6 +1,8 @@ package routeutils import ( + "context" + "fmt" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -9,6 +11,9 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2" "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strings" ) type ServiceBackendConfig struct { @@ -17,6 +22,8 @@ type ServiceBackendConfig struct { servicePort *corev1.ServicePort } +var _ TargetGroupConfigurator = &ServiceBackendConfig{} + func NewServiceBackendConfig(service *corev1.Service, targetGroupProps *elbv2gw.TargetGroupProps, servicePort *corev1.ServicePort) *ServiceBackendConfig { return &ServiceBackendConfig{ service: service, @@ -25,6 +32,14 @@ func NewServiceBackendConfig(service *corev1.Service, targetGroupProps *elbv2gw. } } +func (s *ServiceBackendConfig) GetTargetType(defaultTargetType elbv2model.TargetType) elbv2model.TargetType { + if s.targetGroupProps == nil || s.targetGroupProps.TargetType == nil { + return defaultTargetType + } + + return elbv2model.TargetType(*s.targetGroupProps.TargetType) +} + func (s *ServiceBackendConfig) GetHealthCheckPort(targetType elbv2model.TargetType, isServiceExternalTrafficPolicyTypeLocal bool) (intstr.IntOrString, error) { portConfigNotExist := s.targetGroupProps == nil || s.targetGroupProps.HealthCheckConfig == nil || s.targetGroupProps.HealthCheckConfig.HealthCheckPort == nil @@ -105,4 +120,113 @@ func (s *ServiceBackendConfig) GetTargetGroupProps() *elbv2gw.TargetGroupProps { return s.targetGroupProps } -var _ TargetGroupConfigurator = &ServiceBackendConfig{} +func serviceLoader(ctx context.Context, k8sClient client.Client, routeIdentifier types.NamespacedName, routeKind RouteKind, backendRef gwv1.BackendRef) (*ServiceBackendConfig, error, error) { + if backendRef.Port == nil { + initialErrorMessage := "Port is required" + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedValue, &wrappedGatewayErrorMessage, nil), nil + } + + var svcNamespace string + if backendRef.Namespace == nil { + svcNamespace = routeIdentifier.Namespace + } else { + svcNamespace = string(*backendRef.Namespace) + } + + svcIdentifier := types.NamespacedName{ + Namespace: svcNamespace, + Name: string(backendRef.Name), + } + + // Check for reference grant when performing cross namespace gateway -> route attachment + if svcNamespace != routeIdentifier.Namespace { + allowed, err := referenceGrantCheck(ctx, k8sClient, serviceKind, svcIdentifier, routeIdentifier, routeKind) + if err != nil { + // Currently, this API only fails for a k8s related error message, hence no status update + make the error fatal. + return nil, nil, errors.Wrapf(err, "Unable to perform reference grant check") + } + + // We should not give any hints about the existence of this resource, therefore, we return nil. + // That way, users can't infer if the route is missing because of a misconfigured service reference + // or the sentence grant is not allowing the connection. + if !allowed { + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(referenceGrantNotExists, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", referenceGrantNotExists), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonRefNotPermitted, &wrappedGatewayErrorMessage, nil), nil + } + } + + svc := &corev1.Service{} + err := k8sClient.Get(ctx, svcIdentifier, svc) + if err != nil { + + convertToNotFoundError := client.IgnoreNotFound(err) + + if convertToNotFoundError == nil { + // Svc not found, post an updated status. + initialErrorMessage := fmt.Sprintf("Service (%s:%s) not found)", svcIdentifier.Namespace, svcIdentifier.Name) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + // Otherwise, general error. No need for status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier)) + } + + // TODO -- This should be updated, to handle UDP and TCP on the same service port. + // Currently, it will just arbitrarily take one. + + var servicePort *corev1.ServicePort + + for _, svcPort := range svc.Spec.Ports { + if svcPort.Port == int32(*backendRef.Port) { + servicePort = &svcPort + break + } + } + + if servicePort == nil { + initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + } + + tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, serviceKind, k8s.NamespacedName(svc)) + + if err != nil { + // As of right now, this error can only be thrown because of a k8s api error hence no status update. + return nil, nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) + } + + var tgProps *elbv2gw.TargetGroupProps + + if tgConfig != nil { + tgProps = tgConfigConstructor.ConstructTargetGroupConfigForRoute(tgConfig, routeIdentifier.Name, routeIdentifier.Namespace, string(routeKind)) + } + + // validate if protocol version is compatible with appProtocol + if tgProps != nil && servicePort.AppProtocol != nil { + appProtocol := strings.ToLower(*servicePort.AppProtocol) + if tgProps.ProtocolVersion != nil { + isCompatible := true + switch *tgProps.ProtocolVersion { + case elbv2gw.ProtocolVersionGRPC: + if appProtocol == "http" { + isCompatible = false + } + case elbv2gw.ProtocolVersionHTTP1, elbv2gw.ProtocolVersionHTTP2: + if appProtocol == "grpc" { + isCompatible = false + } + } + if !isCompatible { + initialErrorMessage := fmt.Sprintf("Service port appProtocol %s is not compatible with target group protocolVersion %s", *servicePort.AppProtocol, *tgProps.ProtocolVersion) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + + // This potentially could be fatal, but let's make the reconcile cycle as resilient as possible. + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonUnsupportedProtocol, &wrappedGatewayErrorMessage, nil), nil + } + } + } + + return NewServiceBackendConfig(svc, tgProps, servicePort), nil, nil +} diff --git a/pkg/gateway/routeutils/backend_service_test.go b/pkg/gateway/routeutils/backend_service_test.go index 41e89e209..f3c91d93a 100644 --- a/pkg/gateway/routeutils/backend_service_test.go +++ b/pkg/gateway/routeutils/backend_service_test.go @@ -243,3 +243,24 @@ func Test_buildTargetGroupHealthCheckPort(t *testing.T) { }) } } + +func Test_buildTargetGroupTargetType(t *testing.T) { + svcBackend := NewServiceBackendConfig(nil, nil, nil) + + res := svcBackend.GetTargetType(elbv2model.TargetTypeIP) + assert.Equal(t, elbv2model.TargetTypeIP, res) + + svcBackendEmptyProps := NewServiceBackendConfig(nil, &elbv2gw.TargetGroupProps{}, nil) + + res = svcBackend.GetTargetType(elbv2model.TargetTypeIP) + + res = svcBackendEmptyProps.GetTargetType(elbv2model.TargetTypeIP) + assert.Equal(t, elbv2model.TargetTypeIP, res) + + inst := elbv2gw.TargetTypeInstance + svcBackendWithProps := NewServiceBackendConfig(nil, &elbv2gw.TargetGroupProps{ + TargetType: &inst, + }, nil) + res = svcBackendWithProps.GetTargetType(elbv2model.TargetTypeIP) + assert.Equal(t, elbv2model.TargetTypeInstance, res) +} diff --git a/pkg/gateway/routeutils/backend_test.go b/pkg/gateway/routeutils/backend_test.go index 8b2eae6bd..2d2f4fe2e 100644 --- a/pkg/gateway/routeutils/backend_test.go +++ b/pkg/gateway/routeutils/backend_test.go @@ -425,7 +425,7 @@ func TestCommonBackendLoader_TargetGroupName(t *testing.T) { name: "valid name", backendRef: gwv1.BackendRef{ BackendObjectReference: gwv1.BackendObjectReference{ - Kind: (*gwv1.Kind)(awssdk.String(TargetGroupNameBackend)), + Kind: (*gwv1.Kind)(awssdk.String(targetGroupNameBackend)), Name: "foo", }, }, @@ -462,12 +462,14 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { testCases := []struct { name string allTargetGroupConfigurations []elbv2gw.TargetGroupConfiguration - serviceMetadata types.NamespacedName + objectMetadata types.NamespacedName + kind string expectErr bool expectedTGConfiguration *elbv2gw.TargetGroupConfiguration }{ { - name: "happy path, exactly one tg config", + name: "happy path, exactly one tg config - service", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -482,7 +484,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -499,8 +501,43 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, + { + name: "happy path, exactly one tg config - gateway", + kind: gatewayKind, + allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(gatewayKind), + Name: "svc1", + }, + }, + }, + }, + objectMetadata: types.NamespacedName{ + Namespace: "namespace", + Name: "svc1", + }, + expectedTGConfiguration: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tg1", + Namespace: "namespace", + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Kind: awssdk.String(gatewayKind), + Name: "svc1", + }, + }, + }, + }, { name: "happy path, exactly one tg config (kind not specified)", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -514,7 +551,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -532,6 +569,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, { name: "sad path, svc name different", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -546,13 +584,14 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, }, { name: "sad path, kind not supported", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -567,13 +606,14 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, }, { name: "sad path, many tg none match", + kind: serviceKind, allTargetGroupConfigurations: []elbv2gw.TargetGroupConfiguration{ { ObjectMeta: metav1.ObjectMeta{ @@ -612,7 +652,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, }, }, - serviceMetadata: types.NamespacedName{ + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -620,7 +660,8 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { }, { name: "sad path, no tg none match", - serviceMetadata: types.NamespacedName{ + kind: serviceKind, + objectMetadata: types.NamespacedName{ Namespace: "namespace", Name: "svc1", }, @@ -636,7 +677,7 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { assert.NoError(t, err) } - result, err := LookUpTargetGroupConfiguration(context.Background(), k8sClient, tc.serviceMetadata) + result, err := LookUpTargetGroupConfiguration(context.Background(), k8sClient, tc.kind, tc.objectMetadata) if tc.expectErr { assert.Error(t, err) @@ -655,16 +696,18 @@ func Test_lookUpTargetGroupConfiguration(t *testing.T) { func Test_referenceGrantCheck(t *testing.T) { kind := HTTPRouteKind testCases := []struct { - name string - referenceGrants []gwbeta1.ReferenceGrant - svcIdentifier types.NamespacedName - routeIdentifier types.NamespacedName - expected bool - expectErr bool + name string + kind string + referenceGrants []gwbeta1.ReferenceGrant + objectIdentifier types.NamespacedName + routeIdentifier types.NamespacedName + expected bool + expectErr bool }{ { - name: "happy path", - svcIdentifier: types.NamespacedName{ + name: "happy path - service", + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -696,9 +739,45 @@ func Test_referenceGrantCheck(t *testing.T) { }, expected: true, }, + { + name: "happy path - gateway", + kind: gatewayKind, + objectIdentifier: types.NamespacedName{ + Namespace: "gw-namespace", + Name: "gw-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "gw-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: gatewayKind, + Name: (*gwbeta1.ObjectName)(awssdk.String("gw-name")), + }, + }, + }, + }, + }, + expected: true, + }, { name: "happy path (no name equals wildcard)", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -731,7 +810,8 @@ func Test_referenceGrantCheck(t *testing.T) { }, { name: "no grants, should not allow", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -743,7 +823,8 @@ func Test_referenceGrantCheck(t *testing.T) { }, { name: "from is allowed, but not to", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -777,7 +858,8 @@ func Test_referenceGrantCheck(t *testing.T) { }, { name: "to is allowed, but not from", - svcIdentifier: types.NamespacedName{ + kind: serviceKind, + objectIdentifier: types.NamespacedName{ Namespace: "svc-namespace", Name: "svc-name", }, @@ -808,6 +890,41 @@ func Test_referenceGrantCheck(t *testing.T) { }, expected: false, }, + { + name: "reference grant is for wrong type", + kind: gatewayKind, + objectIdentifier: types.NamespacedName{ + Namespace: "gw-namespace", + Name: "gw-name", + }, + routeIdentifier: types.NamespacedName{ + Namespace: "route-namespace", + Name: "route-name", + }, + referenceGrants: []gwbeta1.ReferenceGrant{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "gw-namespace", + Name: "grant1", + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Kind: gwbeta1.Kind(kind), + Namespace: "route-namespace", + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: serviceKind, + Name: (*gwbeta1.ObjectName)(awssdk.String("gw-name")), + }, + }, + }, + }, + }, + expected: false, + }, } for _, tc := range testCases { @@ -818,7 +935,7 @@ func Test_referenceGrantCheck(t *testing.T) { assert.NoError(t, err) } - result, err := referenceGrantCheck(context.Background(), k8sClient, tc.svcIdentifier, tc.routeIdentifier, kind) + result, err := referenceGrantCheck(context.Background(), k8sClient, tc.kind, tc.objectIdentifier, tc.routeIdentifier, kind) if tc.expectErr { assert.Error(t, err) return diff --git a/pkg/gateway/routeutils/constants.go b/pkg/gateway/routeutils/constants.go index 7a18b048b..956339188 100644 --- a/pkg/gateway/routeutils/constants.go +++ b/pkg/gateway/routeutils/constants.go @@ -19,7 +19,7 @@ const ( ) const ( - TargetGroupNameBackend string = "TargetGroupName" + targetGroupNameBackend string = "TargetGroupName" ) // RouteKind to Route Loader. These functions will pull data directly from the kube api or local cache. diff --git a/pkg/gateway/routeutils/utils.go b/pkg/gateway/routeutils/utils.go index 7a148d558..619d44651 100644 --- a/pkg/gateway/routeutils/utils.go +++ b/pkg/gateway/routeutils/utils.go @@ -79,6 +79,9 @@ func FilterRoutesBySvc(routes []preLoadRouteDescriptor, svc *corev1.Service) []p // Assuming we are only supporting services as backendRefs on Routes func isServiceReferredByRoute(route preLoadRouteDescriptor, svcID types.NamespacedName) bool { for _, backendRef := range route.GetBackendRefs() { + if backendRef.Kind != nil && *backendRef.Kind != "Service" { + continue + } namespace := route.GetRouteNamespacedName().Namespace if backendRef.Namespace != nil { namespace = string(*backendRef.Namespace) diff --git a/pkg/model/elbv2/frontend_nlb_target_group.go b/pkg/model/elbv2/frontend_nlb_target_group.go index 667d225e6..cf3cd6f44 100644 --- a/pkg/model/elbv2/frontend_nlb_target_group.go +++ b/pkg/model/elbv2/frontend_nlb_target_group.go @@ -10,10 +10,12 @@ const ( // FrontendNlbTargetGroupState represents the state of a single ALB Target Type target group with its ALB target type FrontendNlbTargetGroupState struct { - Name string - ARN core.StringToken - Port int32 - TargetARN core.StringToken + Name string + ARN core.StringToken + // Port -> NLB Listener Port + Port int32 + TargetARN core.StringToken + // TargetPort -> ALB Listener Port TargetPort int32 } From 2f442de9890aed680962efe236f2a4c44280a98f Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 3 Nov 2025 17:27:28 -0800 Subject: [PATCH 05/10] fix enum check, the casing was incorrect --- apis/gateway/v1beta1/targetgroupconfig_types.go | 4 ++-- config/crd/gateway/gateway-crds.yaml | 12 ++++++------ .../gateway.k8s.aws_targetgroupconfigurations.yaml | 12 ++++++------ .../crds/gateway-crds.yaml | 12 ++++++------ 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/apis/gateway/v1beta1/targetgroupconfig_types.go b/apis/gateway/v1beta1/targetgroupconfig_types.go index 680f96a69..15c1bae8d 100644 --- a/apis/gateway/v1beta1/targetgroupconfig_types.go +++ b/apis/gateway/v1beta1/targetgroupconfig_types.go @@ -20,7 +20,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Reference defines how to look up the Target Group configuration for a service. +// Reference defines how to look up the Target Group configuration for a kubernetes object. type Reference struct { // Group is the group of the referent. For example, "gateway.networking.k8s.io". // When unspecified or empty string, core API group is inferred. @@ -108,7 +108,7 @@ const ( TargetTypeIP TargetType = "ip" ) -// +kubebuilder:validation:Enum=http;https;tcp +// +kubebuilder:validation:Enum=HTTP;HTTPS;TCP type TargetGroupHealthCheckProtocol string const ( diff --git a/config/crd/gateway/gateway-crds.yaml b/config/crd/gateway/gateway-crds.yaml index 106f8bd7d..93af18d9c 100644 --- a/config/crd/gateway/gateway-crds.yaml +++ b/config/crd/gateway/gateway-crds.yaml @@ -821,9 +821,9 @@ spec: with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, in seconds, @@ -1014,9 +1014,9 @@ spec: and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, diff --git a/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml b/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml index c7b9a6cd7..8327eaf0e 100644 --- a/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml +++ b/config/crd/gateway/gateway.k8s.aws_targetgroupconfigurations.yaml @@ -82,9 +82,9 @@ spec: with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, in seconds, @@ -275,9 +275,9 @@ spec: and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, diff --git a/helm/aws-load-balancer-controller/crds/gateway-crds.yaml b/helm/aws-load-balancer-controller/crds/gateway-crds.yaml index 106f8bd7d..93af18d9c 100644 --- a/helm/aws-load-balancer-controller/crds/gateway-crds.yaml +++ b/helm/aws-load-balancer-controller/crds/gateway-crds.yaml @@ -821,9 +821,9 @@ spec: with the target. The GENEVE, TLS, UDP, and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, in seconds, @@ -1014,9 +1014,9 @@ spec: and TCP_UDP protocols are not supported for health checks. enum: - - http - - https - - tcp + - HTTP + - HTTPS + - TCP type: string healthCheckTimeout: description: healthCheckTimeout The amount of time, From 76da6d4c343a63effcfab87eb8b82afb39167fc2 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 3 Nov 2025 17:29:22 -0800 Subject: [PATCH 06/10] add validation to ensure it is really an ALB gateway --- pkg/deploy/stack_deployer.go | 4 +- pkg/gateway/routeutils/backend_gateway.go | 19 +++++++- .../routeutils/backend_gateway_test.go | 43 +++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index 92adca5d8..12fa5ea96 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -122,9 +122,7 @@ func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack, met if d.enableFrontendNLB { var desiredFENLBState []*elbv2model.FrontendNlbTargetGroupDesiredState - err := stack.ListResources(&desiredFENLBState) - d.logger.Info(fmt.Sprintf("Got this result!!! %+v %+v", desiredFENLBState, err)) - + stack.ListResources(&desiredFENLBState) var frontendNLBState *elbv2model.FrontendNlbTargetGroupDesiredState if len(desiredFENLBState) == 1 { frontendNLBState = desiredFENLBState[0] diff --git a/pkg/gateway/routeutils/backend_gateway.go b/pkg/gateway/routeutils/backend_gateway.go index b74123a16..4fcc3be7c 100644 --- a/pkg/gateway/routeutils/backend_gateway.go +++ b/pkg/gateway/routeutils/backend_gateway.go @@ -14,6 +14,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strings" ) var _ TargetGroupConfigurator = &GatewayBackendConfig{} @@ -160,8 +161,24 @@ func gatewayLoader(ctx context.Context, k8sClient client.Client, routeIdentifier // If the ARN is not available, then the backend is not yet usable. initialErrorMessage := fmt.Sprintf("Gateway (%s:%s) is not usable yet, LB ARN is not provisioned)", gwIdentifier.Namespace, gwIdentifier.Name) wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil + // This needs to be a fatal error, otherwise we will not run another reconcile cycle to pick up the ARN. + return nil, nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil) + } + + err = validateGatewayARN(arn) + if err != nil { + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(err.Error(), routeKind, routeIdentifier) + // This can be a warning, as we know that retrying reconcile will do nothing to fix this situation. + return nil, wrapError(err, gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil), nil } return NewGatewayBackendConfig(gw, tgProps, arn, int32(*backendRef.Port)), nil, nil } + +func validateGatewayARN(arn string) error { + parts := strings.Split(arn, "/") + if len(parts) < 2 || parts[1] != "app" { + return errors.Errorf("invalid gateway ARN: %s, the resource type must be application load balancer", arn) + } + return nil +} diff --git a/pkg/gateway/routeutils/backend_gateway_test.go b/pkg/gateway/routeutils/backend_gateway_test.go index ba496d964..1c32eccd7 100644 --- a/pkg/gateway/routeutils/backend_gateway_test.go +++ b/pkg/gateway/routeutils/backend_gateway_test.go @@ -205,3 +205,46 @@ func TestGatewayBackendConfig_GetHealthCheckPort(t *testing.T) { }) } } + +func TestValidateGatewayARN(t *testing.T) { + tests := []struct { + name string + arn string + wantErr bool + }{ + { + name: "valid ALB ARN", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer/app/k8s-echoserv-testgwal-3c92fc24ed/9604d5627427405c", + wantErr: false, + }, + { + name: "invalid NLB ARN", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer/net/my-nlb/1234567890123456", + wantErr: true, + }, + { + name: "invalid format - no slashes", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer", + wantErr: true, + }, + { + name: "invalid format - only one part", + arn: "arn:aws:elasticloadbalancing:us-east-1:565768096483:loadbalancer/", + wantErr: true, + }, + { + name: "empty string", + arn: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateGatewayARN(tt.arn) + if (err != nil) != tt.wantErr { + t.Errorf("validateGatewayARN() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From 819879ecb41971bb4934be278430e3af60123e80 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 3 Nov 2025 20:25:04 -0800 Subject: [PATCH 07/10] handle tg config updates for gateway based backends --- .../target_group_configuration_events.go | 105 ++++++-- .../target_group_configuration_events_test.go | 232 ++++++++++++++++++ controllers/gateway/gateway_controller.go | 4 +- 3 files changed, 315 insertions(+), 26 deletions(-) create mode 100644 controllers/gateway/eventhandlers/target_group_configuration_events_test.go diff --git a/controllers/gateway/eventhandlers/target_group_configuration_events.go b/controllers/gateway/eventhandlers/target_group_configuration_events.go index 5867b090b..c1a2dbc24 100644 --- a/controllers/gateway/eventhandlers/target_group_configuration_events.go +++ b/controllers/gateway/eventhandlers/target_group_configuration_events.go @@ -5,6 +5,7 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" @@ -13,16 +14,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // NewEnqueueRequestsForTargetGroupConfigurationEvent creates handler for TargetGroupConfiguration resources -func NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan chan<- event.TypedGenericEvent[*corev1.Service], +func NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan chan<- event.TypedGenericEvent[*corev1.Service], tcpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TCPRoute], k8sClient client.Client, eventRecorder record.EventRecorder, logger logr.Logger) handler.TypedEventHandler[*elbv2gw.TargetGroupConfiguration, reconcile.Request] { return &enqueueRequestsForTargetGroupConfigurationEvent{ - svcEventChan: svcEventChan, - k8sClient: k8sClient, - eventRecorder: eventRecorder, - logger: logger, + svcEventChan: svcEventChan, + tcpRouteEventChan: tcpRouteEventChan, + k8sClient: k8sClient, + eventRecorder: eventRecorder, + logger: logger, } } @@ -30,49 +33,103 @@ var _ handler.TypedEventHandler[*elbv2gw.TargetGroupConfiguration, reconcile.Req // enqueueRequestsForTargetGroupConfigurationEvent handles TargetGroupConfiguration events type enqueueRequestsForTargetGroupConfigurationEvent struct { - svcEventChan chan<- event.TypedGenericEvent[*corev1.Service] - k8sClient client.Client - eventRecorder record.EventRecorder - logger logr.Logger + svcEventChan chan<- event.TypedGenericEvent[*corev1.Service] + tcpRouteEventChan chan<- event.TypedGenericEvent[*gwalpha2.TCPRoute] + k8sClient client.Client + eventRecorder record.EventRecorder + logger logr.Logger } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Create(ctx context.Context, e event.TypedCreateEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfigNew := e.Object h.logger.V(1).Info("enqueue targetgroupconfiguration create event", "targetgroupconfiguration", tgconfigNew.Name) - h.enqueueImpactedService(ctx, tgconfigNew, queue) + h.enqueueImpactedObject(ctx, tgconfigNew, queue) } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfigNew := e.ObjectNew h.logger.V(1).Info("enqueue targetgroupconfiguration update event", "targetgroupconfiguration", tgconfigNew.Name) - h.enqueueImpactedService(ctx, tgconfigNew, queue) + h.enqueueImpactedObject(ctx, tgconfigNew, queue) } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfig := e.Object h.logger.V(1).Info("enqueue targetgroupconfiguration delete event", "targetgroupconfiguration", tgconfig.Name) - h.enqueueImpactedService(ctx, tgconfig, queue) + h.enqueueImpactedObject(ctx, tgconfig, queue) } func (h *enqueueRequestsForTargetGroupConfigurationEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*elbv2gw.TargetGroupConfiguration], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { tgconfig := e.Object h.logger.V(1).Info("enqueue targetgroupconfiguration generic event", "targetgroupconfiguration", tgconfig.Name) - h.enqueueImpactedService(ctx, tgconfig, queue) + h.enqueueImpactedObject(ctx, tgconfig, queue) } -func (h *enqueueRequestsForTargetGroupConfigurationEvent) enqueueImpactedService(ctx context.Context, tgconfig *elbv2gw.TargetGroupConfiguration, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { - svcName := types.NamespacedName{Namespace: tgconfig.Namespace, Name: tgconfig.Spec.TargetReference.Name} - svc := &corev1.Service{} - if err := h.k8sClient.Get(ctx, svcName, svc); err != nil { - h.logger.V(1).Info("ignoring targetgroupconfiguration event for unknown service", +func (h *enqueueRequestsForTargetGroupConfigurationEvent) enqueueImpactedObject(ctx context.Context, tgconfig *elbv2gw.TargetGroupConfiguration, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + objName := types.NamespacedName{Namespace: tgconfig.Namespace, Name: tgconfig.Spec.TargetReference.Name} + + if tgconfig.Spec.TargetReference.Kind == nil || *tgconfig.Spec.TargetReference.Kind == "Service" { + svc := &corev1.Service{} + if err := h.k8sClient.Get(ctx, objName, svc); err != nil { + h.logger.V(1).Info("ignoring targetgroupconfiguration event for unknown service", + "targetgroupconfiguration", k8s.NamespacedName(tgconfig), + "service", k8s.NamespacedName(svc)) + return + } + h.logger.V(1).Info("enqueue service for targetgroupconfiguration event", "targetgroupconfiguration", k8s.NamespacedName(tgconfig), "service", k8s.NamespacedName(svc)) - return + h.svcEventChan <- event.TypedGenericEvent[*corev1.Service]{ + Object: svc, + } + } + + // TODO - We should probably use an indexer here, we have a task to do this. + if tgconfig.Spec.TargetReference.Kind != nil && *tgconfig.Spec.TargetReference.Kind == "Gateway" && h.tcpRouteEventChan != nil { + tcpRouteList := &gwalpha2.TCPRouteList{} + + if err := h.k8sClient.List(ctx, tcpRouteList); err != nil { + h.logger.V(1).Info("failed to list tcp routes for target group configuration event", "targetgroupconfiguration", k8s.NamespacedName(tgconfig)) + return + } + + impactedRoutes := getImpactedTCPRoutes(tcpRouteList, tgconfig) + for i := range impactedRoutes { + h.tcpRouteEventChan <- event.TypedGenericEvent[*gwalpha2.TCPRoute]{ + Object: impactedRoutes[i], + } + } + } - h.logger.V(1).Info("enqueue service for targetgroupconfiguration event", - "targetgroupconfiguration", k8s.NamespacedName(tgconfig), - "service", k8s.NamespacedName(svc)) - h.svcEventChan <- event.TypedGenericEvent[*corev1.Service]{ - Object: svc, +} + +func getImpactedTCPRoutes(list *gwalpha2.TCPRouteList, tgconfig *elbv2gw.TargetGroupConfiguration) []*gwalpha2.TCPRoute { + seen := sets.Set[types.NamespacedName]{} + res := make([]*gwalpha2.TCPRoute, 0) + + for i, route := range list.Items { + nsn := k8s.NamespacedName(&route) + for _, rule := range route.Spec.Rules { + for _, beRef := range rule.BackendRefs { + if beRef.Kind != nil && *beRef.Kind == "Gateway" { + if string(beRef.Name) == tgconfig.Spec.TargetReference.Name { + + // The route backend ns + var routeNs string + if beRef.Namespace == nil { + routeNs = route.Namespace + } else { + routeNs = string(*beRef.Namespace) + } + + if routeNs == tgconfig.Namespace && !seen.Has(nsn) { + res = append(res, &list.Items[i]) + seen.Insert(nsn) + } + + } + } + } + } } + return res } diff --git a/controllers/gateway/eventhandlers/target_group_configuration_events_test.go b/controllers/gateway/eventhandlers/target_group_configuration_events_test.go new file mode 100644 index 000000000..25c925c52 --- /dev/null +++ b/controllers/gateway/eventhandlers/target_group_configuration_events_test.go @@ -0,0 +1,232 @@ +package eventhandlers + +import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "testing" +) + +func TestGetImpactedTCPRoutes(t *testing.T) { + tests := []struct { + name string + list *gwalpha2.TCPRouteList + tgconfig *elbv2gw.TargetGroupConfiguration + want []types.NamespacedName + }{ + { + name: "no routes", + list: &gwalpha2.TCPRouteList{}, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{}, + }, + { + name: "matching gateway backend", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "test-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{ + {Name: "route1", Namespace: "test-ns"}, + }, + }, + { + name: "non-matching gateway name", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "test-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "other-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{}, + }, + { + name: "different namespace", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "other-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{}, + }, + { + name: "cross-namespace with explicit namespace", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "route-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + Namespace: (*gwalpha2.Namespace)(awssdk.String("test-ns")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{ + {Name: "route1", Namespace: "route-ns"}, + }, + }, + { + name: "duplicate routes filtered", + list: &gwalpha2.TCPRouteList{ + Items: []gwalpha2.TCPRoute{ + { + ObjectMeta: metav1.ObjectMeta{Name: "route1", Namespace: "test-ns"}, + Spec: gwalpha2.TCPRouteSpec{ + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: "test-gateway", + Kind: (*gwalpha2.Kind)(awssdk.String("Gateway")), + }, + }, + }, + }, + }, + }, + }, + }, + }, + tgconfig: &elbv2gw.TargetGroupConfiguration{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test-ns"}, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Name: "test-gateway", + Kind: awssdk.String("Gateway"), + }, + }, + }, + want: []types.NamespacedName{ + {Name: "route1", Namespace: "test-ns"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getImpactedTCPRoutes(tt.list, tt.tgconfig) + res := make([]types.NamespacedName, 0) + + for i := range got { + res = append(res, k8s.NamespacedName(got[i])) + } + + assert.Equal(t, tt.want, res) + }) + } +} diff --git a/controllers/gateway/gateway_controller.go b/controllers/gateway/gateway_controller.go index 13b4ef3e3..d2aa22226 100644 --- a/controllers/gateway/gateway_controller.go +++ b/controllers/gateway/gateway_controller.go @@ -531,7 +531,7 @@ func (r *gatewayReconciler) setupALBGatewayControllerWatches(ctrl controller.Con grpcRouteEventChan := make(chan event.TypedGenericEvent[*gwv1.GRPCRoute]) svcEventChan := make(chan event.TypedGenericEvent[*corev1.Service]) secretEventsChan := make(chan event.TypedGenericEvent[*corev1.Secret]) - tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, r.k8sClient, r.eventRecorder, + tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, nil, r.k8sClient, r.eventRecorder, loggerPrefix.WithName("TargetGroupConfiguration")) listenerRuleConfigEventHandler := eventhandlers.NewEnqueueRequestsForListenerRuleConfigurationEvent(httpRouteEventChan, grpcRouteEventChan, r.k8sClient, loggerPrefix.WithName("ListenerRuleConfiguration")) grpcRouteEventHandler := eventhandlers.NewEnqueueRequestsForGRPCRouteEvent(r.k8sClient, r.eventRecorder, @@ -591,7 +591,7 @@ func (r *gatewayReconciler) setupNLBGatewayControllerWatches(ctrl controller.Con udpRouteEventChan := make(chan event.TypedGenericEvent[*gwalpha2.UDPRoute]) tlsRouteEventChan := make(chan event.TypedGenericEvent[*gwalpha2.TLSRoute]) svcEventChan := make(chan event.TypedGenericEvent[*corev1.Service]) - tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, r.k8sClient, r.eventRecorder, + tgConfigEventHandler := eventhandlers.NewEnqueueRequestsForTargetGroupConfigurationEvent(svcEventChan, tcpRouteEventChan, r.k8sClient, r.eventRecorder, loggerPrefix.WithName("TargetGroupConfiguration")) tcpRouteEventHandler := eventhandlers.NewEnqueueRequestsForTCPRouteEvent(r.k8sClient, r.eventRecorder, loggerPrefix.WithName("TCPRoute")) From 66fa78cfafe38ee70449d75cec5f4b9d14a8148e Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 3 Nov 2025 20:25:23 -0800 Subject: [PATCH 08/10] add e2e test for gateway api alb target of nlb --- test/e2e/gateway/alb_nlb_test.go | 277 ++++ test/e2e/gateway/nlb_test_helper.go | 116 ++ .../gateway/shared_resource_definitions.go | 33 + test/e2e/ingress/vanilla_ingress_test.go | 1468 ++++++++--------- 4 files changed, 1158 insertions(+), 736 deletions(-) create mode 100644 test/e2e/gateway/alb_nlb_test.go diff --git a/test/e2e/gateway/alb_nlb_test.go b/test/e2e/gateway/alb_nlb_test.go new file mode 100644 index 000000000..4872bcc07 --- /dev/null +++ b/test/e2e/gateway/alb_nlb_test.go @@ -0,0 +1,277 @@ +package gateway + +import ( + "context" + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "strings" + "time" +) + +var _ = Describe("test combined ALB and NLB gateways with HTTPRoute and TCPRoute", func() { + var ( + ctx context.Context + albStack ALBTestStack + nlbStack NLBTestStack + ) + + BeforeEach(func() { + if !tf.Options.EnableGatewayTests { + Skip("Skipping gateway tests") + } + ctx = context.Background() + albStack = ALBTestStack{} + nlbStack = NLBTestStack{} + }) + + AfterEach(func() { + albStack.Cleanup(ctx, tf) + nlbStack.Cleanup(ctx, tf) + }) + + Context("with ALB and NLB gateways using IP targets", func() { + var albDnsName string + var albARN string + var nlbDnsName string + var nlbARN string + var refGrant *gwbeta1.ReferenceGrant + It("should provision both ALB and NLB load balancers with HTTPRoute and TCPRoute", func() { + // ALB Configuration + albInterf := elbv2gw.LoadBalancerSchemeInternal + albLbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ + Scheme: &albInterf, + } + + // NLB Configuration + nlbInterf := elbv2gw.LoadBalancerSchemeInternetFacing + nlbLbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ + Scheme: &nlbInterf, + } + + // Configure TLS for both if certificates are available + var hasTLS bool + if len(tf.Options.CertificateARNs) > 0 { + cert := strings.Split(tf.Options.CertificateARNs, ",")[0] + + // ALB HTTPS listener + albLsConfig := elbv2gw.ListenerConfiguration{ + ProtocolPort: "HTTPS:443", + DefaultCertificate: &cert, + } + albLbcSpec.ListenerConfigurations = &[]elbv2gw.ListenerConfiguration{albLsConfig} + hasTLS = true + } + + // IP target type for both + ipTargetType := elbv2gw.TargetTypeIP + tgSpec := elbv2gw.TargetGroupConfigurationSpec{ + DefaultConfiguration: elbv2gw.TargetGroupProps{ + TargetType: &ipTargetType, + }, + } + lrcSpec := elbv2gw.ListenerRuleConfigurationSpec{} + + // ALB Gateway listeners + albGwListeners := []gwv1.Listener{ + { + Name: "http80", + Port: 80, + Protocol: gwv1.HTTPProtocolType, + }, + } + if hasTLS { + albGwListeners = append(albGwListeners, gwv1.Listener{ + Name: "https443", + Port: 443, + Protocol: gwv1.HTTPSProtocolType, + }) + } + + // HTTPRoute for ALB + httpr := buildHTTPRoute([]string{}, []gwv1.HTTPRouteRule{}, nil) + + By("deploying ALB stack", func() { + err := albStack.DeployHTTP(ctx, nil, tf, albGwListeners, []*gwv1.HTTPRoute{httpr}, albLbcSpec, tgSpec, lrcSpec, nil, true) + Expect(err).NotTo(HaveOccurred()) + }) + + By("deploying NLB stack", func() { + err := nlbStack.DeployFrontendNLB(ctx, albStack, tf, nlbLbcSpec, hasTLS, true) + Expect(err).NotTo(HaveOccurred()) + }) + By("checking alb gateway status for lb dns name", func() { + time.Sleep(2 * time.Minute) + albDnsName = albStack.GetLoadBalancerIngressHostName() + Expect(albDnsName).ToNot(BeEmpty()) + }) + By("querying AWS loadbalancer from the dns name", func() { + var err error + albARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, albDnsName) + Expect(err).NotTo(HaveOccurred()) + Expect(albARN).ToNot(BeEmpty()) + }) + By("checking nlb gateway status for lb dns name", func() { + nlbDnsName = nlbStack.GetLoadBalancerIngressHostName() + Expect(nlbDnsName).ToNot(BeEmpty()) + }) + By("querying AWS loadbalancer from the dns name", func() { + var err error + nlbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, nlbDnsName) + Expect(err).NotTo(HaveOccurred()) + Expect(nlbARN).ToNot(BeEmpty()) + }) + By("verify alb configuration", func() { + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "HTTP", + Port: 80, + NumTargets: int(*albStack.albResourceStack.commonStack.dps[0].Spec.Replicas), + TargetType: "ip", + TargetGroupHC: DEFAULT_ALB_TARGET_GROUP_HC, + }, + } + + listenerPortMap := albStack.albResourceStack.getListenersPortMap() + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, albARN, verifier.LoadBalancerExpectation{ + Type: "application", + Scheme: "internal", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("verify nlb configuration", func() { + // No ref grants, means no tg or listener. + expectedTargetGroups := []verifier.ExpectedTargetGroup{} + + listenerPortMap := map[string]string{} + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, nlbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("deploy reference grant that allows nlb <-> alb attachment", func() { + var err error + refGrant, err = nlbStack.CreateFENLBReferenceGrant(ctx, tf, albStack.albResourceStack.commonStack.ns) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(2 * time.Minute) + }) + By("deploy reference grant that allows nlb <-> alb attachment", func() { + // No ref grants, means no tg or listener. + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "TCP", + Port: 80, + NumTargets: 1, + TargetType: "alb", + TargetGroupHC: &verifier.TargetGroupHC{ + Protocol: "HTTP", + Port: "traffic-port", + Path: "/", + Interval: 15, + Timeout: 5, + HealthyThreshold: 3, + UnhealthyThreshold: 3, + }, + }, + } + + if hasTLS { + expectedTargetGroups = append(expectedTargetGroups, verifier.ExpectedTargetGroup{ + Protocol: "TCP", + Port: 443, + NumTargets: 1, + TargetType: "alb", + TargetGroupHC: &verifier.TargetGroupHC{ + Protocol: "HTTPS", + Port: "traffic-port", + Path: "/", + Interval: 15, + Timeout: 5, + HealthyThreshold: 3, + UnhealthyThreshold: 3, + }, + }) + } + + fmt.Printf("%+v\n", refGrant) + + listenerPortMap := nlbStack.nlbResourceStack.getListenersPortMap() + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, nlbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("verify port 80 works", func() { + url := fmt.Sprintf("http://%v/any-path", nlbDnsName) + err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + }) + if hasTLS { + By("verify port 443 works", func() { + url := fmt.Sprintf("https://%v/any-path", nlbDnsName) + urlOptions := http.URLOptions{ + InsecureSkipVerify: true, + } + err := tf.HTTPVerifier.VerifyURLWithOptions(url, urlOptions, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + }) + } + By("remove reference grant should remove nlb listener but keep alb listener intact", func() { + err := tf.K8sClient.Delete(ctx, refGrant) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(2 * time.Minute) + }) + By("verify alb configuration", func() { + expectedTargetGroups := []verifier.ExpectedTargetGroup{ + { + Protocol: "HTTP", + Port: 80, + NumTargets: int(*albStack.albResourceStack.commonStack.dps[0].Spec.Replicas), + TargetType: "ip", + TargetGroupHC: DEFAULT_ALB_TARGET_GROUP_HC, + }, + } + + listenerPortMap := albStack.albResourceStack.getListenersPortMap() + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, albARN, verifier.LoadBalancerExpectation{ + Type: "application", + Scheme: "internal", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + By("verify nlb configuration", func() { + // No ref grants, means no tg or listener. + expectedTargetGroups := []verifier.ExpectedTargetGroup{} + + listenerPortMap := map[string]string{} + + err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, nlbARN, verifier.LoadBalancerExpectation{ + Type: "network", + Scheme: "internet-facing", + Listeners: listenerPortMap, + TargetGroups: expectedTargetGroups, + }) + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) +}) diff --git a/test/e2e/gateway/nlb_test_helper.go b/test/e2e/gateway/nlb_test_helper.go index 9c79198df..5fdca62ae 100644 --- a/test/e2e/gateway/nlb_test_helper.go +++ b/test/e2e/gateway/nlb_test_helper.go @@ -2,14 +2,18 @@ package gateway import ( "context" + "fmt" + awssdk "github.com/aws/aws-sdk-go-v2/aws" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/test/framework" gwv1 "sigs.k8s.io/gateway-api/apis/v1" gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) type NLBTestStack struct { @@ -78,6 +82,118 @@ func (s *NLBTestStack) Deploy(ctx context.Context, f *framework.Framework, auxil return s.nlbResourceStack.Deploy(ctx, f) } +func (s *NLBTestStack) DeployFrontendNLB(ctx context.Context, albStack ALBTestStack, f *framework.Framework, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, hasTLS bool, readinessGateEnabled bool) error { + gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb") + + if f.Options.IPFamily == framework.IPv6 { + v6 := elbv2gw.LoadBalancerIpAddressTypeDualstack + lbConfSpec.IpAddressType = &v6 + } + + listeners := []gwv1.Listener{ + { + Name: "port80", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + } + + tcprs := []*gwalpha2.TCPRoute{buildFENLBTCPRoute(albStack.albResourceStack.commonStack.gw.Name, albStack.albResourceStack.commonStack.gw.Namespace, gwalpha2.PortNumber(80))} + + if hasTLS { + listeners = append(listeners, gwv1.Listener{ + Name: "port443", + Port: 443, + Protocol: gwv1.TCPProtocolType, + }) + tcpForHTTPS := buildFENLBTCPRoute(albStack.albResourceStack.commonStack.gw.Name, albStack.albResourceStack.commonStack.gw.Namespace, gwalpha2.PortNumber(443)) + tcprs = append(tcprs, tcpForHTTPS) + + } + + gw := buildBasicGatewaySpec(gwc, listeners) + + lbc := buildLoadBalancerConfig(lbConfSpec) + + s.nlbResourceStack = newNLBResourceStack([]*appsv1.Deployment{}, []*corev1.Service{}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{}, tcprs, []*gwalpha2.UDPRoute{}, nil, "nlb-gateway-e2e", readinessGateEnabled) + + err := s.nlbResourceStack.Deploy(ctx, f) + if err != nil { + return err + } + + // The special TGC is just to support HTTPS and HTTP health check routes on the same underlying gateway. + if !hasTLS { + return nil + } + + // Hack to get TargetGroupConfiguration working correctly, as it needs the namespace which is allocated in the deploy step. + + http := elbv2gw.TargetGroupHealthCheckProtocolHTTP + https := elbv2gw.TargetGroupHealthCheckProtocolHTTPS + + return createTargetGroupConfigs(ctx, f, []*elbv2gw.TargetGroupConfiguration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("alb-https-hc-config"), + Namespace: albStack.albResourceStack.commonStack.gw.Namespace, + }, + Spec: elbv2gw.TargetGroupConfigurationSpec{ + TargetReference: elbv2gw.Reference{ + Group: nil, + Kind: awssdk.String("Gateway"), + Name: albStack.albResourceStack.commonStack.gw.Name, + }, + DefaultConfiguration: elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{HealthCheckProtocol: &http}, + }, + RouteConfigurations: []elbv2gw.RouteConfiguration{ + { + RouteIdentifier: elbv2gw.RouteIdentifier{ + RouteName: tcprs[1].Name, + RouteNamespace: tcprs[1].Namespace, + RouteKind: "TCPRoute", + }, + TargetGroupProps: elbv2gw.TargetGroupProps{ + HealthCheckConfig: &elbv2gw.HealthCheckConfiguration{HealthCheckProtocol: &https}, + }, + }, + }, + }, + }, + }) +} + +func (s *NLBTestStack) CreateFENLBReferenceGrant(ctx context.Context, f *framework.Framework, albNamespace *corev1.Namespace) (*gwbeta1.ReferenceGrant, error) { + refGrant := &gwbeta1.ReferenceGrant{ + + ObjectMeta: metav1.ObjectMeta{ + Name: "refgrant-fe-nlb", + Namespace: albNamespace.Name, + }, + Spec: gwbeta1.ReferenceGrantSpec{ + From: []gwbeta1.ReferenceGrantFrom{ + { + Group: gwbeta1.Group(gwbeta1.GroupName), + Kind: gwbeta1.Kind("TCPRoute"), + Namespace: gwbeta1.Namespace(s.nlbResourceStack.commonStack.ns.Name), + }, + }, + To: []gwbeta1.ReferenceGrantTo{ + { + Kind: "Gateway", + }, + }, + }, + } + + if err := createReferenceGrants(ctx, f, []*gwbeta1.ReferenceGrant{refGrant}); err != nil { + return nil, err + } + + return refGrant, nil +} + func (s *NLBTestStack) Cleanup(ctx context.Context, f *framework.Framework) { s.nlbResourceStack.Cleanup(ctx, f) } diff --git a/test/e2e/gateway/shared_resource_definitions.go b/test/e2e/gateway/shared_resource_definitions.go index 1f3b943d5..87fa4b487 100644 --- a/test/e2e/gateway/shared_resource_definitions.go +++ b/test/e2e/gateway/shared_resource_definitions.go @@ -315,6 +315,39 @@ func buildTCPRoute() *gwalpha2.TCPRoute { return tcpr } +func buildFENLBTCPRoute(albGatewayName, albNamespace string, port gwalpha2.PortNumber) *gwalpha2.TCPRoute { + tcpr := &gwalpha2.TCPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("fenlb-tcp-route-%d", port), + }, + Spec: gwalpha2.TCPRouteSpec{ + CommonRouteSpec: gwalpha2.CommonRouteSpec{ + ParentRefs: []gwv1.ParentReference{ + { + Name: defaultName, + Port: &port, + }, + }, + }, + Rules: []gwalpha2.TCPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: gwv1.ObjectName(albGatewayName), + Kind: (*gwv1.Kind)(awssdk.String("Gateway")), + Namespace: (*gwv1.Namespace)(&albNamespace), + Port: &port, + }, + }, + }, + }, + }, + }, + } + return tcpr +} + func buildUDPRoute() *gwalpha2.UDPRoute { port := gwalpha2.PortNumber(8080) udpr := &gwalpha2.UDPRoute{ diff --git a/test/e2e/ingress/vanilla_ingress_test.go b/test/e2e/ingress/vanilla_ingress_test.go index 3d1291207..8fda3cf90 100644 --- a/test/e2e/ingress/vanilla_ingress_test.go +++ b/test/e2e/ingress/vanilla_ingress_test.go @@ -4,12 +4,9 @@ import ( "context" "fmt" "net/http" - "strings" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" "github.com/gavv/httpexpect/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -17,7 +14,6 @@ import ( corev1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/test/framework" "sigs.k8s.io/aws-load-balancer-controller/test/framework/fixture" @@ -71,833 +67,833 @@ var _ = Describe("vanilla ingress tests", func() { }) } }) - - Context("with basic settings", func() { - It("[ingress-class] in IP mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + /* + Context("with basic settings", func() { + It("[ingress-class] in IP mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - ingClass := &networking.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxNS.Name, - }, - Spec: networking.IngressClassSpec{ - Controller: "ingress.k8s.aws/alb", - }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithIngressClassName(ingClass.Name). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + ingClass := &networking.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxNS.Name, + }, + Spec: networking.IngressClassSpec{ + Controller: "ingress.k8s.aws/alb", + }, + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithIngressClassName(ingClass.Name). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) - It("[ingress-class] in Instance mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) + It("[ingress-class] in Instance mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - ingClass := &networking.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxNS.Name, - }, - Spec: networking.IngressClassSpec{ - Controller: "ingress.k8s.aws/alb", - }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithIngressClassName(ingClass.Name). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + ingClass := &networking.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxNS.Name, + }, + Spec: networking.IngressClassSpec{ + Controller: "ingress.k8s.aws/alb", + }, + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithIngressClassName(ingClass.Name). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) - It("ingress in IP mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + It("ingress in IP mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) - It("ingress in Instance mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) + It("ingress in Instance mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) }) - }) - Context("with IngressClass variant settings", func() { - It("[ingress-class] with IngressClass configured with 'nginx' controller, no ALB shall be created", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + Context("with IngressClass variant settings", func() { + It("[ingress-class] with IngressClass configured with 'nginx' controller, no ALB shall be created", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - ingClass := &networking.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxNS.Name, - }, - Spec: networking.IngressClassSpec{ - Controller: "kubernetes.io/nginx", - }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithIngressClassName(ingClass.Name). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + ingClass := &networking.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxNS.Name, + }, + Spec: networking.IngressClassSpec{ + Controller: "kubernetes.io/nginx", + }, + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithIngressClassName(ingClass.Name). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - ExpectNoLBProvisionedForIngress(ctx, tf, ing) - }) + ExpectNoLBProvisionedForIngress(ctx, tf, ing) + }) - It("with 'kubernetes.io/ingress.class' annotation set to 'nginx', no ALB shall be created", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + It("with 'kubernetes.io/ingress.class' annotation set to 'nginx', no ALB shall be created", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "nginx", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "nginx", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - ExpectNoLBProvisionedForIngress(ctx, tf, ing) - }) + ExpectNoLBProvisionedForIngress(ctx, tf, ing) + }) - It("without IngressClass or 'kubernetes.io/ingress.class' annotation, no ALB shall be created", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + It("without IngressClass or 'kubernetes.io/ingress.class' annotation, no ALB shall be created", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - ExpectNoLBProvisionedForIngress(ctx, tf, ing) + ExpectNoLBProvisionedForIngress(ctx, tf, ing) + }) }) - }) - Context("with `alb.ingress.kubernetes.io/load-balancer-name` variant settings", func() { - It("with 'alb.ingress.kubernetes.io/load-balancer-name' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + Context("with `alb.ingress.kubernetes.io/load-balancer-name` variant settings", func() { + It("with 'alb.ingress.kubernetes.io/load-balancer-name' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - safeClusterName := strings.ReplaceAll(tf.Options.ClusterName, ".", "-") - lbName := fmt.Sprintf("%.16s-%.15s", safeClusterName, sandboxNS.Name) - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/load-balancer-name": lbName, - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + safeClusterName := strings.ReplaceAll(tf.Options.ClusterName, ".", "-") + lbName := fmt.Sprintf("%.16s-%.15s", safeClusterName, sandboxNS.Name) + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/load-balancer-name": lbName, + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - sdkLB, err := tf.LBManager.GetLoadBalancerFromARN(ctx, lbARN) - Expect(err).NotTo(HaveOccurred()) - Expect(awssdk.ToString(sdkLB.LoadBalancerName)).Should(Equal(lbName)) + sdkLB, err := tf.LBManager.GetLoadBalancerFromARN(ctx, lbARN) + Expect(err).NotTo(HaveOccurred()) + Expect(awssdk.ToString(sdkLB.LoadBalancerName)).Should(Equal(lbName)) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) }) - }) - Context("with ALB IP targets and named target port", func() { - It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + Context("with ALB IP targets and named target port", func() { + It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) - - defer resStack.TearDown(ctx) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + defer resStack.TearDown(ctx) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) - }) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - Context("with ALB IP targets, named target port and endPointSlices enabled", func() { - BeforeEach(func() { - ctx = context.Background() - if tf.Options.ControllerImage != "" { - By(fmt.Sprintf("upgrade controller with endPointSlices enabled."), func() { - err := tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage, true) - Expect(err).NotTo(HaveOccurred()) - time.Sleep(60 * time.Second) - }) - } - }) - It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, and endPointSlices enabled, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) + }) + + Context("with ALB IP targets, named target port and endPointSlices enabled", func() { + BeforeEach(func() { + ctx = context.Background() + if tf.Options.ControllerImage != "" { + By(fmt.Sprintf("upgrade controller with endPointSlices enabled."), func() { + err := tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage, true) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(60 * time.Second) + }) + } + }) + It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, and endPointSlices enabled, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) }) - }) - Context("with `alb.ingress.kubernetes.io/actions.${action-name}` variant settings", func() { - It("with annotation based actions, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp1, svc1 := appBuilder.WithHTTPBody("app-1").Build(sandboxNS.Name, "app-1", tf.Options.TestImageRegistry) - dp2, svc2 := appBuilder.WithHTTPBody("app-2").Build(sandboxNS.Name, "app-2", tf.Options.TestImageRegistry) - ingResponse503Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "response-503", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + Context("with `alb.ingress.kubernetes.io/actions.${action-name}` variant settings", func() { + It("with annotation based actions, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp1, svc1 := appBuilder.WithHTTPBody("app-1").Build(sandboxNS.Name, "app-1", tf.Options.TestImageRegistry) + dp2, svc2 := appBuilder.WithHTTPBody("app-2").Build(sandboxNS.Name, "app-2", tf.Options.TestImageRegistry) + ingResponse503Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "response-503", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingRedirectToAWSBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "redirect-to-aws", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingRedirectToAWSBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "redirect-to-aws", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingForwardSingleTGBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "forward-single-tg", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingForwardSingleTGBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "forward-single-tg", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingForwardMultipleTGBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "forward-multiple-tg", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingForwardMultipleTGBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "forward-multiple-tg", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/actions.response-503": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"503\",\"messageBody\":\"503 error text\"}}", - "alb.ingress.kubernetes.io/actions.redirect-to-aws": "{\"type\":\"redirect\",\"redirectConfig\":{\"host\":\"aws.amazon.com\",\"path\":\"/eks/\",\"port\":\"443\",\"protocol\":\"HTTPS\",\"query\":\"k=v\",\"statusCode\":\"HTTP_302\"}}", - "alb.ingress.kubernetes.io/actions.forward-single-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\"}]}}", - "alb.ingress.kubernetes.io/actions.forward-multiple-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\",\"weight\":20},{\"serviceName\":\"app-2\",\"servicePort\":80,\"weight\":80}],\"targetGroupStickinessConfig\":{\"enabled\":true,\"durationSeconds\":200}}}", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/response-503", PathType: &exact, Backend: ingResponse503Backend}). - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/redirect-to-aws", PathType: &exact, Backend: ingRedirectToAWSBackend}). - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-single-tg", PathType: &exact, Backend: ingForwardSingleTGBackend}). - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-multiple-tg", PathType: &exact, Backend: ingForwardMultipleTGBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp1, svc1, dp2, svc2, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/actions.response-503": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"503\",\"messageBody\":\"503 error text\"}}", + "alb.ingress.kubernetes.io/actions.redirect-to-aws": "{\"type\":\"redirect\",\"redirectConfig\":{\"host\":\"aws.amazon.com\",\"path\":\"/eks/\",\"port\":\"443\",\"protocol\":\"HTTPS\",\"query\":\"k=v\",\"statusCode\":\"HTTP_302\"}}", + "alb.ingress.kubernetes.io/actions.forward-single-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\"}]}}", + "alb.ingress.kubernetes.io/actions.forward-multiple-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\",\"weight\":20},{\"serviceName\":\"app-2\",\"servicePort\":80,\"weight\":80}],\"targetGroupStickinessConfig\":{\"enabled\":true,\"durationSeconds\":200}}}", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/response-503", PathType: &exact, Backend: ingResponse503Backend}). + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/redirect-to-aws", PathType: &exact, Backend: ingRedirectToAWSBackend}). + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-single-tg", PathType: &exact, Backend: ingForwardSingleTGBackend}). + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-multiple-tg", PathType: &exact, Backend: ingForwardMultipleTGBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp1, svc1, dp2, svc2, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/response-503").Expect(). - Status(http.StatusServiceUnavailable). - Body().Equal("503 error text") - httpExp.GET("/redirect-to-aws").WithRedirectPolicy(httpexpect.DontFollowRedirects).Expect(). - Status(http.StatusFound). - Header("Location").Equal("https://aws.amazon.com:443/eks/?k=v") - httpExp.GET("/forward-single-tg").Expect(). - Status(http.StatusOK). - Body().Equal("app-1") - httpExp.GET("/forward-multiple-tg").Expect(). - Status(http.StatusOK). - Body().Match("app-1|app-2") + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/response-503").Expect(). + Status(http.StatusServiceUnavailable). + Body().Equal("503 error text") + httpExp.GET("/redirect-to-aws").WithRedirectPolicy(httpexpect.DontFollowRedirects).Expect(). + Status(http.StatusFound). + Header("Location").Equal("https://aws.amazon.com:443/eks/?k=v") + httpExp.GET("/forward-single-tg").Expect(). + Status(http.StatusOK). + Body().Equal("app-1") + httpExp.GET("/forward-multiple-tg").Expect(). + Status(http.StatusOK). + Body().Match("app-1|app-2") + }) }) - }) - Context("with `alb.ingress.kubernetes.io/conditions.${conditions-name}` variant settings", func() { - It("with annotation based conditions, one ALB shall be created and functional", func() { - ingBuilder := manifest.NewIngressBuilder() - ingRulePath1Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path1", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + Context("with `alb.ingress.kubernetes.io/conditions.${conditions-name}` variant settings", func() { + It("with annotation based conditions, one ALB shall be created and functional", func() { + ingBuilder := manifest.NewIngressBuilder() + ingRulePath1Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path1", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingRulePath2Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path2", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingRulePath2Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path2", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingRulePath3Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path3", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingRulePath3Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path3", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingRulePath4Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path4", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingRulePath4Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path4", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingRulePath5Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path5", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingRulePath5Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path5", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingRulePath6Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path6", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingRulePath6Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path6", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } - ingRulePath7Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path7", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", + } + ingRulePath7Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path7", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", + }, }, - }, - } + } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/actions.rule-path1": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Host is www.example.com OR anno.example.com\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path1": "[{\"field\":\"host-header\",\"hostHeaderConfig\":{\"values\":[\"anno.example.com\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path2": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Path is /path2 OR /anno/path2\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path2": "[{\"field\":\"path-pattern\",\"pathPatternConfig\":{\"values\":[\"/anno/path2\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path3": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http header HeaderName is HeaderValue1 OR HeaderValue2\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path3": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue1\", \"HeaderValue2\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path4": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http request method is GET OR HEAD\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path4": "[{\"field\":\"http-request-method\",\"httpRequestMethodConfig\":{\"Values\":[\"GET\", \"HEAD\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path5": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Query string is paramA:valueA1 OR paramA:valueA2\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path5": "[{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA1\"},{\"key\":\"paramA\",\"value\":\"valueA2\"}]}}]", - "alb.ingress.kubernetes.io/actions.rule-path6": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Source IP is 192.168.0.0/16 OR 172.16.0.0/16\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path6": "[{\"field\":\"source-ip\",\"sourceIpConfig\":{\"values\":[\"192.168.0.0/16\", \"172.16.0.0/16\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path7": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"multiple conditions applies\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path7": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue\"]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA\"}]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramB\",\"value\":\"valueB\"}]}}]", - } - if tf.Options.IPFamily == "framework.IPv6" { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path1", PathType: &exact, Backend: ingRulePath1Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path2", PathType: &exact, Backend: ingRulePath2Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path3", PathType: &exact, Backend: ingRulePath3Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path4", PathType: &exact, Backend: ingRulePath4Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path5", PathType: &exact, Backend: ingRulePath5Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path6", PathType: &exact, Backend: ingRulePath6Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path7", PathType: &exact, Backend: ingRulePath7Backend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/actions.rule-path1": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Host is www.example.com OR anno.example.com\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path1": "[{\"field\":\"host-header\",\"hostHeaderConfig\":{\"values\":[\"anno.example.com\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path2": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Path is /path2 OR /anno/path2\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path2": "[{\"field\":\"path-pattern\",\"pathPatternConfig\":{\"values\":[\"/anno/path2\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path3": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http header HeaderName is HeaderValue1 OR HeaderValue2\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path3": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue1\", \"HeaderValue2\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path4": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http request method is GET OR HEAD\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path4": "[{\"field\":\"http-request-method\",\"httpRequestMethodConfig\":{\"Values\":[\"GET\", \"HEAD\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path5": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Query string is paramA:valueA1 OR paramA:valueA2\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path5": "[{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA1\"},{\"key\":\"paramA\",\"value\":\"valueA2\"}]}}]", + "alb.ingress.kubernetes.io/actions.rule-path6": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Source IP is 192.168.0.0/16 OR 172.16.0.0/16\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path6": "[{\"field\":\"source-ip\",\"sourceIpConfig\":{\"values\":[\"192.168.0.0/16\", \"172.16.0.0/16\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path7": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"multiple conditions applies\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path7": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue\"]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA\"}]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramB\",\"value\":\"valueB\"}]}}]", + } + if tf.Options.IPFamily == "framework.IPv6" { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path1", PathType: &exact, Backend: ingRulePath1Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path2", PathType: &exact, Backend: ingRulePath2Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path3", PathType: &exact, Backend: ingRulePath3Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path4", PathType: &exact, Backend: ingRulePath4Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path5", PathType: &exact, Backend: ingRulePath5Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path6", PathType: &exact, Backend: ingRulePath6Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path7", PathType: &exact, Backend: ingRulePath7Backend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path1").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Host is www.example.com OR anno.example.com") - httpExp.GET("/path1").WithHost("anno.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Host is www.example.com OR anno.example.com") - httpExp.GET("/path1").WithHost("other.example.com").Expect(). - Status(http.StatusNotFound) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path1").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Host is www.example.com OR anno.example.com") + httpExp.GET("/path1").WithHost("anno.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Host is www.example.com OR anno.example.com") + httpExp.GET("/path1").WithHost("other.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path2").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Path is /path2 OR /anno/path2") - httpExp.GET("/anno/path2").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Path is /path2 OR /anno/path2") - httpExp.GET("/other/path2").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path2").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Path is /path2 OR /anno/path2") + httpExp.GET("/anno/path2").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Path is /path2 OR /anno/path2") + httpExp.GET("/other/path2").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue1").Expect(). - Status(http.StatusOK). - Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") - httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue2").Expect(). - Status(http.StatusOK). - Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") - httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue3").Expect(). - Status(http.StatusNotFound) - httpExp.GET("/path3").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue1").Expect(). + Status(http.StatusOK). + Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") + httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue2").Expect(). + Status(http.StatusOK). + Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") + httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue3").Expect(). + Status(http.StatusNotFound) + httpExp.GET("/path3").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path4").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Http request method is GET OR HEAD") - httpExp.HEAD("/path4").WithHost("www.example.com").Expect(). - Status(http.StatusOK) - httpExp.POST("/path4").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path4").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Http request method is GET OR HEAD") + httpExp.HEAD("/path4").WithHost("www.example.com").Expect(). + Status(http.StatusOK) + httpExp.POST("/path4").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA1").Expect(). - Status(http.StatusOK). - Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") - httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA2").Expect(). - Status(http.StatusOK). - Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") - httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA3").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA1").Expect(). + Status(http.StatusOK). + Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") + httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA2").Expect(). + Status(http.StatusOK). + Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") + httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA3").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path6").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path6").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path7").WithHost("www.example.com"). - WithHeader("HeaderName", "HeaderValue"). - WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). - Status(http.StatusOK). - Body().Equal("multiple conditions applies") - httpExp.GET("/path7").WithHost("www.example.com"). - WithHeader("HeaderName", "OtherHeaderValue"). - WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). - Status(http.StatusNotFound) - httpExp.GET("/path7").WithHost("www.example.com"). - WithHeader("HeaderName", "HeaderValue"). - WithQuery("paramA", "valueB").WithQuery("paramB", "valueB").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path7").WithHost("www.example.com"). + WithHeader("HeaderName", "HeaderValue"). + WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). + Status(http.StatusOK). + Body().Equal("multiple conditions applies") + httpExp.GET("/path7").WithHost("www.example.com"). + WithHeader("HeaderName", "OtherHeaderValue"). + WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). + Status(http.StatusNotFound) + httpExp.GET("/path7").WithHost("www.example.com"). + WithHeader("HeaderName", "HeaderValue"). + WithQuery("paramA", "valueB").WithQuery("paramB", "valueB").Expect(). + Status(http.StatusNotFound) + }) }) - }) - Context("with `alb.ingress.kubernetes.io/ip-address-type` variant settings", func() { - It("with 'alb.ingress.kubernetes.io/ip-address-type' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + Context("with `alb.ingress.kubernetes.io/ip-address-type` variant settings", func() { + It("with 'alb.ingress.kubernetes.io/ip-address-type' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - "alb.ingress.kubernetes.io/ip-address-type": "ipv4", - } + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + "alb.ingress.kubernetes.io/ip-address-type": "ipv4", + } - if tf.Options.IPFamily == framework.IPv6 { - // TODO: annotate to "dualstack-without-public-ipv4" for all regions once it's GA - if tf.Options.AWSRegion == "us-west-2" { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack-without-public-ipv4" - } else { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + if tf.Options.IPFamily == framework.IPv6 { + // TODO: annotate to "dualstack-without-public-ipv4" for all regions once it's GA + if tf.Options.AWSRegion == "us-west-2" { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack-without-public-ipv4" + } else { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } } - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - //TODO: update the traffic test for dualstack-without-public-ipv4 ALB - // as it may need additional setup compared to dualstack ALB - if annotation["alb.ingress.kubernetes.io/ip-address-type"] != "dualstack-without-public-ipv4" { - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - } + //TODO: update the traffic test for dualstack-without-public-ipv4 ALB + // as it may need additional setup compared to dualstack ALB + if annotation["alb.ingress.kubernetes.io/ip-address-type"] != "dualstack-without-public-ipv4" { + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + } + }) }) - }) - Context("with `alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}` variant settings", func() { - It("with 'alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + Context("with `alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}` variant settings", func() { + It("with 'alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, - "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=false", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=false", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - sdkListeners, err := tf.LBManager.GetLoadBalancerListeners(ctx, lbARN) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + sdkListeners, err := tf.LBManager.GetLoadBalancerListeners(ctx, lbARN) - Eventually(func() bool { - return verifyListenerAttributes(ctx, tf, *sdkListeners[0].ListenerArn, map[string]string{ - "routing.http.response.server.enabled": "false", - }) == nil - }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) + Eventually(func() bool { + return verifyListenerAttributes(ctx, tf, *sdkListeners[0].ListenerArn, map[string]string{ + "routing.http.response.server.enabled": "false", + }) == nil + }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) }) - }) - Context("When created with managed alb with tags provided via annotations", func() { - It("ALB will have correct tags", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, + Context("When created with managed alb with tags provided via annotations", func() { + It("ALB will have correct tags", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, + }, }, - }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/tags": "k1=v1,k2=v2", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/tags": "k1=v1,k2=v2", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, _ := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, _ := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // Verify tags - expectedTags := map[string]string{ - "k1": "v1", - "k2": "v2", - } - // Verify tags using DescribeTags - var tagDescriptions []elbv2types.TagDescription - Eventually(func() (bool, error) { - req := &elasticloadbalancingv2.DescribeTagsInput{ - ResourceArns: []string{lbARN}, - } - resp, err := tf.Cloud.ELBV2().DescribeTagsWithContext(ctx, req) - if err != nil { - return false, err - } - if len(resp.TagDescriptions) > 0 { - tagDescriptions = resp.TagDescriptions - return true, nil + // Verify tags + expectedTags := map[string]string{ + "k1": "v1", + "k2": "v2", } - return false, nil - }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) - - // At this point we should have exactly one TagDescription since we only queried one LB - Expect(tagDescriptions).To(HaveLen(1), "Expected exactly one TagDescription") - foundTags := 0 - for _, tag := range tagDescriptions[0].Tags { - if val, ok := expectedTags[awssdk.ToString(tag.Key)]; ok && val == awssdk.ToString(tag.Value) { - foundTags++ + // Verify tags using DescribeTags + var tagDescriptions []elbv2types.TagDescription + Eventually(func() (bool, error) { + req := &elasticloadbalancingv2.DescribeTagsInput{ + ResourceArns: []string{lbARN}, + } + resp, err := tf.Cloud.ELBV2().DescribeTagsWithContext(ctx, req) + if err != nil { + return false, err + } + if len(resp.TagDescriptions) > 0 { + tagDescriptions = resp.TagDescriptions + return true, nil + } + return false, nil + }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) + + // At this point we should have exactly one TagDescription since we only queried one LB + Expect(tagDescriptions).To(HaveLen(1), "Expected exactly one TagDescription") + foundTags := 0 + for _, tag := range tagDescriptions[0].Tags { + if val, ok := expectedTags[awssdk.ToString(tag.Key)]; ok && val == awssdk.ToString(tag.Value) { + foundTags++ + } } - } - Expect(foundTags).To(Equal(len(expectedTags)), "Not all expected tags were found on the ALB") + Expect(foundTags).To(Equal(len(expectedTags)), "Not all expected tags were found on the ALB") + }) }) - }) - + */ Context("with frontend NLB enabled", func() { It("should create a frontend NLB and route traffic correctly", func() { appBuilder := manifest.NewFixedResponseServiceBuilder() From 2f550bea6f6e35b59409f5e29691d3280a2bde3c Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Mon, 3 Nov 2025 20:25:47 -0800 Subject: [PATCH 09/10] remove ununused value --- .../elbv2/frontend_nlb_target_synthesizer.go | 1 - .../model/model_build_listener_test.go | 28 ++++++------------- .../model/model_build_target_group_test.go | 1 - 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go index 64e46632e..e2ee67eea 100644 --- a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go +++ b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go @@ -146,7 +146,6 @@ func (s *frontendNlbTargetSynthesizer) PostSynthesize(ctx context.Context) error }) if err != nil { - s.logger.Error(err, "Got this error!") requeueMsg := "Failed to register target, retrying after deplay for target group: " + resTG.Status.TargetGroupARN return ctrlerrors.NewRequeueNeededAfter(requeueMsg, 15*time.Second) } diff --git a/pkg/gateway/model/model_build_listener_test.go b/pkg/gateway/model/model_build_listener_test.go index e0738b4f5..0eb414e0c 100644 --- a/pkg/gateway/model/model_build_listener_test.go +++ b/pkg/gateway/model/model_build_listener_test.go @@ -1317,11 +1317,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - //Service: &corev1.Service{}, - //ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, }, @@ -1390,11 +1387,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - //Service: &corev1.Service{}, - //ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, }, @@ -1450,11 +1444,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - //Service: &corev1.Service{}, - //ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, ListenerRuleConfig: &elbv2gw.ListenerRuleConfiguration{ @@ -1525,11 +1516,8 @@ func Test_BuildListenerRules(t *testing.T) { }, BackendRefs: []routeutils.Backend{ { - ServiceBackend: &routeutils.ServiceBackendConfig{ - //Service: &corev1.Service{}, - //ServicePort: &corev1.ServicePort{Name: "svcport"}, - }, - Weight: 1, + ServiceBackend: &routeutils.ServiceBackendConfig{}, + Weight: 1, }, }, ListenerRuleConfig: &elbv2gw.ListenerRuleConfiguration{ diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index 5b7246ccb..64880fd0f 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -1782,7 +1782,6 @@ func Test_buildTargetGroupFromGateway(t *testing.T) { assert.True(t, exists) assert.Equal(t, result.Spec.Name, frontendData.Name) assert.Equal(t, tc.listenerPort, frontendData.Port) - //assert.Equal(t, tc.backendConfig.GetALBARN(), frontendData.TargetARN.Resolve(context.Background())) assert.Equal(t, *result.Spec.Port, frontendData.TargetPort) } }) From 5599bdaa1a70ab3c807dbe261ea3221397897814 Mon Sep 17 00:00:00 2001 From: Zachary Nixon Date: Wed, 5 Nov 2025 14:35:34 -0800 Subject: [PATCH 10/10] review fixes --- .../gateway/eventhandlers/tcp_route_events.go | 12 - .../elbv2/frontend_nlb_target_synthesizer.go | 5 + pkg/gateway/gatewayutils/gateway_utils.go | 43 - test/e2e/ingress/vanilla_ingress_test.go | 1464 +++++++++-------- 4 files changed, 739 insertions(+), 785 deletions(-) diff --git a/controllers/gateway/eventhandlers/tcp_route_events.go b/controllers/gateway/eventhandlers/tcp_route_events.go index d609d57b2..8ec77e6be 100644 --- a/controllers/gateway/eventhandlers/tcp_route_events.go +++ b/controllers/gateway/eventhandlers/tcp_route_events.go @@ -69,16 +69,4 @@ func (h *enqueueRequestsForTCPRouteEvent) enqueueImpactedGateways(ctx context.Co "gateway", gw) queue.Add(reconcile.Request{NamespacedName: gw}) } - - gateways, err = gatewayutils.GetImpactedGatewaysFromBackendRefs(ctx, h.k8sClient, route.Spec.Rules, route.Namespace, constants.NLBGatewayController) - if err != nil { - h.logger.V(1).Info("ignoring unknown gateways referred by", "tcproute", route.Name, "error", err) - } - for _, gw := range gateways { - h.logger.V(1).Info("enqueue gateway for tcproute event", - "tcproute", k8s.NamespacedName(route), - "gateway", gw) - queue.Add(reconcile.Request{NamespacedName: gw}) - } - } diff --git a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go index e2ee67eea..dda03c75f 100644 --- a/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go +++ b/pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go @@ -49,6 +49,7 @@ type frontendNlbTargetSynthesizer struct { // Synthesize processes AWS target groups and deregisters ALB targets based on the desired state. func (s *frontendNlbTargetSynthesizer) Synthesize(ctx context.Context) error { + var resTGs []*elbv2model.TargetGroup s.stack.ListResources(&resTGs) res := s.findSDKTargetGroups() @@ -109,6 +110,10 @@ func (s *frontendNlbTargetSynthesizer) deregisterCurrentTarget(ctx context.Conte } func (s *frontendNlbTargetSynthesizer) PostSynthesize(ctx context.Context) error { + if s.frontendNlbTargetGroupDesiredState == nil { + return nil + } + var resTGs []*elbv2model.TargetGroup s.stack.ListResources(&resTGs) diff --git a/pkg/gateway/gatewayutils/gateway_utils.go b/pkg/gateway/gatewayutils/gateway_utils.go index aa1edf8a9..e5197e2af 100644 --- a/pkg/gateway/gatewayutils/gateway_utils.go +++ b/pkg/gateway/gatewayutils/gateway_utils.go @@ -9,7 +9,6 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/pkg/gateway/constants" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" - gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // IsGatewayManagedByLBController checks if a Gateway is managed by the ALB/NLB Gateway Controller @@ -104,48 +103,6 @@ func GetImpactedGatewaysFromParentRefs(ctx context.Context, k8sClient client.Cli return impactedGateways, err } -func GetImpactedGatewaysFromBackendRefs(ctx context.Context, k8sClient client.Client, tcpRoutes []gwalpha2.TCPRouteRule, resourceNamespace string, gwController string) ([]types.NamespacedName, error) { - resultSet := sets.New[types.NamespacedName]() - processedSet := sets.New[types.NamespacedName]() - errorSet := sets.New[types.NamespacedName]() - for _, route := range tcpRoutes { - for _, ref := range route.BackendRefs { - if ref.Kind == nil || *ref.Kind != "Gateway" { - continue - } - - ns := resourceNamespace - if ref.Namespace != nil { - ns = string(*ref.Namespace) - } - nsn := types.NamespacedName{ - Namespace: ns, - Name: string(ref.Name), - } - if processedSet.Has(nsn) { - continue - } - - gw := &gwv1.Gateway{} - if err := k8sClient.Get(ctx, nsn, gw); err != nil { - // Ignore and continue processing other refs - errorSet.Insert(nsn) - continue - } - processedSet.Insert(nsn) - - if IsGatewayManagedByLBController(ctx, k8sClient, gw, gwController) { - resultSet.Insert(nsn) - } - } - } - var err error - if resultSet.Len() > 0 { - err = fmt.Errorf("failed to list gateways, %s", resultSet.UnsortedList()) - } - return resultSet.UnsortedList(), err -} - // GetImpactedGatewayClassesFromLbConfig identifies GatewayClasses affected by LoadBalancer configuration changes. // Returns GatewayClasses that reference the specified LoadBalancer configuration. func GetImpactedGatewayClassesFromLbConfig(ctx context.Context, k8sClient client.Client, lbconfig *elbv2gw.LoadBalancerConfiguration, gwControllers sets.Set[string]) (map[string]*gwv1.GatewayClass, error) { diff --git a/test/e2e/ingress/vanilla_ingress_test.go b/test/e2e/ingress/vanilla_ingress_test.go index 8fda3cf90..031c83ece 100644 --- a/test/e2e/ingress/vanilla_ingress_test.go +++ b/test/e2e/ingress/vanilla_ingress_test.go @@ -3,7 +3,11 @@ package ingress import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net/http" + "strings" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -67,833 +71,833 @@ var _ = Describe("vanilla ingress tests", func() { }) } }) - /* - Context("with basic settings", func() { - It("[ingress-class] in IP mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - } - ingClass := &networking.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxNS.Name, - }, - Spec: networking.IngressClassSpec{ - Controller: "ingress.k8s.aws/alb", + + Context("with basic settings", func() { + It("[ingress-class] in IP mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithIngressClassName(ingClass.Name). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + ingClass := &networking.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxNS.Name, + }, + Spec: networking.IngressClassSpec{ + Controller: "ingress.k8s.aws/alb", + }, + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithIngressClassName(ingClass.Name). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) - It("[ingress-class] in Instance mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - } - ingClass := &networking.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxNS.Name, - }, - Spec: networking.IngressClassSpec{ - Controller: "ingress.k8s.aws/alb", + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) + It("[ingress-class] in Instance mode, with IngressClass configured with 'ingress.k8s.aws/alb' controller, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithIngressClassName(ingClass.Name). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + ingClass := &networking.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxNS.Name, + }, + Spec: networking.IngressClassSpec{ + Controller: "ingress.k8s.aws/alb", + }, + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithIngressClassName(ingClass.Name). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) - It("ingress in IP mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + It("ingress in IP mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) - It("ingress in Instance mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + }) + It("ingress in Instance mode, with 'kubernetes.io/ingress.class' annotation set to 'alb', one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") }) + }) - Context("with IngressClass variant settings", func() { - It("[ingress-class] with IngressClass configured with 'nginx' controller, no ALB shall be created", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - } - ingClass := &networking.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxNS.Name, - }, - Spec: networking.IngressClassSpec{ - Controller: "kubernetes.io/nginx", + Context("with IngressClass variant settings", func() { + It("[ingress-class] with IngressClass configured with 'nginx' controller, no ALB shall be created", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithIngressClassName(ingClass.Name). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + ingClass := &networking.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxNS.Name, + }, + Spec: networking.IngressClassSpec{ + Controller: "kubernetes.io/nginx", + }, + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithIngressClassName(ingClass.Name). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ingClass, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - ExpectNoLBProvisionedForIngress(ctx, tf, ing) - }) + ExpectNoLBProvisionedForIngress(ctx, tf, ing) + }) - It("with 'kubernetes.io/ingress.class' annotation set to 'nginx', no ALB shall be created", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + It("with 'kubernetes.io/ingress.class' annotation set to 'nginx', no ALB shall be created", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "nginx", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "nginx", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - ExpectNoLBProvisionedForIngress(ctx, tf, ing) - }) + ExpectNoLBProvisionedForIngress(ctx, tf, ing) + }) - It("without IngressClass or 'kubernetes.io/ingress.class' annotation, no ALB shall be created", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + It("without IngressClass or 'kubernetes.io/ingress.class' annotation, no ALB shall be created", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "alb.ingress.kubernetes.io/scheme": "internet-facing", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "alb.ingress.kubernetes.io/scheme": "internet-facing", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - ExpectNoLBProvisionedForIngress(ctx, tf, ing) - }) + ExpectNoLBProvisionedForIngress(ctx, tf, ing) }) + }) - Context("with `alb.ingress.kubernetes.io/load-balancer-name` variant settings", func() { - It("with 'alb.ingress.kubernetes.io/load-balancer-name' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Context("with `alb.ingress.kubernetes.io/load-balancer-name` variant settings", func() { + It("with 'alb.ingress.kubernetes.io/load-balancer-name' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - safeClusterName := strings.ReplaceAll(tf.Options.ClusterName, ".", "-") - lbName := fmt.Sprintf("%.16s-%.15s", safeClusterName, sandboxNS.Name) - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/load-balancer-name": lbName, - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + safeClusterName := strings.ReplaceAll(tf.Options.ClusterName, ".", "-") + lbName := fmt.Sprintf("%.16s-%.15s", safeClusterName, sandboxNS.Name) + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/load-balancer-name": lbName, + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - sdkLB, err := tf.LBManager.GetLoadBalancerFromARN(ctx, lbARN) - Expect(err).NotTo(HaveOccurred()) - Expect(awssdk.ToString(sdkLB.LoadBalancerName)).Should(Equal(lbName)) + sdkLB, err := tf.LBManager.GetLoadBalancerFromARN(ctx, lbARN) + Expect(err).NotTo(HaveOccurred()) + Expect(awssdk.ToString(sdkLB.LoadBalancerName)).Should(Equal(lbName)) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") }) + }) - Context("with ALB IP targets and named target port", func() { - It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Context("with ALB IP targets and named target port", func() { + It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") }) + }) - Context("with ALB IP targets, named target port and endPointSlices enabled", func() { - BeforeEach(func() { - ctx = context.Background() - if tf.Options.ControllerImage != "" { - By(fmt.Sprintf("upgrade controller with endPointSlices enabled."), func() { - err := tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage, true) - Expect(err).NotTo(HaveOccurred()) - time.Sleep(60 * time.Second) - }) - } - }) - It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, and endPointSlices enabled, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Context("with ALB IP targets, named target port and endPointSlices enabled", func() { + BeforeEach(func() { + ctx = context.Background() + if tf.Options.ControllerImage != "" { + By(fmt.Sprintf("upgrade controller with endPointSlices enabled."), func() { + err := tf.CTRLInstallationManager.UpgradeController(tf.Options.ControllerImage, true) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(60 * time.Second) + }) + } + }) + It("with 'alb.ingress.kubernetes.io/target-type' annotation explicitly specified, and endPointSlices enabled, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder().WithTargetPortName("e2e-targetport") + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") }) + }) - Context("with `alb.ingress.kubernetes.io/actions.${action-name}` variant settings", func() { - It("with annotation based actions, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp1, svc1 := appBuilder.WithHTTPBody("app-1").Build(sandboxNS.Name, "app-1", tf.Options.TestImageRegistry) - dp2, svc2 := appBuilder.WithHTTPBody("app-2").Build(sandboxNS.Name, "app-2", tf.Options.TestImageRegistry) - ingResponse503Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "response-503", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + Context("with `alb.ingress.kubernetes.io/actions.${action-name}` variant settings", func() { + It("with annotation based actions, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp1, svc1 := appBuilder.WithHTTPBody("app-1").Build(sandboxNS.Name, "app-1", tf.Options.TestImageRegistry) + dp2, svc2 := appBuilder.WithHTTPBody("app-2").Build(sandboxNS.Name, "app-2", tf.Options.TestImageRegistry) + ingResponse503Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "response-503", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingRedirectToAWSBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "redirect-to-aws", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingRedirectToAWSBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "redirect-to-aws", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingForwardSingleTGBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "forward-single-tg", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingForwardSingleTGBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "forward-single-tg", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingForwardMultipleTGBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "forward-multiple-tg", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingForwardMultipleTGBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "forward-multiple-tg", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/actions.response-503": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"503\",\"messageBody\":\"503 error text\"}}", - "alb.ingress.kubernetes.io/actions.redirect-to-aws": "{\"type\":\"redirect\",\"redirectConfig\":{\"host\":\"aws.amazon.com\",\"path\":\"/eks/\",\"port\":\"443\",\"protocol\":\"HTTPS\",\"query\":\"k=v\",\"statusCode\":\"HTTP_302\"}}", - "alb.ingress.kubernetes.io/actions.forward-single-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\"}]}}", - "alb.ingress.kubernetes.io/actions.forward-multiple-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\",\"weight\":20},{\"serviceName\":\"app-2\",\"servicePort\":80,\"weight\":80}],\"targetGroupStickinessConfig\":{\"enabled\":true,\"durationSeconds\":200}}}", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/response-503", PathType: &exact, Backend: ingResponse503Backend}). - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/redirect-to-aws", PathType: &exact, Backend: ingRedirectToAWSBackend}). - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-single-tg", PathType: &exact, Backend: ingForwardSingleTGBackend}). - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-multiple-tg", PathType: &exact, Backend: ingForwardMultipleTGBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp1, svc1, dp2, svc2, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/actions.response-503": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"503\",\"messageBody\":\"503 error text\"}}", + "alb.ingress.kubernetes.io/actions.redirect-to-aws": "{\"type\":\"redirect\",\"redirectConfig\":{\"host\":\"aws.amazon.com\",\"path\":\"/eks/\",\"port\":\"443\",\"protocol\":\"HTTPS\",\"query\":\"k=v\",\"statusCode\":\"HTTP_302\"}}", + "alb.ingress.kubernetes.io/actions.forward-single-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\"}]}}", + "alb.ingress.kubernetes.io/actions.forward-multiple-tg": "{\"type\":\"forward\",\"forwardConfig\":{\"targetGroups\":[{\"serviceName\":\"app-1\",\"servicePort\":\"80\",\"weight\":20},{\"serviceName\":\"app-2\",\"servicePort\":80,\"weight\":80}],\"targetGroupStickinessConfig\":{\"enabled\":true,\"durationSeconds\":200}}}", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/response-503", PathType: &exact, Backend: ingResponse503Backend}). + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/redirect-to-aws", PathType: &exact, Backend: ingRedirectToAWSBackend}). + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-single-tg", PathType: &exact, Backend: ingForwardSingleTGBackend}). + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/forward-multiple-tg", PathType: &exact, Backend: ingForwardMultipleTGBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp1, svc1, dp2, svc2, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/response-503").Expect(). - Status(http.StatusServiceUnavailable). - Body().Equal("503 error text") - httpExp.GET("/redirect-to-aws").WithRedirectPolicy(httpexpect.DontFollowRedirects).Expect(). - Status(http.StatusFound). - Header("Location").Equal("https://aws.amazon.com:443/eks/?k=v") - httpExp.GET("/forward-single-tg").Expect(). - Status(http.StatusOK). - Body().Equal("app-1") - httpExp.GET("/forward-multiple-tg").Expect(). - Status(http.StatusOK). - Body().Match("app-1|app-2") - }) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/response-503").Expect(). + Status(http.StatusServiceUnavailable). + Body().Equal("503 error text") + httpExp.GET("/redirect-to-aws").WithRedirectPolicy(httpexpect.DontFollowRedirects).Expect(). + Status(http.StatusFound). + Header("Location").Equal("https://aws.amazon.com:443/eks/?k=v") + httpExp.GET("/forward-single-tg").Expect(). + Status(http.StatusOK). + Body().Equal("app-1") + httpExp.GET("/forward-multiple-tg").Expect(). + Status(http.StatusOK). + Body().Match("app-1|app-2") }) + }) - Context("with `alb.ingress.kubernetes.io/conditions.${conditions-name}` variant settings", func() { - It("with annotation based conditions, one ALB shall be created and functional", func() { - ingBuilder := manifest.NewIngressBuilder() - ingRulePath1Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path1", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + Context("with `alb.ingress.kubernetes.io/conditions.${conditions-name}` variant settings", func() { + It("with annotation based conditions, one ALB shall be created and functional", func() { + ingBuilder := manifest.NewIngressBuilder() + ingRulePath1Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path1", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingRulePath2Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path2", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingRulePath2Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path2", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingRulePath3Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path3", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingRulePath3Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path3", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingRulePath4Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path4", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingRulePath4Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path4", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingRulePath5Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path5", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingRulePath5Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path5", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingRulePath6Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path6", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingRulePath6Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path6", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } - ingRulePath7Backend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "rule-path7", - Port: networking.ServiceBackendPort{ - Name: "use-annotation", - }, + }, + } + ingRulePath7Backend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "rule-path7", + Port: networking.ServiceBackendPort{ + Name: "use-annotation", }, - } + }, + } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/actions.rule-path1": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Host is www.example.com OR anno.example.com\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path1": "[{\"field\":\"host-header\",\"hostHeaderConfig\":{\"values\":[\"anno.example.com\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path2": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Path is /path2 OR /anno/path2\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path2": "[{\"field\":\"path-pattern\",\"pathPatternConfig\":{\"values\":[\"/anno/path2\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path3": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http header HeaderName is HeaderValue1 OR HeaderValue2\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path3": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue1\", \"HeaderValue2\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path4": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http request method is GET OR HEAD\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path4": "[{\"field\":\"http-request-method\",\"httpRequestMethodConfig\":{\"Values\":[\"GET\", \"HEAD\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path5": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Query string is paramA:valueA1 OR paramA:valueA2\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path5": "[{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA1\"},{\"key\":\"paramA\",\"value\":\"valueA2\"}]}}]", - "alb.ingress.kubernetes.io/actions.rule-path6": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Source IP is 192.168.0.0/16 OR 172.16.0.0/16\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path6": "[{\"field\":\"source-ip\",\"sourceIpConfig\":{\"values\":[\"192.168.0.0/16\", \"172.16.0.0/16\"]}}]", - "alb.ingress.kubernetes.io/actions.rule-path7": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"multiple conditions applies\"}}", - "alb.ingress.kubernetes.io/conditions.rule-path7": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue\"]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA\"}]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramB\",\"value\":\"valueB\"}]}}]", - } - if tf.Options.IPFamily == "framework.IPv6" { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path1", PathType: &exact, Backend: ingRulePath1Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path2", PathType: &exact, Backend: ingRulePath2Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path3", PathType: &exact, Backend: ingRulePath3Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path4", PathType: &exact, Backend: ingRulePath4Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path5", PathType: &exact, Backend: ingRulePath5Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path6", PathType: &exact, Backend: ingRulePath6Backend}). - AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path7", PathType: &exact, Backend: ingRulePath7Backend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/actions.rule-path1": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Host is www.example.com OR anno.example.com\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path1": "[{\"field\":\"host-header\",\"hostHeaderConfig\":{\"values\":[\"anno.example.com\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path2": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Path is /path2 OR /anno/path2\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path2": "[{\"field\":\"path-pattern\",\"pathPatternConfig\":{\"values\":[\"/anno/path2\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path3": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http header HeaderName is HeaderValue1 OR HeaderValue2\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path3": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue1\", \"HeaderValue2\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path4": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Http request method is GET OR HEAD\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path4": "[{\"field\":\"http-request-method\",\"httpRequestMethodConfig\":{\"Values\":[\"GET\", \"HEAD\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path5": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Query string is paramA:valueA1 OR paramA:valueA2\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path5": "[{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA1\"},{\"key\":\"paramA\",\"value\":\"valueA2\"}]}}]", + "alb.ingress.kubernetes.io/actions.rule-path6": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"Source IP is 192.168.0.0/16 OR 172.16.0.0/16\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path6": "[{\"field\":\"source-ip\",\"sourceIpConfig\":{\"values\":[\"192.168.0.0/16\", \"172.16.0.0/16\"]}}]", + "alb.ingress.kubernetes.io/actions.rule-path7": "{\"type\":\"fixed-response\",\"fixedResponseConfig\":{\"contentType\":\"text/plain\",\"statusCode\":\"200\",\"messageBody\":\"multiple conditions applies\"}}", + "alb.ingress.kubernetes.io/conditions.rule-path7": "[{\"field\":\"http-header\",\"httpHeaderConfig\":{\"httpHeaderName\": \"HeaderName\", \"values\":[\"HeaderValue\"]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramA\",\"value\":\"valueA\"}]}},{\"field\":\"query-string\",\"queryStringConfig\":{\"values\":[{\"key\":\"paramB\",\"value\":\"valueB\"}]}}]", + } + if tf.Options.IPFamily == "framework.IPv6" { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path1", PathType: &exact, Backend: ingRulePath1Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path2", PathType: &exact, Backend: ingRulePath2Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path3", PathType: &exact, Backend: ingRulePath3Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path4", PathType: &exact, Backend: ingRulePath4Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path5", PathType: &exact, Backend: ingRulePath5Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path6", PathType: &exact, Backend: ingRulePath6Backend}). + AddHTTPRoute("www.example.com", networking.HTTPIngressPath{Path: "/path7", PathType: &exact, Backend: ingRulePath7Backend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path1").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Host is www.example.com OR anno.example.com") - httpExp.GET("/path1").WithHost("anno.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Host is www.example.com OR anno.example.com") - httpExp.GET("/path1").WithHost("other.example.com").Expect(). - Status(http.StatusNotFound) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path1").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Host is www.example.com OR anno.example.com") + httpExp.GET("/path1").WithHost("anno.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Host is www.example.com OR anno.example.com") + httpExp.GET("/path1").WithHost("other.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path2").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Path is /path2 OR /anno/path2") - httpExp.GET("/anno/path2").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Path is /path2 OR /anno/path2") - httpExp.GET("/other/path2").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path2").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Path is /path2 OR /anno/path2") + httpExp.GET("/anno/path2").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Path is /path2 OR /anno/path2") + httpExp.GET("/other/path2").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue1").Expect(). - Status(http.StatusOK). - Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") - httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue2").Expect(). - Status(http.StatusOK). - Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") - httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue3").Expect(). - Status(http.StatusNotFound) - httpExp.GET("/path3").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue1").Expect(). + Status(http.StatusOK). + Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") + httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue2").Expect(). + Status(http.StatusOK). + Body().Equal("Http header HeaderName is HeaderValue1 OR HeaderValue2") + httpExp.GET("/path3").WithHost("www.example.com").WithHeader("HeaderName", "HeaderValue3").Expect(). + Status(http.StatusNotFound) + httpExp.GET("/path3").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path4").WithHost("www.example.com").Expect(). - Status(http.StatusOK). - Body().Equal("Http request method is GET OR HEAD") - httpExp.HEAD("/path4").WithHost("www.example.com").Expect(). - Status(http.StatusOK) - httpExp.POST("/path4").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path4").WithHost("www.example.com").Expect(). + Status(http.StatusOK). + Body().Equal("Http request method is GET OR HEAD") + httpExp.HEAD("/path4").WithHost("www.example.com").Expect(). + Status(http.StatusOK) + httpExp.POST("/path4").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA1").Expect(). - Status(http.StatusOK). - Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") - httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA2").Expect(). - Status(http.StatusOK). - Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") - httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA3").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA1").Expect(). + Status(http.StatusOK). + Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") + httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA2").Expect(). + Status(http.StatusOK). + Body().Equal("Query string is paramA:valueA1 OR paramA:valueA2") + httpExp.GET("/path5").WithHost("www.example.com").WithQuery("paramA", "valueA3").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path6").WithHost("www.example.com").Expect(). - Status(http.StatusNotFound) + httpExp.GET("/path6").WithHost("www.example.com").Expect(). + Status(http.StatusNotFound) - httpExp.GET("/path7").WithHost("www.example.com"). - WithHeader("HeaderName", "HeaderValue"). - WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). - Status(http.StatusOK). - Body().Equal("multiple conditions applies") - httpExp.GET("/path7").WithHost("www.example.com"). - WithHeader("HeaderName", "OtherHeaderValue"). - WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). - Status(http.StatusNotFound) - httpExp.GET("/path7").WithHost("www.example.com"). - WithHeader("HeaderName", "HeaderValue"). - WithQuery("paramA", "valueB").WithQuery("paramB", "valueB").Expect(). - Status(http.StatusNotFound) - }) + httpExp.GET("/path7").WithHost("www.example.com"). + WithHeader("HeaderName", "HeaderValue"). + WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). + Status(http.StatusOK). + Body().Equal("multiple conditions applies") + httpExp.GET("/path7").WithHost("www.example.com"). + WithHeader("HeaderName", "OtherHeaderValue"). + WithQuery("paramA", "valueA").WithQuery("paramB", "valueB").Expect(). + Status(http.StatusNotFound) + httpExp.GET("/path7").WithHost("www.example.com"). + WithHeader("HeaderName", "HeaderValue"). + WithQuery("paramA", "valueB").WithQuery("paramB", "valueB").Expect(). + Status(http.StatusNotFound) }) + }) - Context("with `alb.ingress.kubernetes.io/ip-address-type` variant settings", func() { - It("with 'alb.ingress.kubernetes.io/ip-address-type' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Context("with `alb.ingress.kubernetes.io/ip-address-type` variant settings", func() { + It("with 'alb.ingress.kubernetes.io/ip-address-type' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/target-type": "ip", - "alb.ingress.kubernetes.io/ip-address-type": "ipv4", - } + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/target-type": "ip", + "alb.ingress.kubernetes.io/ip-address-type": "ipv4", + } - if tf.Options.IPFamily == framework.IPv6 { - // TODO: annotate to "dualstack-without-public-ipv4" for all regions once it's GA - if tf.Options.AWSRegion == "us-west-2" { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack-without-public-ipv4" - } else { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - } + if tf.Options.IPFamily == framework.IPv6 { + // TODO: annotate to "dualstack-without-public-ipv4" for all regions once it's GA + if tf.Options.AWSRegion == "us-west-2" { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack-without-public-ipv4" + } else { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" } + } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - //TODO: update the traffic test for dualstack-without-public-ipv4 ALB - // as it may need additional setup compared to dualstack ALB - if annotation["alb.ingress.kubernetes.io/ip-address-type"] != "dualstack-without-public-ipv4" { - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - } - }) + //TODO: update the traffic test for dualstack-without-public-ipv4 ALB + // as it may need additional setup compared to dualstack ALB + if annotation["alb.ingress.kubernetes.io/ip-address-type"] != "dualstack-without-public-ipv4" { + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") + } }) + }) - Context("with `alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}` variant settings", func() { - It("with 'alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}' annotation explicitly specified, one ALB shall be created and functional", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Context("with `alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}` variant settings", func() { + It("with 'alb.ingress.kubernetes.io/listener-attributes.{Protocol}-{Port}' annotation explicitly specified, one ALB shall be created and functional", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, - "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=false", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/listen-ports": `[{"HTTP": 80}]`, + "alb.ingress.kubernetes.io/listener-attributes.HTTP-80": "routing.http.response.server.enabled=false", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - sdkListeners, err := tf.LBManager.GetLoadBalancerListeners(ctx, lbARN) + lbARN, lbDNS := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + sdkListeners, err := tf.LBManager.GetLoadBalancerListeners(ctx, lbARN) - Eventually(func() bool { - return verifyListenerAttributes(ctx, tf, *sdkListeners[0].ListenerArn, map[string]string{ - "routing.http.response.server.enabled": "false", - }) == nil - }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) + Eventually(func() bool { + return verifyListenerAttributes(ctx, tf, *sdkListeners[0].ListenerArn, map[string]string{ + "routing.http.response.server.enabled": "false", + }) == nil + }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) - // test traffic - ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) - httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) - httpExp.GET("/path").Expect(). - Status(http.StatusOK). - Body().Equal("Hello World!") - }) + // test traffic + ExpectLBDNSBeAvailable(ctx, tf, lbARN, lbDNS) + httpExp := httpexpect.New(tf.LoggerReporter, fmt.Sprintf("http://%v", lbDNS)) + httpExp.GET("/path").Expect(). + Status(http.StatusOK). + Body().Equal("Hello World!") }) + }) - Context("When created with managed alb with tags provided via annotations", func() { - It("ALB will have correct tags", func() { - appBuilder := manifest.NewFixedResponseServiceBuilder() - ingBuilder := manifest.NewIngressBuilder() - dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) - ingBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: svc.Name, - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Context("When created with managed alb with tags provided via annotations", func() { + It("ALB will have correct tags", func() { + appBuilder := manifest.NewFixedResponseServiceBuilder() + ingBuilder := manifest.NewIngressBuilder() + dp, svc := appBuilder.Build(sandboxNS.Name, "app", tf.Options.TestImageRegistry) + ingBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: svc.Name, + Port: networking.ServiceBackendPort{ + Number: 80, }, - } - annotation := map[string]string{ - "kubernetes.io/ingress.class": "alb", - "alb.ingress.kubernetes.io/scheme": "internet-facing", - "alb.ingress.kubernetes.io/tags": "k1=v1,k2=v2", - } - if tf.Options.IPFamily == framework.IPv6 { - annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" - annotation["alb.ingress.kubernetes.io/target-type"] = "ip" - } - ing := ingBuilder. - AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). - WithAnnotations(annotation).Build(sandboxNS.Name, "ing") - resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) - err := resStack.Setup(ctx) - Expect(err).NotTo(HaveOccurred()) + }, + } + annotation := map[string]string{ + "kubernetes.io/ingress.class": "alb", + "alb.ingress.kubernetes.io/scheme": "internet-facing", + "alb.ingress.kubernetes.io/tags": "k1=v1,k2=v2", + } + if tf.Options.IPFamily == framework.IPv6 { + annotation["alb.ingress.kubernetes.io/ip-address-type"] = "dualstack" + annotation["alb.ingress.kubernetes.io/target-type"] = "ip" + } + ing := ingBuilder. + AddHTTPRoute("", networking.HTTPIngressPath{Path: "/path", PathType: &exact, Backend: ingBackend}). + WithAnnotations(annotation).Build(sandboxNS.Name, "ing") + resStack := fixture.NewK8SResourceStack(tf, dp, svc, ing) + err := resStack.Setup(ctx) + Expect(err).NotTo(HaveOccurred()) - defer resStack.TearDown(ctx) + defer resStack.TearDown(ctx) - lbARN, _ := ExpectOneLBProvisionedForIngress(ctx, tf, ing) + lbARN, _ := ExpectOneLBProvisionedForIngress(ctx, tf, ing) - // Verify tags - expectedTags := map[string]string{ - "k1": "v1", - "k2": "v2", + // Verify tags + expectedTags := map[string]string{ + "k1": "v1", + "k2": "v2", + } + // Verify tags using DescribeTags + var tagDescriptions []elbv2types.TagDescription + Eventually(func() (bool, error) { + req := &elasticloadbalancingv2.DescribeTagsInput{ + ResourceArns: []string{lbARN}, } - // Verify tags using DescribeTags - var tagDescriptions []elbv2types.TagDescription - Eventually(func() (bool, error) { - req := &elasticloadbalancingv2.DescribeTagsInput{ - ResourceArns: []string{lbARN}, - } - resp, err := tf.Cloud.ELBV2().DescribeTagsWithContext(ctx, req) - if err != nil { - return false, err - } - if len(resp.TagDescriptions) > 0 { - tagDescriptions = resp.TagDescriptions - return true, nil - } - return false, nil - }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) - - // At this point we should have exactly one TagDescription since we only queried one LB - Expect(tagDescriptions).To(HaveLen(1), "Expected exactly one TagDescription") - foundTags := 0 - for _, tag := range tagDescriptions[0].Tags { - if val, ok := expectedTags[awssdk.ToString(tag.Key)]; ok && val == awssdk.ToString(tag.Value) { - foundTags++ - } + resp, err := tf.Cloud.ELBV2().DescribeTagsWithContext(ctx, req) + if err != nil { + return false, err } - Expect(foundTags).To(Equal(len(expectedTags)), "Not all expected tags were found on the ALB") - }) + if len(resp.TagDescriptions) > 0 { + tagDescriptions = resp.TagDescriptions + return true, nil + } + return false, nil + }, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue()) + + // At this point we should have exactly one TagDescription since we only queried one LB + Expect(tagDescriptions).To(HaveLen(1), "Expected exactly one TagDescription") + foundTags := 0 + for _, tag := range tagDescriptions[0].Tags { + if val, ok := expectedTags[awssdk.ToString(tag.Key)]; ok && val == awssdk.ToString(tag.Value) { + foundTags++ + } + } + Expect(foundTags).To(Equal(len(expectedTags)), "Not all expected tags were found on the ALB") }) - */ + }) + Context("with frontend NLB enabled", func() { It("should create a frontend NLB and route traffic correctly", func() { appBuilder := manifest.NewFixedResponseServiceBuilder()