Merge pull request #4323 from zac-nixon/main

[gw api] add grpc e2e tests

Author: k8s-ci-robot

test/e2e/gateway/alb_instance_target_test.go

@@ -10,11 +10,12 @@ import (
 	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
-func newALBResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, tgc *elbv2gw.TargetGroupConfiguration, lrc *elbv2gw.ListenerRuleConfiguration, httpr []*gwv1.HTTPRoute, baseName string, enablePodReadinessGate bool) *albResourceStack {
+func newALBResourceStack(dps []*appsv1.Deployment, svcs []*corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, tgcs []*elbv2gw.TargetGroupConfiguration, lrc *elbv2gw.ListenerRuleConfiguration, httpr []*gwv1.HTTPRoute, grpcrs []*gwv1.GRPCRoute, baseName string, enablePodReadinessGate bool) *albResourceStack {
 
-	commonStack := newCommonResourceStack([]*appsv1.Deployment{dp}, []*corev1.Service{svc}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{tgc}, []*elbv2gw.ListenerRuleConfiguration{lrc}, baseName, enablePodReadinessGate)
+	commonStack := newCommonResourceStack(dps, svcs, gwc, gw, lbc, tgcs, []*elbv2gw.ListenerRuleConfiguration{lrc}, baseName, enablePodReadinessGate)
 	return &albResourceStack{
 		httprs:      httpr,
+		grpcrs:      grpcrs,
 		commonStack: commonStack,
 	}
 }
@@ -23,13 +24,21 @@ func newALBResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv1.G
 type albResourceStack struct {
 	commonStack *commonResourceStack
 	httprs      []*gwv1.HTTPRoute
+	grpcrs      []*gwv1.GRPCRoute
 }
 
 func (s *albResourceStack) Deploy(ctx context.Context, f *framework.Framework) error {
 	return s.commonStack.Deploy(ctx, f, func(ctx context.Context, f *framework.Framework, namespace string) error {
-		for _, httpr := range s.httprs {
-			httpr.Namespace = namespace
-			if err := s.createHTTPRoute(ctx, f, httpr); err != nil {
+		for i := range s.httprs {
+			s.httprs[i].Namespace = namespace
+			if err := s.createHTTPRoute(ctx, f, s.httprs[i]); err != nil {
+				return err
+			}
+		}
+
+		for i := range s.grpcrs {
+			s.grpcrs[i].Namespace = namespace
+			if err := s.createGRPCRoute(ctx, f, s.grpcrs[i]); err != nil {
 				return err
 			}
 		}
@@ -58,6 +67,16 @@ func (s *albResourceStack) createHTTPRoute(ctx context.Context, f *framework.Fra
 	return f.K8sClient.Create(ctx, httpr)
 }
 
+func (s *albResourceStack) createGRPCRoute(ctx context.Context, f *framework.Framework, grpcr *gwv1.GRPCRoute) error {
+	f.Logger.Info("creating grpc route", "grpc", k8s.NamespacedName(grpcr))
+	return f.K8sClient.Create(ctx, grpcr)
+}
+
+func (s *albResourceStack) updateGRPCRoute(ctx context.Context, f *framework.Framework, grpcr *gwv1.GRPCRoute) error {
+	f.Logger.Info("updating grpc route", "grpc", k8s.NamespacedName(grpcr))
+	return f.K8sClient.Update(ctx, grpcr)
+}
+
 func (s *albResourceStack) deleteHTTPRoute(ctx context.Context, f *framework.Framework, httpr *gwv1.HTTPRoute) error {
 	return f.K8sClient.Delete(ctx, httpr)
 }

test/e2e/gateway/alb_ip_target_test.go

@@ -2,8 +2,14 @@ package gateway
 
 import (
 	"context"
+	"crypto/tls"
+	"fmt"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
+	"sigs.k8s.io/aws-load-balancer-controller/test/e2e/gateway/grpc/echo"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
 	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
@@ -12,7 +18,7 @@ type ALBTestStack struct {
 	albResourceStack *albResourceStack
 }
 
-func (s *ALBTestStack) Deploy(ctx context.Context, auxiliaryStack *auxiliaryResourceStack, f *framework.Framework, gwListeners []gwv1.Listener, httprs []*gwv1.HTTPRoute, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, tgConfSpec elbv2gw.TargetGroupConfigurationSpec, lrConfSpec elbv2gw.ListenerRuleConfigurationSpec, readinessGateEnabled bool) error {
+func (s *ALBTestStack) DeployHTTP(ctx context.Context, auxiliaryStack *auxiliaryResourceStack, f *framework.Framework, gwListeners []gwv1.Listener, httprs []*gwv1.HTTPRoute, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, tgConfSpec elbv2gw.TargetGroupConfigurationSpec, lrConfSpec elbv2gw.ListenerRuleConfigurationSpec, readinessGateEnabled bool) error {
 	if auxiliaryStack != nil {
 		gwListeners = append(gwListeners, gwv1.Listener{
 			Name:     "other-ns",
@@ -23,15 +29,38 @@ func (s *ALBTestStack) Deploy(ctx context.Context, auxiliaryStack *auxiliaryReso
 		httprs = append(httprs, buildOtherNsRefHttpRoute("other-ns", auxiliaryStack.ns))
 	}
 
-	dp := buildDeploymentSpec(f.Options.TestImageRegistry)
 	svc := buildServiceSpec()
+	tgc := buildTargetGroupConfig(defaultTgConfigName, tgConfSpec, svc)
+	return s.deploy(ctx, f, gwListeners, httprs, []*gwv1.GRPCRoute{}, []*appsv1.Deployment{buildDeploymentSpec(f.Options.TestImageRegistry)}, []*corev1.Service{svc}, lbConfSpec, []*elbv2gw.TargetGroupConfiguration{tgc}, lrConfSpec, readinessGateEnabled)
+}
+
+func (s *ALBTestStack) DeployGRPC(ctx context.Context, f *framework.Framework, gwListeners []gwv1.Listener, grpcrs []*gwv1.GRPCRoute, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, tgConfSpec elbv2gw.TargetGroupConfigurationSpec, lrConfSpec elbv2gw.ListenerRuleConfigurationSpec, readinessGateEnabled bool) error {
+	labels := map[string]string{
+		"app.kubernetes.io/instance": grpcDefaultName,
+	}
+
+	otherLabels := map[string]string{
+		"app.kubernetes.io/instance": "other",
+	}
+
+	svc := buildGRPCServiceSpec(grpcDefaultName, labels)
+	dp := buildGRPCDeploymentSpec(grpcDefaultName, "Hello World", labels)
+	tgc := buildTargetGroupConfig(defaultTgConfigName, tgConfSpec, svc)
+
+	svcOther := buildGRPCServiceSpec(grpcDefaultName+"-other", otherLabels)
+	dpOther := buildGRPCDeploymentSpec(grpcDefaultName+"-other", "Hello World - Other", otherLabels)
+	tgcOther := buildTargetGroupConfig(defaultTgConfigName+"-other", tgConfSpec, svcOther)
+
+	return s.deploy(ctx, f, gwListeners, []*gwv1.HTTPRoute{}, grpcrs, []*appsv1.Deployment{dp, dpOther}, []*corev1.Service{svc, svcOther}, lbConfSpec, []*elbv2gw.TargetGroupConfiguration{tgc, tgcOther}, lrConfSpec, readinessGateEnabled)
+}
+
+func (s *ALBTestStack) deploy(ctx context.Context, f *framework.Framework, gwListeners []gwv1.Listener, httprs []*gwv1.HTTPRoute, grpcrs []*gwv1.GRPCRoute, dps []*appsv1.Deployment, svcs []*corev1.Service, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, tgcs []*elbv2gw.TargetGroupConfiguration, lrConfSpec elbv2gw.ListenerRuleConfigurationSpec, readinessGateEnabled bool) error {
 	gwc := buildGatewayClassSpec("gateway.k8s.aws/alb")
 	gw := buildBasicGatewaySpec(gwc, gwListeners)
 	lbc := buildLoadBalancerConfig(lbConfSpec)
-	tgc := buildTargetGroupConfig(defaultTgConfigName, tgConfSpec, svc)
 	lrc := buildListenerRuleConfig(defaultLRConfigName, lrConfSpec)
 
-	s.albResourceStack = newALBResourceStack(dp, svc, gwc, gw, lbc, tgc, lrc, httprs, "alb-gateway-e2e", readinessGateEnabled)
+	s.albResourceStack = newALBResourceStack(dps, svcs, gwc, gw, lbc, tgcs, lrc, httprs, grpcrs, "alb-gateway-e2e", readinessGateEnabled)
 
 	return s.albResourceStack.Deploy(ctx, f)
 }
@@ -58,3 +87,16 @@ func (s *ALBTestStack) GetWorkerNodes(ctx context.Context, f *framework.Framewor
 	}
 	return nodeList, nil
 }
+
+func generateGRPCClient(dnsName string) (echo.EchoServiceClient, error) {
+	target := fmt.Sprintf("%s:443", dnsName)
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true, // This skips all certificate verification, including expiry.
+	}
+
+	conn, err := grpc.NewClient(target, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
+	if err != nil {
+		return nil, err
+	}
+	return echo.NewEchoServiceClient(conn), nil
+}

test/e2e/gateway/alb_resource_stack.go

@@ -11,9 +11,11 @@ import (
 const (
 	appContainerPort        = 80
 	udpContainerPort        = 8080
+	grpcContainerPort       = 50051
 	defaultNumReplicas      = 3
 	defaultName             = "gateway-e2e"
 	udpDefaultName          = defaultName + "-udp"
+	grpcDefaultName         = defaultName + "-grpc"
 	defaultGatewayClassName = "gwclass-e2e"
 	defaultLbConfigName     = "lbconfig-e2e"
 	defaultTgConfigName     = "tgconfig-e2e"
@@ -47,6 +49,17 @@ var DEFAULT_ALB_TARGET_GROUP_HC = &verifier.TargetGroupHC{
 	UnhealthyThreshold: 3,
 }
 
+// Common settings for ALB target group health checks using GRPC
+var DEFAULT_ALB_TARGET_GROUP_HC_GRPC = &verifier.TargetGroupHC{
+	Protocol:           "HTTP",
+	Port:               "traffic-port",
+	Path:               "/AWS.ALB/healthcheck",
+	Interval:           15,
+	Timeout:            5,
+	HealthyThreshold:   3,
+	UnhealthyThreshold: 3,
+}
+
 var listenerConfigurationForHeaderModification = &[]elbv2gw.ListenerConfiguration{
 	{
 		ProtocolPort: "HTTP:80",
@@ -75,6 +88,18 @@ var DefaultHttpRouteRuleBackendRefs = []gwv1.HTTPBackendRef{
 	},
 }
 
+var defaultGrpcPort = gwalpha2.PortNumber(50051)
+var DefaultGrpcRouteRuleBackendRefs = []gwv1.GRPCBackendRef{
+	{
+		BackendRef: gwv1.BackendRef{
+			BackendObjectReference: gwv1.BackendObjectReference{
+				Name: grpcDefaultName,
+				Port: &defaultGrpcPort,
+			},
+		},
+	},
+}
+
 var portNew = gwalpha2.PortNumber(8443)
 var httpRouteRuleWithMatchesAndFilters = []gwv1.HTTPRouteRule{
 	{