Skip to content

Commit 650fb5a

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #4361 from mmiller-sh/main
feat: Add flag to limit the max number of targets added to an ELB
2 parents d847890 + 1becaac commit 650fb5a

File tree

4 files changed

+58
-24
lines changed

4 files changed

+58
-24
lines changed

‎docs/deploy/configurations.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne
117117
| kube-ca-pem-filepath | string | | The file path to the CA to validate webhook callers, when unspecified all webhook callers are permitted. |
118118
| alb-gateway-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for ALB gateways, if enabled |
119119
| nlb-gateway-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for NLB gateways, if enabled |
120+
| max-targets-per-target-group | int | 0 | Maximum number of targets that will be added to a given Target Group. The default value of zero will leave the number of targets unlimited |
120121

121122
### disable-ingress-class-annotation
122123
`--disable-ingress-class-annotation` controls whether to disable new usage of the `kubernetes.io/ingress.class` annotation.

‎main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ func main() {
185185
tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(),
186186
podInfoRepo, networkingManager, vpcInfoProvider, multiClusterManager, lbcMetricsCollector,
187187
cloud.VpcID(), controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices,
188-
mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log)
188+
mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log, controllerCFG.MaxTargetsPerTargetGroup)
189189
backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup,
190190
cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, nlbGatewayEnabled || albGatewayEnabled, ctrl.Log.WithName("backend-sg-provider"))
191191
sgResolver := networking.NewDefaultSecurityGroupResolver(cloud.EC2(), cloud.VpcID())

‎pkg/config/controller_config.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ const (
3434
flagBackendSecurityGroup = "backend-security-group"
3535
flagEnableEndpointSlices = "enable-endpoint-slices"
3636
flagDisableRestrictedSGRules = "disable-restricted-sg-rules"
37+
flagMaxTargetsPerTargetGroup = "max-targets-per-target-group"
3738
defaultLogLevel = "info"
3839
defaultMaxConcurrentReconciles = 3
3940
defaultMaxExponentialBackoffDelay = time.Second * 1000
@@ -43,6 +44,7 @@ const (
4344
defaultEnableEndpointSlices = true
4445
defaultDisableRestrictedSGRules = false
4546
defaultLbStabilizationMonitorInterval = time.Second * 120
47+
defaultMaxTargetsPerTargetGroup = 0
4648
)
4749

4850
var (
@@ -133,6 +135,9 @@ type ControllerConfig struct {
133135
// LBStabilizationMonitorInterval specifies the duration of interval to monitor the load balancer state for stabilization
134136
LBStabilizationMonitorInterval time.Duration
135137

138+
// MaxTargetsPerTargetGroup limits the number of targets that will be added to an ELB instance
139+
MaxTargetsPerTargetGroup int
140+
136141
FeatureGates FeatureGates
137142
}
138143

@@ -177,6 +182,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
177182
"Disable the usage of restricted security group rules")
178183
fs.StringToStringVar(&cfg.ServiceTargetENISGTags, flagServiceTargetENISGTags, nil,
179184
"AWS Tags, in addition to cluster tags, for finding the target ENI security group to which to add inbound rules from NLBs")
185+
fs.IntVar(&cfg.MaxTargetsPerTargetGroup, flagMaxTargetsPerTargetGroup, defaultMaxTargetsPerTargetGroup,
186+
"Maximum number of targets that can be added to an ELB instance. Use this to prevent TargetGroup quotas being exceeded from blocking reconciliation.")
180187
cfg.FeatureGates.BindFlags(fs)
181188
cfg.AWSConfig.BindFlags(fs)
182189
cfg.RuntimeConfig.BindFlags(fs)

‎pkg/targetgroupbinding/resource_manager.go‎

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -48,21 +48,23 @@ func NewDefaultResourceManager(k8sClient client.Client, elbv2Client services.ELB
4848
podInfoRepo k8s.PodInfoRepo, networkingManager networking.NetworkingManager,
4949
vpcInfoProvider networking.VPCInfoProvider, multiClusterManager MultiClusterManager, metricsCollector lbcmetrics.MetricCollector,
5050
vpcID string, failOpenEnabled bool, endpointSliceEnabled bool,
51-
eventRecorder record.EventRecorder, logger logr.Logger) *defaultResourceManager {
51+
eventRecorder record.EventRecorder, logger logr.Logger, maxTargetsPerTargetGroup int) *defaultResourceManager {
52+
5253
targetsManager := NewCachedTargetsManager(elbv2Client, logger)
5354
endpointResolver := backend.NewDefaultEndpointResolver(k8sClient, podInfoRepo, failOpenEnabled, endpointSliceEnabled, logger)
5455
return &defaultResourceManager{
55-
k8sClient: k8sClient,
56-
targetsManager: targetsManager,
57-
endpointResolver: endpointResolver,
58-
networkingManager: networkingManager,
59-
eventRecorder: eventRecorder,
60-
logger: logger,
61-
vpcID: vpcID,
62-
vpcInfoProvider: vpcInfoProvider,
63-
podInfoRepo: podInfoRepo,
64-
multiClusterManager: multiClusterManager,
65-
metricsCollector: metricsCollector,
56+
k8sClient: k8sClient,
57+
targetsManager: targetsManager,
58+
endpointResolver: endpointResolver,
59+
networkingManager: networkingManager,
60+
eventRecorder: eventRecorder,
61+
logger: logger,
62+
vpcID: vpcID,
63+
vpcInfoProvider: vpcInfoProvider,
64+
podInfoRepo: podInfoRepo,
65+
maxTargetsPerTargetGroup: maxTargetsPerTargetGroup,
66+
multiClusterManager: multiClusterManager,
67+
metricsCollector: metricsCollector,
6668

6769
invalidVpcCache: cache.NewExpiring(),
6870
invalidVpcCacheTTL: defaultTargetsCacheTTL,
@@ -75,17 +77,18 @@ var _ ResourceManager = &defaultResourceManager{}
7577

7678
// default implementation for ResourceManager.
7779
type defaultResourceManager struct {
78-
k8sClient client.Client
79-
targetsManager TargetsManager
80-
endpointResolver backend.EndpointResolver
81-
networkingManager networking.NetworkingManager
82-
eventRecorder record.EventRecorder
83-
logger logr.Logger
84-
vpcInfoProvider networking.VPCInfoProvider
85-
podInfoRepo k8s.PodInfoRepo
86-
multiClusterManager MultiClusterManager
87-
metricsCollector lbcmetrics.MetricCollector
88-
vpcID string
80+
k8sClient client.Client
81+
targetsManager TargetsManager
82+
endpointResolver backend.EndpointResolver
83+
networkingManager networking.NetworkingManager
84+
eventRecorder record.EventRecorder
85+
logger logr.Logger
86+
vpcInfoProvider networking.VPCInfoProvider
87+
podInfoRepo k8s.PodInfoRepo
88+
maxTargetsPerTargetGroup int
89+
multiClusterManager MultiClusterManager
90+
metricsCollector lbcmetrics.MetricCollector
91+
vpcID string
8992

9093
invalidVpcCache *cache.Expiring
9194
invalidVpcCacheTTL time.Duration
@@ -240,6 +243,11 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context,
240243
return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "update_tracked_ip_targets_error", err, m.metricsCollector)
241244
}
242245

246+
if m.maxTargetsPerTargetGroup != 0 {
247+
eligibleTargetsCount := m.getMaxNewTargets(len(unmatchedEndpoints), len(targets), tgbScopedLogger)
248+
unmatchedEndpoints = unmatchedEndpoints[:eligibleTargetsCount]
249+
}
250+
243251
if err := m.registerPodEndpoints(ctx, tgb, unmatchedEndpoints); err != nil {
244252
return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "register_pod_endpoint_error", err, m.metricsCollector)
245253
}
@@ -341,6 +349,11 @@ func (m *defaultResourceManager) reconcileWithInstanceTargetType(ctx context.Con
341349
return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "update_tracked_instance_targets_error", err, m.metricsCollector)
342350
}
343351

352+
if m.maxTargetsPerTargetGroup != 0 {
353+
eligibleTargetsCount := m.getMaxNewTargets(len(unmatchedEndpoints), len(targets), tgbScopedLogger)
354+
unmatchedEndpoints = unmatchedEndpoints[:eligibleTargetsCount]
355+
}
356+
344357
if err := m.registerNodePortEndpoints(ctx, tgb, unmatchedEndpoints); err != nil {
345358
return "", "", false, ctrlerrors.NewErrorWithMetrics(controllerName, "update_node_port_endpoints_error", err, m.metricsCollector)
346359
}
@@ -803,3 +816,16 @@ func isVPCNotFoundError(err error) bool {
803816
}
804817
return false
805818
}
819+
820+
func (m *defaultResourceManager) getMaxNewTargets(newTargetCount int, currentTargetCount int, tgbScopedLogger logr.Logger) (maxAdditions int) {
821+
if newTargetCount+currentTargetCount > m.maxTargetsPerTargetGroup {
822+
maxAdditions = m.maxTargetsPerTargetGroup - currentTargetCount
823+
tgbScopedLogger.Info("Limiting target additions due to max-targets-per-instance configuration",
824+
"currentTargets", currentTargetCount,
825+
"maxTargetsPerTargetGroup", m.maxTargetsPerTargetGroup,
826+
"proposedAdditions", newTargetCount)
827+
return maxAdditions
828+
}
829+
830+
return newTargetCount
831+
}

0 commit comments

Comments
 (0)