From d55e395ddc535e198ebc63b444229f92f7686f9e Mon Sep 17 00:00:00 2001 From: Tamilmani Date: Fri, 5 Sep 2025 14:16:05 -0700 Subject: [PATCH 1/7] refactor: Update apipa nic as separate entry in podIPInfo This PR updates both CNS and CNI code to construct apipa nic as separate entry in podIpInfo if either of allowhostonc or allownctohost set. This allows CNI to treat this as separate endpoint and align with current cni design/model of 1 nic per endpoint info. CNI then iterates through endpoint info and creates one nic at a time. Signed-off-by: Tamilmani --- cni/network/invoker_cns.go | 28 ++++++++++++++++++++++++++++ cns/NetworkContainerContract.go | 7 +++++++ cns/restserver/ipam.go | 16 ++++++++++++++++ network/network_windows.go | 3 ++- 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 928096b361..12fdcd6af9 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -192,6 +192,11 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro if err := addBackendNICToResult(&info, &addResult, key); err != nil { return IPAMAddResult{}, err } + case cns.ApipaNIC: + if err := configureApipaAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig, key); err != nil { + return IPAMAddResult{}, err + } + case cns.InfraNIC, "": // if we change from legacy cns, the nicType will be empty, so we assume it is infra nic info.nicType = cns.InfraNIC @@ -508,6 +513,29 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p return nil } +func configureApipaAddResult(info *IPResultInfo, addResult *IPAMAddResult, podIPConfig *cns.IPSubnet, key string) error { + ip, ipnet, err := podIPConfig.GetIPNet() + if ip == nil { + return errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") + } + + addResult.interfaceInfo[key] = network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: ip, + Mask: ipnet.Mask, + }, + Gateway: net.ParseIP(info.ncGatewayIPAddress), + }, + }, + NICType: info.nicType, + SkipDefaultRoutes: true, + } + + return nil +} + func addBackendNICToResult(info *IPResultInfo, addResult *IPAMAddResult, key string) error { macAddress, err := net.ParseMAC(info.macAddress) if err != nil { diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 406c45b554..72f136ab13 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -93,6 +93,9 @@ const ( NodeNetworkInterfaceFrontendNIC NICType = "FrontendNIC" // NodeNetworkInterfaceBackendNIC is the new name for BackendNIC NodeNetworkInterfaceBackendNIC NICType = "BackendNIC" + + // ApipaNIC is used for internal communication between host and container + ApipaNIC NICType = "ApipaNIC" ) // ChannelMode :- CNS channel modes @@ -516,6 +519,10 @@ type PodIpInfo struct { PnPID string // Default Deny ACL's to configure on HNS endpoints for Swiftv2 window nodes EndpointPolicies []policy.Policy + // This flag is in effect only if nic type is apipa. This allows connection originating from host to container via apipa nic and not other way. + AllowHostToNCCommunication bool + // This flag is in effect only if nic type is apipa. This allows connection originating from container to host via apipa nic and not other way. + AllowNCToHostCommunication bool } type HostIPInfo struct { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 883d9aaef3..9cf5638696 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -149,6 +149,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte // assign NICType and MAC Address for SwiftV2. we assume that there won't be any SwiftV1 NCs here podIPInfoList := make([]cns.PodIpInfo, 0, len(resp)) + apipaIndex := -1 for i := range resp { podIPInfo := cns.PodIpInfo{ PodIPConfig: resp[i].IPConfiguration.IPSubnet, @@ -157,6 +158,21 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte NetworkContainerPrimaryIPConfig: resp[i].IPConfiguration, } podIPInfoList = append(podIPInfoList, podIPInfo) + if resp[i].AllowHostToNCCommunication || resp[i].AllowNCToHostCommunication { + apipaIndex = i + } + } + + if apipaIndex != -1 { + apipaPodIPInfo := cns.PodIpInfo{ + PodIPConfig: resp[apipaIndex].LocalIPConfiguration.IPSubnet, + NICType: cns.ApipaNIC, + NetworkContainerPrimaryIPConfig: resp[apipaIndex].LocalIPConfiguration, + SkipDefaultRoutes: true, + AllowHostToNCCommunication: resp[apipaIndex].AllowHostToNCCommunication, + AllowNCToHostCommunication: resp[apipaIndex].AllowNCToHostCommunication, + } + podIPInfoList = append(podIPInfoList, apipaPodIPInfo) } ipConfigsResp := &cns.IPConfigsResponse{ diff --git a/network/network_windows.go b/network/network_windows.go index a467b20983..61e5159caf 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -342,7 +342,8 @@ func (nm *networkManager) addIPv6DefaultRoute() error { // newNetworkImplHnsV2 creates a new container network for HNSv2. func (nm *networkManager) newNetworkImplHnsV2(nwInfo *EndpointInfo, extIf *externalInterface) (*network, error) { // network creation is not required for IB - if nwInfo.NICType == cns.BackendNIC { + // For apipa nic, we create network as part of endpoint creation + if nwInfo.NICType == cns.BackendNIC || nwInfo.NICType == cns.ApipaNIC { return &network{Endpoints: make(map[string]*endpoint)}, nil } From 1efdde8b905b94ed48568beb08365669ba5a3720 Mon Sep 17 00:00:00 2001 From: Tamilmani Date: Wed, 15 Oct 2025 12:13:33 -0700 Subject: [PATCH 2/7] Introduced NetworkContainerID, AllowHostToNC, AllowNCToHost fields in podipinfo to support apipa endpoint create request. persist networkcontainer id in cns state to support delete apipa endpoint when called as endpoint name based on networkcontainerid. Signed-off-by: Tamilmani --- cni/network/invoker_cns.go | 19 ++++++++------ cni/network/network.go | 20 +++++++++------ cns/NetworkContainerContract.go | 2 ++ cns/restserver/ipam.go | 7 ++++++ cns/restserver/ipam_test.go | 11 +++++---- cns/restserver/restserver.go | 15 +++++------ network/endpoint.go | 25 ++++++++++--------- network/endpoint_windows.go | 44 +++++++++++++++++++++++++-------- network/manager.go | 27 +++++++++++--------- 9 files changed, 110 insertions(+), 60 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 12fdcd6af9..380421425f 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -165,7 +165,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } logger.Info("Received info for pod", - zap.Any("ipInfo", info), + zap.Any("ipInfo", response.PodIPInfo[i]), zap.Any("podInfo", podInfo)) //nolint:exhaustive // ignore exhaustive types check @@ -193,7 +193,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro return IPAMAddResult{}, err } case cns.ApipaNIC: - if err := configureApipaAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig, key); err != nil { + if err := configureApipaAddResult(&addResult, &response.PodIPInfo[i], key); err != nil { return IPAMAddResult{}, err } @@ -513,10 +513,10 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p return nil } -func configureApipaAddResult(info *IPResultInfo, addResult *IPAMAddResult, podIPConfig *cns.IPSubnet, key string) error { - ip, ipnet, err := podIPConfig.GetIPNet() +func configureApipaAddResult(addResult *IPAMAddResult, info *cns.PodIpInfo, key string) error { + ip, ipnet, err := info.PodIPConfig.GetIPNet() if ip == nil { - return errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") + return errors.Wrap(err, "Unable to parse IP from response: "+info.PodIPConfig.IPAddress+" with err %w") } addResult.interfaceInfo[key] = network.InterfaceInfo{ @@ -526,11 +526,14 @@ func configureApipaAddResult(info *IPResultInfo, addResult *IPAMAddResult, podIP IP: ip, Mask: ipnet.Mask, }, - Gateway: net.ParseIP(info.ncGatewayIPAddress), + Gateway: net.ParseIP(info.NetworkContainerPrimaryIPConfig.GatewayIPAddress), }, }, - NICType: info.nicType, - SkipDefaultRoutes: true, + NICType: info.NICType, + SkipDefaultRoutes: true, + NetworkContainerID: info.NetworkContainerID, + AllowHostToNCCommunication: info.AllowHostToNCCommunication, + AllowNCToHostCommunication: info.AllowNCToHostCommunication, } return nil diff --git a/cni/network/network.go b/cni/network/network.go index a09db694c0..3abc4a25bf 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -675,11 +675,14 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() - // populate endpoint info section - masterIfName := plugin.findMasterInterface(opt) - if masterIfName == "" { - err := plugin.Errorf("Failed to find the master interface") - return nil, err + var masterIfName string + if opt.ifInfo.NICType != cns.ApipaNIC { + // populate endpoint info section + masterIfName = plugin.findMasterInterface(opt) + if masterIfName == "" { + err := plugin.Errorf("Failed to find the master interface") + return nil, err + } } networkPolicies := opt.policies // save network policies before we modify the slice pointer for ep policies @@ -757,8 +760,11 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn IPAddresses: addresses, MacAddress: opt.ifInfo.MacAddress, // the following is used for creating an external interface if we can't find an existing network - HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(), - PnPID: opt.ifInfo.PnPID, + HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(), + PnPID: opt.ifInfo.PnPID, + NetworkContainerID: opt.ifInfo.NetworkContainerID, + AllowInboundFromHostToNC: opt.ifInfo.AllowHostToNCCommunication, + AllowInboundFromNCToHost: opt.ifInfo.AllowNCToHostCommunication, } if err = addSubnetToEndpointInfo(*opt.ifInfo, &endpointInfo); err != nil { diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 72f136ab13..d8982186ed 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -523,6 +523,8 @@ type PodIpInfo struct { AllowHostToNCCommunication bool // This flag is in effect only if nic type is apipa. This allows connection originating from container to host via apipa nic and not other way. AllowNCToHostCommunication bool + // NetworkContainerID is the ID of the network container to which this Pod IP belongs + NetworkContainerID string } type HostIPInfo struct { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 9cf5638696..c53ae2c70c 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -156,6 +156,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte MacAddress: resp[i].NetworkInterfaceInfo.MACAddress, NICType: resp[i].NetworkInterfaceInfo.NICType, NetworkContainerPrimaryIPConfig: resp[i].IPConfiguration, + NetworkContainerID: resp[i].NetworkContainerID, } podIPInfoList = append(podIPInfoList, podIPInfo) if resp[i].AllowHostToNCCommunication || resp[i].AllowNCToHostCommunication { @@ -171,6 +172,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte SkipDefaultRoutes: true, AllowHostToNCCommunication: resp[apipaIndex].AllowHostToNCCommunication, AllowNCToHostCommunication: resp[apipaIndex].AllowNCToHostCommunication, + NetworkContainerID: resp[apipaIndex].NetworkContainerID, } podIPInfoList = append(podIPInfoList, apipaPodIPInfo) } @@ -1329,6 +1331,11 @@ func updateIPInfoMap(iPInfo map[string]*IPInfo, interfaceInfo *IPInfo, ifName, e iPInfo[ifName].MacAddress = interfaceInfo.MacAddress logger.Printf("[updateEndpoint] update the endpoint %s with MacAddress %s", endpointID, interfaceInfo.MacAddress) } + + if interfaceInfo.NetworkContainerID != "" { + iPInfo[ifName].NetworkContainerID = interfaceInfo.NetworkContainerID + logger.Printf("[updateEndpoint] update the endpoint %s with NetworkContainerID %s", endpointID, interfaceInfo.NetworkContainerID) + } } // verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 52c5b6d0d6..26b229d07f 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -2176,11 +2176,12 @@ func TestStatelessCNIStateFile(t *testing.T) { endpointInfo2ContainerID := "1b4917617e15d24dc495e407d8eb5c88e4406e58fa209e4eb75a2c2fb7045eea" endpointInfo2 := &EndpointInfo{IfnameToIPMap: make(map[string]*IPInfo)} endpointInfo2.IfnameToIPMap["eth2"] = &IPInfo{ - IPv4: nil, - NICType: cns.DelegatedVMNIC, - HnsEndpointID: "5c15cccc-830a-4dff-81f3-4b1e55cb7dcb", - HnsNetworkID: "5c0712cd-824c-4898-b1c0-2fcb16ede4fb", - MacAddress: "7c:1e:52:06:d3:4b", + IPv4: nil, + NICType: cns.DelegatedVMNIC, + HnsEndpointID: "5c15cccc-830a-4dff-81f3-4b1e55cb7dcb", + HnsNetworkID: "5c0712cd-824c-4898-b1c0-2fcb16ede4fb", + MacAddress: "7c:1e:52:06:d3:4b", + NetworkContainerID: testNCID, } // test cases tests := []struct { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 84589292ce..f14c69d12c 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -126,13 +126,14 @@ type EndpointInfo struct { } type IPInfo struct { - IPv4 []net.IPNet - IPv6 []net.IPNet `json:",omitempty"` - HnsEndpointID string `json:",omitempty"` - HnsNetworkID string `json:",omitempty"` - HostVethName string `json:",omitempty"` - MacAddress string `json:",omitempty"` - NICType cns.NICType + IPv4 []net.IPNet + IPv6 []net.IPNet `json:",omitempty"` + HnsEndpointID string `json:",omitempty"` + HnsNetworkID string `json:",omitempty"` + HostVethName string `json:",omitempty"` + MacAddress string `json:",omitempty"` + NetworkContainerID string `json:",omitempty"` + NICType cns.NICType } type GetHTTPServiceDataResponse struct { diff --git a/network/endpoint.go b/network/endpoint.go index bd9fa7fc9b..f1eb9b731e 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -128,17 +128,20 @@ type RouteInfo struct { // InterfaceInfo contains information for secondary interfaces type InterfaceInfo struct { - Name string - MacAddress net.HardwareAddr - IPConfigs []*IPConfig - Routes []RouteInfo - DNS DNSInfo - NICType cns.NICType - SkipDefaultRoutes bool - HostSubnetPrefix net.IPNet // Move this field from ipamAddResult - NCResponse *cns.GetNetworkContainerResponse - PnPID string - EndpointPolicies []policy.Policy + Name string + MacAddress net.HardwareAddr + IPConfigs []*IPConfig + Routes []RouteInfo + DNS DNSInfo + NICType cns.NICType + SkipDefaultRoutes bool + HostSubnetPrefix net.IPNet // Move this field from ipamAddResult + NCResponse *cns.GetNetworkContainerResponse + PnPID string + EndpointPolicies []policy.Policy + NetworkContainerID string + AllowNCToHostCommunication bool + AllowHostToNCCommunication bool } type IPConfig struct { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index edd52327f2..ad5c72e4da 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -157,12 +157,17 @@ func (nw *network) newEndpointImpl( return nw.getEndpointWithVFDevice(plc, epInfo) } + if epInfo.NICType == cns.ApipaNIC { + return nw.createHostNCApipaEndpoint(cli, epInfo) + } + if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err } return nw.newEndpointImplHnsV2(cli, epInfo) + } return nw.newEndpointImplHnsV1(epInfo, plc) @@ -354,14 +359,16 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE return hcnEndpoint, nil } +func getApipaEndpointName(networkContainerID string) string { + endpointName := fmt.Sprintf("%s-%s", hostNCApipaEndpointNamePrefix, networkContainerID) + return endpointName +} + func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { // TODO: this code is duplicated in cns/hnsclient, but that code has logging messages that require a CNSLogger, // which makes is hard to use in this package. We should refactor this into a common package with no logging deps // so it can be called in both places - - // HostNCApipaEndpoint name is derived from NC ID - endpointName := fmt.Sprintf("%s-%s", hostNCApipaEndpointNamePrefix, networkContainerID) - logger.Info("Deleting HostNCApipaEndpoint for NC", zap.String("endpointName", endpointName), zap.String("networkContainerID", networkContainerID)) + endpointName := getApipaEndpointName(networkContainerID) // Check if the endpoint exists endpoint, err := Hnsv2.GetEndpointByName(endpointName) @@ -376,6 +383,7 @@ func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { return nil } + logger.Info("Deleting Apipa Endpoint", zap.String("endpointName", endpointName)) if err := Hnsv2.DeleteEndpoint(endpoint); err != nil { return fmt.Errorf("failed to delete HostNCApipa endpoint: %+v: %w", endpoint, err) } @@ -387,7 +395,7 @@ func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { // createHostNCApipaEndpoint creates a new endpoint in the HostNCApipaNetwork // for host container connectivity -func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointInfo) error { +func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointInfo) (*endpoint, error) { var ( err error hostNCApipaEndpointID string @@ -395,7 +403,7 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn ) if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return fmt.Errorf("Failed to retrieve namespace with GetNamespaceByID for NetNsPath: %s"+ + return nil, fmt.Errorf("Failed to retrieve namespace with GetNamespaceByID for NetNsPath: %s"+ " due to error: %v", epInfo.NetNsPath, err) } @@ -403,7 +411,7 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn zap.String("NetworkContainerID", epInfo.NetworkContainerID)) if hostNCApipaEndpointID, err = cli.CreateHostNCApipaEndpoint(context.TODO(), epInfo.NetworkContainerID); err != nil { - return err + return nil, err } defer func() { @@ -413,10 +421,19 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn }() if err = hcn.AddNamespaceEndpoint(namespace.Id, hostNCApipaEndpointID); err != nil { - return fmt.Errorf("Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", hostNCApipaEndpointID, namespace.Id, err) //nolint + return nil, fmt.Errorf("Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", hostNCApipaEndpointID, namespace.Id, err) //nolint } - return nil + ep := &endpoint{ + Id: getApipaEndpointName(epInfo.NetworkContainerID), + HnsId: hostNCApipaEndpointID, + IfName: getApipaEndpointName(epInfo.NetworkContainerID), + ContainerID: epInfo.ContainerID, + NICType: cns.ApipaNIC, + NetworkContainerID: epInfo.NetworkContainerID, + } + + return ep, nil } // newEndpointImplHnsV2 creates a new endpoint in the network using Hnsv2 @@ -464,7 +481,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( // If the Host - container connectivity is requested, create endpoint in HostNCApipaNetwork if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { - if err = nw.createHostNCApipaEndpoint(cli, epInfo); err != nil { + if _, err = nw.createHostNCApipaEndpoint(cli, epInfo); err != nil { return nil, fmt.Errorf("Failed to create HostNCApipaEndpoint due to error: %v", err) } } @@ -529,6 +546,13 @@ func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.Exe return nil } + if ep.NICType == cns.ApipaNIC { + if err := nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID); err != nil { + logger.Error("Failed to delete HostNCApipaEndpoint due to error", zap.Error(err)) + return err + } + } + if ep.HnsId == "" { logger.Error("No HNS id found. Skip endpoint deletion", zap.Any("nicType", ep.NICType), zap.String("containerId", ep.ContainerID)) return fmt.Errorf("No HNS id found. Skip endpoint deletion for nicType %v, containerID %s", ep.NICType, ep.ContainerID) //nolint diff --git a/network/manager.go b/network/manager.go index 7bc1441fea..a70a667b3e 100644 --- a/network/manager.go +++ b/network/manager.go @@ -424,7 +424,8 @@ func (nm *networkManager) UpdateEndpointState(eps []*endpoint) error { ifnameToIPInfoMap := generateCNSIPInfoMap(eps) // key : interface name, value : IPInfo for key, ipinfo := range ifnameToIPInfoMap { logger.Info("Update endpoint state", zap.String("ifname", key), zap.String("hnsEndpointID", ipinfo.HnsEndpointID), zap.String("hnsNetworkID", ipinfo.HnsNetworkID), - zap.String("hostVethName", ipinfo.HostVethName), zap.String("macAddress", ipinfo.MacAddress), zap.String("nicType", string(ipinfo.NICType))) + zap.String("hostVethName", ipinfo.HostVethName), zap.String("macAddress", ipinfo.MacAddress), zap.String("nicType", string(ipinfo.NICType)), + zap.String("networkContainerID", ipinfo.NetworkContainerID)) } // we assume all endpoints have the same container id cnsEndpointID := eps[0].ContainerID @@ -784,12 +785,11 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI for ifName, ipInfo := range endpointInfo.IfnameToIPMap { epInfo := &EndpointInfo{ - EndpointID: endpointID, // endpoint id is always the same, but we shouldn't use it in the stateless path - IfIndex: EndpointIfIndex, // Azure CNI supports only one interface - ContainerID: endpointID, - PODName: endpointInfo.PodName, - PODNameSpace: endpointInfo.PodNamespace, - NetworkContainerID: endpointID, + EndpointID: endpointID, // endpoint id is always the same, but we shouldn't use it in the stateless path + IfIndex: EndpointIfIndex, // Azure CNI supports only one interface + ContainerID: endpointID, + PODName: endpointInfo.PodName, + PODNameSpace: endpointInfo.PodNamespace, } // If we create an endpoint state with stateful cni and then swap to a stateless cni binary, ifname would not be populated @@ -809,6 +809,8 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI epInfo.NICType = ipInfo.NICType epInfo.HNSNetworkID = ipInfo.HnsNetworkID epInfo.MacAddress = net.HardwareAddr(ipInfo.MacAddress) + epInfo.NetworkContainerID = ipInfo.NetworkContainerID + ret = append(ret, epInfo) } return ret @@ -837,11 +839,12 @@ func generateCNSIPInfoMap(eps []*endpoint) map[string]*restserver.IPInfo { for _, ep := range eps { ifNametoIPInfoMap[ep.IfName] = &restserver.IPInfo{ // in windows, the nicname is args ifname, in linux, it's ethX - NICType: ep.NICType, - HnsEndpointID: ep.HnsId, - HnsNetworkID: ep.HNSNetworkID, - HostVethName: ep.HostIfName, - MacAddress: ep.MacAddress.String(), + NICType: ep.NICType, + HnsEndpointID: ep.HnsId, + HnsNetworkID: ep.HNSNetworkID, + HostVethName: ep.HostIfName, + MacAddress: ep.MacAddress.String(), + NetworkContainerID: ep.NetworkContainerID, } } From 3db8140befdcd03fad1f64c73cfce80f6313933f Mon Sep 17 00:00:00 2001 From: Tamilmani Date: Thu, 16 Oct 2025 15:55:36 -0700 Subject: [PATCH 3/7] dummy create network for apipa nic as like backend nic Signed-off-by: Tamilmani --- cni/network/network.go | 15 ++++++++------- cni/network/network_windows.go | 5 ++++- network/network.go | 3 +-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 3abc4a25bf..6dcc633a1f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -52,6 +52,7 @@ const ( ipv4FullMask = 32 ipv6FullMask = 128 ibInterfacePrefix = "ib" + apipaInterfacePrefix = "apipa" ) // CNI Operation Types @@ -643,6 +644,8 @@ func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { // when the VF is dismounted, this interface will go away // return an unique interface name to containerd return ibInterfacePrefix + strconv.Itoa(opt.endpointIndex) + case cns.ApipaNIC: + return apipaInterfacePrefix + strconv.Itoa(opt.endpointIndex) default: return "" } @@ -676,13 +679,11 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() var masterIfName string - if opt.ifInfo.NICType != cns.ApipaNIC { - // populate endpoint info section - masterIfName = plugin.findMasterInterface(opt) - if masterIfName == "" { - err := plugin.Errorf("Failed to find the master interface") - return nil, err - } + // populate endpoint info section + masterIfName = plugin.findMasterInterface(opt) + if masterIfName == "" { + err := plugin.Errorf("Failed to find the master interface") + return nil, err } networkPolicies := opt.policies // save network policies before we modify the slice pointer for ep policies diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index f7d2e5defb..c76af56b1f 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -75,10 +75,13 @@ func (plugin *NetPlugin) getNetworkName(netNs string, interfaceInfo *network.Int // Swiftv2 L1VH Network Name swiftv2NetworkNamePrefix := "azure-" if interfaceInfo != nil && (interfaceInfo.NICType == cns.NodeNetworkInterfaceFrontendNIC || interfaceInfo.NICType == cns.BackendNIC) { - logger.Info("swiftv2", zap.String("network name", interfaceInfo.MacAddress.String())) return swiftv2NetworkNamePrefix + interfaceInfo.MacAddress.String(), nil } + if interfaceInfo != nil && interfaceInfo.NICType == cns.ApipaNIC { + return swiftv2NetworkNamePrefix + "apipa", nil + } + // For singletenancy, the network name is simply the nwCfg.Name if !nwCfg.MultiTenancy { return nwCfg.Name, nil diff --git a/network/network.go b/network/network.go index 31537b6522..41e1bf4307 100644 --- a/network/network.go +++ b/network/network.go @@ -108,7 +108,7 @@ func (nm *networkManager) newExternalInterface(ifName, subnet, nicType string) e // Find the host interface. macAddress := net.HardwareAddr{} - if nicType != string(cns.BackendNIC) { + if nicType != string(cns.BackendNIC) && nicType != string(cns.ApipaNIC) { hostIf, err := net.InterfaceByName(ifName) if err != nil { return errors.Wrap(err, "failed to find host interface") @@ -337,7 +337,6 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo return err } } - ep, err := nm.createEndpoint(cnsclient, epInfo.NetworkID, epInfo) if err != nil { return err From f052db3ffb8954241afc107c1d185a3b815b6a81 Mon Sep 17 00:00:00 2001 From: Tamilmani Date: Thu, 23 Oct 2025 21:31:28 -0700 Subject: [PATCH 4/7] Added DeleteEndpointState handler for removing endpoint state from CNS in stateless cni case Signed-off-by: Tamilmani --- cni/network/network.go | 4 +- cns/client/client.go | 32 +++++++++ cns/endpointmanager/endpointmanager.go | 1 + .../endpointmanager_windows.go | 15 ++++- cns/restserver/ipam.go | 65 +++++++++++++++++++ network/manager.go | 16 ++++- 6 files changed, 129 insertions(+), 4 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 6dcc633a1f..8226c38d5b 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -1159,10 +1159,10 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { } } } - logger.Info("Deleting the state from the cni statefile") + logger.Info("Deleting endpoint state from statefile") err = plugin.nm.DeleteState(epInfos) if err != nil { - return plugin.RetriableError(fmt.Errorf("failed to save state: %w", err)) + return plugin.RetriableError(fmt.Errorf("failed to delete state: %w", err)) } return err diff --git a/cns/client/client.go b/cns/client/client.go index 1021d6f412..ad0dc2dca5 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1100,3 +1100,35 @@ func (c *Client) UpdateEndpoint(ctx context.Context, endpointID string, ipInfo m return &response, nil } + +func (c *Client) DeleteEndpointState(ctx context.Context, endpointID string) (*cns.Response, error) { + // build the request + u := c.routes[cns.EndpointAPI] + uString := u.String() + endpointID + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, uString, http.NoBody) + if err != nil { + return nil, errors.Wrap(err, "failed to build request") + } + req.Header.Set(headerContentType, contentTypeJSON) + res, err := c.client.Do(req) + if err != nil { + return nil, &ConnectionFailureErr{cause: err} + } + + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, errors.Errorf("http response %d", res.StatusCode) + } + + var response cns.Response + err = json.NewDecoder(res.Body).Decode(&response) + if err != nil { + return nil, errors.Wrap(err, "failed to decode CNS Response") + } + + if response.ReturnCode != 0 { + return nil, errors.New(response.Message) + } + + return &response, nil +} diff --git a/cns/endpointmanager/endpointmanager.go b/cns/endpointmanager/endpointmanager.go index e71f0a6cbc..3f71ecfa80 100644 --- a/cns/endpointmanager/endpointmanager.go +++ b/cns/endpointmanager/endpointmanager.go @@ -14,6 +14,7 @@ type EndpointManager struct { type releaseIPsClient interface { ReleaseIPs(ctx context.Context, ipconfig cns.IPConfigsRequest) error GetEndpoint(ctx context.Context, endpointID string) (*restserver.GetEndpointResponse, error) + DeleteEndpointState(ctx context.Context, endpointID string) (*cns.Response, error) } func WithPlatformReleaseIPsManager(cli releaseIPsClient) *EndpointManager { diff --git a/cns/endpointmanager/endpointmanager_windows.go b/cns/endpointmanager/endpointmanager_windows.go index dd928b721d..60c77f1057 100644 --- a/cns/endpointmanager/endpointmanager_windows.go +++ b/cns/endpointmanager/endpointmanager_windows.go @@ -6,6 +6,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/hnsclient" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/types" "github.com/pkg/errors" ) @@ -16,7 +17,19 @@ func (em *EndpointManager) ReleaseIPs(ctx context.Context, ipconfigreq cns.IPCon if err := em.deleteEndpoint(ctx, ipconfigreq.InfraContainerID); err != nil { logger.Errorf("failed to remove HNS endpoint %s", err.Error()) } - return errors.Wrap(em.cli.ReleaseIPs(ctx, ipconfigreq), "failed to release IP from CNS") + + if err := em.cli.ReleaseIPs(ctx, ipconfigreq); err != nil { + return errors.Wrap(err, "failed to release IP from CNS") + } + + res, err := em.cli.DeleteEndpointState(ctx, ipconfigreq.InfraContainerID) + if err != nil { + if res.ReturnCode != types.NotFound { + return errors.Wrap(err, "") + } + } + + return nil } // deleteEndpoint API to get the state and then remove assiciated HNS diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index c53ae2c70c..75e8b1210c 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -1141,11 +1141,76 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt service.GetEndpointHandler(w, r) case http.MethodPatch: service.UpdateEndpointHandler(w, r) + case http.MethodDelete: + service.DeleteEndpointStateHandler(w, r) default: logger.Errorf("[EndpointHandlerAPI] EndpointHandler API expect http Get or Patch method") } } +func (service *HTTPRestService) DeleteEndpointStateHandler(w http.ResponseWriter, r *http.Request) { + opName := "DeleteEndpointStateHandler" + logger.Printf("[DeleteEndpointStateHandler] DeleteEndpointState for %s", r.URL.Path) //nolint:staticcheck // reason: using deprecated call until migration to new API + endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath) + + if service.EndpointStateStore == nil { + response := cns.Response{ + ReturnCode: types.NilEndpointStateStore, + Message: "[DeleteEndpointStateHandler] EndpointStateStore is not initialized", + } + err := common.Encode(w, &response) + logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API + return + } + + // Delete the endpoint from state + err := service.DeleteEndpointStateHelper(endpointID) + if err != nil { + response := cns.Response{ + ReturnCode: types.UnexpectedError, + Message: fmt.Sprintf("[DeleteEndpointStateHandler] Failed to delete endpoint state for %s with error: %s", endpointID, err.Error()), + } + + if errors.Is(err, ErrEndpointStateNotFound) { + response.ReturnCode = types.NotFound + } + + err = common.Encode(w, &response) + logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API + return + } + + response := cns.Response{ + ReturnCode: types.Success, + Message: "[DeleteEndpointStateHandler] Endpoint state deleted successfully", + } + err = common.Encode(w, &response) + logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API +} + +func (service *HTTPRestService) DeleteEndpointStateHelper(endpointID string) error { + if service.EndpointStateStore == nil { + return ErrStoreEmpty + } + logger.Printf("[deleteEndpointState] Deleting Endpoint state from state file %s", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API + _, endpointExist := service.EndpointState[endpointID] + if !endpointExist { + logger.Printf("[deleteEndpointState] endpoint could not be found in the statefile %s", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API + return fmt.Errorf("[deleteEndpointState] endpoint %s: %w", endpointID, ErrEndpointStateNotFound) + } + + // Delete the endpoint from the state + delete(service.EndpointState, endpointID) + + // Write the updated state back to the store + err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState) + if err != nil { + return fmt.Errorf("[deleteEndpointState] failed to write endpoint state to store: %w", err) + } + logger.Printf("[deleteEndpointState] successfully deleted endpoint %s from state file", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API + return nil +} + // GetEndpointHandler handles the incoming GetEndpoint requests with http Get method func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *http.Request) { opName := "getEndpointState" diff --git a/network/manager.go b/network/manager.go index a70a667b3e..f2dd90d910 100644 --- a/network/manager.go +++ b/network/manager.go @@ -762,7 +762,7 @@ func (nm *networkManager) SaveState(eps []*endpoint) error { return nm.save() } -func (nm *networkManager) DeleteState(_ []*EndpointInfo) error { +func (nm *networkManager) DeleteState(epInfos []*EndpointInfo) error { nm.Lock() defer nm.Unlock() @@ -772,6 +772,20 @@ func (nm *networkManager) DeleteState(_ []*EndpointInfo) error { // For stateless cni, plugin.ipamInvoker.Delete takes care of removing the state in the main Delete function if nm.IsStatelessCNIMode() { + for _, epInfo := range epInfos { + if epInfo.NICType == cns.NodeNetworkInterfaceFrontendNIC || epInfo.NICType == cns.NodeNetworkInterfaceAccelnetFrontendNIC { + response, err := nm.CnsClient.DeleteEndpointState(context.TODO(), epInfo.EndpointID) + if err != nil { + if response != nil && response.ReturnCode == types.NotFound { + logger.Info("Endpoint state not found in CNS", zap.String("endpointID", epInfo.EndpointID)) + return nil + } + return errors.Wrapf(err, "Delete endpoint API returned with error for endpoint %s", epInfo.EndpointID) + } + logger.Info("Delete endpoint succeeded", zap.String("endpointID", epInfo.EndpointID), zap.String("returnCode", response.ReturnCode.String())) + break + } + } return nil } From 0134df3ca1719f0a183ce0ea95cf8fef5ed73794 Mon Sep 17 00:00:00 2001 From: Tamilmani Date: Mon, 27 Oct 2025 14:07:08 -0700 Subject: [PATCH 5/7] add config for disabling async delete Signed-off-by: Tamilmani --- cni/netconfig.go | 1 + cni/network/network.go | 2 +- cns/endpointmanager/endpointmanager.go | 1 - cns/endpointmanager/endpointmanager_windows.go | 14 +------------- network/manager.go | 1 + 5 files changed, 4 insertions(+), 15 deletions(-) diff --git a/cni/netconfig.go b/cni/netconfig.go index c7e0c0ca7e..f69a5bcbe2 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -72,6 +72,7 @@ type NetworkConfig struct { EnableExactMatchForPodName bool `json:"enableExactMatchForPodName,omitempty"` DisableHairpinOnHostInterface bool `json:"disableHairpinOnHostInterface,omitempty"` DisableIPTableLock bool `json:"disableIPTableLock,omitempty"` + DisableAsyncDelete bool `json:"disableAsyncDelete,omitempty"` CNSUrl string `json:"cnsurl,omitempty"` ExecutionMode string `json:"executionMode,omitempty"` IPAM IPAM `json:"ipam,omitempty"` diff --git a/cni/network/network.go b/cni/network/network.go index 8226c38d5b..29181b42a4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -1079,7 +1079,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID) // if stateless CNI fail to get the endpoint from CNS for any reason other than Endpoint Not found if err != nil { - if errors.Is(err, network.ErrConnectionFailure) { + if errors.Is(err, network.ErrConnectionFailure) && !nwCfg.DisableAsyncDelete { logger.Info("failed to connect to CNS", zap.String("containerID", args.ContainerID), zap.Error(err)) addErr := fsnotify.AddFile(args.ContainerID, args.ContainerID, watcherPath) logger.Info("add containerid file for Asynch delete", zap.String("containerID", args.ContainerID), zap.Error(addErr)) diff --git a/cns/endpointmanager/endpointmanager.go b/cns/endpointmanager/endpointmanager.go index 3f71ecfa80..e71f0a6cbc 100644 --- a/cns/endpointmanager/endpointmanager.go +++ b/cns/endpointmanager/endpointmanager.go @@ -14,7 +14,6 @@ type EndpointManager struct { type releaseIPsClient interface { ReleaseIPs(ctx context.Context, ipconfig cns.IPConfigsRequest) error GetEndpoint(ctx context.Context, endpointID string) (*restserver.GetEndpointResponse, error) - DeleteEndpointState(ctx context.Context, endpointID string) (*cns.Response, error) } func WithPlatformReleaseIPsManager(cli releaseIPsClient) *EndpointManager { diff --git a/cns/endpointmanager/endpointmanager_windows.go b/cns/endpointmanager/endpointmanager_windows.go index 60c77f1057..5e87b362da 100644 --- a/cns/endpointmanager/endpointmanager_windows.go +++ b/cns/endpointmanager/endpointmanager_windows.go @@ -6,7 +6,6 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/hnsclient" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/types" "github.com/pkg/errors" ) @@ -18,18 +17,7 @@ func (em *EndpointManager) ReleaseIPs(ctx context.Context, ipconfigreq cns.IPCon logger.Errorf("failed to remove HNS endpoint %s", err.Error()) } - if err := em.cli.ReleaseIPs(ctx, ipconfigreq); err != nil { - return errors.Wrap(err, "failed to release IP from CNS") - } - - res, err := em.cli.DeleteEndpointState(ctx, ipconfigreq.InfraContainerID) - if err != nil { - if res.ReturnCode != types.NotFound { - return errors.Wrap(err, "") - } - } - - return nil + return em.cli.ReleaseIPs(ctx, ipconfigreq) } // deleteEndpoint API to get the state and then remove assiciated HNS diff --git a/network/manager.go b/network/manager.go index f2dd90d910..08b4332160 100644 --- a/network/manager.go +++ b/network/manager.go @@ -773,6 +773,7 @@ func (nm *networkManager) DeleteState(epInfos []*EndpointInfo) error { if nm.IsStatelessCNIMode() { for _, epInfo := range epInfos { + // this cleanup happens only for standalone swiftv2 to delete endpoint state from CNS. if epInfo.NICType == cns.NodeNetworkInterfaceFrontendNIC || epInfo.NICType == cns.NodeNetworkInterfaceAccelnetFrontendNIC { response, err := nm.CnsClient.DeleteEndpointState(context.TODO(), epInfo.EndpointID) if err != nil { From ac7f244fb2ab39f9298b2441236be25b3089806a Mon Sep 17 00:00:00 2001 From: Tamilmani Date: Fri, 31 Oct 2025 13:35:01 -0700 Subject: [PATCH 6/7] lint fixes and address comment Signed-off-by: Tamilmani --- cni/network/invoker_cns.go | 2 +- cni/network/network.go | 3 +-- cni/network/network_windows.go | 2 +- cns/Dockerfile | 2 +- cns/endpointmanager/endpointmanager_windows.go | 2 +- cns/restserver/ipam.go | 3 ++- network/endpoint_windows.go | 5 ++++- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 380421425f..909b2a6366 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -516,7 +516,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p func configureApipaAddResult(addResult *IPAMAddResult, info *cns.PodIpInfo, key string) error { ip, ipnet, err := info.PodIPConfig.GetIPNet() if ip == nil { - return errors.Wrap(err, "Unable to parse IP from response: "+info.PodIPConfig.IPAddress+" with err %w") + return errors.Wrap(err, "GetIPNet failed while configuring apipa AddResult") } addResult.interfaceInfo[key] = network.InterfaceInfo{ diff --git a/cni/network/network.go b/cni/network/network.go index 29181b42a4..b9a620d1ae 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -678,9 +678,8 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn opt.ifInfo.HostSubnetPrefix.IP = opt.ifInfo.HostSubnetPrefix.IP.Mask(opt.ifInfo.HostSubnetPrefix.Mask) opt.ipamAddConfig.nwCfg.IPAM.Subnet = opt.ifInfo.HostSubnetPrefix.String() - var masterIfName string // populate endpoint info section - masterIfName = plugin.findMasterInterface(opt) + masterIfName := plugin.findMasterInterface(opt) if masterIfName == "" { err := plugin.Errorf("Failed to find the master interface") return nil, err diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index c76af56b1f..255fad470f 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -79,7 +79,7 @@ func (plugin *NetPlugin) getNetworkName(netNs string, interfaceInfo *network.Int } if interfaceInfo != nil && interfaceInfo.NICType == cns.ApipaNIC { - return swiftv2NetworkNamePrefix + "apipa", nil + return swiftv2NetworkNamePrefix + apipaInterfacePrefix, nil } // For singletenancy, the network name is simply the nwCfg.Name diff --git a/cns/Dockerfile b/cns/Dockerfile index 3f834c42a7..d205242ee6 100644 --- a/cns/Dockerfile +++ b/cns/Dockerfile @@ -38,4 +38,4 @@ FROM --platform=windows/${ARCH} mcr.microsoft.com/oss/kubernetes/windows-host-pr FROM hpc as windows COPY --from=builder /go/bin/azure-cns /azure-cns.exe ENTRYPOINT ["azure-cns.exe"] -EXPOSE 10090 +EXPOSE 10090 \ No newline at end of file diff --git a/cns/endpointmanager/endpointmanager_windows.go b/cns/endpointmanager/endpointmanager_windows.go index 5e87b362da..71ba854d47 100644 --- a/cns/endpointmanager/endpointmanager_windows.go +++ b/cns/endpointmanager/endpointmanager_windows.go @@ -17,7 +17,7 @@ func (em *EndpointManager) ReleaseIPs(ctx context.Context, ipconfigreq cns.IPCon logger.Errorf("failed to remove HNS endpoint %s", err.Error()) } - return em.cli.ReleaseIPs(ctx, ipconfigreq) + return errors.Wrap(em.cli.ReleaseIPs(ctx, ipconfigreq), "failed to release IP from CNS") } // deleteEndpoint API to get the state and then remove assiciated HNS diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 75e8b1210c..d187d6b3d3 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -161,6 +161,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte podIPInfoList = append(podIPInfoList, podIPInfo) if resp[i].AllowHostToNCCommunication || resp[i].AllowNCToHostCommunication { apipaIndex = i + break } } @@ -1399,7 +1400,7 @@ func updateIPInfoMap(iPInfo map[string]*IPInfo, interfaceInfo *IPInfo, ifName, e if interfaceInfo.NetworkContainerID != "" { iPInfo[ifName].NetworkContainerID = interfaceInfo.NetworkContainerID - logger.Printf("[updateEndpoint] update the endpoint %s with NetworkContainerID %s", endpointID, interfaceInfo.NetworkContainerID) + logger.Printf("[updateEndpoint] update the endpoint %s with NetworkContainerID %s", endpointID, interfaceInfo.NetworkContainerID) //nolint } } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index ad5c72e4da..2e95572a35 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -403,6 +403,7 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn ) if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + //nolint return nil, fmt.Errorf("Failed to retrieve namespace with GetNamespaceByID for NetNsPath: %s"+ " due to error: %v", epInfo.NetNsPath, err) } @@ -411,6 +412,7 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn zap.String("NetworkContainerID", epInfo.NetworkContainerID)) if hostNCApipaEndpointID, err = cli.CreateHostNCApipaEndpoint(context.TODO(), epInfo.NetworkContainerID); err != nil { + //nolint return nil, err } @@ -421,7 +423,8 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn }() if err = hcn.AddNamespaceEndpoint(namespace.Id, hostNCApipaEndpointID); err != nil { - return nil, fmt.Errorf("Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", hostNCApipaEndpointID, namespace.Id, err) //nolint + //nolint + return nil, fmt.Errorf("Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", hostNCApipaEndpointID, namespace.Id, err) } ep := &endpoint{ From 35edfc1bfbd57dcc2ff5cf4f6bf73bc63f189550 Mon Sep 17 00:00:00 2001 From: Tamilmani Date: Wed, 5 Nov 2025 13:58:54 -0800 Subject: [PATCH 7/7] Address review comments Signed-off-by: Tamilmani --- cni/network/invoker_cns_test.go | 312 ++++++++++++++++++ cni/network/network.go | 1 + cni/network/network_windows_test.go | 39 +++ cns/client/client.go | 2 + .../endpointmanager_windows.go | 1 - cns/restserver/ipam.go | 4 +- network/endpoint.go | 27 +- network/endpoint_windows.go | 5 +- network/endpoint_windows_test.go | 6 +- network/manager.go | 12 +- 10 files changed, 381 insertions(+), 28 deletions(-) diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index b28798cc28..311a231845 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -2298,3 +2298,315 @@ func TestMultipleIBNICsToResult(t *testing.T) { }) } } + +func TestCNSIPAMInvoker_Add_ApipaNIC(t *testing.T) { + require := require.New(t) + + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + ipamMode util.IpamMode + } + type args struct { + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + options map[string]interface{} + } + + tests := []struct { + name string + fields fields + args args + wantApipaResult network.InterfaceInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Test CNI Add with InfraNIC + ApipaNIC", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname", + InfraContainerID: "testcontainerid", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + GatewayIPAddress: "10.0.1.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "169.254.128.10", + PrefixLength: 17, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + GatewayIPAddress: "169.254.128.1", + }, + NICType: cns.ApipaNIC, + NetworkContainerID: "test-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + wantApipaResult: network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("169.254.128.10"), + Mask: net.CIDRMask(17, 32), + }, + Gateway: net.ParseIP("169.254.128.1"), + }, + }, + NICType: cns.ApipaNIC, + SkipDefaultRoutes: true, + NetworkContainerID: "test-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + wantErr: false, + }, + { + name: "Test CNI add with Frontend Nic + ApipaNIC", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname", + InfraContainerID: "testcontainerid", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + GatewayIPAddress: "10.0.1.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + MacAddress: "bc:9a:78:56:34:12", + NICType: cns.NodeNetworkInterfaceFrontendNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "169.254.5.50", + PrefixLength: 16, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + GatewayIPAddress: "169.254.5.1", + }, + NICType: cns.ApipaNIC, + NetworkContainerID: "mixed-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + wantApipaResult: network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("169.254.5.50"), + Mask: net.CIDRMask(16, 32), + }, + Gateway: net.ParseIP("169.254.5.1"), + }, + }, + NICType: cns.ApipaNIC, + SkipDefaultRoutes: true, + NetworkContainerID: "mixed-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + wantErr: false, + }, + { + name: "Test CNI add with ApipaNIC fails when GetIPNet fails", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname", + InfraContainerID: "testcontainerid", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "invalid-ip-address", + PrefixLength: 16, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + GatewayIPAddress: "169.254.1.1", + }, + NICType: cns.ApipaNIC, + NetworkContainerID: "failed-nc-id", + AllowHostToNCCommunication: false, + AllowNCToHostCommunication: false, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(_ *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + if tt.fields.ipamMode != "" { + invoker.ipamMode = tt.fields.ipamMode + } + + ipamAddResult, err := invoker.Add(IPAMAddConfig{ + nwCfg: tt.args.nwCfg, + args: tt.args.args, + options: tt.args.options, + }) + + if tt.wantErr { + require.Error(err) + return + } + + require.NoError(err) + + // Find the ApipaNIC interface in the result + var apipaInterfaceFound bool + var actualApipaResult network.InterfaceInfo + + for _, ifInfo := range ipamAddResult.interfaceInfo { + if ifInfo.NICType == cns.ApipaNIC { + apipaInterfaceFound = true + actualApipaResult = ifInfo + break + } + } + + require.True(apipaInterfaceFound, "ApipaNIC interface should be found in the result") + + // Verify the ApipaNIC interface info + // Lines around 2586-2590 should be: + require.Equal(string(tt.wantApipaResult.NICType), string(actualApipaResult.NICType), "NICType should match expected value") + require.Equal(tt.wantApipaResult.SkipDefaultRoutes, actualApipaResult.SkipDefaultRoutes) + require.Equal(tt.wantApipaResult.NetworkContainerID, actualApipaResult.NetworkContainerID) + require.Equal(tt.wantApipaResult.AllowHostToNCCommunication, actualApipaResult.AllowHostToNCCommunication) + require.Equal(tt.wantApipaResult.AllowNCToHostCommunication, actualApipaResult.AllowNCToHostCommunication) + + // Verify IP configs + require.Len(actualApipaResult.IPConfigs, 1, "Should have exactly one IP config for ApipaNIC") + actualIPConfig := actualApipaResult.IPConfigs[0] + expectedIPConfig := tt.wantApipaResult.IPConfigs[0] + + require.True(actualIPConfig.Address.IP.Equal(expectedIPConfig.Address.IP), + "IP addresses should match: expected %s, got %s", + expectedIPConfig.Address.IP, actualIPConfig.Address.IP) + require.Equal(expectedIPConfig.Address.Mask, actualIPConfig.Address.Mask, + "IP masks should match") + + if expectedIPConfig.Gateway != nil { + require.NotNil(actualIPConfig.Gateway, "Gateway should not be nil") + require.True(actualIPConfig.Gateway.Equal(expectedIPConfig.Gateway), + "Gateway IPs should match: expected %s, got %s", + expectedIPConfig.Gateway, actualIPConfig.Gateway) + } else { + require.Nil(actualIPConfig.Gateway, "Gateway should be nil") + } + }) + } +} diff --git a/cni/network/network.go b/cni/network/network.go index b9a620d1ae..29b3839b40 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -1078,6 +1078,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID) // if stateless CNI fail to get the endpoint from CNS for any reason other than Endpoint Not found if err != nil { + // async delete should be disabled for standalone scenario if errors.Is(err, network.ErrConnectionFailure) && !nwCfg.DisableAsyncDelete { logger.Info("failed to connect to CNS", zap.String("containerID", args.ContainerID), zap.Error(err)) addErr := fsnotify.AddFile(args.ContainerID, args.ContainerID, watcherPath) diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 8c47d739ca..bcf3199e31 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -661,6 +661,45 @@ func TestGetNetworkNameSwiftv2FromCNS(t *testing.T) { want: "", wantErr: false, }, + { + name: "Get Network Name from CNS for swiftv2 ApipaNIC", + plugin: &NetPlugin{ + Plugin: plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), + }, + netNs: "azure", + nwCfg: &cni.NetworkConfig{ + CNIVersion: "0.3.0", + MultiTenancy: false, + }, + interfaceInfo: &network.InterfaceInfo{ + Name: "apipa-interface", + MacAddress: parsedMacAddress, + NICType: cns.ApipaNIC, + }, + want: swiftv2NetworkNamePrefix + "apipa", // "azure-apipa" + wantErr: false, + }, + { + name: "Get Network Name from CNS for swiftv2 ApipaNIC with empty MacAddress", + plugin: &NetPlugin{ + Plugin: plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), + }, + netNs: "azure", + nwCfg: &cni.NetworkConfig{ + CNIVersion: "0.3.0", + MultiTenancy: false, + }, + interfaceInfo: &network.InterfaceInfo{ + Name: "apipa-test-interface", + NICType: cns.ApipaNIC, + }, + want: swiftv2NetworkNamePrefix + "apipa", // "azure-apipa" + wantErr: false, + }, } for _, tt := range tests { diff --git a/cns/client/client.go b/cns/client/client.go index ad0dc2dca5..7ef711f1cb 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1101,6 +1101,8 @@ func (c *Client) UpdateEndpoint(ctx context.Context, endpointID string, ipInfo m return &response, nil } +// DeleteEndpointState calls the DeleteEndpointHandler API in CNS to delete the state of a given EndpointID(containerID) +// This api is called for swiftv2 standalone scenario to cleanup state in CNS func (c *Client) DeleteEndpointState(ctx context.Context, endpointID string) (*cns.Response, error) { // build the request u := c.routes[cns.EndpointAPI] diff --git a/cns/endpointmanager/endpointmanager_windows.go b/cns/endpointmanager/endpointmanager_windows.go index 71ba854d47..dd928b721d 100644 --- a/cns/endpointmanager/endpointmanager_windows.go +++ b/cns/endpointmanager/endpointmanager_windows.go @@ -16,7 +16,6 @@ func (em *EndpointManager) ReleaseIPs(ctx context.Context, ipconfigreq cns.IPCon if err := em.deleteEndpoint(ctx, ipconfigreq.InfraContainerID); err != nil { logger.Errorf("failed to remove HNS endpoint %s", err.Error()) } - return errors.Wrap(em.cli.ReleaseIPs(ctx, ipconfigreq), "failed to release IP from CNS") } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index d187d6b3d3..cb38e5a93f 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -161,7 +161,6 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte podIPInfoList = append(podIPInfoList, podIPInfo) if resp[i].AllowHostToNCCommunication || resp[i].AllowNCToHostCommunication { apipaIndex = i - break } } @@ -1145,7 +1144,8 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt case http.MethodDelete: service.DeleteEndpointStateHandler(w, r) default: - logger.Errorf("[EndpointHandlerAPI] EndpointHandler API expect http Get or Patch method") + //nolint + logger.Errorf("[EndpointHandlerAPI] EndpointHandler API expect http Get or Patch or Delete method") } } diff --git a/network/endpoint.go b/network/endpoint.go index f1eb9b731e..08f2e81f6c 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -128,17 +128,18 @@ type RouteInfo struct { // InterfaceInfo contains information for secondary interfaces type InterfaceInfo struct { - Name string - MacAddress net.HardwareAddr - IPConfigs []*IPConfig - Routes []RouteInfo - DNS DNSInfo - NICType cns.NICType - SkipDefaultRoutes bool - HostSubnetPrefix net.IPNet // Move this field from ipamAddResult - NCResponse *cns.GetNetworkContainerResponse - PnPID string - EndpointPolicies []policy.Policy + Name string + MacAddress net.HardwareAddr + IPConfigs []*IPConfig + Routes []RouteInfo + DNS DNSInfo + NICType cns.NICType + SkipDefaultRoutes bool + HostSubnetPrefix net.IPNet // Move this field from ipamAddResult + NCResponse *cns.GetNetworkContainerResponse + PnPID string + EndpointPolicies []policy.Policy + // these fields will be required for swiftv2 apipa nic NetworkContainerID string AllowNCToHostCommunication bool AllowHostToNCCommunication bool @@ -170,10 +171,10 @@ func FormatSliceOfPointersToString[T any](slice []*T) string { func (epInfo *EndpointInfo) PrettyString() string { return fmt.Sprintf("EndpointID:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v NICType: %s "+ - "NetworkContainerID: %s HostIfName: %s NetNs: %s Options: %v MasterIfName: %s HNSEndpointID: %s HNSNetworkID: %s", + "NetworkContainerID: %s HostIfName: %s NetNs: %s Options: %v MasterIfName: %s HNSEndpointID: %s HNSNetworkID: %s AllowHostToNC:%t AllowNCToHost:%t", epInfo.EndpointID, epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName, epInfo.IfIndex, epInfo.MacAddress.String(), epInfo.IPAddresses, epInfo.Gateways, epInfo.Data, epInfo.NICType, epInfo.NetworkContainerID, epInfo.HostIfName, epInfo.NetNs, epInfo.Options, epInfo.MasterIfName, - epInfo.HNSEndpointID, epInfo.HNSNetworkID) + epInfo.HNSEndpointID, epInfo.HNSNetworkID, epInfo.AllowInboundFromHostToNC, epInfo.AllowInboundFromNCToHost) } func (ifInfo *InterfaceInfo) PrettyString() string { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 2e95572a35..418d052dce 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -550,10 +550,7 @@ func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.Exe } if ep.NICType == cns.ApipaNIC { - if err := nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID); err != nil { - logger.Error("Failed to delete HostNCApipaEndpoint due to error", zap.Error(err)) - return err - } + return nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID) } if ep.HnsId == "" { diff --git a/network/endpoint_windows_test.go b/network/endpoint_windows_test.go index 1dfb414bbc..a65add6215 100644 --- a/network/endpoint_windows_test.go +++ b/network/endpoint_windows_test.go @@ -666,15 +666,15 @@ func TestDeleteEndpointStateForInfraDelegatedNIC(t *testing.T) { HNSNetworkID: networkID, } - // mock DeleteEndpointState() to make sure endpoint and network is deleted from cache + // mock DeleteEndpointStateless() to make sure endpoint and network is deleted from cache // network and endpoint should be deleted from cache for delegatedNIC - err = nm.DeleteEndpointState(networkID, delegatedEpInfo) + err = nm.DeleteEndpointStateless(networkID, delegatedEpInfo) if err != nil { t.Fatalf("Failed to delete endpoint for delegatedNIC state due to %v", err) } // endpoint should be deleted from cache for delegatedNIC and network is still there - err = nm.DeleteEndpointState(infraNetworkID, infraEpInfo) + err = nm.DeleteEndpointStateless(infraNetworkID, infraEpInfo) if err != nil { t.Fatalf("Failed to delete endpoint for delegatedNIC state due to %v", err) } diff --git a/network/manager.go b/network/manager.go index 08b4332160..bef8858087 100644 --- a/network/manager.go +++ b/network/manager.go @@ -492,7 +492,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E if nm.IsStatelessCNIMode() { // Calls deleteEndpointImpl directly, skipping the get network check; does not call cns - return nm.DeleteEndpointState(networkID, epInfo) + return nm.DeleteEndpointStateless(networkID, epInfo) } nw, err := nm.getNetwork(networkID) @@ -508,7 +508,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E return nil } -func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *EndpointInfo) error { +func (nm *networkManager) DeleteEndpointStateless(networkID string, epInfo *EndpointInfo) error { // we want to always use hnsv2 in stateless // hnsv2 is only enabled if NetNs has a valid guid and the hnsv2 api is supported // by passing in a dummy guid, we satisfy the first condition @@ -767,14 +767,16 @@ func (nm *networkManager) DeleteState(epInfos []*EndpointInfo) error { defer nm.Unlock() logger.Info("Deleting state") - // We do not use DeleteEndpointState for stateless cni because we already call it in DeleteEndpoint - // This function is only for saving to stateless cni or the cni statefile - // For stateless cni, plugin.ipamInvoker.Delete takes care of removing the state in the main Delete function + // For AKS stateless cni, plugin.ipamInvoker.Delete takes care of removing the state in the main Delete function. + // For swiftv2 stateless cni, this call will delete the endpoint state from CNS. if nm.IsStatelessCNIMode() { for _, epInfo := range epInfos { // this cleanup happens only for standalone swiftv2 to delete endpoint state from CNS. if epInfo.NICType == cns.NodeNetworkInterfaceFrontendNIC || epInfo.NICType == cns.NodeNetworkInterfaceAccelnetFrontendNIC { + // swiftv2 multitenancy does not call plugin.ipamInvoker.Delete and so state does not automatically clean up. this call is required to + // cleanup state in CNS + // One Delete call for endpointID will remove all interface info associated with that endpointID in CNS response, err := nm.CnsClient.DeleteEndpointState(context.TODO(), epInfo.EndpointID) if err != nil { if response != nil && response.ReturnCode == types.NotFound {