diff --git a/cni/netconfig.go b/cni/netconfig.go index c7e0c0ca7e..f69a5bcbe2 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -72,6 +72,7 @@ type NetworkConfig struct { EnableExactMatchForPodName bool `json:"enableExactMatchForPodName,omitempty"` DisableHairpinOnHostInterface bool `json:"disableHairpinOnHostInterface,omitempty"` DisableIPTableLock bool `json:"disableIPTableLock,omitempty"` + DisableAsyncDelete bool `json:"disableAsyncDelete,omitempty"` CNSUrl string `json:"cnsurl,omitempty"` ExecutionMode string `json:"executionMode,omitempty"` IPAM IPAM `json:"ipam,omitempty"` diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 928096b361..909b2a6366 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -165,7 +165,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } logger.Info("Received info for pod", - zap.Any("ipInfo", info), + zap.Any("ipInfo", response.PodIPInfo[i]), zap.Any("podInfo", podInfo)) //nolint:exhaustive // ignore exhaustive types check @@ -192,6 +192,11 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro if err := addBackendNICToResult(&info, &addResult, key); err != nil { return IPAMAddResult{}, err } + case cns.ApipaNIC: + if err := configureApipaAddResult(&addResult, &response.PodIPInfo[i], key); err != nil { + return IPAMAddResult{}, err + } + case cns.InfraNIC, "": // if we change from legacy cns, the nicType will be empty, so we assume it is infra nic info.nicType = cns.InfraNIC @@ -508,6 +513,32 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p return nil } +func configureApipaAddResult(addResult *IPAMAddResult, info *cns.PodIpInfo, key string) error { + ip, ipnet, err := info.PodIPConfig.GetIPNet() + if ip == nil { + return errors.Wrap(err, "GetIPNet failed while configuring apipa AddResult") + } + + addResult.interfaceInfo[key] = network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: ip, + Mask: ipnet.Mask, + }, + Gateway: net.ParseIP(info.NetworkContainerPrimaryIPConfig.GatewayIPAddress), + }, + }, + NICType: info.NICType, + SkipDefaultRoutes: true, + NetworkContainerID: info.NetworkContainerID, + AllowHostToNCCommunication: info.AllowHostToNCCommunication, + AllowNCToHostCommunication: info.AllowNCToHostCommunication, + } + + return nil +} + func addBackendNICToResult(info *IPResultInfo, addResult *IPAMAddResult, key string) error { macAddress, err := net.ParseMAC(info.macAddress) if err != nil { diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index b28798cc28..311a231845 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -2298,3 +2298,315 @@ func TestMultipleIBNICsToResult(t *testing.T) { }) } } + +func TestCNSIPAMInvoker_Add_ApipaNIC(t *testing.T) { + require := require.New(t) + + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + ipamMode util.IpamMode + } + type args struct { + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + options map[string]interface{} + } + + tests := []struct { + name string + fields fields + args args + wantApipaResult network.InterfaceInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Test CNI Add with InfraNIC + ApipaNIC", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname", + InfraContainerID: "testcontainerid", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + GatewayIPAddress: "10.0.1.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "169.254.128.10", + PrefixLength: 17, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + GatewayIPAddress: "169.254.128.1", + }, + NICType: cns.ApipaNIC, + NetworkContainerID: "test-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + wantApipaResult: network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("169.254.128.10"), + Mask: net.CIDRMask(17, 32), + }, + Gateway: net.ParseIP("169.254.128.1"), + }, + }, + NICType: cns.ApipaNIC, + SkipDefaultRoutes: true, + NetworkContainerID: "test-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + wantErr: false, + }, + { + name: "Test CNI add with Frontend Nic + ApipaNIC", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname", + InfraContainerID: "testcontainerid", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + GatewayIPAddress: "10.0.1.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + MacAddress: "bc:9a:78:56:34:12", + NICType: cns.NodeNetworkInterfaceFrontendNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "169.254.5.50", + PrefixLength: 16, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + GatewayIPAddress: "169.254.5.1", + }, + NICType: cns.ApipaNIC, + NetworkContainerID: "mixed-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + wantApipaResult: network.InterfaceInfo{ + IPConfigs: []*network.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("169.254.5.50"), + Mask: net.CIDRMask(16, 32), + }, + Gateway: net.ParseIP("169.254.5.1"), + }, + }, + NICType: cns.ApipaNIC, + SkipDefaultRoutes: true, + NetworkContainerID: "mixed-nc-id", + AllowHostToNCCommunication: true, + AllowNCToHostCommunication: false, + }, + wantErr: false, + }, + { + name: "Test CNI add with ApipaNIC fails when GetIPNet fails", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname", + InfraContainerID: "testcontainerid", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "invalid-ip-address", + PrefixLength: 16, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + GatewayIPAddress: "169.254.1.1", + }, + NICType: cns.ApipaNIC, + NetworkContainerID: "failed-nc-id", + AllowHostToNCCommunication: false, + AllowNCToHostCommunication: false, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(_ *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + if tt.fields.ipamMode != "" { + invoker.ipamMode = tt.fields.ipamMode + } + + ipamAddResult, err := invoker.Add(IPAMAddConfig{ + nwCfg: tt.args.nwCfg, + args: tt.args.args, + options: tt.args.options, + }) + + if tt.wantErr { + require.Error(err) + return + } + + require.NoError(err) + + // Find the ApipaNIC interface in the result + var apipaInterfaceFound bool + var actualApipaResult network.InterfaceInfo + + for _, ifInfo := range ipamAddResult.interfaceInfo { + if ifInfo.NICType == cns.ApipaNIC { + apipaInterfaceFound = true + actualApipaResult = ifInfo + break + } + } + + require.True(apipaInterfaceFound, "ApipaNIC interface should be found in the result") + + // Verify the ApipaNIC interface info + // Lines around 2586-2590 should be: + require.Equal(string(tt.wantApipaResult.NICType), string(actualApipaResult.NICType), "NICType should match expected value") + require.Equal(tt.wantApipaResult.SkipDefaultRoutes, actualApipaResult.SkipDefaultRoutes) + require.Equal(tt.wantApipaResult.NetworkContainerID, actualApipaResult.NetworkContainerID) + require.Equal(tt.wantApipaResult.AllowHostToNCCommunication, actualApipaResult.AllowHostToNCCommunication) + require.Equal(tt.wantApipaResult.AllowNCToHostCommunication, actualApipaResult.AllowNCToHostCommunication) + + // Verify IP configs + require.Len(actualApipaResult.IPConfigs, 1, "Should have exactly one IP config for ApipaNIC") + actualIPConfig := actualApipaResult.IPConfigs[0] + expectedIPConfig := tt.wantApipaResult.IPConfigs[0] + + require.True(actualIPConfig.Address.IP.Equal(expectedIPConfig.Address.IP), + "IP addresses should match: expected %s, got %s", + expectedIPConfig.Address.IP, actualIPConfig.Address.IP) + require.Equal(expectedIPConfig.Address.Mask, actualIPConfig.Address.Mask, + "IP masks should match") + + if expectedIPConfig.Gateway != nil { + require.NotNil(actualIPConfig.Gateway, "Gateway should not be nil") + require.True(actualIPConfig.Gateway.Equal(expectedIPConfig.Gateway), + "Gateway IPs should match: expected %s, got %s", + expectedIPConfig.Gateway, actualIPConfig.Gateway) + } else { + require.Nil(actualIPConfig.Gateway, "Gateway should be nil") + } + }) + } +} diff --git a/cni/network/network.go b/cni/network/network.go index a09db694c0..29b3839b40 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -52,6 +52,7 @@ const ( ipv4FullMask = 32 ipv6FullMask = 128 ibInterfacePrefix = "ib" + apipaInterfacePrefix = "apipa" ) // CNI Operation Types @@ -643,6 +644,8 @@ func (plugin *NetPlugin) findMasterInterface(opt *createEpInfoOpt) string { // when the VF is dismounted, this interface will go away // return an unique interface name to containerd return ibInterfacePrefix + strconv.Itoa(opt.endpointIndex) + case cns.ApipaNIC: + return apipaInterfacePrefix + strconv.Itoa(opt.endpointIndex) default: return "" } @@ -757,8 +760,11 @@ func (plugin *NetPlugin) createEpInfo(opt *createEpInfoOpt) (*network.EndpointIn IPAddresses: addresses, MacAddress: opt.ifInfo.MacAddress, // the following is used for creating an external interface if we can't find an existing network - HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(), - PnPID: opt.ifInfo.PnPID, + HostSubnetPrefix: opt.ifInfo.HostSubnetPrefix.String(), + PnPID: opt.ifInfo.PnPID, + NetworkContainerID: opt.ifInfo.NetworkContainerID, + AllowInboundFromHostToNC: opt.ifInfo.AllowHostToNCCommunication, + AllowInboundFromNCToHost: opt.ifInfo.AllowNCToHostCommunication, } if err = addSubnetToEndpointInfo(*opt.ifInfo, &endpointInfo); err != nil { @@ -1072,7 +1078,8 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { epInfos, err = plugin.nm.GetEndpointState(networkID, args.ContainerID) // if stateless CNI fail to get the endpoint from CNS for any reason other than Endpoint Not found if err != nil { - if errors.Is(err, network.ErrConnectionFailure) { + // async delete should be disabled for standalone scenario + if errors.Is(err, network.ErrConnectionFailure) && !nwCfg.DisableAsyncDelete { logger.Info("failed to connect to CNS", zap.String("containerID", args.ContainerID), zap.Error(err)) addErr := fsnotify.AddFile(args.ContainerID, args.ContainerID, watcherPath) logger.Info("add containerid file for Asynch delete", zap.String("containerID", args.ContainerID), zap.Error(addErr)) @@ -1152,10 +1159,10 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { } } } - logger.Info("Deleting the state from the cni statefile") + logger.Info("Deleting endpoint state from statefile") err = plugin.nm.DeleteState(epInfos) if err != nil { - return plugin.RetriableError(fmt.Errorf("failed to save state: %w", err)) + return plugin.RetriableError(fmt.Errorf("failed to delete state: %w", err)) } return err diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index f7d2e5defb..255fad470f 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -75,10 +75,13 @@ func (plugin *NetPlugin) getNetworkName(netNs string, interfaceInfo *network.Int // Swiftv2 L1VH Network Name swiftv2NetworkNamePrefix := "azure-" if interfaceInfo != nil && (interfaceInfo.NICType == cns.NodeNetworkInterfaceFrontendNIC || interfaceInfo.NICType == cns.BackendNIC) { - logger.Info("swiftv2", zap.String("network name", interfaceInfo.MacAddress.String())) return swiftv2NetworkNamePrefix + interfaceInfo.MacAddress.String(), nil } + if interfaceInfo != nil && interfaceInfo.NICType == cns.ApipaNIC { + return swiftv2NetworkNamePrefix + apipaInterfacePrefix, nil + } + // For singletenancy, the network name is simply the nwCfg.Name if !nwCfg.MultiTenancy { return nwCfg.Name, nil diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 8c47d739ca..bcf3199e31 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -661,6 +661,45 @@ func TestGetNetworkNameSwiftv2FromCNS(t *testing.T) { want: "", wantErr: false, }, + { + name: "Get Network Name from CNS for swiftv2 ApipaNIC", + plugin: &NetPlugin{ + Plugin: plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), + }, + netNs: "azure", + nwCfg: &cni.NetworkConfig{ + CNIVersion: "0.3.0", + MultiTenancy: false, + }, + interfaceInfo: &network.InterfaceInfo{ + Name: "apipa-interface", + MacAddress: parsedMacAddress, + NICType: cns.ApipaNIC, + }, + want: swiftv2NetworkNamePrefix + "apipa", // "azure-apipa" + wantErr: false, + }, + { + name: "Get Network Name from CNS for swiftv2 ApipaNIC with empty MacAddress", + plugin: &NetPlugin{ + Plugin: plugin, + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), + }, + netNs: "azure", + nwCfg: &cni.NetworkConfig{ + CNIVersion: "0.3.0", + MultiTenancy: false, + }, + interfaceInfo: &network.InterfaceInfo{ + Name: "apipa-test-interface", + NICType: cns.ApipaNIC, + }, + want: swiftv2NetworkNamePrefix + "apipa", // "azure-apipa" + wantErr: false, + }, } for _, tt := range tests { diff --git a/cns/Dockerfile b/cns/Dockerfile index 3f834c42a7..d205242ee6 100644 --- a/cns/Dockerfile +++ b/cns/Dockerfile @@ -38,4 +38,4 @@ FROM --platform=windows/${ARCH} mcr.microsoft.com/oss/kubernetes/windows-host-pr FROM hpc as windows COPY --from=builder /go/bin/azure-cns /azure-cns.exe ENTRYPOINT ["azure-cns.exe"] -EXPOSE 10090 +EXPOSE 10090 \ No newline at end of file diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 406c45b554..d8982186ed 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -93,6 +93,9 @@ const ( NodeNetworkInterfaceFrontendNIC NICType = "FrontendNIC" // NodeNetworkInterfaceBackendNIC is the new name for BackendNIC NodeNetworkInterfaceBackendNIC NICType = "BackendNIC" + + // ApipaNIC is used for internal communication between host and container + ApipaNIC NICType = "ApipaNIC" ) // ChannelMode :- CNS channel modes @@ -516,6 +519,12 @@ type PodIpInfo struct { PnPID string // Default Deny ACL's to configure on HNS endpoints for Swiftv2 window nodes EndpointPolicies []policy.Policy + // This flag is in effect only if nic type is apipa. This allows connection originating from host to container via apipa nic and not other way. + AllowHostToNCCommunication bool + // This flag is in effect only if nic type is apipa. This allows connection originating from container to host via apipa nic and not other way. + AllowNCToHostCommunication bool + // NetworkContainerID is the ID of the network container to which this Pod IP belongs + NetworkContainerID string } type HostIPInfo struct { diff --git a/cns/client/client.go b/cns/client/client.go index 1021d6f412..7ef711f1cb 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1100,3 +1100,37 @@ func (c *Client) UpdateEndpoint(ctx context.Context, endpointID string, ipInfo m return &response, nil } + +// DeleteEndpointState calls the DeleteEndpointHandler API in CNS to delete the state of a given EndpointID(containerID) +// This api is called for swiftv2 standalone scenario to cleanup state in CNS +func (c *Client) DeleteEndpointState(ctx context.Context, endpointID string) (*cns.Response, error) { + // build the request + u := c.routes[cns.EndpointAPI] + uString := u.String() + endpointID + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, uString, http.NoBody) + if err != nil { + return nil, errors.Wrap(err, "failed to build request") + } + req.Header.Set(headerContentType, contentTypeJSON) + res, err := c.client.Do(req) + if err != nil { + return nil, &ConnectionFailureErr{cause: err} + } + + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, errors.Errorf("http response %d", res.StatusCode) + } + + var response cns.Response + err = json.NewDecoder(res.Body).Decode(&response) + if err != nil { + return nil, errors.Wrap(err, "failed to decode CNS Response") + } + + if response.ReturnCode != 0 { + return nil, errors.New(response.Message) + } + + return &response, nil +} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 883d9aaef3..cb38e5a93f 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -149,14 +149,32 @@ func (service *HTTPRestService) requestIPConfigHandlerHelperStandalone(ctx conte // assign NICType and MAC Address for SwiftV2. we assume that there won't be any SwiftV1 NCs here podIPInfoList := make([]cns.PodIpInfo, 0, len(resp)) + apipaIndex := -1 for i := range resp { podIPInfo := cns.PodIpInfo{ PodIPConfig: resp[i].IPConfiguration.IPSubnet, MacAddress: resp[i].NetworkInterfaceInfo.MACAddress, NICType: resp[i].NetworkInterfaceInfo.NICType, NetworkContainerPrimaryIPConfig: resp[i].IPConfiguration, + NetworkContainerID: resp[i].NetworkContainerID, } podIPInfoList = append(podIPInfoList, podIPInfo) + if resp[i].AllowHostToNCCommunication || resp[i].AllowNCToHostCommunication { + apipaIndex = i + } + } + + if apipaIndex != -1 { + apipaPodIPInfo := cns.PodIpInfo{ + PodIPConfig: resp[apipaIndex].LocalIPConfiguration.IPSubnet, + NICType: cns.ApipaNIC, + NetworkContainerPrimaryIPConfig: resp[apipaIndex].LocalIPConfiguration, + SkipDefaultRoutes: true, + AllowHostToNCCommunication: resp[apipaIndex].AllowHostToNCCommunication, + AllowNCToHostCommunication: resp[apipaIndex].AllowNCToHostCommunication, + NetworkContainerID: resp[apipaIndex].NetworkContainerID, + } + podIPInfoList = append(podIPInfoList, apipaPodIPInfo) } ipConfigsResp := &cns.IPConfigsResponse{ @@ -1123,9 +1141,75 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt service.GetEndpointHandler(w, r) case http.MethodPatch: service.UpdateEndpointHandler(w, r) + case http.MethodDelete: + service.DeleteEndpointStateHandler(w, r) default: - logger.Errorf("[EndpointHandlerAPI] EndpointHandler API expect http Get or Patch method") + //nolint + logger.Errorf("[EndpointHandlerAPI] EndpointHandler API expect http Get or Patch or Delete method") + } +} + +func (service *HTTPRestService) DeleteEndpointStateHandler(w http.ResponseWriter, r *http.Request) { + opName := "DeleteEndpointStateHandler" + logger.Printf("[DeleteEndpointStateHandler] DeleteEndpointState for %s", r.URL.Path) //nolint:staticcheck // reason: using deprecated call until migration to new API + endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath) + + if service.EndpointStateStore == nil { + response := cns.Response{ + ReturnCode: types.NilEndpointStateStore, + Message: "[DeleteEndpointStateHandler] EndpointStateStore is not initialized", + } + err := common.Encode(w, &response) + logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API + return + } + + // Delete the endpoint from state + err := service.DeleteEndpointStateHelper(endpointID) + if err != nil { + response := cns.Response{ + ReturnCode: types.UnexpectedError, + Message: fmt.Sprintf("[DeleteEndpointStateHandler] Failed to delete endpoint state for %s with error: %s", endpointID, err.Error()), + } + + if errors.Is(err, ErrEndpointStateNotFound) { + response.ReturnCode = types.NotFound + } + + err = common.Encode(w, &response) + logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API + return + } + + response := cns.Response{ + ReturnCode: types.Success, + Message: "[DeleteEndpointStateHandler] Endpoint state deleted successfully", } + err = common.Encode(w, &response) + logger.Response(opName, response, response.ReturnCode, err) //nolint:staticcheck // reason: using deprecated call until migration to new API +} + +func (service *HTTPRestService) DeleteEndpointStateHelper(endpointID string) error { + if service.EndpointStateStore == nil { + return ErrStoreEmpty + } + logger.Printf("[deleteEndpointState] Deleting Endpoint state from state file %s", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API + _, endpointExist := service.EndpointState[endpointID] + if !endpointExist { + logger.Printf("[deleteEndpointState] endpoint could not be found in the statefile %s", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API + return fmt.Errorf("[deleteEndpointState] endpoint %s: %w", endpointID, ErrEndpointStateNotFound) + } + + // Delete the endpoint from the state + delete(service.EndpointState, endpointID) + + // Write the updated state back to the store + err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState) + if err != nil { + return fmt.Errorf("[deleteEndpointState] failed to write endpoint state to store: %w", err) + } + logger.Printf("[deleteEndpointState] successfully deleted endpoint %s from state file", endpointID) //nolint:staticcheck // reason: using deprecated call until migration to new API + return nil } // GetEndpointHandler handles the incoming GetEndpoint requests with http Get method @@ -1313,6 +1397,11 @@ func updateIPInfoMap(iPInfo map[string]*IPInfo, interfaceInfo *IPInfo, ifName, e iPInfo[ifName].MacAddress = interfaceInfo.MacAddress logger.Printf("[updateEndpoint] update the endpoint %s with MacAddress %s", endpointID, interfaceInfo.MacAddress) } + + if interfaceInfo.NetworkContainerID != "" { + iPInfo[ifName].NetworkContainerID = interfaceInfo.NetworkContainerID + logger.Printf("[updateEndpoint] update the endpoint %s with NetworkContainerID %s", endpointID, interfaceInfo.NetworkContainerID) //nolint + } } // verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 52c5b6d0d6..26b229d07f 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -2176,11 +2176,12 @@ func TestStatelessCNIStateFile(t *testing.T) { endpointInfo2ContainerID := "1b4917617e15d24dc495e407d8eb5c88e4406e58fa209e4eb75a2c2fb7045eea" endpointInfo2 := &EndpointInfo{IfnameToIPMap: make(map[string]*IPInfo)} endpointInfo2.IfnameToIPMap["eth2"] = &IPInfo{ - IPv4: nil, - NICType: cns.DelegatedVMNIC, - HnsEndpointID: "5c15cccc-830a-4dff-81f3-4b1e55cb7dcb", - HnsNetworkID: "5c0712cd-824c-4898-b1c0-2fcb16ede4fb", - MacAddress: "7c:1e:52:06:d3:4b", + IPv4: nil, + NICType: cns.DelegatedVMNIC, + HnsEndpointID: "5c15cccc-830a-4dff-81f3-4b1e55cb7dcb", + HnsNetworkID: "5c0712cd-824c-4898-b1c0-2fcb16ede4fb", + MacAddress: "7c:1e:52:06:d3:4b", + NetworkContainerID: testNCID, } // test cases tests := []struct { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 84589292ce..f14c69d12c 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -126,13 +126,14 @@ type EndpointInfo struct { } type IPInfo struct { - IPv4 []net.IPNet - IPv6 []net.IPNet `json:",omitempty"` - HnsEndpointID string `json:",omitempty"` - HnsNetworkID string `json:",omitempty"` - HostVethName string `json:",omitempty"` - MacAddress string `json:",omitempty"` - NICType cns.NICType + IPv4 []net.IPNet + IPv6 []net.IPNet `json:",omitempty"` + HnsEndpointID string `json:",omitempty"` + HnsNetworkID string `json:",omitempty"` + HostVethName string `json:",omitempty"` + MacAddress string `json:",omitempty"` + NetworkContainerID string `json:",omitempty"` + NICType cns.NICType } type GetHTTPServiceDataResponse struct { diff --git a/network/endpoint.go b/network/endpoint.go index bd9fa7fc9b..08f2e81f6c 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -139,6 +139,10 @@ type InterfaceInfo struct { NCResponse *cns.GetNetworkContainerResponse PnPID string EndpointPolicies []policy.Policy + // these fields will be required for swiftv2 apipa nic + NetworkContainerID string + AllowNCToHostCommunication bool + AllowHostToNCCommunication bool } type IPConfig struct { @@ -167,10 +171,10 @@ func FormatSliceOfPointersToString[T any](slice []*T) string { func (epInfo *EndpointInfo) PrettyString() string { return fmt.Sprintf("EndpointID:%s ContainerID:%s NetNsPath:%s IfName:%s IfIndex:%d MacAddr:%s IPAddrs:%v Gateways:%v Data:%+v NICType: %s "+ - "NetworkContainerID: %s HostIfName: %s NetNs: %s Options: %v MasterIfName: %s HNSEndpointID: %s HNSNetworkID: %s", + "NetworkContainerID: %s HostIfName: %s NetNs: %s Options: %v MasterIfName: %s HNSEndpointID: %s HNSNetworkID: %s AllowHostToNC:%t AllowNCToHost:%t", epInfo.EndpointID, epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName, epInfo.IfIndex, epInfo.MacAddress.String(), epInfo.IPAddresses, epInfo.Gateways, epInfo.Data, epInfo.NICType, epInfo.NetworkContainerID, epInfo.HostIfName, epInfo.NetNs, epInfo.Options, epInfo.MasterIfName, - epInfo.HNSEndpointID, epInfo.HNSNetworkID) + epInfo.HNSEndpointID, epInfo.HNSNetworkID, epInfo.AllowInboundFromHostToNC, epInfo.AllowInboundFromNCToHost) } func (ifInfo *InterfaceInfo) PrettyString() string { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index edd52327f2..418d052dce 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -157,12 +157,17 @@ func (nw *network) newEndpointImpl( return nw.getEndpointWithVFDevice(plc, epInfo) } + if epInfo.NICType == cns.ApipaNIC { + return nw.createHostNCApipaEndpoint(cli, epInfo) + } + if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err } return nw.newEndpointImplHnsV2(cli, epInfo) + } return nw.newEndpointImplHnsV1(epInfo, plc) @@ -354,14 +359,16 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE return hcnEndpoint, nil } +func getApipaEndpointName(networkContainerID string) string { + endpointName := fmt.Sprintf("%s-%s", hostNCApipaEndpointNamePrefix, networkContainerID) + return endpointName +} + func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { // TODO: this code is duplicated in cns/hnsclient, but that code has logging messages that require a CNSLogger, // which makes is hard to use in this package. We should refactor this into a common package with no logging deps // so it can be called in both places - - // HostNCApipaEndpoint name is derived from NC ID - endpointName := fmt.Sprintf("%s-%s", hostNCApipaEndpointNamePrefix, networkContainerID) - logger.Info("Deleting HostNCApipaEndpoint for NC", zap.String("endpointName", endpointName), zap.String("networkContainerID", networkContainerID)) + endpointName := getApipaEndpointName(networkContainerID) // Check if the endpoint exists endpoint, err := Hnsv2.GetEndpointByName(endpointName) @@ -376,6 +383,7 @@ func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { return nil } + logger.Info("Deleting Apipa Endpoint", zap.String("endpointName", endpointName)) if err := Hnsv2.DeleteEndpoint(endpoint); err != nil { return fmt.Errorf("failed to delete HostNCApipa endpoint: %+v: %w", endpoint, err) } @@ -387,7 +395,7 @@ func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { // createHostNCApipaEndpoint creates a new endpoint in the HostNCApipaNetwork // for host container connectivity -func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointInfo) error { +func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointInfo) (*endpoint, error) { var ( err error hostNCApipaEndpointID string @@ -395,7 +403,8 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn ) if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return fmt.Errorf("Failed to retrieve namespace with GetNamespaceByID for NetNsPath: %s"+ + //nolint + return nil, fmt.Errorf("Failed to retrieve namespace with GetNamespaceByID for NetNsPath: %s"+ " due to error: %v", epInfo.NetNsPath, err) } @@ -403,7 +412,8 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn zap.String("NetworkContainerID", epInfo.NetworkContainerID)) if hostNCApipaEndpointID, err = cli.CreateHostNCApipaEndpoint(context.TODO(), epInfo.NetworkContainerID); err != nil { - return err + //nolint + return nil, err } defer func() { @@ -413,10 +423,20 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn }() if err = hcn.AddNamespaceEndpoint(namespace.Id, hostNCApipaEndpointID); err != nil { - return fmt.Errorf("Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", hostNCApipaEndpointID, namespace.Id, err) //nolint + //nolint + return nil, fmt.Errorf("Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", hostNCApipaEndpointID, namespace.Id, err) } - return nil + ep := &endpoint{ + Id: getApipaEndpointName(epInfo.NetworkContainerID), + HnsId: hostNCApipaEndpointID, + IfName: getApipaEndpointName(epInfo.NetworkContainerID), + ContainerID: epInfo.ContainerID, + NICType: cns.ApipaNIC, + NetworkContainerID: epInfo.NetworkContainerID, + } + + return ep, nil } // newEndpointImplHnsV2 creates a new endpoint in the network using Hnsv2 @@ -464,7 +484,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( // If the Host - container connectivity is requested, create endpoint in HostNCApipaNetwork if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { - if err = nw.createHostNCApipaEndpoint(cli, epInfo); err != nil { + if _, err = nw.createHostNCApipaEndpoint(cli, epInfo); err != nil { return nil, fmt.Errorf("Failed to create HostNCApipaEndpoint due to error: %v", err) } } @@ -529,6 +549,10 @@ func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.Exe return nil } + if ep.NICType == cns.ApipaNIC { + return nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID) + } + if ep.HnsId == "" { logger.Error("No HNS id found. Skip endpoint deletion", zap.Any("nicType", ep.NICType), zap.String("containerId", ep.ContainerID)) return fmt.Errorf("No HNS id found. Skip endpoint deletion for nicType %v, containerID %s", ep.NICType, ep.ContainerID) //nolint diff --git a/network/endpoint_windows_test.go b/network/endpoint_windows_test.go index 1dfb414bbc..a65add6215 100644 --- a/network/endpoint_windows_test.go +++ b/network/endpoint_windows_test.go @@ -666,15 +666,15 @@ func TestDeleteEndpointStateForInfraDelegatedNIC(t *testing.T) { HNSNetworkID: networkID, } - // mock DeleteEndpointState() to make sure endpoint and network is deleted from cache + // mock DeleteEndpointStateless() to make sure endpoint and network is deleted from cache // network and endpoint should be deleted from cache for delegatedNIC - err = nm.DeleteEndpointState(networkID, delegatedEpInfo) + err = nm.DeleteEndpointStateless(networkID, delegatedEpInfo) if err != nil { t.Fatalf("Failed to delete endpoint for delegatedNIC state due to %v", err) } // endpoint should be deleted from cache for delegatedNIC and network is still there - err = nm.DeleteEndpointState(infraNetworkID, infraEpInfo) + err = nm.DeleteEndpointStateless(infraNetworkID, infraEpInfo) if err != nil { t.Fatalf("Failed to delete endpoint for delegatedNIC state due to %v", err) } diff --git a/network/manager.go b/network/manager.go index 7bc1441fea..bef8858087 100644 --- a/network/manager.go +++ b/network/manager.go @@ -424,7 +424,8 @@ func (nm *networkManager) UpdateEndpointState(eps []*endpoint) error { ifnameToIPInfoMap := generateCNSIPInfoMap(eps) // key : interface name, value : IPInfo for key, ipinfo := range ifnameToIPInfoMap { logger.Info("Update endpoint state", zap.String("ifname", key), zap.String("hnsEndpointID", ipinfo.HnsEndpointID), zap.String("hnsNetworkID", ipinfo.HnsNetworkID), - zap.String("hostVethName", ipinfo.HostVethName), zap.String("macAddress", ipinfo.MacAddress), zap.String("nicType", string(ipinfo.NICType))) + zap.String("hostVethName", ipinfo.HostVethName), zap.String("macAddress", ipinfo.MacAddress), zap.String("nicType", string(ipinfo.NICType)), + zap.String("networkContainerID", ipinfo.NetworkContainerID)) } // we assume all endpoints have the same container id cnsEndpointID := eps[0].ContainerID @@ -491,7 +492,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E if nm.IsStatelessCNIMode() { // Calls deleteEndpointImpl directly, skipping the get network check; does not call cns - return nm.DeleteEndpointState(networkID, epInfo) + return nm.DeleteEndpointStateless(networkID, epInfo) } nw, err := nm.getNetwork(networkID) @@ -507,7 +508,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *E return nil } -func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *EndpointInfo) error { +func (nm *networkManager) DeleteEndpointStateless(networkID string, epInfo *EndpointInfo) error { // we want to always use hnsv2 in stateless // hnsv2 is only enabled if NetNs has a valid guid and the hnsv2 api is supported // by passing in a dummy guid, we satisfy the first condition @@ -761,16 +762,33 @@ func (nm *networkManager) SaveState(eps []*endpoint) error { return nm.save() } -func (nm *networkManager) DeleteState(_ []*EndpointInfo) error { +func (nm *networkManager) DeleteState(epInfos []*EndpointInfo) error { nm.Lock() defer nm.Unlock() logger.Info("Deleting state") - // We do not use DeleteEndpointState for stateless cni because we already call it in DeleteEndpoint - // This function is only for saving to stateless cni or the cni statefile - // For stateless cni, plugin.ipamInvoker.Delete takes care of removing the state in the main Delete function + // For AKS stateless cni, plugin.ipamInvoker.Delete takes care of removing the state in the main Delete function. + // For swiftv2 stateless cni, this call will delete the endpoint state from CNS. if nm.IsStatelessCNIMode() { + for _, epInfo := range epInfos { + // this cleanup happens only for standalone swiftv2 to delete endpoint state from CNS. + if epInfo.NICType == cns.NodeNetworkInterfaceFrontendNIC || epInfo.NICType == cns.NodeNetworkInterfaceAccelnetFrontendNIC { + // swiftv2 multitenancy does not call plugin.ipamInvoker.Delete and so state does not automatically clean up. this call is required to + // cleanup state in CNS + // One Delete call for endpointID will remove all interface info associated with that endpointID in CNS + response, err := nm.CnsClient.DeleteEndpointState(context.TODO(), epInfo.EndpointID) + if err != nil { + if response != nil && response.ReturnCode == types.NotFound { + logger.Info("Endpoint state not found in CNS", zap.String("endpointID", epInfo.EndpointID)) + return nil + } + return errors.Wrapf(err, "Delete endpoint API returned with error for endpoint %s", epInfo.EndpointID) + } + logger.Info("Delete endpoint succeeded", zap.String("endpointID", epInfo.EndpointID), zap.String("returnCode", response.ReturnCode.String())) + break + } + } return nil } @@ -784,12 +802,11 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI for ifName, ipInfo := range endpointInfo.IfnameToIPMap { epInfo := &EndpointInfo{ - EndpointID: endpointID, // endpoint id is always the same, but we shouldn't use it in the stateless path - IfIndex: EndpointIfIndex, // Azure CNI supports only one interface - ContainerID: endpointID, - PODName: endpointInfo.PodName, - PODNameSpace: endpointInfo.PodNamespace, - NetworkContainerID: endpointID, + EndpointID: endpointID, // endpoint id is always the same, but we shouldn't use it in the stateless path + IfIndex: EndpointIfIndex, // Azure CNI supports only one interface + ContainerID: endpointID, + PODName: endpointInfo.PodName, + PODNameSpace: endpointInfo.PodNamespace, } // If we create an endpoint state with stateful cni and then swap to a stateless cni binary, ifname would not be populated @@ -809,6 +826,8 @@ func cnsEndpointInfotoCNIEpInfos(endpointInfo restserver.EndpointInfo, endpointI epInfo.NICType = ipInfo.NICType epInfo.HNSNetworkID = ipInfo.HnsNetworkID epInfo.MacAddress = net.HardwareAddr(ipInfo.MacAddress) + epInfo.NetworkContainerID = ipInfo.NetworkContainerID + ret = append(ret, epInfo) } return ret @@ -837,11 +856,12 @@ func generateCNSIPInfoMap(eps []*endpoint) map[string]*restserver.IPInfo { for _, ep := range eps { ifNametoIPInfoMap[ep.IfName] = &restserver.IPInfo{ // in windows, the nicname is args ifname, in linux, it's ethX - NICType: ep.NICType, - HnsEndpointID: ep.HnsId, - HnsNetworkID: ep.HNSNetworkID, - HostVethName: ep.HostIfName, - MacAddress: ep.MacAddress.String(), + NICType: ep.NICType, + HnsEndpointID: ep.HnsId, + HnsNetworkID: ep.HNSNetworkID, + HostVethName: ep.HostIfName, + MacAddress: ep.MacAddress.String(), + NetworkContainerID: ep.NetworkContainerID, } } diff --git a/network/network.go b/network/network.go index 31537b6522..41e1bf4307 100644 --- a/network/network.go +++ b/network/network.go @@ -108,7 +108,7 @@ func (nm *networkManager) newExternalInterface(ifName, subnet, nicType string) e // Find the host interface. macAddress := net.HardwareAddr{} - if nicType != string(cns.BackendNIC) { + if nicType != string(cns.BackendNIC) && nicType != string(cns.ApipaNIC) { hostIf, err := net.InterfaceByName(ifName) if err != nil { return errors.Wrap(err, "failed to find host interface") @@ -337,7 +337,6 @@ func (nm *networkManager) EndpointCreate(cnsclient apipaClient, epInfos []*Endpo return err } } - ep, err := nm.createEndpoint(cnsclient, epInfo.NetworkID, epInfo) if err != nil { return err diff --git a/network/network_windows.go b/network/network_windows.go index a467b20983..61e5159caf 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -342,7 +342,8 @@ func (nm *networkManager) addIPv6DefaultRoute() error { // newNetworkImplHnsV2 creates a new container network for HNSv2. func (nm *networkManager) newNetworkImplHnsV2(nwInfo *EndpointInfo, extIf *externalInterface) (*network, error) { // network creation is not required for IB - if nwInfo.NICType == cns.BackendNIC { + // For apipa nic, we create network as part of endpoint creation + if nwInfo.NICType == cns.BackendNIC || nwInfo.NICType == cns.ApipaNIC { return &network{Endpoints: make(map[string]*endpoint)}, nil }