From 4370e6e93c7623a70446580417df1ab0fc2b4ff9 Mon Sep 17 00:00:00 2001 From: Behzad Mirkhanzadeh Date: Tue, 7 May 2024 17:07:03 -0700 Subject: [PATCH] feat: Adding MACAddress to CNS endpoint State and refactoring stateless CNI code (#2722) * Adding MACAddress to CNS endpointState and also refactoring statelss cni code * Addressing the comments. * Addressing the comments --- cni/windows.Dockerfile | 2 +- cns/cnireconciler/podinfoprovider.go | 2 +- cns/restserver/ipam.go | 65 +++++++++++++++++++--------- cns/restserver/restserver.go | 2 + network/endpoint.go | 4 +- network/endpoint_windows.go | 10 ++--- network/manager.go | 31 ++++++++----- 7 files changed, 76 insertions(+), 40 deletions(-) diff --git a/cni/windows.Dockerfile b/cni/windows.Dockerfile index cef1b6a2dd..e363c74281 100644 --- a/cni/windows.Dockerfile +++ b/cni/windows.Dockerfile @@ -11,7 +11,7 @@ COPY . . RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/plugin/main.go RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-telemetry -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/telemetry/service/telemetrymain.go RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-ipam -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go -RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azurecni-stateless -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/stateless/main.go +RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-stateless -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/stateless/main.go FROM --platform=linux/${ARCH} mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor ARG OS diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 69bb7db519..4a19a91d2c 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -149,7 +149,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) map[string]*restserver // extractEndpointInfo extract Interface Name and endpointID for each endpoint based the CNI state func extractEndpointInfo(epID, containerID string) (endpointID, interfaceName string) { - ifName := restserver.InterfaceName + ifName := restserver.InfraInterfaceName if strings.Contains(epID, "-eth") { ifName = epID[len(epID)-4:] } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 8371bea1f5..689221ea69 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -30,8 +30,8 @@ var ( ) const ( - ContainerIDLength = 8 - InterfaceName = "eth0" + ContainerIDLength = 8 + InfraInterfaceName = "eth0" ) // requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs @@ -1071,7 +1071,7 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointI // This part is a temprory fix if we have endpoint states belong to CNI version 1.4.X on Windows since the states don't have the containerID // In case there was no endpoint founded with ContainerID as the key, // then [First 8 character of containerid]-eth0 will be tried - legacyEndpointID := endpointID[:ContainerIDLength] + "-" + InterfaceName + legacyEndpointID := endpointID[:ContainerIDLength] + "-" + InfraInterfaceName if endpointInfo, ok := service.EndpointState[legacyEndpointID]; ok { logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", legacyEndpointID) return endpointInfo, nil @@ -1129,42 +1129,67 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r * logger.Response(service.Name, response, response.ReturnCode, err) } -// UpdateEndpointHelper updates the state of the given endpointId with HNSId or VethName +// UpdateEndpointHelper updates the state of the given endpointId with HNSId, VethName or other InterfaceInfo fields func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req map[string]*IPInfo) error { if service.EndpointStateStore == nil { return ErrStoreEmpty } logger.Printf("[updateEndpoint] Updating endpoint state for infra container %s", endpointID) if endpointInfo, ok := service.EndpointState[endpointID]; ok { - for ifName, InterfaceInfo := range req { - logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID) - if InterfaceInfo.HnsEndpointID != "" { - service.EndpointState[endpointID].IfnameToIPMap[ifName].HnsEndpointID = InterfaceInfo.HnsEndpointID - logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, InterfaceInfo.HnsEndpointID) - } - if InterfaceInfo.HostVethName != "" { - service.EndpointState[endpointID].IfnameToIPMap[ifName].HostVethName = InterfaceInfo.HostVethName - logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, InterfaceInfo.HostVethName) - } - if InterfaceInfo.NICType != "" { - service.EndpointState[endpointID].IfnameToIPMap[ifName].NICType = InterfaceInfo.NICType - logger.Printf("[updateEndpoint] update the endpoint %s with NICType %s", endpointID, InterfaceInfo.NICType) - } + // Updating the InterfaceInfo map of endpoint states with the interfaceInfo map that is given by Stateless Azure CNI + for ifName, interfaceInfo := range req { + // updating the ipInfoMap + updateIPInfoMap(endpointInfo.IfnameToIPMap, interfaceInfo, ifName, endpointID) } err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState) if err != nil { return fmt.Errorf("[updateEndpoint] failed to write endpoint state to store for pod %s : %w", endpointInfo.PodName, err) } + logger.Printf("[updateEndpoint] successfully write the state to the file %s", endpointID) return nil } return errors.New("[updateEndpoint] endpoint could not be found in the statefile") } +// updateIPInfoMap updates the IfnameToIPMap of endpoint states with the interfaceInfo map that is given by Stateless Azure CNI +func updateIPInfoMap(iPInfo map[string]*IPInfo, interfaceInfo *IPInfo, ifName, endpointID string) { + // This codition will create a map for SecodaryNIC and also also creates MAP entry for InfraNic in case that the initial goalState is using empty InterfaceName + if _, keyExist := iPInfo[ifName]; !keyExist { + iPInfo[ifName] = &IPInfo{} + if val, emptyKeyExist := iPInfo[""]; emptyKeyExist && ifName == InfraInterfaceName { + iPInfo[ifName].IPv4 = val.IPv4 + iPInfo[ifName].IPv6 = val.IPv6 + delete(iPInfo, "") + } + } + logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s with %s , [%+v]", endpointID, ifName, interfaceInfo) + if interfaceInfo.HnsEndpointID != "" { + iPInfo[ifName].HnsEndpointID = interfaceInfo.HnsEndpointID + logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, interfaceInfo.HnsEndpointID) + } + if interfaceInfo.HnsNetworkID != "" { + iPInfo[ifName].HnsNetworkID = interfaceInfo.HnsNetworkID + logger.Printf("[updateEndpoint] update the endpoint %s with HnsNetworkID %s", endpointID, interfaceInfo.HnsEndpointID) + } + if interfaceInfo.HostVethName != "" { + iPInfo[ifName].HostVethName = interfaceInfo.HostVethName + logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, interfaceInfo.HostVethName) + } + if interfaceInfo.NICType != "" { + iPInfo[ifName].NICType = interfaceInfo.NICType + logger.Printf("[updateEndpoint] update the endpoint %s with NICType %s", endpointID, interfaceInfo.NICType) + } + if interfaceInfo.MacAddress != "" { + iPInfo[ifName].MacAddress = interfaceInfo.MacAddress + logger.Printf("[updateEndpoint] update the endpoint %s with MacAddress %s", endpointID, interfaceInfo.MacAddress) + } +} + // verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API func verifyUpdateEndpointStateRequest(req map[string]*IPInfo) error { for ifName, InterfaceInfo := range req { - if InterfaceInfo.HostVethName == "" && InterfaceInfo.HnsEndpointID == "" && InterfaceInfo.NICType == "" { - return errors.New("[updateEndpoint] No NicType, HnsEndpointID or HostVethName has been provided") + if InterfaceInfo.HostVethName == "" && InterfaceInfo.HnsEndpointID == "" && InterfaceInfo.NICType == "" && InterfaceInfo.MacAddress == "" { + return errors.New("[updateEndpoint] No NicType, MacAddress, HnsEndpointID or HostVethName has been provided") } if ifName == "" { return errors.New("[updateEndpoint] No Interface has been provided") diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 268c97ef14..cc17ef8724 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -99,7 +99,9 @@ type IPInfo struct { IPv4 []net.IPNet IPv6 []net.IPNet `json:",omitempty"` HnsEndpointID string `json:",omitempty"` + HnsNetworkID string `json:",omitempty"` HostVethName string `json:",omitempty"` + MacAddress string `json:",omitempty"` NICType cns.NICType } diff --git a/network/endpoint.go b/network/endpoint.go index db2c2cf01e..e530810e51 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -30,6 +30,7 @@ type AzureHNSEndpoint struct{} type endpoint struct { Id string HnsId string `json:",omitempty"` + HNSNetworkID string `json:",omitempty"` SandboxKey string IfName string HostIfName string @@ -92,6 +93,7 @@ type EndpointInfo struct { NICType cns.NICType SkipDefaultRoutes bool HNSEndpointID string + HNSNetworkID string HostIfName string } @@ -351,7 +353,7 @@ func GetPodNameWithoutSuffix(podName string) string { // IsEndpointStateInComplete returns true if both HNSEndpointID and HostVethName are missing. func (epInfo *EndpointInfo) IsEndpointStateIncomplete() bool { - if epInfo.HNSEndpointID == "" && epInfo.IfName == "" { + if epInfo.HNSEndpointID == "" && epInfo.HostIfName == "" { return true } return false diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index c0ddf486bf..b1f4e9ab98 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -499,10 +499,10 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi // GetEndpointInfoByIPImpl returns an endpointInfo with the corrsponding HNS Endpoint ID that matches an specific IP Address. func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) { - // check if network exists, only create the network does not exist + logger.Info("Fetching missing HNS endpoint id for endpoints in network with id", zap.String("id", networkID)) hnsResponse, err := Hnsv2.GetNetworkByName(networkID) - if err != nil { - return epInfo, errors.Wrapf(err, "HNS Network not found") + if err != nil || hnsResponse == nil { + return epInfo, errors.Wrapf(err, "HNS Network or endpoints not found") } hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id) if err != nil { @@ -511,8 +511,8 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, net for i := range hcnEndpoints { for _, ipConfiguration := range hcnEndpoints[i].IpConfigurations { for _, ipAddress := range ipAddresses { - prefixLength, _ := ipAddress.Mask.Size() - if ipConfiguration.IpAddress == ipAddress.IP.String() && ipConfiguration.PrefixLength == uint8(prefixLength) { + if ipConfiguration.IpAddress == ipAddress.IP.String() { + logger.Info("Successfully found hcn endpoint id for endpoint with ip", zap.String("id", hcnEndpoints[i].Id), zap.String("ip", ipAddress.IP.String())) epInfo.HNSEndpointID = hcnEndpoints[i].Id return epInfo, nil } diff --git a/network/manager.go b/network/manager.go index d82a95684f..9775c84ff7 100644 --- a/network/manager.go +++ b/network/manager.go @@ -38,6 +38,7 @@ const ( InfraInterfaceName = "eth0" ContainerIDLength = 8 EndpointIfIndex = 0 // Azure CNI supports only one interface + DefaultNetworkID = "azure" ) var Ipv4DefaultRouteDstPrefix = net.IPNet{ @@ -441,8 +442,10 @@ func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*Endpo return nil, errors.Wrapf(err, "Get endpoint API returend with error") } epInfo := cnsEndpointInfotoCNIEpInfo(endpointResponse.EndpointInfo, endpointID) - if epInfo.IsEndpointStateIncomplete() { + if networkID == "" { + networkID = DefaultNetworkID + } epInfo, err = epInfo.GetEndpointInfoByIPImpl(epInfo.IPAddresses, networkID) if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") @@ -689,6 +692,7 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { return containerID + "-" + ifName } +// cnsEndpointInfotoCNIEpInfo convert a CNS endpoint state to CNI EndpointInfo func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID string) *EndpointInfo { epInfo := &EndpointInfo{ Id: endpointID, @@ -700,31 +704,34 @@ func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID } for ifName, ipInfo := range endpointInfo.IfnameToIPMap { + // This is an special case for endpoint state that are being crated by statefull CNI + if ifName == "" { + ifName = InfraInterfaceName + } + // TODO: DelegatedNIC state will be added in a future PR if ifName != InfraInterfaceName { - // TODO: filling out the SecondaryNICs from the state for Swift 2.0 continue } - // filling out the InfraNIC from the state epInfo.IPAddresses = ipInfo.IPv4 epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) epInfo.IfName = ifName epInfo.HostIfName = ipInfo.HostVethName epInfo.HNSEndpointID = ipInfo.HnsEndpointID + epInfo.HNSNetworkID = ipInfo.HnsNetworkID + epInfo.MacAddress = net.HardwareAddr(ipInfo.MacAddress) } return epInfo } +// generateCNSIPInfoMap generates a CNS ifNametoIPInfoMap structure based on CNI endpoint func generateCNSIPInfoMap(ep *endpoint) map[string]*restserver.IPInfo { ifNametoIPInfoMap := make(map[string]*restserver.IPInfo) // key : interface name, value : IPInfo - if ep.IfName != "" { - ifNametoIPInfoMap[ep.IfName].NICType = cns.InfraNIC - ifNametoIPInfoMap[ep.IfName].HnsEndpointID = ep.HnsId - ifNametoIPInfoMap[ep.IfName].HostVethName = ep.HostIfName - } - if ep.SecondaryInterfaces != nil { - for ifName, InterfaceInfo := range ep.SecondaryInterfaces { - ifNametoIPInfoMap[ifName].NICType = InterfaceInfo.NICType - } + ifNametoIPInfoMap[ep.IfName] = &restserver.IPInfo{ + NICType: cns.InfraNIC, + HnsEndpointID: ep.HnsId, + HnsNetworkID: ep.HNSNetworkID, + HostVethName: ep.HostIfName, + MacAddress: ep.MacAddress.String(), } return ifNametoIPInfoMap }