feat: Consume the NCStatus to be able to append subnet is full error …

…to Pod IP reservation failures (#2202)

* Added Network Conatiner Status to include the latest error code for a Network Container

* Updated the crd to have the Status field included into the Network Container

* Updated the names and added Status and ErrorText as two fields in NC Status

* Fixed the casing and json values for these variables

* Propagated the NC Status inside the CNS and IPAM Monitor pool states

* Fixed the lint error of missing comma

* Saved the updated NC Status into the CNS statefile

* Updated the IP assignment to check and error out subnet is Full when there are no more available IPs for CNS to assign

* Fixed a minor compilation issue

* Fixed lint failures

* Fixed lint failures

* Removed the reference from the metastate of the ipam monitor

* Added Update Success and Update Failed statuses to the NC Status to be able to clearly indicate response status inside the NNC from DNC-RC

* Updated the error to use errors pkg instead of fmt

* Updating the cns reconcillation logic to skip if there is a failure updating the NC and there are no IPs allocated for the NC

* Handled PR comments:
* Updated the code to have the NC status be part of the error directly so that it can be consumed by containerD
 and cx can perform actions on it.
 * Code update to not use dynamic slices.
 * Removed the logic which handled 0 IPs allocated to NNC in CNS reconcile

Signed-off-by: GitHub <[email protected]>

* Addressed the PR comment which helped delete a block of code to store ncIDs and also added more error codes to the NCStatus

---------

Signed-off-by: GitHub <[email protected]>

cns/NetworkContainerContract.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -95,16 +96,17 @@ type CreateNetworkContainerRequest struct {
 	AllowHostToNCCommunication bool
 	AllowNCToHostCommunication bool
 	EndpointPolicies           []NetworkContainerRequestPolicies
+	NCStatus                   v1alpha.NCStatus
 }
 
 // CreateNetworkContainerRequest implements fmt.Stringer for logging
 func (req *CreateNetworkContainerRequest) String() string {
 	return fmt.Sprintf("CreateNetworkContainerRequest"+
 		"{Version: %s, NetworkContainerType: %s, NetworkContainerid: %s, PrimaryInterfaceIdentifier: %s, "+
 		"LocalIPConfiguration: %+v, IPConfiguration: %+v, SecondaryIPConfigs: %+v, MultitenancyInfo: %+v, "+
-		"AllowHostToNCCommunication: %t, AllowNCToHostCommunication: %t}",
+		"AllowHostToNCCommunication: %t, AllowNCToHostCommunication: %t, NCStatus: %s}",
 		req.Version, req.NetworkContainerType, req.NetworkContainerid, req.PrimaryInterfaceIdentifier, req.LocalIPConfiguration,
-		req.IPConfiguration, req.SecondaryIPConfigs, req.MultiTenancyInfo, req.AllowHostToNCCommunication, req.AllowNCToHostCommunication)
+		req.IPConfiguration, req.SecondaryIPConfigs, req.MultiTenancyInfo, req.AllowHostToNCCommunication, req.AllowNCToHostCommunication, string(req.NCStatus))
 }
 
 // NetworkContainerRequestPolicies - specifies policies associated with create network request

cns/kubecontroller/nodenetworkconfig/conversion.go

@@ -66,6 +66,7 @@ func CreateNCRequestFromDynamicNC(nc v1alpha.NetworkContainer) (*cns.CreateNetwo
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
+		NCStatus: nc.Status,
 	}, nil
 }
 

cns/kubecontroller/nodenetworkconfig/conversion_linux.go

@@ -56,5 +56,6 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
+		NCStatus: nc.Status,
 	}, nil
 }

cns/kubecontroller/nodenetworkconfig/conversion_windows.go

@@ -66,5 +66,6 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
+		NCStatus: nc.Status,
 	}, nil
 }

cns/restserver/ipam.go

@@ -816,8 +816,13 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([
 
 	// Checks to make sure we found one IP for each NC
 	if len(ipsToAssign) != numOfNCs {
-		//nolint:goerr113 // return error
-		return podIPInfo, fmt.Errorf("not enough IPs available, waiting on Azure CNS to allocate more")
+		for ncID := range service.state.ContainerStatus {
+			if _, found := ipsToAssign[ncID]; found {
+				continue
+			}
+			return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s",
+				ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
+		}
 	}
 
 	failedToAssignIP := false

crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go

@@ -59,12 +59,16 @@ const (
 )
 
 // NCStatus indicates the latest NC request status
-// +kubebuilder:validation:Enum=SubnetFull
+// +kubebuilder:validation:Enum=NCUpdateSubnetFullError;NCUpdateInternalServerError;NCUpdateUnauthorizedError;NCUpdateSuccess;NCUpdateFailed
 // +kubebuilder:validation:Optional
 type NCStatus string
 
 const (
-	NCStatusSubnetFull NCStatus = "SubnetFull"
+	NCUpdateSubnetFull          NCStatus = "NCUpdateSubnetFullError"
+	NCUpdateInternalServerError NCStatus = "NCUpdateInternalServerError"
+	NCUpdateUnauthorizedError   NCStatus = "NCUpdateUnauthorizedError"
+	NCUpdateSuccess             NCStatus = "NCUpdateSuccess"
+	NCUpdateFailed              NCStatus = "NCUpdateFailed"
 )
 
 // NodeNetworkConfigStatus defines the observed state of NetworkConfig

crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml

@@ -118,7 +118,11 @@ spec:
                     status:
                       description: NCStatus indicates the latest NC request status
                       enum:
-                      - SubnetFull
+                      - NCUpdateSubnetFullError
+                      - NCUpdateInternalServerError
+                      - NCUpdateUnauthorizedError
+                      - NCUpdateSuccess
+                      - NCUpdateFailed
                       type: string
                     subcriptionID:
                       type: string