feat: create a dummy NC to store secondary IPs in nodesubnet deployments with Cilium

cns/NetworkContainerContract.go

@@ -55,6 +55,7 @@ const (
 	JobObject              = "JobObject"
 	COW                    = "COW" // Container on Windows
 	BackendNICNC           = "BackendNICNC"
+	NodeSubnet             = "NodeSubnet" // Artificial Container Type for NodeSubnet
 )
 
 // Orchestrator Types
@@ -66,6 +67,7 @@ const (
 	AzureFirstParty = "AzureFirstParty"
 	KubernetesCRD   = "KubernetesCRD"
 	// TODO: Add OrchastratorType as CRD: https://msazure.visualstudio.com/One/_workitems/edit/7711872
+	KubernetesNodeSubnet = "KubernetesNodeSubnet"
 )
 
 // Encap Types
@@ -99,10 +101,13 @@ const (
 	Managed        = "Managed"
 	CRD            = "CRD"
 	MultiTenantCRD = "MultiTenantCRD"
+	AzureHost      = "AzureHost"
 )
 
-var ErrInvalidNCID = errors.New("invalid NetworkContainerID")
-var ErrInvalidIP = errors.New("invalid IP")
+var (
+	ErrInvalidNCID = errors.New("invalid NetworkContainerID")
+	ErrInvalidIP   = errors.New("invalid IP")
+)
 
 // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary.
 type CreateNetworkContainerRequest struct {

cns/nodesubnet/initialization.go

@@ -0,0 +1,36 @@
+package nodesubnet
+
+import (
+	"context"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	cnstypes "github.com/Azure/azure-container-networking/cns/types"
+	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
+)
+
+type ipamReconciler interface {
+	ReconcileIPAMStateForNodeSubnet(ncRequests []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo) cnstypes.ResponseCode
+}
+
+func ReconcileInitialCNSState(_ context.Context, ipamReconciler ipamReconciler, podInfoByIPProvider cns.PodInfoByIPProvider) (int, error) {
+	// Get previous PodInfo state from podInfoByIPProvider
+	podInfoByIP, err := podInfoByIPProvider.PodInfoByIP()
+	if err != nil {
+		return 0, errors.Wrap(err, "provider failed to provide PodInfoByIP")
+	}
+
+	logger.Printf("Reconciling initial CNS state with %d IPs", len(podInfoByIP))
+
+	// Create a network container request that holds all the IPs from PodInfoByIP
+	secondaryIPs := maps.Keys(podInfoByIP)
+	ncRequest := CreateNodeSubnetNCRequest(secondaryIPs)
+	responseCode := ipamReconciler.ReconcileIPAMStateForNodeSubnet([]*cns.CreateNetworkContainerRequest{ncRequest}, podInfoByIP)
+
+	if responseCode != cnstypes.Success {
+		return 0, errors.Errorf("failed to reconcile initial CNS state: %d", responseCode)
+	}
+
+	return len(secondaryIPs), nil
+}

cns/nodesubnet/initialization_test.go

@@ -0,0 +1,114 @@
+package nodesubnet_test
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/cnireconciler"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
+	"github.com/Azure/azure-container-networking/cns/restserver"
+	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/Azure/azure-container-networking/store"
+)
+
+func getMockStore() store.KeyValueStore {
+	mockStore := store.NewMockStore("")
+	endpointState := map[string]*restserver.EndpointInfo{
+		"12e65d89e58cb23c784e97840cf76866bfc9902089bdc8e87e9f64032e312b0b": {
+			PodName:      "coredns-54b69f46b8-ldmwr",
+			PodNamespace: "kube-system",
+			IfnameToIPMap: map[string]*restserver.IPInfo{
+				"eth0": {
+					IPv4: []net.IPNet{
+						{
+							IP:   net.IPv4(10, 10, 0, 52),
+							Mask: net.CIDRMask(24, 32),
+						},
+					},
+				},
+			},
+		},
+		"1fc5176913a3a1a7facfb823dde3b4ded404041134fef4f4a0c8bba140fc0413": {
+			PodName:      "load-test-7f7d49687d-wxc9p",
+			PodNamespace: "load-test",
+			IfnameToIPMap: map[string]*restserver.IPInfo{
+				"eth0": {
+					IPv4: []net.IPNet{
+						{
+							IP:   net.IPv4(10, 10, 0, 63),
+							Mask: net.CIDRMask(24, 32),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	err := mockStore.Write(restserver.EndpointStoreKey, endpointState)
+	if err != nil {
+		return nil
+	}
+	return mockStore
+}
+
+type MockIpamStateReconciler struct{}
+
+func (m *MockIpamStateReconciler) ReconcileIPAMStateForNodeSubnet(ncRequests []*cns.CreateNetworkContainerRequest, podInfoByIP map[string]cns.PodInfo) types.ResponseCode {
+	if len(ncRequests) == 1 && len(ncRequests[0].SecondaryIPConfigs) == len(podInfoByIP) {
+		return types.Success
+	}
+
+	return types.UnexpectedError
+}
+
+func TestNewCNSPodInfoProvider(t *testing.T) {
+	tests := []struct {
+		name       string
+		store      store.KeyValueStore
+		wantErr    bool
+		reconciler *MockIpamStateReconciler
+		exp        int
+	}{
+		{
+			name:       "happy_path",
+			store:      getMockStore(),
+			wantErr:    false,
+			reconciler: &MockIpamStateReconciler{},
+			exp:        2,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := testContext(t)
+			defer cancel()
+
+			podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(tt.store)
+			checkErr(t, err, false)
+
+			got, err := nodesubnet.ReconcileInitialCNSState(ctx, tt.reconciler, podInfoByIPProvider)
+			checkErr(t, err, tt.wantErr)
+			if got != tt.exp {
+				t.Errorf("got %d IPs reconciled, expected %d", got, tt.exp)
+			}
+		})
+	}
+}
+
+// testContext creates a context from the provided testing.T that will be
+// canceled if the test suite is terminated.
+func testContext(t *testing.T) (context.Context, context.CancelFunc) {
+	if deadline, ok := t.Deadline(); ok {
+		return context.WithDeadline(context.Background(), deadline)
+	}
+	return context.WithCancel(context.Background())
+}
+
+func init() {
+	logger.InitLogger("testlogs", 0, 0, "./")
+}

cns/nodesubnet/nodesubnet_nc.go

@@ -0,0 +1,40 @@
+package nodesubnet
+
+import (
+	"strconv"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+)
+
+const (
+	// ID for fake NC that we create to store NodeSubnet IPS
+	NodeSubnetNCID          = "55022629-3854-499b-7133-5e6887959f4ea" // md5sum of "NodeSubnetNC_IPv4"
+	NodeSubnetNCVersion     = 0
+	NodeSubnetHostVersion   = "0"
+	NodeSubnetNCStatus      = v1alpha.NCUpdateSuccess
+	NodeSubnetHostPrimaryIP = ""
+)
+
+// CreateNodeSubnetNCRequest generates a CreateNetworkContainerRequest that simply stores the static secondary IPs.
+func CreateNodeSubnetNCRequest(secondaryIPs []string) *cns.CreateNetworkContainerRequest {
+	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
+
+	for _, secondaryIP := range secondaryIPs {
+		// iterate through all secondary IP addresses add them to the request as secondary IPConfigs.
+		secondaryIPConfigs[secondaryIP] = cns.SecondaryIPConfig{
+			IPAddress: secondaryIP,
+			NCVersion: NodeSubnetNCVersion,
+		}
+	}
+
+	return &cns.CreateNetworkContainerRequest{
+		HostPrimaryIP:        NodeSubnetHostPrimaryIP,
+		SecondaryIPConfigs:   secondaryIPConfigs,
+		NetworkContainerid:   NodeSubnetNCID,
+		NetworkContainerType: cns.NodeSubnet,
+		Version:              strconv.FormatInt(NodeSubnetNCVersion, 10), //nolint:gomnd // it's decimal
+		IPConfiguration:      cns.IPConfiguration{},
+		NCStatus:             NodeSubnetNCStatus,
+	}
+}

cns/nodesubnet/nodesubnet_nc_test.go

@@ -0,0 +1,54 @@
+package nodesubnet_test
+
+import (
+	"testing"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
+	"github.com/Azure/azure-container-networking/cns/nodesubnet"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestCreateNodeSubnetNCRequest_EmptySecondaryIPs(t *testing.T) {
+	secondaryIPs := []string{}
+	expectedRequest := &cns.CreateNetworkContainerRequest{
+		HostPrimaryIP:        nodesubnet.NodeSubnetHostPrimaryIP,
+		SecondaryIPConfigs:   map[string]cns.SecondaryIPConfig{},
+		NetworkContainerid:   nodesubnet.NodeSubnetNCID,
+		NetworkContainerType: cns.NodeSubnet,
+		Version:              "0",
+		IPConfiguration:      cns.IPConfiguration{},
+		NCStatus:             v1alpha.NCUpdateSuccess,
+	}
+
+	request := nodesubnet.CreateNodeSubnetNCRequest(secondaryIPs)
+	if !cmp.Equal(request, expectedRequest) {
+		t.Errorf("Unexepected diff in NodeSubnetNCRequest: %v", cmp.Diff(request, expectedRequest))
+	}
+}
+
+func TestCreateNodeSubnetNCRequest_NonEmptySecondaryIPs(t *testing.T) {
+	secondaryIPs := []string{"10.0.0.1", "10.0.0.2"}
+	expectedRequest := &cns.CreateNetworkContainerRequest{
+		HostPrimaryIP: nodesubnet.NodeSubnetHostPrimaryIP,
+		SecondaryIPConfigs: map[string]cns.SecondaryIPConfig{
+			"10.0.0.1": {IPAddress: "10.0.0.1", NCVersion: nodesubnet.NodeSubnetNCVersion},
+			"10.0.0.2": {IPAddress: "10.0.0.2", NCVersion: nodesubnet.NodeSubnetNCVersion},
+		},
+		NetworkContainerid:   nodesubnet.NodeSubnetNCID,
+		NetworkContainerType: cns.NodeSubnet,
+		Version:              "0",
+		IPConfiguration:      cns.IPConfiguration{},
+		NCStatus:             v1alpha.NCUpdateSuccess,
+	}
+
+	request := nodesubnet.CreateNodeSubnetNCRequest(secondaryIPs)
+	if !cmp.Equal(request, expectedRequest) {
+		t.Errorf("Unexepected diff in NodeSubnetNCRequest: %v", cmp.Diff(request, expectedRequest))
+	}
+}
+
+func init() {
+	logger.InitLogger("testlogs", 0, 0, "./")
+}