From aea37e9df12f4e38dfdb10ba43bb961ba5cfd785 Mon Sep 17 00:00:00 2001 From: Jose Villalta Date: Thu, 21 Nov 2024 15:44:04 -0800 Subject: [PATCH] Adds branch ENI config to non-firecracker platforms --- ecs-agent/netlib/platform/common_linux.go | 18 ++++--- .../netlib/platform/common_linux_test.go | 20 +++++--- .../netlib/platform/firecracker_linux.go | 51 ++++++++++++++++++- .../netlib/platform/firecracker_linux_test.go | 37 ++++++++++++++ 4 files changed, 113 insertions(+), 13 deletions(-) diff --git a/ecs-agent/netlib/platform/common_linux.go b/ecs-agent/netlib/platform/common_linux.go index 5709d571b16..437f5011993 100644 --- a/ecs-agent/netlib/platform/common_linux.go +++ b/ecs-agent/netlib/platform/common_linux.go @@ -695,22 +695,28 @@ func (c *common) configureBranchENI(ctx context.Context, netNSPath string, eni * "NetNSPath": netNSPath, }) - var cniNetConf ecscni.PluginConfig + // Set the path for the IPAM CNI local db to track assigned IPs. + // Default path is /data but in some linux distros (i.e.Amazon BottleRocket) the root volume is read-only. + c.os.Setenv(IPAMDataPathEnv, filepath.Join(c.stateDBDir, IPAMDataFileName)) + + var cniNetConf []ecscni.PluginConfig var err error add := true // Generate CNI network configuration based on the ENI's desired state. switch eni.DesiredStatus { case status.NetworkReadyPull: - cniNetConf = createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeVlan) - case status.NetworkReady: - cniNetConf = createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeTap) + // Setup bridge to connect task network namespace to TMDS running in host's primary netns. + if eni.IsPrimary() { + cniNetConf = append(cniNetConf, createBridgePluginConfig(netNSPath)) + } + cniNetConf = append(cniNetConf, createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeVlan)) case status.NetworkDeleted: - cniNetConf = createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeTap) + cniNetConf = append(cniNetConf, createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeVlan)) add = false } - _, err = c.executeCNIPlugin(ctx, add, cniNetConf) + _, err = c.executeCNIPlugin(ctx, add, cniNetConf...) if err != nil { err = errors.Wrap(err, "failed to setup branch eni") } diff --git a/ecs-agent/netlib/platform/common_linux_test.go b/ecs-agent/netlib/platform/common_linux_test.go index 38f25169970..6da1efc6c07 100644 --- a/ecs-agent/netlib/platform/common_linux_test.go +++ b/ecs-agent/netlib/platform/common_linux_test.go @@ -342,27 +342,35 @@ func testBranchENIConfiguration(t *testing.T) { defer ctrl.Finish() ctx := context.TODO() + osWrapper := mock_oswrapper.NewMockOS(ctrl) cniClient := mock_ecscni2.NewMockCNI(ctrl) commonPlatform := &common{ - cniClient: cniClient, + os: osWrapper, + cniClient: cniClient, + stateDBDir: "dummy-db-dir", } branchENI := getTestBranchENI() - + branchENI.DesiredStatus = status.NetworkReadyPull + bridgeConfig := createBridgePluginConfig(netNSPath) cniConfig := createBranchENIConfig(netNSPath, branchENI, VPCBranchENIInterfaceTypeVlan) - cniClient.EXPECT().Add(gomock.Any(), cniConfig).Return(nil, nil).Times(1) + gomock.InOrder( + osWrapper.EXPECT().Setenv("IPAM_DB_PATH", filepath.Join(commonPlatform.stateDBDir, "eni-ipam.db")), + cniClient.EXPECT().Add(gomock.Any(), bridgeConfig).Return(nil, nil).Times(1), + cniClient.EXPECT().Add(gomock.Any(), cniConfig).Return(nil, nil).Times(1), + ) err := commonPlatform.configureInterface(ctx, netNSPath, branchENI, nil) require.NoError(t, err) + // Ready-Pull to Ready transition branchENI.DesiredStatus = status.NetworkReady - cniConfig = createBranchENIConfig(netNSPath, branchENI, VPCBranchENIInterfaceTypeTap) - cniClient.EXPECT().Add(gomock.Any(), cniConfig).Return(nil, nil).Times(1) + osWrapper.EXPECT().Setenv("IPAM_DB_PATH", filepath.Join(commonPlatform.stateDBDir, "eni-ipam.db")) err = commonPlatform.configureInterface(ctx, netNSPath, branchENI, nil) require.NoError(t, err) // Delete workflow. branchENI.DesiredStatus = status.NetworkDeleted - cniConfig = createBranchENIConfig(netNSPath, branchENI, VPCBranchENIInterfaceTypeTap) + osWrapper.EXPECT().Setenv("IPAM_DB_PATH", filepath.Join(commonPlatform.stateDBDir, "eni-ipam.db")) cniClient.EXPECT().Del(gomock.Any(), cniConfig).Return(nil).Times(1) err = commonPlatform.configureInterface(ctx, netNSPath, branchENI, nil) require.NoError(t, err) diff --git a/ecs-agent/netlib/platform/firecracker_linux.go b/ecs-agent/netlib/platform/firecracker_linux.go index 85915782d80..c08db1be78f 100644 --- a/ecs-agent/netlib/platform/firecracker_linux.go +++ b/ecs-agent/netlib/platform/firecracker_linux.go @@ -20,9 +20,12 @@ import ( netlibdata "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/data" "github.com/aws/amazon-ecs-agent/ecs-agent/acs/model/ecsacs" + "github.com/aws/amazon-ecs-agent/ecs-agent/logger" "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/appmesh" + "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/ecscni" "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/networkinterface" "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/serviceconnect" + "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/status" "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/tasknetworkconfig" "github.com/aws/aws-sdk-go/aws" @@ -59,13 +62,29 @@ func (f *firecraker) CreateDNSConfig(taskID string, netNS *tasknetworkconfig.Net return f.configureSecondaryDNSConfig(taskID, netNS) } +// ConfigureInterface is a firecracker-specific method that adds network interfaces to tasks running on +// Firecracker microVMs. It calls a FC-specific method that configures and connect Branch ENIs to a TAP interface. func (f *firecraker) ConfigureInterface( ctx context.Context, netNSPath string, iface *networkinterface.NetworkInterface, netDAO netlibdata.NetworkDataClient, ) error { - return f.common.configureInterface(ctx, netNSPath, iface, netDAO) + var err error + switch iface.InterfaceAssociationProtocol { + case networkinterface.DefaultInterfaceAssociationProtocol: + err = f.common.configureRegularENI(ctx, netNSPath, iface) + case networkinterface.VLANInterfaceAssociationProtocol: + err = f.configureBranchENI(ctx, netNSPath, iface) + case networkinterface.V2NInterfaceAssociationProtocol: + err = f.common.configureGENEVEInterface(ctx, netNSPath, iface, netDAO) + case networkinterface.VETHInterfaceAssociationProtocol: + // Do nothing. Virtual Ethernet Interfaces do not need to be configured by the Linux Kernel. + return nil + default: + err = errors.New("invalid interface association protocol " + iface.InterfaceAssociationProtocol) + } + return err } func (f *firecraker) ConfigureAppMesh(ctx context.Context, netNSPath string, cfg *appmesh.AppMesh) error { @@ -171,3 +190,33 @@ func assignInterfacesToNamespaces(taskPayload *ecsacs.Task) (map[string]string, return i2n, nil } + +// configureBranchENI configures a network interface for a branch ENI. +func (f *firecraker) configureBranchENI(ctx context.Context, netNSPath string, eni *networkinterface.NetworkInterface) error { + logger.Info("Configuring branch ENI", map[string]interface{}{ + "ENIName": eni.Name, + "NetNSPath": netNSPath, + }) + + var cniNetConf ecscni.PluginConfig + var err error + add := true + + // Generate CNI network configuration based on the ENI's desired state. + switch eni.DesiredStatus { + case status.NetworkReadyPull: + cniNetConf = createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeVlan) + case status.NetworkReady: + cniNetConf = createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeTap) + case status.NetworkDeleted: + cniNetConf = createBranchENIConfig(netNSPath, eni, VPCBranchENIInterfaceTypeTap) + add = false + } + + _, err = f.common.executeCNIPlugin(ctx, add, cniNetConf) + if err != nil { + err = errors.Wrap(err, "failed to setup branch eni") + } + + return err +} diff --git a/ecs-agent/netlib/platform/firecracker_linux_test.go b/ecs-agent/netlib/platform/firecracker_linux_test.go index 49f5df3bf26..a8c211f8e2f 100644 --- a/ecs-agent/netlib/platform/firecracker_linux_test.go +++ b/ecs-agent/netlib/platform/firecracker_linux_test.go @@ -17,13 +17,16 @@ package platform import ( + "context" "fmt" "io/fs" "os" "testing" + mock_ecscni2 "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/ecscni/mocks_ecscni" mock_ecscni "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/ecscni/mocks_nsutil" "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/networkinterface" + "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/status" "github.com/aws/amazon-ecs-agent/ecs-agent/netlib/model/tasknetworkconfig" mock_ioutilwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/ioutilwrapper/mocks" mock_oswrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/oswrapper/mocks" @@ -142,3 +145,37 @@ func TestFirecracker_CreateDNSConfig(t *testing.T) { err := fc.CreateDNSConfig(taskID, netns) require.NoError(t, err) } + +func TestFirecracker_BranchENIConfiguration(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ctx := context.TODO() + cniClient := mock_ecscni2.NewMockCNI(ctrl) + commonPlatform := common{ + cniClient: cniClient, + } + fc := &firecraker{ + common: commonPlatform, + } + + branchENI := getTestBranchENI() + + cniConfig := createBranchENIConfig(netNSPath, branchENI, VPCBranchENIInterfaceTypeVlan) + cniClient.EXPECT().Add(gomock.Any(), cniConfig).Return(nil, nil).Times(1) + err := fc.ConfigureInterface(ctx, netNSPath, branchENI, nil) + require.NoError(t, err) + + branchENI.DesiredStatus = status.NetworkReady + cniConfig = createBranchENIConfig(netNSPath, branchENI, VPCBranchENIInterfaceTypeTap) + cniClient.EXPECT().Add(gomock.Any(), cniConfig).Return(nil, nil).Times(1) + err = fc.ConfigureInterface(ctx, netNSPath, branchENI, nil) + require.NoError(t, err) + + // Delete workflow. + branchENI.DesiredStatus = status.NetworkDeleted + cniConfig = createBranchENIConfig(netNSPath, branchENI, VPCBranchENIInterfaceTypeTap) + cniClient.EXPECT().Del(gomock.Any(), cniConfig).Return(nil).Times(1) + err = fc.ConfigureInterface(ctx, netNSPath, branchENI, nil) + require.NoError(t, err) +}