Merge appmesh to dev branch

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "amazon-ecs-cni-plugins"]
 	path = amazon-ecs-cni-plugins
 	url = https://github.com/aws/amazon-ecs-cni-plugins.git
+[submodule "amazon-vpc-cni-plugins"]
+	path = amazon-vpc-cni-plugins
+	url = https://github.com/aws/amazon-vpc-cni-plugins.git

Makefile

@@ -27,7 +27,7 @@ gobuild:
 
 # create output directories
 .out-stamp:
-	mkdir -p ./out/test-artifacts ./out/cni-plugins
+	mkdir -p ./out/test-artifacts ./out/cni-plugins ./out/amazon-ecs-cni-plugins ./out/amazon-vpc-cni-plugins
 	touch .out-stamp
 
 # Basic go build
@@ -237,21 +237,37 @@ ECS_CNI_REPOSITORY_REVISION=master
 
 # Variable to override cni repository location
 ECS_CNI_REPOSITORY_SRC_DIR=$(PWD)/amazon-ecs-cni-plugins
+VPC_CNI_REPOSITORY_SRC_DIR=$(PWD)/amazon-vpc-cni-plugins
 
 get-cni-sources:
-	git submodule update --init --checkout
+	git submodule update --init --recursive --remote
 
-cni-plugins: get-cni-sources .out-stamp
-	@docker build -f scripts/dockerfiles/Dockerfile.buildCNIPlugins -t "amazon/amazon-ecs-build-cniplugins:make" .
+build-ecs-cni-plugins:
+	@docker build -f scripts/dockerfiles/Dockerfile.buildECSCNIPlugins -t "amazon/amazon-ecs-build-ecs-cni-plugins:make" .
 	docker run --rm --net=none \
 		-e GIT_SHORT_HASH=$(shell cd $(ECS_CNI_REPOSITORY_SRC_DIR) && git rev-parse --short=8 HEAD) \
 		-e GIT_PORCELAIN=$(shell cd $(ECS_CNI_REPOSITORY_SRC_DIR) && git status --porcelain 2> /dev/null | wc -l | sed 's/^ *//') \
 		-u "$(USERID)" \
-		-v "$(PWD)/out/cni-plugins:/go/src/github.com/aws/amazon-ecs-cni-plugins/bin/plugins" \
+		-v "$(PWD)/out/amazon-ecs-cni-plugins:/go/src/github.com/aws/amazon-ecs-cni-plugins/bin/plugins" \
 		-v "$(ECS_CNI_REPOSITORY_SRC_DIR):/go/src/github.com/aws/amazon-ecs-cni-plugins" \
-		"amazon/amazon-ecs-build-cniplugins:make"
+		"amazon/amazon-ecs-build-ecs-cni-plugins:make"
 	@echo "Built amazon-ecs-cni-plugins successfully."
 
+build-vpc-cni-plugins:
+	@docker build -f scripts/dockerfiles/Dockerfile.buildVPCCNIPlugins -t "amazon/amazon-ecs-build-vpc-cni-plugins:make" .
+	docker run --rm --net=none \
+		-e GIT_SHORT_HASH=$(shell cd $(VPC_CNI_REPOSITORY_SRC_DIR) && git rev-parse --short=8 HEAD) \
+		-u "$(USERID)" \
+		-v "$(PWD)/out/amazon-vpc-cni-plugins:/go/src/github.com/aws/amazon-vpc-cni-plugins/build/linux_amd64" \
+		-v "$(VPC_CNI_REPOSITORY_SRC_DIR):/go/src/github.com/aws/amazon-vpc-cni-plugins" \
+		"amazon/amazon-ecs-build-vpc-cni-plugins:make"
+	@echo "Built amazon-vpc-cni-plugins successfully."
+
+cni-plugins: get-cni-sources .out-stamp build-ecs-cni-plugins build-vpc-cni-plugins
+	mv $(PWD)/out/amazon-ecs-cni-plugins/* $(PWD)/out/cni-plugins
+	mv $(PWD)/out/amazon-vpc-cni-plugins/* $(PWD)/out/cni-plugins
+	@echo "Built all cni plugins successfully."
+
 ifeq (${BUILD_PLATFORM},aarch64)
 run-integ-tests: test-registry gremlin container-health-check-image run-sudo-tests
 	. ./scripts/shared_env && go test -tags integration -timeout=20m -v ./agent/engine/... ./agent/stats/... ./agent/app/...

agent/acs/handler/payload_handler.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/aws/amazon-ecs-agent/agent/acs/model/ecsacs"
 	"github.com/aws/amazon-ecs-agent/agent/api"
+	apiappmesh "github.com/aws/amazon-ecs-agent/agent/api/appmesh"
 	apieni "github.com/aws/amazon-ecs-agent/agent/api/eni"
 	apitask "github.com/aws/amazon-ecs-agent/agent/api/task"
 	apitaskstatus "github.com/aws/amazon-ecs-agent/agent/api/task/status"
@@ -222,6 +223,16 @@ func (payloadHandler *payloadRequestHandler) addPayloadTasks(payload *ecsacs.Pay
 
 			apiTask.SetTaskENI(eni)
 		}
+		// Add the app mesh information to task struct
+		if task.ProxyConfiguration != nil {
+			appmesh, err := apiappmesh.AppMeshFromACS(task.ProxyConfiguration)
+			if err != nil {
+				payloadHandler.handleUnrecognizedTask(task, err, payload)
+				allTasksOK = false
+				continue
+			}
+			apiTask.SetAppMesh(appmesh)
+		}
 		if task.ExecutionRoleCredentials != nil {
 			// The payload message contains execution credentials for the task.
 			// Add the credentials to the credentials manager and set the

agent/acs/handler/payload_handler_test.go

@@ -666,6 +666,78 @@ func TestPayloadHandlerAddedENIToTask(t *testing.T) {
 	assert.Equal(t, aws.StringValue(expectedENI.Ipv6Addresses[0].Address), taskeni.IPV6Addresses[0].Address)
 }
 
+func TestPayloadHandlerAddedAppMeshToTask(t *testing.T) {
+	appMeshType := "APPMESH"
+	mockEgressIgnoredIP1 := "128.0.0.1"
+	mockEgressIgnoredIP2 := "171.1.3.24"
+	mockAppPort1 := "8000"
+	mockAppPort2 := "8001"
+	mockEgressIgnoredPort1 := "13000"
+	mockEgressIgnoredPort2 := "13001"
+	mockIgnoredUID := "1337"
+	mockIgnoredGID := "2339"
+	mockProxyIngressPort := "9000"
+	mockProxyEgressPort := "9001"
+	mockAppPorts := mockAppPort1 + "," + mockAppPort2
+	mockEgressIgnoredIPs := mockEgressIgnoredIP1 + "," + mockEgressIgnoredIP2
+	mockEgressIgnoredPorts := mockEgressIgnoredPort1 + "," + mockEgressIgnoredPort2
+	mockContainerName := "testEnvoyContainer"
+	taskMetadataEndpointIP     := "169.254.170.2"
+	instanceMetadataEndpointIP := "169.254.169.254"
+	tester := setup(t)
+	defer tester.ctrl.Finish()
+
+	var addedTask *apitask.Task
+	tester.mockTaskEngine.EXPECT().AddTask(gomock.Any()).Do(
+		func(task *apitask.Task) {
+			addedTask = task
+		})
+
+	payloadMessage := &ecsacs.PayloadMessage{
+		Tasks: []*ecsacs.Task{
+			{
+				Arn: aws.String("arn"),
+				ProxyConfiguration: &ecsacs.ProxyConfiguration{
+					Type: aws.String(appMeshType),
+					Properties: map[string]*string{
+						"IgnoredUID":         aws.String(mockIgnoredUID),
+						"IgnoredGID":         aws.String(mockIgnoredGID),
+						"ProxyIngressPort":   aws.String(mockProxyIngressPort),
+						"ProxyEgressPort":    aws.String(mockProxyEgressPort),
+						"AppPorts":           aws.String(mockAppPorts),
+						"EgressIgnoredIPs":   aws.String(mockEgressIgnoredIPs),
+						"EgressIgnoredPorts": aws.String(mockEgressIgnoredPorts),
+					},
+					ContainerName: aws.String(mockContainerName),
+				},
+			},
+		},
+		MessageId: aws.String(payloadMessageId),
+	}
+
+	err := tester.payloadHandler.handleSingleMessage(payloadMessage)
+	assert.NoError(t, err)
+
+	// Validate the added task has the eni information as expected
+	appMesh := addedTask.GetAppMesh()
+	assert.NotNil(t, appMesh)
+	assert.Equal(t, mockIgnoredUID, appMesh.IgnoredUID)
+	assert.Equal(t, mockIgnoredGID, appMesh.IgnoredGID)
+	assert.Equal(t, mockProxyIngressPort, appMesh.ProxyIngressPort)
+	assert.Equal(t, mockProxyEgressPort, appMesh.ProxyEgressPort)
+	assert.Equal(t, 2, len(appMesh.AppPorts))
+	assert.Equal(t, mockAppPort1, appMesh.AppPorts[0])
+	assert.Equal(t, mockAppPort2, appMesh.AppPorts[1])
+	assert.Equal(t, 4, len(appMesh.EgressIgnoredIPs))
+	assert.Equal(t, mockEgressIgnoredIP1, appMesh.EgressIgnoredIPs[0])
+	assert.Equal(t, mockEgressIgnoredIP2, appMesh.EgressIgnoredIPs[1])
+	assert.Equal(t, taskMetadataEndpointIP, appMesh.EgressIgnoredIPs[2])
+	assert.Equal(t, instanceMetadataEndpointIP, appMesh.EgressIgnoredIPs[3])
+	assert.Equal(t, 2, len(appMesh.EgressIgnoredPorts))
+	assert.Equal(t, mockEgressIgnoredPort1, appMesh.EgressIgnoredPorts[0])
+	assert.Equal(t, mockEgressIgnoredPort2, appMesh.EgressIgnoredPorts[1])
+}
+
 func TestPayloadHandlerAddedECRAuthData(t *testing.T) {
 	tester := setup(t)
 	defer tester.ctrl.Finish()

agent/acs/model/api/api-2.json

@@ -138,8 +138,8 @@
     "AssociationType":{
       "type":"string",
       "enum":[
-        "gpu",
-        "elastic-inference"
+        "elastic-inference",
+        "gpu"
       ]
     },
     "Associations":{
@@ -432,6 +432,18 @@
       "type":"list",
       "member":{"shape":"PortMapping"}
     },
+    "ProxyConfiguration":{
+      "type":"structure",
+      "members":{
+        "type":{"shape":"ProxyConfigurationType"},
+        "containerName":{"shape":"String"},
+        "properties":{"shape":"StringMap"}
+      }
+    },
+    "ProxyConfigurationType":{
+      "type":"string",
+      "enum":["APPMESH"]
+    },
     "RegistryAuthenticationData":{
       "type":"structure",
       "members":{
@@ -531,7 +543,8 @@
         "memory":{"shape":"Integer"},
         "associations":{"shape":"Associations"},
         "pidMode":{"shape":"String"},
-        "ipcMode":{"shape":"String"}
+        "ipcMode":{"shape":"String"},
+        "proxyConfiguration":{"shape":"ProxyConfiguration"}
       }
     },
     "TaskList":{
@@ -592,4 +605,4 @@
       ]
     }
   }
-}
+}

agent/acs/model/ecsacs/api.go

@@ -885,6 +885,26 @@ func (s PortMapping) GoString() string {
 	return s.String()
 }
 
+type ProxyConfiguration struct {
+	_ struct{} `type:"structure"`
+
+	ContainerName *string `locationName:"containerName" type:"string"`
+
+	Properties map[string]*string `locationName:"properties" type:"map"`
+
+	Type *string `locationName:"type" type:"string" enum:"ProxyConfigurationType"`
+}
+
+// String returns the string representation
+func (s ProxyConfiguration) String() string {
+	return awsutil.Prettify(s)
+}
+
+// GoString returns the string representation
+func (s ProxyConfiguration) GoString() string {
+	return s.String()
+}
+
 type RefreshTaskIAMRoleCredentialsInput struct {
 	_ struct{} `type:"structure"`
 
@@ -1080,6 +1100,8 @@ type Task struct {
 
 	PidMode *string `locationName:"pidMode" type:"string"`
 
+	ProxyConfiguration *ProxyConfiguration `locationName:"proxyConfiguration" type:"structure"`
+
 	RoleCredentials *IAMRoleCredentials `locationName:"roleCredentials" type:"structure"`
 
 	TaskDefinitionAccountId *string `locationName:"taskDefinitionAccountId" type:"string"`

agent/api/appmesh/appmesh.go

@@ -0,0 +1,123 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+package appmesh
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/amazon-ecs-agent/agent/acs/model/ecsacs"
+	"github.com/aws/aws-sdk-go/aws"
+)
+
+const (
+	appMesh                    = "APPMESH"
+	splitter                   = ","
+	ignoredUID                 = "IgnoredUID"
+	ignoredGID                 = "IgnoredGID"
+	proxyIngressPort           = "ProxyIngressPort"
+	proxyEgressPort            = "ProxyEgressPort"
+	appPorts                   = "AppPorts"
+	egressIgnoredIPs           = "EgressIgnoredIPs"
+	egressIgnoredPorts         = "EgressIgnoredPorts"
+	taskMetadataEndpointIP     = "169.254.170.2"
+	instanceMetadataEndpointIP = "169.254.169.254"
+)
+
+// AppMesh contains information of app mesh config
+type AppMesh struct {
+	// IgnoredUID is egress traffic from the processes owned by the UID will be ignored
+	IgnoredUID string
+	// IgnoredGID specifies egress traffic from the processes owned by the GID will be ignored
+	IgnoredGID string
+	// ProxyIngressPort is the ingress port number that proxy is listening on
+	ProxyIngressPort string
+	// ProxyEgressPort is the egress port number that proxy is listening on
+	ProxyEgressPort string
+	// AppPorts is the port number that application is listening on
+	AppPorts []string
+	// EgressIgnoredIPs is the list of ports for which egress traffic will be ignored
+	EgressIgnoredIPs []string
+	// EgressIgnoredPorts is the list of IPs for which egress traffic will be ignored
+	EgressIgnoredPorts []string
+}
+
+// AppMeshFromACS validates proxy config if it is app mesh type and creates AppMesh object
+func AppMeshFromACS(proxyConfig *ecsacs.ProxyConfiguration) (*AppMesh, error) {
+
+	if *proxyConfig.Type != appMesh {
+		return nil, fmt.Errorf("agent does not support proxy type other than app mesh")
+	}
+
+	return &AppMesh{
+		IgnoredUID:         aws.StringValue(proxyConfig.Properties[ignoredUID]),
+		IgnoredGID:         aws.StringValue(proxyConfig.Properties[ignoredGID]),
+		ProxyIngressPort:   aws.StringValue(proxyConfig.Properties[proxyIngressPort]),
+		ProxyEgressPort:    aws.StringValue(proxyConfig.Properties[proxyEgressPort]),
+		AppPorts:           buildAppPorts(proxyConfig),
+		EgressIgnoredIPs:   buildEgressIgnoredIPs(proxyConfig),
+		EgressIgnoredPorts: buildEgressIgnoredPorts(proxyConfig),
+	}, nil
+}
+
+// buildAppPorts creates app ports from proxy config
+func buildAppPorts(proxyConfig *ecsacs.ProxyConfiguration) []string {
+	var inputAppPorts []string
+	if proxyConfig.Properties[appPorts] != nil {
+		inputAppPorts = strings.Split(*proxyConfig.Properties[appPorts], splitter)
+	}
+	return inputAppPorts
+}
+
+// buildEgressIgnoredIPs creates egress ignored IPs from proxy config
+func buildEgressIgnoredIPs(proxyConfig *ecsacs.ProxyConfiguration) []string {
+	var inputEgressIgnoredIPs []string
+	if proxyConfig.Properties[egressIgnoredIPs] != nil {
+		inputEgressIgnoredIPs = strings.Split(*proxyConfig.Properties[egressIgnoredIPs], splitter)
+	}
+	// append agent default egress ignored IPs
+	return appendDefaultEgressIgnoredIPs(inputEgressIgnoredIPs)
+}
+
+// buildEgressIgnoredPorts creates egress ignored ports from proxy config
+func buildEgressIgnoredPorts(proxyConfig *ecsacs.ProxyConfiguration) []string {
+	var inputEgressIgnoredPorts []string
+	if proxyConfig.Properties[egressIgnoredPorts] != nil {
+		inputEgressIgnoredPorts = strings.Split(*proxyConfig.Properties[egressIgnoredPorts], splitter)
+	}
+	return inputEgressIgnoredPorts
+}
+
+// appendDefaultEgressIgnoredIPs append task metadata endpoint ip and
+// instance metadata ip address to egress ignored IPs if does not exist
+func appendDefaultEgressIgnoredIPs(egressIgnoredIPs []string) []string {
+	hasTaskMetadataEndpointIP := false
+	hasInstanceMetadataEndpointIP := false
+	for _, egressIgnoredIP := range egressIgnoredIPs {
+		if strings.TrimSpace(egressIgnoredIP) == taskMetadataEndpointIP {
+			hasTaskMetadataEndpointIP = true
+		}
+		if strings.TrimSpace(egressIgnoredIP) == instanceMetadataEndpointIP {
+			hasInstanceMetadataEndpointIP = true
+		}
+	}
+
+	if !hasTaskMetadataEndpointIP {
+		egressIgnoredIPs = append(egressIgnoredIPs, taskMetadataEndpointIP)
+	}
+	if !hasInstanceMetadataEndpointIP {
+		egressIgnoredIPs = append(egressIgnoredIPs, instanceMetadataEndpointIP)
+	}
+
+	return egressIgnoredIPs
+}