Progagating Container Instance and Task Tags to Task Metadata endpoint; Bumping functional test timeout from 30m to 32m

Author: linkar-ec2

Makefile

@@ -193,7 +193,7 @@ test-in-docker:
 	docker run --net=none -v "$(PWD):/go/src/github.com/aws/amazon-ecs-agent" --privileged "amazon/amazon-ecs-agent-test:make"
 
 run-functional-tests: testnnp test-registry ecr-execution-role-image telemetry-test-image
-	. ./scripts/shared_env && go test -tags functional -timeout=30m -v ./agent/functional_tests/...
+	. ./scripts/shared_env && go test -tags functional -timeout=32m -v ./agent/functional_tests/...
 
 .PHONY: build-image-for-ecr ecr-execution-role-image-for-upload upload-images replicate-images
 

agent/api/ecsclient/client.go

@@ -523,3 +523,13 @@ func (client *APIECSClient) discoverPollEndpoint(containerInstanceArn string) (*
 	client.pollEndpoinCache.Set(containerInstanceArn, output)
 	return output, nil
 }
+
+func (client *APIECSClient) GetResourceTags(resourceArn string) ([]*ecs.Tag, error) {
+	output, err := client.standardClient.ListTagsForResource(&ecs.ListTagsForResourceInput{
+		ResourceArn: &resourceArn,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return output.Tags, nil
+}

agent/api/interface.go

@@ -40,6 +40,8 @@ type ECSClient interface {
 	// DiscoverTelemetryEndpoint takes a ContainerInstanceARN and returns the
 	// endpoint at which this Agent should contact Telemetry Service
 	DiscoverTelemetryEndpoint(containerInstanceArn string) (string, error)
+	// GetTaskTags retrieves the Tags associated with a certain Task
+	GetResourceTags(resourceArn string) ([]*ecs.Tag, error)
 }
 
 // ECSSDK is an interface that specifies the subset of the AWS Go SDK's ECS
@@ -49,6 +51,7 @@ type ECSSDK interface {
 	CreateCluster(*ecs.CreateClusterInput) (*ecs.CreateClusterOutput, error)
 	RegisterContainerInstance(*ecs.RegisterContainerInstanceInput) (*ecs.RegisterContainerInstanceOutput, error)
 	DiscoverPollEndpoint(*ecs.DiscoverPollEndpointInput) (*ecs.DiscoverPollEndpointOutput, error)
+	ListTagsForResource(*ecs.ListTagsForResourceInput) (*ecs.ListTagsForResourceOutput, error)
 }
 
 // ECSSubmitStateSDK is an interface with customized ecs client that

agent/api/mocks/api_mocks.go

@@ -559,7 +559,7 @@ func (agent *ecsAgent) startAsyncRoutines(
 	statsEngine := stats.NewDockerStatsEngine(agent.cfg, agent.dockerClient, containerChangeEventStream)
 
 	// Start serving the endpoint to fetch IAM Role credentials and other task metadata
-	go handlers.ServeTaskHTTPEndpoint(credentialsManager, state, agent.containerInstanceARN, agent.cfg, statsEngine, agent.availabilityZone)
+	go handlers.ServeTaskHTTPEndpoint(credentialsManager, state, client, agent.containerInstanceARN, agent.cfg, statsEngine, agent.availabilityZone)
 
 	// Start sending events to the backend
 	go eventhandler.HandleEngineEvents(taskEngine, client, taskHandler)

agent/app/agent.go

@@ -777,7 +777,8 @@
         "attachments":{"shape":"Attachments"},
         "clientToken":{
           "shape":"String"
-        }
+        },
+        "tags":{"shape":"Tags"}
       }
     },
     "ContainerInstanceStatus":{
@@ -1623,6 +1624,12 @@
         "stringSetValue":{"shape":"StringList"}
       }
     },
+    "ResourceNotFoundException":{
+      "type":"structure",
+      "members":{
+      },
+      "exception":true
+    },
     "Resources":{
       "type":"list",
       "member":{"shape":"Resource"}
@@ -1665,6 +1672,13 @@
         "DAEMON"
       ]
     },
+    "Scope":{
+      "type":"string",
+      "enum":[
+        "task",
+        "shared"
+      ]
+    },
     "Secret":{
       "type":"structure",
       "required":[
@@ -1940,7 +1954,8 @@
         "launchType":{"shape":"LaunchType"},
         "platformVersion":{"shape":"String"},
         "attachments":{"shape":"Attachments"},
-        "healthStatus":{"shape":"HealthStatus"}
+        "healthStatus":{"shape":"HealthStatus"},
+        "tags":{"shape":"Tags"}
       }
     },
     "TaskDefinition":{

agent/ecs_client/model/api/api-2.json

@@ -676,6 +676,16 @@
       "refs": {
       }
     },
+    "ListTagsForResourceRequest": {
+      "base": null,
+      "refs": {
+      }
+    },
+    "ListTagsForResourceResponse": {
+      "base": null,
+      "refs": {
+      }
+    },
     "ListTaskDefinitionFamiliesRequest": {
       "base": null,
       "refs": {
@@ -951,7 +961,13 @@
       "refs": {
         "CreateServiceRequest$schedulingStrategy": "<p>The scheduling strategy to use for the service. For more information, see <a href=\"http://docs.aws.amazon.com/AmazonECS/latest/developerguideecs_services.html\">Services</a>.</p> <p>There are two service scheduler strategies available:</p> <ul> <li> <p> <code>REPLICA</code>-The replica scheduling strategy places and maintains the desired number of tasks across your cluster. By default, the service scheduler spreads tasks across Availability Zones. You can use task placement strategies and constraints to customize task placement decisions.</p> </li> <li> <p> <code>DAEMON</code>-The daemon scheduling strategy deploys exactly one task on each active container instance that meets all of the task placement constraints that you specify in your cluster. When using this strategy, there is no need to specify a desired number of tasks, a task placement strategy, or use Service Auto Scaling policies.</p> <note> <p>Fargate tasks do not support the <code>DAEMON</code> scheduling strategy.</p> </note> </li> </ul>",
         "ListServicesRequest$schedulingStrategy": "<p>The scheduling strategy for services to list.</p>",
-        "Service$schedulingStrategy": "<p>The scheduling strategy to use for the service. For more information, see <a href=\"http://docs.aws.amazon.com/AmazonECS/latest/developerguideecs_services.html\">Services</a>.</p> <p>There are two service scheduler strategies available:</p> <ul> <li> <p> <code>REPLICA</code>-The replica scheduling strategy places and maintains the desired number of tasks across your cluster. By default, the service scheduler spreads tasks across Availability Zones. You can use task placement strategies and constraints to customize task placement decisions.</p> </li> <li> <p> <code>DAEMON</code>-The daemon scheduling strategy deploys exactly one task on each container instance in your cluster. When using this strategy, do not specify a desired number of tasks or any task placement strategies.</p> <note> <p>Fargate tasks do not support the <code>DAEMON</code> scheduling strategy.</p> </note> </li> </ul>"
+        "Service$schedulingStrategy": "<p>The scheduling strategy to use for the service. For more information, see <a href=\"http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html\">Services</a>.</p> <p>There are two service scheduler strategies available:</p> <ul> <li> <p> <code>REPLICA</code>-The replica scheduling strategy places and maintains the desired number of tasks across your cluster. By default, the service scheduler spreads tasks across Availability Zones. You can use task placement strategies and constraints to customize task placement decisions.</p> </li> <li> <p> <code>DAEMON</code>-The daemon scheduling strategy deploys exactly one task on each container instance in your cluster. When you are using this strategy, do not specify a desired number of tasks or any task placement strategies.</p> <note> <p>Fargate tasks do not support the <code>DAEMON</code> scheduling strategy.</p> </note> </li> </ul>"
+      }
+    },
+    "Scope": {
+      "base": null,
+      "refs": {
+        "DockerVolumeConfiguration$scope": "<p>The scope for the Docker volume that determines its lifecycle. Docker volumes that are scoped to a <code>task</code> are automatically provisioned when the task starts and destroyed when the task stops. Docker volumes that are scoped as <code>shared</code> persist after the task stops.</p>"
       }
     },
     "Secret": {
@@ -1130,6 +1146,7 @@
         "ListServicesRequest$cluster": "<p>The short name or full Amazon Resource Name (ARN) of the cluster that hosts the services to list. If you do not specify a cluster, the default cluster is assumed.</p>",
         "ListServicesRequest$nextToken": "<p>The <code>nextToken</code> value returned from a previous paginated <code>ListServices</code> request where <code>maxResults</code> was used and the results exceeded the value of that parameter. Pagination continues from the end of the previous results that returned the <code>nextToken</code> value.</p> <note> <p>This token should be treated as an opaque identifier that is only used to retrieve the next items in a list and not for other programmatic purposes.</p> </note>",
         "ListServicesResponse$nextToken": "<p>The <code>nextToken</code> value to include in a future <code>ListServices</code> request. When the results of a <code>ListServices</code> request exceed <code>maxResults</code>, this value can be used to retrieve the next page of results. This value is <code>null</code> when there are no more results to return.</p>",
+        "ListTagsForResourceRequest$resourceArn": "<p>The Amazon Resource Name (ARN) that identifies the resource for which to list the tags. Currently, the supported resources are Amazon ECS tasks, services, task definitions, clusters, and container instances.</p>",
         "ListTaskDefinitionFamiliesRequest$familyPrefix": "<p>The <code>familyPrefix</code> is a string that is used to filter the results of <code>ListTaskDefinitionFamilies</code>. If you specify a <code>familyPrefix</code>, only task definition family names that begin with the <code>familyPrefix</code> string are returned.</p>",
         "ListTaskDefinitionFamiliesRequest$nextToken": "<p>The <code>nextToken</code> value returned from a previous paginated <code>ListTaskDefinitionFamilies</code> request where <code>maxResults</code> was used and the results exceeded the value of that parameter. Pagination continues from the end of the previous results that returned the <code>nextToken</code> value.</p> <note> <p>This token should be treated as an opaque identifier that is only used to retrieve the next items in a list and not for other programmatic purposes.</p> </note>",
         "ListTaskDefinitionFamiliesResponse$nextToken": "<p>The <code>nextToken</code> value to include in a future <code>ListTaskDefinitionFamilies</code> request. When the results of a <code>ListTaskDefinitionFamilies</code> request exceed <code>maxResults</code>, this value can be used to retrieve the next page of results. This value is <code>null</code> when there are no more results to return.</p>",
@@ -1294,6 +1311,14 @@
       "refs": {
       }
     },
+    "Tags": {
+      "base": null,
+      "refs": {
+        "ContainerInstance$tags": "<p>The metadata that you apply to the container instance to help you categorize and organize them. Each tag consists of a key and an optional value, both of which you define. Tag keys can have a maximum character length of 128 characters, and tag values can have a maximum length of 256 characters.</p>",
+        "ListTagsForResourceResponse$tags": "<p>The tags for the resource.</p>",
+        "Task$tags": "<p>The metadata that you apply to the task to help you categorize and organize them. Each tag consists of a key and an optional value, both of which you define. Tag keys can have a maximum character length of 128 characters, and tag values can have a maximum length of 256 characters.</p>"
+      }
+    },
     "TargetNotFoundException": {
       "base": "<p>The specified target could not be found. You can view your available container instances with <a>ListContainerInstances</a>. Amazon ECS container instances are cluster-specific and region-specific.</p>",
       "refs": {

agent/ecs_client/model/api/docs-2.json

@@ -4939,6 +4939,12 @@ type ContainerInstance struct {
 	// in the Amazon Elastic Container Service Developer Guide.
 	Status *string `locationName:"status" type:"string"`
 
+	// The metadata that you apply to the container instance to help you categorize
+	// and organize them. Each tag consists of a key and an optional value, both
+	// of which you define. Tag keys can have a maximum character length of 128
+	// characters, and tag values can have a maximum length of 256 characters.
+	Tags []*Tag `locationName:"tags" type:"list"`
+
 	// The version counter for the container instance. Every time a container instance
 	// experiences a change that triggers a CloudWatch event, the version counter
 	// is incremented. If you are replicating your Amazon ECS container instance
@@ -5041,6 +5047,12 @@ func (s *ContainerInstance) SetStatus(v string) *ContainerInstance {
 	return s
 }
 
+// SetTags sets the Tags field's value.
+func (s *ContainerInstance) SetTags(v []*Tag) *ContainerInstance {
+	s.Tags = v
+	return s
+}
+
 // SetVersion sets the Version field's value.
 func (s *ContainerInstance) SetVersion(v int64) *ContainerInstance {
 	s.Version = &v
@@ -6678,6 +6690,10 @@ type DockerVolumeConfiguration struct {
 
 	Labels map[string]*string `locationName:"labels" type:"map"`
 
+	// The scope for the Docker volume that determines its lifecycle. Docker volumes
+	// that are scoped to a task are automatically provisioned when the task starts
+	// and destroyed when the task stops. Docker volumes that are scoped as shared
+	// persist after the task stops.
 	Scope *string `locationName:"scope" type:"string" enum:"Scope"`
 }
 
@@ -7597,6 +7613,10 @@ func (s *ListServicesOutput) SetServiceArns(v []*string) *ListServicesOutput {
 type ListTagsForResourceInput struct {
 	_ struct{} `type:"structure"`
 
+	// The Amazon Resource Name (ARN) that identifies the resource for which to
+	// list the tags. Currently, the supported resources are Amazon ECS tasks, services,
+	// task definitions, clusters, and container instances.
+	//
 	// ResourceArn is a required field
 	ResourceArn *string `locationName:"resourceArn" type:"string" required:"true"`
 }
@@ -7633,6 +7653,7 @@ func (s *ListTagsForResourceInput) SetResourceArn(v string) *ListTagsForResource
 type ListTagsForResourceOutput struct {
 	_ struct{} `type:"structure"`
 
+	// The tags for the resource.
 	Tags []*Tag `locationName:"tags" type:"list"`
 }
 
@@ -9564,7 +9585,7 @@ type Service struct {
 	RunningCount *int64 `locationName:"runningCount" type:"integer"`
 
 	// The scheduling strategy to use for the service. For more information, see
-	// Services (http://docs.aws.amazon.com/AmazonECS/latest/developerguideecs_services.html).
+	// Services (http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html).
 	//
 	// There are two service scheduler strategies available:
 	//
@@ -9574,8 +9595,8 @@ type Service struct {
 	//    and constraints to customize task placement decisions.
 	//
 	//    * DAEMON-The daemon scheduling strategy deploys exactly one task on each
-	//    container instance in your cluster. When using this strategy, do not specify
-	//    a desired number of tasks or any task placement strategies.
+	//    container instance in your cluster. When you are using this strategy,
+	//    do not specify a desired number of tasks or any task placement strategies.
 	//
 	// Fargate tasks do not support the DAEMON scheduling strategy.
 	SchedulingStrategy *string `locationName:"schedulingStrategy" type:"string" enum:"SchedulingStrategy"`
@@ -10591,6 +10612,12 @@ type Task struct {
 	// state to STOPPED).
 	StoppingAt *time.Time `locationName:"stoppingAt" type:"timestamp"`
 
+	// The metadata that you apply to the task to help you categorize and organize
+	// them. Each tag consists of a key and an optional value, both of which you
+	// define. Tag keys can have a maximum character length of 128 characters, and
+	// tag values can have a maximum length of 256 characters.
+	Tags []*Tag `locationName:"tags" type:"list"`
+
 	// The Amazon Resource Name (ARN) of the task.
 	TaskArn *string `locationName:"taskArn" type:"string"`
 
@@ -10760,6 +10787,12 @@ func (s *Task) SetStoppingAt(v time.Time) *Task {
 	return s
 }
 
+// SetTags sets the Tags field's value.
+func (s *Task) SetTags(v []*Tag) *Task {
+	s.Tags = v
+	return s
+}
+
 // SetTaskArn sets the TaskArn field's value.
 func (s *Task) SetTaskArn(v string) *Task {
 	s.TaskArn = &v

agent/ecs_client/model/ecs/api.go

@@ -73,6 +73,7 @@ type TaskEngineState interface {
 	DockerIDByV3EndpointID(v3EndpointID string) (string, bool)
 	// TaskARNByV3EndpointID returns a taskARN for a given v3 endpoint ID
 	TaskARNByV3EndpointID(v3EndpointID string) (string, bool)
+
 	json.Marshaler
 	json.Unmarshaler
 }

agent/engine/dockerstate/docker_task_engine_state.go

@@ -12,6 +12,7 @@
         "retries": 2,
         "startPeriod": 1
     },
+    "command": ["$$$CHECK_TAGS$$$"],
     "logConfiguration": {
         "logDriver": "awslogs",
         "options": {

agent/functional_tests/testdata/taskdefinitions/taskmetadata-validator-awsvpc/task-definition.json

@@ -13,7 +13,7 @@
       "startPeriod": 1
     },
     "entryPoint": ["powershell"],
-    "command": [".\\application.ps1; $env:AWS_REGION=\"$$$TEST_REGION$$$\";.\\v3-task-endpoint-validator-windows.exe; exit $LASTEXITCODE"],
+    "command": [".\\application.ps1; $env:AWS_REGION=\"$$$TEST_REGION$$$\";.\\v3-task-endpoint-validator-windows.exe $$$CHECK_TAGS$$$; exit $LASTEXITCODE"],
     "logConfiguration": {
       "logDriver": "awslogs",
       "options": {

agent/functional_tests/testdata/taskdefinitions/v3-task-endpoint-validator-windows/task-definition.json

@@ -12,6 +12,7 @@
       "retries": 2,
       "startPeriod": 1
     },
+    "command": ["$$$CHECK_TAGS$$$"],
     "logConfiguration": {
       "logDriver": "awslogs",
       "options": {