diff --git a/README.md b/README.md index 990ee6a4cec..8907339c9ec 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ additional details on each available environment variable. | `ECS_APPARMOR_CAPABLE` | `true` | Whether AppArmor is available on the container instance. | `false` | `false` | | `ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION` | 10m | Time to wait to delete containers for a stopped task. If set to less than 1 minute, the value is ignored. | 3h | 3h | | `ECS_CONTAINER_STOP_TIMEOUT` | 10m | Instance scoped configuration for time to wait for the container to exit normally before being forcibly killed. | 30s | 30s | -| `ECS_CONTAINER_START_TIMEOUT` | 10m | Instance scoped configuration for timeout before giving up on starting a container. | 3m | 8m | +| `ECS_CONTAINER_START_TIMEOUT` | 10m | Timeout before giving up on starting a container. | 3m | 8m | | `ECS_ENABLE_TASK_IAM_ROLE` | `true` | Whether to enable IAM Roles for Tasks on the Container Instance | `false` | `false` | | `ECS_ENABLE_TASK_IAM_ROLE_NETWORK_HOST` | `true` | Whether to enable IAM Roles for Tasks when launched with `host` network mode on the Container Instance | `false` | `false` | | `ECS_DISABLE_IMAGE_CLEANUP` | `true` | Whether to disable automated image cleanup for the ECS Agent. | `false` | `false` | diff --git a/agent/acs/model/api/api-2.json b/agent/acs/model/api/api-2.json index a5004f3da42..3e3faedf8cc 100644 --- a/agent/acs/model/api/api-2.json +++ b/agent/acs/model/api/api-2.json @@ -138,8 +138,8 @@ "AssociationType":{ "type":"string", "enum":[ - "gpu", - "elastic-inference" + "elastic-inference", + "gpu" ] }, "Associations":{ @@ -203,7 +203,30 @@ "healthCheckType":{"shape":"HealthCheckType"}, "registryAuthentication":{"shape":"RegistryAuthenticationData"}, "logsAuthStrategy":{"shape":"AuthStrategy"}, - "secrets":{"shape":"SecretList"} + "secrets":{"shape":"SecretList"}, + "dependsOn":{"shape":"ContainerDependencies"}, + "startTimeout":{"shape":"Integer"}, + "stopTimeout":{"shape":"Integer"} + } + }, + "ContainerCondition":{ + "type":"string", + "enum":[ + "START", + "COMPLETE", + "SUCCESS", + "HEALTHY" + ] + }, + "ContainerDependencies":{ + "type":"list", + "member":{"shape":"ContainerDependency"} + }, + "ContainerDependency":{ + "type":"structure", + "members":{ + "containerName":{"shape":"String"}, + "condition":{"shape":"ContainerCondition"} } }, "ContainerList":{ @@ -479,8 +502,7 @@ "SecretType":{ "type":"string", "enum":[ - "ENVIRONMENT_VARIABLE", - "MOUNT_POINT" + "ENVIRONMENT_VARIABLE" ] }, "SensitiveString":{ @@ -592,4 +614,4 @@ ] } } -} \ No newline at end of file +} diff --git a/agent/acs/model/ecsacs/api.go b/agent/acs/model/ecsacs/api.go index 5d1b3cfcdee..7908018ab90 100644 --- a/agent/acs/model/ecsacs/api.go +++ b/agent/acs/model/ecsacs/api.go @@ -206,6 +206,8 @@ type Container struct { Cpu *int64 `locationName:"cpu" type:"integer"` + DependsOn []*ContainerDependency `locationName:"dependsOn" type:"list"` + DockerConfig *DockerConfig `locationName:"dockerConfig" type:"structure"` EntryPoint []*string `locationName:"entryPoint" type:"list"` @@ -236,6 +238,10 @@ type Container struct { Secrets []*Secret `locationName:"secrets" type:"list"` + StartTimeout *int64 `locationName:"startTimeout" type:"integer"` + + StopTimeout *int64 `locationName:"stopTimeout" type:"integer"` + VolumesFrom []*VolumeFrom `locationName:"volumesFrom" type:"list"` } @@ -249,6 +255,24 @@ func (s Container) GoString() string { return s.String() } +type ContainerDependency struct { + _ struct{} `type:"structure"` + + Condition *string `locationName:"condition" type:"string" enum:"ContainerCondition"` + + ContainerName *string `locationName:"containerName" type:"string"` +} + +// String returns the string representation +func (s ContainerDependency) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ContainerDependency) GoString() string { + return s.String() +} + type DockerConfig struct { _ struct{} `type:"structure"` diff --git a/agent/api/container/container.go b/agent/api/container/container.go index 1b5d5982804..d561f78730a 100644 --- a/agent/api/container/container.go +++ b/agent/api/container/container.go @@ -93,6 +93,8 @@ type HealthStatus struct { type Container struct { // Name is the name of the container specified in the task definition Name string + // DependsOn is the field which specifies the ordering for container startup and shutdown. + DependsOn []DependsOn `json:"dependsOn,omitempty"` // V3EndpointID is a container identifier used to construct v3 metadata endpoint; it's unique among // all the containers managed by the agent V3EndpointID string @@ -130,7 +132,7 @@ type Container struct { DockerConfig DockerConfig `json:"dockerConfig"` // RegistryAuthentication is the auth data used to pull image RegistryAuthentication *RegistryAuthenticationData `json:"registryAuthentication"` - // HealthCheckType is the mechnism to use for the container health check + // HealthCheckType is the mechanism to use for the container health check // currently it only supports 'DOCKER' HealthCheckType string `json:"healthCheckType,omitempty"` // Health contains the health check information of container health check @@ -138,6 +140,13 @@ type Container struct { // LogsAuthStrategy specifies how the logs driver for the container will be // authenticated LogsAuthStrategy string + // StartTimeout specifies the time value after which if a container has a dependency + // on another container and the dependency conditions are 'SUCCESS', 'COMPLETE', 'HEALTHY', + // then that dependency will not be resolved. + StartTimeout uint + // StopTimeout specifies the time value to be passed as StopContainer api call + StopTimeout uint + // lock is used for fields that are accessed and updated concurrently lock sync.RWMutex @@ -236,6 +245,11 @@ type Container struct { labels map[string]string } +type DependsOn struct { + ContainerName string `json:"containerName"` + Condition string `json:"condition"` +} + // DockerContainer is a mapping between containers-as-docker-knows-them and // containers-as-we-know-them. // This is primarily used in DockerState, but lives here such that tasks and @@ -860,3 +874,17 @@ func (c *Container) HasSecretAsEnv() bool { } return false } + +func (c *Container) GetStartTimeout() time.Duration { + c.lock.Lock() + defer c.lock.Unlock() + + return time.Duration(c.StartTimeout) * time.Second +} + +func (c *Container) GetStopTimeout() time.Duration { + c.lock.Lock() + defer c.lock.Unlock() + + return time.Duration(c.StopTimeout) * time.Second +} diff --git a/agent/api/container/container_test.go b/agent/api/container/container_test.go index 91296735a9e..d588e109ae5 100644 --- a/agent/api/container/container_test.go +++ b/agent/api/container/container_test.go @@ -179,7 +179,7 @@ func TestSetupExecutionRoleFlag(t *testing.T) { } } -func TestSetHealtStatus(t *testing.T) { +func TestSetHealthStatus(t *testing.T) { container := Container{} // set the container status to be healthy @@ -455,3 +455,16 @@ func TestHasSecretAsEnv(t *testing.T) { } } + +func TestPerContainerTimeouts(t *testing.T) { + timeout := uint(10) + expectedTimeout := time.Duration(timeout) * time.Second + + container := Container{ + StartTimeout: timeout, + StopTimeout: timeout, + } + + assert.Equal(t, container.GetStartTimeout(), expectedTimeout) + assert.Equal(t, container.GetStopTimeout(), expectedTimeout) +} diff --git a/agent/api/task/task.go b/agent/api/task/task.go index ff8120ea248..333de6e3e85 100644 --- a/agent/api/task/task.go +++ b/agent/api/task/task.go @@ -72,6 +72,9 @@ const ( NvidiaVisibleDevicesEnvVar = "NVIDIA_VISIBLE_DEVICES" GPUAssociationType = "gpu" + ContainerOrderingCreateCondition = "CREATE" + ContainerOrderingStartCondition = "START" + arnResourceSections = 2 arnResourceDelimiter = "/" // networkModeNone specifies the string used to define the `none` docker networking mode @@ -250,6 +253,18 @@ func (task *Task) PostUnmarshalTask(cfg *config.Config, } } + err := task.initializeContainerOrderingForVolumes() + if err != nil { + seelog.Errorf("Task [%s]: could not initialize volumes dependency for container: %v", task.Arn, err) + return apierrors.NewResourceInitError(task.Arn, err) + } + + err = task.initializeContainerOrderingForLinks() + if err != nil { + seelog.Errorf("Task [%s]: could not initialize links dependency for container: %v", task.Arn, err) + return apierrors.NewResourceInitError(task.Arn, err) + } + if task.requiresASMDockerAuthData() { task.initializeASMAuthResource(credentialsManager, resourceFields) } @@ -262,7 +277,7 @@ func (task *Task) PostUnmarshalTask(cfg *config.Config, task.initializeASMSecretResource(credentialsManager, resourceFields) } - err := task.initializeDockerLocalVolumes(dockerClient, ctx) + err = task.initializeDockerLocalVolumes(dockerClient, ctx) if err != nil { return apierrors.NewResourceInitError(task.Arn, err) } @@ -1250,6 +1265,41 @@ func (task *Task) shouldOverrideIPCMode(container *apicontainer.Container, docke } } +func (task *Task) initializeContainerOrderingForVolumes() error { + for _, container := range task.Containers { + if len(container.VolumesFrom) > 0 { + for _, volume := range container.VolumesFrom { + if _, ok := task.ContainerByName(volume.SourceContainer); !ok { + return fmt.Errorf("could not find container with name %s", volume.SourceContainer) + } + dependOn := apicontainer.DependsOn{ContainerName: volume.SourceContainer, Condition: ContainerOrderingCreateCondition} + container.DependsOn = append(container.DependsOn, dependOn) + } + } + } + return nil +} + +func (task *Task) initializeContainerOrderingForLinks() error { + for _, container := range task.Containers { + if len(container.Links) > 0 { + for _, link := range container.Links { + linkParts := strings.Split(link, ":") + if len(linkParts) > 2 { + return fmt.Errorf("Invalid link format") + } + linkName := linkParts[0] + if _, ok := task.ContainerByName(linkName); !ok { + return fmt.Errorf("could not find container with name %s", linkName) + } + dependOn := apicontainer.DependsOn{ContainerName: linkName, Condition: ContainerOrderingStartCondition} + container.DependsOn = append(container.DependsOn, dependOn) + } + } + } + return nil +} + func (task *Task) dockerLinks(container *apicontainer.Container, dockerContainerMap map[string]*apicontainer.DockerContainer) ([]string, error) { dockerLinkArr := make([]string, len(container.Links)) for i, link := range container.Links { diff --git a/agent/api/task/task_test.go b/agent/api/task/task_test.go index c2d346e4461..47737336aa9 100644 --- a/agent/api/task/task_test.go +++ b/agent/api/task/task_test.go @@ -2845,6 +2845,122 @@ func TestTaskGPUDisabled(t *testing.T) { }, }, } - assert.False(t, testTask.isGPUEnabled()) } + +func TestInitializeContainerOrderingWithLinksAndVolumesFrom(t *testing.T) { + containerWithOnlyVolume := &apicontainer.Container{ + Name: "myName", + Image: "image:tag", + VolumesFrom: []apicontainer.VolumeFrom{{SourceContainer: "myName1"}}, + } + + containerWithOnlyLink := &apicontainer.Container{ + Name: "myName1", + Image: "image:tag", + Links: []string{"myName"}, + } + + containerWithBothVolumeAndLink := &apicontainer.Container{ + Name: "myName2", + Image: "image:tag", + VolumesFrom: []apicontainer.VolumeFrom{{SourceContainer: "myName"}}, + Links: []string{"myName1"}, + } + + containerWithNoVolumeOrLink := &apicontainer.Container{ + Name: "myName3", + Image: "image:tag", + } + + task := &Task{ + Arn: "test", + ResourcesMapUnsafe: make(map[string][]taskresource.TaskResource), + Containers: []*apicontainer.Container{containerWithOnlyVolume, containerWithOnlyLink, + containerWithBothVolumeAndLink, containerWithNoVolumeOrLink}, + } + + err := task.initializeContainerOrderingForVolumes() + assert.NoError(t, err) + err = task.initializeContainerOrderingForLinks() + assert.NoError(t, err) + + containerResultWithVolume := task.Containers[0] + assert.Equal(t, "myName1", containerResultWithVolume.DependsOn[0].ContainerName) + assert.Equal(t, ContainerOrderingCreateCondition, containerResultWithVolume.DependsOn[0].Condition) + + containerResultWithLink := task.Containers[1] + assert.Equal(t, "myName", containerResultWithLink.DependsOn[0].ContainerName) + assert.Equal(t, ContainerOrderingStartCondition, containerResultWithLink.DependsOn[0].Condition) + + containerResultWithBothVolumeAndLink := task.Containers[2] + assert.Equal(t, "myName", containerResultWithBothVolumeAndLink.DependsOn[0].ContainerName) + assert.Equal(t, ContainerOrderingCreateCondition, containerResultWithBothVolumeAndLink.DependsOn[0].Condition) + assert.Equal(t, "myName1", containerResultWithBothVolumeAndLink.DependsOn[1].ContainerName) + assert.Equal(t, ContainerOrderingStartCondition, containerResultWithBothVolumeAndLink.DependsOn[1].Condition) + + containerResultWithNoVolumeOrLink := task.Containers[3] + assert.Equal(t, 0, len(containerResultWithNoVolumeOrLink.DependsOn)) +} + +func TestInitializeContainerOrderingWithError(t *testing.T) { + containerWithVolumeError := &apicontainer.Container{ + Name: "myName", + Image: "image:tag", + VolumesFrom: []apicontainer.VolumeFrom{{SourceContainer: "dummyContainer"}}, + } + + containerWithLinkError1 := &apicontainer.Container{ + Name: "myName1", + Image: "image:tag", + Links: []string{"dummyContainer"}, + } + + containerWithLinkError2 := &apicontainer.Container{ + Name: "myName2", + Image: "image:tag", + Links: []string{"myName:link1:link2"}, + } + + task1 := &Task{ + Arn: "test", + ResourcesMapUnsafe: make(map[string][]taskresource.TaskResource), + Containers: []*apicontainer.Container{containerWithVolumeError, containerWithLinkError1}, + } + + task2 := &Task{ + Arn: "test", + ResourcesMapUnsafe: make(map[string][]taskresource.TaskResource), + Containers: []*apicontainer.Container{containerWithVolumeError, containerWithLinkError2}, + } + + errVolume1 := task1.initializeContainerOrderingForVolumes() + assert.Error(t, errVolume1) + errLink1 := task1.initializeContainerOrderingForLinks() + assert.Error(t, errLink1) + + errVolume2 := task2.initializeContainerOrderingForVolumes() + assert.Error(t, errVolume2) + errLink2 := task2.initializeContainerOrderingForLinks() + assert.Error(t, errLink2) +} + +func TestTaskFromACSPerContainerTimeouts(t *testing.T) { + modelTimeout := int64(10) + expectedTimeout := uint(modelTimeout) + + taskFromACS := ecsacs.Task{ + Containers: []*ecsacs.Container{ + { + StartTimeout: aws.Int64(modelTimeout), + StopTimeout: aws.Int64(modelTimeout), + }, + }, + } + seqNum := int64(42) + task, err := TaskFromACS(&taskFromACS, &ecsacs.PayloadMessage{SeqNum: &seqNum}) + assert.Nil(t, err, "Should be able to handle acs task") + + assert.Equal(t, task.Containers[0].StartTimeout, expectedTimeout) + assert.Equal(t, task.Containers[0].StopTimeout, expectedTimeout) +} diff --git a/agent/app/agent_capability.go b/agent/app/agent_capability.go index 9fcd3324d0b..da3ca78347c 100644 --- a/agent/app/agent_capability.go +++ b/agent/app/agent_capability.go @@ -42,6 +42,7 @@ const ( capabiltyPIDAndIPCNamespaceSharing = "pid-ipc-namespace-sharing" capabilityNvidiaDriverVersionInfix = "nvidia-driver-version." capabilityECREndpoint = "ecr-endpoint" + capabilityContainerOrdering = "container-ordering" taskEIAAttributeSuffix = "task-eia" ) @@ -141,6 +142,9 @@ func (agent *ecsAgent) capabilities() ([]*ecs.Attribute, error) { // support elastic inference in agent capabilities = appendNameOnlyAttribute(capabilities, attributePrefix+taskEIAAttributeSuffix) + // support container ordering in agent + capabilities = appendNameOnlyAttribute(capabilities, attributePrefix+capabilityContainerOrdering) + return capabilities, nil } diff --git a/agent/app/agent_capability_test.go b/agent/app/agent_capability_test.go index dc5ce1d1b8e..b4baee1ab18 100644 --- a/agent/app/agent_capability_test.go +++ b/agent/app/agent_capability_test.go @@ -126,6 +126,9 @@ func TestCapabilities(t *testing.T) { { Name: aws.String(attributePrefix + taskEIAAttributeSuffix), }, + { + Name: aws.String(attributePrefix + capabilityContainerOrdering), + }, }...) ctx, cancel := context.WithCancel(context.TODO()) diff --git a/agent/dockerclient/dockerapi/docker_client.go b/agent/dockerclient/dockerapi/docker_client.go index 2eb0a4c17ed..7ee1eede261 100644 --- a/agent/dockerclient/dockerapi/docker_client.go +++ b/agent/dockerclient/dockerapi/docker_client.go @@ -91,6 +91,8 @@ const ( pullRetryJitterMultiplier = 0.2 ) +var ctxTimeoutStopContainer = dockerclient.StopContainerTimeout + // DockerClient interface to make testing it easier type DockerClient interface { // SupportedVersions returns a slice of the supported docker versions (or at least supposedly supported). @@ -654,13 +656,16 @@ func (dg *dockerGoClient) inspectContainer(ctx context.Context, dockerID string) } func (dg *dockerGoClient) StopContainer(ctx context.Context, dockerID string, timeout time.Duration) DockerContainerMetadata { - ctx, cancel := context.WithTimeout(ctx, timeout) + // ctxTimeout is sum of timeout(applied to the StopContainer api call) and a fixed constant dockerclient.StopContainerTimeout + // the context's timeout should be greater than the sigkill timout for the StopContainer call + ctxTimeout := timeout + ctxTimeoutStopContainer + ctx, cancel := context.WithTimeout(ctx, ctxTimeout) defer cancel() defer metrics.MetricsEngineGlobal.RecordDockerMetric("STOP_CONTAINER")() // Buffered channel so in the case of timeout it takes one write, never gets // read, and can still be GC'd response := make(chan DockerContainerMetadata, 1) - go func() { response <- dg.stopContainer(ctx, dockerID) }() + go func() { response <- dg.stopContainer(ctx, dockerID, timeout) }() select { case resp := <-response: return resp @@ -669,19 +674,18 @@ func (dg *dockerGoClient) StopContainer(ctx context.Context, dockerID string, ti // send back the DockerTimeoutError err := ctx.Err() if err == context.DeadlineExceeded { - return DockerContainerMetadata{Error: &DockerTimeoutError{timeout, "stopped"}} + return DockerContainerMetadata{Error: &DockerTimeoutError{ctxTimeout, "stopped"}} } return DockerContainerMetadata{Error: CannotStopContainerError{err}} } } -func (dg *dockerGoClient) stopContainer(ctx context.Context, dockerID string) DockerContainerMetadata { +func (dg *dockerGoClient) stopContainer(ctx context.Context, dockerID string, timeout time.Duration) DockerContainerMetadata { client, err := dg.sdkDockerClient() if err != nil { return DockerContainerMetadata{Error: CannotGetDockerClientError{version: dg.version, err: err}} } - - err = client.ContainerStop(ctx, dockerID, &dg.config.DockerStopTimeout) + err = client.ContainerStop(ctx, dockerID, &timeout) metadata := dg.containerMetadata(ctx, dockerID) if err != nil { seelog.Infof("DockerGoClient: error stopping container %s: %v", dockerID, err) diff --git a/agent/dockerclient/dockerapi/docker_client_test.go b/agent/dockerclient/dockerapi/docker_client_test.go index 2a487aa8b44..70db0d83bf7 100644 --- a/agent/dockerclient/dockerapi/docker_client_test.go +++ b/agent/dockerclient/dockerapi/docker_client_test.go @@ -468,6 +468,7 @@ func TestStopContainerTimeout(t *testing.T) { cfg.DockerStopTimeout = xContainerShortTimeout mockDockerSDK, client, _, _, _, done := dockerClientSetupWithConfig(t, cfg) defer done() + ctxTimeoutStopContainer = xContainerShortTimeout wait := &sync.WaitGroup{} wait.Add(1) diff --git a/agent/engine/common_integ_test.go b/agent/engine/common_integ_test.go index 174c4e3b8cb..262c57de662 100644 --- a/agent/engine/common_integ_test.go +++ b/agent/engine/common_integ_test.go @@ -41,6 +41,7 @@ import ( func defaultTestConfigIntegTest() *config.Config { cfg, _ := config.NewConfig(ec2.NewBlackholeEC2MetadataClient()) cfg.TaskCPUMemLimit = config.ExplicitlyDisabled + cfg.ImagePullBehavior = config.ImagePullPreferCachedBehavior return cfg } @@ -107,8 +108,8 @@ func createTestContainerWithImageAndName(image string, name string) *apicontaine Command: []string{}, Essential: true, DesiredStatusUnsafe: apicontainerstatus.ContainerRunning, - CPU: 100, - Memory: 80, + CPU: 1024, + Memory: 128, } } diff --git a/agent/engine/common_unix_integ_test.go b/agent/engine/common_unix_integ_test.go index b6d7ec8ff2d..dbf76d2e809 100644 --- a/agent/engine/common_unix_integ_test.go +++ b/agent/engine/common_unix_integ_test.go @@ -32,8 +32,6 @@ func createTestContainer() *apicontainer.Container { } func isDockerRunning() bool { - if _, err := os.Stat("/var/run/docker.sock"); err != nil { - return false - } - return true + _, err := os.Stat("/var/run/docker.sock") + return err == nil } diff --git a/agent/engine/dependencygraph/graph.go b/agent/engine/dependencygraph/graph.go index 05bb1c20b71..f9853e8832d 100644 --- a/agent/engine/dependencygraph/graph.go +++ b/agent/engine/dependencygraph/graph.go @@ -14,8 +14,7 @@ package dependencygraph import ( - "strings" - + "fmt" apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" apicontainerstatus "github.com/aws/amazon-ecs-agent/agent/api/container/status" apitask "github.com/aws/amazon-ecs-agent/agent/api/task" @@ -23,6 +22,24 @@ import ( "github.com/aws/amazon-ecs-agent/agent/taskresource" log "github.com/cihub/seelog" "github.com/pkg/errors" + "strings" + "time" +) + +const ( + // CreateCondition ensures that a container progresses to next state only when dependency container has started + createCondition = "CREATE" + // StartCondition ensures that a container progresses to next state only when dependency container is running + startCondition = "START" + // SuccessCondition ensures that a container progresses to next state only when + // dependency container has successfully completed with exit code 0 + successCondition = "SUCCESS" + // CompleteCondition ensures that a container progresses to next state only when dependency container has completed + completeCondition = "COMPLETE" + // HealthyCondition ensures that a container progresses to next state only when dependency container is healthy + healthyCondition = "HEALTHY" + // 0 is the standard exit code for success. + successExitCode = 0 ) var ( @@ -33,8 +50,6 @@ var ( DependentContainerNotResolvedErr = errors.New("dependency graph: dependent container not in expected state") // ContainerPastDesiredStatusErr is the error where the container status is bigger than desired status ContainerPastDesiredStatusErr = errors.New("container transition: container status is equal or greater than desired status") - volumeUnresolvedErr = errors.New("dependency graph: container volume dependency not resolved") - linkUnresolvedErr = errors.New("dependency graph: container links dependency not resolved") // ErrContainerDependencyNotResolved is when the container's dependencies // on other containers are not resolved ErrContainerDependencyNotResolved = errors.New("dependency graph: dependency on containers not resolved") @@ -43,16 +58,9 @@ var ( ErrResourceDependencyNotResolved = errors.New("dependency graph: dependency on resources not resolved") ) -// Because a container may depend on another container being created -// (volumes-from) or running (links) it makes sense to abstract it out -// to each container having dependencies on another container being in any -// particular state set. For now, these are resolved here and support only -// volume/link (created/run) - // ValidDependencies takes a task and verifies that it is possible to allow all // containers within it to reach the desired status by proceeding in some -// order. ValidDependencies is called during DockerTaskEngine.AddTask to -// verify that a startup order can exist. +// order. func ValidDependencies(task *apitask.Task) bool { unresolved := make([]*apicontainer.Container, len(task.Containers)) resolved := make([]*apicontainer.Container, 0, len(task.Containers)) @@ -90,14 +98,11 @@ func dependenciesCanBeResolved(target *apicontainer.Container, by []*apicontaine for _, cont := range by { nameMap[cont.Name] = cont } - neededVolumeContainers := make([]string, len(target.VolumesFrom)) - for i, volume := range target.VolumesFrom { - neededVolumeContainers[i] = volume.SourceContainer - } - return verifyStatusResolvable(target, nameMap, neededVolumeContainers, volumeCanResolve) && - verifyStatusResolvable(target, nameMap, linksToContainerNames(target.Links), linkCanResolve) && - verifyStatusResolvable(target, nameMap, target.SteadyStateDependencies, onSteadyStateCanResolve) + if _, err := verifyContainerOrderingStatusResolvable(target, nameMap, containerOrderingDependenciesCanResolve); err != nil { + return false + } + return verifyStatusResolvable(target, nameMap, target.SteadyStateDependencies, onSteadyStateCanResolve) } // DependenciesAreResolved validates that the `target` container can be @@ -111,9 +116,9 @@ func DependenciesAreResolved(target *apicontainer.Container, by []*apicontainer.Container, id string, manager credentials.Manager, - resources []taskresource.TaskResource) error { + resources []taskresource.TaskResource) (*apicontainer.DependsOn, error) { if !executionCredentialsResolved(target, id, manager) { - return CredentialsNotResolvedErr + return nil, CredentialsNotResolvedErr } nameMap := make(map[string]*apicontainer.Container) @@ -130,22 +135,26 @@ func DependenciesAreResolved(target *apicontainer.Container, resourcesMap[resource.GetName()] = resource } - if !verifyStatusResolvable(target, nameMap, neededVolumeContainers, volumeIsResolved) { - return volumeUnresolvedErr - } - - if !verifyStatusResolvable(target, nameMap, linksToContainerNames(target.Links), linkIsResolved) { - return linkUnresolvedErr + if blocked, err := verifyContainerOrderingStatusResolvable(target, nameMap, containerOrderingDependenciesIsResolved); err != nil { + return blocked, err } if !verifyStatusResolvable(target, nameMap, target.SteadyStateDependencies, onSteadyStateIsResolved) { - return DependentContainerNotResolvedErr + return nil, DependentContainerNotResolvedErr } if err := verifyTransitionDependenciesResolved(target, nameMap, resourcesMap); err != nil { - return err + return nil, err } - return nil + // If the target is desired terminal and isn't stopped, we should validate that it doesn't have any containers + // that are dependent on it that need to shut down first. + if target.DesiredTerminal() && !target.KnownTerminal() { + if err := verifyShutdownOrder(target, nameMap); err != nil { + return nil, err + } + } + + return nil, nil } func linksToContainerNames(links []string) []string { @@ -193,6 +202,51 @@ func verifyStatusResolvable(target *apicontainer.Container, existingContainers m return true } +// verifyContainerOrderingStatusResolvable validates that `target` can be resolved given that +// the dependsOn containers are resolved and there are `existingContainers` +// (map from name to container). The `resolves` function passed should return true if the named container is resolved. + +func verifyContainerOrderingStatusResolvable(target *apicontainer.Container, existingContainers map[string]*apicontainer.Container, + resolves func(*apicontainer.Container, *apicontainer.Container, string) bool) (*apicontainer.DependsOn, error) { + + targetGoal := target.GetDesiredStatus() + targetKnown := target.GetKnownStatus() + if targetGoal != target.GetSteadyStateStatus() && targetGoal != apicontainerstatus.ContainerCreated { + // A container can always stop, die, or reach whatever other state it + // wants regardless of what dependencies it has + return nil, nil + } + + for _, dependency := range target.DependsOn { + dependencyContainer, ok := existingContainers[dependency.ContainerName] + if !ok { + return nil, fmt.Errorf("dependency graph: container ordering dependency [%v] for target [%v] does not exist.", dependencyContainer, target) + } + + // We want to check whether the dependency container has timed out only if target has not been created yet. + // If the target is already created, then everything is normal and dependency can be and is resolved. + // However, if dependency container has already stopped, then it cannot time out. + if targetKnown < apicontainerstatus.ContainerCreated && dependencyContainer.GetKnownStatus() != apicontainerstatus.ContainerStopped { + if hasDependencyTimedOut(dependencyContainer, dependency.Condition) { + return nil, fmt.Errorf("dependency graph: container ordering dependency [%v] for target [%v] has timed out.", dependencyContainer, target) + } + } + + // We want to fail fast if the dependency container did not exit successfully' because target container + // can then never progress to its desired state when the dependency condition is 'SUCCESS' + if dependency.Condition == successCondition && dependencyContainer.GetKnownExitCode() != nil { + if !hasDependencyStoppedSuccessfully(dependencyContainer, dependency.Condition) { + return nil, fmt.Errorf("dependency graph: failed to resolve container ordering dependency [%v] for target [%v] as dependency did not exit successfully.", dependencyContainer, target) + } + } + + if !resolves(target, dependencyContainer, dependency.Condition) { + return &dependency, fmt.Errorf("dependency graph: failed to resolve the container ordering dependency [%v] for target [%v]", dependencyContainer, target) + } + } + return nil, nil +} + func verifyTransitionDependenciesResolved(target *apicontainer.Container, existingContainers map[string]*apicontainer.Container, existingResources map[string]taskresource.TaskResource) error { @@ -236,74 +290,153 @@ func verifyResourceDependenciesResolved(target *apicontainer.Container, existing return true } -func linkCanResolve(target *apicontainer.Container, link *apicontainer.Container) bool { +func containerOrderingDependenciesCanResolve(target *apicontainer.Container, + dependsOnContainer *apicontainer.Container, + dependsOnStatus string) bool { + targetDesiredStatus := target.GetDesiredStatus() - linkDesiredStatus := link.GetDesiredStatus() - if targetDesiredStatus == apicontainerstatus.ContainerCreated { - // The 'target' container desires to be moved to 'Created' state. - // Allow this only if the desired status of the linked container is - // 'Created' or if the linked container is in 'steady state' - return linkDesiredStatus == apicontainerstatus.ContainerCreated || linkDesiredStatus == link.GetSteadyStateStatus() - } else if targetDesiredStatus == target.GetSteadyStateStatus() { - // The 'target' container desires to be moved to its 'steady' state. - // Allow this only if the linked container is in 'steady state' as well - return linkDesiredStatus == link.GetSteadyStateStatus() + dependsOnContainerDesiredStatus := dependsOnContainer.GetDesiredStatus() + + switch dependsOnStatus { + case createCondition: + return verifyContainerOrderingStatus(dependsOnContainer) + + case startCondition: + if targetDesiredStatus == apicontainerstatus.ContainerCreated { + // The 'target' container desires to be moved to 'Created' state. + // Allow this only if the desired status of the dependency container is + // 'Created' or if the linked container is in 'steady state' + return dependsOnContainerDesiredStatus == apicontainerstatus.ContainerCreated || + dependsOnContainerDesiredStatus == dependsOnContainer.GetSteadyStateStatus() + + } else if targetDesiredStatus == target.GetSteadyStateStatus() { + // The 'target' container desires to be moved to its 'steady' state. + // Allow this only if the dependency container also desires to be in 'steady state' + return dependsOnContainerDesiredStatus == dependsOnContainer.GetSteadyStateStatus() + } + return false + + case successCondition: + var dependencyStoppedSuccessfully bool + + if dependsOnContainer.GetKnownExitCode() != nil { + dependencyStoppedSuccessfully = dependsOnContainer.GetKnownStatus() == apicontainerstatus.ContainerStopped && + *dependsOnContainer.GetKnownExitCode() == successExitCode + } + return verifyContainerOrderingStatus(dependsOnContainer) || dependencyStoppedSuccessfully + + case completeCondition: + return verifyContainerOrderingStatus(dependsOnContainer) + + case healthyCondition: + return verifyContainerOrderingStatus(dependsOnContainer) && dependsOnContainer.HealthStatusShouldBeReported() + + default: + return false } - log.Errorf("Failed to resolve the desired status of the link [%v] for the target [%v]", link, target) - return false } -func linkIsResolved(target *apicontainer.Container, link *apicontainer.Container) bool { +func containerOrderingDependenciesIsResolved(target *apicontainer.Container, + dependsOnContainer *apicontainer.Container, + dependsOnStatus string) bool { + targetDesiredStatus := target.GetDesiredStatus() - if targetDesiredStatus == apicontainerstatus.ContainerCreated { - // The 'target' container desires to be moved to 'Created' state. - // Allow this only if the known status of the linked container is - // 'Created' or if the linked container is in 'steady state' - linkKnownStatus := link.GetKnownStatus() - return linkKnownStatus == apicontainerstatus.ContainerCreated || link.IsKnownSteadyState() - } else if targetDesiredStatus == target.GetSteadyStateStatus() { - // The 'target' container desires to be moved to its 'steady' state. - // Allow this only if the linked container is in 'steady state' as well - return link.IsKnownSteadyState() + dependsOnContainerKnownStatus := dependsOnContainer.GetKnownStatus() + + switch dependsOnStatus { + case createCondition: + // The 'target' container desires to be moved to 'Created' or the 'steady' state. + // Allow this only if the known status of the dependency container state is already started + // i.e it's state is any of 'Created', 'steady state' or 'Stopped' + return dependsOnContainerKnownStatus >= apicontainerstatus.ContainerCreated + + case startCondition: + if targetDesiredStatus == apicontainerstatus.ContainerCreated { + // The 'target' container desires to be moved to 'Created' state. + // Allow this only if the known status of the linked container is + // 'Created' or if the dependency container is in 'steady state' + return dependsOnContainerKnownStatus == apicontainerstatus.ContainerCreated || dependsOnContainer.IsKnownSteadyState() + } else if targetDesiredStatus == target.GetSteadyStateStatus() { + // The 'target' container desires to be moved to its 'steady' state. + // Allow this only if the dependency container is in 'steady state' as well + return dependsOnContainer.IsKnownSteadyState() + } + return false + + case successCondition: + // The 'target' container desires to be moved to 'Created' or the 'steady' state. + // Allow this only if the known status of the dependency container state is stopped with an exit code of 0 + if dependsOnContainer.GetKnownExitCode() != nil { + return dependsOnContainerKnownStatus == apicontainerstatus.ContainerStopped && + *dependsOnContainer.GetKnownExitCode() == successExitCode + } + return false + + case completeCondition: + // The 'target' container desires to be moved to 'Created' or the 'steady' state. + // Allow this only if the known status of the dependency container state is stopped with any exit code + return dependsOnContainerKnownStatus == apicontainerstatus.ContainerStopped && dependsOnContainer.GetKnownExitCode() != nil + + case healthyCondition: + return dependsOnContainer.HealthStatusShouldBeReported() && + dependsOnContainer.GetHealthStatus().Status == apicontainerstatus.ContainerHealthy + + default: + return false } - log.Errorf("Failed to resolve if the link [%v] has been resolved for the target [%v]", link, target) - return false } -func volumeCanResolve(target *apicontainer.Container, volume *apicontainer.Container) bool { - targetDesiredStatus := target.GetDesiredStatus() - if targetDesiredStatus != apicontainerstatus.ContainerCreated && targetDesiredStatus != target.GetSteadyStateStatus() { - // The 'target' container doesn't desire to move to either 'Created' or the 'steady' state, - // which is not allowed - log.Errorf("Failed to resolve the desired status of the volume [%v] for the target [%v]", volume, target) +func hasDependencyTimedOut(dependOnContainer *apicontainer.Container, dependencyCondition string) bool { + if dependOnContainer.GetStartedAt().IsZero() || dependOnContainer.GetStartTimeout() <= 0 { return false } + switch dependencyCondition { + case successCondition, completeCondition, healthyCondition: + return time.Now().After(dependOnContainer.GetStartedAt().Add(dependOnContainer.GetStartTimeout())) + default: + return false + } +} +func hasDependencyStoppedSuccessfully(dependency *apicontainer.Container, condition string) bool { + isDependencyStoppedSuccessfully := dependency.GetKnownStatus() == apicontainerstatus.ContainerStopped && + *dependency.GetKnownExitCode() == 0 + return isDependencyStoppedSuccessfully +} + +func verifyContainerOrderingStatus(dependsOnContainer *apicontainer.Container) bool { + dependsOnContainerDesiredStatus := dependsOnContainer.GetDesiredStatus() // The 'target' container desires to be moved to 'Created' or the 'steady' state. - // Allow this only if the known status of the source volume container is - // any of 'Created', 'steady state' or 'Stopped' - volumeDesiredStatus := volume.GetDesiredStatus() - return volumeDesiredStatus == apicontainerstatus.ContainerCreated || - volumeDesiredStatus == volume.GetSteadyStateStatus() || - volumeDesiredStatus == apicontainerstatus.ContainerStopped + // Allow this only if the dependency container also desires to be started + // i.e it's status is any of 'Created', 'steady state' or 'Stopped' + return dependsOnContainerDesiredStatus == apicontainerstatus.ContainerCreated || + dependsOnContainerDesiredStatus == apicontainerstatus.ContainerStopped || + dependsOnContainerDesiredStatus == dependsOnContainer.GetSteadyStateStatus() } -func volumeIsResolved(target *apicontainer.Container, volume *apicontainer.Container) bool { - targetDesiredStatus := target.GetDesiredStatus() - if targetDesiredStatus != apicontainerstatus.ContainerCreated && targetDesiredStatus != apicontainerstatus.ContainerRunning { - // The 'target' container doesn't desire to be moved to 'Created' or the 'steady' state. - // Do not allow it. - log.Errorf("Failed to resolve if the volume [%v] has been resolved for the target [%v]", volume, target) - return false +func verifyShutdownOrder(target *apicontainer.Container, existingContainers map[string]*apicontainer.Container) error { + // We considered adding this to the task state, but this will be at most 45 loops, + // so we err'd on the side of having less state. + missingShutdownDependencies := []string{} + + for _, existingContainer := range existingContainers { + for _, dependency := range existingContainer.DependsOn { + // If another container declares a dependency on our target, we will want to verify that the container is + // stopped. + if dependency.ContainerName == target.Name { + if !existingContainer.KnownTerminal() { + missingShutdownDependencies = append(missingShutdownDependencies, existingContainer.Name) + } + } + } } - // The 'target' container desires to be moved to 'Created' or the 'steady' state. - // Allow this only if the known status of the source volume container is - // any of 'Created', 'steady state' or 'Stopped' - knownStatus := volume.GetKnownStatus() - return knownStatus == apicontainerstatus.ContainerCreated || - knownStatus == volume.GetSteadyStateStatus() || - knownStatus == apicontainerstatus.ContainerStopped + if len(missingShutdownDependencies) == 0 { + return nil + } + + return fmt.Errorf("dependency graph: target %s needs other containers stopped before it can stop: [%s]", + target.Name, strings.Join(missingShutdownDependencies, "], [")) } func onSteadyStateCanResolve(target *apicontainer.Container, run *apicontainer.Container) bool { diff --git a/agent/engine/dependencygraph/graph_test.go b/agent/engine/dependencygraph/graph_test.go index fcc2322d618..70053c38eb8 100644 --- a/agent/engine/dependencygraph/graph_test.go +++ b/agent/engine/dependencygraph/graph_test.go @@ -18,6 +18,7 @@ package dependencygraph import ( "fmt" "testing" + "time" apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" apicontainerstatus "github.com/aws/amazon-ecs-agent/agent/api/container/status" @@ -25,7 +26,7 @@ import ( "github.com/aws/amazon-ecs-agent/agent/taskresource" "github.com/aws/amazon-ecs-agent/agent/taskresource/mocks" resourcestatus "github.com/aws/amazon-ecs-agent/agent/taskresource/status" - + "github.com/aws/aws-sdk-go/aws" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" ) @@ -38,20 +39,18 @@ func volumeStrToVol(vols []string) []apicontainer.VolumeFrom { return ret } -func steadyStateContainer(name string, links, volumes []string, desiredState apicontainerstatus.ContainerStatus, steadyState apicontainerstatus.ContainerStatus) *apicontainer.Container { +func steadyStateContainer(name string, dependsOn []apicontainer.DependsOn, desiredState apicontainerstatus.ContainerStatus, steadyState apicontainerstatus.ContainerStatus) *apicontainer.Container { container := apicontainer.NewContainerWithSteadyState(steadyState) container.Name = name - container.Links = links - container.VolumesFrom = volumeStrToVol(volumes) + container.DependsOn = dependsOn container.DesiredStatusUnsafe = desiredState return container } -func createdContainer(name string, links, volumes []string, steadyState apicontainerstatus.ContainerStatus) *apicontainer.Container { +func createdContainer(name string, dependsOn []apicontainer.DependsOn, steadyState apicontainerstatus.ContainerStatus) *apicontainer.Container { container := apicontainer.NewContainerWithSteadyState(steadyState) container.Name = name - container.Links = links - container.VolumesFrom = volumeStrToVol(volumes) + container.DependsOn = dependsOn container.DesiredStatusUnsafe = apicontainerstatus.ContainerCreated return container } @@ -74,12 +73,12 @@ func TestValidDependencies(t *testing.T) { assert.True(t, resolveable, "One container should resolve trivially") // Webserver stack - php := steadyStateContainer("php", []string{"db"}, []string{}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - db := steadyStateContainer("db", []string{}, []string{"dbdatavolume"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - dbdata := createdContainer("dbdatavolume", []string{}, []string{}, apicontainerstatus.ContainerRunning) - webserver := steadyStateContainer("webserver", []string{"php"}, []string{"htmldata"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - htmldata := steadyStateContainer("htmldata", []string{}, []string{"sharedcssfiles"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - sharedcssfiles := createdContainer("sharedcssfiles", []string{}, []string{}, apicontainerstatus.ContainerRunning) + php := steadyStateContainer("php", []apicontainer.DependsOn{{ContainerName: "db", Condition: startCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + db := steadyStateContainer("db", []apicontainer.DependsOn{{ContainerName: "dbdatavolume", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + dbdata := createdContainer("dbdatavolume", []apicontainer.DependsOn{}, apicontainerstatus.ContainerRunning) + webserver := steadyStateContainer("webserver", []apicontainer.DependsOn{{ContainerName: "php", Condition: startCondition}, {ContainerName: "htmldata", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + htmldata := steadyStateContainer("htmldata", []apicontainer.DependsOn{{ContainerName: "sharedcssfiles", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + sharedcssfiles := createdContainer("sharedcssfiles", []apicontainer.DependsOn{}, apicontainerstatus.ContainerRunning) task = &apitask.Task{ Containers: []*apicontainer.Container{ @@ -95,8 +94,8 @@ func TestValidDependenciesWithCycles(t *testing.T) { // Unresolveable: cycle task := &apitask.Task{ Containers: []*apicontainer.Container{ - steadyStateContainer("a", []string{"b"}, []string{}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning), - steadyStateContainer("b", []string{"a"}, []string{}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning), + steadyStateContainer("a", []apicontainer.DependsOn{{ContainerName: "b", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning), + steadyStateContainer("b", []apicontainer.DependsOn{{ContainerName: "a", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning), }, } resolveable := ValidDependencies(task) @@ -107,7 +106,7 @@ func TestValidDependenciesWithUnresolvedReference(t *testing.T) { // Unresolveable, reference doesn't exist task := &apitask.Task{ Containers: []*apicontainer.Container{ - steadyStateContainer("php", []string{"db"}, []string{}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning), + steadyStateContainer("php", []apicontainer.DependsOn{{ContainerName: "db", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning), }, } resolveable := ValidDependencies(task) @@ -123,16 +122,16 @@ func TestDependenciesAreResolvedWhenSteadyStateIsRunning(t *testing.T) { }, }, } - err := DependenciesAreResolved(task.Containers[0], task.Containers, "", nil, nil) + _, err := DependenciesAreResolved(task.Containers[0], task.Containers, "", nil, nil) assert.NoError(t, err, "One container should resolve trivially") // Webserver stack - php := steadyStateContainer("php", []string{"db"}, []string{}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - db := steadyStateContainer("db", []string{}, []string{"dbdatavolume"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - dbdata := createdContainer("dbdatavolume", []string{}, []string{}, apicontainerstatus.ContainerRunning) - webserver := steadyStateContainer("webserver", []string{"php"}, []string{"htmldata"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - htmldata := steadyStateContainer("htmldata", []string{}, []string{"sharedcssfiles"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - sharedcssfiles := createdContainer("sharedcssfiles", []string{}, []string{}, apicontainerstatus.ContainerRunning) + php := steadyStateContainer("php", []apicontainer.DependsOn{{ContainerName: "db", Condition: startCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + db := steadyStateContainer("db", []apicontainer.DependsOn{{ContainerName: "dbdatavolume", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + dbdata := createdContainer("dbdatavolume", []apicontainer.DependsOn{}, apicontainerstatus.ContainerRunning) + webserver := steadyStateContainer("webserver", []apicontainer.DependsOn{{ContainerName: "php", Condition: startCondition}, {ContainerName: "htmldata", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + htmldata := steadyStateContainer("htmldata", []apicontainer.DependsOn{{ContainerName: "sharedcssfiles", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + sharedcssfiles := createdContainer("sharedcssfiles", []apicontainer.DependsOn{}, apicontainerstatus.ContainerRunning) task = &apitask.Task{ Containers: []*apicontainer.Container{ @@ -140,28 +139,28 @@ func TestDependenciesAreResolvedWhenSteadyStateIsRunning(t *testing.T) { }, } - err = DependenciesAreResolved(php, task.Containers, "", nil, nil) + _, err = DependenciesAreResolved(php, task.Containers, "", nil, nil) assert.Error(t, err, "Shouldn't be resolved; db isn't running") - err = DependenciesAreResolved(db, task.Containers, "", nil, nil) + _, err = DependenciesAreResolved(db, task.Containers, "", nil, nil) assert.Error(t, err, "Shouldn't be resolved; dbdatavolume isn't created") - err = DependenciesAreResolved(dbdata, task.Containers, "", nil, nil) + _, err = DependenciesAreResolved(dbdata, task.Containers, "", nil, nil) assert.NoError(t, err, "data volume with no deps should resolve") - dbdata.KnownStatusUnsafe = apicontainerstatus.ContainerCreated - err = DependenciesAreResolved(php, task.Containers, "", nil, nil) + dbdata.SetKnownStatus(apicontainerstatus.ContainerCreated) + _, err = DependenciesAreResolved(php, task.Containers, "", nil, nil) assert.Error(t, err, "Php shouldn't run, db is not created") - db.KnownStatusUnsafe = apicontainerstatus.ContainerCreated - err = DependenciesAreResolved(php, task.Containers, "", nil, nil) + db.SetKnownStatus(apicontainerstatus.ContainerCreated) + _, err = DependenciesAreResolved(php, task.Containers, "", nil, nil) assert.Error(t, err, "Php shouldn't run, db is not running") - err = DependenciesAreResolved(db, task.Containers, "", nil, nil) + _, err = DependenciesAreResolved(db, task.Containers, "", nil, nil) assert.NoError(t, err, "db should be resolved, dbdata volume is Created") - db.KnownStatusUnsafe = apicontainerstatus.ContainerRunning + db.SetKnownStatus(apicontainerstatus.ContainerRunning) - err = DependenciesAreResolved(php, task.Containers, "", nil, nil) + _, err = DependenciesAreResolved(php, task.Containers, "", nil, nil) assert.NoError(t, err, "Php should resolve") } @@ -177,28 +176,32 @@ func TestRunDependencies(t *testing.T) { SteadyStateDependencies: []string{"a"}, } task := &apitask.Task{Containers: []*apicontainer.Container{c1, c2}} + _, err := DependenciesAreResolved(c2, task.Containers, "", nil, nil) + assert.Error(t, err, "Dependencies should not be resolved") - assert.Error(t, DependenciesAreResolved(c2, task.Containers, "", nil, nil), "Dependencies should not be resolved") task.Containers[1].SetDesiredStatus(apicontainerstatus.ContainerRunning) - assert.Error(t, DependenciesAreResolved(c2, task.Containers, "", nil, nil), "Dependencies should not be resolved") + _, err = DependenciesAreResolved(c2, task.Containers, "", nil, nil) + assert.Error(t, err, "Dependencies should not be resolved") - task.Containers[0].KnownStatusUnsafe = apicontainerstatus.ContainerRunning - assert.NoError(t, DependenciesAreResolved(c2, task.Containers, "", nil, nil), "Dependencies should be resolved") + task.Containers[0].SetKnownStatus(apicontainerstatus.ContainerRunning) + _, err = DependenciesAreResolved(c2, task.Containers, "", nil, nil) + assert.NoError(t, err, "Dependencies should be resolved") task.Containers[1].SetDesiredStatus(apicontainerstatus.ContainerCreated) - assert.NoError(t, DependenciesAreResolved(c1, task.Containers, "", nil, nil), "Dependencies should be resolved") + _, err = DependenciesAreResolved(c1, task.Containers, "", nil, nil) + assert.NoError(t, err, "Dependencies should be resolved") } func TestRunDependenciesWhenSteadyStateIsResourcesProvisionedForOneContainer(t *testing.T) { // Webserver stack - php := steadyStateContainer("php", []string{"db"}, []string{}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - db := steadyStateContainer("db", []string{}, []string{"dbdatavolume"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - dbdata := createdContainer("dbdatavolume", []string{}, []string{}, apicontainerstatus.ContainerRunning) - webserver := steadyStateContainer("webserver", []string{"php"}, []string{"htmldata"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - htmldata := steadyStateContainer("htmldata", []string{}, []string{"sharedcssfiles"}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) - sharedcssfiles := createdContainer("sharedcssfiles", []string{}, []string{}, apicontainerstatus.ContainerRunning) + php := steadyStateContainer("php", []apicontainer.DependsOn{{ContainerName: "db", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + db := steadyStateContainer("db", []apicontainer.DependsOn{{ContainerName: "dbdatavolume", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + dbdata := createdContainer("dbdatavolume", []apicontainer.DependsOn{}, apicontainerstatus.ContainerRunning) + webserver := steadyStateContainer("webserver", []apicontainer.DependsOn{{ContainerName: "php", Condition: createCondition}, {ContainerName: "htmldata", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + htmldata := steadyStateContainer("htmldata", []apicontainer.DependsOn{{ContainerName: "sharedcssfiles", Condition: createCondition}}, apicontainerstatus.ContainerRunning, apicontainerstatus.ContainerRunning) + sharedcssfiles := createdContainer("sharedcssfiles", []apicontainer.DependsOn{}, apicontainerstatus.ContainerRunning) // The Pause container, being added to the webserver stack - pause := steadyStateContainer("pause", []string{}, []string{}, apicontainerstatus.ContainerResourcesProvisioned, apicontainerstatus.ContainerResourcesProvisioned) + pause := steadyStateContainer("pause", []apicontainer.DependsOn{}, apicontainerstatus.ContainerResourcesProvisioned, apicontainerstatus.ContainerResourcesProvisioned) task := &apitask.Task{ Containers: []*apicontainer.Container{ @@ -212,11 +215,11 @@ func TestRunDependenciesWhenSteadyStateIsResourcesProvisionedForOneContainer(t * continue } container.SteadyStateDependencies = []string{"pause"} - err := DependenciesAreResolved(container, task.Containers, "", nil, nil) + _, err := DependenciesAreResolved(container, task.Containers, "", nil, nil) assert.Error(t, err, "Shouldn't be resolved; pause isn't running") } - err := DependenciesAreResolved(pause, task.Containers, "", nil, nil) + _, err := DependenciesAreResolved(pause, task.Containers, "", nil, nil) assert.NoError(t, err, "Pause container's dependencies should be resolved") // Transition pause container to RUNNING @@ -230,166 +233,16 @@ func TestRunDependenciesWhenSteadyStateIsResourcesProvisionedForOneContainer(t * } // Assert that dependencies remain unresolved until the pause container reaches // RESOURCES_PROVISIONED - err = DependenciesAreResolved(container, task.Containers, "", nil, nil) + _, err = DependenciesAreResolved(container, task.Containers, "", nil, nil) assert.Error(t, err, "Shouldn't be resolved; pause isn't running") } pause.KnownStatusUnsafe = apicontainerstatus.ContainerResourcesProvisioned // Dependecies should be resolved now that the 'pause' container has // transitioned into RESOURCES_PROVISIONED - err = DependenciesAreResolved(php, task.Containers, "", nil, nil) + _, err = DependenciesAreResolved(php, task.Containers, "", nil, nil) assert.NoError(t, err, "Php should resolve") } -func TestVolumeCanResolve(t *testing.T) { - testcases := []struct { - TargetDesired apicontainerstatus.ContainerStatus - VolumeDesired apicontainerstatus.ContainerStatus - Resolvable bool - }{ - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeDesired: apicontainerstatus.ContainerStatusNone, - Resolvable: false, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeDesired: apicontainerstatus.ContainerCreated, - Resolvable: true, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeDesired: apicontainerstatus.ContainerRunning, - Resolvable: true, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeDesired: apicontainerstatus.ContainerStopped, - Resolvable: true, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeDesired: apicontainerstatus.ContainerZombie, - Resolvable: false, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeDesired: apicontainerstatus.ContainerStatusNone, - Resolvable: false, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeDesired: apicontainerstatus.ContainerCreated, - Resolvable: true, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeDesired: apicontainerstatus.ContainerRunning, - Resolvable: true, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeDesired: apicontainerstatus.ContainerStopped, - Resolvable: true, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeDesired: apicontainerstatus.ContainerZombie, - Resolvable: false, - }, - { - TargetDesired: apicontainerstatus.ContainerStatusNone, - Resolvable: false, - }, - { - TargetDesired: apicontainerstatus.ContainerStopped, - Resolvable: false, - }, - { - TargetDesired: apicontainerstatus.ContainerZombie, - Resolvable: false, - }, - } - for _, tc := range testcases { - t.Run(fmt.Sprintf("T:%s+V:%s", tc.TargetDesired.String(), tc.VolumeDesired.String()), - assertCanResolve(volumeCanResolve, tc.TargetDesired, tc.VolumeDesired, tc.Resolvable)) - } -} - -func TestVolumeIsResolved(t *testing.T) { - testcases := []struct { - TargetDesired apicontainerstatus.ContainerStatus - VolumeKnown apicontainerstatus.ContainerStatus - Resolved bool - }{ - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeKnown: apicontainerstatus.ContainerStatusNone, - Resolved: false, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeKnown: apicontainerstatus.ContainerCreated, - Resolved: true, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeKnown: apicontainerstatus.ContainerRunning, - Resolved: true, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeKnown: apicontainerstatus.ContainerStopped, - Resolved: true, - }, - { - TargetDesired: apicontainerstatus.ContainerCreated, - VolumeKnown: apicontainerstatus.ContainerZombie, - Resolved: false, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeKnown: apicontainerstatus.ContainerStatusNone, - Resolved: false, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeKnown: apicontainerstatus.ContainerCreated, - Resolved: true, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeKnown: apicontainerstatus.ContainerRunning, - Resolved: true, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeKnown: apicontainerstatus.ContainerStopped, - Resolved: true, - }, - { - TargetDesired: apicontainerstatus.ContainerRunning, - VolumeKnown: apicontainerstatus.ContainerZombie, - Resolved: false, - }, - { - TargetDesired: apicontainerstatus.ContainerStatusNone, - Resolved: false, - }, - { - TargetDesired: apicontainerstatus.ContainerStopped, - Resolved: false, - }, - { - TargetDesired: apicontainerstatus.ContainerZombie, - Resolved: false, - }, - } - for _, tc := range testcases { - t.Run(fmt.Sprintf("T:%s+V:%s", tc.TargetDesired.String(), tc.VolumeKnown.String()), - assertResolved(volumeIsResolved, tc.TargetDesired, tc.VolumeKnown, tc.Resolved)) - } -} - func TestOnSteadyStateIsResolved(t *testing.T) { testcases := []struct { TargetDesired apicontainerstatus.ContainerStatus @@ -426,19 +279,6 @@ func TestOnSteadyStateIsResolved(t *testing.T) { } } -func assertCanResolve(f func(target *apicontainer.Container, dep *apicontainer.Container) bool, targetDesired, depKnown apicontainerstatus.ContainerStatus, expectedResolvable bool) func(t *testing.T) { - return func(t *testing.T) { - target := &apicontainer.Container{ - DesiredStatusUnsafe: targetDesired, - } - dep := &apicontainer.Container{ - DesiredStatusUnsafe: depKnown, - } - resolvable := f(target, dep) - assert.Equal(t, expectedResolvable, resolvable) - } -} - func assertResolved(f func(target *apicontainer.Container, dep *apicontainer.Container) bool, targetDesired, depKnown apicontainerstatus.ContainerStatus, expectedResolved bool) func(t *testing.T) { return func(t *testing.T) { target := &apicontainer.Container{ @@ -728,3 +568,448 @@ func TestTransitionDependencyResourceNotFound(t *testing.T) { resolved := verifyTransitionDependenciesResolved(target, nil, resources) assert.Equal(t, ErrResourceDependencyNotResolved, resolved) } + +func TestContainerOrderingCanResolve(t *testing.T) { + testcases := []struct { + TargetDesired apicontainerstatus.ContainerStatus + DependencyDesired apicontainerstatus.ContainerStatus + DependencyKnown apicontainerstatus.ContainerStatus + DependencyCondition string + ExitCode int + Resolvable bool + }{ + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyDesired: apicontainerstatus.ContainerStatusNone, + DependencyCondition: createCondition, + Resolvable: false, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyDesired: apicontainerstatus.ContainerStopped, + DependencyCondition: createCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyDesired: apicontainerstatus.ContainerZombie, + DependencyCondition: createCondition, + Resolvable: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerStatusNone, + DependencyCondition: createCondition, + Resolvable: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerCreated, + DependencyCondition: createCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerRunning, + DependencyCondition: createCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerStopped, + DependencyCondition: createCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyDesired: apicontainerstatus.ContainerCreated, + DependencyCondition: startCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerRunning, + DependencyCondition: startCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyDesired: apicontainerstatus.ContainerRunning, + DependencyCondition: startCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerZombie, + DependencyCondition: startCondition, + Resolvable: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerStopped, + DependencyCondition: successCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerRunning, + DependencyCondition: successCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStopped, + ExitCode: 0, + DependencyCondition: successCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStopped, + ExitCode: 1, + DependencyCondition: successCondition, + Resolvable: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerStopped, + DependencyCondition: completeCondition, + Resolvable: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyDesired: apicontainerstatus.ContainerRunning, + DependencyCondition: completeCondition, + Resolvable: true, + }, + } + for _, tc := range testcases { + t.Run(fmt.Sprintf("T:%s+V:%s", tc.TargetDesired.String(), tc.DependencyDesired.String()), + assertContainerOrderingCanResolve(containerOrderingDependenciesCanResolve, tc.TargetDesired, tc.DependencyDesired, tc.DependencyKnown, tc.DependencyCondition, tc.ExitCode, tc.Resolvable)) + } +} + +func TestContainerOrderingIsResolved(t *testing.T) { + testcases := []struct { + TargetDesired apicontainerstatus.ContainerStatus + DependencyKnown apicontainerstatus.ContainerStatus + DependencyCondition string + Resolved bool + ExitCode int + }{ + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnown: apicontainerstatus.ContainerStatusNone, + DependencyCondition: createCondition, + Resolved: false, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnown: apicontainerstatus.ContainerCreated, + DependencyCondition: createCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStopped, + DependencyCondition: createCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnown: apicontainerstatus.ContainerStopped, + DependencyCondition: createCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStatusNone, + DependencyCondition: createCondition, + Resolved: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerCreated, + DependencyCondition: createCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnown: apicontainerstatus.ContainerCreated, + DependencyCondition: startCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnown: apicontainerstatus.ContainerRunning, + DependencyCondition: startCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnown: apicontainerstatus.ContainerZombie, + DependencyCondition: startCondition, + Resolved: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerRunning, + DependencyCondition: startCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerZombie, + DependencyCondition: startCondition, + Resolved: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStopped, + DependencyCondition: successCondition, + ExitCode: 0, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStopped, + DependencyCondition: successCondition, + ExitCode: 1, + Resolved: false, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStopped, + DependencyCondition: completeCondition, + ExitCode: 0, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerStopped, + DependencyCondition: completeCondition, + ExitCode: 1, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerRunning, + DependencyKnown: apicontainerstatus.ContainerRunning, + DependencyCondition: completeCondition, + Resolved: false, + }, + } + for _, tc := range testcases { + t.Run(fmt.Sprintf("T:%s+V:%s", tc.TargetDesired.String(), tc.DependencyKnown.String()), + assertContainerOrderingResolved(containerOrderingDependenciesIsResolved, tc.TargetDesired, tc.DependencyKnown, tc.DependencyCondition, tc.ExitCode, tc.Resolved)) + } +} + +func assertContainerOrderingCanResolve(f func(target *apicontainer.Container, dep *apicontainer.Container, depCond string) bool, targetDesired, depDesired, depKnown apicontainerstatus.ContainerStatus, depCond string, exitcode int, expectedResolvable bool) func(t *testing.T) { + return func(t *testing.T) { + target := &apicontainer.Container{ + DesiredStatusUnsafe: targetDesired, + } + dep := &apicontainer.Container{ + DesiredStatusUnsafe: depDesired, + KnownStatusUnsafe: depKnown, + KnownExitCodeUnsafe: aws.Int(exitcode), + } + resolvable := f(target, dep, depCond) + assert.Equal(t, expectedResolvable, resolvable) + } +} + +func assertContainerOrderingResolved(f func(target *apicontainer.Container, dep *apicontainer.Container, depCond string) bool, targetDesired, depKnown apicontainerstatus.ContainerStatus, depCond string, exitcode int, expectedResolved bool) func(t *testing.T) { + return func(t *testing.T) { + target := &apicontainer.Container{ + DesiredStatusUnsafe: targetDesired, + } + dep := &apicontainer.Container{ + KnownStatusUnsafe: depKnown, + KnownExitCodeUnsafe: aws.Int(exitcode), + } + resolved := f(target, dep, depCond) + assert.Equal(t, expectedResolved, resolved) + } +} + +func TestContainerOrderingHealthyConditionIsResolved(t *testing.T) { + testcases := []struct { + TargetDesired apicontainerstatus.ContainerStatus + DependencyKnownHealthStatus apicontainerstatus.ContainerHealthStatus + HealthCheckType string + DependencyKnownHealthExitCode int + DependencyCondition string + Resolved bool + }{ + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnownHealthStatus: apicontainerstatus.ContainerHealthy, + HealthCheckType: "docker", + DependencyCondition: healthyCondition, + Resolved: true, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + DependencyKnownHealthStatus: apicontainerstatus.ContainerUnhealthy, + HealthCheckType: "docker", + DependencyKnownHealthExitCode: 1, + DependencyCondition: healthyCondition, + Resolved: false, + }, + { + TargetDesired: apicontainerstatus.ContainerCreated, + Resolved: false, + }, + } + for _, tc := range testcases { + t.Run(fmt.Sprintf("T:%s+V:%s", tc.TargetDesired.String(), tc.DependencyKnownHealthStatus.String()), + assertContainerOrderingHealthyConditionResolved(containerOrderingDependenciesIsResolved, tc.TargetDesired, tc.DependencyKnownHealthStatus, tc.HealthCheckType, tc.DependencyKnownHealthExitCode, tc.DependencyCondition, tc.Resolved)) + } +} + +func assertContainerOrderingHealthyConditionResolved(f func(target *apicontainer.Container, dep *apicontainer.Container, depCond string) bool, targetDesired apicontainerstatus.ContainerStatus, depHealthKnown apicontainerstatus.ContainerHealthStatus, healthCheckEnabled string, depHealthKnownExitCode int, depCond string, expectedResolved bool) func(t *testing.T) { + return func(t *testing.T) { + target := &apicontainer.Container{ + DesiredStatusUnsafe: targetDesired, + } + dep := &apicontainer.Container{ + Health: apicontainer.HealthStatus{ + Status: depHealthKnown, + ExitCode: depHealthKnownExitCode, + }, + HealthCheckType: healthCheckEnabled, + } + resolved := f(target, dep, depCond) + assert.Equal(t, expectedResolved, resolved) + } +} + +func dependsOn(vals ...string) []apicontainer.DependsOn { + d := make([]apicontainer.DependsOn, len(vals)) + for i, val := range vals { + d[i] = apicontainer.DependsOn{ContainerName: val} + } + return d +} + +// TestVerifyShutdownOrder validates that the shutdown graph works in the inverse order of the startup graph +// This test always uses a running target +func TestVerifyShutdownOrder(t *testing.T) { + + // dependencies aren't transitive, so we can check multiple levels within here + others := map[string]*apicontainer.Container{ + "A": { + Name: "A", + DependsOn: dependsOn("B"), + KnownStatusUnsafe: apicontainerstatus.ContainerStopped, + }, + "B": { + Name: "B", + DependsOn: dependsOn("C", "D"), + KnownStatusUnsafe: apicontainerstatus.ContainerRunning, + }, + "C": { + Name: "C", + DependsOn: dependsOn("E", "F"), + KnownStatusUnsafe: apicontainerstatus.ContainerStopped, + }, + "D": { + Name: "D", + DependsOn: dependsOn("E"), + KnownStatusUnsafe: apicontainerstatus.ContainerRunning, + }, + } + + testCases := []struct { + TestName string + TargetName string + ShouldResolve bool + }{ + { + TestName: "Dependency is already stopped", + TargetName: "F", + ShouldResolve: true, + }, + { + TestName: "Running with a running dependency", + TargetName: "D", + ShouldResolve: false, + }, + { + TestName: "One dependency running, one stopped", + TargetName: "E", + ShouldResolve: false, + }, + { + TestName: "No dependencies declared", + TargetName: "Z", + ShouldResolve: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.TestName, func(t *testing.T) { + // create a target container + target := &apicontainer.Container{ + Name: tc.TargetName, + KnownStatusUnsafe: apicontainerstatus.ContainerRunning, + DesiredStatusUnsafe: apicontainerstatus.ContainerStopped, + } + + // Validation + if tc.ShouldResolve { + assert.NoError(t, verifyShutdownOrder(target, others)) + } else { + assert.Error(t, verifyShutdownOrder(target, others)) + } + }) + } +} + +func TestStartTimeoutForContainerOrdering(t *testing.T) { + testcases := []struct { + DependencyStartedAt time.Time + DependencyStartTimeout uint + DependencyCondition string + ExpectedTimedOut bool + }{ + { + DependencyStartedAt: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + DependencyStartTimeout: 10, + DependencyCondition: healthyCondition, + ExpectedTimedOut: true, + }, + { + DependencyStartedAt: time.Date(4000, 1, 1, 0, 0, 0, 0, time.UTC), + DependencyStartTimeout: 10, + DependencyCondition: successCondition, + ExpectedTimedOut: false, + }, + { + DependencyStartedAt: time.Time{}, + DependencyStartTimeout: 10, + DependencyCondition: successCondition, + ExpectedTimedOut: false, + }, + } + + for _, tc := range testcases { + t.Run(fmt.Sprintf("T:dependency time out: %v", tc.ExpectedTimedOut), + assertContainerOrderingNotTimedOut(hasDependencyTimedOut, tc.DependencyStartedAt, tc.DependencyStartTimeout, tc.DependencyCondition, tc.ExpectedTimedOut)) + } +} + +func assertContainerOrderingNotTimedOut(f func(dep *apicontainer.Container, depCond string) bool, startedAt time.Time, timeout uint, depCond string, expectedTimedOut bool) func(t *testing.T) { + return func(t *testing.T) { + dep := &apicontainer.Container{ + Name: "dep", + StartTimeout: timeout, + } + + dep.SetStartedAt(startedAt) + + timedOut := f(dep, depCond) + assert.Equal(t, expectedTimedOut, timedOut) + } +} diff --git a/agent/engine/docker_image_manager_test.go b/agent/engine/docker_image_manager_test.go index 4c1be98587e..05992c47f3f 100644 --- a/agent/engine/docker_image_manager_test.go +++ b/agent/engine/docker_image_manager_test.go @@ -145,8 +145,8 @@ func TestRecordContainerReferenceInspectError(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -179,8 +179,8 @@ func TestRecordContainerReferenceWithNoImageName(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -406,8 +406,8 @@ func TestRemoveContainerReferenceFromImageStateWithNoReference(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -442,8 +442,8 @@ func TestGetCandidateImagesForDeletionImageNoImageState(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -462,8 +462,8 @@ func TestGetCandidateImagesForDeletionImageJustPulled(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -487,8 +487,8 @@ func TestGetCandidateImagesForDeletionImageHasContainerReference(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -528,8 +528,8 @@ func TestGetCandidateImagesForDeletionImageHasMoreContainerReferences(t *testing client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -606,8 +606,8 @@ func TestImageCleanupExclusionListWithSingleName(t *testing.T) { PulledAt: time.Now().AddDate(0, -2, 0), } imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -665,8 +665,8 @@ func TestImageCleanupExclusionListWithMultipleNames(t *testing.T) { PulledAt: time.Now().AddDate(0, -2, 0), } imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -731,8 +731,8 @@ func TestGetLeastRecentlyUsedImagesLessThanFive(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -765,8 +765,8 @@ func TestRemoveAlreadyExistingImageNameWithDifferentID(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -816,8 +816,8 @@ func TestImageCleanupHappyPath(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: 1 * time.Millisecond, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -873,8 +873,8 @@ func TestImageCleanupCannotRemoveImage(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -931,8 +931,8 @@ func TestImageCleanupRemoveImageById(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -984,8 +984,8 @@ func TestNonECSImageAndContainersCleanupRemoveImage(t *testing.T) { defer ctrl.Finish() client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -1220,8 +1220,8 @@ func TestConcurrentRemoveUnusedImages(t *testing.T) { client := mock_dockerapi.NewMockDockerClient(ctrl) imageManager := &dockerImageManager{ - client: client, - state: dockerstate.NewTaskEngineState(), + client: client, + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index e3e17a45407..788a7e9746b 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -1095,9 +1095,13 @@ func (engine *DockerTaskEngine) stopContainer(task *apitask.Task, container *api } seelog.Infof("Task engine [%s]: cleaned pause container network namespace", task.Arn) } - // timeout is defined by the const 'stopContainerTimeout' and the 'DockerStopTimeout' in the config - timeout := engine.cfg.DockerStopTimeout + dockerclient.StopContainerTimeout - return engine.client.StopContainer(engine.ctx, dockerContainer.DockerID, timeout) + + apiTimeoutStopContainer := container.GetStopTimeout() + if apiTimeoutStopContainer <= 0 { + apiTimeoutStopContainer = engine.cfg.DockerStopTimeout + } + + return engine.client.StopContainer(engine.ctx, dockerContainer.DockerID, apiTimeoutStopContainer) } func (engine *DockerTaskEngine) removeContainer(task *apitask.Task, container *apicontainer.Container) error { diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index eccffc202f3..89aa3ae504a 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -1271,7 +1271,7 @@ func TestStopPauseContainerCleanupCalled(t *testing.T) { mockCNIClient.EXPECT().CleanupNS(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), dockerClient.EXPECT().StopContainer(gomock.Any(), containerID, - defaultConfig.DockerStopTimeout+dockerclient.StopContainerTimeout, + defaultConfig.DockerStopTimeout, ).Return(dockerapi.DockerContainerMetadata{}), ) diff --git a/agent/engine/engine_integ_test.go b/agent/engine/engine_integ_test.go index 4425b86d751..b4b101f152c 100644 --- a/agent/engine/engine_integ_test.go +++ b/agent/engine/engine_integ_test.go @@ -51,6 +51,15 @@ const ( localhost = "127.0.0.1" waitForDockerDuration = 50 * time.Millisecond removeVolumeTimeout = 5 * time.Second + + alwaysHealthyHealthCheckConfig = `{ + "HealthCheck":{ + "Test":["CMD-SHELL", "echo hello"], + "Interval":100000000, + "Timeout":100000000, + "StartPeriod":100000000, + "Retries":3} + }` ) func init() { diff --git a/agent/engine/engine_unix_integ_test.go b/agent/engine/engine_unix_integ_test.go index bdcb4d07313..5ad20fb6ab8 100644 --- a/agent/engine/engine_unix_integ_test.go +++ b/agent/engine/engine_unix_integ_test.go @@ -94,14 +94,7 @@ func createTestHealthCheckTask(arn string) *apitask.Task { testTask.Containers[0].HealthCheckType = "docker" testTask.Containers[0].Command = []string{"sh", "-c", "sleep 300"} testTask.Containers[0].DockerConfig = apicontainer.DockerConfig{ - Config: aws.String(`{ - "HealthCheck":{ - "Test":["CMD-SHELL", "echo hello"], - "Interval":100000000, - "Timeout":100000000, - "StartPeriod":100000000, - "Retries":3} - }`), + Config: aws.String(alwaysHealthyHealthCheckConfig), } return testTask } @@ -1447,3 +1440,42 @@ func TestGPUAssociationTask(t *testing.T) { go taskEngine.AddTask(&taskUpdate) verifyTaskIsStopped(stateChangeEvents, testTask) } + +func TestPerContainerStopTimeout(t *testing.T) { + // set the global stop timemout, but this should be ignored since the per container value is set + globalStopContainerTimeout := 1000 * time.Second + os.Setenv("ECS_CONTAINER_STOP_TIMEOUT", globalStopContainerTimeout.String()) + defer os.Unsetenv("ECS_CONTAINER_STOP_TIMEOUT") + cfg := defaultTestConfigIntegTest() + + taskEngine, _, _ := setup(cfg, nil, t) + + dockerTaskEngine := taskEngine.(*DockerTaskEngine) + + if dockerTaskEngine.cfg.DockerStopTimeout != globalStopContainerTimeout { + t.Errorf("Expect ECS_CONTAINER_STOP_TIMEOUT to be set to , %v", dockerTaskEngine.cfg.DockerStopTimeout) + } + + testTask := createTestTask("TestDockerStopTimeout") + testTask.Containers[0].Command = []string{"sh", "-c", "trap 'echo hello' SIGTERM; while true; do echo `date +%T`; sleep 1s; done;"} + testTask.Containers[0].Image = testBusyboxImage + testTask.Containers[0].Name = "test-docker-timeout" + testTask.Containers[0].StopTimeout = uint(testDockerStopTimeout.Seconds()) + + go dockerTaskEngine.AddTask(testTask) + + verifyContainerRunningStateChange(t, taskEngine) + verifyTaskRunningStateChange(t, taskEngine) + + startTime := ttime.Now() + dockerTaskEngine.stopContainer(testTask, testTask.Containers[0]) + + verifyContainerStoppedStateChange(t, taskEngine) + + if ttime.Since(startTime) < testDockerStopTimeout { + t.Errorf("Container stopped before the timeout: %v", ttime.Since(startTime)) + } + if ttime.Since(startTime) > testDockerStopTimeout+1*time.Second { + t.Errorf("Container should have stopped eariler, but stopped after %v", ttime.Since(startTime)) + } +} diff --git a/agent/engine/engine_windows_integ_test.go b/agent/engine/engine_windows_integ_test.go index 661b9b74aaf..c0ef0b6190d 100644 --- a/agent/engine/engine_windows_integ_test.go +++ b/agent/engine/engine_windows_integ_test.go @@ -58,7 +58,7 @@ func isDockerRunning() bool { return true } func createTestContainer() *apicontainer.Container { return &apicontainer.Container{ Name: "windows", - Image: "microsoft/windowsservercore:latest", + Image: "microsoft/windowsservercore", Essential: true, DesiredStatusUnsafe: apicontainerstatus.ContainerRunning, CPU: 512, @@ -96,19 +96,12 @@ func createTestHealthCheckTask(arn string) *apitask.Task { DesiredStatusUnsafe: apitaskstatus.TaskRunning, Containers: []*apicontainer.Container{createTestContainer()}, } - testTask.Containers[0].Image = "microsoft/nanoserver:latest" + testTask.Containers[0].Image = "microsoft/nanoserver" testTask.Containers[0].Name = "test-health-check" testTask.Containers[0].HealthCheckType = "docker" testTask.Containers[0].Command = []string{"powershell", "-command", "Start-Sleep -s 300"} testTask.Containers[0].DockerConfig = apicontainer.DockerConfig{ - Config: aws.String(`{ - "HealthCheck":{ - "Test":["CMD-SHELL", "echo hello"], - "Interval":100000000, - "Timeout":100000000, - "StartPeriod":100000000, - "Retries":3} - }`), + Config: aws.String(alwaysHealthyHealthCheckConfig), } return testTask } diff --git a/agent/engine/ordering_integ_test.go b/agent/engine/ordering_integ_test.go new file mode 100644 index 00000000000..7de2939e4c0 --- /dev/null +++ b/agent/engine/ordering_integ_test.go @@ -0,0 +1,444 @@ +// +build integration + +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package engine + +import ( + "testing" + "time" + + "github.com/aws/amazon-ecs-agent/agent/api" + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + "github.com/aws/amazon-ecs-agent/agent/api/container/status" + "github.com/aws/aws-sdk-go/aws" + "github.com/stretchr/testify/assert" +) + +const orderingTimeout = 90 * time.Second + +// TestDependencyHealthCheck is a happy-case integration test that considers a workflow with a HEALTHY dependency +// condition. We ensure that the task can be both started and stopped. +func TestDependencyHealthCheck(t *testing.T) { + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "testDependencyHealth" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + dependency := createTestContainerWithImageAndName(baseImageForOS, "dependency") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"sleep 5 && exit 1"} + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "dependency", + Condition: "HEALTHY", + }, + } + + dependency.EntryPoint = &entryPointForOS + dependency.Command = []string{"sleep 60 && exit 0"} + dependency.HealthCheckType = apicontainer.DockerHealthCheckType + dependency.DockerConfig.Config = aws.String(alwaysHealthyHealthCheckConfig) + + testTask.Containers = []*apicontainer.Container{ + parent, + dependency, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + // Both containers should start + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerRunningStateChange(t, taskEngine) + verifyTaskIsRunning(stateChangeEvents, testTask) + + // Task should stop all at once + verifyContainerStoppedStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + verifyTaskIsStopped(stateChangeEvents, testTask) + close(finished) + }() + + waitFinished(t, finished, orderingTimeout) +} + +// TestDependencyComplete validates that the COMPLETE dependency condition will resolve when the child container exits +// with exit code 1. It ensures that the child is started and stopped before the parent starts. +func TestDependencyComplete(t *testing.T) { + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "testDependencyComplete" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + dependency := createTestContainerWithImageAndName(baseImageForOS, "dependency") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"sleep 5"} + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "dependency", + Condition: "COMPLETE", + }, + } + + dependency.EntryPoint = &entryPointForOS + dependency.Command = []string{"sleep 10 && exit 1"} + dependency.Essential = false + + testTask.Containers = []*apicontainer.Container{ + parent, + dependency, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + // First container should run to completion and then exit + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + + // Second container starts after the first stops, task becomes running + verifyContainerRunningStateChange(t, taskEngine) + verifyTaskIsRunning(stateChangeEvents, testTask) + + // Last container stops and then the task stops + verifyContainerStoppedStateChange(t, taskEngine) + verifyTaskIsStopped(stateChangeEvents, testTask) + close(finished) + }() + + waitFinished(t, finished, orderingTimeout) +} + +// TestDependencySuccess validates that the SUCCESS dependency condition will resolve when the child container exits +// with exit code 0. It ensures that the child is started and stopped before the parent starts. +func TestDependencySuccess(t *testing.T) { + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "testDependencySuccess" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + dependency := createTestContainerWithImageAndName(baseImageForOS, "dependency") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"exit 0"} + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "dependency", + Condition: "SUCCESS", + }, + } + + dependency.EntryPoint = &entryPointForOS + dependency.Command = []string{"sleep 10"} + dependency.Essential = false + + testTask.Containers = []*apicontainer.Container{ + parent, + dependency, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + // First container should run to completion + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + + // Second container starts after the first stops, task becomes running + verifyContainerRunningStateChange(t, taskEngine) + verifyTaskIsRunning(stateChangeEvents, testTask) + + // Last container stops and then the task stops + verifyContainerStoppedStateChange(t, taskEngine) + verifyTaskIsStopped(stateChangeEvents, testTask) + close(finished) + }() + + waitFinished(t, finished, orderingTimeout) +} + +// TestDependencySuccess validates that the SUCCESS dependency condition will fail when the child exits 1. This is a +// contrast to how COMPLETE behaves. Instead of starting the parent, the task should simply exit. +func TestDependencySuccessErrored(t *testing.T) { + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "testDependencySuccessErrored" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + dependency := createTestContainerWithImageAndName(baseImageForOS, "dependency") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"exit 0"} + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "dependency", + Condition: "SUCCESS", + }, + } + + dependency.EntryPoint = &entryPointForOS + dependency.Command = []string{"sleep 10 && exit 1"} + dependency.Essential = false + + testTask.Containers = []*apicontainer.Container{ + parent, + dependency, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + // First container should run to completion + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + + // task should transition to stopped + verifyTaskIsStopped(stateChangeEvents, testTask) + close(finished) + }() + + waitFinished(t, finished, orderingTimeout) +} + +// TestDependencySuccessTimeout +func TestDependencySuccessTimeout(t *testing.T) { + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "testDependencySuccessTimeout" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + dependency := createTestContainerWithImageAndName(baseImageForOS, "dependency") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"exit 0"} + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "dependency", + Condition: "SUCCESS", + }, + } + + dependency.EntryPoint = &entryPointForOS + dependency.Command = []string{"sleep 15"} + dependency.Essential = false + + // set the timeout to be shorter than the amount of time it takes to stop + dependency.StartTimeout = 8 + + testTask.Containers = []*apicontainer.Container{ + parent, + dependency, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + // First container should run to completion + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + + // task should transition to stopped + verifyTaskIsStopped(stateChangeEvents, testTask) + close(finished) + }() + + waitFinished(t, finished, orderingTimeout) +} + +// TestDependencyHealthyTimeout +func TestDependencyHealthyTimeout(t *testing.T) { + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "testDependencyHealthyTimeout" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + dependency := createTestContainerWithImageAndName(baseImageForOS, "dependency") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"exit 0"} + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "dependency", + Condition: "HEALTHY", + }, + } + + dependency.EntryPoint = &entryPointForOS + dependency.Command = []string{"sleep 30"} + dependency.HealthCheckType = apicontainer.DockerHealthCheckType + + // enter a healthcheck that will fail + dependency.DockerConfig.Config = aws.String(`{ + "HealthCheck":{ + "Test":["CMD-SHELL", "exit 1"] + } + }`) + + // set the timeout. Duration doesn't matter since healthcheck will always be unhealthy. + dependency.StartTimeout = 8 + + testTask.Containers = []*apicontainer.Container{ + parent, + dependency, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + // First container should run to completion + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + + // task should transition to stopped + verifyTaskIsStopped(stateChangeEvents, testTask) + close(finished) + }() + + waitFinished(t, finished, orderingTimeout) +} + +// TestShutdownOrder +func TestShutdownOrder(t *testing.T) { + shutdownOrderingTimeout := 120 * time.Second + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "testShutdownOrder" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + A := createTestContainerWithImageAndName(baseImageForOS, "A") + B := createTestContainerWithImageAndName(baseImageForOS, "B") + C := createTestContainerWithImageAndName(baseImageForOS, "C") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"echo hi"} + parent.Essential = true + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "A", + Condition: "START", + }, + } + + A.EntryPoint = &entryPointForOS + A.Command = []string{"sleep 100"} + A.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "B", + Condition: "START", + }, + } + + B.EntryPoint = &entryPointForOS + B.Command = []string{"sleep 100"} + B.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "C", + Condition: "START", + }, + } + + C.EntryPoint = &entryPointForOS + C.Command = []string{"sleep 100"} + + testTask.Containers = []*apicontainer.Container{ + parent, + A, + B, + C, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + // Everything should first progress to running + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerRunningStateChange(t, taskEngine) + verifyTaskIsRunning(stateChangeEvents, testTask) + + // The shutdown order will now proceed. Parent will exit first since it has an explicit exit command. + event := <-stateChangeEvents + assert.Equal(t, event.(api.ContainerStateChange).Status, status.ContainerStopped) + assert.Equal(t, event.(api.ContainerStateChange).ContainerName, "parent") + + // The dependency chain is A -> B -> C. We expect the inverse order to be followed for shutdown: + // A shuts down, then B, then C + expectedC := <-stateChangeEvents + assert.Equal(t, expectedC.(api.ContainerStateChange).Status, status.ContainerStopped) + assert.Equal(t, expectedC.(api.ContainerStateChange).ContainerName, "A") + + expectedB := <-stateChangeEvents + assert.Equal(t, expectedB.(api.ContainerStateChange).Status, status.ContainerStopped) + assert.Equal(t, expectedB.(api.ContainerStateChange).ContainerName, "B") + + expectedA := <-stateChangeEvents + assert.Equal(t, expectedA.(api.ContainerStateChange).Status, status.ContainerStopped) + assert.Equal(t, expectedA.(api.ContainerStateChange).ContainerName, "C") + + close(finished) + }() + + waitFinished(t, finished, shutdownOrderingTimeout) +} + + +func waitFinished(t *testing.T, finished <-chan interface{}, duration time.Duration) { + select { + case <-finished: + t.Log("Finished successfully.") + return + case <-time.After(duration): + t.Error("timed out after: ", duration) + t.FailNow() + } +} diff --git a/agent/engine/ordering_integ_unix_test.go b/agent/engine/ordering_integ_unix_test.go new file mode 100644 index 00000000000..b392c43d42e --- /dev/null +++ b/agent/engine/ordering_integ_unix_test.go @@ -0,0 +1,98 @@ +// +build integration,!windows + +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package engine + +import ( + "testing" + + apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container" + "github.com/stretchr/testify/assert" +) + +const ( + baseImageForOS = testRegistryHost + "/" + "busybox" +) + +var ( + entryPointForOS = []string{"sh", "-c"} +) + +func TestGranularStopTimeout(t *testing.T) { + taskEngine, done, _ := setupWithDefaultConfig(t) + defer done() + + stateChangeEvents := taskEngine.StateChangeEvents() + + taskArn := "TestGranularStopTimeout" + testTask := createTestTask(taskArn) + + parent := createTestContainerWithImageAndName(baseImageForOS, "parent") + dependency1 := createTestContainerWithImageAndName(baseImageForOS, "dependency1") + dependency2 := createTestContainerWithImageAndName(baseImageForOS, "dependency2") + + parent.EntryPoint = &entryPointForOS + parent.Command = []string{"sleep 30"} + parent.Essential = true + parent.DependsOn = []apicontainer.DependsOn{ + { + ContainerName: "dependency1", + Condition: "START", + }, + { + ContainerName: "dependency2", + Condition: "START", + }, + } + + dependency1.EntryPoint = &entryPointForOS + dependency1.Command = []string{"trap 'echo caught' SIGTERM; sleep 60"} + dependency1.StopTimeout = 5 + + dependency2.EntryPoint = &entryPointForOS + dependency2.Command = []string{"trap 'echo caught' SIGTERM; sleep 60"} + dependency2.StopTimeout = 50 + + testTask.Containers = []*apicontainer.Container{ + dependency1, + dependency2, + parent, + } + + go taskEngine.AddTask(testTask) + + finished := make(chan interface{}) + go func() { + + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerRunningStateChange(t, taskEngine) + verifyContainerRunningStateChange(t, taskEngine) + + verifyTaskIsRunning(stateChangeEvents, testTask) + + verifyContainerStoppedStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + verifyContainerStoppedStateChange(t, taskEngine) + + verifyTaskIsStopped(stateChangeEvents, testTask) + + assert.Equal(t, 137, *testTask.Containers[0].GetKnownExitCode(), "Dependency1 should exit with code 137") + assert.Equal(t, 0, *testTask.Containers[1].GetKnownExitCode(), "Dependency2 should exit with code 0") + + close(finished) + }() + + waitFinished(t, finished, orderingTimeout) +} diff --git a/agent/engine/ordering_integ_windows_test.go b/agent/engine/ordering_integ_windows_test.go new file mode 100644 index 00000000000..295a989eb73 --- /dev/null +++ b/agent/engine/ordering_integ_windows_test.go @@ -0,0 +1,24 @@ +// +build windows,integration + +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package engine + +const ( + baseImageForOS = "microsoft/windowsservercore" +) + +var ( + entryPointForOS = []string{"powershell"} +) diff --git a/agent/engine/task_manager.go b/agent/engine/task_manager.go index 81e6ebfc0b4..853ab4fb295 100644 --- a/agent/engine/task_manager.go +++ b/agent/engine/task_manager.go @@ -47,6 +47,7 @@ const ( // and start processing the task or start another round of waiting waitForPullCredentialsTimeout = 1 * time.Minute defaultTaskSteadyStatePollInterval = 5 * time.Minute + transitionPollTime = 5 * time.Second stoppedSentWaitInterval = 30 * time.Second maxStoppedWaitTimes = 72 * time.Hour / stoppedSentWaitInterval taskUnableToTransitionToStoppedReason = "TaskStateError: Agent could not progress task's state to stopped" @@ -83,6 +84,7 @@ type acsTransition struct { type containerTransition struct { nextState apicontainerstatus.ContainerStatus actionRequired bool + blockedOn *apicontainer.DependsOn reason error } @@ -163,21 +165,21 @@ type managedTask struct { func (engine *DockerTaskEngine) newManagedTask(task *apitask.Task) *managedTask { ctx, cancel := context.WithCancel(engine.ctx) t := &managedTask{ - ctx: ctx, - cancel: cancel, - Task: task, - acsMessages: make(chan acsTransition), - dockerMessages: make(chan dockerContainerChange), - resourceStateChangeEvent: make(chan resourceStateChange), + ctx: ctx, + cancel: cancel, + Task: task, + acsMessages: make(chan acsTransition), + dockerMessages: make(chan dockerContainerChange), + resourceStateChangeEvent: make(chan resourceStateChange), engine: engine, cfg: engine.cfg, stateChangeEvents: engine.stateChangeEvents, containerChangeEventStream: engine.containerChangeEventStream, - saver: engine.saver, - credentialsManager: engine.credentialsManager, - cniClient: engine.cniClient, - taskStopWG: engine.taskStopGroup, - steadyStatePollInterval: engine.taskSteadyStatePollInterval, + saver: engine.saver, + credentialsManager: engine.credentialsManager, + cniClient: engine.cniClient, + taskStopWG: engine.taskStopGroup, + steadyStatePollInterval: engine.taskSteadyStatePollInterval, } engine.managedTasks[task.Arn] = t return t @@ -311,8 +313,14 @@ func (mtask *managedTask) waitSteady() { // steadyState returns if the task is in a steady state. Steady state is when task's desired // and known status are both RUNNING func (mtask *managedTask) steadyState() bool { - taskKnownStatus := mtask.GetKnownStatus() - return taskKnownStatus == apitaskstatus.TaskRunning && taskKnownStatus >= mtask.GetDesiredStatus() + select { + case <-mtask.ctx.Done(): + seelog.Info("Context expired. No longer steady.") + return false + default: + taskKnownStatus := mtask.GetKnownStatus() + return taskKnownStatus == apitaskstatus.TaskRunning && taskKnownStatus >= mtask.GetDesiredStatus() + } } // cleanupCredentials removes credentials for a stopped task @@ -740,16 +748,36 @@ func (mtask *managedTask) progressTask() { transitionChangeEntity <- resource.GetName() }) - anyContainerTransition, contTransitions, reasons := mtask.startContainerTransitions( + anyContainerTransition, blockedDependencies, contTransitions, reasons := mtask.startContainerTransitions( func(container *apicontainer.Container, nextStatus apicontainerstatus.ContainerStatus) { mtask.engine.transitionContainer(mtask.Task, container, nextStatus) transitionChange <- struct{}{} transitionChangeEntity <- container.Name }) - if !anyContainerTransition && !anyResourceTransition { - if !mtask.waitForExecutionCredentialsFromACS(reasons) { - mtask.onContainersUnableToTransitionState() + atLeastOneTransitionStarted := anyResourceTransition || anyContainerTransition + + blockedByOrderingDependencies := len(blockedDependencies) > 0 + + // If no transitions happened and we aren't blocked by ordering dependencies, then we are possibly in a state where + // its impossible for containers to move forward. We will do an additional check to see if we are waiting for ACS + // execution credentials. If not, then we will abort the task progression. + if !atLeastOneTransitionStarted && !blockedByOrderingDependencies { + if !mtask.isWaitingForACSExecutionCredentials(reasons) { + mtask.handleContainersUnableToTransitionState() + } + return + } + + // If no containers are starting and we are blocked on ordering dependencies, we should watch for the task to change + // over time. This will update the containers if they become healthy or stop, which makes it possible for the + // conditions "HEALTHY" and "SUCCESS" to succeed. + if !atLeastOneTransitionStarted && blockedByOrderingDependencies { + go mtask.engine.checkTaskState(mtask.Task) + ctx, cancel := context.WithTimeout(context.Background(), transitionPollTime) + defer cancel() + for timeout := mtask.waitEvent(ctx.Done()); !timeout; { + timeout = mtask.waitEvent(ctx.Done()) } return } @@ -781,9 +809,9 @@ func (mtask *managedTask) progressTask() { } } -// waitForExecutionCredentialsFromACS checks if the container that can't be transitioned +// isWaitingForACSExecutionCredentials checks if the container that can't be transitioned // was caused by waiting for credentials and start waiting -func (mtask *managedTask) waitForExecutionCredentialsFromACS(reasons []error) bool { +func (mtask *managedTask) isWaitingForACSExecutionCredentials(reasons []error) bool { for _, reason := range reasons { if reason == dependencygraph.CredentialsNotResolvedErr { seelog.Debugf("Managed task [%s]: waiting for credentials to pull from ECR", mtask.Arn) @@ -803,15 +831,19 @@ func (mtask *managedTask) waitForExecutionCredentialsFromACS(reasons []error) bo // startContainerTransitions steps through each container in the task and calls // the passed transition function when a transition should occur. -func (mtask *managedTask) startContainerTransitions(transitionFunc containerTransitionFunc) (bool, map[string]apicontainerstatus.ContainerStatus, []error) { +func (mtask *managedTask) startContainerTransitions(transitionFunc containerTransitionFunc) (bool, map[string]apicontainer.DependsOn, map[string]apicontainerstatus.ContainerStatus, []error) { anyCanTransition := false var reasons []error + blocked := make(map[string]apicontainer.DependsOn) transitions := make(map[string]apicontainerstatus.ContainerStatus) for _, cont := range mtask.Containers { transition := mtask.containerNextState(cont) if transition.reason != nil { // container can't be transitioned reasons = append(reasons, transition.reason) + if transition.blockedOn != nil { + blocked[cont.Name] = *transition.blockedOn + } continue } @@ -844,7 +876,7 @@ func (mtask *managedTask) startContainerTransitions(transitionFunc containerTran go transitionFunc(cont, transition.nextState) } - return anyCanTransition, transitions, reasons + return anyCanTransition, blocked, transitions, reasons } // startResourceTransitions steps through each resource in the task and calls @@ -956,7 +988,7 @@ func (mtask *managedTask) containerNextState(container *apicontainer.Container) reason: dependencygraph.ContainerPastDesiredStatusErr, } } - if err := dependencygraph.DependenciesAreResolved(container, mtask.Containers, + if blocked, err := dependencygraph.DependenciesAreResolved(container, mtask.Containers, mtask.Task.GetExecutionCredentialsID(), mtask.credentialsManager, mtask.GetResources()); err != nil { seelog.Debugf("Managed task [%s]: can't apply state to container [%s] yet due to unresolved dependencies: %v", mtask.Arn, container.Name, err) @@ -964,6 +996,7 @@ func (mtask *managedTask) containerNextState(container *apicontainer.Container) nextState: apicontainerstatus.ContainerStatusNone, actionRequired: false, reason: err, + blockedOn: blocked, } } @@ -1018,7 +1051,7 @@ func (mtask *managedTask) resourceNextState(resource taskresource.TaskResource) } } -func (mtask *managedTask) onContainersUnableToTransitionState() { +func (mtask *managedTask) handleContainersUnableToTransitionState() { seelog.Criticalf("Managed task [%s]: task in a bad state; it's not steadystate but no containers want to transition", mtask.Arn) if mtask.GetDesiredStatus().Terminal() { diff --git a/agent/engine/task_manager_test.go b/agent/engine/task_manager_test.go index 0f40d42b1e2..f7acc4eca9d 100644 --- a/agent/engine/task_manager_test.go +++ b/agent/engine/task_manager_test.go @@ -635,7 +635,7 @@ func TestStartContainerTransitionsWhenForwardTransitionPossible(t *testing.T) { } waitForAssertions.Add(2) - canTransition, transitions, _ := task.startContainerTransitions( + canTransition, _, transitions, _ := task.startContainerTransitions( func(cont *apicontainer.Container, nextStatus apicontainerstatus.ContainerStatus) { if cont.Name == firstContainerName { assert.Equal(t, nextStatus, apicontainerstatus.ContainerRunning, "Mismatch for first container next status") @@ -697,7 +697,7 @@ func TestStartContainerTransitionsWhenForwardTransitionIsNotPossible(t *testing. engine: &DockerTaskEngine{}, } - canTransition, transitions, _ := task.startContainerTransitions( + canTransition, _, transitions, _ := task.startContainerTransitions( func(cont *apicontainer.Container, nextStatus apicontainerstatus.ContainerStatus) { t.Error("Transition function should not be called when no transitions are possible") }) @@ -765,7 +765,7 @@ func TestStartContainerTransitionsInvokesHandleContainerChange(t *testing.T) { }() go task.waitEvent(nil) - canTransition, transitions, _ := task.startContainerTransitions( + canTransition, _, transitions, _ := task.startContainerTransitions( func(cont *apicontainer.Container, nextStatus apicontainerstatus.ContainerStatus) { t.Error("Invalid code path. The transition function should not be invoked when transitioning container from CREATED -> STOPPED") }) @@ -875,7 +875,7 @@ func TestOnContainersUnableToTransitionStateForDesiredStoppedTask(t *testing.T) eventsGenerated.Done() }() - task.onContainersUnableToTransitionState() + task.handleContainersUnableToTransitionState() eventsGenerated.Wait() assert.Equal(t, task.GetDesiredStatus(), apitaskstatus.TaskStopped) @@ -897,7 +897,7 @@ func TestOnContainersUnableToTransitionStateForDesiredRunningTask(t *testing.T) }, } - task.onContainersUnableToTransitionState() + task.handleContainersUnableToTransitionState() assert.Equal(t, task.GetDesiredStatus(), apitaskstatus.TaskStopped) assert.Equal(t, task.Containers[0].GetDesiredStatus(), apicontainerstatus.ContainerStopped) } @@ -1314,7 +1314,7 @@ func TestTaskWaitForExecutionCredentials(t *testing.T) { go func() { task.acsMessages <- acsTransition{desiredStatus: apitaskstatus.TaskRunning} }() } - assert.Equal(t, tc.result, task.waitForExecutionCredentialsFromACS(tc.errs), tc.msg) + assert.Equal(t, tc.result, task.isWaitingForACSExecutionCredentials(tc.errs), tc.msg) }) } } diff --git a/agent/engine/task_manager_unix_test.go b/agent/engine/task_manager_unix_test.go index baae542e517..b025f09ccc1 100644 --- a/agent/engine/task_manager_unix_test.go +++ b/agent/engine/task_manager_unix_test.go @@ -263,7 +263,7 @@ func TestStartResourceTransitionsEmpty(t *testing.T) { ResourcesMapUnsafe: make(map[string][]taskresource.TaskResource), DesiredStatusUnsafe: apitaskstatus.TaskRunning, }, - ctx: ctx, + ctx: ctx, resourceStateChangeEvent: make(chan resourceStateChange), } mtask.Task.AddResource("cgroup", res) diff --git a/agent/functional_tests/testdata/simpletests_unix/link-volume-depencies.json b/agent/functional_tests/testdata/simpletests_unix/link-volume-depencies.json index 51d9a0b9320..651f46cc0d2 100644 --- a/agent/functional_tests/testdata/simpletests_unix/link-volume-depencies.json +++ b/agent/functional_tests/testdata/simpletests_unix/link-volume-depencies.json @@ -3,7 +3,7 @@ "Description": "Tests that the dependency graph of task definitions is resolved correctly", "TaskDefinition": "network-link-2", "Version": ">=1.0.0", - "Timeout": "2m", + "Timeout": "5m", "ExitCodes": { "exit": 42 } diff --git a/agent/functional_tests/testdata/simpletests_windows/datavolume-windows.json b/agent/functional_tests/testdata/simpletests_windows/datavolume-windows.json index f22689ea320..d3c27e99775 100644 --- a/agent/functional_tests/testdata/simpletests_windows/datavolume-windows.json +++ b/agent/functional_tests/testdata/simpletests_windows/datavolume-windows.json @@ -3,7 +3,7 @@ "Description": "Check that basic data volumes work", "TaskDefinition": "datavolume-windows", "Version": ">=1.0.0", - "Timeout": "2m", + "Timeout": "5m", "ExitCodes": { "exit": 42 } diff --git a/agent/functional_tests/testdata/taskdefinitions/awslogs-datetime-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/awslogs-datetime-windows/task-definition.json index f503fb39098..eb18daff644 100644 --- a/agent/functional_tests/testdata/taskdefinitions/awslogs-datetime-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/awslogs-datetime-windows/task-definition.json @@ -5,7 +5,7 @@ "memory": 512, "name": "awslogs-datetime-windows", "cpu": 1024, - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "entryPoint": ["powershell"], "command": ["echo", "\"May 01, 2017 19:00:01 ECS\nMay 01, 2017 19:00:04 Agent\nRunning\nin the instance\""], "logConfiguration": { diff --git a/agent/functional_tests/testdata/taskdefinitions/awslogs-multiline-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/awslogs-multiline-windows/task-definition.json index 415b57fe2f8..e89eb61093e 100644 --- a/agent/functional_tests/testdata/taskdefinitions/awslogs-multiline-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/awslogs-multiline-windows/task-definition.json @@ -5,7 +5,7 @@ "memory": 512, "name": "awslogs-multiline-windows", "cpu": 1024, - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "entryPoint": ["powershell"], "command": ["echo", "\"INFO: ECS Agent\nRunning\nINFO: Instance\""], "logConfiguration": { diff --git a/agent/functional_tests/testdata/taskdefinitions/awslogs-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/awslogs-windows/task-definition.json index 344211c1fd5..66661f8a7c2 100644 --- a/agent/functional_tests/testdata/taskdefinitions/awslogs-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/awslogs-windows/task-definition.json @@ -5,7 +5,7 @@ "memory": 512, "name": "awslogs", "cpu": 1024, - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "logConfiguration": { "logDriver": "awslogs", "options": { diff --git a/agent/functional_tests/testdata/taskdefinitions/cleanup-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/cleanup-windows/task-definition.json index 91ab3c575ee..87c38348d16 100644 --- a/agent/functional_tests/testdata/taskdefinitions/cleanup-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/cleanup-windows/task-definition.json @@ -1,7 +1,7 @@ { "family": "ecsftest-cleanup-windows", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "cleanup-windows", "cpu": 1024, "memory": 512, diff --git a/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator-windows/task-definition.json index 04862dbf8b4..40e03b946cd 100644 --- a/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator-windows/task-definition.json @@ -24,4 +24,4 @@ } }] } - \ No newline at end of file + diff --git a/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator/task-definition.json index 92e29ec7577..ca136a9a8e8 100644 --- a/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/container-metadata-file-validator/task-definition.json @@ -21,4 +21,4 @@ } }] } - \ No newline at end of file + diff --git a/agent/functional_tests/testdata/taskdefinitions/datavolume-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/datavolume-windows/task-definition.json index 85e56bac546..4d6175c19be 100644 --- a/agent/functional_tests/testdata/taskdefinitions/datavolume-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/datavolume-windows/task-definition.json @@ -5,7 +5,7 @@ "host": {} }], "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "exit", "cpu": 512, "memory": 256, @@ -15,7 +15,7 @@ }], "command": ["powershell", "-c", "while (1) { sleep 1; if (test-path \"C:/data/success\") { exit 42 }}; done"] }, { - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "dataSource", "cpu": 512, "memory": 256, @@ -25,7 +25,7 @@ }], "command": ["powershell", "-c", "New-Item -ItemType file C:/data/success"] }, { - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "data-volume-source", "cpu": 512, "memory": 256, @@ -34,6 +34,6 @@ "sourceVolume": "test", "containerPath": "C:/data/" }], - "command": ["data volumes shouldn't need to run"] + "command": ["powershell", "-c", "exit"] }] } diff --git a/agent/functional_tests/testdata/taskdefinitions/hostname-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/hostname-windows/task-definition.json index 891e029201e..a1dd5c4cfcb 100644 --- a/agent/functional_tests/testdata/taskdefinitions/hostname-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/hostname-windows/task-definition.json @@ -1,7 +1,7 @@ { "family": "ecsftest-hostname", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "exit", "cpu": 1024, "memory": 512, diff --git a/agent/functional_tests/testdata/taskdefinitions/labels-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/labels-windows/task-definition.json index 3672df20cee..0672c2cb9a1 100644 --- a/agent/functional_tests/testdata/taskdefinitions/labels-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/labels-windows/task-definition.json @@ -1,7 +1,7 @@ { "family": "ecsftest-labels-windows", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "labeled", "cpu": 1024, "memory": 512, diff --git a/agent/functional_tests/testdata/taskdefinitions/logdriver-jsonfile-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/logdriver-jsonfile-windows/task-definition.json index 6e608d52c80..2bf70504d9f 100644 --- a/agent/functional_tests/testdata/taskdefinitions/logdriver-jsonfile-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/logdriver-jsonfile-windows/task-definition.json @@ -1,7 +1,7 @@ { "family": "ecsinteg-json-file-rollover", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "exit", "memory": 512, "cpu": 1024, diff --git a/agent/functional_tests/testdata/taskdefinitions/network-mode-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/network-mode-windows/task-definition.json index 6b381ceb38d..a8835b9d404 100644 --- a/agent/functional_tests/testdata/taskdefinitions/network-mode-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/network-mode-windows/task-definition.json @@ -2,7 +2,7 @@ "family": "ecsftest-networkmode", "networkMode": "$$$$NETWORK_MODE$$$$", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "entryPoint": ["powershell"], "command": ["sleep", "60"], "name": "network-$$$$NETWORK_MODE$$$$", diff --git a/agent/functional_tests/testdata/taskdefinitions/oom-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/oom-windows/task-definition.json index caa1bd420c1..52be1f38851 100644 --- a/agent/functional_tests/testdata/taskdefinitions/oom-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/oom-windows/task-definition.json @@ -5,8 +5,8 @@ "memory": 256, "name": "memory-overcommit", "cpu": 512, - "image": "amazon/amazon-ecs-windows-python:make", - "command": ["python", "-c", "import time; time.sleep(30); foo=' '*1024*1024*1024;"] + "image": "amazon/amazon-ecs-windows-telemetry-test:make", + "command": ["-concurrency", "10", "-memory", "1024"] }] } diff --git a/agent/functional_tests/testdata/taskdefinitions/savedstate-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/savedstate-windows/task-definition.json index ec831ba5cac..cb9fc368e9a 100644 --- a/agent/functional_tests/testdata/taskdefinitions/savedstate-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/savedstate-windows/task-definition.json @@ -1,7 +1,7 @@ { "family": "ecsftest-savedstate-windows", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "savedstate-windows", "cpu": 1024, "memory": 512, diff --git a/agent/functional_tests/testdata/taskdefinitions/secrets-environment-variables/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/secrets-environment-variables/task-definition.json index f98daf3a07e..d366d64f21e 100644 --- a/agent/functional_tests/testdata/taskdefinitions/secrets-environment-variables/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/secrets-environment-variables/task-definition.json @@ -17,4 +17,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/agent/functional_tests/testdata/taskdefinitions/simple-exit-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/simple-exit-windows/task-definition.json index b5dbec1a275..9d10d3647d0 100644 --- a/agent/functional_tests/testdata/taskdefinitions/simple-exit-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/simple-exit-windows/task-definition.json @@ -1,7 +1,7 @@ { "family": "ecsinteg-simple-exit-windows", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "exit", "cpu": 1024, "memory": 512, diff --git a/agent/functional_tests/testdata/taskdefinitions/task-elastic-inference/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/task-elastic-inference/task-definition.json index 61938d975d8..4a9fa4b1b2b 100644 --- a/agent/functional_tests/testdata/taskdefinitions/task-elastic-inference/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/task-elastic-inference/task-definition.json @@ -23,4 +23,4 @@ "deviceType": "eia1.medium" } ] -} \ No newline at end of file +} diff --git a/agent/functional_tests/testdata/taskdefinitions/task-local-vol-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/task-local-vol-windows/task-definition.json index 056b06c4533..bb69a9f7be2 100644 --- a/agent/functional_tests/testdata/taskdefinitions/task-local-vol-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/task-local-vol-windows/task-definition.json @@ -2,7 +2,7 @@ "family": "ecsftest-task-local-volume", "containerDefinitions": [ { - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "exit", "cpu": 512, "memory": 256, diff --git a/agent/functional_tests/testdata/taskdefinitions/task-local-vol/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/task-local-vol/task-definition.json index f83b54c495e..045c670525d 100644 --- a/agent/functional_tests/testdata/taskdefinitions/task-local-vol/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/task-local-vol/task-definition.json @@ -32,4 +32,4 @@ } } ] -} \ No newline at end of file +} diff --git a/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-read-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-read-windows/task-definition.json index 64e34d7bf3d..19723e0cd24 100644 --- a/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-read-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-read-windows/task-definition.json @@ -3,7 +3,7 @@ "containerDefinitions": [ { "name": "task-shared-vol-read", - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "cpu": 512, "memory": 256, "essential": true, diff --git a/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-write-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-write-windows/task-definition.json index dae08a148b9..a7aa9d288de 100644 --- a/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-write-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/task-shared-vol-write-windows/task-definition.json @@ -2,7 +2,7 @@ "family": "ecsftest-task-local-volume", "containerDefinitions": [ { - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "task-shared-vol-write-windows", "cpu": 512, "memory": 256, diff --git a/agent/functional_tests/testdata/taskdefinitions/working-dir-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/working-dir-windows/task-definition.json index 10c584389ab..87ea6631c2c 100644 --- a/agent/functional_tests/testdata/taskdefinitions/working-dir-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/working-dir-windows/task-definition.json @@ -1,7 +1,7 @@ { "family": "ecsftest-working-dir-windows", "containerDefinitions": [{ - "image": "microsoft/windowsservercore:latest", + "image": "microsoft/windowsservercore", "name": "exit", "cpu": 1024, "memory": 512, diff --git a/agent/functional_tests/tests/generated/simpletests_unix/simpletests_generated_unix_test.go b/agent/functional_tests/tests/generated/simpletests_unix/simpletests_generated_unix_test.go index 8778e55f4cb..5fc385a0676 100644 --- a/agent/functional_tests/tests/generated/simpletests_unix/simpletests_generated_unix_test.go +++ b/agent/functional_tests/tests/generated/simpletests_unix/simpletests_generated_unix_test.go @@ -735,7 +735,7 @@ func TestLinkVolumeDependencies(t *testing.T) { } } - timeout, err := time.ParseDuration("2m") + timeout, err := time.ParseDuration("5m") if err != nil { t.Fatalf("Could not parse timeout: %#v", err) } diff --git a/agent/functional_tests/tests/generated/simpletests_windows/simpletests_generated_windows_test.go b/agent/functional_tests/tests/generated/simpletests_windows/simpletests_generated_windows_test.go index b9a49e75300..e27222e9d28 100644 --- a/agent/functional_tests/tests/generated/simpletests_windows/simpletests_generated_windows_test.go +++ b/agent/functional_tests/tests/generated/simpletests_windows/simpletests_generated_windows_test.go @@ -63,7 +63,7 @@ func TestDataVolume(t *testing.T) { } } - timeout, err := time.ParseDuration("2m") + timeout, err := time.ParseDuration("5m") if err != nil { t.Fatalf("Could not parse timeout: %#v", err) } diff --git a/agent/functional_tests/util/utils_windows.go b/agent/functional_tests/util/utils_windows.go index f1dfcbe0df7..4bd388b32b7 100644 --- a/agent/functional_tests/util/utils_windows.go +++ b/agent/functional_tests/util/utils_windows.go @@ -93,6 +93,7 @@ func RunAgent(t *testing.T, options *AgentOptions) *TestAgent { os.Setenv("ECS_AUDIT_LOGFILE", logdir+"/audit.log") os.Setenv("ECS_LOGLEVEL", "debug") os.Setenv("ECS_AVAILABLE_LOGGING_DRIVERS", `["json-file","awslogs"]`) + os.Setenv("ECS_IMAGE_PULL_BEHAVIOR", "prefer-cached") t.Log("datadir", datadir) os.Mkdir(logdir, 0755) diff --git a/agent/statemanager/state_manager.go b/agent/statemanager/state_manager.go index 999bbb28875..86f731ae38d 100644 --- a/agent/statemanager/state_manager.go +++ b/agent/statemanager/state_manager.go @@ -78,7 +78,11 @@ const ( // a) Add 'Associations' field to 'api.task.task' // b) Add 'GPUIDs' field to 'apicontainer.Container' // c) Add 'NvidiaRuntime' field to 'api.task.task' - ECSDataVersion = 19 + // 20) + // a) Add 'DependsOn' field to 'apicontainer.Container' + // b) Add 'StartTime' field to 'api.container.Container' + // c) Add 'StopTime' field to 'api.container.Container' + ECSDataVersion = 20 // ecsDataFile specifies the filename in the ECS_DATADIR ecsDataFile = "ecs_agent_data.json" diff --git a/agent/statemanager/state_manager_test.go b/agent/statemanager/state_manager_test.go index 7291487b9ff..3dfe4332a7b 100644 --- a/agent/statemanager/state_manager_test.go +++ b/agent/statemanager/state_manager_test.go @@ -326,3 +326,74 @@ func TestLoadsDataForASMSecretsTask(t *testing.T) { assert.Equal(t, "secret-value-from", secret.ValueFrom) assert.Equal(t, "asm", secret.Provider) } + +// verify that the state manager correctly loads container ordering related fields in state file +func TestLoadsDataForContainerOrdering(t *testing.T) { + cleanup, err := setupWindowsTest(filepath.Join(".", "testdata", "v20", "containerOrdering", "ecs_agent_data.json")) + require.Nil(t, err, "Failed to set up test") + defer cleanup() + cfg := &config.Config{DataDir: filepath.Join(".", "testdata", "v20", "containerOrdering")} + taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewTaskEngineState(), nil, nil) + var containerInstanceArn, cluster, savedInstanceID string + var sequenceNumber int64 + stateManager, err := statemanager.NewStateManager(cfg, + statemanager.AddSaveable("TaskEngine", taskEngine), + statemanager.AddSaveable("ContainerInstanceArn", &containerInstanceArn), + statemanager.AddSaveable("Cluster", &cluster), + statemanager.AddSaveable("EC2InstanceID", &savedInstanceID), + statemanager.AddSaveable("SeqNum", &sequenceNumber), + ) + assert.NoError(t, err) + err = stateManager.Load() + assert.NoError(t, err) + assert.Equal(t, "state-file", cluster) + assert.EqualValues(t, 0, sequenceNumber) + + tasks, err := taskEngine.ListTasks() + assert.NoError(t, err) + assert.Equal(t, 1, len(tasks)) + + task := tasks[0] + assert.Equal(t, "arn:aws:ecs:us-west-2:1234567890:task/33425c99-5db7-45fb-8244-bc94d00661e4", task.Arn) + assert.Equal(t, "container-ordering-state", task.Family) + assert.Equal(t, 2, len(task.Containers)) + + dependsOn := task.Containers[1].DependsOn + assert.Equal(t, "container_1", dependsOn[0].ContainerName) + assert.Equal(t, "START", dependsOn[0].Condition) +} + +func TestLoadsDataForPerContainerTimeouts(t *testing.T) { + cleanup, err := setupWindowsTest(filepath.Join(".", "testdata", "v20", "perContainerTimeouts", "ecs_agent_data.json")) + require.Nil(t, err, "Failed to set up test") + defer cleanup() + cfg := &config.Config{DataDir: filepath.Join(".", "testdata", "v20", "perContainerTimeouts")} + taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewTaskEngineState(), nil, nil) + var containerInstanceArn, cluster, savedInstanceID string + var sequenceNumber int64 + stateManager, err := statemanager.NewStateManager(cfg, + statemanager.AddSaveable("TaskEngine", taskEngine), + statemanager.AddSaveable("ContainerInstanceArn", &containerInstanceArn), + statemanager.AddSaveable("Cluster", &cluster), + statemanager.AddSaveable("EC2InstanceID", &savedInstanceID), + statemanager.AddSaveable("SeqNum", &sequenceNumber), + ) + assert.NoError(t, err) + err = stateManager.Load() + assert.NoError(t, err) + assert.Equal(t, "state-file", cluster) + assert.EqualValues(t, 0, sequenceNumber) + + tasks, err := taskEngine.ListTasks() + assert.NoError(t, err) + assert.Equal(t, 1, len(tasks)) + + task := tasks[0] + assert.Equal(t, "arn:aws:ecs:us-west-2:1234567890:task/33425c99-5db7-45fb-8244-bc94d00661e4", task.Arn) + assert.Equal(t, "per-container-timeouts", task.Family) + assert.Equal(t, 1, len(task.Containers)) + + c1 := task.Containers[0] + assert.Equal(t, uint(10), c1.StartTimeout) + assert.Equal(t, uint(10), c1.StopTimeout) +} diff --git a/agent/statemanager/testdata/v20/containerOrdering/ecs_agent_data.json b/agent/statemanager/testdata/v20/containerOrdering/ecs_agent_data.json new file mode 100644 index 00000000000..38a27246687 --- /dev/null +++ b/agent/statemanager/testdata/v20/containerOrdering/ecs_agent_data.json @@ -0,0 +1,259 @@ +{ + "Data": { + "Cluster": "state-file", + "ContainerInstanceArn": "arn:aws:ecs:us-west-2:1234567890:container-instance/46efd519-df3f-4096-8f34-faebb1747752", + "EC2InstanceID": "i-0da29eb1a8a98768b", + "TaskEngine": { + "Tasks": [ + { + "Arn": "arn:aws:ecs:us-west-2:1234567890:task/33425c99-5db7-45fb-8244-bc94d00661e4", + "Family": "container-ordering-state", + "Version": "1", + "Containers": [ + { + "Name": "container_1", + "Image": "amazonlinux:1", + "ImageID": "sha256:7f929d2604c7e504a568eac9a2523c1b9e9b15e1fcee4076e1411a552913d08e", + "Command": [ + "sleep", + "3600" + ], + "Cpu": 0, + "GPUIDs": ["0", "1"], + "Memory": 512, + "Links": null, + "volumesFrom": [], + "mountPoints": [], + "portMappings": [], + "Essential": true, + "EntryPoint": null, + "environment": { + "NVIDIA_VISIBLE_DEVICES": "0,1" + }, + "overrides": { + "command": null + }, + "dockerConfig": { + "config": "{}", + "hostConfig": "{\"CapAdd\":[],\"CapDrop\":[]}", + "version": "1.17" + }, + "registryAuthentication": null, + "secrets": [], + "LogsAuthStrategy": "", + "desiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "TransitionDependencySet": { + "1": { + "ContainerDependencies": null, + "ResourceDependencies": [ + { + "Name": "cgroup", + "RequiredStatus": 1 + } + ] + } + }, + "RunDependencies": null, + "IsInternal": "NORMAL", + "ApplyingError": { + "error": "API error (500): Get https://registry-1.docker.io/v2/library/amazonlinux/manifests/1: toomanyrequests: too many failed login attempts for username or IP address\n", + "name": "CannotPullContainerError" + }, + "SentStatus": "RUNNING", + "metadataFileUpdated": false, + "KnownExitCode": null, + "KnownPortBindings": null + }, + { + "Name": "container_2", + "DependsOn": [ + { + "ContainerName":"container_1", + "Condition":"START" + } + ], + "Image": "amazonlinux:1", + "ImageID": "sha256:7f929d2604c7e504a568eac9a2523c1b9e9b15e1fcee4076e1411a552913d08e", + "Command": [ + "sleep", + "3600" + ], + "Cpu": 0, + "GPUIDs": [], + "Memory": 512, + "Links": null, + "volumesFrom": [], + "mountPoints": [], + "portMappings": [], + "Essential": true, + "EntryPoint": null, + "environment": {}, + "overrides": { + "command": null + }, + "dockerConfig": { + "config": "{}", + "hostConfig": "{\"CapAdd\":[],\"CapDrop\":[]}", + "version": "1.17" + }, + "registryAuthentication": null, + "secrets": [], + "LogsAuthStrategy": "", + "desiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "TransitionDependencySet": { + "1": { + "ContainerDependencies": null, + "ResourceDependencies": [ + { + "Name": "cgroup", + "RequiredStatus": 1 + } + ] + } + }, + "RunDependencies": null, + "IsInternal": "NORMAL", + "ApplyingError": { + "error": "API error (500): Get https://registry-1.docker.io/v2/library/amazonlinux/manifests/1: toomanyrequests: too many failed login attempts for username or IP address\n", + "name": "CannotPullContainerError" + }, + "SentStatus": "RUNNING", + "metadataFileUpdated": false, + "KnownExitCode": null, + "KnownPortBindings": null + } + ], + "Associations": [ + { + "Containers": ["container_1"], + "Content": { + "Encoding": "base64", + "Value": "val" + }, + "Name": "0", + "Type": "gpu" + }, + { + "Containers": ["container_1"], + "Content": { + "Encoding": "base64", + "Value": "val" + }, + "Name": "1", + "Type": "gpu" + } + ], + "resources": { + "cgroup": [ + { + "cgroupRoot": "/ecs/33425c99-5db7-45fb-8244-bc94d00661e4", + "cgroupMountPath": "/sys/fs/cgroup", + "createdAt": "0001-01-01T00:00:00Z", + "desiredStatus": "CREATED", + "knownStatus": "CREATED", + "resourceSpec": { + "cpu": { + "shares": 2 + } + } + } + ] + }, + "volumes": [], + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "KnownTime": "2018-10-04T18:05:49.121835686Z", + "PullStartedAt": "2018-10-04T18:05:34.359798761Z", + "PullStoppedAt": "2018-10-04T18:05:48.445985904Z", + "ExecutionStoppedAt": "0001-01-01T00:00:00Z", + "SentStatus": "RUNNING", + "StartSequenceNumber": 2, + "StopSequenceNumber": 0, + "executionCredentialsID": "b1a6ede6-1a9f-4ab3-a02e-bd3e51b11244", + "ENI": null, + "MemoryCPULimitsEnabled": true, + "PlatformFields": {} + } + ], + "IdToContainer": { + "8f5e6e3091f221c876103289ddabcbcdeb64acd7ac7e2d0cf4da2be2be9d8956": { + "DockerId": "8f5e6e3091f221c876103289ddabcbcdeb64acd7ac7e2d0cf4da2be2be9d8956", + "DockerName": "ecs-private-registry-state-1-container1-a68ef4b6e0fba38d3500", + "Container": { + "Name": "container_1", + "Image": "amazonlinux:1", + "ImageID": "sha256:7f929d2604c7e504a568eac9a2523c1b9e9b15e1fcee4076e1411a552913d08e", + "Command": [ + "sleep", + "3600" + ], + "Cpu": 0, + "Memory": 512, + "Links": null, + "volumesFrom": [], + "mountPoints": [], + "portMappings": [], + "Essential": true, + "EntryPoint": null, + "environment": {}, + "overrides": { + "command": null + }, + "dockerConfig": { + "config": "{}", + "hostConfig": "{\"CapAdd\":[],\"CapDrop\":[]}", + "version": "1.17" + }, + "registryAuthentication": null, + "LogsAuthStrategy": "", + "desiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "TransitionDependencySet": { + "1": { + "ContainerDependencies": null, + "ResourceDependencies": [ + { + "Name": "cgroup", + "RequiredStatus": 1 + } + ] + } + }, + "RunDependencies": null, + "IsInternal": "NORMAL", + "ApplyingError": { + "error": "API error (500): Get https://registry-1.docker.io/v2/library/amazonlinux/manifests/1: toomanyrequests: too many failed login attempts for username or IP address\n", + "name": "CannotPullContainerError" + }, + "SentStatus": "RUNNING", + "metadataFileUpdated": false, + "KnownExitCode": null, + "KnownPortBindings": null + } + } + }, + "IdToTask": { + "8f5e6e3091f221c876103289ddabcbcdeb64acd7ac7e2d0cf4da2be2be9d8956": "arn:aws:ecs:us-west-2:1234567890:task/33425c99-5db7-45fb-8244-bc94d00661e4" + }, + "ImageStates": [ + { + "Image": { + "ImageID": "sha256:7f929d2604c7e504a568eac9a2523c1b9e9b15e1fcee4076e1411a552913d08e", + "Names": [ + "amazonlinux:1" + ], + "Size": 165452304 + }, + "PulledAt": "2018-10-04T18:05:48.445644088Z", + "LastUsedAt": "2018-10-04T18:05:48.445645342Z", + "PullSucceeded": false + } + ], + "ENIAttachments": null, + "IPToTask": {} + } + }, + "Version": 20 +} diff --git a/agent/statemanager/testdata/v20/perContainerTimeouts/ecs_agent_data.json b/agent/statemanager/testdata/v20/perContainerTimeouts/ecs_agent_data.json new file mode 100644 index 00000000000..459f2bd3cbe --- /dev/null +++ b/agent/statemanager/testdata/v20/perContainerTimeouts/ecs_agent_data.json @@ -0,0 +1,202 @@ +{ + "Data": { + "Cluster": "state-file", + "ContainerInstanceArn": "arn:aws:ecs:us-west-2:1234567890:container-instance/46efd519-df3f-4096-8f34-faebb1747752", + "EC2InstanceID": "i-0da29eb1a8a98768b", + "TaskEngine": { + "Tasks": [ + { + "Arn": "arn:aws:ecs:us-west-2:1234567890:task/33425c99-5db7-45fb-8244-bc94d00661e4", + "Family": "per-container-timeouts", + "Version": "1", + "Containers": [ + { + "Name": "container_1", + "Image": "amazonlinux:1", + "ImageID": "sha256:7f929d2604c7e504a568eac9a2523c1b9e9b15e1fcee4076e1411a552913d08e", + "Command": [ + "sleep", + "3600" + ], + "Cpu": 0, + "GPUIDs": ["0", "1"], + "Memory": 512, + "Links": null, + "volumesFrom": [], + "mountPoints": [], + "portMappings": [], + "Essential": true, + "StartTimeout":10, + "StopTimeout":10, + "EntryPoint": null, + "environment": { + "NVIDIA_VISIBLE_DEVICES": "0,1" + }, + "overrides": { + "command": null + }, + "dockerConfig": { + "config": "{}", + "hostConfig": "{\"CapAdd\":[],\"CapDrop\":[]}", + "version": "1.17" + }, + "registryAuthentication": null, + "secrets": [], + "LogsAuthStrategy": "", + "desiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "TransitionDependencySet": { + "1": { + "ContainerDependencies": null, + "ResourceDependencies": [ + { + "Name": "cgroup", + "RequiredStatus": 1 + } + ] + } + }, + "RunDependencies": null, + "IsInternal": "NORMAL", + "ApplyingError": { + "error": "API error (500): Get https://registry-1.docker.io/v2/library/amazonlinux/manifests/1: toomanyrequests: too many failed login attempts for username or IP address\n", + "name": "CannotPullContainerError" + }, + "SentStatus": "RUNNING", + "metadataFileUpdated": false, + "KnownExitCode": null, + "KnownPortBindings": null + } + ], + "Associations": [ + { + "Containers": ["container_1"], + "Content": { + "Encoding": "base64", + "Value": "val" + }, + "Name": "0", + "Type": "gpu" + }, + { + "Containers": ["container_1"], + "Content": { + "Encoding": "base64", + "Value": "val" + }, + "Name": "1", + "Type": "gpu" + } + ], + "resources": { + "cgroup": [ + { + "cgroupRoot": "/ecs/33425c99-5db7-45fb-8244-bc94d00661e4", + "cgroupMountPath": "/sys/fs/cgroup", + "createdAt": "0001-01-01T00:00:00Z", + "desiredStatus": "CREATED", + "knownStatus": "CREATED", + "resourceSpec": { + "cpu": { + "shares": 2 + } + } + } + ] + }, + "volumes": [], + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "KnownTime": "2018-10-04T18:05:49.121835686Z", + "PullStartedAt": "2018-10-04T18:05:34.359798761Z", + "PullStoppedAt": "2018-10-04T18:05:48.445985904Z", + "ExecutionStoppedAt": "0001-01-01T00:00:00Z", + "SentStatus": "RUNNING", + "StartSequenceNumber": 2, + "StopSequenceNumber": 0, + "executionCredentialsID": "b1a6ede6-1a9f-4ab3-a02e-bd3e51b11244", + "ENI": null, + "MemoryCPULimitsEnabled": true, + "PlatformFields": {} + } + ], + "IdToContainer": { + "8f5e6e3091f221c876103289ddabcbcdeb64acd7ac7e2d0cf4da2be2be9d8956": { + "DockerId": "8f5e6e3091f221c876103289ddabcbcdeb64acd7ac7e2d0cf4da2be2be9d8956", + "DockerName": "ecs-private-registry-state-1-container1-a68ef4b6e0fba38d3500", + "Container": { + "Name": "container_1", + "Image": "amazonlinux:1", + "ImageID": "sha256:7f929d2604c7e504a568eac9a2523c1b9e9b15e1fcee4076e1411a552913d08e", + "Command": [ + "sleep", + "3600" + ], + "Cpu": 0, + "Memory": 512, + "Links": null, + "volumesFrom": [], + "mountPoints": [], + "portMappings": [], + "Essential": true, + "EntryPoint": null, + "environment": {}, + "overrides": { + "command": null + }, + "dockerConfig": { + "config": "{}", + "hostConfig": "{\"CapAdd\":[],\"CapDrop\":[]}", + "version": "1.17" + }, + "registryAuthentication": null, + "LogsAuthStrategy": "", + "desiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "TransitionDependencySet": { + "1": { + "ContainerDependencies": null, + "ResourceDependencies": [ + { + "Name": "cgroup", + "RequiredStatus": 1 + } + ] + } + }, + "RunDependencies": null, + "IsInternal": "NORMAL", + "ApplyingError": { + "error": "API error (500): Get https://registry-1.docker.io/v2/library/amazonlinux/manifests/1: toomanyrequests: too many failed login attempts for username or IP address\n", + "name": "CannotPullContainerError" + }, + "SentStatus": "RUNNING", + "metadataFileUpdated": false, + "KnownExitCode": null, + "KnownPortBindings": null + } + } + }, + "IdToTask": { + "8f5e6e3091f221c876103289ddabcbcdeb64acd7ac7e2d0cf4da2be2be9d8956": "arn:aws:ecs:us-west-2:1234567890:task/33425c99-5db7-45fb-8244-bc94d00661e4" + }, + "ImageStates": [ + { + "Image": { + "ImageID": "sha256:7f929d2604c7e504a568eac9a2523c1b9e9b15e1fcee4076e1411a552913d08e", + "Names": [ + "amazonlinux:1" + ], + "Size": 165452304 + }, + "PulledAt": "2018-10-04T18:05:48.445644088Z", + "LastUsedAt": "2018-10-04T18:05:48.445645342Z", + "PullSucceeded": false + } + ], + "ENIAttachments": null, + "IPToTask": {} + } + }, + "Version": 20 +} diff --git a/agent/stats/common_test.go b/agent/stats/common_test.go index 1a3db782a75..652eeaf0a4e 100644 --- a/agent/stats/common_test.go +++ b/agent/stats/common_test.go @@ -63,6 +63,7 @@ var ( func init() { cfg.EngineAuthData = config.NewSensitiveRawMessage([]byte{}) + cfg.ImagePullBehavior = config.ImagePullPreferCachedBehavior } // eventStream returns the event stream used to receive container change events diff --git a/misc/container-health-windows/windows.dockerfile b/misc/container-health-windows/windows.dockerfile index 83c1f8214cc..3500f4ed529 100644 --- a/misc/container-health-windows/windows.dockerfile +++ b/misc/container-health-windows/windows.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/container-metadata-file-validator-windows/container-metadata-file-validator-windows.dockerfile b/misc/container-metadata-file-validator-windows/container-metadata-file-validator-windows.dockerfile index 68d01abdc6c..3391340e615 100644 --- a/misc/container-metadata-file-validator-windows/container-metadata-file-validator-windows.dockerfile +++ b/misc/container-metadata-file-validator-windows/container-metadata-file-validator-windows.dockerfile @@ -11,6 +11,6 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/nanoserver:latest +FROM microsoft/windowsservercore ADD container-metadata-file-validator-windows.exe container-metadata-file-validator-windows.exe diff --git a/misc/container-metadata-file-validator-windows/setup-container-metadata-file-validator.ps1 b/misc/container-metadata-file-validator-windows/setup-container-metadata-file-validator.ps1 index bb9fada1201..35c8ce1b1fb 100644 --- a/misc/container-metadata-file-validator-windows/setup-container-metadata-file-validator.ps1 +++ b/misc/container-metadata-file-validator-windows/setup-container-metadata-file-validator.ps1 @@ -14,23 +14,7 @@ $oldPref = $ErrorActionPreference $ErrorActionPreference = 'Stop' -Invoke-Expression ${PSScriptRoot}\..\windows-deploy\hostsetup.ps1 - # Create amazon/amazon-ecs-container-metadata-file-validator-windows for tests -$buildscript = @" -mkdir C:\md -cp C:\ecs\container-metadata-file-validator-windows.go C:\md -go build -o C:\md\container-metadata-file-validator-windows.exe C:\md\container-metadata-file-validator-windows.go -cp C:\md\container-metadata-file-validator-windows.exe C:\ecs -"@ - -$buildimage="golang:1.7-nanoserver" -docker pull $buildimage - -docker run ` - --volume ${PSScriptRoot}:C:\ecs ` - $buildimage ` - powershell ${buildscript} - +Invoke-Expression "go build -o ${PSScriptRoot}\container-metadata-file-validator-windows.exe ${PSScriptRoot}\container-metadata-file-validator-windows.go" Invoke-Expression "docker build -t amazon/amazon-ecs-container-metadata-file-validator-windows --file ${PSScriptRoot}\container-metadata-file-validator-windows.dockerfile ${PSScriptRoot}" $ErrorActionPreference = $oldPref diff --git a/misc/image-cleanup-test-images/windows0.dockerfile b/misc/image-cleanup-test-images/windows0.dockerfile index 0c74c1e1437..795a83eee69 100644 --- a/misc/image-cleanup-test-images/windows0.dockerfile +++ b/misc/image-cleanup-test-images/windows0.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows1.dockerfile b/misc/image-cleanup-test-images/windows1.dockerfile index 87ea24df410..0eea3f3b725 100644 --- a/misc/image-cleanup-test-images/windows1.dockerfile +++ b/misc/image-cleanup-test-images/windows1.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows2.dockerfile b/misc/image-cleanup-test-images/windows2.dockerfile index eecb823eac3..402fc7a0972 100644 --- a/misc/image-cleanup-test-images/windows2.dockerfile +++ b/misc/image-cleanup-test-images/windows2.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows3.dockerfile b/misc/image-cleanup-test-images/windows3.dockerfile index 0467862d05d..1e46fd28a99 100644 --- a/misc/image-cleanup-test-images/windows3.dockerfile +++ b/misc/image-cleanup-test-images/windows3.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows4.dockerfile b/misc/image-cleanup-test-images/windows4.dockerfile index badb152dad6..66687cefe5a 100644 --- a/misc/image-cleanup-test-images/windows4.dockerfile +++ b/misc/image-cleanup-test-images/windows4.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows5.dockerfile b/misc/image-cleanup-test-images/windows5.dockerfile index 73e513aac2b..96528688811 100644 --- a/misc/image-cleanup-test-images/windows5.dockerfile +++ b/misc/image-cleanup-test-images/windows5.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows6.dockerfile b/misc/image-cleanup-test-images/windows6.dockerfile index 2a925dbfc7c..7e5f49366e7 100644 --- a/misc/image-cleanup-test-images/windows6.dockerfile +++ b/misc/image-cleanup-test-images/windows6.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows7.dockerfile b/misc/image-cleanup-test-images/windows7.dockerfile index c42ca9c428a..7b8115a08fb 100644 --- a/misc/image-cleanup-test-images/windows7.dockerfile +++ b/misc/image-cleanup-test-images/windows7.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows8.dockerfile b/misc/image-cleanup-test-images/windows8.dockerfile index f87c4367476..9c889a22919 100644 --- a/misc/image-cleanup-test-images/windows8.dockerfile +++ b/misc/image-cleanup-test-images/windows8.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/image-cleanup-test-images/windows9.dockerfile b/misc/image-cleanup-test-images/windows9.dockerfile index 8b243414b5e..b9a069ca3b2 100644 --- a/misc/image-cleanup-test-images/windows9.dockerfile +++ b/misc/image-cleanup-test-images/windows9.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/netkitten/build.ps1 b/misc/netkitten/build.ps1 index cf053df214d..7b49f124fc8 100644 --- a/misc/netkitten/build.ps1 +++ b/misc/netkitten/build.ps1 @@ -11,16 +11,5 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -$buildscript = @" -mkdir C:\nk -cp C:\netkitten\netkitten.go C:\nk -go build -o C:\nk\netkitten.exe C:\nk\netkitten.go -cp C:\nk\netkitten.exe C:\netkitten -"@ - -docker run ` - --volume ${PSScriptRoot}:C:\netkitten ` - golang:1.7-windowsservercore ` - powershell ${buildscript} - +Invoke-Expression "go build -o ${PSScriptRoot}\netkitten.exe ${PSScriptRoot}\netkitten.go" docker build -t "amazon/amazon-ecs-netkitten:make" -f "${PSScriptRoot}/windows.dockerfile" ${PSScriptRoot} diff --git a/misc/netkitten/windows.dockerfile b/misc/netkitten/windows.dockerfile index 67ed5844d4b..83f0a4575e2 100644 --- a/misc/netkitten/windows.dockerfile +++ b/misc/netkitten/windows.dockerfile @@ -10,10 +10,9 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. ADD netkitten.exe C:/netkitten.exe - ENTRYPOINT ["C:\\netkitten.exe"] diff --git a/misc/stats-windows/windows.dockerfile b/misc/stats-windows/windows.dockerfile index 402ad9a72f8..d7888f802f9 100644 --- a/misc/stats-windows/windows.dockerfile +++ b/misc/stats-windows/windows.dockerfile @@ -10,7 +10,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. diff --git a/misc/v3-task-endpoint-validator-windows/setup-v3-task-endpoint-validator.ps1 b/misc/v3-task-endpoint-validator-windows/setup-v3-task-endpoint-validator.ps1 index 4968934ee53..e3c349994b3 100644 --- a/misc/v3-task-endpoint-validator-windows/setup-v3-task-endpoint-validator.ps1 +++ b/misc/v3-task-endpoint-validator-windows/setup-v3-task-endpoint-validator.ps1 @@ -14,23 +14,7 @@ $oldPref = $ErrorActionPreference $ErrorActionPreference = 'Stop' -Invoke-Expression ${PSScriptRoot}\..\windows-deploy\hostsetup.ps1 - # Create amazon/amazon-ecs-v3-task-endpoint-validator-windows for tests -$buildscript = @" -mkdir C:\V3 -cp C:\ecs\v3-task-endpoint-validator-windows.go C:\V3 -go build -o C:\V3\v3-task-endpoint-validator-windows.exe C:\V3\v3-task-endpoint-validator-windows.go -cp C:\V3\v3-task-endpoint-validator-windows.exe C:\ecs -"@ - -$buildimage="golang:1.7-windowsservercore" -docker pull $buildimage - -docker run ` - --volume ${PSScriptRoot}:C:\ecs ` - $buildimage ` - powershell ${buildscript} - +Invoke-Expression "go build -o ${PSScriptRoot}\v3-task-endpoint-validator-windows.exe ${PSScriptRoot}\v3-task-endpoint-validator-windows.go" Invoke-Expression "docker build -t amazon/amazon-ecs-v3-task-endpoint-validator-windows --file ${PSScriptRoot}\v3-task-endpoint-validator-windows.dockerfile ${PSScriptRoot}" $ErrorActionPreference = $oldPref diff --git a/misc/v3-task-endpoint-validator-windows/v3-task-endpoint-validator-windows.dockerfile b/misc/v3-task-endpoint-validator-windows/v3-task-endpoint-validator-windows.dockerfile index f5ce1cfc72f..49282b83dbe 100644 --- a/misc/v3-task-endpoint-validator-windows/v3-task-endpoint-validator-windows.dockerfile +++ b/misc/v3-task-endpoint-validator-windows/v3-task-endpoint-validator-windows.dockerfile @@ -11,7 +11,7 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore ADD application.ps1 application.ps1 ADD v3-task-endpoint-validator-windows.exe v3-task-endpoint-validator-windows.exe diff --git a/misc/volumes-test/windows.dockerfile b/misc/volumes-test/windows.dockerfile index 1382115de0d..50c55bb3ab8 100644 --- a/misc/volumes-test/windows.dockerfile +++ b/misc/volumes-test/windows.dockerfile @@ -11,7 +11,7 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore MAINTAINER Amazon Web Services, Inc. SHELL ["powershell", "-command"] diff --git a/misc/windows-iam/Setup_Iam.ps1 b/misc/windows-iam/Setup_Iam.ps1 index 0d019452247..0597d301ce5 100644 --- a/misc/windows-iam/Setup_Iam.ps1 +++ b/misc/windows-iam/Setup_Iam.ps1 @@ -16,23 +16,9 @@ $ErrorActionPreference = 'Stop' Invoke-Expression ${PSScriptRoot}\..\windows-deploy\hostsetup.ps1 -# Create amazon/amazon-ecs-iamrolecontainer for tests -$buildscript = @" -mkdir C:\IAM -cp C:\ecs\ec2.go C:\IAM -go get -u github.com/aws/aws-sdk-go -go get -u github.com/aws/aws-sdk-go/aws -go build -o C:\IAM\ec2.exe C:\IAM\ec2.go -cp C:\IAM\ec2.exe C:\ecs -"@ - -$buildimage="golang:1.7-windowsservercore" -docker pull $buildimage - -docker run ` - --volume ${PSScriptRoot}:C:\ecs ` - $buildimage ` - powershell ${buildscript} +Invoke-Expression "go get -u github.com/aws/aws-sdk-go" +Invoke-Expression "go get -u github.com/aws/aws-sdk-go/aws" +Invoke-Expression "go build -o ${PSScriptRoot}\ec2.exe ${PSScriptRoot}\ec2.go" Invoke-Expression "docker build -t amazon/amazon-ecs-iamrolecontainer --file ${PSScriptRoot}\iamroles.dockerfile ${PSScriptRoot}" $ErrorActionPreference = $oldPref diff --git a/misc/windows-iam/Setup_Iam_Images.ps1 b/misc/windows-iam/Setup_Iam_Images.ps1 index 310d43bd834..3e793193866 100644 --- a/misc/windows-iam/Setup_Iam_Images.ps1 +++ b/misc/windows-iam/Setup_Iam_Images.ps1 @@ -16,22 +16,10 @@ $ErrorActionPreference = 'Stop' # Create amazon/amazon-ecs-iamrolecontainer for tests -$buildscript = @" -mkdir C:\IAM -cp C:\ecs\ec2.go C:\IAM -go get -u github.com/aws/aws-sdk-go -go get -u github.com/aws/aws-sdk-go/aws -go build -o C:\IAM\ec2.exe C:\IAM\ec2.go -cp C:\IAM\ec2.exe C:\ecs -"@ +Invoke-Expression "go get -u github.com/aws/aws-sdk-go' +Invoke-Expression "go get -u github.com/aws/aws-sdk-go/aws" +Invoke-Expression "go build -o ${PSScriptRoot}\ec2.exe ${PSScriptRoot}\ec2.go" -$buildimage="golang:1.7-windowsservercore" -docker pull $buildimage - -docker run ` - --volume ${PSScriptRoot}:C:\ecs ` - $buildimage ` - powershell ${buildscript} Invoke-Expression "docker build -t amazon/amazon-ecs-iamrolecontainer --file ${PSScriptRoot}\iamroles.dockerfile ${PSScriptRoot}" $ErrorActionPreference = $oldPref diff --git a/misc/windows-iam/iamroles.dockerfile b/misc/windows-iam/iamroles.dockerfile index 11fc5875fd8..5dbf4234d38 100644 --- a/misc/windows-iam/iamroles.dockerfile +++ b/misc/windows-iam/iamroles.dockerfile @@ -11,7 +11,7 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore ADD application.ps1 application.ps1 ADD ec2.exe ec2.exe diff --git a/misc/windows-listen80/Setup_Listen80.ps1 b/misc/windows-listen80/Setup_Listen80.ps1 index 4f5732486d3..50cb5ee831e 100644 --- a/misc/windows-listen80/Setup_Listen80.ps1 +++ b/misc/windows-listen80/Setup_Listen80.ps1 @@ -10,7 +10,9 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. +$oldPref = $ErrorActionPreference +$ErrorActionPreference = 'Stop' -docker pull microsoft/windowsservercore Invoke-Expression "go build -o ${PSScriptRoot}\listen80.exe ${PSScriptRoot}\listen80.go" Invoke-Expression "docker build -t amazon/amazon-ecs-listen80 --file ${PSScriptRoot}\listen80.dockerfile ${PSScriptRoot}" +$ErrorActionPreference = $oldPref diff --git a/misc/windows-listen80/listen80.dockerfile b/misc/windows-listen80/listen80.dockerfile index 7bad06d8cb6..d544f13a806 100644 --- a/misc/windows-listen80/listen80.dockerfile +++ b/misc/windows-listen80/listen80.dockerfile @@ -11,5 +11,5 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM microsoft/windowsservercore:latest +FROM microsoft/windowsservercore ADD listen80.exe listen80.exe diff --git a/misc/windows-python/build.ps1 b/misc/windows-python/build.ps1 deleted file mode 100644 index 89a596e023d..00000000000 --- a/misc/windows-python/build.ps1 +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may -# not use this file except in compliance with the License. A copy of the -# License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -docker pull python:3-windowsservercore-ltsc2016 -docker tag python:3-windowsservercore-ltsc2016 amazon/amazon-ecs-windows-python:make diff --git a/misc/windows-telemetry/build.ps1 b/misc/windows-telemetry/build.ps1 index 2c3b7143d12..ed00e3b846a 100644 --- a/misc/windows-telemetry/build.ps1 +++ b/misc/windows-telemetry/build.ps1 @@ -11,4 +11,6 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. + +Invoke-Expression "go build -o ${PSScriptRoot}\stress.exe ${PSScriptRoot}\main.go" docker build -t "amazon/amazon-ecs-windows-telemetry-test:make" -f "${PSScriptRoot}/windows.dockerfile" ${PSScriptRoot} diff --git a/misc/windows-telemetry/windows.dockerfile b/misc/windows-telemetry/windows.dockerfile index f27985383ad..a409398ba7a 100644 --- a/misc/windows-telemetry/windows.dockerfile +++ b/misc/windows-telemetry/windows.dockerfile @@ -10,11 +10,8 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -FROM golang:nanoserver +FROM microsoft/windowsservercore -WORKDIR /gopath -COPY main.go . - -RUN go build -o stress main.go -ENTRYPOINT ["./stress"] +ADD stress.exe stress.exe +ENTRYPOINT ["./stress.exe"] CMD [ "-concurrency", "1000", "-memory", "1024"] diff --git a/scripts/run-functional-tests.ps1 b/scripts/run-functional-tests.ps1 index 382ad1d6814..506716aedec 100755 --- a/scripts/run-functional-tests.ps1 +++ b/scripts/run-functional-tests.ps1 @@ -14,7 +14,6 @@ Invoke-Expression "${PSScriptRoot}\..\misc\windows-iam\Setup_Iam.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\windows-listen80\Setup_Listen80.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\windows-telemetry\build.ps1" -Invoke-Expression "${PSScriptRoot}\..\misc\windows-python\build.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\container-health-windows\build.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\v3-task-endpoint-validator-windows\setup-v3-task-endpoint-validator.ps1" Invoke-Expression "${PSScriptRoot}\..\misc\container-metadata-file-validator-windows\setup-container-metadata-file-validator.ps1" diff --git a/scripts/run-integ-tests.ps1 b/scripts/run-integ-tests.ps1 index b99d2792c2d..7d333eb779e 100755 --- a/scripts/run-integ-tests.ps1 +++ b/scripts/run-integ-tests.ps1 @@ -22,7 +22,7 @@ Invoke-Expression "${PSScriptRoot}\..\misc\netkitten\build.ps1" $cwd = (pwd).Path try { cd "${PSScriptRoot}" - go test -race -tags integration -timeout=25m -v ../agent/engine ../agent/stats ../agent/app + go test -race -tags integration -timeout=35m -v ../agent/engine ../agent/stats ../agent/app $testsExitCode = $LastExitCode } finally { cd "$cwd"