Skip to content

Commit 4505743

Browse files
committed
container stats: ignore context.Canceled errors
we often receive context.Canceled errors when a container exits during docker stats collection. there is a benign race condition where if we process the error first before processing that the context is "Done", then we will log this canceled error as a warning message here: https://github.com/aws/amazon-ecs-agent/blob/5be7aa08bed215a557f48c16d8201ad3db59a9be/agent/stats/container.go#L118-L122 this change ignores these context.Canceled errors so that we don't log them. This will eliminate log messages that look like this when a container exits: level=warn time=2020-12-25T07:51:33Z msg="Error encountered processing metrics stream from docker, this may affect cloudwatch metric accuracy: DockerGoClient: Unable to decode stats for container REDACTED: context canceled" module=container.go
1 parent 6cd32aa commit 4505743

File tree

1 file changed

+12
-0
lines changed

1 file changed

+12
-0
lines changed

agent/dockerclient/dockerapi/docker_client.go

+12
Original file line numberDiff line numberDiff line change
@@ -1393,6 +1393,10 @@ func (dg *dockerGoClient) Stats(ctx context.Context, id string, inactivityTimeou
13931393
stream := true
13941394
resp, err = client.ContainerStats(subCtx, id, stream)
13951395
if err != nil {
1396+
if errors.Is(err, context.Canceled) {
1397+
// ignore context cancelled errors, which indicate container has exited already
1398+
return
1399+
}
13961400
errC <- fmt.Errorf("DockerGoClient: Unable to retrieve stats for container %s: %v", id, err)
13971401
return
13981402
}
@@ -1408,6 +1412,10 @@ func (dg *dockerGoClient) Stats(ctx context.Context, id string, inactivityTimeou
14081412
data := new(types.StatsJSON)
14091413
for err := decoder.Decode(data); err != io.EOF; err = decoder.Decode(data) {
14101414
if err != nil {
1415+
if errors.Is(err, context.Canceled) {
1416+
// ignore context cancelled errors, which indicate container has exited already
1417+
return
1418+
}
14111419
errC <- fmt.Errorf("DockerGoClient: Unable to decode stats for container %s: %v", id, err)
14121420
return
14131421
}
@@ -1466,6 +1474,10 @@ func getContainerStatsNotStreamed(client sdkclient.Client, ctx context.Context,
14661474
response := make(chan statsResponse, 1)
14671475
go func() {
14681476
stats, err := client.ContainerStats(ctxWithTimeout, id, false)
1477+
if errors.Is(err, context.Canceled) {
1478+
// ignore context cancelled errors, which indicate container has exited already
1479+
err = nil
1480+
}
14691481
response <- statsResponse{stats, err}
14701482
}()
14711483
select {

0 commit comments

Comments
 (0)