V1.42.0 stage

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 1.42.0
+* Feature - Support for sub second precision in FluentD [#2538](https://github.com/aws/amazon-ecs-agent/pull/2538).
+* Bug - Fixed a bug that caused configured values for ImageCleanupExclusionList
+to be ignored in some situations [#2513](https://github.com/aws/amazon-ecs-agent/pull/2513)
+
 ## 1.41.1
 * Bug - Fixed a bug [#2476](https://github.com/aws/amazon-ecs-agent/issues/2476) where HostPort is not present in ECS Task Metadata Endpoint response with bridge network type [#2495](https://github.com/aws/amazon-ecs-agent/pull/2495)
 

VERSION

@@ -1 +1 @@
-1.41.1
+1.42.0

agent/config/config.go

@@ -132,7 +132,6 @@ const (
 	DefaultTaskMetadataBurstRate = 60
 
 	//Known cached image names
-	CachedImageNamePauseContainer = "amazon/amazon-ecs-pause:0.1.0"
 	CachedImageNameAgentContainer = "amazon/amazon-ecs-agent:latest"
 
 	// DefaultNvidiaRuntime is the name of the runtime to pass Nvidia GPUs to containers

agent/config/config_test.go

@@ -419,7 +419,7 @@ func TestValidForImagesCleanupExclusion(t *testing.T) {
 	defer setTestRegion()()
 	defer setTestEnv("ECS_EXCLUDE_UNTRACKED_IMAGE", "amazonlinux:2,amazonlinux:3")()
 	imagesNotDelete := parseImageCleanupExclusionList("ECS_EXCLUDE_UNTRACKED_IMAGE")
-	expectedImages := []string{"amazonlinux:2", "amazonlinux:3", CachedImageNameAgentContainer, CachedImageNamePauseContainer}
+	expectedImages := []string{"amazonlinux:2", "amazonlinux:3"}
 	assert.Equal(t, expectedImages, imagesNotDelete, "unexpected imageCleanupExclusionList")
 }
 

agent/config/parse.go

@@ -309,16 +309,11 @@ func parseImageCleanupExclusionList(envVar string) []string {
 	var imageCleanupExclusionList []string
 	if imageEnv == "" {
 		seelog.Debugf("Environment variable empty: %s", imageEnv)
+		return nil
 	} else {
 		imageCleanupExclusionList = strings.Split(imageEnv, ",")
 	}
 
-	// append known cached internal images to imageCleanupExclusionLis
-	imageCleanupExclusionList = append(imageCleanupExclusionList, CachedImageNameAgentContainer, CachedImageNamePauseContainer)
-
-	for _, image := range imageCleanupExclusionList {
-		seelog.Infof("Image excluded from cleanup: %s", image)
-	}
 	return imageCleanupExclusionList
 }
 

agent/engine/docker_image_manager.go

@@ -81,14 +81,27 @@ func NewImageManager(cfg *config.Config, client dockerapi.DockerClient, state do
 		numImagesToDelete:                  cfg.NumImagesToDeletePerCycle,
 		imageCleanupTimeInterval:           cfg.ImageCleanupInterval,
 		imagePullBehavior:                  cfg.ImagePullBehavior,
-		imageCleanupExclusionList:          cfg.ImageCleanupExclusionList,
+		imageCleanupExclusionList:          buildImageCleanupExclusionList(cfg),
 		deleteNonECSImagesEnabled:          cfg.DeleteNonECSImagesEnabled,
 		nonECSContainerCleanupWaitDuration: cfg.TaskCleanupWaitDuration,
 		numNonECSContainersToDelete:        cfg.NumNonECSContainersToDeletePerCycle,
 		nonECSMinimumAgeBeforeDeletion:     cfg.NonECSMinimumImageDeletionAge,
 	}
 }
 
+func buildImageCleanupExclusionList(cfg *config.Config) []string {
+	// append known cached internal images to imageCleanupExclusionList
+	excludedImages := append(cfg.ImageCleanupExclusionList,
+		cfg.PauseContainerImageName+":"+cfg.PauseContainerTag,
+		config.DefaultPauseContainerImageName+":"+config.DefaultPauseContainerTag,
+		config.CachedImageNameAgentContainer,
+	)
+	for _, image := range excludedImages {
+		seelog.Infof("Image excluded from cleanup: %s", image)
+	}
+	return excludedImages
+}
+
 func (imageManager *dockerImageManager) SetSaver(stateManager statemanager.Saver) {
 	imageManager.saver = stateManager
 }

agent/engine/docker_image_manager_test.go

@@ -44,6 +44,23 @@ func defaultTestConfig() *config.Config {
 	return cfg
 }
 
+func TestNewImageManagerExcludesCachedImages(t *testing.T) {
+	cfg := defaultTestConfig()
+	cfg.PauseContainerImageName = "pause-name"
+	cfg.PauseContainerTag = "pause-tag"
+	cfg.ImageCleanupExclusionList = []string{"excluded:1"}
+	expected := []string{
+		"excluded:1",
+		"pause-name:pause-tag",
+		config.DefaultPauseContainerImageName + ":" + config.DefaultPauseContainerTag,
+		config.CachedImageNameAgentContainer,
+	}
+	imageManager := NewImageManager(cfg, nil, nil)
+	dockerImageManager, ok := imageManager.(*dockerImageManager)
+	require.True(t, ok, "imageManager must be *dockerImageManager")
+	assert.ElementsMatch(t, expected, dockerImageManager.imageCleanupExclusionList)
+}
+
 // TestImagePullRemoveDeadlock tests if there's a deadlock when trying to
 // pull an image while image clean up is in progress
 func TestImagePullRemoveDeadlock(t *testing.T) {

agent/engine/docker_task_engine.go

@@ -79,14 +79,15 @@ const (
 	engineConnectRetryJitterMultiplier = 0.20
 	engineConnectRetryDelayMultiplier  = 1.5
 	// logDriverTypeFirelens is the log driver type for containers that want to use the firelens container to send logs.
-	logDriverTypeFirelens   = "awsfirelens"
-	logDriverTypeFluentd    = "fluentd"
-	logDriverTag            = "tag"
-	logDriverFluentdAddress = "fluentd-address"
-	dataLogDriverPath       = "/data/firelens/"
-	logDriverAsyncConnect   = "fluentd-async-connect"
-	dataLogDriverSocketPath = "/socket/fluent.sock"
-	socketPathPrefix        = "unix://"
+	logDriverTypeFirelens       = "awsfirelens"
+	logDriverTypeFluentd        = "fluentd"
+	logDriverTag                = "tag"
+	logDriverFluentdAddress     = "fluentd-address"
+	dataLogDriverPath           = "/data/firelens/"
+	logDriverAsyncConnect       = "fluentd-async-connect"
+	logDriverSubSecondPrecision = "fluentd-sub-second-precision"
+	dataLogDriverSocketPath     = "/socket/fluent.sock"
+	socketPathPrefix            = "unix://"
 
 	// fluentTagDockerFormat is the format for the log tag, which is "containerName-firelens-taskID"
 	fluentTagDockerFormat = "%s-firelens-%s"
@@ -1102,6 +1103,7 @@ func getFirelensLogConfig(task *apitask.Task, container *apicontainer.Container,
 	logConfig.Config[logDriverTag] = tag
 	logConfig.Config[logDriverFluentdAddress] = fluentd
 	logConfig.Config[logDriverAsyncConnect] = strconv.FormatBool(true)
+	logConfig.Config[logDriverSubSecondPrecision] = strconv.FormatBool(true)
 	seelog.Debugf("Applying firelens log config for container %s: %v", container.Name, logConfig)
 	return logConfig
 }

agent/engine/docker_task_engine_test.go

@@ -2858,6 +2858,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 		expectedLogConfigTag           string
 		expectedLogConfigFluentAddress string
 		expectedFluentdAsyncConnect    string
+		expectedSubSecondPrecision     string
 		expectedIPAddress              string
 		expectedPort                   string
 	}{
@@ -2867,6 +2868,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 			expectedLogConfigType:          logDriverTypeFluentd,
 			expectedLogConfigTag:           taskName + "-firelens-" + taskID,
 			expectedFluentdAsyncConnect:    strconv.FormatBool(true),
+			expectedSubSecondPrecision:     strconv.FormatBool(true),
 			expectedLogConfigFluentAddress: socketPathPrefix + filepath.Join(defaultConfig.DataDirOnHost, dataLogDriverPath, taskID, dataLogDriverSocketPath),
 			expectedIPAddress:              envVarBridgeMode,
 			expectedPort:                   envVarPort,
@@ -2877,6 +2879,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 			expectedLogConfigType:          logDriverTypeFluentd,
 			expectedLogConfigTag:           taskName + "-firelens-" + taskID,
 			expectedFluentdAsyncConnect:    strconv.FormatBool(true),
+			expectedSubSecondPrecision:     strconv.FormatBool(true),
 			expectedLogConfigFluentAddress: socketPathPrefix + filepath.Join(defaultConfig.DataDirOnHost, dataLogDriverPath, taskID, dataLogDriverSocketPath),
 			expectedIPAddress:              envVarBridgeMode,
 			expectedPort:                   envVarPort,
@@ -2887,6 +2890,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 			expectedLogConfigType:          logDriverTypeFluentd,
 			expectedLogConfigTag:           taskName + "-firelens-" + taskID,
 			expectedFluentdAsyncConnect:    strconv.FormatBool(true),
+			expectedSubSecondPrecision:     strconv.FormatBool(true),
 			expectedLogConfigFluentAddress: socketPathPrefix + filepath.Join(defaultConfig.DataDirOnHost, dataLogDriverPath, taskID, dataLogDriverSocketPath),
 			expectedIPAddress:              envVarAWSVPCMode,
 			expectedPort:                   envVarPort,
@@ -2911,6 +2915,7 @@ func TestCreateContainerAddFirelensLogDriverConfig(t *testing.T) {
 					assert.Equal(t, tc.expectedLogConfigTag, hostConfig.LogConfig.Config["tag"])
 					assert.Equal(t, tc.expectedLogConfigFluentAddress, hostConfig.LogConfig.Config["fluentd-address"])
 					assert.Equal(t, tc.expectedFluentdAsyncConnect, hostConfig.LogConfig.Config["fluentd-async-connect"])
+					assert.Equal(t, tc.expectedSubSecondPrecision, hostConfig.LogConfig.Config["fluentd-sub-second-precision"])
 					assert.Contains(t, config.Env, tc.expectedIPAddress)
 					assert.Contains(t, config.Env, tc.expectedPort)
 				})

agent/version/version.go

@@ -22,10 +22,10 @@ package version
 // repository. Only the 'Version' const should change in checked-in source code
 
 // Version is the version of the Agent
-const Version = "1.41.1"
+const Version = "1.42.0"
 
 // GitDirty indicates the cleanliness of the git repo when this agent was built
 const GitDirty = true
 
 // GitShortHash is the short hash of this agent build
-const GitShortHash = "0d002a15"
+const GitShortHash = "1a88cff9"

misc/awscli/Dockerfile

@@ -1,8 +1 @@
-FROM debian:stable
-
-RUN apt-get update && apt-get install -y \
-    python2.7 curl
-
-RUN curl -O https://bootstrap.pypa.io/get-pip.py
-
-RUN python2.7 get-pip.py && pip install awscli
+FROM amazon/aws-cli:2.0.27

misc/container-health/Dockerfile

@@ -10,7 +10,7 @@
 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
-FROM busybox:1.29.3
+FROM busybox:1.32.0
 
 HEALTHCHECK --interval=1s --timeout=1s --retries=3 CMD echo hello
 

misc/fluentd/Dockerfile

@@ -1,4 +1,4 @@
-#    fluent/fluentd:v1.2.5
-FROM fluent/fluentd@sha256:c99d55c78d383df803d49f2f54755e41c654ea8e3c1ef34ba695d3a1af33151c
+#    fluent/fluentd:v1.11-1
+FROM fluent/fluentd@sha256:708cb509799785a40a07c4269e5dded4bfa6df1ae50631f24e4921ae98fda1e5
 
 COPY fluent.conf /fluentd/etc/

misc/image-cleanup-test-images/linux0.dockerfile

@@ -10,7 +10,7 @@
 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
-FROM busybox:1.29.3
+FROM busybox:1.32.0
 
 MAINTAINER Amazon Web Services, Inc.
 

misc/image-cleanup-test-images/linux1.dockerfile

@@ -10,7 +10,7 @@
 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
-FROM busybox:1.29.3
+FROM busybox:1.32.0
 
 MAINTAINER Amazon Web Services, Inc.
 

misc/image-cleanup-test-images/linux2.dockerfile

@@ -10,7 +10,7 @@
 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
-FROM busybox:1.29.3
+FROM busybox:1.32.0
 
 MAINTAINER Amazon Web Services, Inc.
 

misc/taskmetadata-validator/Dockerfile

@@ -10,7 +10,7 @@
 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
-FROM busybox:1.29.3
+FROM busybox:1.32.0
 
 MAINTAINER Amazon Web Services, Inc.
 COPY taskmetadata-validator /

misc/volumes-test/linux.dockerfile

@@ -10,7 +10,7 @@
 # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
-FROM busybox:1.29.3
+FROM busybox:1.32.0
 MAINTAINER Amazon Web Services, Inc.
 
 RUN mkdir /data

scripts/setup-test-registry

@@ -17,7 +17,7 @@
 set -e
 
 REGISTRY_IMAGE="registry:2.7.0"
-NGINX_IMAGE="nginx:1.15.8"
+NGINX_IMAGE="nginx:stable"
 
 ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
 cd "${ROOT}"
@@ -47,8 +47,8 @@ mirror_local_image() {
 setup_parallel_pull_image() {
   local image_name="127.0.0.1:51670/$1:parallel-pull-fts"
   echo "Generating parallel pull image $image_name"
- 
-  # Get new random bits every time we build 
+
+  # Get new random bits every time we build
   head -c 25m < /dev/urandom > docker-context/random-bits
   docker build -t "$image_name" docker-context
 
@@ -67,15 +67,15 @@ done
 # Remove the tag so this image can be deleted successfully in the docker image cleanup integ tests
 docker rmi amazon/image-cleanup-test-image1:make amazon/image-cleanup-test-image2:make amazon/image-cleanup-test-image3:make
 
-mirror_image busybox:1.29.3 "127.0.0.1:51670/busybox:latest"
+mirror_image busybox:1.32.0 "127.0.0.1:51670/busybox:latest"
 mirror_image "$NGINX_IMAGE" "127.0.0.1:51670/nginx:latest"
-mirror_image ubuntu:16.04 "127.0.0.1:51670/ubuntu:latest"
+mirror_image ubuntu:20.04 "127.0.0.1:51670/ubuntu:latest"
 
 # create a context folder used by docker build. It will only have a file
 # full of random bits so that the parallel pull images are different.
 mkdir -p docker-context
 cat << EOF > docker-context/Dockerfile
-FROM amazon/amazon-ecs-pause:0.1.0 
+FROM amazon/amazon-ecs-pause:0.1.0
 ADD random-bits /random-bits
 EOF
 
@@ -84,7 +84,7 @@ for image in "busybox" "ubuntu" "consul" "debian" "httpd" "crux" "nginx" "redis"
 done
 
 # cleanup the context
-rm -rf docker-context 
+rm -rf docker-context
 
 mirror_local_image "${REGISTRY_IMAGE}" "127.0.0.1:51670/registry:parallel-pull-fts"
 

scripts/upload-images

@@ -48,10 +48,10 @@ fi
 
 ECR_ROLE_IMAGE="executionrole"
 
-BUSYBOX_IMAGE="busybox:1.29.3"
-NGINX_IMAGE="nginx:1.15.8"
+BUSYBOX_IMAGE="busybox:1.32.0"
+NGINX_IMAGE="nginx:stable"
 PYTHON2_IMAGE="python:2.7.15"
-UBUNTU_IMAGE="ubuntu:16.04"
+UBUNTU_IMAGE="ubuntu:20.04"
 
 PARALLEL_PULL_FTS_BUSYBOX=${BUSYBOX_IMAGE}
 PARALLEL_PULL_FTS_UBUNTU=${UBUNTU_IMAGE}