aws · fierlion · Mar 13, 2022 · Mar 11, 2022 · Mar 11, 2022 · lydiafilipe
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -28,3 +28,5 @@ jobs:
           cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent
           make test-silent
           make analyze-cover-profile
+          make test-init
+          make analyze-cover-profile-init
diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml
@@ -29,6 +29,32 @@ jobs:
           make get-deps
           make static-check
 
+  init-check:
+    name: Static Analysis Init
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: src/github.com/aws/amazon-ecs-agent
+      - name: get GO_VERSION
+        run:  |
+          cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent
+          echo "GO_VERSION=$(cat GO_VERSION)" >> $GITHUB_ENV
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v2
+        with:
+          path: src/github.com/aws/amazon-ecs-agent
+      - name: run static checks
+        run: |
+          export GOPATH=$GITHUB_WORKSPACE
+          export PATH=$PATH:$(go env GOPATH)/bin
+          export GO111MODULE=auto
+          cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent
+          make get-deps-init
+          make static-check-init
+
   x-platform-build:
     name: Cross platform build
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -39,7 +39,6 @@ all: docker
 gobuild:
 	./scripts/build false
 
-
 # create output directories
 .out-stamp:
 	mkdir -p ./out/test-artifacts ./out/cni-plugins ./out/amazon-ecs-cni-plugins ./out/amazon-vpc-cni-plugins
@@ -49,6 +48,9 @@ gobuild:
 static:
 	./scripts/build
 
+static-with-pause:
+	./scripts/build true "" false true
+
 # Cross-platform build target for static checks
 xplatform-build:
 	GOOS=linux GOARCH=arm64 ./scripts/build true "" false
@@ -119,6 +121,10 @@ gogenerate:
 	go generate -x ./agent/...
 	$(MAKE) goimports
 
+gogenerate-init:
+	PATH=$(PATH):$(shell pwd)/scripts go generate -x ./ecs-init/...
+	$(MAKE) goimports
+
 # 'go' may not be on the $PATH for sudo tests
 GO_EXECUTABLE=$(shell command -v go 2> /dev/null)
 
@@ -140,6 +146,10 @@ test:
 	${GOTEST} -tags unit -coverprofile cover.out -timeout=60s ./agent/...
 	go tool cover -func cover.out > coverprofile.out
 
+test-init:
+	go test -count=1 -short -v -coverprofile cover.out ./ecs-init/...
+	go tool cover -func cover.out > coverprofile-init.out
+
 test-silent:
 	$(eval VERBOSE=)
 	${GOTEST} -tags unit -coverprofile cover.out -timeout=60s ./agent/...
@@ -149,6 +159,10 @@ test-silent:
 analyze-cover-profile: coverprofile.out
 	./scripts/analyze-cover-profile
 
+.PHONY: analyze-cover-profile-init
+analyze-cover-profile-init: coverprofile-init.out
+	./scripts/analyze-cover-profile-init
+
 run-integ-tests: test-registry gremlin container-health-check-image run-sudo-tests
 	ECS_LOGLEVEL=debug ${GOTEST} -tags integration -timeout=30m ./agent/...
 
@@ -158,7 +172,6 @@ run-sudo-tests:
 benchmark-test:
 	go test -run=XX -bench=. ./agent/...
 
-
 .PHONY: build-image-for-ecr upload-images replicate-images
 
 build-image-for-ecr: netkitten volumes-test image-cleanup-test-images fluentd exec-command-agent-test
@@ -223,6 +236,21 @@ cni-plugins: get-cni-sources .out-stamp build-ecs-cni-plugins build-vpc-cni-plug
 	mv $(PWD)/out/amazon-vpc-cni-plugins/* $(PWD)/out/cni-plugins
 	@echo "Built all cni plugins successfully."
 
+# dockerfree build process will build the agent container image from scratch
+# and with minimal dependencies
+# requires glibc-static
+
+dockerfree-pause:
+	GOOS=linux GOARCH=amd64 ./scripts/build-pause
+
+dockerfree-certs:
+	GOOS=linux GOARCH=amd64 ./scripts/get-host-certs
+
+dockerfree-cni-plugins: get-cni-sources
+	GOOS=linux GOARCH=amd64 ./scripts/build-cni-plugins
+
+dockerfree-agent-image: dockerfree-pause dockerfree-certs dockerfree-cni-plugins static-with-pause
+	GOOS=linux GOARCH=amd64 ./scripts/build-agent-image
 
 .PHONY: codebuild
 codebuild: .out-stamp
@@ -259,7 +287,6 @@ image-cleanup-test-images:
 container-health-check-image:
 	$(MAKE) -C misc/container-health $(MFLAGS)
 
-
 # all .go files in the agent, excluding vendor/, model/ and testutils/ directories, and all *_test.go and *_mocks.go files
 GOFILES:=$(shell go list -f '{{$$p := .}}{{range $$f := .GoFiles}}{{$$p.Dir}}/{{$$f}} {{end}}' ./agent/... \
 		| grep -v /testutils/ | grep -v _test\.go$ | grep -v _mocks\.go$ | grep -v /model)
@@ -286,12 +313,23 @@ gogenerate-check: gogenerate
 	# check that gogenerate does not generate a diff.
 	git diff --exit-code
 
+.PHONY: gogenerate-check-init
+gogenerate-check-init: gogenerate-init
+	# check that gogenerate does not generate a diff.
+	git diff --exit-code
+
 .PHONY: static-check
 static-check: gocyclo govet importcheck gogenerate-check
 	# use default checks of staticcheck tool, except style checks (-ST*) and depracation checks (-SA1019)
 	# depracation checks have been left out for now; removing their warnings requires error handling for newer suggested APIs, changes in function signatures and their usages.
 	# https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
-	staticcheck -tests=false -checks "inherit,-ST*,-SA1019,-SA9002" ./agent/...
+	staticcheck -tests=false -checks "inherit,-ST*,-SA1019,-SA9002,-SA4006" ./agent/...
+
+.PHONY: static-check-init
+static-check-init: gocyclo govet importcheck gogenerate-check-init
+	# use default checks of staticcheck tool, except style checks (-ST*)
+	# https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
+	staticcheck -tests=false -checks "inherit,-ST*" ./ecs-init/...
 
 .PHONY: goimports
 goimports:
@@ -309,13 +347,53 @@ GOPATH=$(shell go env GOPATH)
 
 get-deps: .get-deps-stamp
 
+get-deps-init:
+	go get golang.org/x/tools/cover
+	go get golang.org/x/tools/cmd/cover
+	go get github.com/golang/mock/mockgen
+	cd "${GOPATH}/src/github.com/golang/mock/mockgen" && git checkout 1.3.1 && go get ./... && go install ./... && cd -
+	GO111MODULE=on go get github.com/fzipp/gocyclo/cmd/[email protected]
+	go get golang.org/x/tools/cmd/goimports
+	go get honnef.co/go/tools/cmd/staticcheck
+
+.generic-rpm-done:
+	./scripts/update-version.sh
+	cp packaging/generic-rpm/amazon-ecs-init.spec amazon-ecs-init.spec
+	cp packaging/generic-rpm/ecs.service ecs.service
+	cp packaging/generic-rpm/amazon-ecs-volume-plugin.service amazon-ecs-volume-plugin.service
+	cp packaging/generic-rpm/amazon-ecs-volume-plugin.socket amazon-ecs-volume-plugin.socket
+	tar -czf ./sources.tgz ecs-init scripts
+	test -e SOURCES || ln -s . SOURCES
+	rpmbuild --define "%_topdir $(PWD)" -bb amazon-ecs-init.spec
+	find RPMS/ -type f -exec cp {} . \;
+	touch .rpm-done
+
+generic-rpm: .generic-rpm-done
+
+dockerfree-all: dockerfree-agent-image generic-rpm
+
+.deb-done: BUILDROOT/ecs-agent.tar
+	./scripts/update-version.sh
+	tar -czf ./amazon-ecs-init_${VERSION}.orig.tar.gz ecs-init scripts README.md
+	cp -r packaging/generic-deb/debian ecs-init scripts README.md BUILDROOT
+	cd BUILDROOT && debuild -uc -us --lintian-opts --suppress-tags bad-distribution-in-changes-file,file-in-unusual-dir
+	touch .deb-done
+
+deb: .deb-done
+
 clean:
 	# ensure docker is running and we can talk to it, abort if not:
 	docker ps > /dev/null
 	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild:make"
 	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild-windows:make"
 	rm -f misc/certs/ca-certificates.crt &> /dev/null
+	rm -f misc/certs/ca-bundle.crt &> /dev/null
+	rm -rf misc/pause-container/image/
+	rm -rf misc/pause-container/rootfs/
+	rm -rf misc/plugins/
+	rm -f misc/pause-container/amazon-ecs-pause.tar
 	rm -rf out/
+	rm -rf rootfs/
 	-$(MAKE) -C $(ECS_CNI_REPOSITORY_SRC_DIR) clean
 	-$(MAKE) -C misc/netkitten $(MFLAGS) clean
 	-$(MAKE) -C misc/volumes-test $(MFLAGS) clean
@@ -329,4 +407,25 @@ clean:
 	-rm -rf $(PWD)/bin
 	-rm -rf cover.out
 	-rm -rf coverprofile.out
-
+	-rm -rf coverprofile-init.out
+	# ecs-init & rpm cleanup
+	-rm -f ecs-init.spec
+	-rm -f amazon-ecs-init.spec
+	-rm -f ecs.conf
+	-rm -f ecs.service
+	-rm -f amazon-ecs-volume-plugin.conf
+	-rm -f amazon-ecs-volume-plugin.service
+	-rm -f amazon-ecs-volume-plugin.socket
+	-rm -rf ./bin
+	-rm -f ./sources.tgz
+	-rm -f ./amazon-ecs-init
+	-rm -f ./amazon-ecs-init-*.rpm
+	-rm -f ./ecs-agent-*.tar
+	-rm -f ./ecs-init-*.src.rpm
+	-rm -rf ./ecs-init-*
+	-rm -rf ./BUILDROOT BUILD RPMS SRPMS SOURCES SPECS
+	-rm -rf ./x86_64
+	-rm -f ./amazon-ecs-init_${VERSION}*
+	-rm -f .srpm-done .rpm-done .generic-rpm-done
+	-rm -f .deb-done
+	-rm -f amazon-ecs-volume-plugin
diff --git a/agent-container/agent-config.json b/agent-container/agent-config.json
@@ -0,0 +1,2 @@
+{"author":"Amazon Web Services, Inc.","config":{"Cmd":["/agent"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Amazon Web Services, Inc.","created_by":"[] + [] === \"\"","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}}
+
diff --git a/agent-container/agent-image-VERSION b/agent-container/agent-image-VERSION
@@ -0,0 +1 @@
+1.0
diff --git a/agent-container/agent-manifest.json b/agent-container/agent-manifest.json
@@ -0,0 +1,2 @@
+[{"Config":"config.json","RepoTags":["amazon/amazon-ecs-agent:1.60.0"],"Layers":["rootfs/layer.tar"]}]
+
diff --git a/agent-container/agent-repositories b/agent-container/agent-repositories
@@ -0,0 +1 @@
+{"amazon/amazon-ecs-agent":{"amazon-ecs":"rootfs"}}
diff --git a/dependencies_mocks.go b/dependencies_mocks.go
@@ -0,0 +1,15 @@
+// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+//
+// Source: dependencies.go in package cache
+
diff --git a/ecs-init/cache/cache_test.go b/ecs-init/cache/cache_test.go
@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may

diff --git a/ecs-init/cache/dependencies.go b/ecs-init/cache/dependencies.go
@@ -18,7 +18,7 @@ package cache
 // package-level functions.  These interfaces are then used to create mocks
 // for the unit tests.
 
-//go:generate mockgen.sh $GOPACKAGE $GOFILE
+//go:generate mockgen.sh cache $GOFILE  
 
 import (
 	"io"

diff --git a/ecs-init/cache/dependencies_mocks.go b/ecs-init/cache/dependencies_mocks.go
diff --git a/ecs-init/config/config_al2.go b/ecs-init/config/config_al2.go
@@ -1,3 +1,4 @@
+//go:build al2
 // +build al2
 
 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

diff --git a/ecs-init/config/config_generic_rpm.go b/ecs-init/config/config_generic_rpm.go
@@ -1,3 +1,4 @@
+//go:build generic_rpm
 // +build generic_rpm
 
 // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.

diff --git a/ecs-init/config/config_suse_ubuntu_debian.go b/ecs-init/config/config_suse_ubuntu_debian.go
@@ -1,3 +1,4 @@
+//go:build suse || ubuntu || debian
 // +build suse ubuntu debian
 
 // Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

diff --git a/ecs-init/config/config_unspecified.go b/ecs-init/config/config_unspecified.go
@@ -1,3 +1,4 @@
+//go:build !suse && !ubuntu && !al2 && !debian && !generic_rpm
 // +build !suse,!ubuntu,!al2,!debian,!generic_rpm
 
 // Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

diff --git a/ecs-init/config/development.go b/ecs-init/config/development.go
@@ -1,3 +1,4 @@
+//go:build development
 // +build development
 
 // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.

diff --git a/ecs-init/config/logger.go b/ecs-init/config/logger.go
@@ -34,7 +34,7 @@ and limitations under the License.
 <seelog type="asyncloop">
 	<outputs formatid="main">
 		<console formatid="console" />
-		<rollingfile filename="` + initLogFile() + `" type="date"
+		<rollingfile filename="`+initLogFile()+`" type="date"
 			 datepattern="2006-01-02-15" archivetype="zip" maxrolls="5" />
 	</outputs>
 	<formats>

diff --git a/ecs-init/config/release.go b/ecs-init/config/release.go
@@ -1,3 +1,4 @@
+//go:build !development
 // +build !development
 
 // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.

diff --git a/ecs-init/docker/backoff_mocks.go b/ecs-init/docker/backoff_mocks.go
diff --git a/ecs-init/docker/dependencies_mocks.go b/ecs-init/docker/dependencies_mocks.go
diff --git a/ecs-init/docker/dependencies_test.go b/ecs-init/docker/dependencies_test.go
@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may

diff --git a/ecs-init/docker/docker_test.go b/ecs-init/docker/docker_test.go
@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2015-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may

diff --git a/ecs-init/ecs-init b/ecs-init/ecs-init
diff --git a/ecs-init/engine/dependencies_mocks.go b/ecs-init/engine/dependencies_mocks.go
diff --git a/ecs-init/engine/engine_test.go b/ecs-init/engine/engine_test.go
@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may

diff --git a/ecs-init/exec/iptables/cmd_mocks.go b/ecs-init/exec/iptables/cmd_mocks.go
diff --git a/ecs-init/exec/iptables/exec_mocks.go b/ecs-init/exec/iptables/exec_mocks.go
diff --git a/ecs-init/exec/sysctl/cmd_mocks.go b/ecs-init/exec/sysctl/cmd_mocks.go
diff --git a/ecs-init/exec/sysctl/exec_mocks.go b/ecs-init/exec/sysctl/exec_mocks.go
diff --git a/ecs-init/gpu/nvidia_gpu_manager_mocks.go b/ecs-init/gpu/nvidia_gpu_manager_mocks.go
diff --git a/ecs-init/version/version.go b/ecs-init/version/version.go
@@ -18,10 +18,10 @@
 package version
 
 // Version is the version of the ecs-init
-var Version string = "unknown"
+var Version string = "1.60.0"
 
 // GitDirty indicates the cleanliness of the git repo when this ecs-init was built
 var GitDirty string = "true"
 
 // GitShortHash is the short hash of this ecs-init build
-var GitShortHash string = "unknown"
+var GitShortHash string = "11173cad"
diff --git a/misc/pause-container/pause-config.json b/misc/pause-container/pause-config.json
@@ -0,0 +1,2 @@
+{"author":"Amazon Web Services, Inc.","config":{"Cmd":["/pause"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Amazon Web Services, Inc.","created_by":"[] + [] === \"\"","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}}
+
diff --git a/misc/pause-container/pause-image-VERSION b/misc/pause-container/pause-image-VERSION
@@ -0,0 +1 @@
+1.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		{"author":"Amazon Web Services, Inc.","config":{"Cmd":["/agent"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Amazon Web Services, Inc.","created_by":"[] + [] === \"\"","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}}
Copy link Contributor sparrc Mar 11, 2022 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. why are these static files? shouldn't they be generated at build-time? Copy link Member Author fierlion Mar 11, 2022 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. The diff_ids field is being generated at build-time. Otherwise I agree maybe we can update the history.created with an actual timestamp. These are generated by Docker otherwise -- if we don't have Docker, we need to create/provide these config files manually. Bottlerocket uses static configs to build its pause container: https://github.com/bottlerocket-os/bottlerocket/blob/develop/packages/ecs-agent/pause-config.json Copy link Contributor lydiafilipe Mar 11, 2022 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. So do we have to ensure these files get rebuilt/committed each time then from current agent changes? Might need some sort of check to reinforce that Copy link Member Author fierlion Mar 11, 2022 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Updated to string replace and use the date util to match expected formatting: `sed -i "s/~~timestamp~~/$(date +"%FT%T.%NZ")/g" image/config.json` Otherwise we'll use https://github.com/aws/amazon-ecs-agent/blob/master/VERSION to fill in our version.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		[{"Config":"config.json","RepoTags":["amazon/amazon-ecs-agent:1.60.0"],"Layers":["rootfs/layer.tar"]}]
fierlion marked this conversation as resolved. Show resolved Hide resolved
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"amazon/amazon-ecs-agent":{"amazon-ecs":"rootfs"}}