From 78367500aa5446107f14e6a9761637cae1107ff3 Mon Sep 17 00:00:00 2001 From: Ray Allan Date: Sun, 13 Mar 2022 13:28:13 -0700 Subject: [PATCH] Dockerfree build and ecs-init make integration (#3149) * Resolve go modules/vendor conflicts and add dockerfree-all make target * add build-time configuration and update scripts/comments --- .github/workflows/linux.yml | 2 + .github/workflows/static.yml | 26 +++++ .gitignore | 19 ++++ Makefile | 111 ++++++++++++++++++- agent-container/agent-config.json | 1 + agent-container/agent-image-VERSION | 1 + agent-container/agent-manifest.json | 2 + agent-container/agent-repositories | 1 + dependencies_mocks.go | 15 +++ ecs-init/cache/cache_test.go | 1 + ecs-init/cache/dependencies.go | 2 +- ecs-init/cache/dependencies_mocks.go | 2 +- ecs-init/config/config_al2.go | 1 + ecs-init/config/config_generic_rpm.go | 1 + ecs-init/config/config_suse_ubuntu_debian.go | 1 + ecs-init/config/config_unspecified.go | 1 + ecs-init/config/development.go | 1 + ecs-init/config/logger.go | 2 +- ecs-init/config/release.go | 1 + ecs-init/docker/backoff_mocks.go | 2 +- ecs-init/docker/dependencies_mocks.go | 2 +- ecs-init/docker/dependencies_test.go | 1 + ecs-init/docker/docker_test.go | 1 + ecs-init/engine/dependencies_mocks.go | 2 +- ecs-init/engine/engine_test.go | 1 + ecs-init/exec/iptables/cmd_mocks.go | 2 +- ecs-init/exec/iptables/exec_mocks.go | 2 +- ecs-init/exec/sysctl/cmd_mocks.go | 2 +- ecs-init/exec/sysctl/exec_mocks.go | 2 +- ecs-init/gpu/nvidia_gpu_manager_mocks.go | 2 +- misc/pause-container/pause-config.json | 2 + misc/pause-container/pause-image-VERSION | 1 + misc/pause-container/pause-manifest.json | 2 + misc/pause-container/pause-repositories | 1 + packaging/generic-rpm/amazon-ecs-init.spec | 1 - scripts/analyze-cover-profile-init | 27 +++++ scripts/build | 9 ++ scripts/build-agent-image | 56 ++++++++++ scripts/build-cni-plugins | 32 ++++++ scripts/build-pause | 32 ++++++ scripts/get-host-certs | 42 +++++++ 41 files changed, 397 insertions(+), 18 deletions(-) create mode 100644 agent-container/agent-config.json create mode 100644 agent-container/agent-image-VERSION create mode 100644 agent-container/agent-manifest.json create mode 100644 agent-container/agent-repositories create mode 100644 dependencies_mocks.go create mode 100644 misc/pause-container/pause-config.json create mode 100644 misc/pause-container/pause-image-VERSION create mode 100644 misc/pause-container/pause-manifest.json create mode 100644 misc/pause-container/pause-repositories create mode 100755 scripts/analyze-cover-profile-init create mode 100755 scripts/build-agent-image create mode 100755 scripts/build-cni-plugins create mode 100755 scripts/build-pause create mode 100755 scripts/get-host-certs diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 527fb295a31..62941cb32f4 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,3 +28,5 @@ jobs: cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent make test-silent make analyze-cover-profile + make test-init + make analyze-cover-profile-init diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index 0b5c4a48755..776e47a68cb 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -29,6 +29,32 @@ jobs: make get-deps make static-check + init-check: + name: Static Analysis Init + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + path: src/github.com/aws/amazon-ecs-agent + - name: get GO_VERSION + run: | + cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent + echo "GO_VERSION=$(cat GO_VERSION)" >> $GITHUB_ENV + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - uses: actions/checkout@v2 + with: + path: src/github.com/aws/amazon-ecs-agent + - name: run static checks + run: | + export GOPATH=$GITHUB_WORKSPACE + export PATH=$PATH:$(go env GOPATH)/bin + export GO111MODULE=auto + cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent + make get-deps-init + make static-check-init + x-platform-build: name: Cross platform build runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6cdbe76d8ad..77d3f530b58 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,22 @@ _bin/ *.iml cover.out coverprofile.out +/amazon-ecs-init* +/BUILDROOT/ +/x86_64/ +/sources.tar +/ecs-init-* +/ecs.conf +/.deb-done +/.rpm-done +/.srpm-done +/BUILD +/RPMS +/SOURCES +/SRPMS +/ecs-init.spec +/sources.tgz +ecs-agent-*.tar +/ecs.service +*.log +*.DS_Store diff --git a/Makefile b/Makefile index 8c244ff01b9..2a2b8771d15 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,6 @@ all: docker gobuild: ./scripts/build false - # create output directories .out-stamp: mkdir -p ./out/test-artifacts ./out/cni-plugins ./out/amazon-ecs-cni-plugins ./out/amazon-vpc-cni-plugins @@ -49,6 +48,9 @@ gobuild: static: ./scripts/build +static-with-pause: + ./scripts/build true "" false true + # Cross-platform build target for static checks xplatform-build: GOOS=linux GOARCH=arm64 ./scripts/build true "" false @@ -119,6 +121,10 @@ gogenerate: go generate -x ./agent/... $(MAKE) goimports +gogenerate-init: + PATH=$(PATH):$(shell pwd)/scripts go generate -x ./ecs-init/... + $(MAKE) goimports + # 'go' may not be on the $PATH for sudo tests GO_EXECUTABLE=$(shell command -v go 2> /dev/null) @@ -140,6 +146,10 @@ test: ${GOTEST} -tags unit -coverprofile cover.out -timeout=60s ./agent/... go tool cover -func cover.out > coverprofile.out +test-init: + go test -count=1 -short -v -coverprofile cover.out ./ecs-init/... + go tool cover -func cover.out > coverprofile-init.out + test-silent: $(eval VERBOSE=) ${GOTEST} -tags unit -coverprofile cover.out -timeout=60s ./agent/... @@ -149,6 +159,10 @@ test-silent: analyze-cover-profile: coverprofile.out ./scripts/analyze-cover-profile +.PHONY: analyze-cover-profile-init +analyze-cover-profile-init: coverprofile-init.out + ./scripts/analyze-cover-profile-init + run-integ-tests: test-registry gremlin container-health-check-image run-sudo-tests ECS_LOGLEVEL=debug ${GOTEST} -tags integration -timeout=30m ./agent/... @@ -158,7 +172,6 @@ run-sudo-tests: benchmark-test: go test -run=XX -bench=. ./agent/... - .PHONY: build-image-for-ecr upload-images replicate-images build-image-for-ecr: netkitten volumes-test image-cleanup-test-images fluentd exec-command-agent-test @@ -223,6 +236,21 @@ cni-plugins: get-cni-sources .out-stamp build-ecs-cni-plugins build-vpc-cni-plug mv $(PWD)/out/amazon-vpc-cni-plugins/* $(PWD)/out/cni-plugins @echo "Built all cni plugins successfully." +# dockerfree build process will build the agent container image from scratch +# and with minimal dependencies +# requires glibc-static + +dockerfree-pause: + GOOS=linux GOARCH=amd64 ./scripts/build-pause + +dockerfree-certs: + GOOS=linux GOARCH=amd64 ./scripts/get-host-certs + +dockerfree-cni-plugins: get-cni-sources + GOOS=linux GOARCH=amd64 ./scripts/build-cni-plugins + +dockerfree-agent-image: dockerfree-pause dockerfree-certs dockerfree-cni-plugins static-with-pause + GOOS=linux GOARCH=amd64 ./scripts/build-agent-image .PHONY: codebuild codebuild: .out-stamp @@ -259,7 +287,6 @@ image-cleanup-test-images: container-health-check-image: $(MAKE) -C misc/container-health $(MFLAGS) - # all .go files in the agent, excluding vendor/, model/ and testutils/ directories, and all *_test.go and *_mocks.go files GOFILES:=$(shell go list -f '{{$$p := .}}{{range $$f := .GoFiles}}{{$$p.Dir}}/{{$$f}} {{end}}' ./agent/... \ | grep -v /testutils/ | grep -v _test\.go$ | grep -v _mocks\.go$ | grep -v /model) @@ -286,12 +313,23 @@ gogenerate-check: gogenerate # check that gogenerate does not generate a diff. git diff --exit-code +.PHONY: gogenerate-check-init +gogenerate-check-init: gogenerate-init + # check that gogenerate does not generate a diff. + git diff --exit-code + .PHONY: static-check static-check: gocyclo govet importcheck gogenerate-check # use default checks of staticcheck tool, except style checks (-ST*) and depracation checks (-SA1019) # depracation checks have been left out for now; removing their warnings requires error handling for newer suggested APIs, changes in function signatures and their usages. # https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck - staticcheck -tests=false -checks "inherit,-ST*,-SA1019,-SA9002" ./agent/... + staticcheck -tests=false -checks "inherit,-ST*,-SA1019,-SA9002,-SA4006" ./agent/... + +.PHONY: static-check-init +static-check-init: gocyclo govet importcheck gogenerate-check-init + # use default checks of staticcheck tool, except style checks (-ST*) + # https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck + staticcheck -tests=false -checks "inherit,-ST*" ./ecs-init/... .PHONY: goimports goimports: @@ -309,13 +347,52 @@ GOPATH=$(shell go env GOPATH) get-deps: .get-deps-stamp +get-deps-init: + go get golang.org/x/tools/cover + go get golang.org/x/tools/cmd/cover + go get github.com/golang/mock/mockgen + cd "${GOPATH}/src/github.com/golang/mock/mockgen" && git checkout 1.3.1 && go get ./... && go install ./... && cd - + GO111MODULE=on go get github.com/fzipp/gocyclo/cmd/gocyclo@v0.3.1 + go get golang.org/x/tools/cmd/goimports + go get honnef.co/go/tools/cmd/staticcheck + +.generic-rpm-done: + ./scripts/update-version.sh + cp packaging/generic-rpm/amazon-ecs-init.spec amazon-ecs-init.spec + cp packaging/generic-rpm/ecs.service ecs.service + cp packaging/generic-rpm/amazon-ecs-volume-plugin.service amazon-ecs-volume-plugin.service + cp packaging/generic-rpm/amazon-ecs-volume-plugin.socket amazon-ecs-volume-plugin.socket + tar -czf ./sources.tgz ecs-init scripts + test -e SOURCES || ln -s . SOURCES + rpmbuild --define "%_topdir $(PWD)" -bb amazon-ecs-init.spec + find RPMS/ -type f -exec cp {} . \; + touch .rpm-done + +generic-rpm: .generic-rpm-done + +dockerfree-all: dockerfree-agent-image generic-rpm + +.deb-done: BUILDROOT/ecs-agent.tar + ./scripts/update-version.sh + tar -czf ./amazon-ecs-init_${VERSION}.orig.tar.gz ecs-init scripts README.md + cp -r packaging/generic-deb/debian ecs-init scripts README.md BUILDROOT + cd BUILDROOT && debuild -uc -us --lintian-opts --suppress-tags bad-distribution-in-changes-file,file-in-unusual-dir + touch .deb-done + +deb: .deb-done + clean: # ensure docker is running and we can talk to it, abort if not: docker ps > /dev/null -docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild:make" -docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild-windows:make" - rm -f misc/certs/ca-certificates.crt &> /dev/null + rm -f misc/certs/host-certs.crt &> /dev/null + rm -rf misc/pause-container/image/ + rm -rf misc/pause-container/rootfs/ + rm -rf misc/plugins/ + rm -f misc/pause-container/amazon-ecs-pause.tar rm -rf out/ + rm -rf rootfs/ -$(MAKE) -C $(ECS_CNI_REPOSITORY_SRC_DIR) clean -$(MAKE) -C misc/netkitten $(MFLAGS) clean -$(MAKE) -C misc/volumes-test $(MFLAGS) clean @@ -329,4 +406,26 @@ clean: -rm -rf $(PWD)/bin -rm -rf cover.out -rm -rf coverprofile.out - + -rm -rf coverprofile-init.out + # ecs-init & rpm cleanup + -rm -f ecs-init.spec + -rm -f amazon-ecs-init.spec + -rm -f ecs.conf + -rm -f ecs.service + -rm -f amazon-ecs-volume-plugin.conf + -rm -f amazon-ecs-volume-plugin.service + -rm -f amazon-ecs-volume-plugin.socket + -rm -rf ./bin + -rm -f ./sources.tgz + -rm -f ./amazon-ecs-init + -rm -f ./ecs-init/ecs-init + -rm -f ./amazon-ecs-init-*.rpm + -rm -f ./ecs-agent-*.tar + -rm -f ./ecs-init-*.src.rpm + -rm -rf ./ecs-init-* + -rm -rf ./BUILDROOT BUILD RPMS SRPMS SOURCES SPECS + -rm -rf ./x86_64 + -rm -f ./amazon-ecs-init_${VERSION}* + -rm -f .srpm-done .rpm-done .generic-rpm-done + -rm -f .deb-done + -rm -f amazon-ecs-volume-plugin diff --git a/agent-container/agent-config.json b/agent-container/agent-config.json new file mode 100644 index 00000000000..30594914aac --- /dev/null +++ b/agent-container/agent-config.json @@ -0,0 +1 @@ +{"author":"Amazon Web Services, Inc.","config":{"Cmd":["/agent"],"ArgsEscaped":true},"created":"~~timestamp~~","history":[{"created":"~~timestamp~~","author":"Amazon Web Services, Inc.","created_by":"[] + [] === \"\"","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}} diff --git a/agent-container/agent-image-VERSION b/agent-container/agent-image-VERSION new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/agent-container/agent-image-VERSION @@ -0,0 +1 @@ +1.0 diff --git a/agent-container/agent-manifest.json b/agent-container/agent-manifest.json new file mode 100644 index 00000000000..01681945ec2 --- /dev/null +++ b/agent-container/agent-manifest.json @@ -0,0 +1,2 @@ +[{"Config":"config.json","RepoTags":["amazon/amazon-ecs-agent:~~agentversion~~"],"Layers":["rootfs/layer.tar"]}] + diff --git a/agent-container/agent-repositories b/agent-container/agent-repositories new file mode 100644 index 00000000000..91624061cc3 --- /dev/null +++ b/agent-container/agent-repositories @@ -0,0 +1 @@ +{"amazon/amazon-ecs-agent":{"amazon-ecs":"rootfs"}} diff --git a/dependencies_mocks.go b/dependencies_mocks.go new file mode 100644 index 00000000000..801b41dcfbd --- /dev/null +++ b/dependencies_mocks.go @@ -0,0 +1,15 @@ +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +// +// Source: dependencies.go in package cache + diff --git a/ecs-init/cache/cache_test.go b/ecs-init/cache/cache_test.go index e9d9e8b4a06..52e8446b905 100644 --- a/ecs-init/cache/cache_test.go +++ b/ecs-init/cache/cache_test.go @@ -1,3 +1,4 @@ +// +build test // Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may diff --git a/ecs-init/cache/dependencies.go b/ecs-init/cache/dependencies.go index 735d81419d4..161964a25a7 100644 --- a/ecs-init/cache/dependencies.go +++ b/ecs-init/cache/dependencies.go @@ -18,7 +18,7 @@ package cache // package-level functions. These interfaces are then used to create mocks // for the unit tests. -//go:generate mockgen.sh $GOPACKAGE $GOFILE +//go:generate mockgen.sh cache $GOFILE import ( "io" diff --git a/ecs-init/cache/dependencies_mocks.go b/ecs-init/cache/dependencies_mocks.go index 95780f23ccd..0d41276ce13 100644 --- a/ecs-init/cache/dependencies_mocks.go +++ b/ecs-init/cache/dependencies_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/config/config_al2.go b/ecs-init/config/config_al2.go index b6d94cd0f37..c7ef64bc872 100644 --- a/ecs-init/config/config_al2.go +++ b/ecs-init/config/config_al2.go @@ -1,3 +1,4 @@ +//go:build al2 // +build al2 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/ecs-init/config/config_generic_rpm.go b/ecs-init/config/config_generic_rpm.go index 695c639f410..5dc81bd611a 100644 --- a/ecs-init/config/config_generic_rpm.go +++ b/ecs-init/config/config_generic_rpm.go @@ -1,3 +1,4 @@ +//go:build generic_rpm // +build generic_rpm // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/ecs-init/config/config_suse_ubuntu_debian.go b/ecs-init/config/config_suse_ubuntu_debian.go index b291c4f46d3..cdbc4e543cf 100644 --- a/ecs-init/config/config_suse_ubuntu_debian.go +++ b/ecs-init/config/config_suse_ubuntu_debian.go @@ -1,3 +1,4 @@ +//go:build suse || ubuntu || debian // +build suse ubuntu debian // Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/ecs-init/config/config_unspecified.go b/ecs-init/config/config_unspecified.go index 0977af8fe3c..04c77b44c2f 100644 --- a/ecs-init/config/config_unspecified.go +++ b/ecs-init/config/config_unspecified.go @@ -1,3 +1,4 @@ +//go:build !suse && !ubuntu && !al2 && !debian && !generic_rpm // +build !suse,!ubuntu,!al2,!debian,!generic_rpm // Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/ecs-init/config/development.go b/ecs-init/config/development.go index 071231eb89e..c4afbd4c427 100644 --- a/ecs-init/config/development.go +++ b/ecs-init/config/development.go @@ -1,3 +1,4 @@ +//go:build development // +build development // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/ecs-init/config/logger.go b/ecs-init/config/logger.go index 20699f3c885..9e1093dc894 100644 --- a/ecs-init/config/logger.go +++ b/ecs-init/config/logger.go @@ -34,7 +34,7 @@ and limitations under the License. - diff --git a/ecs-init/config/release.go b/ecs-init/config/release.go index e5338f65acc..ee182136e89 100644 --- a/ecs-init/config/release.go +++ b/ecs-init/config/release.go @@ -1,3 +1,4 @@ +//go:build !development // +build !development // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/ecs-init/docker/backoff_mocks.go b/ecs-init/docker/backoff_mocks.go index 5e78e9298f4..e00aef56d12 100644 --- a/ecs-init/docker/backoff_mocks.go +++ b/ecs-init/docker/backoff_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/docker/dependencies_mocks.go b/ecs-init/docker/dependencies_mocks.go index ba0f695fae6..8b5ff299e41 100644 --- a/ecs-init/docker/dependencies_mocks.go +++ b/ecs-init/docker/dependencies_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/docker/dependencies_test.go b/ecs-init/docker/dependencies_test.go index d5418c665e6..01220bc92a6 100644 --- a/ecs-init/docker/dependencies_test.go +++ b/ecs-init/docker/dependencies_test.go @@ -1,3 +1,4 @@ +// +build test // Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may diff --git a/ecs-init/docker/docker_test.go b/ecs-init/docker/docker_test.go index 644f4ff6db1..2059acbd65b 100644 --- a/ecs-init/docker/docker_test.go +++ b/ecs-init/docker/docker_test.go @@ -1,3 +1,4 @@ +// +build test // Copyright 2015-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may diff --git a/ecs-init/engine/dependencies_mocks.go b/ecs-init/engine/dependencies_mocks.go index a3646d5b6ab..07d50263831 100644 --- a/ecs-init/engine/dependencies_mocks.go +++ b/ecs-init/engine/dependencies_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/engine/engine_test.go b/ecs-init/engine/engine_test.go index e1582cc7c48..2b706fd487e 100644 --- a/ecs-init/engine/engine_test.go +++ b/ecs-init/engine/engine_test.go @@ -1,3 +1,4 @@ +// +build test // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may diff --git a/ecs-init/exec/iptables/cmd_mocks.go b/ecs-init/exec/iptables/cmd_mocks.go index b539a3c34c5..19a74282468 100644 --- a/ecs-init/exec/iptables/cmd_mocks.go +++ b/ecs-init/exec/iptables/cmd_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/exec/iptables/exec_mocks.go b/ecs-init/exec/iptables/exec_mocks.go index ef516d91f69..75ebcc4c825 100644 --- a/ecs-init/exec/iptables/exec_mocks.go +++ b/ecs-init/exec/iptables/exec_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/exec/sysctl/cmd_mocks.go b/ecs-init/exec/sysctl/cmd_mocks.go index 27775fbf55a..6398c63252d 100644 --- a/ecs-init/exec/sysctl/cmd_mocks.go +++ b/ecs-init/exec/sysctl/cmd_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/exec/sysctl/exec_mocks.go b/ecs-init/exec/sysctl/exec_mocks.go index 2c24e8bfe9f..74fcbcd94bb 100644 --- a/ecs-init/exec/sysctl/exec_mocks.go +++ b/ecs-init/exec/sysctl/exec_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/ecs-init/gpu/nvidia_gpu_manager_mocks.go b/ecs-init/gpu/nvidia_gpu_manager_mocks.go index 0f89774a3da..c86e30ea77b 100644 --- a/ecs-init/gpu/nvidia_gpu_manager_mocks.go +++ b/ecs-init/gpu/nvidia_gpu_manager_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/misc/pause-container/pause-config.json b/misc/pause-container/pause-config.json new file mode 100644 index 00000000000..f59eee6a7ef --- /dev/null +++ b/misc/pause-container/pause-config.json @@ -0,0 +1,2 @@ +{"author":"Amazon Web Services, Inc.","config":{"Cmd":["/pause"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Amazon Web Services, Inc.","created_by":"[] + [] === \"\"","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}} + diff --git a/misc/pause-container/pause-image-VERSION b/misc/pause-container/pause-image-VERSION new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/misc/pause-container/pause-image-VERSION @@ -0,0 +1 @@ +1.0 diff --git a/misc/pause-container/pause-manifest.json b/misc/pause-container/pause-manifest.json new file mode 100644 index 00000000000..92f89910ffd --- /dev/null +++ b/misc/pause-container/pause-manifest.json @@ -0,0 +1,2 @@ +[{"Config":"config.json","RepoTags":["amazon/amazon-ecs-pause:0.1.0"],"Layers":["rootfs/layer.tar"]}] + diff --git a/misc/pause-container/pause-repositories b/misc/pause-container/pause-repositories new file mode 100644 index 00000000000..e2564e0b801 --- /dev/null +++ b/misc/pause-container/pause-repositories @@ -0,0 +1 @@ +{"amazon/amazon-ecs-pause":{"amazon-ecs":"rootfs"}} diff --git a/packaging/generic-rpm/amazon-ecs-init.spec b/packaging/generic-rpm/amazon-ecs-init.spec index ea3bca2b6b5..ceb7e79291c 100644 --- a/packaging/generic-rpm/amazon-ecs-init.spec +++ b/packaging/generic-rpm/amazon-ecs-init.spec @@ -42,7 +42,6 @@ Source4: https://s3.amazonaws.com/amazon-ecs-agent/ecs-agent-arm64-v%{bun Source5: amazon-ecs-volume-plugin.service Source6: amazon-ecs-volume-plugin.socket -BuildRequires: golang >= 1.7 BuildRequires: systemd Requires: systemd Requires: iptables diff --git a/scripts/analyze-cover-profile-init b/scripts/analyze-cover-profile-init new file mode 100755 index 00000000000..fa9be691767 --- /dev/null +++ b/scripts/analyze-cover-profile-init @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +MINIMUM_TEST_COVERAGE = 73.7 + +# coverprofile-init.out (see Makefile for how this is generated) will look something like this: +# +# github.com/aws/amazon-ecs-agent/agent/wsclient/error.go:66: Retry 0.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/errors.go:23: Error 0.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/errors.go:28: Retry 0.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/errors.go:41: Retry 0.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/errors.go:46: Error 0.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/errors.go:59: Error 0.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/types.go:40: BuildTypeDecoder 100.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/types.go:49: NewOfType 0.0% +# github.com/aws/amazon-ecs-agent/agent/wsclient/types.go:57: GetRecognizedTypes 100.0% +# total: (statements) 73.8% + +with open("coverprofile-init.out") as f: + for line in f: + if line.startswith("total:"): + splitline = line.split() + print("Total unit test coverage: " + splitline[2]) + coverage = float(splitline[2].rstrip("%")) + if coverage < MINIMUM_TEST_COVERAGE: + raise BaseException( + "Unit test coverage ({0}%) is below the minimum ({1}%)". + format(coverage, MINIMUM_TEST_COVERAGE)) diff --git a/scripts/build b/scripts/build index ffc5969c882..0990096a8b0 100755 --- a/scripts/build +++ b/scripts/build @@ -25,6 +25,11 @@ set -ex static=${1:-true} output_directory=${2:-} version_gen=${3:-true} +with_pause=${4:-false} + +PAUSE_CONTAINER_IMAGE="amazon/amazon-ecs-pause" +PAUSE_CONTAINER_TAG="0.1.0" +PAUSE_CONTAINER_TARBALL="amazon-ecs-pause.tar" # Normalize to working directory being build root (up one level from ./scripts) ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd ) @@ -48,6 +53,10 @@ if [[ "${version_gen}" == "true" ]]; then GO111MODULE=off go run gen/version-gen.go fi +if [[ "${with_pause}" == "true" ]]; then + LDFLAGS="-X github.com/aws/amazon-ecs-agent/agent/config.DefaultPauseContainerTag=$PAUSE_CONTAINER_TAG -X github.com/aws/amazon-ecs-agent/agent/config.DefaultPauseContainerImageName=$PAUSE_CONTAINER_IMAGE" +fi + if [ "${TARGET_OS}" == "windows" ]; then unset static build_exe="out/amazon-ecs-agent.exe" diff --git a/scripts/build-agent-image b/scripts/build-agent-image new file mode 100755 index 00000000000..32f18926128 --- /dev/null +++ b/scripts/build-agent-image @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# This script builds a copy of the agent. +# It exists to wrap go build and properly make a static binary, as well as to +# correctly setup versioning before creating the binary + +set -ex + +ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd ) +cd "${ROOT}" +AGENT_VERSION=$(cat VERSION) + +# add cni-plugins +mkdir -p rootfs/amazon-ecs-cni-plugins/ +cp ./misc/plugins/aws-appmesh rootfs/amazon-ecs-cni-plugins/aws-appmesh +cp ./misc/plugins/ecs-bridge rootfs/amazon-ecs-cni-plugins/ecs-bridge +cp ./misc/plugins/ecs-eni rootfs/amazon-ecs-cni-plugins/ecs-eni +cp ./misc/plugins/ecs-ipam rootfs/amazon-ecs-cni-plugins/ecs-ipam +cp ./misc/plugins/vpc-branch-eni rootfs/amazon-ecs-cni-plugins/vpc-branch-eni + +# add certs +mkdir -p rootfs/etc/ssl/certs/ +cp ./misc/certs/host-certs.crt rootfs/etc/ssl/certs/ca-certificates.crt + +# add pause container +mkdir -p rootfs/images/ +cp ./misc/pause-container/amazon-ecs-pause.tar rootfs/images/amazon-ecs-pause.tar + +# add agent +cp ./out/amazon-ecs-agent rootfs/agent + +# build container +mkdir -p image/rootfs +tar --mtime="@1492525740" --owner=0 --group=0 --numeric-owner -cf image/rootfs/layer.tar -C rootfs . +DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//') +install -m 0644 ./agent-container/agent-image-VERSION image/rootfs/VERSION +install -m 0644 ./agent-container/agent-config.json image/config.json +sed -i "s/~~digest~~/${DIGEST}/" image/config.json +sed -i "s/~~timestamp~~/$(date +"%FT%T.%NZ")/g" image/config.json +install -m 0644 ./agent-container/agent-manifest.json image/manifest.json +sed -i "s/~~agentversion~~/${AGENT_VERSION}/" image/manifest.json +install -m 0644 ./agent-container/agent-repositories image/repositories +tar --mtime="@1492525740" --owner=0 --group=0 --numeric-owner -cf ./ecs-agent-v${AGENT_VERSION}.tar -C image . +rm -rf image/ diff --git a/scripts/build-cni-plugins b/scripts/build-cni-plugins new file mode 100755 index 00000000000..097471f1861 --- /dev/null +++ b/scripts/build-cni-plugins @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# this script builds the ecs/vpc cni plugins from the submodules +# by copying them out of agent into their expected location in the +# gopath + +set -ex + +# this script assumes we've run the get-cni-sources make target to update the cni submodules +ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd ) +cd "${ROOT}" +# copy submodules to expected go build path github.com/aws/amazon-ecs-cni-plugins +cp -r amazon-ecs-cni-plugins ../amazon-ecs-cni-plugins +cp -r amazon-vpc-cni-plugins ../amazon-vpc-cni-plugins +cd ../amazon-ecs-cni-plugins && GO111MODULE=auto make plugins +mkdir -p ../amazon-ecs-agent/misc/plugins && cp -a ./bin/plugins/. ../amazon-ecs-agent/misc/plugins/ +make clean +cd ../amazon-vpc-cni-plugins && GO111MODULE=auto make build +cp -a ./build/linux_amd64/. ../amazon-ecs-agent/misc/plugins/ +make clean diff --git a/scripts/build-pause b/scripts/build-pause new file mode 100755 index 00000000000..d013a6b65f4 --- /dev/null +++ b/scripts/build-pause @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# This script builds the pause container without Docker +# The pause container configuration is static, except for the DIGEST + +set -ex + +ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd ) +cd "${ROOT}/misc/pause-container" +mkdir -p rootfs/ +gcc -static pause.c -o rootfs/pause +mkdir -p image/rootfs +tar --mtime="@1492525740" --owner=0 --group=0 --numeric-owner -cf image/rootfs/layer.tar -C rootfs . +DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//') +install -m 0644 pause-image-VERSION image/rootfs/VERSION +install -m 0644 pause-config.json image/config.json +sed -i "s/~~digest~~/${DIGEST}/" image/config.json +install -m 0644 pause-manifest.json image/manifest.json +install -m 0644 pause-repositories image/repositories +tar --mtime="@1492525740" --owner=0 --group=0 --numeric-owner -cf ./amazon-ecs-pause.tar -C image . diff --git a/scripts/get-host-certs b/scripts/get-host-certs new file mode 100755 index 00000000000..e53f1e6eb18 --- /dev/null +++ b/scripts/get-host-certs @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# this script searches through known locations for certs (borrowed from +# https://go.dev/src/crypto/x509/root_linux.go) and writes them to the +# current dir + +set -ex + +ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd ) +cd "${ROOT}/misc/certs" + +# Possible certificate files +certFiles=("/etc/ssl/certs/ca-certificates.crt" "/etc/pki/tls/certs/ca-bundle.crt" "/etc/ssl/ca-bundle.pem" "/etc/pki/tls/cacert.pem" "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" "/etc/ssl/cert.pem") + +# search and stop once certs are found +thisCertFile="" +for file in ${certFiles[@]}; do + if test -f $file; then + thisCertFile=$file + break + fi +done + +# if we found a cert file, we'll copy else return 0 +if [ ! -z "$thisCertFile" ] +then + cp $thisCertFile ./host-certs.crt +else + exit 1 +fi