From 6c4a8ad7b83352f3fb62ee860a3a70dd373319e5 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 26 Dec 2019 22:38:07 +0000 Subject: [PATCH 01/26] bin: use /usr/bin/env for portability --- bin/amiize.sh | 2 +- bin/upload-sources | 2 +- tools/docker-go | 2 +- tools/gen-docs.sh | 2 +- tools/rpm2img | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/amiize.sh b/bin/amiize.sh index d69b19a55fd..89ad9b69a73 100755 --- a/bin/amiize.sh +++ b/bin/amiize.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Register partitioned root and data images as an AMI in EC2. # Only registers with HVM virtualization type and GP2 EBS volume type. diff --git a/bin/upload-sources b/bin/upload-sources index cb25185577d..e7bbf21d708 100755 --- a/bin/upload-sources +++ b/bin/upload-sources @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/tools/docker-go b/tools/docker-go index 25c52005a1a..15516443fc4 100755 --- a/tools/docker-go +++ b/tools/docker-go @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Helper script for running commands in a golang build/runtime environment for testing/vendoring/building a go module diff --git a/tools/gen-docs.sh b/tools/gen-docs.sh index 2ae32cea42f..a83dd20c858 100755 --- a/tools/gen-docs.sh +++ b/tools/gen-docs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash DOCS=(START.md README.md INSTALL.md CHANGELOG.md extras/dogswatch/README.md) EXTRAS=(extras/dogswatch/{dogswatch,dev/deployment}.yaml) diff --git a/tools/rpm2img b/tools/rpm2img index 3c92cbb69b6..65411776d0b 100755 --- a/tools/rpm2img +++ b/tools/rpm2img @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu -o pipefail shopt -qs failglob From 49dae10e5bc67797f086c365b152c67aee5b856c Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 26 Dec 2019 22:38:43 +0000 Subject: [PATCH 02/26] ci: use environment's desired certificate bundle --- Makefile.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.toml b/Makefile.toml index f7b5ce06082..a536e6cc8d3 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -57,6 +57,8 @@ script = [ ''' set -o pipefail if ! docker image inspect ${BUILDSYS_SDK_IMAGE} >/dev/null 2>&1 ; then + # Let curl resolve the certificates instead of the tasks resolved bundle. + unset SSL_CERT_FILE SSL_CERT_DIR if ! curl https://thar-upstream-lookaside-cache.s3.us-west-2.amazonaws.com/${BUILDSYS_SDK_IMAGE}.tar.gz \ | gunzip | docker load ; then echo "failed to load '${BUILDSYS_SDK_IMAGE}'" >&2 From 74522c2caeea463750ee953a320d5fdc2f2d87b8 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 26 Dec 2019 22:34:57 +0000 Subject: [PATCH 03/26] ci: create common environment image --- tools/infra/.dockerignore | 4 ++++ tools/infra/Dockerfile | 18 ++++++++++++++++ tools/infra/env/bin/setup-rust-builder | 2 +- tools/infra/env/bin/start-build-environment | 5 +++++ tools/infra/env/bin/start-docker-daemon | 23 +++++++++++++++++++++ tools/infra/env/bin/start-log-listener | 13 ++++++++++++ tools/infra/env/container/entrypoint.sh | 4 ++++ 7 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tools/infra/.dockerignore create mode 100644 tools/infra/Dockerfile create mode 100755 tools/infra/env/bin/start-build-environment create mode 100755 tools/infra/env/bin/start-docker-daemon create mode 100755 tools/infra/env/bin/start-log-listener create mode 100755 tools/infra/env/container/entrypoint.sh diff --git a/tools/infra/.dockerignore b/tools/infra/.dockerignore new file mode 100644 index 00000000000..1a29ad78ca8 --- /dev/null +++ b/tools/infra/.dockerignore @@ -0,0 +1,4 @@ +stacks/ +buildspec/ +.direnv/ +.envrc diff --git a/tools/infra/Dockerfile b/tools/infra/Dockerfile new file mode 100644 index 00000000000..51d0ea2f4e0 --- /dev/null +++ b/tools/infra/Dockerfile @@ -0,0 +1,18 @@ +FROM amazonlinux:2 as base +RUN yum update -y \ + && yum groupinstall -y 'Development Tools' \ + && yum install -y socat procps-ng \ + && amazon-linux-extras install -y docker \ + && yum clean all \ + && rm -rf /var/cache/yum + +FROM base +ENV PATH="$PATH:/build/env/bin:/build/.cargo/bin" +ENV CARGO_HOME="/build/.cargo" +COPY env/ /build/env/ +WORKDIR /build + +RUN source setup-rust-builder + +ENTRYPOINT ["/build/env/container/entrypoint.sh"] +CMD [ "bash" ] diff --git a/tools/infra/env/bin/setup-rust-builder b/tools/infra/env/bin/setup-rust-builder index f242e7885b6..d305f328a05 100644 --- a/tools/infra/env/bin/setup-rust-builder +++ b/tools/infra/env/bin/setup-rust-builder @@ -57,7 +57,7 @@ install_rust_toolchain() { logger -s -t ERROR "could not fetch rustup, needed for managing rust toolchain" exit 1 fi - if ! bash rustup-init.sh -y --no-modify-path --default-toolchain stable ; then + if ! bash rustup-init.sh -y --profile minimal --no-modify-path --default-toolchain stable ; then logger -s -t ERROR "could not setup rustup & rust toolchain for build" exit 1 fi diff --git a/tools/infra/env/bin/start-build-environment b/tools/infra/env/bin/start-build-environment new file mode 100755 index 00000000000..2f2bc477a17 --- /dev/null +++ b/tools/infra/env/bin/start-build-environment @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -xe +start-log-listener +logger -s -t INFO "starting docker daemon" +start-docker-daemon diff --git a/tools/infra/env/bin/start-docker-daemon b/tools/infra/env/bin/start-docker-daemon new file mode 100755 index 00000000000..de26bbb0c15 --- /dev/null +++ b/tools/infra/env/bin/start-docker-daemon @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -e + +nohup dockerd \ + --host=unix:///var/run/docker.sock \ + &>/var/log/docker.log & +daemon_pid="$!" +sleep 1 +if [[ ! -e "/proc/$daemon_pid" ]]; then + echo "ERROR: docker daemon did not start up" + exit 1 +fi + +tries=0 +max_tries=5 +until docker info &>/dev/null; do + ((tries++)) + if [[ "$tries" -gt "$max_tries" ]]; then + echo "ERROR: unable to start dockerd for build" >&2 + exit 1 + fi + sleep 1 +done diff --git a/tools/infra/env/bin/start-log-listener b/tools/infra/env/bin/start-log-listener new file mode 100755 index 00000000000..f9b4e47c9b8 --- /dev/null +++ b/tools/infra/env/bin/start-log-listener @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +hash socat +# Listen on syslog domain socket to make for cleaner output. +nohup socat UNIX-LISTEN:/dev/log,fork - &>/dev/null & +tries=0 +until test -S /dev/log; do + ((tries++)) + if [[ "$tries" -gt 3 ]]; then + echo "ERROR: unable to start log listener" >&2 + exit 1 + fi + sleep 0.1s +done diff --git a/tools/infra/env/container/entrypoint.sh b/tools/infra/env/container/entrypoint.sh new file mode 100755 index 00000000000..6c31cdf3d3d --- /dev/null +++ b/tools/infra/env/container/entrypoint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e +start-build-environment +exec -- "$@" From 560d1d1852fa7d97daf8f54a7ea3c0f330065526 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Wed, 1 Jan 2020 01:17:40 +0000 Subject: [PATCH 04/26] ci: add image builder with docs --- tools/infra/container/.dockerignore | 1 + tools/infra/container/Makefile | 20 +++++++++++++++ tools/infra/container/README.md | 39 +++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 tools/infra/container/.dockerignore create mode 100644 tools/infra/container/Makefile create mode 100644 tools/infra/container/README.md diff --git a/tools/infra/container/.dockerignore b/tools/infra/container/.dockerignore new file mode 100644 index 00000000000..f3c7a7c5da6 --- /dev/null +++ b/tools/infra/container/.dockerignore @@ -0,0 +1 @@ +Makefile diff --git a/tools/infra/container/Makefile b/tools/infra/container/Makefile new file mode 100644 index 00000000000..8274d2d6c75 --- /dev/null +++ b/tools/infra/container/Makefile @@ -0,0 +1,20 @@ +DOCKERFILES = $(filter-out %~,$(wildcard Dockerfile.*)) +NAMES = $(DOCKERFILES:Dockerfile.%=%) + +IMAGE_REPO_PREFIX ?= infra/ +IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(IMAGE_TAG) + +.PHONY: force all release $(NAMES) +.DEFAULT: all +force: + +all : IMAGE_TAG ?= develop +all: $(NAMES) + +release : IMAGE_TAG ?= latest +release: $(if $(NAME),$(NAME),$(NAMES)) + +$(NAMES) : NAME = $@ +$(NAMES): force + @echo "Building container image for '$(NAME)'" + docker build -t $(IMAGE_NAME) -f Dockerfile.$(NAME) . diff --git a/tools/infra/container/README.md b/tools/infra/container/README.md new file mode 100644 index 00000000000..13383be6156 --- /dev/null +++ b/tools/infra/container/README.md @@ -0,0 +1,39 @@ +# Container Environments + +Container images, defined in this directory, provide environments for infra's build and automation needs. + +## Images + +Each image is defined in their own `Dockerfile` and suffixed with its name. For example the `builder` container - used in CI builds - is defined by `Dockerfile.builder`. +The containers copy in common resources and others as needed from this shared root context. + +**`builder` image** + +The `builder` image provides an environment in which packages and images may be built. +`builder`'s container image is created with all required dependencies used by the build driver, `buildsys`, and the supporting tools & scripts used by it (including many of the `cargo-make` tasks' dependencies). + +# Building + +## Development Images + +To all build images locally, a single `make` call can be made: + +```bash +make all +``` + +Each `Dockerfile.` can be built individually with `make $name` as needed. + +## Release Images + +As with the development images, all images may be built at once: + +```bash +make release +``` + +To build a specific image, for instance named `builder`, `make` may be provided this name to build its release image: + +```bash +make release NAME=builder +``` From 22bcf85dc0ce52031f8ca9170fec3facd3dfe35c Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 26 Dec 2019 22:57:05 +0000 Subject: [PATCH 05/26] ci: add stage dependencies Additional tools are required to run the AMI building stage and are common enough to merit being installed by default - eg: aws cli. --- tools/infra/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/infra/Dockerfile b/tools/infra/Dockerfile index 51d0ea2f4e0..15ff124a06c 100644 --- a/tools/infra/Dockerfile +++ b/tools/infra/Dockerfile @@ -1,10 +1,10 @@ FROM amazonlinux:2 as base RUN yum update -y \ && yum groupinstall -y 'Development Tools' \ - && yum install -y socat procps-ng \ + && yum install -y socat procps-ng awscli jq openssh rsync \ && amazon-linux-extras install -y docker \ && yum clean all \ - && rm -rf /var/cache/yum + && rm -rf /var/cache/yum /var/cache/amzn2extras FROM base ENV PATH="$PATH:/build/env/bin:/build/.cargo/bin" From 715637d51f807e7586a4a9321fbe14626abd6eb8 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 26 Dec 2019 15:10:47 -0800 Subject: [PATCH 06/26] ci: run with build environment This uses the commands provided in the build environment instead of performing the on-demand installation for building within. --- tools/infra/buildspec/thar-pr-build.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tools/infra/buildspec/thar-pr-build.yml b/tools/infra/buildspec/thar-pr-build.yml index 2c793d43bcf..2cfc7e5937f 100644 --- a/tools/infra/buildspec/thar-pr-build.yml +++ b/tools/infra/buildspec/thar-pr-build.yml @@ -6,20 +6,13 @@ env: INFRA_DIR: "./tools/infra" phases: - install: - runtime-versions: - docker: 18 - commands: - - . "${INFRA_DIR}/env/lib/environment-setup" - - . setup-rust-builder pre_build: commands: + - start-build-environment - environment-report - write-build-meta build: commands: - # Retry fetches a few times before failing - - retry 2 'fetch dependencies' -- cargo make fetch - cargo make world artifacts: From 7c57aeb07786d87a4b1c8e655356565ac3142455 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 26 Dec 2019 15:29:44 -0800 Subject: [PATCH 07/26] ci: fix CI env checker --- tools/infra/env/bin/write-build-meta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/infra/env/bin/write-build-meta b/tools/infra/env/bin/write-build-meta index 8636072b919..1e15292271d 100755 --- a/tools/infra/env/bin/write-build-meta +++ b/tools/infra/env/bin/write-build-meta @@ -31,7 +31,7 @@ fi write_common -if [[ -z "${CODEBUILD_BUILD_ID}" ]]; then +if [[ -n "${CODEBUILD_BUILD_ID}" ]]; then write_codebuild fi From 544913d1d212ab33e245f421dc4aab446bd95807 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Wed, 1 Jan 2020 01:12:28 +0000 Subject: [PATCH 08/26] ci: use container dir for images and supporting scripts --- tools/infra/.dockerignore | 4 -- .../Dockerfile.builder} | 13 ++++-- .../builder}/entrypoint.sh | 0 .../runtime}/bin/environment-report | 0 .../{env => container/runtime}/bin/retry | 0 .../runtime}/bin/start-build-environment | 0 .../runtime}/bin/start-docker-daemon | 0 .../runtime}/bin/start-log-listener | 0 .../runtime}/bin/write-build-meta | 0 .../scripts/configure-rust} | 41 ++----------------- tools/infra/container/scripts/install-crates | 22 ++++++++++ tools/infra/container/scripts/install-rust | 24 +++++++++++ tools/infra/env/README.md | 7 ---- tools/infra/env/lib/environment-setup | 15 ------- 14 files changed, 58 insertions(+), 68 deletions(-) delete mode 100644 tools/infra/.dockerignore rename tools/infra/{Dockerfile => container/Dockerfile.builder} (56%) rename tools/infra/{env/container => container/builder}/entrypoint.sh (100%) rename tools/infra/{env => container/runtime}/bin/environment-report (100%) rename tools/infra/{env => container/runtime}/bin/retry (100%) rename tools/infra/{env => container/runtime}/bin/start-build-environment (100%) rename tools/infra/{env => container/runtime}/bin/start-docker-daemon (100%) rename tools/infra/{env => container/runtime}/bin/start-log-listener (100%) rename tools/infra/{env => container/runtime}/bin/write-build-meta (100%) rename tools/infra/{env/bin/setup-rust-builder => container/scripts/configure-rust} (50%) mode change 100644 => 100755 create mode 100755 tools/infra/container/scripts/install-crates create mode 100755 tools/infra/container/scripts/install-rust delete mode 100644 tools/infra/env/README.md delete mode 100644 tools/infra/env/lib/environment-setup diff --git a/tools/infra/.dockerignore b/tools/infra/.dockerignore deleted file mode 100644 index 1a29ad78ca8..00000000000 --- a/tools/infra/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -stacks/ -buildspec/ -.direnv/ -.envrc diff --git a/tools/infra/Dockerfile b/tools/infra/container/Dockerfile.builder similarity index 56% rename from tools/infra/Dockerfile rename to tools/infra/container/Dockerfile.builder index 15ff124a06c..14ee95de6dd 100644 --- a/tools/infra/Dockerfile +++ b/tools/infra/container/Dockerfile.builder @@ -7,12 +7,17 @@ RUN yum update -y \ && rm -rf /var/cache/yum /var/cache/amzn2extras FROM base -ENV PATH="$PATH:/build/env/bin:/build/.cargo/bin" +ENV PATH="$PATH:/build/runtime/bin:/build/scripts:/build/.cargo/bin" ENV CARGO_HOME="/build/.cargo" -COPY env/ /build/env/ + +COPY scripts /build/scripts WORKDIR /build -RUN source setup-rust-builder +RUN install-rust && configure-rust && install-crates + +COPY runtime /build/runtime +COPY builder/entrypoint.sh /build/entrypoint.sh + +ENTRYPOINT ["/build/entrypoint.sh"] -ENTRYPOINT ["/build/env/container/entrypoint.sh"] CMD [ "bash" ] diff --git a/tools/infra/env/container/entrypoint.sh b/tools/infra/container/builder/entrypoint.sh similarity index 100% rename from tools/infra/env/container/entrypoint.sh rename to tools/infra/container/builder/entrypoint.sh diff --git a/tools/infra/env/bin/environment-report b/tools/infra/container/runtime/bin/environment-report similarity index 100% rename from tools/infra/env/bin/environment-report rename to tools/infra/container/runtime/bin/environment-report diff --git a/tools/infra/env/bin/retry b/tools/infra/container/runtime/bin/retry similarity index 100% rename from tools/infra/env/bin/retry rename to tools/infra/container/runtime/bin/retry diff --git a/tools/infra/env/bin/start-build-environment b/tools/infra/container/runtime/bin/start-build-environment similarity index 100% rename from tools/infra/env/bin/start-build-environment rename to tools/infra/container/runtime/bin/start-build-environment diff --git a/tools/infra/env/bin/start-docker-daemon b/tools/infra/container/runtime/bin/start-docker-daemon similarity index 100% rename from tools/infra/env/bin/start-docker-daemon rename to tools/infra/container/runtime/bin/start-docker-daemon diff --git a/tools/infra/env/bin/start-log-listener b/tools/infra/container/runtime/bin/start-log-listener similarity index 100% rename from tools/infra/env/bin/start-log-listener rename to tools/infra/container/runtime/bin/start-log-listener diff --git a/tools/infra/env/bin/write-build-meta b/tools/infra/container/runtime/bin/write-build-meta similarity index 100% rename from tools/infra/env/bin/write-build-meta rename to tools/infra/container/runtime/bin/write-build-meta diff --git a/tools/infra/env/bin/setup-rust-builder b/tools/infra/container/scripts/configure-rust old mode 100644 new mode 100755 similarity index 50% rename from tools/infra/env/bin/setup-rust-builder rename to tools/infra/container/scripts/configure-rust index d305f328a05..89042e494b7 --- a/tools/infra/env/bin/setup-rust-builder +++ b/tools/infra/container/scripts/configure-rust @@ -1,13 +1,10 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash # -# setup-rust-builder - install rust and configure shell for its use by a builder -# -# This script is expected to be sourced by callers in order to affect their PATH -# directly AND is used in CodeBuild environments where the SHELL is /bin/sh. +# confgure-rust - configure container environment for rust usage # # usage: # -# . setup-rust-builder +# configure-rust # configure_color_output() { @@ -42,37 +39,5 @@ configure_color_output() { unset cargo_make_config } -cargo_dep() { - cargo_install_package="$@" - logger -s -t INFO "installing cargo dep with 'cargo install $cargo_install_package'" - if ! cargo install --force $cargo_install_package; then - logger -s -t ERROR "failed to install dep with 'cargo install $cargo_install_package'" - exit 1 - fi -} - -install_rust_toolchain() { - test -f rustup-init.sh || rm -f rustup-init.sh - if ! curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sS "https://sh.rustup.rs"; then - logger -s -t ERROR "could not fetch rustup, needed for managing rust toolchain" - exit 1 - fi - if ! bash rustup-init.sh -y --profile minimal --no-modify-path --default-toolchain stable ; then - logger -s -t ERROR "could not setup rustup & rust toolchain for build" - exit 1 - fi - rm -f rustup-init.sh -} - configure_color_output -install_rust_toolchain - -# Install build tooling dependencies -cargo_dep --version 0.23.0 cargo-make -cargo_dep --version 0.2.6 cargo-deny - -unset -f configure_color_output -unset -f install_rust_toolchain -unset -f cargo_dep - diff --git a/tools/infra/container/scripts/install-crates b/tools/infra/container/scripts/install-crates new file mode 100755 index 00000000000..6ebcf628790 --- /dev/null +++ b/tools/infra/container/scripts/install-crates @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# install-crates - install binary crate dependencies +# +# usage: +# +# install-rust +# + +cargo_dep() { + cargo_install_package="$@" + logger -s -t INFO "installing cargo dep with 'cargo install $cargo_install_package'" + if ! cargo install --force $cargo_install_package; then + logger -s -t ERROR "failed to install dep with 'cargo install $cargo_install_package'" + exit 1 + fi +} + +# Install build tooling dependencies +cargo_dep --version 0.23.0 cargo-make +cargo_dep --version 0.2.6 cargo-deny + diff --git a/tools/infra/container/scripts/install-rust b/tools/infra/container/scripts/install-rust new file mode 100755 index 00000000000..9ab46d49c1d --- /dev/null +++ b/tools/infra/container/scripts/install-rust @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# install-rust - install rust for use in builder containers +# +# usage: +# +# install-rust +# + +install_rust_toolchain() { + local toolchain="${1:-stable}" + test -f rustup-init.sh || rm -f rustup-init.sh + if ! curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sS "https://sh.rustup.rs"; then + logger -s -t ERROR "could not fetch rustup, needed for managing rust toolchain" + exit 1 + fi + if ! bash rustup-init.sh -y --profile minimal --no-modify-path --default-toolchain "$toolchain" ; then + logger -s -t ERROR "could not setup rustup & rust toolchain for build" + exit 1 + fi + rm -f rustup-init.sh +} + +install_rust_toolchain "stable" diff --git a/tools/infra/env/README.md b/tools/infra/env/README.md deleted file mode 100644 index 97189cf3585..00000000000 --- a/tools/infra/env/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# CI Builder Environment - -This directory provides the working environment within the context of a CI build. -The `lib/` directory contains bootstrap and support resources that are not otherwise not directly executed. -The `bin/` directory contains scripts and executables that are directly used in the build process. -Scripts named `setup-$NAME` are intended to be sourced into the build environment; scripts sourced into the build environment must be compatible with the bourne shell - `/bin/sh`. - diff --git a/tools/infra/env/lib/environment-setup b/tools/infra/env/lib/environment-setup deleted file mode 100644 index 5397068b3d3..00000000000 --- a/tools/infra/env/lib/environment-setup +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env sh -# -# environment-setup - setup shell for building in -# -if [ -d "${INFRA_DIR:-./tools/infra}/env/bin" ]; then - PATH="$(pwd -P)/tools/infra/env/bin:$PATH" -fi - -for src_path in $(env | awk -F'=' '/^CODEBUILD_SRC_DIR/ { print $2; }'); do - for src_bin in "$src_path/bin" "$src_path/ci/bin"; do - if [ -d "$src_bin" ]; then - PATH="$src_bin:$PATH" - fi - done -done From 6f78ea6bd2c39a9d67c593f55bf91a7900cc48fb Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Mon, 6 Jan 2020 22:27:41 +0000 Subject: [PATCH 09/26] ci: add logger helper to shared script lib --- tools/infra/container/Dockerfile.builder | 3 ++- tools/infra/container/runtime/lib/lib.bash | 6 ++++++ tools/infra/container/scripts/configure-rust | 3 +++ tools/infra/container/scripts/install-crates | 13 ++++++++----- tools/infra/container/scripts/install-rust | 3 +++ 5 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 tools/infra/container/runtime/lib/lib.bash diff --git a/tools/infra/container/Dockerfile.builder b/tools/infra/container/Dockerfile.builder index 14ee95de6dd..ade8d816a98 100644 --- a/tools/infra/container/Dockerfile.builder +++ b/tools/infra/container/Dockerfile.builder @@ -9,13 +9,14 @@ RUN yum update -y \ FROM base ENV PATH="$PATH:/build/runtime/bin:/build/scripts:/build/.cargo/bin" ENV CARGO_HOME="/build/.cargo" +ENV RUNTIME_SCRIPT_LIB="/build/runtime/lib" COPY scripts /build/scripts +COPY runtime /build/runtime WORKDIR /build RUN install-rust && configure-rust && install-crates -COPY runtime /build/runtime COPY builder/entrypoint.sh /build/entrypoint.sh ENTRYPOINT ["/build/entrypoint.sh"] diff --git a/tools/infra/container/runtime/lib/lib.bash b/tools/infra/container/runtime/lib/lib.bash new file mode 100644 index 00000000000..431ae1bc3fe --- /dev/null +++ b/tools/infra/container/runtime/lib/lib.bash @@ -0,0 +1,6 @@ +# shellcheck shell=bash + +# Logger provides the corrected interface to log to stderr. +logger() { + command logger --no-act -s "$@" +} diff --git a/tools/infra/container/scripts/configure-rust b/tools/infra/container/scripts/configure-rust index 89042e494b7..38e9631043e 100755 --- a/tools/infra/container/scripts/configure-rust +++ b/tools/infra/container/scripts/configure-rust @@ -7,6 +7,9 @@ # configure-rust # +# shellcheck source=../runtime/lib/lib.bash +source "$RUNTIME_SCRIPT_LIB/lib.bash" + configure_color_output() { # Prepare Cargo for build and CI usage PATH="$HOME/.cargo/bin:$PATH" diff --git a/tools/infra/container/scripts/install-crates b/tools/infra/container/scripts/install-crates index 6ebcf628790..145460bc366 100755 --- a/tools/infra/container/scripts/install-crates +++ b/tools/infra/container/scripts/install-crates @@ -4,14 +4,17 @@ # # usage: # -# install-rust +# install-crates # +# shellcheck source=../runtime/lib/lib.bash +source "$RUNTIME_SCRIPT_LIB/lib.bash" + cargo_dep() { - cargo_install_package="$@" - logger -s -t INFO "installing cargo dep with 'cargo install $cargo_install_package'" - if ! cargo install --force $cargo_install_package; then - logger -s -t ERROR "failed to install dep with 'cargo install $cargo_install_package'" + local cargo_install_package=( "$@" ) + logger -s -t INFO "installing cargo dep with 'cargo install ${cargo_install_package[*]}'" + if ! cargo install --force "${cargo_install_package[@]}"; then + logger -s -t ERROR "failed to install dep with 'cargo install ${cargo_install_package[*]}'" exit 1 fi } diff --git a/tools/infra/container/scripts/install-rust b/tools/infra/container/scripts/install-rust index 9ab46d49c1d..e056fd08ff1 100755 --- a/tools/infra/container/scripts/install-rust +++ b/tools/infra/container/scripts/install-rust @@ -7,6 +7,9 @@ # install-rust # +# shellcheck source=../runtime/lib/lib.bash +source "$RUNTIME_SCRIPT_LIB/lib.bash" + install_rust_toolchain() { local toolchain="${1:-stable}" test -f rustup-init.sh || rm -f rustup-init.sh From 5a3928813e0affc863741aeed4052ff379670ecb Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 7 Jan 2020 10:16:57 -0800 Subject: [PATCH 10/26] ci: add doc strings and clarify conditionals --- tools/infra/container/Dockerfile.builder | 9 +++ .../runtime/bin/start-build-environment | 14 +++- .../container/runtime/bin/start-docker-daemon | 75 ++++++++++++++++--- .../container/runtime/bin/start-log-listener | 60 ++++++++++++--- 4 files changed, 139 insertions(+), 19 deletions(-) diff --git a/tools/infra/container/Dockerfile.builder b/tools/infra/container/Dockerfile.builder index ade8d816a98..56eb9dec4d3 100644 --- a/tools/infra/container/Dockerfile.builder +++ b/tools/infra/container/Dockerfile.builder @@ -1,3 +1,12 @@ +# Dockerfile.builder - Base build environment container image +# +# The builder image provides an environment in which packages and images may be +# built. This includes the necessary compilers, libraries, services, and +# executable dependencies used in the course of the build process. +# +# Facilitating scripts may be found in the ./runtime and ./scripts directory +# where scripts are generally participants in the build of the environment. +# FROM amazonlinux:2 as base RUN yum update -y \ && yum groupinstall -y 'Development Tools' \ diff --git a/tools/infra/container/runtime/bin/start-build-environment b/tools/infra/container/runtime/bin/start-build-environment index 2f2bc477a17..99f2f1f5a3a 100755 --- a/tools/infra/container/runtime/bin/start-build-environment +++ b/tools/infra/container/runtime/bin/start-build-environment @@ -1,5 +1,17 @@ #!/usr/bin/env bash +# +# start-build-environment - Setup the build environment & start supporting services +# +# Intended primarily for use in the builder container image, the +# start-build-environment provides a single call setup for the common use cases +# in the context of builds. +# +# usage: +# +# start-build-environment +# set -xe +# Ensure we have a logging facility (or stubbed service) start-log-listener -logger -s -t INFO "starting docker daemon" +# Ensure the docker daemon is running. start-docker-daemon diff --git a/tools/infra/container/runtime/bin/start-docker-daemon b/tools/infra/container/runtime/bin/start-docker-daemon index de26bbb0c15..0a95438e6fb 100755 --- a/tools/infra/container/runtime/bin/start-docker-daemon +++ b/tools/infra/container/runtime/bin/start-docker-daemon @@ -1,23 +1,80 @@ #!/usr/bin/env bash -set -e +# +# start-docker-daemon - Start up a docker daemon for the build environment +# +# Builds depend on a series of docker images and containers used to run builds +# for various stages of the system. In the CodeBuild task, the docker daemon +# must be started as part of the environment startup (in the buildspec) as the +# CodeBuild Agent does not execute an image's ENTRYPOINT. +# +# usage: +# +# start-docker-daemon +# +if ! hash docker dockerd; then + logger -s --no-act -t ERROR "missing required commands" +fi + +# If not configured, we'll use the default docker daemon socket address. +export DOCKER_HOST="${DOCKER_HOST:-unix:///var/run/docker.sock}" +# Don't startup our own daemon if the environment has access configured for the +if docker version &>/dev/null; then + logger -s --no-act -t INFO "delegating to running & configured dockerd (via $DOCKER_HOST)" + exit 0 +fi + +# Also check to see if the daemon is running but we're unable to access it, the +# environment may already have docker daemon running but not fully configured +# for use. +if [[ -S "${DOCKER_HOST:+${DOCKER_HOST##unix://}}" ]]; then + logger -s --no-act -t ERROR "unable to access running dockerd (via $DOCKER_HOST)" + exit 1 +fi + +# Verify we're a user that can start the docker daemon (assuming for now that +# being root means that). +euid="$(id -u)" +if [[ "$euid" -ne 0 ]]; then + logger -s --no-act -t ERROR "unable to start dockerd as non-root user (euid: $euid != 0)" + exit 1 +fi + +# In all other cases, start the docker daemon +logger -s --no-act -t INFO "starting dockerd" nohup dockerd \ - --host=unix:///var/run/docker.sock \ + --host="$DOCKER_HOST" \ &>/var/log/docker.log & daemon_pid="$!" + sleep 1 + if [[ ! -e "/proc/$daemon_pid" ]]; then - echo "ERROR: docker daemon did not start up" + logger -s --no-act -t ERROR "dockerd did not start" exit 1 fi -tries=0 -max_tries=5 +# Starting up the daemon asynchronously may take a moment or may fail to start +# in the background, tries are made to check in on the brief time between exec +# and the daemon ready to work. +# +# sleep interval before attempting each try +try_interval="0.1s" +# maximum try attempts that will be made +try_max_count=3 +# try attempts +try_count=0 + until docker info &>/dev/null; do - ((tries++)) - if [[ "$tries" -gt "$max_tries" ]]; then - echo "ERROR: unable to start dockerd for build" >&2 + ((try_count++)) + if [[ "$try_count" -gt "$try_max_count" ]]; then + logger -s --no-act -t ERROR "dockerd start exceeded deadline (may be slow or failed to start, check logs)" + if [[ -s "/var/log/docker.log" ]]; then + sed 's/^/docker.log: /' /var/log/docker.log + else + logger -s --no-act -t WARN "dockerd logs are empty" + fi exit 1 fi - sleep 1 + sleep "$try_interval" done diff --git a/tools/infra/container/runtime/bin/start-log-listener b/tools/infra/container/runtime/bin/start-log-listener index f9b4e47c9b8..ee8e85a1055 100755 --- a/tools/infra/container/runtime/bin/start-log-listener +++ b/tools/infra/container/runtime/bin/start-log-listener @@ -1,13 +1,55 @@ #!/usr/bin/env bash -hash socat -# Listen on syslog domain socket to make for cleaner output. -nohup socat UNIX-LISTEN:/dev/log,fork - &>/dev/null & -tries=0 -until test -S /dev/log; do - ((tries++)) - if [[ "$tries" -gt 3 ]]; then - echo "ERROR: unable to start log listener" >&2 +# +# start-log-listener - Setup a socket at /dev/log when missing +# +# The kernel's /dev/log socket may not be present in some environments so this +# script stands up a blackhole listener, to avoid logger from complaining and +# not outputting our messages. +# +# usage: +# +# start-log-listener +# +if ! hash socat; then + echo "ERROR: missing required commands" +fi + +log_sock="/dev/log" + +# Starting up the socket asynchronously may take a moment or may fail to start +# in the background, tries are made to check in on the brief time between exec +# and the listener bound. +# +# sleep interval before attempting each try +try_interval="0.1s" +# maximum try attempts that will be made +try_max_count=3 + +# Let the existing socket be. +if [[ -S "$log_sock" ]]; then + logger -s --no-act -t INFO -- "kernel's log socket is already present /dev/log" + logger -s --no-act -t INFO -- "not replacing /dev/log" + exit 0 +fi + +# Require EUID 0 as we're mucking about in /dev +euid="$(id -u)" +if [[ "$euid" -ne 0 ]]; then + echo "ERROR: unable to start log listener socket as non-root user (euid: $euid != 0)" + exit 1 +fi + +# Listen on syslog unix socket and dump any sent datagrams. +nohup socat "UNIX-LISTEN:$log_sock,fork" - &>/dev/null & + +# Wait for the listener to start and assume the process didn't successfully bind +# to the socket otherwise. +try_count=0 +until [[ -S "$log_sock" ]]; do + ((try_count++)) + if [[ "$try_count" -gt "$try_max_count" ]]; then + echo "ERROR: unable to start log listener at $log_sock" >&2 exit 1 fi - sleep 0.1s + sleep "$try_interval" done From b3322ebf433fe0de8bc73aadbd507bb23d960459 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 7 Jan 2020 15:36:34 -0800 Subject: [PATCH 11/26] ci: add credential helper for ECR images Projects producing images will utilize this tool to push images to ECR repositories as part of their build. --- tools/infra/container/Dockerfile.builder | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/infra/container/Dockerfile.builder b/tools/infra/container/Dockerfile.builder index 56eb9dec4d3..888dff5410e 100644 --- a/tools/infra/container/Dockerfile.builder +++ b/tools/infra/container/Dockerfile.builder @@ -11,7 +11,8 @@ FROM amazonlinux:2 as base RUN yum update -y \ && yum groupinstall -y 'Development Tools' \ && yum install -y socat procps-ng awscli jq openssh rsync \ - && amazon-linux-extras install -y docker \ + && amazon-linux-extras enable docker \ + && yum install -y docker amazon-ecr-credential-helper \ && yum clean all \ && rm -rf /var/cache/yum /var/cache/amzn2extras From f48c6fc90b29daf9af39d25c4dd241b251152201 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 7 Jan 2020 16:43:12 -0800 Subject: [PATCH 12/26] ci: add stack for supporting ECR repositories --- tools/infra/stacks/infra-container.yml | 163 +++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 tools/infra/stacks/infra-container.yml diff --git a/tools/infra/stacks/infra-container.yml b/tools/infra/stacks/infra-container.yml new file mode 100644 index 00000000000..ba48c506df8 --- /dev/null +++ b/tools/infra/stacks/infra-container.yml @@ -0,0 +1,163 @@ +# stack: infra-container + +AWSTemplateFormatVersion: "2010-09-09" + +Description: >- + Infra's container images ECR repositories used in release automation. + +Parameters: + SSMPathNamespace: + Type: String + Default: /infra/container + AllowedPattern: '^/.*[^/]$' + Description: > + Namespace under which SSM Parameters will be created for container images (should start but *not* end with '/') + +Resources: + SDKx8664Repo: + Type: AWS::ECR::Repository + Metadata: + Source: extras/sdk-container + Properties: + RepositoryName: thar/sdk-x86_64 + + SDKx8664Parameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "${SSMPathNamespace}/${SDKx8664Repo}" + Type: String + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SDKx8664Repo}" + + SDKaarch64Repo: + Type: AWS::ECR::Repository + Metadata: + Source: extras/sdk-container + Properties: + RepositoryName: thar/sdk-aarch64 + + SDKaarch64Parameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "${SSMPathNamespace}/${SDKaarch64Repo}" + Type: String + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SDKaarch64Repo}" + + BuilderRepo: + Type: AWS::ECR::Repository + Metadata: + Source: tools/infra/container/Dockerfile.builder + Properties: + RepositoryName: infra/builder + RepositoryPolicyText: + Version: "2012-10-17" + Statement: + - Sid: "codeBuildPull" + Effect: Allow + Principal: + Service: "codebuild.amazonaws.com" + Action: + - "ecr:GetDownloadUrlForLayer" + - "ecr:BatchGetImage" + - "ecr:BatchCheckLayerAvailability" + + BuilderParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "${SSMPathNamespace}/${BuilderRepo}" + Type: String + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${BuilderRepo}" + + SigningRepo: + Type: AWS::ECR::Repository + Metadata: + Source: tools/infra/container/Dockerfile.signing + Properties: + RepositoryName: infra/signing + RepositoryPolicyText: + Version: "2012-10-17" + Statement: + - Sid: "codeBuildPull" + Effect: Allow + Principal: + Service: "codebuild.amazonaws.com" + Action: + - "ecr:GetDownloadUrlForLayer" + - "ecr:BatchGetImage" + - "ecr:BatchCheckLayerAvailability" + + SigningParameter: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "${SSMPathNamespace}/${SigningRepo}" + Type: String + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SigningRepo}" + + PullPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + ManagedPolicyName: pull + Path: !Sub "/${AWS::StackName}/" + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: "imagePull" + Effect: Allow + Resource: + - !GetAtt BuilderRepo.Arn + - !GetAtt SigningRepo.Arn + - !GetAtt SDKx8664Repo.Arn + - !GetAtt SDKaarch64Repo.Arn + Action: + - "ecr:GetAuthorizationToken" + - "ecr:BatchCheckLayerAvailability" + - "ecr:GetDownloadUrlForLayer" + - "ecr:ListImages" + - "ecr:DescribeImages" + - "ecr:BatchGetImage" + - "ecr:ListTagsForResource" + - Sid: "imageResolve" + Effect: Allow + Resource: + - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*" + Action: + - "ssm:GetParameter" + - "ssm:GetParameters" + - "ssm:GetParametersByPath" + - "ssm:DescribeParameters" + + PushPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + ManagedPolicyName: push + Path: !Sub "/${AWS::StackName}/" + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: "imagePush" + Effect: Allow + Resource: + - !GetAtt BuilderRepo.Arn + - !GetAtt SigningRepo.Arn + - !GetAtt SDKx8664Repo.Arn + - !GetAtt SDKaarch64Repo.Arn + Action: + - "ecr:GetAuthorizationToken" + - "ecr:BatchCheckLayerAvailability" + - "ecr:GetDownloadUrlForLayer" + - "ecr:DescribeImages" + - "ecr:BatchGetImage" + - "ecr:ListTagsForResource" + - "ecr:InitiateLayerUpload" + - "ecr:UploadLayerPart" + - "ecr:CompleteLayerUpload" + - "ecr:PutImage" + +Outputs: + PullPolicy: + Export: + Name: !Sub "${AWS::StackName}-pull-policy" + Value: !Ref PullPolicy + PushPolicy: + Export: + Name: !Sub "${AWS::StackName}-push-policy" + Value: !Ref PushPolicy From 8ecb5fb6582701a37809b11cc360fdf1414b58f8 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Wed, 8 Jan 2020 16:40:44 -0800 Subject: [PATCH 13/26] ci: update codebuild stack using SSM pointers This uses the SSM pointers defined and provisioned in another stack that provides a pre-established parameter with the Container Image and its Tag to use. --- tools/infra/stacks/infra-pr-build.yml | 74 +++++++++++++++++---------- tools/infra/stacks/thar-pr-build.yml | 59 +++++++++++++-------- 2 files changed, 87 insertions(+), 46 deletions(-) diff --git a/tools/infra/stacks/infra-pr-build.yml b/tools/infra/stacks/infra-pr-build.yml index aa458427fcd..ebb7c60fac9 100644 --- a/tools/infra/stacks/infra-pr-build.yml +++ b/tools/infra/stacks/infra-pr-build.yml @@ -1,3 +1,5 @@ +# stack: infra-pr-build + Parameters: BuildSpecPath: Type: String @@ -12,6 +14,28 @@ Parameters: Description: >- The GitHub repository that builds run for. Your account must be authorized to modify this repository's settings. + EnvironmentImageName: + Type: AWS::SSM::Parameter::Value + Default: /infra/container/infra/builder + Description: >- + Parameter that defines the image name the builder uses as its execution + environment *without* a tag (eg: registry/image-name, not + registry/image-name:tag). The EnvironmentImageTag Parameter provides the + appropriate tag separately. + EnvironmentImageTag: + Type: String + Default: latest + Description: >- + The image 'tag' (as in registry/image-name:tag) to select of the EnvironmentImage + provided. + ImageCredentialsType: + Type: String + Default: CODEBUILD + AllowedValues: [ CODEBUILD, SERVICE_ROLE ] + Description: >- + If image policy does not trust codebuild.amazonaws.com OR cross-account + role is needed, then the SERVICE_ROLE must be specified to use the role + assigned to the build project. Resources: BuildArtifactBucket: @@ -41,40 +65,37 @@ Resources: BuildRolePolicy: Type: AWS::IAM::Policy Properties: - Roles: - - Ref: BuildRole + Roles: !Ref BuildRole PolicyName: BuildRolePolicy PolicyDocument: Version: "2012-10-17" Statement: # For validation API usage performed on templates during build. - - Action: - - cloudformation:ValidateTemplate + - Sid: "validateCfnTemplates" Effect: Allow + Action: + - cloudformation:ValidateTemplate + Resource: "*" # For managing cache, logs, and artifacts in the build's buckets. - - Action: + - Sid: "manageBuildArtifacts" + Effect: Allow + Action: - s3:GetObject* - s3:GetBucket* - s3:List* - s3:PutObject* - s3:Abort* - Effect: Allow Resource: - !GetAtt BuildArtifactBucket.Arn - - Fn::Join: - - "" - - - !GetAtt BuildArtifactBucket.Arn - - /* + - !Sub "${BuildArtifactBucket.Arn}/*" - !GetAtt BuildLogBucket.Arn - - Fn::Join: - - "" - - - !GetAtt BuildLogBucket.Arn - - /* + - !Sub "${BuildLogBucket.Arn}/*" # For writing to CloudWatch Logs Streams for each build. - - Action: + - Sid: "manageBuildLogs" + Effect: Allow + Action: - logs:CreateLogStream - logs:PutLogEvents - Effect: Allow Resource: - !GetAtt BuildLogGroup.Arn @@ -86,14 +107,19 @@ Resources: - BuildLogGroup - BuildRole Properties: + Artifacts: + Location: !Ref BuildArtifactBucket + Name: / + NamespaceType: BUILD_ID + Packaging: NONE + Path: artifact/ + Type: S3 Environment: ComputeType: BUILD_GENERAL1_SMALL - Image: aws/codebuild/standard:2.0 Type: LINUX_CONTAINER - ServiceRole: - Fn::GetAtt: - - BuildRole - - Arn + Image: !Sub "${EnvironmentImageName}:${EnvironmentImageTag}" + ImagePullCredentialsType: !Ref ImageCredentialsType + ServiceRole: !GetAtt BuildRole.Arn Source: BuildSpec: !Ref BuildSpecPath Location: !Ref SourceGitHubRepositoryURL @@ -102,11 +128,7 @@ Resources: LogsConfig: S3Logs: Status: ENABLED - Location: - Fn::Join: - - "/" - - - !GetAtt BuildArtifactBucket.Arn - - "codebuild/log" + Location: !Sub "${BuildArtifactBucket.Arn}/codebuild/log" CloudWatchLogs: Status: ENABLED GroupName: !Ref BuildLogGroup diff --git a/tools/infra/stacks/thar-pr-build.yml b/tools/infra/stacks/thar-pr-build.yml index 37b2886f716..250f415c890 100644 --- a/tools/infra/stacks/thar-pr-build.yml +++ b/tools/infra/stacks/thar-pr-build.yml @@ -1,9 +1,12 @@ +# stack: thar-pr-build + Parameters: BuildSpecPath: Type: String AllowedPattern: '.+\.yml$' Default: "./tools/infra/buildspec/thar-pr-build.yml" - Description: "The path to the buildspec.yml file to use." + Description: >- + The path to the buildspec.yml file to use. SourceGitHubRepositoryURL: Type: String Default: "https://github.com/amazonlinux/PRIVATE-thar.git" @@ -12,6 +15,29 @@ Parameters: Description: >- The GitHub repository that builds run for. Your account must be authorized to modify this repository's settings. + EnvironmentImageName: + Type: AWS::SSM::Parameter::Value + Default: /infra/container/infra/builder + Description: >- + Parameter that defines the image name the builder uses as its execution + environment *without* a tag (eg: registry/image-name, not + registry/image-name:tag). The EnvironmentImageTag Parameter provides the + appropriate tag separately. + EnvironmentImageTag: + Type: String + Default: latest + Description: >- + The image 'tag' (as in registry/image-name:tag) to select of the EnvironmentImage + provided. + ImageCredentialsType: + Type: String + Default: CODEBUILD + AllowedValues: [ CODEBUILD, SERVICE_ROLE ] + Description: >- + If image policy does not trust codebuild.amazonaws.com OR cross-account + role is needed, then the SERVICE_ROLE must be specified to use the role + assigned to the build project. + Resources: BuildArtifactBucket: Type: AWS::S3::Bucket @@ -47,29 +73,25 @@ Resources: Version: "2012-10-17" Statement: # For managing cache, logs, and artifacts in the build's buckets. - - Action: + - Sid: "manageBuildArtifacts" + Effect: Allow + Action: - s3:GetObject* - s3:GetBucket* - s3:List* - s3:PutObject* - s3:Abort* - Effect: Allow Resource: - !GetAtt BuildArtifactBucket.Arn - - Fn::Join: - - "" - - - !GetAtt BuildArtifactBucket.Arn - - /* + - !Sub "${BuildArtifactBucket.Arn}/*" - !GetAtt BuildLogBucket.Arn - - Fn::Join: - - "" - - - !GetAtt BuildLogBucket.Arn - - /* + - !Sub "${BuildLogBucket.Arn}/*" # For writing to CloudWatch Logs Streams for each build. - - Action: + - Sid: "manageBuildLogs" + Effect: Allow + Action: - logs:CreateLogStream - logs:PutLogEvents - Effect: Allow Resource: - !GetAtt BuildLogGroup.Arn @@ -90,9 +112,10 @@ Resources: Type: S3 Environment: ComputeType: BUILD_GENERAL1_LARGE - Image: aws/codebuild/standard:2.0 - PrivilegedMode: true Type: LINUX_CONTAINER + PrivilegedMode: true + Image: !Sub "${EnvironmentImageName}:${EnvironmentImageTag}" + ImagePullCredentialsType: !Ref ImageCredentialsType ServiceRole: !GetAtt BuildRole.Arn Source: BuildSpec: !Ref BuildSpecPath @@ -102,11 +125,7 @@ Resources: LogsConfig: S3Logs: Status: ENABLED - Location: - Fn::Join: - - "/" - - - !GetAtt BuildLogBucket.Arn - - "codebuild/log" + Location: !Sub "${BuildLogBucket.Arn}/codebuild/log" CloudWatchLogs: Status: ENABLED GroupName: !Ref BuildLogGroup From 71210dbf6b60ef9d18cf30a31ca85e0634b3fb8c Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 9 Jan 2020 17:07:39 -0800 Subject: [PATCH 14/26] ci: fix Makefile image tagging --- tools/infra/container/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/infra/container/Makefile b/tools/infra/container/Makefile index 8274d2d6c75..03594feb2e6 100644 --- a/tools/infra/container/Makefile +++ b/tools/infra/container/Makefile @@ -2,13 +2,13 @@ DOCKERFILES = $(filter-out %~,$(wildcard Dockerfile.*)) NAMES = $(DOCKERFILES:Dockerfile.%=%) IMAGE_REPO_PREFIX ?= infra/ -IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(IMAGE_TAG) +DEFAULT_IMAGE_TAG = develop +IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(if $(IMAGE_TAG),$(IMAGE_TAG),$(DEFAULT_IMAGE_TAG)) .PHONY: force all release $(NAMES) .DEFAULT: all force: -all : IMAGE_TAG ?= develop all: $(NAMES) release : IMAGE_TAG ?= latest From 15e8e68a5d4c5351dcbed28f73b996392fa8e0ce Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 9 Jan 2020 17:07:55 -0800 Subject: [PATCH 15/26] ci: configure ecr-credential-helper in image --- tools/infra/container/Dockerfile.builder | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/infra/container/Dockerfile.builder b/tools/infra/container/Dockerfile.builder index 888dff5410e..7b1cb53062a 100644 --- a/tools/infra/container/Dockerfile.builder +++ b/tools/infra/container/Dockerfile.builder @@ -15,6 +15,8 @@ RUN yum update -y \ && yum install -y docker amazon-ecr-credential-helper \ && yum clean all \ && rm -rf /var/cache/yum /var/cache/amzn2extras +RUN install -D /dev/null /root/.docker/config.json \ + && echo '{ "credsStore": "ecr-login" }' >> /root/.docker/config.json FROM base ENV PATH="$PATH:/build/runtime/bin:/build/scripts:/build/.cargo/bin" From 63719706a5cc8b1ebee747080e3708ed3465c8f9 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Wed, 8 Jan 2020 16:40:44 -0800 Subject: [PATCH 16/26] ci: update codebuild stack using SSM pointers This uses the SSM pointers defined and provisioned in another stack that provides a pre-established parameter with the Container Image and its Tag to use. --- tools/infra/stacks/infra-pr-build.yml | 6 ++++-- tools/infra/stacks/thar-pr-build.yml | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/infra/stacks/infra-pr-build.yml b/tools/infra/stacks/infra-pr-build.yml index ebb7c60fac9..9e9094b242e 100644 --- a/tools/infra/stacks/infra-pr-build.yml +++ b/tools/infra/stacks/infra-pr-build.yml @@ -1,4 +1,5 @@ -# stack: infra-pr-build +# stack-name: infra-pr-build +# stack-require: infra-container Parameters: BuildSpecPath: @@ -65,8 +66,9 @@ Resources: BuildRolePolicy: Type: AWS::IAM::Policy Properties: - Roles: !Ref BuildRole PolicyName: BuildRolePolicy + Roles: + - !Ref BuildRole PolicyDocument: Version: "2012-10-17" Statement: diff --git a/tools/infra/stacks/thar-pr-build.yml b/tools/infra/stacks/thar-pr-build.yml index 250f415c890..17a6b9ff695 100644 --- a/tools/infra/stacks/thar-pr-build.yml +++ b/tools/infra/stacks/thar-pr-build.yml @@ -1,4 +1,5 @@ -# stack: thar-pr-build +# stack-name: thar-pr-build +# stack-require: infra-container Parameters: BuildSpecPath: @@ -66,9 +67,9 @@ Resources: BuildRolePolicy: Type: AWS::IAM::Policy Properties: + PolicyName: BuildRolePolicy Roles: - !Ref BuildRole - PolicyName: BuildRolePolicy PolicyDocument: Version: "2012-10-17" Statement: From fba30baf183cc41766ce5109504d46d83fedc08a Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Thu, 9 Jan 2020 17:07:39 -0800 Subject: [PATCH 17/26] ci: update Makefile for pushing infra containers --- tools/infra/container/Makefile | 46 ++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/tools/infra/container/Makefile b/tools/infra/container/Makefile index 03594feb2e6..7fbae8be7f4 100644 --- a/tools/infra/container/Makefile +++ b/tools/infra/container/Makefile @@ -1,20 +1,50 @@ +# SHELL is bash, silly sh. +SHELL = bash +# DOCKERFILES are the detected container images that are being worked +# with. It is expected that NAME be part of the file name, as in +# Dockerfile.NAME, which is used throughout the infrastructure. DOCKERFILES = $(filter-out %~,$(wildcard Dockerfile.*)) +# NAMES are the detected NAMES given the provided Dockerfiles. NAMES = $(DOCKERFILES:Dockerfile.%=%) - +# IMAGE_REPO_PREFIX is prepended to the image's tag. In the case of +# `push', the IMAGE_REPO_PREFIX provides the ECR repository URI prefix +# for each image. IMAGE_REPO_PREFIX ?= infra/ -DEFAULT_IMAGE_TAG = develop -IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(if $(IMAGE_TAG),$(IMAGE_TAG),$(DEFAULT_IMAGE_TAG)) +# IMAGE_TAG provides the registry/image-name:IMAGE_TAG portion of the +# URI tagged to images. +IMAGE_TAG ?= develop +# IMAGE_NAME is the name that the container image is tagged with. +IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(IMAGE_TAG) +# ECR_URI_PREFIX is the ECR URI prefix based on the resolved builder +# image URI which, like other container images, is discoverable under +# its in-region SSM parameter - so we can lob off the builder part and +# use it as our model for the pushed repository name. +ECR_URI_PREFIX = $(shell aws ssm get-parameter --name /infra/container/infra/builder --query Parameter.Value --output text | sed 's/builder$$//') +# ECR_NAME_PREFIX provides a prefix to derive the ECR repository-name +# (the attribute) from the images' NAME - the infra/ prefix is +# conventional across automations' consumed images. +ECR_NAME_PREFIX ?= infra/ -.PHONY: force all release $(NAMES) .DEFAULT: all +.PHONY: force all release $(NAMES) force: -all: $(NAMES) - -release : IMAGE_TAG ?= latest -release: $(if $(NAME),$(NAME),$(NAMES)) +all: $(if $(NAME),$(NAME),$(NAMES)) $(NAMES) : NAME = $@ $(NAMES): force @echo "Building container image for '$(NAME)'" docker build -t $(IMAGE_NAME) -f Dockerfile.$(NAME) . + +# Push images (must explicitly provide IMAGE_TAG=release to be pulled +# by consumers). +push: IMAGE_REPO_PREFIX = $(ECR_URI_PREFIX) +push: IMAGE_TAG = staging +push: all + @echo "Pushing container images with tag '$(IMAGE_TAG)'" + @echo "Images: $(foreach NAME,$(NAMES),$(IMAGE_NAME))" + @$(foreach NAME,$(NAMES),\ + echo "Pushing '$(NAME)' to '$(IMAGE_NAME)'" && \ + aws ecr describe-repositories --repository-names $(ECR_NAME_PREFIX)$(NAME) &> /dev/null \ + && docker push $(IMAGE_NAME) \ + || echo "Could not push $(NAME) to ECR repository as $(IMAGE_NAME)";) From 175bc08c68c45b19e9544071c7df858ba790d617 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Fri, 10 Jan 2020 14:57:43 -0800 Subject: [PATCH 18/26] ci: add README for stacks with general description --- tools/infra/container/README.md | 28 +++++++++++++++++++++++++--- tools/infra/stacks/Makefile | 8 ++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tools/infra/container/README.md b/tools/infra/container/README.md index 13383be6156..37b3ee2d8dc 100644 --- a/tools/infra/container/README.md +++ b/tools/infra/container/README.md @@ -24,16 +24,38 @@ make all Each `Dockerfile.` can be built individually with `make $name` as needed. -## Release Images +## Release Images (using a tag) As with the development images, all images may be built at once: ```bash -make release +make all IMAGE_TAG=release ``` To build a specific image, for instance named `builder`, `make` may be provided this name to build its release image: ```bash -make release NAME=builder +make all NAME=builder IMAGE_TAG=release ``` + +# Releasing + +The `push` target is provided to build & push release container images for use, at least in the context of build and release automation. + +The default target will prepare to push the images using the environment's AWS profile to confirm that the ECR repositories line up and subsequently pushing with a default of `IMAGE_TAG=staging`. +This invocation **will** push to the ECR repository, but with the image tagged as "staging". +Doing a push this way will stage the layers in the ECR repository so that subsequent pushes update lightweight references only (pushing a tag that refers to the same layers). + +``` bash +make push +``` + +To push a container image tagged as a release image, which is required for the CodeBuild project to use, the `IMAGE_TAG` must be set explicitly to the same tag that's configured to be pulled by projects. +If the release tag is `release`, then the call to `push` these images would be: + +``` bash +make push IMAGE_TAG=release +``` + +The `Makefile` target would then match the images to their respective ECR repositories, as before, and `docker push` to the images' respective repositories. +If the `make push IMAGE_TAG=release` followed an earlier `make push` then this the `make push IMAGE_TAG=release` call will simply update the references in the remote ECR repository to point to the same layers. diff --git a/tools/infra/stacks/Makefile b/tools/infra/stacks/Makefile index bdce060aacf..594e53a599e 100644 --- a/tools/infra/stacks/Makefile +++ b/tools/infra/stacks/Makefile @@ -10,4 +10,12 @@ validate/%: $(info validating stack: $*) aws cloudformation validate-template --template-body "$$(< $*)" +check: check-readme + +check-readme: + @$(foreach stack,$(stacks:.yml=),\ + grep -Fw -q -e "$(stack)" README.md ||\ + { MISSING=1; echo "Missing README section mentioning $(stack)"; }; )\ + $${MISSING:+exit 1} + .PHONY: test list From c020d9604e78fa35c4a124d62a6a2792be12824c Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Fri, 10 Jan 2020 15:05:15 -0800 Subject: [PATCH 19/26] ci: capture additional images needed from build Related to: #630 --- tools/infra/buildspec/thar-pr-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/infra/buildspec/thar-pr-build.yml b/tools/infra/buildspec/thar-pr-build.yml index 2cfc7e5937f..91fba6f681f 100644 --- a/tools/infra/buildspec/thar-pr-build.yml +++ b/tools/infra/buildspec/thar-pr-build.yml @@ -19,6 +19,8 @@ artifacts: base-directory: 'build/' files: - '*.img*' + - '*.ext4*' + - '*.verity*' secondary-artifacts: meta: base-directory: 'build/meta' From 88127e091283453c8133fa6b63b8aa3a6d57f215 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Fri, 10 Jan 2020 15:06:11 -0800 Subject: [PATCH 20/26] ci: use check target in infra-pr-build --- tools/infra/buildspec/infra-pr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/infra/buildspec/infra-pr-build.yml b/tools/infra/buildspec/infra-pr-build.yml index 62e7042a6fe..17e4c25e74d 100644 --- a/tools/infra/buildspec/infra-pr-build.yml +++ b/tools/infra/buildspec/infra-pr-build.yml @@ -8,4 +8,4 @@ env: phases: build: commands: - - make -C "$INFRA_DIR/stacks" --keep-going validate + - make -C "$INFRA_DIR/stacks" --keep-going validate check From 9ec444bdba4f816348547759887f21f5766fcedf Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Mon, 13 Jan 2020 13:09:20 -0800 Subject: [PATCH 21/26] ci: polyfill logger in edge cases for #541 This polyfill implementation of `logger` handles scenarios where the environment isn't fully initialized or where the logger stub isn't able to be run. Falling back to `printf` eliminates logged errors from these places and in their place prints the expected messages. --- tools/infra/container/runtime/lib/lib.bash | 47 +++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tools/infra/container/runtime/lib/lib.bash b/tools/infra/container/runtime/lib/lib.bash index 431ae1bc3fe..958a202fdb5 100644 --- a/tools/infra/container/runtime/lib/lib.bash +++ b/tools/infra/container/runtime/lib/lib.bash @@ -2,5 +2,50 @@ # Logger provides the corrected interface to log to stderr. logger() { - command logger --no-act -s "$@" + # Use logger if its usable + if test -S /dev/log; then + command logger --no-act -s "$@" + return 0 + fi + + # Otherwise, use a simple polyfill implementation that provides a similar + # enough interface to be used across scripts. + local tag + local message + local format + + # polyfill in a logger that writes to stderr + while [ "$#" -ne 0 ]; do + case "$1" in + -t ) + tag="$2" + shift 1 + ;; + -*|-- ) + # drop options + ;; + * ) + # message + message=( "$@" ) + break + ;; + esac + shift 1 + done + + # Message printer format + format="${tag:+"$tag: "}%s\n" + + # Single message in function call + if [[ "${#message[@]}" -ne 0 ]]; then + printf "$format" "${message[*]}" >&2 + return 0 + fi + + # Stream of messages sent to function as input + while read msg; do + printf "$format" "${msg}" >&2 + done + + return 0 } From e83a912f9d217323db303e0838ea8710532070ab Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Fri, 3 Jan 2020 00:37:13 +0000 Subject: [PATCH 22/26] ci: add amiize CI build harness & supporting tools This adds a CI specific harness for creating AMIs from built disk images. To accomplish the task at hand, the script "create-ami-image" manages the use of build artifacts and kicks off the amiize process according to its build environment. "ensure-key-pair" validates and/or creates an EC2 key pair for its use during automated builds. This key may be rotated (by way of deletion) as needed with additional straightforward & well scoped permissions needed for the build task to manage its own key pair (aside from the overlapping EC2 permissions needed for amiizing): - ssm:PutParameter - ssm:GetParameter - ec2:ImportKey - ec2:DescribeKeyPairs - kms:Encrypt - kms:Decrypt The KMS documentation page regarding SSM Parameter Store has much more outlined on restricting the usage of SSM' AWS-Managed CMK to the SSM Parameters involved as well. Signed-off-by: Jacob Vallejo --- bin/amiize.sh | 22 ++ .../container/runtime/bin/create-image-ami | 271 ++++++++++++++++++ .../container/runtime/bin/ensure-key-pair | 47 +++ 3 files changed, 340 insertions(+) create mode 100755 tools/infra/container/runtime/bin/create-image-ami create mode 100755 tools/infra/container/runtime/bin/ensure-key-pair diff --git a/bin/amiize.sh b/bin/amiize.sh index 89ad9b69a73..7166a0be369 100755 --- a/bin/amiize.sh +++ b/bin/amiize.sh @@ -106,6 +106,7 @@ $(basename "${0}") [ --root-volume-size 1234 ] [ --data-volume-size 5678 ] [ --security-group-name default | --security-group-id sg-abcdef1234 ] + [ --write-output-dir output-dir ] Registers the given images as an AMI in the given EC2 region. @@ -130,6 +131,9 @@ Optional: --security-group-id The ID of a security group name that allows SSH access from this host --security-group-name The name of a security group name that allows SSH access from this host (defaults to "default" if neither name nor ID are specified) + --write-output-dir The directory to write out IDs into attribute named files. + (not written out to anywhere other than log otherwise) + EOF } @@ -161,6 +165,7 @@ parse_args() { --data-volume-size ) shift; DATA_VOLUME_SIZE="${1}" ;; --security-group-name ) shift; SECURITY_GROUP_NAME="${1}" ;; --security-group-id ) shift; SECURITY_GROUP_ID="${1}" ;; + --write-output-dir ) shift; WRITE_OUTPUT_DIR="${1}" ;; --help ) usage; exit 0 ;; *) @@ -323,6 +328,19 @@ check_return() { return 0 } +# Helper to conditionally write out attribute if WRITE_OUTPUT_DIR is +# configured. +write_output() { + local name="$1" + local value="$2" + + if [[ -z "${WRITE_OUTPUT_DIR}" ]]; then + return + fi + + mkdir -p "${WRITE_OUTPUT_DIR}/$(dirname "$name")" + echo -n "$value" > "${WRITE_OUTPUT_DIR}/${name}" +} # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= @@ -623,6 +641,7 @@ while true; do echo "* Warning: Could not delete root volume!" # Don't die though, we got what we want... fi + write_output "root_snapshot_id" "$root_snapshot" if aws ec2 delete-volume \ --output text \ @@ -635,6 +654,7 @@ while true; do echo "* Warning: Could not delete data volume!" # Don't die though, we got what we want... fi + write_output "data_snapshot_id" "$data_snapshot" # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= @@ -657,6 +677,8 @@ while true; do check_return ${?} "AMI registration failed!" || continue echo "Registered ${registered_ami}" + write_output "ami_id" "$registered_ami" + echo "Waiting for the AMI to appear in a describe query" waits=0 while [ ${waits} -lt 20 ]; do diff --git a/tools/infra/container/runtime/bin/create-image-ami b/tools/infra/container/runtime/bin/create-image-ami new file mode 100755 index 00000000000..09958a8742c --- /dev/null +++ b/tools/infra/container/runtime/bin/create-image-ami @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +# +# create-image-ami - Create an AMI from a build's image +# +# usage: +# +# create-image-ami +# +# Environment variable overrides for building, testing, and otherwise +# using this script (other variables, undocumented here, may be +# overridden also - see script below): +# +# BUILD_AMI_NAME +# +# Name the AMI exactly instead of calculating the image's name +# +# BUILD_REGION +# +# Region to create and register the AMI and its snapshots in. +# +# The region must be configured and provisioned with the +# required resources for both EC2 AMI creation and the SSM resources +# used for manipulating the launched image building instance. +# +# BUILD_IMAGE_ROOT, BUILD_IMAGE_DATA +# +# Paths, to the ROOT and DATA images respectively, for +# specifying exact disk images to use instead of relying on path +# construction. +# +# BUILD_INSTANCE_AMI, BUILD_INSTANCE_TYPE +# +# AMI ID and the EC2 Instance Type may be specified to +# explicitly choose an AMI and the Instance Type used to launch, +# instead of querying for the latest Amazon Linux 2 in-region +# AMI and the default EC2 Instance Type. +# +# KEYPAIR_NAME +# +# Name of the EC2 Key Pair (as named during creation or import) +# provisioned and accessible in the SSM Parameter (configured +# with KEYPAIR_PARAMETER). +# +# KEYPAIR_PARAMETER +# +# SSM Parameter Name (eg: "/Prod/ami-build/builder-ssh-key") +# that holds the Private SSH Key that corresponds to the EC2 Key +# Pair (specified in KEYPAIR_NAME) used to access the image +# building instance. +# + +set -o pipefail + +# BUILD_OUTPUT is the directory in which resource data will be written +# to as files. +BUILD_OUTPUT="${BUILD_OUTPUT:-build/ami}" +# BUILD_KEEP_OUTPUT can be set to non-nil to retain existing output in +# BUILD_OUTPUT. +BUILD_KEEP_OUTPUT="${BUILD_KEEP_OUTPUT:+yes}" +# BUILD_REGION is the region the image should be built in. +BUILD_REGION="${AWS_DEFAULT_REGION:-us-west-2}" +# BUILD_INSTANCE_TYPE is the instance type chosen to run the amiize +# build on. +BUILD_INSTANCE_TYPE="${BUILD_INSTANCE_TYPE:-m3.xlarge}" +# BUILD_AMI provides an override to choose an image to use, otherwise +# the latest release of Amazon Linux 2 is used. +BUILD_INSTANCE_AMI="${BUILD_INSTANCE_AMI:-}" +# BUILDSYS_VARIANT specifies the image's variant to be amiized. +BUILDSYS_VARIANT="${BUILDSYS_VARIANT:-aws-k8s}" +# BUILD_ARCH is the image's architecture. +BUILD_ARCH="${BUILD_ARCH:-x86_64}" +# ami_suffix provides a the dynamic portion of an AMI name using the +# environment of a build. +# +# CodeBuild based defaults use pre-defined Environment Variables as +# documented: +# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html +# +if [[ -n "$CODEBUILD_RESOLVED_SOURCE_VERSION" ]]; then + ami_suffix="${CODEBUILD_RESOLVED_SOURCE_VERSION:0:8}" +else + ami_suffix="$USER-$(date +'%s')" +fi +# BUILD_AMI_NAME is the created AMI's EC2 AMI Name. +BUILD_AMI_NAME="${BUILD_AMI_NAME:-thar-${BUILD_ARCH}-${ami_suffix}}" +# BUILD_IMAGE_ROOT may be specified to provide a specific root disk +# image to write out. +BUILD_IMAGE_ROOT="${BUILD_IMAGE_ROOT:-build/thar-$BUILD_ARCH-${BUILDSYS_VARIANT}.img.lz4}" +# BUILD_IMAGE_DATA may be specified to provide a specific data disk +# image to write out. +BUILD_IMAGE_DATA="${BUILD_IMAGE_ROOT:-build/thar-$BUILD_ARCH-${BUILDSYS_VARIANT}-data.img.lz4}" +# EC2 Key Pair used to spin up and access instance for AMIizing disk +# images. These are created automatically if the build task is +# configured with access to the SSM, EC2, and KMS resources involved. +# +# KEYPAIR_NAME is the EC2 KeyPair Name +KEYPAIR_NAME="${KEYPAIR_NAME:-ami-build-key}" +# KEYPAIR_PARAMETER is the SecureString SSM Parameter used to hold the +# private key for the specified EC2 Key Pair (in KEYPAIR_NAME). +KEYPAIR_PARAMETER="${KEYPAIR_PARAMETER:-/Prod/ami-build/$KEYPAIR_NAME}" + +WORK_DIR="$(mktemp -d -t create-image-ami.XXX)" + +# Shim for `logger` to write out to STDERR correctly throughout. +# +# shellcheck disable=2032 +logger() { + command logger --stderr --no-act "$@" +} + +# Explicitly configured aws-cli for making API calls. +aws() { + command aws --region "$BUILD_REGION" "$@" +} + +# prepareImage massages a provided disk image into a format suitable +# for use in the amiize process and returns that file's name. +# +# usage: prepareImage +prepareImage() { + local image_file="${1:?need image file to prepare it}" + local out + case "$image_file" in + *.lz4 ) + un_name="${image_file%%.lz4}" + out="$WORK_DIR/${un_name##*/}" + logger -t INFO "decompressing LZ4 disk image: $image_file to $out" + lz4 -dc "$image_file" > "$out" || return 1 + ;; + *.img ) + out="$image_file" + logger -t INFO "using provided raw disk image: $out" + ;; + * ) + logger -t ERROR "unknown image file provided: $image_file" + return 1 + ;; + esac + + echo "$out" +} + +# cleanup our files and helping processes before exiting +cleanup() { + if [[ -n "$SSH_AGENT_PID" ]]; then + logger -t INFO "terminating key-wielding ssh-agent" + eval "$(ssh-agent -k | grep -v '^echo')" + fi + + if [[ -d "$WORK_DIR" ]]; then + logger -t INFO "removing work directory $WORK_DIR" + rm -rf -- "$WORK_DIR" + fi +} + +trap "cleanup" EXIT + +mkdir -p "$WORK_DIR" + +if ! [[ -s "$BUILD_IMAGE_ROOT" ]]; then + logger -t ERROR "missing root disk image" + exit 1 +fi +if ! [[ -s "$BUILD_IMAGE_DATA" ]]; then + logger -t ERROR "missing data disk image" + exit 1 +fi + +logger -t INFO "using ssh key from SSM parameter '$KEYPAIR_PARAMETER'" +if ! ( + # shellcheck disable=SC2030 + export KEYPAIR_NAME KEYPAIR_PARAMETER + export AWS_DEFAULT_REGION="$BUILD_REGION" + ensure-key-pair + ); then + logger -t ERROR "unable to setup ssh key from SSM parameter" + exit 1 +fi + +logger -t INFO "configuring ssh for SSM parameter ssh key" +# shellcheck disable=SC2091 +eval "$(ssh-agent | grep -v '^echo')" +# shellcheck disable=SC2031 +if ! ssh-add <(aws ssm get-parameter --name "$KEYPAIR_PARAMETER" --with-decryption --query Parameter.Value --output text) ; then + logger -t ERROR "unable to load ssh key from SSM Parameter $KEYPAIR_PARAMETER" + exit 1 +fi + +if [[ -z "$BUILD_AMI" ]]; then + logger -t INFO "querying SSM for latest Amazon Linux 2 AMI" + BUILD_INSTANCE_AMI="$(aws ssm get-parameter --output text --query Parameter.Value \ + --name /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2)" + # shellcheck disable=2181 + if [[ "$?" -ne 0 ]]; then + logger -t ERROR "AMI query failed, cannot proceed without image" + exit 1 + fi + if [[ -z "$BUILD_INSTANCE_AMI" ]]; then + logger -t ERROR "AMI ID is empty, cannot proceed without image" + exit 1 + fi +fi +logger -t INFO "using $BUILD_INSTANCE_AMI for amiizing instance" + +logger -t INFO "preparing disk images for amiizing" +if ! BUILD_IMAGE_ROOT="$(prepareImage "$BUILD_IMAGE_ROOT")"; then + logger -t ERROR "failed to prepare root image for amiizing" + exit 1 +fi + +if ! BUILD_IMAGE_DATA="$(prepareImage "$BUILD_IMAGE_DATA")"; then + logger -t ERROR "failed to prepare data image for amiizing" + exit 1 +fi + +userdata="$(base64 -w 0 < "$amiize_output/region" + + # When we're writing to an existing directory, we allow the caller to retain + # existing resources there also while (effectively) overwriting the + # overlapping resources. + if [[ -d "$BUILD_OUTPUT" ]]; then + if [[ -z "$BUILD_KEEP_OUTPUT" ]]; then + logger -t WARN -- "removing prior data (in $BUILD_OUTPUT) to record newly created resources" + # Remove overlapping files in the $BUILD_OUTPUT first. + find "$amiize_output" -mindepth 1 -printf "$BUILD_OUTPUT/%P\0" | xargs -0 -- rm -rf -- + fi + fi + + logger -t INFO "recording created resource IDs in $BUILD_OUTPUT" + mkdir -p "$BUILD_OUTPUT" + cp -rT "$amiize_output/" "$BUILD_OUTPUT/" +fi + +exit "$amiize_ret" diff --git a/tools/infra/container/runtime/bin/ensure-key-pair b/tools/infra/container/runtime/bin/ensure-key-pair new file mode 100755 index 00000000000..ac40b347723 --- /dev/null +++ b/tools/infra/container/runtime/bin/ensure-key-pair @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +KEYPAIR_NAME="${KEYPAIR_NAME:?Need a keypair name}" +KEYPAIR_PARAMETER="${KEYPAIR_PARAMETER:?Need a parameter name}" + +logger() { + command logger --stderr --no-act "$@" +} + +ssh_keypair="$(mktemp -u -t "ssm-key-pair.XXX")" +# shellcheck disable=2064 +trap "rm -f -- '$ssh_keypair' '$ssh_keypair.pub'" EXIT + +if aws ssm get-parameter --name "$KEYPAIR_PARAMETER" &>/dev/null; then + : SSM Secure Parameter exists +else + logger -t INFO "creating SSM parameter ssh key-pair" + mkfifo "$ssh_keypair" "${ssh_keypair}.pub" + yes | ssh-keygen -P '' -C "$KEYPAIR_NAME" -f "$ssh_keypair" & + aws ssm put-parameter --overwrite --name "$KEYPAIR_PARAMETER" --type SecureString --value "$(<"$ssh_keypair")" >/dev/null +fi + +if aws ec2 describe-key-pairs --key-name "$KEYPAIR_NAME" &>/dev/null ; then + : EC2 Key Pair exists +else + if [ -e "${ssh_keypair}.pub" ]; then + logger -t INFO "importing ssh key from newly generated pair" + else + logger -t INFO "importing ssh key from SSM parameter '$KEYPAIR_PARAMETER'" + + ssh-keygen -y -f <(aws ssm get-parameter --name "$KEYPAIR_PARAMETER" \ + --with-decryption \ + --query Parameter.Value \ + --output text) \ + > "${ssh_keypair}.pub" + # shellcheck disable=SC2181 + if [ "$?" -ne 0 ]; then + logger -t ERROR "could not import ssh key from SSM parameter" + exit 1 + fi + fi + + + aws ec2 import-key-pair --key-name "$KEYPAIR_NAME" --public-key-material file://"${ssh_keypair}.pub" + logger -t INFO "imported ssh key as EC2 key pair '$KEYPAIR_NAME'" +fi From b348e58a24addba790dae8f2b0b8b7c44ce32029 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 14 Jan 2020 15:30:44 -0800 Subject: [PATCH 23/26] ci: use bash function library for logger --- tools/infra/container/runtime/bin/create-image-ami | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/infra/container/runtime/bin/create-image-ami b/tools/infra/container/runtime/bin/create-image-ami index 09958a8742c..de517852ee9 100755 --- a/tools/infra/container/runtime/bin/create-image-ami +++ b/tools/infra/container/runtime/bin/create-image-ami @@ -101,12 +101,8 @@ KEYPAIR_PARAMETER="${KEYPAIR_PARAMETER:-/Prod/ami-build/$KEYPAIR_NAME}" WORK_DIR="$(mktemp -d -t create-image-ami.XXX)" -# Shim for `logger` to write out to STDERR correctly throughout. -# -# shellcheck disable=2032 -logger() { - command logger --stderr --no-act "$@" -} +# shellcheck source=../lib/lib.bash +source "${RUNTIME_SCRIPT_LIB:-../lib}/lib.bash" # Explicitly configured aws-cli for making API calls. aws() { From e9b8da753b9ff52611538e4ca77277c6a45b7c11 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 14 Jan 2020 17:33:51 -0800 Subject: [PATCH 24/26] ci: update stack-name tokens --- tools/infra/stacks/host-containers-pr-build.yml | 2 ++ tools/infra/stacks/infra-container.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/infra/stacks/host-containers-pr-build.yml b/tools/infra/stacks/host-containers-pr-build.yml index 3bf1e796924..9d907c89c19 100644 --- a/tools/infra/stacks/host-containers-pr-build.yml +++ b/tools/infra/stacks/host-containers-pr-build.yml @@ -1,3 +1,5 @@ +# stack-name: host-containers-pr-build + Parameters: BuildSpecPath: Type: String diff --git a/tools/infra/stacks/infra-container.yml b/tools/infra/stacks/infra-container.yml index ba48c506df8..6ef40a927bb 100644 --- a/tools/infra/stacks/infra-container.yml +++ b/tools/infra/stacks/infra-container.yml @@ -1,4 +1,4 @@ -# stack: infra-container +# stack-name: infra-container AWSTemplateFormatVersion: "2010-09-09" From ec2e4acc993eff4ad2c5377ac44772941f711a13 Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 14 Jan 2020 17:49:47 -0800 Subject: [PATCH 25/26] ci: remove unused environment variable --- tools/infra/buildspec/thar-pr-build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/infra/buildspec/thar-pr-build.yml b/tools/infra/buildspec/thar-pr-build.yml index 91fba6f681f..5038b9ed12e 100644 --- a/tools/infra/buildspec/thar-pr-build.yml +++ b/tools/infra/buildspec/thar-pr-build.yml @@ -1,10 +1,5 @@ version: 0.2 -env: - variables: - # Path to infra tooling directory. - INFRA_DIR: "./tools/infra" - phases: pre_build: commands: From 2f011e0eb85b8b22e780238a9cd81b5c7721bf0b Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 14 Jan 2020 17:50:20 -0800 Subject: [PATCH 26/26] ci: add systemd headers in image for test stage --- .../infra/buildspec/thar-develop-pipeline-test.yml | 14 +------------- tools/infra/container/Dockerfile.builder | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/tools/infra/buildspec/thar-develop-pipeline-test.yml b/tools/infra/buildspec/thar-develop-pipeline-test.yml index 30e824d0f06..c40747f74c8 100644 --- a/tools/infra/buildspec/thar-develop-pipeline-test.yml +++ b/tools/infra/buildspec/thar-develop-pipeline-test.yml @@ -1,21 +1,9 @@ version: 0.2 -env: - variables: - # Path to infra tooling directory. - INFRA_DIR: "./tools/infra" - phases: - install: - runtime-versions: - docker: 18 - commands: - - . "${INFRA_DIR}/env/lib/environment-setup" - - . setup-rust-builder - # TODO: rely on libsystemd in build container instead, see #545 - - apt install -y libsystemd-dev pre_build: commands: + - start-build-environment - environment-report - write-build-meta build: diff --git a/tools/infra/container/Dockerfile.builder b/tools/infra/container/Dockerfile.builder index 7b1cb53062a..bc48992bcc2 100644 --- a/tools/infra/container/Dockerfile.builder +++ b/tools/infra/container/Dockerfile.builder @@ -10,7 +10,7 @@ FROM amazonlinux:2 as base RUN yum update -y \ && yum groupinstall -y 'Development Tools' \ - && yum install -y socat procps-ng awscli jq openssh rsync \ + && yum install -y socat procps-ng awscli jq openssh rsync systemd-devel \ && amazon-linux-extras enable docker \ && yum install -y docker amazon-ecr-credential-helper \ && yum clean all \