Add CodeBuild Infra containers supporting resources

tools/infra/buildspec/infra-pr-build.yml

@@ -8,4 +8,4 @@ env:
 phases:
   build:
     commands:
-      - make -C "$INFRA_DIR/stacks" --keep-going validate
+      - make -C "$INFRA_DIR/stacks" --keep-going validate check

tools/infra/buildspec/thar-pr-build.yml

@@ -19,6 +19,8 @@ artifacts:
   base-directory: 'build/'
   files:
     - '*.img*'
+    - '*.ext4*'
+    - '*.verity*'
   secondary-artifacts:
     meta:
       base-directory: 'build/meta'

tools/infra/container/Dockerfile.builder

@@ -11,9 +11,12 @@ FROM amazonlinux:2 as base
 RUN yum update -y \
 	&& yum groupinstall -y 'Development Tools' \
 	&& yum install -y socat procps-ng awscli jq openssh rsync \
-	&& amazon-linux-extras install -y docker \
+	&& amazon-linux-extras enable docker \
+	&& yum install -y docker amazon-ecr-credential-helper \
 	&& yum clean all \
 	&& rm -rf /var/cache/yum /var/cache/amzn2extras
+RUN install -D /dev/null /root/.docker/config.json \
+	&& echo '{ "credsStore": "ecr-login" }' >> /root/.docker/config.json
 
 FROM base
 ENV PATH="$PATH:/build/runtime/bin:/build/scripts:/build/.cargo/bin"

tools/infra/container/Makefile

@@ -1,20 +1,50 @@
+# SHELL is bash, silly sh.
+SHELL = bash
+# DOCKERFILES are the detected container images that are being worked
+# with. It is expected that NAME be part of the file name, as in
+# Dockerfile.NAME, which is used throughout the infrastructure.
 DOCKERFILES = $(filter-out %~,$(wildcard Dockerfile.*))
+# NAMES are the detected NAMES given the provided Dockerfiles.
 NAMES = $(DOCKERFILES:Dockerfile.%=%)
-
+# IMAGE_REPO_PREFIX is prepended to the image's tag. In the case of
+# `push', the IMAGE_REPO_PREFIX provides the ECR repository URI prefix
+# for each image.
 IMAGE_REPO_PREFIX ?= infra/
+# IMAGE_TAG provides the registry/image-name:IMAGE_TAG portion of the
+# URI tagged to images.
+IMAGE_TAG ?= develop
+# IMAGE_NAME is the name that the container image is tagged with.
 IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(IMAGE_TAG)
+# ECR_URI_PREFIX is the ECR URI prefix based on the resolved builder
+# image URI which, like other container images, is discoverable under
+# its in-region SSM parameter - so we can lob off the builder part and
+# use it as our model for the pushed repository name.
+ECR_URI_PREFIX = $(shell aws ssm get-parameter --name /infra/container/infra/builder --query Parameter.Value --output text | sed 's/builder$$//')
+# ECR_NAME_PREFIX provides a prefix to derive the ECR repository-name
+# (the attribute) from the images' NAME - the infra/ prefix is
+# conventional across automations' consumed images.
+ECR_NAME_PREFIX ?= infra/
 
-.PHONY: force all release $(NAMES)
 .DEFAULT: all
+.PHONY: force all release $(NAMES)
 force:
 
-all : IMAGE_TAG ?= develop
-all: $(NAMES)
-
-release : IMAGE_TAG ?= latest
-release: $(if $(NAME),$(NAME),$(NAMES))
+all: $(if $(NAME),$(NAME),$(NAMES))
 
 $(NAMES) : NAME = $@
 $(NAMES): force
 	@echo "Building container image for '$(NAME)'"
 	docker build -t $(IMAGE_NAME) -f Dockerfile.$(NAME) .
+
+# Push images (must explicitly provide IMAGE_TAG=release to be pulled
+# by consumers).
+push: IMAGE_REPO_PREFIX = $(ECR_URI_PREFIX)
+push: IMAGE_TAG = staging
+push: all
+	@echo "Pushing container images with tag '$(IMAGE_TAG)'"
+	@echo "Images: $(foreach NAME,$(NAMES),$(IMAGE_NAME))"
+	@$(foreach NAME,$(NAMES),\
+		echo "Pushing '$(NAME)' to '$(IMAGE_NAME)'" && \
+		aws ecr describe-repositories --repository-names $(ECR_NAME_PREFIX)$(NAME) &> /dev/null \
+		&& docker push $(IMAGE_NAME) \
+		|| echo "Could not push $(NAME) to ECR repository as $(IMAGE_NAME)";)

tools/infra/container/README.md

@@ -24,16 +24,38 @@ make all
 
 Each `Dockerfile.<name>` can be built individually with `make $name` as needed.
 
-## Release Images
+## Release Images (using a tag)
 
 As with the development images, all images may be built at once:
 
 ```bash
-make release
+make all IMAGE_TAG=release
 ```
 
 To build a specific image, for instance named `builder`, `make` may be provided this name to build its release image:
 
 ```bash
-make release NAME=builder
+make all NAME=builder IMAGE_TAG=release
 ```
+
+# Releasing
+
+The `push` target is provided to build & push release container images for use, at least in the context of build and release automation.
+
+The default target will prepare to push the images using the environment's AWS profile to confirm that the ECR repositories line up and subsequently pushing with a default of `IMAGE_TAG=staging`.
+This invocation **will** push to the ECR repository, but with the image tagged as "staging".
+Doing a push this way will stage the layers in the ECR repository so that subsequent pushes update lightweight references only (pushing a tag that refers to the same layers).
+
+``` bash
+make push
+```
+
+To push a container image tagged as a release image, which is required for the CodeBuild project to use, the `IMAGE_TAG` must be set explicitly to the same tag that's configured to be pulled by projects.
+If the release tag is `release`, then the call to `push` these images would be:
+
+``` bash
+make push IMAGE_TAG=release
+```
+
+The `Makefile` target would then match the images to their respective ECR repositories, as before, and `docker push` to the images' respective repositories.
+If the `make push IMAGE_TAG=release` followed an earlier `make push` then this the `make push IMAGE_TAG=release` call will simply update the references in the remote ECR repository to point to the same layers.

tools/infra/stacks/Makefile

@@ -10,4 +10,12 @@ validate/%:
 	$(info validating stack: $*)
 	aws cloudformation validate-template --template-body "$$(< $*)"
 
+check: check-readme
+
+check-readme:
+	@$(foreach stack,$(stacks:.yml=),\
+		grep -Fw -q -e "$(stack)" README.md ||\
+		{ MISSING=1; echo "Missing README section mentioning $(stack)";  }; )\
+	$${MISSING:+exit 1}
+
 .PHONY: test list

tools/infra/stacks/infra-container.yml

@@ -0,0 +1,163 @@
+# stack: infra-container
+
+AWSTemplateFormatVersion: "2010-09-09"
+
+Description: >-
+  Infra's container images ECR repositories used in release automation.
+
+Parameters:
+  SSMPathNamespace:
+    Type: String
+    Default: /infra/container
+    AllowedPattern: '^/.*[^/]$'
+    Description: >
+      Namespace under which SSM Parameters will be created for container images (should start but *not* end with '/')
+
+Resources:
+  SDKx8664Repo:
+    Type: AWS::ECR::Repository
+    Metadata:
+      Source: extras/sdk-container
+    Properties:
+      RepositoryName: thar/sdk-x86_64
+
+  SDKx8664Parameter:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: !Sub "${SSMPathNamespace}/${SDKx8664Repo}"
+      Type: String
+      Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SDKx8664Repo}"
+
+  SDKaarch64Repo:
+    Type: AWS::ECR::Repository
+    Metadata:
+      Source: extras/sdk-container
+    Properties:
+      RepositoryName: thar/sdk-aarch64
+
+  SDKaarch64Parameter:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: !Sub "${SSMPathNamespace}/${SDKaarch64Repo}"
+      Type: String
+      Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SDKaarch64Repo}"
+
+  BuilderRepo:
+    Type: AWS::ECR::Repository
+    Metadata:
+      Source: tools/infra/container/Dockerfile.builder
+    Properties:
+      RepositoryName: infra/builder
+      RepositoryPolicyText:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: "codeBuildPull"
+            Effect: Allow
+            Principal:
+              Service: "codebuild.amazonaws.com"
+            Action:
+               - "ecr:GetDownloadUrlForLayer"
+               - "ecr:BatchGetImage"
+               - "ecr:BatchCheckLayerAvailability"
+
+  BuilderParameter:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: !Sub "${SSMPathNamespace}/${BuilderRepo}"
+      Type: String
+      Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${BuilderRepo}"
+
+  SigningRepo:
+    Type: AWS::ECR::Repository
+    Metadata:
+      Source: tools/infra/container/Dockerfile.signing
+    Properties:
+      RepositoryName: infra/signing
+      RepositoryPolicyText:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: "codeBuildPull"
+            Effect: Allow
+            Principal:
+              Service: "codebuild.amazonaws.com"
+            Action:
+               - "ecr:GetDownloadUrlForLayer"
+               - "ecr:BatchGetImage"
+               - "ecr:BatchCheckLayerAvailability"
+
+  SigningParameter:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: !Sub "${SSMPathNamespace}/${SigningRepo}"
+      Type: String
+      Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${SigningRepo}"
+
+  PullPolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      ManagedPolicyName: pull
+      Path: !Sub "/${AWS::StackName}/"
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Sid: "imagePull"
+            Effect: Allow
+            Resource:
+              - !GetAtt BuilderRepo.Arn
+              - !GetAtt SigningRepo.Arn
+              - !GetAtt SDKx8664Repo.Arn
+              - !GetAtt SDKaarch64Repo.Arn
+            Action:
+              - "ecr:GetAuthorizationToken"
+              - "ecr:BatchCheckLayerAvailability"
+              - "ecr:GetDownloadUrlForLayer"
+              - "ecr:ListImages"
+              - "ecr:DescribeImages"
+              - "ecr:BatchGetImage"
+              - "ecr:ListTagsForResource"
+          - Sid: "imageResolve"
+            Effect: Allow
+            Resource:
+              - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*"
+            Action:
+              - "ssm:GetParameter"
+              - "ssm:GetParameters"
+              - "ssm:GetParametersByPath"
+              - "ssm:DescribeParameters"
+
+  PushPolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      ManagedPolicyName: push
+      Path: !Sub "/${AWS::StackName}/"
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Sid: "imagePush"
+            Effect: Allow
+            Resource:
+              - !GetAtt BuilderRepo.Arn
+              - !GetAtt SigningRepo.Arn
+              - !GetAtt SDKx8664Repo.Arn
+              - !GetAtt SDKaarch64Repo.Arn
+            Action:
+              - "ecr:GetAuthorizationToken"
+              - "ecr:BatchCheckLayerAvailability"
+              - "ecr:GetDownloadUrlForLayer"
+              - "ecr:DescribeImages"
+              - "ecr:BatchGetImage"
+              - "ecr:ListTagsForResource"
+              - "ecr:InitiateLayerUpload"
+              - "ecr:UploadLayerPart"
+              - "ecr:CompleteLayerUpload"
+              - "ecr:PutImage"
+
+Outputs:
+  PullPolicy:
+    Export:
+      Name: !Sub "${AWS::StackName}-pull-policy"
+    Value: !Ref PullPolicy
+  PushPolicy:
+    Export:
+      Name: !Sub "${AWS::StackName}-push-policy"
+    Value: !Ref PushPolicy