Use Docker Build secrets in API templates (#471)

This is now well-supported by the v5 Buildkite stack and https://github.com/seek-oss/docker-ecr-cache-buildkite-plugin.
seek-oss · Jul 13, 2021 · 985ff9a · 985ff9a
1 parent 3f0f14d
commit 985ff9a
Show file tree

Hide file tree

Showing 14 changed files with 160 additions and 122 deletions.
diff --git a/.changeset/afraid-humans-design.md b/.changeset/afraid-humans-design.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template/express-rest-api:** Add `test:ci` script
diff --git a/.changeset/new-falcons-raise.md b/.changeset/new-falcons-raise.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template/express-rest-api:** Use [Docker Build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information)
diff --git a/.changeset/quiet-actors-live.md b/.changeset/quiet-actors-live.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template/koa-rest-api:** Add `test:ci` script
diff --git a/.changeset/wicked-dodos-occur.md b/.changeset/wicked-dodos-occur.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template/koa-rest-api:** Use [Docker Build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information)
diff --git a/template/express-rest-api/.buildkite/pipeline.yml b/template/express-rest-api/.buildkite/pipeline.yml
@@ -1,52 +1,72 @@
 configs:
-  - &aws-sm
-    seek-oss/aws-sm#v2.0.0:
-      env:
-        NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+  environments:
+    - &dev
+      agents:
+        queue: <%- devBuildkiteQueueName %>
 
-  - &docker-ecr-cache
-    seek-oss/docker-ecr-cache#v1.9.0:
-      build-args:
-        - NPM_READ_TOKEN
-      cache-on:
-        - package.json
-        - yarn.lock
-      target: dev-deps
+    - &prod
+      agents:
+        queue: <%- prodBuildkiteQueueName %>
 
-  - &deploy
-    concurrency: 1
-    retry:
-      manual:
-        # Only use this if you need to roll back a deployment ASAP.
-        # Always follow up with a proper revert or fix in Git history.
-        permit_on_passed: true
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
-  - &dev
-    agents:
-      queue: <%- devBuildkiteQueueName %>
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v1.11.0:
+        cache-on:
+          - package.json
+          - yarn.lock
+        dockerfile: Dockerfile.dev-deps
+        secrets: id=npm,src=.npmrc
 
-  - &prod
-    agents:
-      queue: <%- prodBuildkiteQueueName %>
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+  base-steps:
+    - &deploy
+      concurrency: 1
+      retry:
+        manual:
+          # Only use this if you need to roll back a deployment ASAP.
+          # Always follow up with a proper revert or fix in Git history.
+          permit_on_passed: true
 
 steps:
+  - <<: *prod
+    label: 🧖‍♀️ Warm Prod
+    command: ':'
+    key: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+
   - <<: *prod
     label: 🧪 Test & Lint
     commands:
-      - echo '+++ yarn test'
-      - yarn test
-      - echo '+++ yarn lint'
+      - echo '+++ yarn test:ci'
+      - yarn test:ci
+      - echo '--- yarn lint'
       - yarn lint
+    depends_on: warm-prod
     plugins:
       - *aws-sm
+      - *private-npm
       - *docker-ecr-cache
       - docker-compose#v3.7.0:
           run: app
 
   - <<: *prod
     label: 📦 Build & Package
+    depends_on: warm-prod
     plugins:
       - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
       - seek-jobs/gantry#v1.3.0:
           command: build
           file: gantry.build.yml
@@ -61,6 +81,7 @@ steps:
     <<: *deploy
     label: 🤞 Deploy Dev
     concurrency_group: <%- teamName %>/deploy/gantry/<%- devGantryEnvironmentName %>
+    key: deploy-dev
     plugins:
       - seek-jobs/gantry#v1.3.0:
           command: apply
@@ -71,13 +92,12 @@ steps:
             - .gantry/common.yml
             - .gantry/dev.yml
 
-  - wait
-
   - <<: *prod
     <<: *deploy
     label: 🚀 Deploy Prod
     branches: master
     concurrency_group: <%- teamName %>/deploy/gantry/<%- prodGantryEnvironmentName %>
+    depends_on: deploy-dev
     plugins:
       - seek-jobs/gantry#v1.3.0:
           command: apply

diff --git a/template/express-rest-api/Dockerfile b/template/express-rest-api/Dockerfile
@@ -1,40 +1,15 @@
-# Docker image history includes ARG values, so never target this stage directly
-FROM node:14-alpine AS unsafe-dev-deps
-
-WORKDIR /workdir
-
-COPY package.json yarn.lock ./
-
-ARG NPM_READ_TOKEN
-
-RUN yarn config set cache-folder /yarn
-
-RUN \
-  echo '//registry.npmjs.org/:_authToken=${NPM_READ_TOKEN}' > .npmrc && \
-  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
-  rm .npmrc
-
-###
-
-FROM node:14-alpine AS dev-deps
-
-WORKDIR /workdir
-
-COPY --from=unsafe-dev-deps /workdir .
-
-COPY --from=unsafe-dev-deps /yarn /yarn
-
-RUN yarn config set cache-folder /yarn
+ARG BASE_IMAGE
+ARG BASE_TAG
 
 ###
 
-FROM dev-deps AS deps
+FROM ${BASE_IMAGE}:${BASE_TAG} AS deps
 
 RUN yarn install --ignore-optional --ignore-scripts --non-interactive --offline --production
 
 ###
 
-FROM dev-deps AS build
+FROM ${BASE_IMAGE}:${BASE_TAG} AS build
 
 COPY . .
 

diff --git a/template/express-rest-api/Dockerfile.dev-deps b/template/express-rest-api/Dockerfile.dev-deps
@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1.2
+
+FROM node:14-alpine AS dev-deps
+
+WORKDIR /workdir
+
+COPY package.json yarn.lock ./
+
+RUN \
+  --mount=type=secret,id=npm,dst=/workdir/.npmrc \
+  yarn install --frozen-lockfile --ignore-optional --non-interactive
diff --git a/template/express-rest-api/gantry.build.yml b/template/express-rest-api/gantry.build.yml
@@ -5,6 +5,8 @@ schemaVersion: v0.0
 image: '{{values "image"}}'
 
 buildArgs:
-  NPM_READ_TOKEN: '{{.Env.NPM_READ_TOKEN}}'
+  # https://github.com/seek-oss/docker-ecr-cache-buildkite-plugin#building-on-the-resulting-image
+  BASE_IMAGE: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE}}'
+  BASE_TAG: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG}}'
 
 failOnScanFindings: true
diff --git a/template/express-rest-api/package.json b/template/express-rest-api/package.json
@@ -26,7 +26,8 @@
     "lint": "skuba lint",
     "start": "ENVIRONMENT=local skuba start",
     "start:debug": "yarn start --inspect-brk",
-    "test": "skuba test --coverage",
+    "test": "skuba test",
+    "test:ci": "skuba test --coverage",
     "test:watch": "skuba test --watch"
   }
 }
diff --git a/template/koa-rest-api/.buildkite/pipeline.yml b/template/koa-rest-api/.buildkite/pipeline.yml
@@ -1,52 +1,72 @@
 configs:
-  - &aws-sm
-    seek-oss/aws-sm#v2.0.0:
-      env:
-        NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+  environments:
+    - &dev
+      agents:
+        queue: <%- devBuildkiteQueueName %>
 
-  - &docker-ecr-cache
-    seek-oss/docker-ecr-cache#v1.9.0:
-      build-args:
-        - NPM_READ_TOKEN
-      cache-on:
-        - package.json
-        - yarn.lock
-      target: dev-deps
+    - &prod
+      agents:
+        queue: <%- prodBuildkiteQueueName %>
 
-  - &deploy
-    concurrency: 1
-    retry:
-      manual:
-        # Only use this if you need to roll back a deployment ASAP.
-        # Always follow up with a proper revert or fix in Git history.
-        permit_on_passed: true
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
-  - &dev
-    agents:
-      queue: <%- devBuildkiteQueueName %>
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v1.11.0:
+        cache-on:
+          - package.json
+          - yarn.lock
+        dockerfile: Dockerfile.dev-deps
+        secrets: id=npm,src=.npmrc
 
-  - &prod
-    agents:
-      queue: <%- prodBuildkiteQueueName %>
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+  base-steps:
+    - &deploy
+      concurrency: 1
+      retry:
+        manual:
+          # Only use this if you need to roll back a deployment ASAP.
+          # Always follow up with a proper revert or fix in Git history.
+          permit_on_passed: true
 
 steps:
+  - <<: *prod
+    label: 🧖‍♀️ Warm Prod
+    command: ':'
+    key: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+
   - <<: *prod
     label: 🧪 Test & Lint
     commands:
-      - echo '+++ yarn test'
-      - yarn test
-      - echo '+++ yarn lint'
+      - echo '+++ yarn test:ci'
+      - yarn test:ci
+      - echo '--- yarn lint'
       - yarn lint
+    depends_on: warm-prod
     plugins:
       - *aws-sm
+      - *private-npm
       - *docker-ecr-cache
       - docker-compose#v3.7.0:
           run: app
 
   - <<: *prod
     label: 📦 Build & Package
+    depends_on: warm-prod
     plugins:
       - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
       - seek-jobs/gantry#v1.3.0:
           command: build
           file: gantry.build.yml
@@ -61,6 +81,7 @@ steps:
     <<: *deploy
     label: 🤞 Deploy Dev
     concurrency_group: <%- teamName %>/deploy/gantry/<%- devGantryEnvironmentName %>
+    key: deploy-dev
     plugins:
       - seek-jobs/gantry#v1.3.0:
           command: apply
@@ -71,13 +92,12 @@ steps:
             - .gantry/common.yml
             - .gantry/dev.yml
 
-  - wait
-
   - <<: *prod
     <<: *deploy
     label: 🚀 Deploy Prod
     branches: master
     concurrency_group: <%- teamName %>/deploy/gantry/<%- prodGantryEnvironmentName %>
+    depends_on: deploy-dev
     plugins:
       - seek-jobs/gantry#v1.3.0:
           command: apply

diff --git a/template/koa-rest-api/Dockerfile b/template/koa-rest-api/Dockerfile
@@ -1,40 +1,15 @@
-# Docker image history includes ARG values, so never target this stage directly
-FROM node:14-alpine AS unsafe-dev-deps
-
-WORKDIR /workdir
-
-COPY package.json yarn.lock ./
-
-ARG NPM_READ_TOKEN
-
-RUN yarn config set cache-folder /yarn
-
-RUN \
-  echo '//registry.npmjs.org/:_authToken=${NPM_READ_TOKEN}' > .npmrc && \
-  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
-  rm .npmrc
-
-###
-
-FROM node:14-alpine AS dev-deps
-
-WORKDIR /workdir
-
-COPY --from=unsafe-dev-deps /workdir .
-
-COPY --from=unsafe-dev-deps /yarn /yarn
-
-RUN yarn config set cache-folder /yarn
+ARG BASE_IMAGE
+ARG BASE_TAG
 
 ###
 
-FROM dev-deps AS deps
+FROM ${BASE_IMAGE}:${BASE_TAG} AS deps
 
 RUN yarn install --ignore-optional --ignore-scripts --non-interactive --offline --production
 
 ###
 
-FROM dev-deps AS build
+FROM ${BASE_IMAGE}:${BASE_TAG} AS build
 
 COPY . .