Improve ECR caching in templates (#453)

Editorialised context from an internal PR I forgot to upstream: Our previous attempt at `COMPOSE_DOCKER_CLI_BUILD` works locally for `docker build` and `docker-compose build` interop. The issue in Buildkite is that we use multi-stage builds which means that the cached ECR image only contains layers from the final stage, so Compose will always try to rebuild the earlier stages like `yarn install`. This switches our Compose file over to directly reference the cached ECR image. 1. Upside: we never wastefully rebuild base images through Compose, saving ~2 minutes per step. 2. Downside: Compose file is not usable for local debugging. A workaround is to define separate Compose services for CI and local use, but that isn't implemented in this PR.
seek-oss · Jun 10, 2021 · 3042866 · 3042866
1 parent 21bf2b9
commit 3042866
Show file tree

Hide file tree

Showing 12 changed files with 54 additions and 27 deletions.
diff --git a/.changeset/fresh-feet-trade.md b/.changeset/fresh-feet-trade.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template/greeter:** Use `seek-oss/docker-ecr-cache` Buildkite plugin
diff --git a/.changeset/hungry-parents-repeat.md b/.changeset/hungry-parents-repeat.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template/koa-rest-api:** Use `seek-oss/docker-ecr-cache` Buildkite plugin
diff --git a/.changeset/late-owls-type.md b/.changeset/late-owls-type.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template/express-rest-api:** Use `seek-oss/docker-ecr-cache` Buildkite plugin
diff --git a/.changeset/olive-chicken-doubt.md b/.changeset/olive-chicken-doubt.md
@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**template:** Reuse ECR cache in Docker Compose
diff --git a/template/express-rest-api/.buildkite/pipeline.yml b/template/express-rest-api/.buildkite/pipeline.yml
@@ -4,6 +4,15 @@ configs:
       env:
         NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
+  - &docker-ecr-cache
+    seek-oss/docker-ecr-cache#v1.9.0:
+      build-args:
+        - NPM_READ_TOKEN
+      cache-on:
+        - package.json
+        - yarn.lock
+      target: dev-deps
+
   - &deploy
     concurrency: 1
     retry:
@@ -30,8 +39,7 @@ steps:
       - yarn lint
     plugins:
       - *aws-sm
-      - ecr#v2.3.0:
-          login: true
+      - *docker-ecr-cache
       - docker-compose#v3.7.0:
           run: app
 

diff --git a/template/express-rest-api/docker-compose.yml b/template/express-rest-api/docker-compose.yml
@@ -2,11 +2,7 @@ version: '3.7'
 
 services:
   app:
-    build:
-      args:
-        - NPM_READ_TOKEN
-      context: .
-      target: dev-deps
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE}
     init: true
     volumes:
       - ./:/workdir

diff --git a/template/greeter/.buildkite/pipeline.yml b/template/greeter/.buildkite/pipeline.yml
@@ -4,6 +4,15 @@ configs:
       env:
         NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
+  - &docker-ecr-cache
+    seek-oss/docker-ecr-cache#v1.5.0:
+      build-args:
+        - NPM_READ_TOKEN
+      cache-on:
+        - package.json
+        - yarn.lock
+      target: dev-deps
+
   - &prod
     agents:
       queue: <%- prodBuildkiteQueueName %>
@@ -18,7 +27,6 @@ steps:
       - yarn lint
     plugins:
       - *aws-sm
-      - ecr#v2.3.0:
-          login: true
+      - *docker-ecr-cache
       - docker-compose#v3.7.0:
           run: app
diff --git a/template/greeter/docker-compose.yml b/template/greeter/docker-compose.yml
@@ -2,11 +2,7 @@ version: '3.7'
 
 services:
   app:
-    build:
-      args:
-        - NPM_READ_TOKEN
-      context: .
-      target: dev-deps
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE}
     init: true
     volumes:
       - ./:/workdir

diff --git a/template/koa-rest-api/.buildkite/pipeline.yml b/template/koa-rest-api/.buildkite/pipeline.yml
@@ -4,6 +4,15 @@ configs:
       env:
         NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
+  - &docker-ecr-cache
+    seek-oss/docker-ecr-cache#v1.9.0:
+      build-args:
+        - NPM_READ_TOKEN
+      cache-on:
+        - package.json
+        - yarn.lock
+      target: dev-deps
+
   - &deploy
     concurrency: 1
     retry:
@@ -30,8 +39,7 @@ steps:
       - yarn lint
     plugins:
       - *aws-sm
-      - ecr#v2.3.0:
-          login: true
+      - *docker-ecr-cache
       - docker-compose#v3.7.0:
           run: app
 

diff --git a/template/koa-rest-api/docker-compose.yml b/template/koa-rest-api/docker-compose.yml
@@ -2,11 +2,7 @@ version: '3.7'
 
 services:
   app:
-    build:
-      args:
-        - NPM_READ_TOKEN
-      context: .
-      target: dev-deps
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE}
     init: true
     volumes:
       - ./:/workdir

diff --git a/template/lambda-sqs-worker/.buildkite/pipeline.yml b/template/lambda-sqs-worker/.buildkite/pipeline.yml
@@ -1,5 +1,4 @@
 env:
-  COMPOSE_DOCKER_CLI_BUILD: 1
   VERSION: ${BUILDKITE_COMMIT:0:7}.${BUILDKITE_BUILD_NUMBER}
 
 configs:

diff --git a/template/lambda-sqs-worker/docker-compose.yml b/template/lambda-sqs-worker/docker-compose.yml
@@ -2,11 +2,7 @@ version: '3.7'
 
 services:
   app:
-    build:
-      args:
-        - NPM_READ_TOKEN
-      context: .
-      target: dev-deps
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE}
     environment:
       - BUILDKITE_COMMIT
       - ENVIRONMENT