ENG-2990 updates ci to tag docker images with sha, branch, release tags

.circleci/config.yml

@@ -88,6 +88,9 @@ jobs:
     environment:
       DOCKER_BUILDKIT: 1
     parameters:
+      docker_name:
+        description: Docker image name
+        type: string
       docker_tags:
         description: Docker image tags as csv
         type: string
@@ -101,10 +104,14 @@ jobs:
         description: Docker build target
         type: string
         default: ""
+      registry:
+        description: Docker registry
+        type: string
+        default: "us-central1-docker.pkg.dev"
       repo:
         description: Docker repo
         type: string
-        default: ""
+        default: "bedrock-goerli-development/images"
     machine:
       image: ubuntu-2204:2022.07.1
       resource_class: xlarge
@@ -125,8 +132,10 @@ jobs:
                 name: Build with context
                 command: |
                   echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin
+                  IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
+                  DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
                   docker build \
-                  $(echo -ne "<< parameters.docker_tags >>" | sed "s/,/\n/g" | sed -e 's/^/-t /' | tr '\n' ' ') \
+                  $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
                   -f <<parameters.docker_file>> \
                   --target <<parameters.target>> \
                   <<parameters.docker_context>>
@@ -137,15 +146,41 @@ jobs:
                 name: Build
                 command: |
                   echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin
+                  IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
+                  DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
                   docker build \
-                  $(echo -ne "<< parameters.docker_tags >>" | sed "s/,/\n/g" | sed -e 's/^/-t /' | tr '\n' ' ') \
+                  $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
                   -f <<parameters.docker_file>> \
                   <<parameters.docker_context>>
       - run:
           name: Publish
           command: |
-            gcloud auth configure-docker us-central1-docker.pkg.dev
-            docker push <<parameters.docker_tags>>
+            gcloud auth configure-docker <<parameters.registry>>
+            IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
+            DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|${IMAGE_BASE}:|")
+            echo -ne $DOCKER_TAGS | tr ' ' '\n' | xargs -L1 docker push
+
+  docker-tag-op-stack-release:
+    parameters:
+      registry:
+        description: Docker registry
+        type: string
+        default: "us-central1-docker.pkg.dev"
+      repo:
+        description: Docker repo
+        type: string
+        default: "bedrock-goerli-development/images"
+    docker:
+      - image: google/cloud-sdk:latest
+    resource_class: small
+    steps:
+      - gcp-oidc-authenticate
+      - checkout
+      - run:
+          name: Tag
+          command: |
+            gcloud auth configure-docker <<parameters.registry>>
+            ./ops/ci-docker-tag-op-stack-release.sh <<parameters.registry>>/<<parameters.repo>> $CIRCLE_TAG $CIRCLE_SHA1
 
   contracts-bedrock-tests:
     docker:
@@ -887,25 +922,25 @@ workflows:
       - docker-publish:
           name: op-node-publish-dev
           docker_file: op-node/Dockerfile
-          docker_tags: us-central1-docker.pkg.dev/bedrock-goerli-development/images/op-node:<<pipeline.git.revision>>
+          docker_name: op-node
+          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
           docker_context: .
-          repo: us-central1-docker.pkg.dev
           context:
             - gcr
       - docker-publish:
           name: op-batcher-publish-dev
           docker_file: op-batcher/Dockerfile
-          docker_tags: us-central1-docker.pkg.dev/bedrock-goerli-development/images/op-batcher:<<pipeline.git.revision>>
+          docker_name: op-batcher
+          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
           docker_context: .
-          repo: us-central1-docker.pkg.dev
           context:
             - gcr
       - docker-publish:
           name: op-proposer-publish-dev
           docker_file: op-proposer/Dockerfile
-          docker_tags: us-central1-docker.pkg.dev/bedrock-goerli-development/images/op-proposer:<<pipeline.git.revision>>
+          docker_name: op-proposer
+          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
           docker_context: .
-          repo: us-central1-docker.pkg.dev
           context:
             - gcr
       - hive-test:
@@ -932,3 +967,14 @@ workflows:
             - op-node-publish-dev
             - op-batcher-publish-dev
             - op-proposer-publish-dev
+  release:
+    jobs:
+      - docker-tag-op-stack-release:
+          name: docker-tag-op-stack-release
+          filters:
+            tags:
+              only: /^op-[a-z0-9\-]*\/v.*/
+            branches:
+              ignore: /.*/
+          context:
+            - gcr-release

cloudbuild.yaml

@@ -1,28 +1,4 @@
 steps:
-  - name: 'gcr.io/kaniko-project/executor:latest'
-    args:
-      - --destination=us-central1-docker.pkg.dev/$PROJECT_ID/images/op-node:$_TAG
-      - --destination=us-central1-docker.pkg.dev/$PROJECT_ID/images/op-node:$COMMIT_SHA
-      - --dockerfile=op-node/Dockerfile
-      - --cache=true
-      - --cache-ttl=48h
-    waitFor: ['-']
-  - name: 'gcr.io/kaniko-project/executor:latest'
-    args:
-      - --destination=us-central1-docker.pkg.dev/$PROJECT_ID/images/op-batcher:$_TAG
-      - --destination=us-central1-docker.pkg.dev/$PROJECT_ID/images/op-batcher:$COMMIT_SHA
-      - --dockerfile=./op-batcher/Dockerfile
-      - --cache=true
-      - --cache-ttl=48h
-    waitFor: ['-']
-  - name: 'gcr.io/kaniko-project/executor:latest'
-    args:
-      - --destination=us-central1-docker.pkg.dev/$PROJECT_ID/images/op-proposer:$_TAG
-      - --destination=us-central1-docker.pkg.dev/$PROJECT_ID/images/op-proposer:$COMMIT_SHA
-      - --dockerfile=./op-proposer/Dockerfile
-      - --cache=true
-      - --cache-ttl=48h
-    waitFor: ['-']
   - name: 'gcr.io/kaniko-project/executor:latest'
     args:
       - --destination=us-central1-docker.pkg.dev/$PROJECT_ID/images/deployer-bedrock:$_TAG
@@ -42,4 +18,4 @@ steps:
       - --cache-ttl=48h
     waitFor: ['-']
 options:
-  machineType: N1_HIGHCPU_32
+  machineType: N1_HIGHCPU_32

ops/scripts/ci-docker-tag-op-stack-release.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+DOCKER_REPO=$1
+GIT_TAG=$2
+GIT_SHA=$3
+
+IMAGE_NAME=$(echo "$GIT_TAG" | grep -Eow '^op-[a-z0-9\-]*')
+IMAGE_TAG=$(echo "$GIT_TAG" | grep -Eow 'v.*')
+
+SOURCE_IMAGE_TAG="$DOCKER_REPO/$IMAGE_NAME:$GIT_SHA"
+TARGET_IMAGE_TAG="$DOCKER_REPO/$IMAGE_NAME:$IMAGE_TAG"
+TARGET_IMAGE_TAG_LATEST="$DOCKER_REPO/$IMAGE_NAME:latest"
+
+echo "Checking if docker images exist for '$IMAGE_NAME'"
+echo ""
+tags=$(gcloud container images list-tags "$DOCKER_REPO/$IMAGE_NAME" --limit 1 --format json)
+if [ "$tags" = "[]" ]; then
+  echo "No existing docker images were found for '$IMAGE_NAME'. The code tagged with '$GIT_TAG' may not have an associated dockerfile or docker build job."
+  echo "If this service has a dockerfile, add a docker-publish job for it in the circleci config."
+  echo ""
+  echo "Exiting"
+  exit 0
+fi
+
+echo "Tagging $SOURCE_IMAGE_TAG with '$IMAGE_TAG'"
+gcloud container images add-tag -q "$SOURCE_IMAGE_TAG" "$TARGET_IMAGE_TAG"
+
+echo "Tagging $SOURCE_IMAGE_TAG with 'latest'"
+gcloud container images add-tag -q "$SOURCE_IMAGE_TAG" "$TARGET_IMAGE_TAG_LATEST"