From 324db7e3e17e99fb84a94b17394d4ccebd5a6f5a Mon Sep 17 00:00:00 2001 From: Joshua Gutow Date: Mon, 21 Nov 2022 09:40:54 -0800 Subject: [PATCH 1/2] CI: Remove optional target from docker-publish step The old nightlies used a partial build (which is the purpose of the --target flag), but they were removed in PR #3330 (commit f772f66cfcb59e57d43bdd0419baf5a5ee96228e). --- .circleci/config.yml | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 80d957da22c7e..cde56a1c69d81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,10 +100,6 @@ jobs: docker_context: description: Docker build context type: string - target: - description: Docker build target - type: string - default: "" registry: description: Docker registry type: string @@ -125,33 +121,16 @@ jobs: - run: cat /etc/netplan/50-cloud-init.yaml - run: sudo netplan apply - checkout - - when: - condition: <> - steps: - - run: - name: Build with context - command: | - echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin - IMAGE_BASE="<>/<>/<>" - DOCKER_TAGS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|") - docker build \ - $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \ - -f <> \ - --target <> \ - <> - - unless: - condition: <> - steps: - - run: - name: Build - command: | - echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin - IMAGE_BASE="<>/<>/<>" - DOCKER_TAGS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|") - docker build \ - $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \ - -f <> \ - <> + - run: + name: Build + command: | + echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin + IMAGE_BASE="<>/<>/<>" + DOCKER_TAGS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|") + docker build \ + $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \ + -f <> \ + <> - run: name: Publish command: | From 80e17c002ceac92599ca60dfbd93d302cf96943b Mon Sep 17 00:00:00 2001 From: Joshua Gutow Date: Mon, 21 Nov 2022 12:29:22 -0800 Subject: [PATCH 2/2] ci: Split docker build & publish This splits the docker publish job into a build & publish job. They were previously just steps. The build step uses docker save & circle workspaces to provide the docker images to dependent jobs (like the publish & hive jobs). This makes one change to the hive job to remove the docker.pull=true flag. This is because it loads the image locally but in the case of an external contributor the image will not be published. Docker will use the local image first & then pull other images if they are not present. --- .circleci/config.yml | 124 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 99 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cde56a1c69d81..f3a4eee18b0e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,8 +83,7 @@ jobs: - "packages/contracts/libraries" - "packages/contracts/standards" - - docker-publish: + docker-build: environment: DOCKER_BUILDKIT: 1 parameters: @@ -112,25 +111,69 @@ jobs: image: ubuntu-2204:2022.07.1 resource_class: xlarge steps: - - gcp-oidc-authenticate - # Below is CircleCI recommended way of specifying nameservers on an Ubuntu box: - # https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan - - run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml - - run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml - - run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml - - run: cat /etc/netplan/50-cloud-init.yaml - - run: sudo netplan apply - checkout + - run: + command: mkdir -p /tmp/docker_images - run: name: Build command: | - echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin + # Check to see if DOCKER_HUB_READ_ONLY_TOKEN is set (i.e. we are in repo) before attempting to use secrets. + # Building should work without this read only login, but may get rate limited. + if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then + echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin + fi IMAGE_BASE="<>/<>/<>" DOCKER_TAGS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|") docker build \ $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \ -f <> \ <> + - run: + name: Save + command: | + IMAGE_BASE="<>/<>/<>" + DOCKER_LABELS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g") + echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker save -o /tmp/docker_images/<>_{}.tar $IMAGE_BASE:{} + - persist_to_workspace: + root: /tmp/docker_images + paths: + - "." + + docker-publish: + parameters: + docker_name: + description: Docker image name + type: string + docker_tags: + description: Docker image tags as csv + type: string + registry: + description: Docker registry + type: string + default: "us-central1-docker.pkg.dev" + repo: + description: Docker repo + type: string + default: "bedrock-goerli-development/images" + machine: + image: ubuntu-2204:2022.07.1 + resource_class: xlarge + steps: + - attach_workspace: + at: /tmp/docker_images + - run: + name: Docker load + command: | + DOCKER_LABELS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g") + echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker load -i /tmp/docker_images/<>_{}.tar + - gcp-oidc-authenticate + # Below is CircleCI recommended way of specifying nameservers on an Ubuntu box: + # https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan + - run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml + - run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml + - run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml + - run: cat /etc/netplan/50-cloud-init.yaml + - run: sudo netplan apply - run: name: Publish command: | @@ -669,6 +712,14 @@ jobs: docker_layer_caching: true resource_class: xlarge steps: + - attach_workspace: + at: /tmp/docker_images + - run: + name: Docker Load + command: | + docker load -i /tmp/docker_images/op-batcher_<>.tar + docker load -i /tmp/docker_images/op-proposer_<>.tar + docker load -i /tmp/docker_images/op-node_<>.tar - run: command: git clone https://github.com/ethereum-optimism/hive.git . - go/load-cache @@ -680,7 +731,6 @@ jobs: ./hive \ -sim=<> \ -sim.loglevel=5 \ - -docker.pull=true \ -client=go-ethereum,op-geth_optimism-history,op-proposer_<>,op-batcher_<>,op-node_<> |& tee /tmp/hive.log || echo "failed." - run: command: | @@ -899,8 +949,8 @@ workflows: - op-service-tests - op-e2e-WS-tests - op-e2e-HTTP-tests - - docker-publish: - name: op-node-publish-dev + - docker-build: + name: op-node-docker-build docker_file: op-node/Dockerfile docker_name: op-node docker_tags: <>,<> @@ -908,7 +958,15 @@ workflows: context: - gcr - docker-publish: - name: op-batcher-publish-dev + name: op-node-docker-publish + docker_name: op-node + docker_tags: <>,<> + context: + - gcr + requires: + - op-node-docker-build + - docker-build: + name: op-batcher-docker-build docker_file: op-batcher/Dockerfile docker_name: op-batcher docker_tags: <>,<> @@ -916,37 +974,53 @@ workflows: context: - gcr - docker-publish: - name: op-proposer-publish-dev + name: op-batcher-docker-publish + docker_name: op-batcher + docker_tags: <>,<> + context: + - gcr + requires: + - op-batcher-docker-build + - docker-build: + name: op-proposer-docker-build docker_file: op-proposer/Dockerfile docker_name: op-proposer docker_tags: <>,<> docker_context: . context: - gcr + - docker-publish: + name: op-proposer-docker-publish + docker_name: op-proposer + docker_tags: <>,<> + context: + - gcr + requires: + - op-proposer-docker-build - hive-test: name: hive-test-rpc version: <> sim: optimism/rpc requires: - - op-node-publish-dev - - op-batcher-publish-dev - - op-proposer-publish-dev + - op-node-docker-build + - op-batcher-docker-build + - op-proposer-docker-build - hive-test: name: hive-test-p2p version: <> sim: optimism/p2p requires: - - op-node-publish-dev - - op-batcher-publish-dev - - op-proposer-publish-dev + - op-node-docker-build + - op-batcher-docker-build + - op-proposer-docker-build - hive-test: name: hive-test-l1ops version: <> sim: optimism/l1ops requires: - - op-node-publish-dev - - op-batcher-publish-dev - - op-proposer-publish-dev + - op-node-docker-build + - op-batcher-docker-build + - op-proposer-docker-build release: jobs: - docker-tag-op-stack-release: