From 0fb3050f5eb554bb9e4cd7f5bb0c5a1197ba1d9c Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Mon, 7 Mar 2022 14:06:05 -0700 Subject: [PATCH] ops: Build cleanups Cleans up a bunch of configuration in `ops/`, in preparation for switching most of our build configuration to CircleCI. Changes include: 1. Removing the "builder image" concept in favor of a single multi-target Dockerfile. Once this goes live on CircleCI, we'll be able to leverage Docker Layer caching more effectively. 2. Fixes the nightly build, which is currently failing on the legacy batch submitter. 3. Removes the legacy rpc-proxy since it isn't used anywhere anymore. 4. Re-enables CODEOWNERS. 5. Removes the `docker-compose-nobuild.yml` file, since the main `docker-compose.yml` file now supports images and builds. Meta: - Fixes ENG-1838 --- .circleci/config.yml | 14 -- .github/CODEOWNERS | 36 ++--- .github/workflows/ext-test-snx.yml | 2 +- .github/workflows/publish-canary.yml | 126 ++++------------ .github/workflows/release.yml | 136 ++++-------------- .github/workflows/sync-tests.yml | 2 +- CONTRIBUTING.md | 11 -- ops/docker-compose-nobuild.yml | 151 -------------------- ops/docker-compose-rpc-proxy.yml | 17 --- ops/docker-compose.yml | 13 +- ops/docker/Dockerfile.data-transport-layer | 35 ----- ops/docker/Dockerfile.deployer | 37 ----- ops/docker/Dockerfile.integration-tests | 31 ---- ops/docker/Dockerfile.message-relayer | 35 ----- ops/docker/Dockerfile.monorepo | 68 --------- ops/docker/Dockerfile.packages | 7 +- ops/docker/Dockerfile.replica-healthcheck | 33 ----- ops/docker/Dockerfile.rpc-proxy | 21 --- ops/docker/builder/CHANGELOG.md | 18 --- ops/docker/builder/package.json | 5 - ops/docker/rpc-proxy/CHANGELOG.md | 19 --- ops/docker/rpc-proxy/docker-entrypoint.sh | 19 --- ops/docker/rpc-proxy/eth-jsonrpc-access.lua | 91 ------------ ops/docker/rpc-proxy/nginx.template.conf | 91 ------------ ops/docker/rpc-proxy/package.json | 6 - ops/scripts/build-ci.sh | 21 --- ops/scripts/ci-versions.js | 19 --- 27 files changed, 99 insertions(+), 965 deletions(-) delete mode 100644 ops/docker-compose-nobuild.yml delete mode 100644 ops/docker-compose-rpc-proxy.yml delete mode 100644 ops/docker/Dockerfile.data-transport-layer delete mode 100644 ops/docker/Dockerfile.deployer delete mode 100644 ops/docker/Dockerfile.integration-tests delete mode 100644 ops/docker/Dockerfile.message-relayer delete mode 100644 ops/docker/Dockerfile.monorepo delete mode 100644 ops/docker/Dockerfile.replica-healthcheck delete mode 100644 ops/docker/Dockerfile.rpc-proxy delete mode 100644 ops/docker/builder/CHANGELOG.md delete mode 100644 ops/docker/builder/package.json delete mode 100644 ops/docker/rpc-proxy/CHANGELOG.md delete mode 100755 ops/docker/rpc-proxy/docker-entrypoint.sh delete mode 100644 ops/docker/rpc-proxy/eth-jsonrpc-access.lua delete mode 100644 ops/docker/rpc-proxy/nginx.template.conf delete mode 100644 ops/docker/rpc-proxy/package.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 40772adcbb5b4..7f91c78040f97 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,14 +65,6 @@ jobs: image-name: data-transport-layer target: data-transport-layer dockerfile: ./ops/docker/Dockerfile.packages - build-batch-submitter: - docker: - - image: cimg/base:2021.04 - steps: - - build-dockerfile: - image-name: batch-submitter - target: batch-submitter - dockerfile: ./ops/docker/Dockerfile.packages build-go-batch-submitter: docker: - image: cimg/base:2021.04 @@ -264,11 +256,6 @@ workflows: - optimism - slack <<: *slack-nightly-build-fail-post-step - - build-batch-submitter: - context: - - optimism - - slack - <<: *slack-nightly-build-fail-post-step - build-deployer: context: - optimism @@ -306,7 +293,6 @@ workflows: <<: *slack-nightly-build-fail-post-step requires: - build-dtl - - build-batch-submitter - build-go-batch-submitter - build-deployer - build-l2geth diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 018d44015b875..476c30ed0e2a9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,17 +1,21 @@ -# CODEOWNERS can be disruptive because it automatically requests review from individuals across the -# board. We still like to use this file to track who's working on what, but all lines are commented -# out so that GitHub won't trigger review requests. +go/bss-core @cfromknecht @tynes +go/batch-submitter @cfromknecht @tynes +go/gas-oracle @tynes +go/l2geth-exporter @optimisticben @mslipper +go/op-exporter @optimisticben @mslipper +go/proxyd @mslipper @inphi +go/teleportr @mslipper @cfromknecht -# l2geth/ @smartcontracts @tynes @karlfloersch -# packages/specs/l2geth/ @smartcontracts @tynes @karlfloersch -# packages/contracts/ @smartcontracts @ben-chain @maurelian @elenadimitrova -# packages/specs/protocol/ @smartcontracts @ben-chain @maurelian -# ops/ @tynes @karlfloersch -# packages/core-utils/ @smartcontracts @annieke @ben-chain -# packages/common-ts/ @annieke -# packages/core-utils/src/watcher.ts @K-Ho -# packages/message-relayer/ @K-Ho -# packages/batch-submitter/ @annieke @karlfloersch -# packages/data-transport-layer/ @annieke -# packages/replica-healthcheck/ @annieke -# integration-tests/ @tynes +integration-tests/ @tynes @mslipper + +packages/core-utils @smartcontracts @tynes +packages/common-ts/ @smartcontracts +packages/message-relayer/ @smartcontracts +packages/data-transport-layer/ @tynes @smartcontracts +packages/replica-healthcheck @optimisticben @tynes +packages/sdk @smartcontracts @mslipper +packages/contracts @elenadimitrova @maurelian @smartcontracts + +l2geth @tynes @cfromknecht @smartcontracts + +ops @tynes @optimisticben @mslipper diff --git a/.github/workflows/ext-test-snx.yml b/.github/workflows/ext-test-snx.yml index c9ceafc38aac8..676929c6ed449 100644 --- a/.github/workflows/ext-test-snx.yml +++ b/.github/workflows/ext-test-snx.yml @@ -69,7 +69,7 @@ jobs: if: failure() uses: jwalton/gh-docker-logs@v1 with: - images: 'ethereumoptimism/builder,ethereumoptimism/hardhat,ethereumoptimism/deployer,ethereumoptimism/data-transport-layer,ethereumoptimism/l2geth,ethereumoptimism/message-relayer,ethereumoptimism/batch-submitter,ethereumoptimism/l2geth,ethereumoptimism/integration-tests' + images: 'ethereumoptimism/hardhat,ethereumoptimism/deployer,ethereumoptimism/data-transport-layer,ethereumoptimism/l2geth,ethereumoptimism/message-relayer,ethereumoptimism/batch-submitter,ethereumoptimism/l2geth,ethereumoptimism/integration-tests' dest: '~/logs' - name: Tar logs diff --git a/.github/workflows/publish-canary.yml b/.github/workflows/publish-canary.yml index 0638071037eb5..a466f557d66ef 100644 --- a/.github/workflows/publish-canary.yml +++ b/.github/workflows/publish-canary.yml @@ -15,7 +15,6 @@ jobs: runs-on: ubuntu-latest # map the step outputs to job outputs outputs: - builder: ${{ steps.packages.outputs.builder }} l2geth: ${{ steps.packages.outputs.l2geth }} message-relayer: ${{ steps.packages.outputs.message-relayer }} data-transport-layer: ${{ steps.packages.outputs.data-transport-layer }} @@ -25,7 +24,6 @@ jobs: hardhat-node: ${{ steps.packages.outputs.hardhat-node }} canary-docker-tag: ${{ steps.docker-image-name.outputs.canary-docker-tag }} proxyd: ${{ steps.packages.outputs.proxyd }} - rpc-proxy : ${{ steps.packages.outputs.rpc-proxy }} op-exporter : ${{ steps.packages.outputs.op-exporter }} l2geth-exporter : ${{ steps.packages.outputs.l2geth-exporter }} batch-submitter-service : ${{ steps.packages.outputs.batch-submitter-service }} @@ -91,10 +89,6 @@ jobs: env: CUSTOM_IMAGE_NAME: ${{ github.event.inputs.customImageName }} - # The below code is duplicated, would be ideal if we could use a matrix with a - # key/value being dynamically generated from the `publishedPackages` output - # while also allowing for parallelization (i.e. `l2geth` not depending on `builder`) - # and all jobs executing in parallel once `builder` is built l2geth: name: Publish L2Geth Version ${{ needs.canary-publish.outputs.canary-docker-tag }} needs: canary-publish @@ -173,44 +167,10 @@ jobs: push: true tags: ethereumoptimism/hardhat-node:${{ needs.canary-publish.outputs.canary-docker-tag }} - builder: - name: Prepare the base builder image for the services - needs: canary-publish - runs-on: ubuntu-latest - # we re-output the variables so that the child jobs can access them - outputs: - message-relayer: ${{ needs.canary-publish.outputs.message-relayer }} - data-transport-layer: ${{ needs.canary-publish.outputs.data-transport-layer }} - contracts: ${{ needs.canary-publish.outputs.contracts }} - integration-tests: ${{ needs.canary-publish.outputs.integration-tests }} - replica-healthcheck: ${{ needs.canary-publish.outputs.replica-healthcheck }} - canary-docker-tag: ${{ needs.canary-publish.outputs.canary-docker-tag }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./ops/docker/Dockerfile.monorepo - push: true - tags: ethereumoptimism/builder:${{ needs.canary-publish.outputs.canary-docker-tag }} - message-relayer: - name: Publish Message Relayer Version ${{ needs.builder.outputs.canary-docker-tag }} - needs: builder - if: needs.builder.outputs.message-relayer != '' + name: Publish Message Relayer Version ${{ needs.canary-publish.outputs.canary-docker-tag }} + needs: canary-publish + if: needs.canary-publish.outputs.message-relayer != '' runs-on: ubuntu-latest steps: @@ -229,15 +189,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.message-relayer + file: ./ops/docker/Dockerfile.packages + target: relayer push: true - tags: ethereumoptimism/message-relayer:${{ needs.builder.outputs.canary-docker-tag }} - build-args: BUILDER_TAG=${{ needs.builder.outputs.canary-docker-tag }} + tags: ethereumoptimism/message-relayer:${{ needs.canary-publish.outputs.canary-docker-tag }} data-transport-layer: - name: Publish Data Transport Layer Version ${{ needs.builder.outputs.canary-docker-tag }} - needs: builder - if: needs.builder.outputs.data-transport-layer != '' + name: Publish Data Transport Layer Version ${{ needs.canary-publish.outputs.canary-docker-tag }} + needs: canary-publish + if: needs.canary-publish.outputs.data-transport-layer != '' runs-on: ubuntu-latest steps: @@ -256,15 +216,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.data-transport-layer + file: ./ops/docker/Dockerfile.packages + target: data-transport-layer push: true - tags: ethereumoptimism/data-transport-layer:${{ needs.builder.outputs.canary-docker-tag }} - build-args: BUILDER_TAG=${{ needs.builder.outputs.canary-docker-tag }} + tags: ethereumoptimism/data-transport-layer:${{ needs.canary-publish.outputs.canary-docker-tag }} contracts: - name: Publish Deployer Version ${{ needs.builder.outputs.canary-docker-tag }} - needs: builder - if: needs.builder.outputs.contracts != '' + name: Publish Deployer Version ${{ needs.canary-publish.outputs.canary-docker-tag }} + needs: canary-publish + if: needs.canary-publish.outputs.contracts != '' runs-on: ubuntu-latest steps: @@ -283,15 +243,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.deployer + file: ./ops/docker/Dockerfile.packages + target: deployer push: true - tags: ethereumoptimism/deployer:${{ needs.builder.outputs.canary-docker-tag }} - build-args: BUILDER_TAG=${{ needs.builder.outputs.canary-docker-tag }} + tags: ethereumoptimism/deployer:${{ needs.canary-publish.outputs.canary-docker-tag }} integration_tests: - name: Publish Integration tests ${{ needs.builder.outputs.integration-tests }} - needs: builder - if: needs.builder.outputs.integration-tests != '' + name: Publish Integration tests ${{ needs.canary-publish.outputs.integration-tests }} + needs: canary-publish + if: needs.canary-publish.outputs.integration-tests != '' runs-on: ubuntu-latest steps: @@ -310,15 +270,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.integration-tests + file: ./ops/docker/Dockerfile.packages + target: integration-tests push: true - tags: ethereumoptimism/integration-tests:${{ needs.builder.outputs.canary-docker-tag }} - build-args: BUILDER_TAG=${{ needs.builder.outputs.canary-docker-tag }} + tags: ethereumoptimism/integration-tests:${{ needs.canary-publish.outputs.canary-docker-tag }} replica-healthcheck: - name: Publish Replica Healthcheck Version ${{ needs.builder.outputs.canary-docker-tag }} - needs: builder - if: needs.builder.outputs.replica-healthcheck != '' + name: Publish Replica Healthcheck Version ${{ needs.canary-publish.outputs.canary-docker-tag }} + needs: canary-publish + if: needs.canary-publish.outputs.replica-healthcheck != '' runs-on: ubuntu-latest steps: @@ -337,10 +297,10 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.replica-healthcheck + file: ./ops/docker/Dockerfile.packages + target: replica-healthcheck push: true - tags: ethereumoptimism/replica-healthcheck:${{ needs.builder.outputs.canary-docker-tag }} - build-args: BUILDER_TAG=${{ needs.builder.outputs.canary-docker-tag }} + tags: ethereumoptimism/replica-healthcheck:${{ needs.canary-publish.outputs.canary-docker-tag }} proxyd: name: Publish proxyd Version ${{ needs.canary-publish.outputs.canary-docker-tag }} @@ -453,32 +413,6 @@ jobs: GITCOMMIT=${{ steps.build_args.outputs.GITCOMMIT }} GITVERSION=${{ steps.build_args.outputs.GITVERSION }} - rpc-proxy: - name: Publish rpc-proxy Version ${{ needs.canary-publish.outputs.canary-docker-tag }} - needs: canary-publish - if: needs.canary-publish.outputs.rpc-proxy != '' - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./ops/docker/Dockerfile.rpc-proxy - push: true - tags: ethereumoptimism/rpc-proxy:${{ needs.canary-publish.outputs.rpc-proxy }} - batch-submitter-service: name: Publish batch-submitter-service Version ${{ needs.canary-publish.outputs.canary-docker-tag }} needs: canary-publish diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1db5157f40ab4..52cb989375cd6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,6 @@ jobs: runs-on: ubuntu-latest # map the step outputs to job outputs outputs: - use_builder: ${{ steps.packages.outputs.use_builder }} - builder: ${{ steps.packages.outputs.builder }} l2geth: ${{ steps.packages.outputs.l2geth }} message-relayer: ${{ steps.packages.outputs.message-relayer }} data-transport-layer: ${{ steps.packages.outputs.data-transport-layer }} @@ -20,7 +18,6 @@ jobs: gas-oracle: ${{ steps.packages.outputs.gas-oracle }} replica-healthcheck: ${{ steps.packages.outputs.replica-healthcheck }} proxyd: ${{ steps.packages.outputs.proxyd }} - rpc-proxy: ${{ steps.packages.outputs.rpc-proxy }} hardhat-node: ${{ steps.packages.outputs.hardhat-node }} op-exporter : ${{ steps.packages.outputs.op-exporter }} l2geth-exporter : ${{ steps.packages.outputs.l2geth-exporter }} @@ -70,10 +67,6 @@ jobs: run: | node ops/scripts/ci-versions.js ${{ toJSON(steps.changesets.outputs.publishedPackages) }} - # The below code is duplicated, would be ideal if we could use a matrix with a - # key/value being dynamically generated from the `publishedPackages` output - # while also allowing for parallelization (i.e. `l2geth` not depending on `builder`) - # and all jobs executing in parallel once `builder` is built l2geth: name: Publish L2Geth Version ${{ needs.release.outputs.l2geth }} needs: release @@ -263,79 +256,10 @@ jobs: GITCOMMIT=${{ steps.build_args.outputs.GITCOMMIT }} GITVERSION=${{ steps.build_args.outputs.GITVERSION }} - rpc-proxy: - name: Publish rpc-proxy Version ${{ needs.release.outputs.rpc-proxy }} - needs: release - if: needs.release.outputs.rpc-proxy != '' - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }} - - - name: Set env - run: | - echo "GITDATE=$(date)" >> $GITHUB_ENV" - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./ops/docker/Dockerfile.rpc-proxy - push: true - tags: ethereumoptimism/rpc-proxy:${{ needs.release.outputs.rpc-proxy }},ethereumoptimism/rpc-proxy:latest - - # pushes the base builder image to dockerhub - builder: - name: Prepare/Publish the base builder image for the services ${{ needs.release.outputs.builder }} - needs: release - # Build the builder if a dep of the builder has an update or if the builder - # has had its version bumped. TODO: remove the left hand side once tagged - # releases of the builder are released so that pulled builder images are - # used - if: ${{ needs.release.outputs.use_builder == 'true' || needs.release.outputs.builder != '' }} - runs-on: ubuntu-latest - # we re-output the variables so that the child jobs can access them - outputs: - builder: ${{ needs.release.outputs.builder || 'latest' }} - message-relayer: ${{ needs.release.outputs.message-relayer }} - data-transport-layer: ${{ needs.release.outputs.data-transport-layer }} - contracts: ${{ needs.release.outputs.contracts }} - integration-tests: ${{ needs.release.outputs.integration-tests }} - replica-healthcheck: ${{ needs.release.outputs.replica-healthcheck }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./ops/docker/Dockerfile.monorepo - push: true - tags: ethereumoptimism/builder:latest - message-relayer: - name: Publish Message Relayer Version ${{ needs.builder.outputs.message-relayer }} - needs: builder - if: needs.builder.outputs.message-relayer != '' + name: Publish Message Relayer Version ${{ needs.release.outputs.message-relayer }} + needs: release + if: needs.release.outputs.message-relayer != '' runs-on: ubuntu-latest steps: @@ -354,15 +278,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.message-relayer + file: ./ops/docker/Dockerfile.packages + target: message-relayer push: true - tags: ethereumoptimism/message-relayer:${{ needs.builder.outputs.message-relayer }},ethereumoptimism/message-relayer:latest - build-args: BUILDER_TAG=${{ needs.builder.outputs.builder }} + tags: ethereumoptimism/message-relayer:${{ needs.release.outputs.message-relayer }},ethereumoptimism/message-relayer:latest data-transport-layer: - name: Publish Data Transport Layer Version ${{ needs.builder.outputs.data-transport-layer }} - needs: builder - if: needs.builder.outputs.data-transport-layer != '' + name: Publish Data Transport Layer Version ${{ needs.release.outputs.data-transport-layer }} + needs: release + if: needs.release.outputs.data-transport-layer != '' runs-on: ubuntu-latest steps: @@ -381,15 +305,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.data-transport-layer + file: ./ops/docker/Dockerfile.packages + target: data-transport-layer push: true - tags: ethereumoptimism/data-transport-layer:${{ needs.builder.outputs.data-transport-layer }},ethereumoptimism/data-transport-layer:latest - build-args: BUILDER_TAG=${{ needs.builder.outputs.builder }} + tags: ethereumoptimism/data-transport-layer:${{ needs.release.outputs.data-transport-layer }},ethereumoptimism/data-transport-layer:latest contracts: - name: Publish Deployer Version ${{ needs.builder.outputs.contracts }} - needs: builder - if: needs.builder.outputs.contracts != '' + name: Publish Deployer Version ${{ needs.release.outputs.contracts }} + needs: release + if: needs.release.outputs.contracts != '' runs-on: ubuntu-latest steps: @@ -408,15 +332,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.deployer + file: ./ops/docker/Dockerfile.packages + target: deployer push: true - tags: ethereumoptimism/deployer:${{ needs.builder.outputs.contracts }},ethereumoptimism/deployer:latest - build-args: BUILDER_TAG=${{ needs.builder.outputs.builder }} + tags: ethereumoptimism/deployer:${{ needs.release.outputs.contracts }},ethereumoptimism/deployer:latest integration_tests: - name: Publish Integration tests ${{ needs.builder.outputs.integration-tests }} - needs: builder - if: needs.builder.outputs.integration-tests != '' + name: Publish Integration tests ${{ needs.release.outputs.integration-tests }} + needs: release + if: needs.release.outputs.integration-tests != '' runs-on: ubuntu-latest steps: @@ -435,15 +359,15 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.integration-tests + file: ./ops/docker/Dockerfile.packages + target: integration-tests push: true - tags: ethereumoptimism/integration-tests:${{ needs.builder.outputs.integration-tests }},ethereumoptimism/integration-tests:latest - build-args: BUILDER_TAG=${{ needs.builder.outputs.builder }} + tags: ethereumoptimism/integration-tests:${{ needs.release.outputs.integration-tests }},ethereumoptimism/integration-tests:latest replica-healthcheck: - name: Publish Replica Healthcheck Version ${{ needs.builder.outputs.replica-healthcheck }} - needs: builder - if: needs.builder.outputs.replica-healthcheck != '' + name: Publish Replica Healthcheck Version ${{ needs.release.outputs.replica-healthcheck }} + needs: release + if: needs.release.outputs.replica-healthcheck != '' runs-on: ubuntu-latest steps: @@ -462,10 +386,10 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./ops/docker/Dockerfile.replica-healthcheck + file: ./ops/docker/Dockerfile.packages + target: replica-healthcheck push: true - tags: ethereumoptimism/replica-healthcheck:${{ needs.builder.outputs.replica-healthcheck }},ethereumoptimism/replica-healthcheck:latest - build-args: BUILDER_TAG=${{ needs.builder.outputs.builder }} + tags: ethereumoptimism/replica-healthcheck:${{ needs.release.outputs.replica-healthcheck }},ethereumoptimism/replica-healthcheck:latest batch-submitter-service: name: Publish batch-submitter-service Version ${{ needs.release.outputs.batch-submitter-service }} diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index 9723e2a72cdf3..59d05eceaaa8b 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -61,7 +61,7 @@ jobs: if: failure() uses: jwalton/gh-docker-logs@v1 with: - images: 'ethereumoptimism/builder,ethereumoptimism/hardhat,ethereumoptimism/deployer,ethereumoptimism/data-transport-layer,ethereumoptimism/l2geth,ethereumoptimism/message-relayer,ethereumoptimism/batch-submitter,ethereumoptimism/l2geth' + images: 'ethereumoptimism/hardhat,ethereumoptimism/deployer,ethereumoptimism/data-transport-layer,ethereumoptimism/l2geth,ethereumoptimism/message-relayer,ethereumoptimism/batch-submitter,ethereumoptimism/l2geth' dest: './logs' - name: Tar logs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5485a08be4d99..93ad63620ffc4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,7 +109,6 @@ docker-compose build This will build the following containers: -* [`builder`](https://hub.docker.com/r/ethereumoptimism/builder): used to build the TypeScript packages * [`l1_chain`](https://hub.docker.com/r/ethereumoptimism/hardhat): simulated L1 chain using hardhat-evm as a backend * [`deployer`](https://hub.docker.com/r/ethereumoptimism/deployer): process that deploys L1 smart contracts to the L1 chain * [`dtl`](https://hub.docker.com/r/ethereumoptimism/data-transport-layer): service that indexes transaction data from the L1 chain @@ -129,16 +128,6 @@ docker-compose build -- l2geth docker-compose start l2geth ``` -For the typescript services, you'll need to rebuild the `builder` so that the compiled -files are re-generated, and then your service, e.g. for the batch submitter - -```bash -cd ops -docker-compose stop -- batch_submitter -docker-compose build -- builder batch_submitter -docker-compose start batch_submitter -``` - Source code changes can have an impact on more than one container. **If you're unsure about which containers to rebuild, just rebuild them all**: diff --git a/ops/docker-compose-nobuild.yml b/ops/docker-compose-nobuild.yml deleted file mode 100644 index 3988f589fda4f..0000000000000 --- a/ops/docker-compose-nobuild.yml +++ /dev/null @@ -1,151 +0,0 @@ -version: "3.4" -services: - l1_chain: - image: ethereumoptimism/hardhat-node:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - build: - context: ./docker/hardhat - dockerfile: Dockerfile - ports: - # expose the service to the host for integration testing - - ${L1CHAIN_HTTP_PORT:-9545}:8545 - - deployer: - depends_on: - - l1_chain - image: ethereumoptimism/deployer:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - entrypoint: ./deployer.sh - environment: - FRAUD_PROOF_WINDOW_SECONDS: 0 - L1_NODE_WEB3_URL: http://l1_chain:8545 - # these keys are hardhat's first 3 accounts, DO NOT use in production - DEPLOYER_PRIVATE_KEY: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - SEQUENCER_PRIVATE_KEY: "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d" - PROPOSER_PRIVATE_KEY: "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a" - GAS_PRICE_ORACLE_OWNER: "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" - # setting the whitelist owner to address(0) disables the whitelist - WHITELIST_OWNER: "0x0000000000000000000000000000000000000000" - L1_FEE_WALLET_ADDRESS: "0x391716d440c151c42cdf1c95c1d83a5427bca52c" - L2_CHAIN_ID: 420 - BLOCK_SIGNER_ADDRESS: "0x00000398232E2064F896018496b4b44b3D62751F" - L2_BLOCK_GAS_LIMIT: 15000000 - GAS_PRICE_ORACLE_OVERHEAD: "2750" - GAS_PRICE_ORACLE_SCALAR: "1500000" - GAS_PRICE_ORACLE_L1_BASE_FEE: "1" - GAS_PRICE_ORACLE_GAS_PRICE: "1" - GAS_PRICE_ORACLE_DECIMALS: "6" - # skip compilation when run in docker-compose, since the contracts - # were already compiled in the builder step - NO_COMPILE: 1 - ports: - # expose the service to the host for getting the contract addrs - - ${DEPLOYER_PORT:-8080}:8081 - - dtl: - depends_on: - - l1_chain - - deployer - - l2geth - image: ethereumoptimism/data-transport-layer:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - entrypoint: ./dtl.sh - env_file: - - ./envs/dtl.env - environment: - # used for setting the address manager address - URL: http://deployer:8081/addresses.json - # connect to the 2 layers - DATA_TRANSPORT_LAYER__L1_RPC_ENDPOINT: http://l1_chain:8545 - DATA_TRANSPORT_LAYER__L2_RPC_ENDPOINT: http://l2geth:8545 - DATA_TRANSPORT_LAYER__SYNC_FROM_L2: 'true' - DATA_TRANSPORT_LAYER__L2_CHAIN_ID: 420 - ports: - - ${DTL_PORT:-7878}:7878 - - l2geth: - depends_on: - - l1_chain - - deployer - image: ethereumoptimism/l2geth:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - entrypoint: sh ./geth.sh - env_file: - - ./envs/geth.env - environment: - ETH1_HTTP: http://l1_chain:8545 - ROLLUP_TIMESTAMP_REFRESH: 5s - ROLLUP_STATE_DUMP_PATH: http://deployer:8081/state-dump.latest.json - # connecting to the DTL - ROLLUP_CLIENT_HTTP: http://dtl:7878 - ETH1_CTC_DEPLOYMENT_HEIGHT: 8 - RETRIES: 60 - ports: - - ${L2GETH_HTTP_PORT:-8545}:8545 - - ${L2GETH_WS_PORT:-8546}:8546 - - batch_submitter: - depends_on: - - l1_chain - - deployer - - l2geth - image: ethereumoptimism/batch-submitter-service:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - entrypoint: ./batch-submitter.sh - env_file: - - ./envs/batch-submitter.env - environment: - L1_ETH_RPC: http://l1_chain:8545 - L2_ETH_RPC: http://l2geth:8545 - URL: http://deployer:8081/addresses.json - BATCH_SUBMITTER_SEQUENCER_PRIVATE_KEY: '0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d' - BATCH_SUBMITTER_PROPOSER_PRIVATE_KEY: '0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a' - BATCH_SUBMITTER_SEQUENCER_BATCH_TYPE: ${BATCH_SUBMITTER_SEQUENCER_BATCH_TYPE:-zlib} - - verifier: - depends_on: - - l1_chain - - deployer - - dtl - image: ethereumoptimism/l2geth:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - deploy: - replicas: 0 - entrypoint: sh ./geth.sh - env_file: - - ./envs/geth.env - environment: - ETH1_HTTP: http://l1_chain:8545 - ROLLUP_STATE_DUMP_PATH: http://deployer:8081/state-dump.latest.json - ROLLUP_CLIENT_HTTP: http://dtl:7878 - ROLLUP_BACKEND: 'l1' - ROLLUP_VERIFIER_ENABLE: 'true' - ETH1_CTC_DEPLOYMENT_HEIGHT: 8 - RETRIES: 60 - ports: - - ${VERIFIER_HTTP_PORT:-8547}:8545 - - ${VERIFIER_WS_PORT:-8548}:8546 - - replica: - depends_on: - - dtl - image: ethereumoptimism/l2geth:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - deploy: - replicas: 0 - entrypoint: sh ./geth.sh - env_file: - - ./envs/geth.env - environment: - ETH1_HTTP: http://l1_chain:8545 - ROLLUP_STATE_DUMP_PATH: http://deployer:8081/state-dump.latest.json - ROLLUP_CLIENT_HTTP: http://dtl:7878 - ROLLUP_BACKEND: 'l2' - ROLLUP_VERIFIER_ENABLE: 'true' - ETH1_CTC_DEPLOYMENT_HEIGHT: 8 - RETRIES: 60 - ports: - - ${L2GETH_HTTP_PORT:-8549}:8545 - - ${L2GETH_WS_PORT:-8550}:8546 - - gas_oracle: - image: ethereumoptimism/gas-oracle:${DOCKER_TAG:-prerelease-0.5.0-rc-7-ee217ce} - deploy: - replicas: 0 - entrypoint: ./gas-oracle.sh - environment: - GAS_PRICE_ORACLE_ETHEREUM_HTTP_URL: http://l2geth:8545 - GAS_PRICE_ORACLE_PRIVATE_KEY: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" diff --git a/ops/docker-compose-rpc-proxy.yml b/ops/docker-compose-rpc-proxy.yml deleted file mode 100644 index f7d3037dccfb4..0000000000000 --- a/ops/docker-compose-rpc-proxy.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.4" -services: - rpc-proxy: - depends_on: - - l1_chain - - deployer - - l2geth - image: rpc-proxy - build: - context: .. - dockerfile: ./ops/docker/Dockerfile.rpc-proxy - environment: - SEQUENCER: l2geth:8545 - ETH_CALLS_ALLOWED: eth_blockNumber,eth_sendRawTransaction - ports: - - 9546:8080 - - 9145:9145 diff --git a/ops/docker-compose.yml b/ops/docker-compose.yml index 61a189ea4a213..a2c1519b61c65 100644 --- a/ops/docker-compose.yml +++ b/ops/docker-compose.yml @@ -13,7 +13,7 @@ x-system-addr-env: &system-addr-env services: # this is a helper service used because there's no official hardhat image l1_chain: - image: ethereumoptimism/hardhat:${DOCKER_TAG:-latest} + image: ethereumoptimism/hardhat:${DOCKER_TAG_HARDHAT:-latest} build: context: ./docker/hardhat dockerfile: Dockerfile @@ -30,6 +30,7 @@ services: context: .. dockerfile: ./ops/docker/Dockerfile.packages target: deployer + image: ethereumoptimism/deployer:${DOCKER_TAG_DEPLOYER:-latest} entrypoint: ./deployer.sh environment: # Env vars for the deployment script. @@ -72,6 +73,7 @@ services: context: .. dockerfile: ./ops/docker/Dockerfile.packages target: data-transport-layer + image: ethereumoptimism/data-transport-layer:${DOCKER_TAG_DATA_TRANSPORT_LAYER:-latest} # override with the dtl script and the env vars required for it entrypoint: ./dtl.sh env_file: @@ -96,6 +98,7 @@ services: build: context: .. dockerfile: ./ops/docker/Dockerfile.geth + image: ethereumoptimism/l2geth:${DOCKER_TAG_L2GETH:-latest} # override with the geth script and the env vars required for it entrypoint: sh ./geth.sh env_file: @@ -130,7 +133,8 @@ services: build: context: .. dockerfile: ./ops/docker/Dockerfile.packages - target: relayer + target: message-relayer + image: ethereumoptimism/message-relayer:${DOCKER_TAG_MESSAGE_RELAYER:-latest} entrypoint: ./relayer.sh environment: L1_NODE_WEB3_URL: http://l1_chain:8545 @@ -153,6 +157,7 @@ services: build: context: .. dockerfile: ./ops/docker/Dockerfile.geth + image: ethereumoptimism/l2geth:${DOCKER_TAG_L2GETH:-latest} entrypoint: sh ./geth.sh env_file: - ./envs/geth.env @@ -179,6 +184,7 @@ services: build: context: .. dockerfile: ./ops/docker/Dockerfile.geth + image: ethereumoptimism/l2geth:${DOCKER_TAG_L2GETH:-latest} entrypoint: sh ./geth.sh env_file: - ./envs/geth.env @@ -203,6 +209,7 @@ services: context: .. dockerfile: ./ops/docker/Dockerfile.packages target: integration-tests + image: ethereumoptimism/integration-tests:${DOCKER_TAG_INTEGRATION_TESTS:-latest} entrypoint: ./integration-tests.sh environment: L1_URL: http://l1_chain:8545 @@ -226,6 +233,7 @@ services: build: context: .. dockerfile: ./ops/docker/Dockerfile.gas-oracle + image: ethereumoptimism/l2geth:${DOCKER_TAG_GAS_ORACLE:-latest} entrypoint: ./gas-oracle.sh environment: GAS_PRICE_ORACLE_ETHEREUM_HTTP_URL: http://l2geth:8545 @@ -240,6 +248,7 @@ services: build: context: .. dockerfile: ./ops/docker/Dockerfile.batch-submitter-service + image: ethereumoptimism/batch-submitter-service:${DOCKER_TAG_BATCH_SUBMITTER_SERVICE:-latest} entrypoint: ./batch-submitter.sh env_file: - ./envs/batch-submitter.env diff --git a/ops/docker/Dockerfile.data-transport-layer b/ops/docker/Dockerfile.data-transport-layer deleted file mode 100644 index d83123f605151..0000000000000 --- a/ops/docker/Dockerfile.data-transport-layer +++ /dev/null @@ -1,35 +0,0 @@ -ARG LOCAL_REGISTRY=docker.io -ARG BUILDER_TAG=latest -FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder:${BUILDER_TAG} AS builder - -FROM node:16-alpine - -RUN apk add --no-cache curl bash jq - -WORKDIR /opt/optimism - -# copy top level files -COPY --from=builder /optimism/*.json ./ -COPY --from=builder /optimism/yarn.lock . -COPY --from=builder /optimism/node_modules ./node_modules - -# copy deps (would have been nice if docker followed the symlinks required) -COPY --from=builder /optimism/packages/core-utils/package.json ./packages/core-utils/package.json -COPY --from=builder /optimism/packages/core-utils/dist ./packages/core-utils/dist -COPY --from=builder /optimism/packages/common-ts/package.json ./packages/common-ts/package.json -COPY --from=builder /optimism/packages/common-ts/dist ./packages/common-ts/dist - -COPY --from=builder /optimism/packages/contracts/package.json ./packages/contracts/package.json -COPY --from=builder /optimism/packages/contracts/deployments ./packages/contracts/deployments -COPY --from=builder /optimism/packages/contracts/dist ./packages/contracts/dist -COPY --from=builder /optimism/packages/contracts/artifacts ./packages/contracts/artifacts - -# copy the service -WORKDIR /opt/optimism/packages/data-transport-layer -COPY --from=builder /optimism/packages/data-transport-layer/dist ./dist -COPY --from=builder /optimism/packages/data-transport-layer/package.json . -COPY --from=builder /optimism/packages/data-transport-layer/node_modules ./node_modules - -# copy this over in case you want to run alongside other services -COPY ./ops/scripts/dtl.sh . -ENTRYPOINT ["node", "dist/src/services/run.js"] diff --git a/ops/docker/Dockerfile.deployer b/ops/docker/Dockerfile.deployer deleted file mode 100644 index 86ac125d3c1d5..0000000000000 --- a/ops/docker/Dockerfile.deployer +++ /dev/null @@ -1,37 +0,0 @@ -ARG LOCAL_REGISTRY=docker.io -ARG BUILDER_TAG=latest -FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder:${BUILDER_TAG} AS builder - -FROM node:16-alpine - -RUN apk add --no-cache git curl python3 bash jq - -WORKDIR /opt/optimism/ - -COPY --from=builder /optimism/*.json /optimism/yarn.lock ./ -COPY --from=builder /optimism/node_modules ./node_modules - -# copy deps (would have been nice if docker followed the symlinks required) -COPY --from=builder /optimism/packages/core-utils/package.json ./packages/core-utils/package.json -COPY --from=builder /optimism/packages/core-utils/dist ./packages/core-utils/dist - -# get the needed built artifacts -WORKDIR /opt/optimism/packages/contracts -COPY --from=builder /optimism/packages/contracts/dist ./dist -COPY --from=builder /optimism/packages/contracts/*.json ./ -COPY --from=builder /optimism/packages/contracts/deployments ./deployments -COPY --from=builder /optimism/packages/contracts/node_modules ./node_modules -COPY --from=builder /optimism/packages/contracts/artifacts ./artifacts -COPY --from=builder /optimism/packages/contracts/src ./src - -# get non-build artifacts from the host -COPY packages/contracts/bin ./bin -COPY packages/contracts/contracts ./contracts -COPY packages/contracts/hardhat.config.ts ./ -COPY packages/contracts/deploy ./deploy -COPY packages/contracts/tasks ./tasks -COPY packages/contracts/test/helpers/constants.ts ./test/helpers/constants.ts -COPY packages/contracts/scripts ./scripts - -COPY ./ops/scripts/deployer.sh . -CMD ./ops/scripts/deployer.sh diff --git a/ops/docker/Dockerfile.integration-tests b/ops/docker/Dockerfile.integration-tests deleted file mode 100644 index cf98d55d2df6f..0000000000000 --- a/ops/docker/Dockerfile.integration-tests +++ /dev/null @@ -1,31 +0,0 @@ -ARG LOCAL_REGISTRY=docker.io -ARG BUILDER_TAG=latest -FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder:${BUILDER_TAG} AS builder - -FROM node:16-alpine - -RUN apk add --no-cache git curl python3 bash jq - -WORKDIR /opt/optimism/ - -COPY --from=builder /optimism/*.json /optimism/yarn.lock ./ -COPY --from=builder /optimism/node_modules ./node_modules - -# copy deps (would have been nice if docker followed the symlinks required) -COPY --from=builder /optimism/packages/sdk/package.json ./packages/sdk/package.json -COPY --from=builder /optimism/packages/sdk/dist ./packages/sdk/dist - -COPY --from=builder /optimism/packages/core-utils/package.json ./packages/core-utils/package.json -COPY --from=builder /optimism/packages/core-utils/dist ./packages/core-utils/dist - -COPY --from=builder /optimism/packages/message-relayer/package.json ./packages/message-relayer/package.json -COPY --from=builder /optimism/packages/message-relayer/dist ./packages/message-relayer/dist - -COPY --from=builder /optimism/packages/contracts ./packages/contracts - -# get the needed built artifacts -WORKDIR /opt/optimism/integration-tests -COPY --from=builder /optimism/integration-tests ./ - -COPY ./ops/scripts/integration-tests.sh ./ -CMD ["yarn", "test:integration"] diff --git a/ops/docker/Dockerfile.message-relayer b/ops/docker/Dockerfile.message-relayer deleted file mode 100644 index 3b9cf0569fb0c..0000000000000 --- a/ops/docker/Dockerfile.message-relayer +++ /dev/null @@ -1,35 +0,0 @@ -ARG LOCAL_REGISTRY=docker.io -ARG BUILDER_TAG=latest -FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder:${BUILDER_TAG} AS builder - -FROM node:16-alpine - -RUN apk add --no-cache curl bash jq - -WORKDIR /opt/optimism - -# copy top level files -COPY --from=builder /optimism/*.json ./ -COPY --from=builder /optimism/yarn.lock . -COPY --from=builder /optimism/node_modules ./node_modules - -# copy deps (would have been nice if docker followed the symlinks required) -COPY --from=builder /optimism/packages/core-utils/package.json ./packages/core-utils/package.json -COPY --from=builder /optimism/packages/core-utils/dist ./packages/core-utils/dist -COPY --from=builder /optimism/packages/common-ts/package.json ./packages/common-ts/package.json -COPY --from=builder /optimism/packages/common-ts/dist ./packages/common-ts/dist - -COPY --from=builder /optimism/packages/contracts/package.json ./packages/contracts/package.json -COPY --from=builder /optimism/packages/contracts/deployments ./packages/contracts/deployments -COPY --from=builder /optimism/packages/contracts/dist ./packages/contracts/dist -COPY --from=builder /optimism/packages/contracts/artifacts ./packages/contracts/artifacts - -# copy the service -WORKDIR /opt/optimism/packages/message-relayer -COPY --from=builder /optimism/packages/message-relayer/dist ./dist -COPY --from=builder /optimism/packages/message-relayer/package.json . -COPY --from=builder /optimism/packages/message-relayer/node_modules ./node_modules - -# copy this over in case you want to run alongside other services -COPY ./ops/scripts/relayer.sh . -ENTRYPOINT ["npm", "run", "start"] diff --git a/ops/docker/Dockerfile.monorepo b/ops/docker/Dockerfile.monorepo deleted file mode 100644 index e3b07a6a27df6..0000000000000 --- a/ops/docker/Dockerfile.monorepo +++ /dev/null @@ -1,68 +0,0 @@ -# This Dockerfile builds all the dependencies needed by the monorepo, and should -# be used to build any of the follow-on services -# -# ### BASE: Install deps -# We do not use Alpine because there's a regression causing it to be very slow -# when used with typescript/hardhat: https://github.com/nomiclabs/hardhat/issues/1219 -FROM node:16-buster-slim as node -RUN apt-get update -y && apt-get install -y git - -# Pre-download the compilers so that they do not need to be downloaded inside -# the image when building -FROM alpine as downloader -ARG VERSION=v0.8.9 -ARG SOLC_VERSION=${VERSION}+commit.e5eed63a -ARG SOLC_UPSTREAM=https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-${SOLC_VERSION} - -ADD $SOLC_UPSTREAM ./solc -ADD https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.5.17+commit.d19bba13 ./solc - -FROM node as builder -# copy over the needed configs to run the dep installation -# note: this approach can be a bit unhandy to maintain, but it allows -# us to cache the installation steps -WORKDIR /optimism -COPY .git ./.git -COPY *.json yarn.lock ./ -COPY packages/sdk/package.json ./packages/sdk/package.json -COPY packages/core-utils/package.json ./packages/core-utils/package.json -COPY packages/common-ts/package.json ./packages/common-ts/package.json -COPY packages/contracts/package.json ./packages/contracts/package.json -COPY packages/data-transport-layer/package.json ./packages/data-transport-layer/package.json -COPY packages/message-relayer/package.json ./packages/message-relayer/package.json -COPY packages/replica-healthcheck/package.json ./packages/replica-healthcheck/package.json -COPY integration-tests/package.json ./integration-tests/package.json - -RUN yarn install --frozen-lockfile - -### BUILDER: Builds the typescript -FROM node:16 - -WORKDIR /optimism - -# cache the node_modules copying step since it's expensive -# we run this before copying over any source files to avoid re-copying anytime the -# code changes -COPY --from=builder /optimism/node_modules ./node_modules -COPY --from=builder /optimism/packages ./packages -COPY --from=builder /optimism/integration-tests ./integration-tests -COPY --from=builder /optimism/.git ./.git - -# the following steps are cheap -COPY *.json yarn.lock ./ -# copy over the source -COPY ./packages ./packages -COPY ./integration-tests ./integration-tests -# copy over solc to save time building (35+ seconds vs not doing this step) -COPY --from=downloader solc /root/.cache/hardhat-nodejs/compilers/linux-amd64/solc-linux-amd64-${SOLC_VERSION} -COPY --from=downloader solc /root/.cache/hardhat-nodejs/compilers/linux-amd64/solc-linux-amd64-v0.5.17+commit.d19bba13 - -# build it! -RUN yarn build -# build integration tests' contracts -RUN yarn workspace @eth-optimism/integration-tests build - -# TODO: Consider thinning up the container by trimming non-production -# dependencies -# so that it can be used in docker-compose -CMD ["true"] diff --git a/ops/docker/Dockerfile.packages b/ops/docker/Dockerfile.packages index e06cb37949b07..2764f478e18ac 100644 --- a/ops/docker/Dockerfile.packages +++ b/ops/docker/Dockerfile.packages @@ -49,7 +49,12 @@ COPY ./ops/scripts/integration-tests.sh ./ CMD ["yarn", "test:integration"] -FROM base as relayer +FROM base as message-relayer WORKDIR /opt/optimism/packages/message-relayer COPY ./ops/scripts/relayer.sh . CMD ["npm", "run", "start"] + + +FROM base as replica-healthcheck +WORKDIR /opts/optimism/packages/replica-healthcheck +ENTRYPOINT ["node", "dist/exec/run-healthcheck-server.js"] diff --git a/ops/docker/Dockerfile.replica-healthcheck b/ops/docker/Dockerfile.replica-healthcheck deleted file mode 100644 index c60ea84463da6..0000000000000 --- a/ops/docker/Dockerfile.replica-healthcheck +++ /dev/null @@ -1,33 +0,0 @@ -ARG LOCAL_REGISTRY=docker.io -ARG BUILDER_TAG=latest -FROM ${LOCAL_REGISTRY}/ethereumoptimism/builder:${BUILDER_TAG} AS builder - -FROM node:16-alpine - -WORKDIR /opt/optimism - -# copy top level files -COPY --from=builder /optimism/*.json ./ -COPY --from=builder /optimism/yarn.lock . -COPY --from=builder /optimism/node_modules ./node_modules - -# copy deps (would have been nice if docker followed the symlinks required) -COPY --from=builder /optimism/packages/sdk/package.json ./packages/sdk/package.json -COPY --from=builder /optimism/packages/sdk/dist ./packages/sdk/dist -COPY --from=builder /optimism/packages/core-utils/package.json ./packages/core-utils/package.json -COPY --from=builder /optimism/packages/core-utils/dist ./packages/core-utils/dist -COPY --from=builder /optimism/packages/common-ts/package.json ./packages/common-ts/package.json -COPY --from=builder /optimism/packages/common-ts/dist ./packages/common-ts/dist - -COPY --from=builder /optimism/packages/contracts/package.json ./packages/contracts/package.json -COPY --from=builder /optimism/packages/contracts/deployments ./packages/contracts/deployments -COPY --from=builder /optimism/packages/contracts/dist ./packages/contracts/dist -COPY --from=builder /optimism/packages/contracts/artifacts ./packages/contracts/artifacts - -# copy the service -WORKDIR /opt/optimism/packages/replica-healthcheck -COPY --from=builder /optimism/packages/replica-healthcheck/dist ./dist -COPY --from=builder /optimism/packages/replica-healthcheck/package.json . -COPY --from=builder /optimism/packages/replica-healthcheck/node_modules ./node_modules - -ENTRYPOINT ["node", "dist/exec/run-healthcheck-server.js"] diff --git a/ops/docker/Dockerfile.rpc-proxy b/ops/docker/Dockerfile.rpc-proxy deleted file mode 100644 index 319538036a7de..0000000000000 --- a/ops/docker/Dockerfile.rpc-proxy +++ /dev/null @@ -1,21 +0,0 @@ -FROM openresty/openresty:buster -LABEL maintainer="Optimistic Systems " -ARG GOTEMPLATE_VERSION=v3.9.0 - -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - openresty-opm \ - && opm get knyar/nginx-lua-prometheus - -RUN curl -o /usr/local/bin/gomplate \ - -sSL https://github.com/hairyhenderson/gomplate/releases/download/$GOTEMPLATE_VERSION/gomplate_linux-amd64-slim \ - && chmod +x /usr/local/bin/gomplate - -RUN mkdir -p /var/log/nginx/ \ - && ln -sf /dev/stdout /var/log/nginx/access.log \ - && ln -sf /dev/stderr /var/log/nginx/error.log - -COPY ./ops/docker/rpc-proxy/eth-jsonrpc-access.lua /usr/local/openresty/nginx/eth-jsonrpc-access.lua -COPY ./ops/docker/rpc-proxy/nginx.template.conf /docker-entrypoint.d/nginx.template.conf -COPY ./ops/docker/rpc-proxy/docker-entrypoint.sh /docker-entrypoint.sh -ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/ops/docker/builder/CHANGELOG.md b/ops/docker/builder/CHANGELOG.md deleted file mode 100644 index 43ae376424927..0000000000000 --- a/ops/docker/builder/CHANGELOG.md +++ /dev/null @@ -1,18 +0,0 @@ -# @eth-optimism/builder - -## 0.1.1 - -### Patch Changes - -- 0ab37fc9: Update to node.js version 16 - -## 0.1.0 - -### Minor Changes - -- 81ccd6e4: `regenesis/0.5.0` release - -### Patch Changes - -- 222a3eef: Add 'User-Agent' to the http headers for ethers providers -- 391dbf8c: Create builder release diff --git a/ops/docker/builder/package.json b/ops/docker/builder/package.json deleted file mode 100644 index eab6fb2daa4ed..0000000000000 --- a/ops/docker/builder/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "@eth-optimism/builder", - "version": "0.1.1", - "license": "MIT" -} diff --git a/ops/docker/rpc-proxy/CHANGELOG.md b/ops/docker/rpc-proxy/CHANGELOG.md deleted file mode 100644 index ca451967b0b93..0000000000000 --- a/ops/docker/rpc-proxy/CHANGELOG.md +++ /dev/null @@ -1,19 +0,0 @@ -# @eth-optimism/rpc-proxy - -## 0.0.4 - -### Patch Changes - -- b9d2fbee: Trigger releases - -## 0.0.3 - -### Patch Changes - -- 893623c9: Trigger patch releases for dockerhub - -## 0.0.2 - -### Patch Changes - -- f7c78498: Initial rpc-proxy package diff --git a/ops/docker/rpc-proxy/docker-entrypoint.sh b/ops/docker/rpc-proxy/docker-entrypoint.sh deleted file mode 100755 index 477f1dd8f934a..0000000000000 --- a/ops/docker/rpc-proxy/docker-entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -if [ -z "$SEQUENCER" ];then - echo "SEQUENCER env must be set, exiting" - exit 1 -fi - -if [ -z "$ETH_CALLS_ALLOWED" ];then - echo "ETH_CALLS_ALLOWED env must be set, exiting" - exit 1 -fi - -gomplate -f /docker-entrypoint.d/nginx.template.conf > /usr/local/openresty/nginx/conf/nginx.conf - -cat /usr/local/openresty/nginx/conf/nginx.conf - -exec openresty "$@" diff --git a/ops/docker/rpc-proxy/eth-jsonrpc-access.lua b/ops/docker/rpc-proxy/eth-jsonrpc-access.lua deleted file mode 100644 index 3f2280ec815f4..0000000000000 --- a/ops/docker/rpc-proxy/eth-jsonrpc-access.lua +++ /dev/null @@ -1,91 +0,0 @@ --- Source: https://github.com/adetante/ethereum-nginx-proxy -local cjson = require('cjson') - -local function empty(s) - return s == nil or s == '' -end - -local function split(s) - local res = {} - local i = 1 - for v in string.gmatch(s, "([^,]+)") do - res[i] = v - i = i + 1 - end - return res -end - -local function contains(arr, val) - for i, v in ipairs (arr) do - if v == val then - return true - end - end - return false -end - --- parse conf -local blacklist, whitelist = nil -if not empty(ngx.var.jsonrpc_blacklist) then - blacklist = split(ngx.var.jsonrpc_blacklist) -end -if not empty(ngx.var.jsonrpc_whitelist) then - whitelist = split(ngx.var.jsonrpc_whitelist) -end - --- check conf -if blacklist ~= nil and whitelist ~= nil then - ngx.log(ngx.ERR, 'invalid conf: jsonrpc_blacklist and jsonrpc_whitelist are both set') - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) - return -end - --- get request content -ngx.req.read_body() - --- try to parse the body as JSON -local success, body = pcall(cjson.decode, ngx.var.request_body); -if not success then - ngx.log(ngx.ERR, 'invalid JSON request') - ngx.exit(ngx.HTTP_BAD_REQUEST) - return -end - -local method = body['method'] -local version = body['jsonrpc'] - --- check we have a method and a version -if empty(method) or empty(version) then - ngx.log(ngx.ERR, 'no method and/or jsonrpc attribute') - ngx.exit(ngx.HTTP_BAD_REQUEST) - return -end - -metric_sequencer_requests:inc(1, {method, ngx.var.server_name, ngx.var.status}) - --- check the version is supported -if version ~= "2.0" then - ngx.log(ngx.ERR, 'jsonrpc version not supported: ' .. version) - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) - return -end - --- if whitelist is configured, check that the method is whitelisted -if whitelist ~= nil then - if not contains(whitelist, method) then - ngx.log(ngx.ERR, 'jsonrpc method is not whitelisted: ' .. method) - ngx.exit(ngx.HTTP_FORBIDDEN) - return - end -end - --- if blacklist is configured, check that the method is not blacklisted -if blacklist ~= nil then - if contains(blacklist, method) then - ngx.log(ngx.ERR, 'jsonrpc method is blacklisted: ' .. method) - ngx.exit(ngx.HTTP_FORBIDDEN) - return - end -end - -return diff --git a/ops/docker/rpc-proxy/nginx.template.conf b/ops/docker/rpc-proxy/nginx.template.conf deleted file mode 100644 index f1de13edd85fa..0000000000000 --- a/ops/docker/rpc-proxy/nginx.template.conf +++ /dev/null @@ -1,91 +0,0 @@ -worker_processes 5; -daemon off; -error_log /var/log/nginx/error.log; -worker_rlimit_nofile 8192; -pcre_jit on; - -events { - worker_connections 4096; -} - -http { - include mime.types; - index index.html; - # The JSONRPC POST body must fit inside this allocation for the method parsing to succeed. - # https://github.com/openresty/lua-nginx-module#ngxreqread_body - # http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size - client_body_buffer_size 128k; - # client_max_body_size should match client_body_buffer_size - # Values that exceed client_body_buffer_size will be written to a temporary file, which we don't want - # Requests above this limit will also be denied with an HTTP 413 response (entity too large) - # http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size - client_max_body_size 128k; - - # See Move default writable paths to a dedicated directory (#119) - # https://github.com/openresty/docker-openresty/issues/119 - client_body_temp_path /var/run/openresty/nginx-client-body; - proxy_temp_path /var/run/openresty/nginx-proxy; - fastcgi_temp_path /var/run/openresty/nginx-fastcgi; - uwsgi_temp_path /var/run/openresty/nginx-uwsgi; - scgi_temp_path /var/run/openresty/nginx-scgi; - - keepalive_timeout 0; - - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] $status ' - '"$request" $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - tcp_nopush on; - - lua_shared_dict prometheus_metrics 10M; - init_worker_by_lua_block { - prometheus = require("prometheus").init("prometheus_metrics") - metric_requests = prometheus:counter( - "nginx_http_requests_total", "Number of HTTP requests", {"host", "status"}) - metric_sequencer_requests = prometheus:counter( - "nginx_eth_sequencer_requests", "Number of requests going to the sequencer", {"method", "host", "status"}) - metric_replica_requests = prometheus:counter( - "nginx_eth_replica_requests", "Number of requests going to the replicas", {"host", "status"}) - metric_latency = prometheus:histogram( - "nginx_http_request_duration_seconds", "HTTP request latency", {"host"}) - metric_connections = prometheus:gauge( - "nginx_http_connections", "Number of HTTP connections", {"state"}) - } - log_by_lua_block { - metric_requests:inc(1, {ngx.var.server_name, ngx.var.status}) - metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name}) - } - - upstream sequencer { - server {{env.Getenv "SEQUENCER"}}; - } - - server { # RPC proxy server - listen 8080; - location = /healthz { - return 200 'healthz'; - } - location / { - set $jsonrpc_whitelist {{env.Getenv "ETH_CALLS_ALLOWED"}}; - if ($request_method = POST) { - access_by_lua_file 'eth-jsonrpc-access.lua'; - } - proxy_pass http://sequencer; - } - } - - server { # Metrics server - listen 9145; - location /metrics { - content_by_lua_block { - metric_connections:set(ngx.var.connections_reading, {"reading"}) - metric_connections:set(ngx.var.connections_waiting, {"waiting"}) - metric_connections:set(ngx.var.connections_writing, {"writing"}) - prometheus:collect() - } - } - } - -} \ No newline at end of file diff --git a/ops/docker/rpc-proxy/package.json b/ops/docker/rpc-proxy/package.json deleted file mode 100644 index db2ed8e75f521..0000000000000 --- a/ops/docker/rpc-proxy/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "@eth-optimism/rpc-proxy", - "version": "0.0.4", - "private": true, - "devDependencies": {} -} diff --git a/ops/scripts/build-ci.sh b/ops/scripts/build-ci.sh index 682f29450730b..7380fc42d4256 100755 --- a/ops/scripts/build-ci.sh +++ b/ops/scripts/build-ci.sh @@ -7,7 +7,6 @@ function build() { echo "Context: $4" docker buildx build \ --tag "$2" \ - --build-arg LOCAL_REGISTRY=localhost:5000 \ --cache-from "type=local,src=/tmp/.buildx-cache/$1" \ --cache-to="type=local,dest=/tmp/.buildx-cache-new/$1" \ --file "$3" \ @@ -15,32 +14,12 @@ function build() { & } -# Split across two build stages: -# -# 1. Build the builder and everything that doesn't depend on it, then -# 2. Build everything else. -# -# Each individual build is executed in parallel, so we use wait block all builds -# in each stage are complete. - mkdir -p /tmp/.buildx-cache-new -docker buildx build --tag "localhost:5000/ethereumoptimism/builder:latest" --cache-from "type=local,src=/tmp/.buildx-cache/builder" --cache-to="type=local,mode=max,dest=/tmp/.buildx-cache-new/builder" --file "./ops/docker/Dockerfile.monorepo" --push . & build l2geth "ethereumoptimism/l2geth:latest" "./ops/docker/Dockerfile.geth" . build l1chain "ethereumoptimism/hardhat:latest" "./ops/docker/hardhat/Dockerfile" ./ops/docker/hardhat wait -# BuildX builds everything in a container when docker-container is selected as -# the backend. Unfortunately, this means that the built image must be pushed -# then re-pulled in order to make the container accessible to the Docker daemon. -# We have to use the docker-container backend since the the docker backend does -# not support cache-from and cache-to. -docker pull localhost:5000/ethereumoptimism/builder:latest - -# Re-tag the local registry version of the builder so that docker-compose and -# friends can see it. -docker tag localhost:5000/ethereumoptimism/builder:latest ethereumoptimism/builder:latest - build deployer "ethereumoptimism/deployer:latest" "./ops/docker/Dockerfile.deployer" . build dtl "ethereumoptimism/data-transport-layer:latest" "./ops/docker/Dockerfile.data-transport-layer" . build relayer "ethereumoptimism/message-relayer:latest" "./ops/docker/Dockerfile.message-relayer" . diff --git a/ops/scripts/ci-versions.js b/ops/scripts/ci-versions.js index bac2051d8a8ee..e0a565a2348e8 100644 --- a/ops/scripts/ci-versions.js +++ b/ops/scripts/ci-versions.js @@ -6,27 +6,8 @@ const os = require('os') data = process.argv[2] data = JSON.parse(data) -// Packages that do not depend on the builder. -// There are more packages that depend on the -// builder than not, so keep track of this list instead -const nonBuilders = new Set([ - 'l2geth', - 'gas-oracle', - 'proxyd', - 'rpc-proxy', -]) - -builder = false for (const i of data) { const name = i.name.replace("@eth-optimism/", "") - if (!nonBuilders.has(name)) { - builder = true - } const version = i.version process.stdout.write(`::set-output name=${name}::${version}` + os.EOL) } - -if (builder) { - process.stdout.write(`::set-output name=use_builder::true` + os.EOL) -} -