Combine Cel2 migration scripts

.github/workflows/docker-build-scan.yaml

@@ -1,92 +1,89 @@
 name: Docker Build Scan
 on:
+  pull_request:
+    branches:
+      - 'master'
+      - 'celo*'
   workflow_dispatch:
 
 jobs:
-  Build-Scan-Container-op-ufm:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-ufm/Dockerfile
-
-  Build-Scan-Container-ops-bedrock-l1:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: ops-bedrock/Dockerfile.l1
-      context: ops-bedrock
-
-  Build-Scan-Container-ops-bedrock-l2:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: ops-bedrock/Dockerfile.l2
-      context: ops-bedrock
-
-  Build-Scan-Container-indexer:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: indexer/Dockerfile
-
-  Build-Scan-Container-op-heartbeat:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-heartbeat/Dockerfile
-
-  Build-Scan-Container-op-exporter:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-exporter/Dockerfile
-
-  Build-Scan-Container-op-program:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-program/Dockerfile
-
-  Build-Scan-Container-ops-bedrock:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: ops-bedrock/Dockerfile.stateviz
-
-  Build-Scan-Container-ci-builder:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: ops/docker/ci-builder/Dockerfile
-
-  Build-Scan-Container-proxyd:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: proxyd/Dockerfile
-
-  Build-Scan-Container-op-node:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-node/Dockerfile
-
-  Build-Scan-Container-op-batcher:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-batcher/Dockerfile
-
-  Build-Scan-Container-indexer-ui:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: indexer/ui/Dockerfile
-
-  Build-Scan-Container-op-proposer:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-proposer/Dockerfile
-
-  Build-Scan-Container-op-challenger:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-challenger/Dockerfile
-
-  Build-Scan-Container-endpoint-monitor:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: endpoint-monitor/Dockerfile
-
-  Build-Scan-Container-opwheel:
-    uses: celo-org/reusable-workflows/.github/workflows/[email protected]
-    with:
-      dockerfile: op-wheel/Dockerfile
-
+  detect-files-changed:
+    runs-on: ubuntu-latest
+    outputs:
+      files-changed: ${{ steps.detect-files-changed.outputs.all_changed_files }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Detect files changed
+        id: detect-files-changed
+        uses: tj-actions/changed-files@v44
+        with:
+          separator: ','
+
+  build-cel2-migration-tool:
+    runs-on: ubuntu-latest
+    needs: detect-files-changed
+    if: |
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/cmd/celo-migrate') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-chain-ops/Dockerfile')
+    permissions:
+      contents: read
+      id-token: write
+      security-events: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login at GCP Artifact Registry
+        uses: celo-org/reusable-workflows/.github/actions/[email protected]
+        with:
+          workload-id-provider: 'projects/1094498259535/locations/global/workloadIdentityPools/gh-optimism/providers/github-by-repos'
+          service-account: '[email protected]'
+          docker-gcp-registries: us-west1-docker.pkg.dev
+      - name: Build and push container
+        uses: celo-org/reusable-workflows/.github/actions/[email protected]
+        with:
+          platforms: linux/amd64
+          registry: us-west1-docker.pkg.dev/devopsre/dev-images/cel2-migration-tool
+          tags: ${{ github.sha }}
+          context: ./
+          dockerfile: ./op-chain-ops/Dockerfile
+          push: true
+          trivy: false
+
+  # Build op-node op-batcher op-proposer using docker-bake
+  build-op-stack:
+    runs-on: ubuntu-latest
+    needs: detect-files-changed
+    if: |
+      contains(needs.detect-files-changed.outputs.files-changed, 'go.sum') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'ops/docker') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-node/') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-batcher/') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-proposer/') ||
+      contains(needs.detect-files-changed.outputs.files-changed, 'op-service/')
+    permissions:
+      contents: read
+      id-token: write
+      security-events: write
+    env:
+      GIT_COMMIT: ${{ github.sha }}
+      GIT_DATE: ${{ github.event.head_commit.timestamp }}
+      IMAGE_TAGS: ${{ github.sha }},latest
+      REGISTRY: us-west1-docker.pkg.dev
+      REPOSITORY: blockchaintestsglobaltestnet/dev-images
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login at GCP Artifact Registry
+        uses: celo-org/reusable-workflows/.github/actions/[email protected]
+        with:
+          workload-id-provider: 'projects/1094498259535/locations/global/workloadIdentityPools/gh-optimism/providers/github-by-repos'
+          service-account: '[email protected]'
+          docker-gcp-registries: us-west1-docker.pkg.dev
+      # We need a custom steps as it's using docker bake
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push
+        uses: docker/bake-action@v5
+        with:
+          push: true
+          source: .
+          files: docker-bake.hcl
+          targets: op-node,op-batcher,op-proposer

go.mod

@@ -226,7 +226,7 @@ require (
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
-replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240612131021-96a2a76ceaaa
+replace github.com/ethereum/go-ethereum => github.com/celo-org/op-geth v0.0.0-20240624092059-26b8a1d8617d
 
 //replace github.com/ethereum/go-ethereum v1.13.9 => ../op-geth
 

go.sum

@@ -77,8 +77,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/celo-org/op-geth v0.0.0-20240612131021-96a2a76ceaaa h1:a9OVs0F3EjWwePKDuDUPr0QWe/CWi7uwElH9JzUxq/8=
-github.com/celo-org/op-geth v0.0.0-20240612131021-96a2a76ceaaa/go.mod h1:vObZmT4rKd8hjSblIktsJHtLX8SXbCoaIXEd42HMDB0=
+github.com/celo-org/op-geth v0.0.0-20240624092059-26b8a1d8617d h1:Qs1Sy1TVhOGWLEMj3i+5Tg30gPEtgrCkNcIg8vvfBz4=
+github.com/celo-org/op-geth v0.0.0-20240624092059-26b8a1d8617d/go.mod h1:vObZmT4rKd8hjSblIktsJHtLX8SXbCoaIXEd42HMDB0=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

op-chain-ops/Dockerfile

@@ -0,0 +1,29 @@
+FROM golang:1.21.1-alpine3.18 as builder
+
+RUN apk --no-cache add make
+
+COPY ./go.mod /app/go.mod
+COPY ./go.sum /app/go.sum
+
+WORKDIR /app
+
+RUN go mod download
+
+COPY ./op-service /app/op-service
+COPY ./op-node /app/op-node
+COPY ./op-plasma /app/op-plasma
+COPY ./op-chain-ops /app/op-chain-ops
+WORKDIR /app/op-chain-ops
+RUN make celo-migrate
+
+FROM alpine:3.18
+RUN apk --no-cache add ca-certificates bash rsync
+
+# RUN addgroup -S app && adduser -S app -G app
+# USER app
+WORKDIR /app
+
+COPY --from=builder /app/op-chain-ops/bin/celo-migrate /app
+ENV PATH="/app:${PATH}"
+
+ENTRYPOINT ["/app/celo-migrate"]

op-chain-ops/cmd/celo-migrate/README.md

@@ -0,0 +1,117 @@
+# Celo L2 Migration Script
+
+## Overview
+
+This script has two main sections. The first migrates Celo blocks to a format compatible with `op-geth`, and the second performs necessary state changes such as deploying L2 smart contracts.
+
+### Block migration
+
+The block migration itself has two parts: It first migrates the ancient / frozen blocks, which is all blocks before the last 90000. Because the ancients db is append-only, it copies these blocks into a new database after making the necessary transformations. The script then copies the rest of the chaindata directory (excluding `/ancients`) using the system level `rsync` command. All non-ancient blocks are then transformed in-place in the new db, leaving the old db unchanged.
+
+### State migration
+
+After all blocks have been migrated, the script performs a series of modifications to the state db. This is also done in-place in the `--new-db` directory. First, the state migration deploys the L2 smart contracts by iterating through the genesis allocs passed to the script and setting the nonce, balance, code and storage for each address accordingly, overwritting existing data if necessary. Finally, the state migration will commit the state changes to produce a new state root and create the first Cel2 block.
+
+### Notes
+
+Once the state changes are complete the migration is finished. The longest running section of the script is the ancients migration, and it can be resumed / skipped if interupted part way. The rest of the script cannot be resumed and will restart from the last migrated ancient block if interupted or re-run.
+
+The script outputs a `rollup-config.json` file that is passed to the sequencer in order to start the L2 network.
+
+See `--help` for how to run each portion of the script individually, along with other configuration options.
+
+### Running the script
+
+First, build the script by running
+
+```bash
+make celo-migrate
+```
+
+from the `op-chain-ops` directory.
+
+You can then run the script as follows.
+
+```bash
+go run ./cmd/celo-migrate --help
+```
+
+NOTE: You will need `rsync` to run this script if it's not already installed
+
+#### Running with local test setup (Alfajores / Holesky)
+
+To test the script locally, we can migrate an alfajores database and use Holesky as our L1. The input files needed for this can be found in `./testdata`. The necessary smart contracts have already been deployed on Holesky.
+
+##### Pull down the latest alfajores database snapshot
+
+```bash
+gcloud alpha storage cp gs://celo-chain-backup/alfajores/chaindata-latest.tar.zst alfajores.tar.zst
+```
+
+Unzip and rename
+
+```bash
+tar --use-compress-program=unzstd -xvf alfajores.tar.zst
+mv chaindata ./data/alfajores_old
+```
+
+##### Generate test allocs file
+
+The state migration takes in a allocs file that specifies the l2 state changes to be made during the migration. This file can be generated from the deploy config and l1 contract addresses by running the following from the `contracts-bedrock` directory.
+
+```bash
+CONTRACT_ADDRESSES_PATH=../../op-chain-ops/cmd/celo-migrate/testdata/deployment-l1-holesky.json \
+DEPLOY_CONFIG_PATH=../../op-chain-ops/cmd/celo-migrate/testdata/deploy-config-holesky-alfajores.json \
+STATE_DUMP_PATH=../../op-chain-ops/cmd/celo-migrate/testdata/l2-allocs-alfajores.json \
+forge script ./scripts/L2Genesis.s.sol:L2Genesis \
+--sig 'runWithStateDump()'
+```
+
+This should output the allocs file to `./testdata/l2-allocs-alfajores.json`. If you encounter difficulties with this and want to just continue testing the script, you can alternatively find the allocs file [here](https://gist.github.com/jcortejoso/7f90ba9b67c669791014661ccb6de81a).
+
+##### Run script with test configuration
+
+```bash
+go run ./cmd/celo-migrate full \
+--deploy-config ./cmd/celo-migrate/testdata/deploy-config-holesky-alfajores.json \
+--l1-deployments ./cmd/celo-migrate/testdata/deployment-l1-holesky.json \
+--l1-rpc https://ethereum-holesky-rpc.publicnode.com  \
+--l2-allocs ./cmd/celo-migrate/testdata/l2-allocs-alfajores.json \
+--outfile.rollup-config ./cmd/celo-migrate/testdata/rollup-config.json \
+--old-db ./data/alfajores_old \
+--new-db ./data/alfajores_new
+```
+
+The first time you run the script it should take ~5 minutes. The first part of the script will migrate ancient blocks, and will take the majority of the time.
+
+During the ancients migration you can play around with stopping and re-running the script, which should always resume where it left off. If you run the script subsequent times after ancient migrations have been run, the script should skip ancient migrations and proceed to migrating non-ancient blocks quickly.
+
+Note that partial migration progress beyond the ancient blocks (i.e. non-frozen blocks and state changes) will not be preserved between runs by default.
+
+#### Running for Cel2 migration
+
+##### Generate allocs file
+
+You can generate the allocs file needed to run the migration with the following script in `contracts-bedrock`
+
+```bash
+CONTRACT_ADDRESSES_PATH=<PATH_TO_CONTRACT_ADDRESSES> \
+DEPLOY_CONFIG_PATH=<PATH_TO_MY_DEPLOY_CONFIG> \
+STATE_DUMP_PATH=<PATH_TO_WRITE_L2_ALLOCS> \
+forge script scripts/L2Genesis.s.sol:L2Genesis \
+--sig 'runWithStateDump()'
+```
+
+##### Dress rehearsal / pre-migration
+
+To minimize downtime caused by the migration, node operators can prepare their Cel2 databases by running this script a day ahead of the actual migration. This will pre-populate the new database with most of the ancient blocks needed for the final migration, and will also serve as a dress rehearsal for the rest of the migration.
+
+NOTE: The pre-migration should be run using a chaindata snapshot, rather than a db that is being used by a node. To avoid network downtime, we recommend that node operators do not stop any nodes in order to perform the pre-migration.
+
+Node operators should inspect their migration logs after the dress rehearsal to ensure the migration completed succesfully and direct any questions to the Celo developer community on Discord before the actual migration.
+
+##### Final migration
+
+On the day of the actual cel2 migration, this script can be re-run using the same parameters as for the dress rehearsal but with the latest Celo Mainnet database snapshot as `--old-db`. The script will only need to migrate any ancient blocks frozen after the dress rehearsal, all non-frozen blocks, and state.
+
+Unlike the pre-migration, the final migration can be run directly on the db used by the Celo node rather than a snapshot.