diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index aabb4c2a505ee..ba57aed438e3c 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -941,7 +941,7 @@ jobs: - run: name: Cannon Go lint command: | - make lint + just lint working_directory: cannon - run: name: Cannon Go 64-bit tests @@ -1804,7 +1804,7 @@ jobs: - run: name: run Go linter command: | - make lint-go + just lint-go - save_cache: key: golangci-v1-{{ checksum ".golangci.yaml" }} paths: @@ -1822,7 +1822,7 @@ jobs: namespace: sysgo-go-binaries - run: name: Build Go binaries for sysgo - command: make cannon op-program + command: just cannon op-program - go-save-cache: namespace: sysgo-go-binaries - persist_to_workspace: @@ -1843,7 +1843,7 @@ jobs: - run: name: check op-geth version command: | - make check-op-geth-version + just check-op-geth-version check-nut-locks: docker: @@ -1902,7 +1902,7 @@ jobs: - restore_cache: key: go-tests-v2-{{ checksum "go.mod" }} - run: - name: Run Go tests via Makefile + name: Run Go tests no_output_timeout: <> command: | <> @@ -1910,7 +1910,7 @@ jobs: # set to less than number CPUs (xlarge Docker is 16 CPU) so there's some buffer for things # like Geth export PARALLEL=12 - make <> + just <> - save_cache: key: go-tests-v2-{{ checksum "go.mod" }} paths: @@ -1977,22 +1977,22 @@ jobs: at: . - run: name: build op-program-client - command: make op-program-client + command: just op-program-client working_directory: op-program - run: name: build op-program-host - command: make op-program-host + command: just op-program-host working_directory: op-program - run: name: build cannon - command: make cannon + command: just cannon - run: name: run tests no_output_timeout: <> command: | <> export TEST_TIMEOUT=<> - make go-tests-fraud-proofs-ci + just go-tests-fraud-proofs-ci - codecov/upload: disable_search: true files: ./coverage.out @@ -2365,13 +2365,13 @@ jobs: sudo apt-get install -y binutils-mips-linux-gnu - run: name: Build cannon - command: make cannon + command: just cannon - run: name: Build op-program - command: make op-program + command: just op-program - run: name: Sanitize op-program client - command: make sanitize-program GUEST_PROGRAM=../op-program/bin/op-program-client64.elf + command: GUEST_PROGRAM=../op-program/bin/op-program-client64.elf just sanitize-program working_directory: cannon cannon-prestate: @@ -2383,7 +2383,7 @@ jobs: enable-mise-cache: true - run: name: Build prestates - command: make -j reproducible-prestate + command: just reproducible-prestate - persist_to_workspace: root: . paths: @@ -2485,7 +2485,8 @@ jobs: enable-mise-cache: true - run: name: Verify reproducibility - command: make -C op-program verify-reproducibility + command: just verify-reproducibility + working_directory: op-program - store_artifacts: path: ./op-program/temp/logs when: always @@ -2502,10 +2503,11 @@ jobs: - setup_remote_docker - run: name: Build cannon - command: make cannon + command: just cannon - run: name: Verify the Cannon STF - command: make -C ./cannon cannon-stf-verify + command: just cannon-stf-verify + working_directory: cannon - notify-failures-on-develop: mentions: "@proofs-team" @@ -2572,7 +2574,7 @@ jobs: - run: name: Run Analyzer command: | - make run-vm-compat + just run-vm-compat working_directory: op-program - store_artifacts: path: op-program/bin/vm-compat-output/vm-compat-findings.json @@ -2590,7 +2592,7 @@ jobs: - run: name: Verify Compatibility command: | - make verify-compat + just verify-compat working_directory: op-program check-generated-mocks-op-node: @@ -2605,7 +2607,7 @@ jobs: patterns: op-node - run: name: check-generated-mocks - command: make generate-mocks-op-node && git diff --exit-code + command: just generate-mocks-op-node && git diff --exit-code check-generated-mocks-op-service: docker: @@ -2619,7 +2621,7 @@ jobs: patterns: op-service - run: name: check-generated-mocks - command: make generate-mocks-op-service && git diff --exit-code + command: just generate-mocks-op-service && git diff --exit-code op-deployer-forge-version: docker: diff --git a/.circleci/continue/rust-ci.yml b/.circleci/continue/rust-ci.yml index 2db59a0c8b7d7..6e75f363d7425 100644 --- a/.circleci/continue/rust-ci.yml +++ b/.circleci/continue/rust-ci.yml @@ -858,7 +858,7 @@ jobs: - run: name: Build cannon command: | - cd cannon && make + cd cannon && just cannon sudo mv ./bin/cannon /usr/local/bin/ - run: name: Set run environment diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e851085a56e44..5ecd50abecf22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,7 +108,7 @@ You must install all of the required [Software Dependencies](#software-dependenc Optimism Monorepo. Once you've done so, run the following command to build: ```bash -make build +just build ``` Packages built on one branch may not be compatible with packages on a different branch. diff --git a/Makefile b/Makefile index ea34470a152e5..e71824901670b 100644 --- a/Makefile +++ b/Makefile @@ -1,377 +1,3 @@ -# provide JUSTFLAGS for just-backed targets -include ./justfiles/flags.mk +DEPRECATED_TARGETS := help build build-go build-contracts build-customlint lint-go lint-go-fix check-op-geth-version golang-docker docker-builder-clean docker-builder compute-git-versions cross-op-node contracts-bedrock-docker submodules op-node generate-mocks-op-node generate-mocks-op-service op-batcher op-proposer op-challenger op-dispute-mon op-supernode op-interop-filter op-program cannon reproducible-prestate-op-program reproducible-prestate-kona reproducible-prestate cannon-prestates mod-tidy clean nuke test-unit semgrep-ci op-program-client op-program-host make-pre-test go-tests go-tests-short go-tests-short-ci go-tests-ci go-tests-ci-kona-action go-tests-fraud-proofs-ci test update-op-geth -BEDROCK_TAGS_REMOTE?=origin -OP_STACK_GO_BUILDER?=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest - -# Requires at least Python v3.9; specify a minor version below if needed -PYTHON?=python3 - -help: ## Prints this help message - @grep -h -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -build: build-go build-contracts ## Builds Go components and contracts-bedrock -.PHONY: build - -build-go: submodules op-node op-proposer op-batcher op-challenger op-dispute-mon op-program cannon ## Builds main Go components -.PHONY: build-go - -build-contracts: - (cd packages/contracts-bedrock && just build) -.PHONY: build-contracts - -build-customlint: - make -C linter build -.PHONY: build-customlint - -lint-go: build-customlint ## Lints Go code with specific linters - ./linter/bin/op-golangci-lint run ./... - go mod tidy -diff -.PHONY: lint-go - -lint-go-fix: build-customlint ## Lints Go code with specific linters and fixes reported issues - ./linter/bin/op-golangci-lint run ./... --fix -.PHONY: lint-go-fix - -check-op-geth-version: ## Checks that op-geth version in go.mod is valid - go run ./ops/scripts/check-op-geth-version -.PHONY: check-op-geth-version - -golang-docker: ## Builds Docker images for Go components using buildx - # We don't use a buildx builder here, and just load directly into regular docker, for convenience. - GIT_COMMIT=$$(git rev-parse HEAD) \ - GIT_DATE=$$(git show -s --format='%ct') \ - IMAGE_TAGS=$$(git rev-parse HEAD),latest \ - docker buildx bake \ - --progress plain \ - --load \ - -f docker-bake.hcl \ - op-node op-batcher op-proposer op-challenger op-dispute-mon op-supervisor -.PHONY: golang-docker - -docker-builder-clean: ## Removes the Docker buildx builder - docker buildx rm buildx-build -.PHONY: docker-builder-clean - -docker-builder: ## Creates a Docker buildx builder - docker buildx create \ - --driver=docker-container --name=buildx-build --bootstrap --use -.PHONY: docker-builder - -compute-git-versions: ## Computes GIT_VERSION for all images and outputs JSON - @GIT_COMMIT=$$(git rev-parse HEAD) ./ops/scripts/compute-git-versions.sh -.PHONY: compute-git-versions - -# add --print to dry-run -cross-op-node: ## Builds cross-platform Docker image for op-node - # We don't use a buildx builder here, and just load directly into regular docker, for convenience. - GIT_COMMIT=$$(git rev-parse HEAD) \ - GIT_DATE=$$(git show -s --format='%ct') \ - IMAGE_TAGS=$$(git rev-parse HEAD),latest \ - PLATFORMS="linux/arm64" \ - GIT_VERSION=$(shell tags=$$(git tag --points-at $(GITCOMMIT) | grep '^op-node/' | sed 's/op-node\///' | sort -V); \ - preferred_tag=$$(echo "$$tags" | grep -v -- '-rc' | tail -n 1); \ - if [ -z "$$preferred_tag" ]; then \ - if [ -z "$$tags" ]; then \ - echo "untagged"; \ - else \ - echo "$$tags" | tail -n 1; \ - fi \ - else \ - echo $$preferred_tag; \ - fi) \ - docker buildx bake \ - --progress plain \ - --builder=buildx-build \ - --load \ - --no-cache \ - -f docker-bake.hcl \ - op-node -.PHONY: cross-op-node - -contracts-bedrock-docker: ## Builds Docker image for Bedrock contracts - IMAGE_TAGS=$$(git rev-parse HEAD),latest \ - docker buildx bake \ - --progress plain \ - --load \ - -f docker-bake.hcl \ - contracts-bedrock -.PHONY: contracts-bedrock-docker - -submodules: ## Updates git submodules - git submodule update --init --recursive -.PHONY: submodules - - -op-node: ## Builds op-node binary - just $(JUSTFLAGS) ./op-node/op-node -.PHONY: op-node - -generate-mocks-op-node: ## Generates mocks for op-node - make -C ./op-node generate-mocks -.PHONY: generate-mocks-op-node - -generate-mocks-op-service: ## Generates mocks for op-service - make -C ./op-service generate-mocks -.PHONY: generate-mocks-op-service - -op-batcher: ## Builds op-batcher binary - just $(JUSTFLAGS) ./op-batcher/op-batcher -.PHONY: op-batcher - -op-proposer: ## Builds op-proposer binary - just $(JUSTFLAGS) ./op-proposer/op-proposer -.PHONY: op-proposer - -op-challenger: ## Builds op-challenger binary - make -C ./op-challenger op-challenger -.PHONY: op-challenger - -op-dispute-mon: ## Builds op-dispute-mon binary - make -C ./op-dispute-mon op-dispute-mon -.PHONY: op-dispute-mon - -op-supernode: ## Builds op-supernode binary - just $(JUSTFLAGS) ./op-supernode/op-supernode -.PHONY: op-supernode - -op-interop-filter: ## Builds op-interop-filter binary - just $(JUSTFLAGS) ./op-interop-filter/op-interop-filter -.PHONY: op-interop-filter - -op-program: ## Builds op-program binary - make -C ./op-program op-program -.PHONY: op-program - -cannon: ## Builds cannon binary - make -C ./cannon cannon -.PHONY: cannon - -reproducible-prestate-op-program: - make -C ./op-program build-reproducible-prestate -.PHONY: reproducible-prestate-op-program - -reproducible-prestate-kona: - cd rust && just build-kona-reproducible-prestate -.PHONY: reproducible-prestate-kona - -reproducible-prestate: reproducible-prestate-op-program reproducible-prestate-kona ## Builds reproducible prestates for op-program and kona - # Output the prestate hashes after all the builds complete so they are easy to find at the end of the build logs. - make -C ./op-program output-prestate-hash - cd rust && just output-kona-prestate-hash -.PHONY: reproducible-prestate - -cannon-prestates: cannon op-program - go run ./op-program/builder/main.go build-all-prestates -.PHONY: cannon-prestates - -mod-tidy: ## Cleans up unused dependencies in Go modules - # Below GOPRIVATE line allows mod-tidy to be run immediately after - # releasing new versions. This bypasses the Go modules proxy, which - # can take a while to index new versions. - # - # See https://proxy.golang.org/ for more info. - export GOPRIVATE="github.com/ethereum-optimism" && go mod tidy -.PHONY: mod-tidy - -clean: ## Removes all generated files under bin/ - rm -rf ./bin - cd packages/contracts-bedrock/ && forge clean -.PHONY: clean - -nuke: clean ## Completely clean the project directory - git clean -Xdf -.PHONY: nuke - -test-unit: ## Runs unit tests for individual components - make -C ./op-node test - make -C ./op-proposer test - make -C ./op-batcher test - make -C ./op-e2e test - (cd packages/contracts-bedrock && just test) -.PHONY: test-unit - -# Remove the baseline-commit to generate a base reading & show all issues -semgrep: ## Runs Semgrep checks - $(eval DEV_REF := $(shell git rev-parse develop)) - SEMGREP_REPO_NAME=ethereum-optimism/optimism semgrep ci --baseline-commit=$(DEV_REF) -.PHONY: semgrep - -op-program-client: ## Builds op-program-client binary - make -C ./op-program op-program-client -.PHONY: op-program-client - -op-program-host: ## Builds op-program-host binary - make -C ./op-program op-program-host -.PHONY: op-program-host - -make-pre-test: ## Makes pre-test setup - make -C ./op-e2e pre-test -.PHONY: make-pre-test - -# Common prerequisites and package list for Go tests -TEST_DEPS := op-program-client op-program-host cannon build-contracts cannon-prestates make-pre-test - -# Excludes: op-validator, op-deployer/pkg/{validation,deployer/{bootstrap,manage,opcm,pipeline,upgrade}} (need RPC) -TEST_PKGS := \ - ./op-alt-da/... \ - ./op-batcher/... \ - ./op-chain-ops/... \ - ./op-node/... \ - ./op-proposer/... \ - ./op-challenger/... \ - ./op-faucet/... \ - ./op-dispute-mon/... \ - ./op-conductor/... \ - ./op-program/... \ - ./op-service/... \ - ./op-supervisor/... \ - ./op-test-sequencer/... \ - ./op-fetcher/... \ - ./op-e2e/system/... \ - ./op-e2e/e2eutils/... \ - ./op-e2e/opgeth/... \ - ./op-e2e/interop/... \ - ./op-e2e/actions/altda \ - ./op-e2e/actions/batcher \ - ./op-e2e/actions/derivation \ - ./op-e2e/actions/helpers \ - ./op-e2e/actions/interop \ - ./op-e2e/actions/proofs \ - ./op-e2e/actions/proposer \ - ./op-e2e/actions/safedb \ - ./op-e2e/actions/sequencer \ - ./op-e2e/actions/sync \ - ./op-e2e/actions/upgrades \ - ./packages/contracts-bedrock/scripts/checks/... \ - ./op-dripper/... \ - ./op-devstack/... \ - ./op-deployer/pkg/deployer/artifacts/... \ - ./op-deployer/pkg/deployer/broadcaster/... \ - ./op-deployer/pkg/deployer/clean/... \ - ./op-deployer/pkg/deployer/integration_test/ \ - ./op-deployer/pkg/deployer/integration_test/cli/... \ - ./op-deployer/pkg/deployer/standard/... \ - ./op-deployer/pkg/deployer/state/... \ - ./op-deployer/pkg/deployer/verify/... \ - ./op-sync-tester/... \ - ./op-supernode/... - -FRAUD_PROOF_TEST_PKGS := \ - ./op-e2e/faultproofs/... - -# Includes: op-validator, op-deployer/pkg/{bootstrap,manage,opcm,pipeline,upgrade} (need RPC) -RPC_TEST_PKGS := \ - ./op-validator/pkg/validations/... \ - ./op-deployer/pkg/deployer/bootstrap/... \ - ./op-deployer/pkg/deployer/manage/... \ - ./op-deployer/pkg/deployer/opcm/... \ - ./op-deployer/pkg/deployer/pipeline/... \ - ./op-deployer/pkg/deployer/upgrade/... - -# All test packages used by CI (combination of all package groups) -ALL_TEST_PACKAGES := $(TEST_PKGS) $(RPC_TEST_PKGS) $(FRAUD_PROOF_TEST_PKGS) - -# Common test environment variables -# For setting PARALLEL, nproc is for linux, sysctl for Mac and then fallback to 4 if neither is available -define DEFAULT_TEST_ENV_VARS -export ENABLE_KURTOSIS=true && \ -export OP_E2E_CANNON_ENABLED="false" && \ -export OP_E2E_USE_HTTP=true && \ -export ENABLE_ANVIL=true && \ -export PARALLEL=$$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) -endef - -# Additional CI-specific environment variables -define CI_ENV_VARS -export OP_TESTLOG_FILE_LOGGER_OUTDIR=$$(realpath ./tmp/testlogs) && \ -export SEPOLIA_RPC_URL="https://ci-sepolia-l1-archive.optimism.io" && \ -export MAINNET_RPC_URL="https://ci-mainnet-l1-archive.optimism.io" && \ -export NAT_INTEROP_LOADTEST_TARGET=10 && \ -export NAT_INTEROP_LOADTEST_TIMEOUT=30s -endef - -# Test timeout (can be overridden via environment) -TEST_TIMEOUT ?= 10m - -go-tests: $(TEST_DEPS) ## Runs comprehensive Go tests across all packages (cached for fast repeated runs) - $(DEFAULT_TEST_ENV_VARS) && \ - go test -parallel=$$PARALLEL -timeout=$(TEST_TIMEOUT) $(TEST_PKGS) -.PHONY: go-tests - -go-tests-short: $(TEST_DEPS) ## Runs comprehensive Go tests with -short flag - $(DEFAULT_TEST_ENV_VARS) && \ - go test -short -parallel=$$PARALLEL -timeout=$(TEST_TIMEOUT) $(TEST_PKGS) -.PHONY: go-tests-short - -# Internal target for running Go tests with gotestsum for CI -# Usage: make _go-tests-ci-internal GO_TEST_FLAGS="-short" -_go-tests-ci-internal: - $(MAKE) -C cannon cannon elf # Required for cannon/provider_test TestLastStepCacheAccuracy - @echo "Setting up test directories..." - mkdir -p ./tmp/test-results ./tmp/testlogs - @echo "Running Go tests with gotestsum..." - $(DEFAULT_TEST_ENV_VARS) && \ - $(CI_ENV_VARS) && \ - if [ -n "$$CIRCLE_NODE_TOTAL" ] && [ "$$CIRCLE_NODE_TOTAL" -gt 1 ]; then \ - export NODE_INDEX=$${CIRCLE_NODE_INDEX:-0} && \ - export NODE_TOTAL=$${CIRCLE_NODE_TOTAL:-1} && \ - export PARALLEL_PACKAGES=$$(echo "$(ALL_TEST_PACKAGES)" | tr ' ' '\n' | awk -v idx=$$NODE_INDEX -v total=$$NODE_TOTAL 'NR % total == idx' | tr '\n' ' ') && \ - if [ -n "$$PARALLEL_PACKAGES" ]; then \ - echo "Node $$NODE_INDEX/$$NODE_TOTAL running packages: $$PARALLEL_PACKAGES"; \ - gotestsum --format=testname \ - --junitfile=./tmp/test-results/results-$$NODE_INDEX.xml \ - --jsonfile=./tmp/testlogs/log-$$NODE_INDEX.json \ - --rerun-fails=3 \ - --rerun-fails-max-failures=50 \ - --packages="$$PARALLEL_PACKAGES" \ - -- -parallel=$$PARALLEL -coverprofile=coverage-$$NODE_INDEX.out $(GO_TEST_FLAGS) -timeout=$(TEST_TIMEOUT) -tags="ci"; \ - else \ - echo "ERROR: Node $$NODE_INDEX/$$NODE_TOTAL has no packages to run! Perhaps parallelism is set too high? (ALL_TEST_PACKAGES has $$(echo '$(ALL_TEST_PACKAGES)' | wc -w) packages)"; \ - exit 1; \ - fi; \ - else \ - gotestsum --format=testname \ - --junitfile=./tmp/test-results/results.xml \ - --jsonfile=./tmp/testlogs/log.json \ - --rerun-fails=3 \ - --rerun-fails-max-failures=50 \ - --packages="$(ALL_TEST_PACKAGES)" \ - -- -parallel=$$PARALLEL -coverprofile=coverage.out $(GO_TEST_FLAGS) -timeout=$(TEST_TIMEOUT) -tags="ci"; \ - fi -.PHONY: _go-tests-ci-internal - -go-tests-short-ci: ## Runs short Go tests with gotestsum for CI (assumes deps built by CI) - $(MAKE) _go-tests-ci-internal GO_TEST_FLAGS="-short" -.PHONY: go-tests-short-ci - -go-tests-ci: ## Runs comprehensive Go tests with gotestsum for CI (assumes deps built by CI) - $(MAKE) _go-tests-ci-internal GO_TEST_FLAGS="" -.PHONY: go-tests-ci - -go-tests-ci-kona-action: ## Runs action tests for kona with gotestsum for CI (assumes deps built by CI) - $(MAKE) _go-tests-ci-internal GO_TEST_FLAGS="-count=1 -timeout 60m -run Test_ProgramAction" -.PHONY: go-tests-ci-kona-action - -go-tests-fraud-proofs-ci: ## Runs fraud proofs Go tests with gotestsum for CI (assumes deps built by CI) - @echo "Setting up test directories..." - mkdir -p ./tmp/test-results ./tmp/testlogs - @echo "Running Go tests with gotestsum..." - $(DEFAULT_TEST_ENV_VARS) && \ - $(CI_ENV_VARS) && \ - export OP_E2E_CANNON_ENABLED="true" && \ - gotestsum --format=testname \ - --junitfile=./tmp/test-results/results.xml \ - --jsonfile=./tmp/testlogs/log.json \ - --rerun-fails=3 \ - --rerun-fails-max-failures=50 \ - --packages="$(FRAUD_PROOF_TEST_PKGS)" \ - -- -parallel=$$PARALLEL -coverprofile=coverage.out -timeout=$(TEST_TIMEOUT) -.PHONY: go-tests-fraud-proofs-ci - -test: go-tests ## Runs comprehensive Go tests (alias for go-tests) -.PHONY: test - -update-op-geth: ## Updates the Geth version used in the project - ./ops/scripts/update-op-geth.py -.PHONY: update-op-geth +include ./justfiles/deprecated.mk diff --git a/justfile b/justfile index d5ed8c55affa0..8dab027f3226c 100644 --- a/justfile +++ b/justfile @@ -1,3 +1,347 @@ +import 'justfiles/git.just' + +BEDROCK_TAGS_REMOTE := env('BEDROCK_TAGS_REMOTE', 'origin') +OP_STACK_GO_BUILDER := env('OP_STACK_GO_BUILDER', 'us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest') +PYTHON := env('PYTHON', 'python3') + +TEST_TIMEOUT := env('TEST_TIMEOUT', '10m') + +TEST_PKGS := "./op-alt-da/... ./op-batcher/... ./op-chain-ops/... ./op-node/... ./op-proposer/... ./op-challenger/... ./op-faucet/... ./op-dispute-mon/... ./op-conductor/... ./op-program/... ./op-service/... ./op-supervisor/... ./op-test-sequencer/... ./op-fetcher/... ./op-e2e/system/... ./op-e2e/e2eutils/... ./op-e2e/opgeth/... ./op-e2e/interop/... ./op-e2e/actions/altda ./op-e2e/actions/batcher ./op-e2e/actions/derivation ./op-e2e/actions/helpers ./op-e2e/actions/interop ./op-e2e/actions/proofs ./op-e2e/actions/proposer ./op-e2e/actions/safedb ./op-e2e/actions/sequencer ./op-e2e/actions/sync ./op-e2e/actions/upgrades ./packages/contracts-bedrock/scripts/checks/... ./op-dripper/... ./op-devstack/... ./op-deployer/pkg/deployer/artifacts/... ./op-deployer/pkg/deployer/broadcaster/... ./op-deployer/pkg/deployer/clean/... ./op-deployer/pkg/deployer/integration_test/ ./op-deployer/pkg/deployer/integration_test/cli/... ./op-deployer/pkg/deployer/standard/... ./op-deployer/pkg/deployer/state/... ./op-deployer/pkg/deployer/verify/... ./op-sync-tester/... ./op-supernode/..." + +FRAUD_PROOF_TEST_PKGS := "./op-e2e/faultproofs/..." + +RPC_TEST_PKGS := "./op-validator/pkg/validations/... ./op-deployer/pkg/deployer/bootstrap/... ./op-deployer/pkg/deployer/manage/... ./op-deployer/pkg/deployer/opcm/... ./op-deployer/pkg/deployer/pipeline/... ./op-deployer/pkg/deployer/upgrade/..." + +ALL_TEST_PACKAGES := TEST_PKGS + " " + RPC_TEST_PKGS + " " + FRAUD_PROOF_TEST_PKGS + +# Lists all available targets. +help: + @just --list + +# Builds Go components and contracts-bedrock. +build: build-go build-contracts + +# Builds main Go components. +build-go: submodules op-node op-proposer op-batcher op-challenger op-dispute-mon op-program cannon + +# Builds contracts-bedrock. +build-contracts: + cd packages/contracts-bedrock && just build + +# Builds the custom linter. +build-customlint: + cd linter && just build + +# Lints Go code with specific linters. +lint-go: build-customlint + ./linter/bin/op-golangci-lint run ./... + go mod tidy -diff + +# Lints Go code with specific linters and fixes reported issues. +lint-go-fix: build-customlint + ./linter/bin/op-golangci-lint run ./... --fix + +# Checks that op-geth version in go.mod is valid. +check-op-geth-version: + go run ./ops/scripts/check-op-geth-version + +# Builds Docker images for Go components using buildx. +[script('bash')] +golang-docker: + set -euo pipefail + GIT_COMMIT=$(git rev-parse HEAD) \ + GIT_DATE=$(git show -s --format='%ct') \ + IMAGE_TAGS=$(git rev-parse HEAD),latest \ + docker buildx bake \ + --progress plain \ + --load \ + -f docker-bake.hcl \ + op-node op-batcher op-proposer op-challenger op-dispute-mon op-supervisor + +# Removes the Docker buildx builder. +docker-builder-clean: + docker buildx rm buildx-build + +# Creates a Docker buildx builder. +docker-builder: + docker buildx create \ + --driver=docker-container --name=buildx-build --bootstrap --use + +# Computes GIT_VERSION for all images and outputs JSON. +[script('bash')] +compute-git-versions: + GIT_COMMIT=$(git rev-parse HEAD) ./ops/scripts/compute-git-versions.sh + +# Builds cross-platform Docker image for op-node. +[script('bash')] +cross-op-node: + set -euo pipefail + GITCOMMIT=$(git rev-parse HEAD) + tags=$(git tag --points-at "$GITCOMMIT" | grep '^op-node/' | sed 's/op-node\///' | sort -V) + preferred_tag=$(echo "$tags" | grep -v -- '-rc' | tail -n 1) + if [ -z "$preferred_tag" ]; then + if [ -z "$tags" ]; then + GIT_VERSION="untagged" + else + GIT_VERSION=$(echo "$tags" | tail -n 1) + fi + else + GIT_VERSION="$preferred_tag" + fi + GIT_COMMIT="$GITCOMMIT" \ + GIT_DATE=$(git show -s --format='%ct') \ + IMAGE_TAGS=$(git rev-parse HEAD),latest \ + PLATFORMS="linux/arm64" \ + GIT_VERSION="$GIT_VERSION" \ + docker buildx bake \ + --progress plain \ + --builder=buildx-build \ + --load \ + --no-cache \ + -f docker-bake.hcl \ + op-node + +# Builds Docker image for Bedrock contracts. +[script('bash')] +contracts-bedrock-docker: + set -euo pipefail + IMAGE_TAGS=$(git rev-parse HEAD),latest \ + docker buildx bake \ + --progress plain \ + --load \ + -f docker-bake.hcl \ + contracts-bedrock + +# Updates git submodules. +submodules: + git submodule update --init --recursive + +# Builds op-node binary. +op-node: + just ./op-node/op-node + +# Generates mocks for op-node. +generate-mocks-op-node: + cd op-node && just generate-mocks + +# Generates mocks for op-service. +generate-mocks-op-service: + cd op-service && just generate-mocks + +# Builds op-batcher binary. +op-batcher: + just ./op-batcher/op-batcher + +# Builds op-proposer binary. +op-proposer: + just ./op-proposer/op-proposer + +# Builds op-challenger binary. +op-challenger: + cd op-challenger && just op-challenger + +# Builds op-dispute-mon binary. +op-dispute-mon: + cd op-dispute-mon && just op-dispute-mon + +# Builds op-supernode binary. +op-supernode: + just ./op-supernode/op-supernode + +# Builds op-interop-filter binary. +op-interop-filter: + just ./op-interop-filter/op-interop-filter + +# Builds op-program binary. +op-program: + cd op-program && just op-program + +# Builds cannon binary. +cannon: + cd cannon && just cannon + +# Builds reproducible prestate for op-program. +reproducible-prestate-op-program: + cd op-program && just build-reproducible-prestate + +# Builds reproducible prestate for kona. +reproducible-prestate-kona: + cd rust && just build-kona-reproducible-prestate + +# Builds reproducible prestates for op-program and kona in parallel. +[script('bash')] +reproducible-prestate: + set -euo pipefail + (cd op-program && just build-reproducible-prestate) & + pid1=$! + (cd rust && just build-kona-reproducible-prestate) & + pid2=$! + wait "$pid1" "$pid2" + (cd op-program && just output-prestate-hash) + (cd rust && just output-kona-prestate-hash) + +# Builds cannon prestates. +cannon-prestates: cannon op-program + go run ./op-program/builder/main.go build-all-prestates + +# Cleans up unused dependencies in Go modules. +# Bypasses the Go module proxy for freshly released versions. +# See https://proxy.golang.org/ for more info. +mod-tidy: + GOPRIVATE="github.com/ethereum-optimism" go mod tidy + +# Removes all generated files under bin/. +clean: + rm -rf ./bin + cd packages/contracts-bedrock/ && forge clean + +# Completely clean the project directory. +nuke: clean + git clean -Xdf + +# Runs unit tests for individual components. +test-unit: + cd op-node && just test + cd op-proposer && just test + cd op-batcher && just test + cd op-e2e && just test + cd packages/contracts-bedrock && just test + +# Runs semgrep on the entire monorepo. +semgrep: + semgrep scan --config .semgrep/rules/ --error . + +# Runs semgrep CI checks against develop baseline. +[script('bash')] +semgrep-ci: + set -euo pipefail + DEV_REF=$(git rev-parse develop) + SEMGREP_REPO_NAME=ethereum-optimism/optimism semgrep ci --baseline-commit="$DEV_REF" + +# Builds op-program-client binary. +op-program-client: + cd op-program && just op-program-client + +# Builds op-program-host binary. +op-program-host: + cd op-program && just op-program-host + +# Makes pre-test setup. +make-pre-test: + cd op-e2e && just pre-test + +# Runs comprehensive Go tests across all packages. +[script('bash')] +go-tests: op-program-client op-program-host cannon build-contracts cannon-prestates make-pre-test + set -euo pipefail + export ENABLE_KURTOSIS=true + export OP_E2E_CANNON_ENABLED="false" + export OP_E2E_USE_HTTP=true + export ENABLE_ANVIL=true + export PARALLEL=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) + go test -parallel="$PARALLEL" -timeout={{TEST_TIMEOUT}} {{TEST_PKGS}} + +# Runs comprehensive Go tests with -short flag. +[script('bash')] +go-tests-short: op-program-client op-program-host cannon build-contracts cannon-prestates make-pre-test + set -euo pipefail + export ENABLE_KURTOSIS=true + export OP_E2E_CANNON_ENABLED="false" + export OP_E2E_USE_HTTP=true + export ENABLE_ANVIL=true + export PARALLEL=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) + go test -short -parallel="$PARALLEL" -timeout={{TEST_TIMEOUT}} {{TEST_PKGS}} + +# Internal: runs Go tests with gotestsum for CI. +[script('bash')] +_go-tests-ci-internal go_test_flags="": + set -euo pipefail + (cd cannon && just cannon elf) + echo "Setting up test directories..." + mkdir -p ./tmp/test-results ./tmp/testlogs + echo "Running Go tests with gotestsum..." + export ENABLE_KURTOSIS=true + export OP_E2E_CANNON_ENABLED="false" + export OP_E2E_USE_HTTP=true + export ENABLE_ANVIL=true + export PARALLEL=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) + export OP_TESTLOG_FILE_LOGGER_OUTDIR=$(realpath ./tmp/testlogs) + export SEPOLIA_RPC_URL="https://ci-sepolia-l1-archive.optimism.io" + export MAINNET_RPC_URL="https://ci-mainnet-l1-archive.optimism.io" + export NAT_INTEROP_LOADTEST_TARGET=10 + export NAT_INTEROP_LOADTEST_TIMEOUT=30s + ALL_PACKAGES="{{ALL_TEST_PACKAGES}}" + if [ -n "${CIRCLE_NODE_TOTAL:-}" ] && [ "$CIRCLE_NODE_TOTAL" -gt 1 ]; then + NODE_INDEX=${CIRCLE_NODE_INDEX:-0} + NODE_TOTAL=${CIRCLE_NODE_TOTAL:-1} + PARALLEL_PACKAGES=$(echo "$ALL_PACKAGES" | tr ' ' '\n' | awk -v idx="$NODE_INDEX" -v total="$NODE_TOTAL" 'NR % total == idx' | tr '\n' ' ') + if [ -n "$PARALLEL_PACKAGES" ]; then + echo "Node $NODE_INDEX/$NODE_TOTAL running packages: $PARALLEL_PACKAGES" + gotestsum --format=testname \ + --junitfile=./tmp/test-results/results-"$NODE_INDEX".xml \ + --jsonfile=./tmp/testlogs/log-"$NODE_INDEX".json \ + --rerun-fails=3 \ + --rerun-fails-max-failures=50 \ + --packages="$PARALLEL_PACKAGES" \ + -- -parallel="$PARALLEL" -coverprofile=coverage-"$NODE_INDEX".out {{go_test_flags}} -timeout={{TEST_TIMEOUT}} -tags="ci" + else + echo "ERROR: Node $NODE_INDEX/$NODE_TOTAL has no packages to run! Perhaps parallelism is set too high? (ALL_TEST_PACKAGES has $(echo "$ALL_PACKAGES" | wc -w) packages)" + exit 1 + fi + else + gotestsum --format=testname \ + --junitfile=./tmp/test-results/results.xml \ + --jsonfile=./tmp/testlogs/log.json \ + --rerun-fails=3 \ + --rerun-fails-max-failures=50 \ + --packages="$ALL_PACKAGES" \ + -- -parallel="$PARALLEL" -coverprofile=coverage.out {{go_test_flags}} -timeout={{TEST_TIMEOUT}} -tags="ci" + fi + +# Runs short Go tests with gotestsum for CI. +go-tests-short-ci: + just _go-tests-ci-internal "-short" + +# Runs comprehensive Go tests with gotestsum for CI. +go-tests-ci: + just _go-tests-ci-internal "" + +# Runs action tests for kona with gotestsum for CI. +go-tests-ci-kona-action: + just _go-tests-ci-internal "-count=1 -timeout 60m -run Test_ProgramAction" + +# Runs fraud proofs Go tests with gotestsum for CI. +[script('bash')] +go-tests-fraud-proofs-ci: + set -euo pipefail + echo "Setting up test directories..." + mkdir -p ./tmp/test-results ./tmp/testlogs + echo "Running Go tests with gotestsum..." + export ENABLE_KURTOSIS=true + export OP_E2E_CANNON_ENABLED="true" + export OP_E2E_USE_HTTP=true + export ENABLE_ANVIL=true + export PARALLEL=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) + export OP_TESTLOG_FILE_LOGGER_OUTDIR=$(realpath ./tmp/testlogs) + export SEPOLIA_RPC_URL="https://ci-sepolia-l1-archive.optimism.io" + export MAINNET_RPC_URL="https://ci-mainnet-l1-archive.optimism.io" + export NAT_INTEROP_LOADTEST_TARGET=10 + export NAT_INTEROP_LOADTEST_TIMEOUT=30s + gotestsum --format=testname \ + --junitfile=./tmp/test-results/results.xml \ + --jsonfile=./tmp/testlogs/log.json \ + --rerun-fails=3 \ + --rerun-fails-max-failures=50 \ + --packages="{{FRAUD_PROOF_TEST_PKGS}}" \ + -- -parallel="$PARALLEL" -coverprofile=coverage.out -timeout={{TEST_TIMEOUT}} + +# Runs comprehensive Go tests (alias for go-tests). +test: go-tests + +# Updates the Geth version used in the project. +update-op-geth: + ./ops/scripts/update-op-geth.py + # Build all Rust binaries (release) for sysgo tests. build-rust-release: cd rust && cargo build --release --bin kona-node @@ -12,10 +356,6 @@ check-nut-locks: todo-checker: ./ops/scripts/todo-checker.sh -# Runs semgrep on the entire monorepo. -semgrep: - semgrep scan --config .semgrep/rules/ --error . - # Runs semgrep tests. semgrep-test: semgrep scan --test --config .semgrep/rules/ .semgrep/tests/ @@ -37,11 +377,11 @@ latest-versions: ./ops/scripts/latest-versions.sh # Usage: -# just update-op-geth 2f0528b -# just update-op-geth v1.101602.4 -# just update-op-geth optimism +# just update-op-geth-ref 2f0528b +# just update-op-geth-ref v1.101602.4 +# just update-op-geth-ref optimism [script('bash')] -update-op-geth ref: +update-op-geth-ref ref: set -euo pipefail ref="{{ref}}" if [ -z "$ref" ]; then echo "error: provide a hash/tag/branch"; exit 1; fi diff --git a/op-e2e/justfile b/op-e2e/justfile index 72002fe4b54b6..2fe97ed1098e4 100644 --- a/op-e2e/justfile +++ b/op-e2e/justfile @@ -49,7 +49,7 @@ test-fault-proofs: pre-test # Build cannon prestates cannon-prestates: - make -C .. cannon-prestates + cd .. && just cannon-prestates # Pre-test setup pre-test: pre-test-cannon