From 8ff7f08446616c291ba276daef27bc956366f120 Mon Sep 17 00:00:00 2001 From: Yann Hodique Date: Mon, 25 Nov 2024 14:53:51 +0100 Subject: [PATCH 1/6] build(just): factor out flags code This will enable making use of JUSTFLAGS in the main Makefile for migrated targets. --- just/deprecated.mk | 30 ++++-------------------------- just/flags.mk | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 26 deletions(-) create mode 100644 just/flags.mk diff --git a/just/deprecated.mk b/just/deprecated.mk index a18d665c054..b3e6fc6bd2a 100644 --- a/just/deprecated.mk +++ b/just/deprecated.mk @@ -6,37 +6,15 @@ else banner-style = $(shell tput bold)$(shell tput setaf 1)$1$(shell tput sgr0) endif -# Variable assignments can affect the semantic of the make targets. -# Typical use-case: setting VERSION in a release build, since CI -# doesn't preserve the git environment. -# -# We need to translate: -# "make target VAR=val" to "just VAR=val target" -# -# MAKEFLAGS is a string of the form: -# "abc --foo --bar=baz -- VAR1=val1 VAR2=val2", namely: -# - abc is the concatnation of all short flags -# - --foo and --bar=baz are long options, -# - -- is the separator between flags and variable assignments, -# - VAR1=val1 and VAR2=val2 are variable assignments -# -# Goal: ignore all CLI flags, keep only variable assignments. -# -# First remove the short flags at the beginning, or the first long-flag, -# or if there is no flag at all, the -- separator (which then makes the -# next step a noop). If there's no flag and no variable assignment, the -# result is empty anyway, so the wordlist call is safe (everything is a noop). -tmp-flags = $(wordlist 2,$(words $(MAKEFLAGS)),$(MAKEFLAGS)) -# Then remove all long options, including the -- separator, if needed. That -# leaves only variable assignments. -just-flags = $(patsubst --%,,$(tmp-flags)) +SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +include $(SELF_DIR)/flags.mk define make-deprecated-target $1: @echo - @printf %s\\n '$(call banner-style,"make $1 $(just-flags)" is deprecated. Please use "just $(just-flags) $1" instead.)' + @printf %s\\n '$(call banner-style,"make $1 $(JUSTFLAGS)" is deprecated. Please use "just $(JUSTFLAGS) $1" instead.)' @echo - just $(just-flags) $1 + just $(JUSTFLAGS) $1 endef $(foreach element,$(DEPRECATED_TARGETS),$(eval $(call make-deprecated-target,$(element)))) diff --git a/just/flags.mk b/just/flags.mk new file mode 100644 index 00000000000..121a3eb70e9 --- /dev/null +++ b/just/flags.mk @@ -0,0 +1,24 @@ +# Variable assignments can affect the semantic of the make targets. +# Typical use-case: setting VERSION in a release build, since CI +# doesn't preserve the git environment. +# +# We need to translate: +# "make target VAR=val" to "just VAR=val target" +# +# MAKEFLAGS is a string of the form: +# "abc --foo --bar=baz -- VAR1=val1 VAR2=val2", namely: +# - abc is the concatnation of all short flags +# - --foo and --bar=baz are long options, +# - -- is the separator between flags and variable assignments, +# - VAR1=val1 and VAR2=val2 are variable assignments +# +# Goal: ignore all CLI flags, keep only variable assignments. +# +# First remove the short flags at the beginning, or the first long-flag, +# or if there is no flag at all, the -- separator (which then makes the +# next step a noop). If there's no flag and no variable assignment, the +# result is empty anyway, so the wordlist call is safe (everything is a noop). +tmp-flags := $(wordlist 2,$(words $(MAKEFLAGS)),$(MAKEFLAGS)) +# Then remove all long options, including the -- separator, if needed. That +# leaves only variable assignments. +JUSTFLAGS := $(patsubst --%,,$(tmp-flags)) \ No newline at end of file From c019aeb010fde2183b6df37897dbcf690227cd7a Mon Sep 17 00:00:00 2001 From: Yann Hodique Date: Mon, 25 Nov 2024 14:50:10 +0100 Subject: [PATCH 2/6] fix(just): parallel support For some unclear reason runs in CI environment are invalid (they generate some "empty" calls to the private target). Sidestep the issue for now. --- just/default.just | 5 +++-- just/deprecated.mk | 6 +++++- op-batcher/justfile | 19 ++++++++++--------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/just/default.just b/just/default.just index 220c27dca5c..92503b23ba5 100644 --- a/just/default.just +++ b/just/default.just @@ -1,5 +1,6 @@ set shell := ["bash", "-c"] -PARALLEL := num_cpus() +PARALLEL_JOBS := num_cpus() -MAP_JUST := "/usr/bin/env -S parallel --shebang --jobs " + PARALLEL + " --colsep ' ' -r " + just_executable() +# TODO: this fails in CI for some reason +MAP_JUST := "/usr/bin/env -S parallel --shebang --jobs " + PARALLEL_JOBS + " --colsep ' ' -r " + just_executable() diff --git a/just/deprecated.mk b/just/deprecated.mk index b3e6fc6bd2a..655f901348d 100644 --- a/just/deprecated.mk +++ b/just/deprecated.mk @@ -1,6 +1,9 @@ ifeq (, $(shell which tput)) # CI environment typically does not support tput. banner-style = $1 +else ifeq (, $(TERM)) + # Terminal type not set, so tput would fail. + banner-style = $1 else # print in bold red to bring attention. banner-style = $(shell tput bold)$(shell tput setaf 1)$1$(shell tput sgr0) @@ -12,7 +15,8 @@ include $(SELF_DIR)/flags.mk define make-deprecated-target $1: @echo - @printf %s\\n '$(call banner-style,"make $1 $(JUSTFLAGS)" is deprecated. Please use "just $(JUSTFLAGS) $1" instead.)' + @printf %s\\n '$(call banner-style,Deprecated make call: make $1 $(JUSTFLAGS))' + @printf %s\\n '$(call banner-style,Consider using just instead: just $(JUSTFLAGS) $1)' @echo just $(JUSTFLAGS) $1 endef diff --git a/op-batcher/justfile b/op-batcher/justfile index 2647debbcb4..a0c671ebd28 100644 --- a/op-batcher/justfile +++ b/op-batcher/justfile @@ -24,12 +24,13 @@ batcher_fuzz_task FUZZ TIME='10s': (go_fuzz FUZZ TIME "./batcher") # Run fuzzing tests fuzz: - #!{{MAP_JUST}} batcher_fuzz_task - FuzzChannelConfig_CheckTimeout - FuzzDurationZero - FuzzDurationTimeoutMaxChannelDuration - FuzzDurationTimeoutZeroMaxChannelDuration - FuzzChannelCloseTimeout - FuzzChannelZeroCloseTimeout - FuzzSeqWindowClose - FuzzSeqWindowZeroTimeoutClose + printf "%s\n" \ + "FuzzChannelConfig_CheckTimeout" \ + "FuzzDurationZero" \ + "FuzzDurationTimeoutMaxChannelDuration" \ + "FuzzDurationTimeoutZeroMaxChannelDuration" \ + "FuzzChannelCloseTimeout" \ + "FuzzChannelZeroCloseTimeout" \ + "FuzzSeqWindowClose" \ + "FuzzSeqWindowZeroTimeoutClose" \ + | parallel -j {{PARALLEL_JOBS}} {{just_executable()}} batcher_fuzz_task {} From 20624aaa088a3641a6c10f3affb39d7d4fe26bab Mon Sep 17 00:00:00 2001 From: Yann Hodique Date: Fri, 22 Nov 2024 19:17:41 +0100 Subject: [PATCH 3/6] build(just): add more helpers - VERSION_META variable - go_generate target --- just/git.just | 2 ++ just/go.just | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/just/git.just b/just/git.just index 922286d7ab5..ac50e4e762d 100644 --- a/just/git.just +++ b/just/git.just @@ -24,3 +24,5 @@ VERSION := shell('if [ -z "$1" ]; then else echo $1 fi', _PREFERRED_TAG, _LAST_TAG) + +VERSION_META := "" diff --git a/just/go.just b/just/go.just index 5af76629509..3a394e3ec94 100644 --- a/just/go.just +++ b/just/go.just @@ -25,3 +25,7 @@ go_test SELECTOR *FLAGS: [private] go_fuzz FUZZ TIME='10s' PKG='': (go_test PKG _EXTRALDFLAGS "-fuzztime" TIME "-fuzz" FUZZ "-run" "NOTAREALTEST") + +[private] +go_generate SELECTOR *FLAGS: + go generate -v {{FLAGS}} {{SELECTOR}} \ No newline at end of file From 159b10e8521130d4b2957147e7added7b6f0a9dd Mon Sep 17 00:00:00 2001 From: Yann Hodique Date: Thu, 7 Nov 2024 09:55:43 +0100 Subject: [PATCH 4/6] build(op-proposer): migrate build to just --- op-proposer/Makefile | 35 ++--------------------------------- op-proposer/justfile | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 33 deletions(-) create mode 100644 op-proposer/justfile diff --git a/op-proposer/Makefile b/op-proposer/Makefile index 561d7d32f30..3a036e6d5d5 100644 --- a/op-proposer/Makefile +++ b/op-proposer/Makefile @@ -1,34 +1,3 @@ -GITCOMMIT ?= $(shell git rev-parse HEAD) -GITDATE ?= $(shell git show -s --format='%ct') -# Find the github tag that points to this commit. If none are found, set the version string to "untagged" -# Prioritizes release tag, if one exists, over tags suffixed with "-rc" -VERSION ?= $(shell tags=$$(git tag --points-at $(GITCOMMIT) | grep '^op-proposer/' | sed 's/op-proposer\///' | sort -V); \ - preferred_tag=$$(echo "$$tags" | grep -v -- '-rc' | tail -n 1); \ - if [ -z "$$preferred_tag" ]; then \ - if [ -z "$$tags" ]; then \ - echo "untagged"; \ - else \ - echo "$$tags" | tail -n 1; \ - fi \ - else \ - echo $$preferred_tag; \ - fi) +DEPRECATED_TARGETS := op-proposer clean test -LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT) -LDFLAGSSTRING +=-X main.GitDate=$(GITDATE) -LDFLAGSSTRING +=-X main.Version=$(VERSION) -LDFLAGS := -ldflags "$(LDFLAGSSTRING)" - -op-proposer: - env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) CGO_ENABLED=0 go build -v $(LDFLAGS) -o ./bin/op-proposer ./cmd - -clean: - rm bin/op-proposer - -test: - go test -v ./... - -.PHONY: \ - clean \ - op-proposer \ - test +include ../just/deprecated.mk diff --git a/op-proposer/justfile b/op-proposer/justfile new file mode 100644 index 00000000000..08b1b7b7391 --- /dev/null +++ b/op-proposer/justfile @@ -0,0 +1,20 @@ +import '../just/go.just' + +# Build ldflags string +_LDFLAGSSTRING := "'" + trim( + "-X main.GitCommit=" + GITCOMMIT + " " + \ + "-X main.GitDate=" + GITDATE + " " + \ + "-X main.Version=" + VERSION + " " + \ + "") + "'" + +BINARY := "./bin/op-proposer" + +# Build op-proposer binary +op-proposer: (go_build BINARY "./cmd" "-ldflags" _LDFLAGSSTRING) + +# Clean build artifacts +clean: + rm -f {{BINARY}} + +# Run tests +test: (go_test "./...") From 56f719f43070f17c7da9911665ec2e43f5374154 Mon Sep 17 00:00:00 2001 From: Yann Hodique Date: Fri, 22 Nov 2024 19:19:24 +0100 Subject: [PATCH 5/6] build(op-node): migrate build to just --- op-node/Makefile | 66 ++---------------------------------------------- op-node/justfile | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 64 deletions(-) create mode 100644 op-node/justfile diff --git a/op-node/Makefile b/op-node/Makefile index c1d480d9b71..b63f489925a 100644 --- a/op-node/Makefile +++ b/op-node/Makefile @@ -1,65 +1,3 @@ -GITCOMMIT ?= $(shell git rev-parse HEAD) -GITDATE ?= $(shell git show -s --format='%ct') -# Find the github tag that points to this commit. If none are found, set the version string to "untagged" -# Prioritizes release tag, if one exists, over tags suffixed with "-rc" -VERSION ?= $(shell tags=$$(git tag --points-at $(GITCOMMIT) | grep '^op-node/' | sed 's/op-node\///' | sort -V); \ - preferred_tag=$$(echo "$$tags" | grep -v -- '-rc' | tail -n 1); \ - if [ -z "$$preferred_tag" ]; then \ - if [ -z "$$tags" ]; then \ - echo "untagged"; \ - else \ - echo "$$tags" | tail -n 1; \ - fi \ - else \ - echo $$preferred_tag; \ - fi) +DEPRECATED_TARGETS := op-node clean test fuzz generate-mocks readme -LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT) -LDFLAGSSTRING +=-X main.GitDate=$(GITDATE) -LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-node/version.Version=$(VERSION) -LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-node/version.Meta=$(VERSION_META) -LDFLAGS := -ldflags "$(LDFLAGSSTRING)" - -# Use the old Apple linker to workaround broken xcode - https://github.com/golang/go/issues/65169 -ifeq ($(shell uname),Darwin) - FUZZLDFLAGS := -ldflags=-extldflags=-Wl,-ld_classic -endif - -op-node: - env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) CGO_ENABLED=0 go build -v $(LDFLAGS) -o ./bin/op-node ./cmd/main.go - -clean: - rm bin/op-node - -test: - go test -v ./... - -fuzz: - printf "%s\n" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzL1InfoBedrockRoundTrip ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzL1InfoEcotoneRoundTrip ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzL1InfoAgainstContract ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzUnmarshallLogEvent ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzParseFrames ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzFrameUnmarshalBinary ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzBatchRoundTrip ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzDeriveDepositsRoundTrip ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzDeriveDepositsBadVersion ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzParseL1InfoDepositTxDataValid ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzParseL1InfoDepositTxDataBadLength ./rollup/derive" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzRejectCreateBlockBadTimestamp ./rollup/driver" \ - "go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz FuzzDecodeDepositTxDataToL1Info ./rollup/driver" \ - | parallel -j 8 {} - -generate-mocks: - go generate ./... - -readme: - doctoc README.md - -.PHONY: \ - op-node \ - clean \ - test \ - fuzz \ - readme +include ../just/deprecated.mk diff --git a/op-node/justfile b/op-node/justfile new file mode 100644 index 00000000000..46aadcc84b3 --- /dev/null +++ b/op-node/justfile @@ -0,0 +1,49 @@ +import '../just/go.just' + +# Build ldflags string +_LDFLAGSSTRING := "'" + trim( + "-X main.GitCommit=" + GITCOMMIT + " " + \ + "-X main.GitDate=" + GITDATE + " " + \ + "-X github.com/ethereum-optimism/optimism/op-node/version.Version=" + VERSION + " " + \ + "-X github.com/ethereum-optimism/optimism/op-node/version.Meta=" + VERSION_META + " " + \ + "") + "'" + +BINARY := "./bin/op-node" + +# Build op-node binary +op-node: (go_build BINARY "./cmd" "-ldflags" _LDFLAGSSTRING) + +# Clean build artifacts +clean: + rm -f {{BINARY}} + +# Run tests +test: (go_test "./...") + +# Generate mocks +generate-mocks: (go_generate "./...") + +# Update readme +readme: + doctoc README.md + +[private] +node_fuzz_task FUZZ TIME='10s': (go_fuzz FUZZ TIME "./rollup/derive") + +# Run fuzz tests +fuzz: + printf "%s\n" \ + "FuzzL1InfoBedrockRoundTrip" \ + "FuzzL1InfoEcotoneRoundTrip" \ + "FuzzL1InfoAgainstContract" \ + "FuzzUnmarshallLogEvent" \ + "FuzzParseFrames" \ + "FuzzFrameUnmarshalBinary" \ + "FuzzBatchRoundTrip" \ + "FuzzDeriveDepositsRoundTrip" \ + "FuzzDeriveDepositsBadVersion" \ + "FuzzParseL1InfoDepositTxDataValid" \ + "FuzzParseL1InfoDepositTxDataBadLength" \ + "FuzzRejectCreateBlockBadTimestamp" \ + "FuzzDecodeDepositTxDataToL1Info" \ + | parallel -j {{PARALLEL_JOBS}} {{just_executable()}} node_fuzz_task {} From 1f18bb90bb04ee165452549755965ff9bacd6705 Mon Sep 17 00:00:00 2001 From: Yann Hodique Date: Mon, 25 Nov 2024 14:54:15 +0100 Subject: [PATCH 6/6] fix(build): rewire just-migrated targets --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9830a3060cb..b2c8dba7d5d 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +# provide JUSTFLAGS for just-backed targets +include ./just/flags.mk + COMPOSEFLAGS=-d ITESTS_L2_HOST=http://localhost:9545 BEDROCK_TAGS_REMOTE?=origin @@ -94,7 +97,7 @@ submodules: ## Updates git submodules op-node: ## Builds op-node binary - make -C ./op-node op-node + just $(JUSTFLAGS) ./op-node/op-node .PHONY: op-node generate-mocks-op-node: ## Generates mocks for op-node @@ -106,11 +109,11 @@ generate-mocks-op-service: ## Generates mocks for op-service .PHONY: generate-mocks-op-service op-batcher: ## Builds op-batcher binary - make -C ./op-batcher op-batcher + just $(JUSTFLAGS) ./op-batcher/op-batcher .PHONY: op-batcher op-proposer: ## Builds op-proposer binary - make -C ./op-proposer op-proposer + just $(JUSTFLAGS) ./op-proposer/op-proposer .PHONY: op-proposer op-challenger: ## Builds op-challenger binary