diff --git a/.circleci/config.yml b/.circleci/config.yml index c70c1c69bf5..690d5ed921a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -993,7 +993,7 @@ jobs: - run: name: download espresso-network go sdk command: | - ver=$(grep "github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}') + ver=$(grep "^\s*github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}') url="https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ver}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so" mkdir -p /home/circleci/local-lib wget $url -O /home/circleci/local-lib/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so @@ -1066,7 +1066,7 @@ jobs: - run: name: download espresso-network go sdk command: | - ver=$(grep "github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}') + ver=$(grep "^\s*github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}') url="https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ver}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so" mkdir -p /home/circleci/local-lib wget $url -O /home/circleci/local-lib/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so diff --git a/espresso/docker-compose.yml b/espresso/docker-compose.yml index b11c879bd4e..c9b425b6740 100644 --- a/espresso/docker-compose.yml +++ b/espresso/docker-compose.yml @@ -493,7 +493,7 @@ services: OP_CHALLENGER_TRACE_TYPE: permissioned espresso-dev-node: - image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-colorful-snake + image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-fix-cors depends_on: l1-geth: condition: service_healthy diff --git a/espresso/docker/op-geth/Dockerfile b/espresso/docker/op-geth/Dockerfile index cf11668432b..e2482ef998e 100644 --- a/espresso/docker/op-geth/Dockerfile +++ b/espresso/docker/op-geth/Dockerfile @@ -6,25 +6,6 @@ ARG TARGETARCH ARG GIT_COMMIT ARG GIT_DATE -# Rust builder for Espresso crypto libraries -FROM rust:1.88.0-alpine3.22 AS rust-builder -# TODO: Check the hash of the Espresso GO library when switch to the new one. -# -ARG ESPRESSO_NETWORK_GO_VER=0.0.34 -RUN apk add perl make openssl-dev musl-dev gcc -ADD https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$ESPRESSO_NETWORK_GO_VER.tar.gz /source.tgz -RUN tar -oxzf /source.tgz -WORKDIR /espresso-network-go-$ESPRESSO_NETWORK_GO_VER -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/espresso-network-go/verification/rust/target \ - cargo build --release --locked --manifest-path ./verification/rust/Cargo.toml -RUN mkdir -p /libespresso -RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \ - /libespresso/libespresso_crypto_helper-aarch64-unknown-linux-gnu.a -RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \ - /libespresso/libespresso_crypto_helper-x86_64-unknown-linux-gnu.a - # CGO builder for components that need Espresso crypto linking FROM alpine:3.22 AS op-cgo-builder # Install dependencies @@ -33,11 +14,19 @@ RUN apk add musl-dev gcc go g++ curl tar gzip make gcc linux-headers git jq bash COPY ./mise.toml . RUN curl -L https://github.com/casey/just/releases/download/$(yq '.tools.just' mise.toml)/just-$(yq '.tools.just' mise.toml)-x86_64-unknown-linux-musl.tar.gz | \ tar xz -C /usr/local/bin just +# Fetch rust libs for dynamic linking +ARG ESPRESSO_SDK_VER=0.3.2 +ARG ESPRESSO_SDK_HELPER_HASH_AARCH64=ec6ce7b37edd173206ad338c84a6a771a0e9dc8b184081af7440ebfc0c531a71 +ARG ESPRESSO_SDK_HELPER_HASH_X86_64=49c50949ec1acf52107cb190c90911e05cc9c4e9d72dd7455496163443760b92 +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_AARCH64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so \ + /lib/ +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_X86_64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so \ + /lib/ # Go sources COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum -# Copy rust libs for dynamic linking -COPY --from=rust-builder /libespresso/* /lib # Warm-up the cache WORKDIR /app RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download diff --git a/espresso/docker/op-stack/Dockerfile b/espresso/docker/op-stack/Dockerfile index dd4f8ce08a9..24f34249699 100644 --- a/espresso/docker/op-stack/Dockerfile +++ b/espresso/docker/op-stack/Dockerfile @@ -39,25 +39,6 @@ COPY . /app ARG GIT_COMMIT ARG GIT_DATE -# Rust builder for Espresso crypto libraries -FROM rust:1.88.0-alpine3.22 AS rust-builder -# TODO: Check the hash of the Espresso GO library when switch to the new one. -# -ARG ESPRESSO_NETWORK_GO_VER=0.0.34 -RUN apk add perl make openssl-dev musl-dev gcc -ADD https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$ESPRESSO_NETWORK_GO_VER.tar.gz /source.tgz -RUN tar -oxzf /source.tgz -WORKDIR /espresso-network-go-$ESPRESSO_NETWORK_GO_VER -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/espresso-network-go/verification/rust/target \ - cargo build --release --locked --manifest-path ./verification/rust/Cargo.toml -RUN mkdir -p /libespresso -RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \ - /libespresso/libespresso_crypto_helper-aarch64-unknown-linux-gnu.a -RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \ - /libespresso/libespresso_crypto_helper-x86_64-unknown-linux-gnu.a - # CGO builder for components that need Espresso crypto linking FROM alpine:3.22 AS op-cgo-builder # Install dependencies @@ -70,11 +51,19 @@ RUN case $(uname -m) in \ esac && \ curl -L https://github.com/casey/just/releases/download/$(yq '.tools.just' mise.toml)/just-$(yq '.tools.just' mise.toml)-$JUST_ARCH-unknown-linux-musl.tar.gz | \ tar xz -C /usr/local/bin just +# Fetch rust libs for dynamic linking +ARG ESPRESSO_SDK_VER=0.3.2 +ARG ESPRESSO_SDK_HELPER_HASH_AARCH64=ec6ce7b37edd173206ad338c84a6a771a0e9dc8b184081af7440ebfc0c531a71 +ARG ESPRESSO_SDK_HELPER_HASH_X86_64=49c50949ec1acf52107cb190c90911e05cc9c4e9d72dd7455496163443760b92 +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_AARCH64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so \ + /lib/ +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_X86_64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so \ + /lib/ # Go sources COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum -# Copy rust libs for dynamic linking -COPY --from=rust-builder /libespresso/* /lib # Warm-up the cache WORKDIR /app RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download diff --git a/espresso/environment/12_enforce_majority_rule_test.go b/espresso/environment/12_enforce_majority_rule_test.go index 76c4f0cffb3..165f5b6f06e 100644 --- a/espresso/environment/12_enforce_majority_rule_test.go +++ b/espresso/environment/12_enforce_majority_rule_test.go @@ -5,9 +5,11 @@ import ( "math/big" "net/http" "net/http/httptest" + "strings" "testing" "time" + "github.com/coder/websocket" env "github.com/ethereum-optimism/optimism/espresso/environment" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth" "github.com/ethereum-optimism/optimism/op-e2e/system/e2esys" @@ -24,16 +26,26 @@ const NO_ERROR_EXPECTED = false // @param numBadUrls N as mentioned in the above description // @param expectedError if set to true, we expect a timeout error as the L2 cannot make progress. Otherwise, we expect no error at all. func runWithMultiClient(t *testing.T, numGoodUrls int, numBadUrls int, expectedError bool) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "Hello", http.StatusOK) + if strings.Contains(r.URL.Path, "stream") { + conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{}) + require.NoError(t, err) + + defer conn.Close(websocket.StatusGoingAway, "Bye") + + err = conn.Write(ctx, websocket.MessageText, []byte("Hello")) + require.NoError(t, err) + + } else { + http.Error(w, "Hello", http.StatusOK) + } })) badServerUrl := server.URL - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - launcher := new(env.EspressoDevNodeLauncherDocker) system, devNode, err := launcher.StartE2eDevnet(ctx, t, env.SetEspressoUrls(numGoodUrls, numBadUrls, badServerUrl)) @@ -57,7 +69,7 @@ func runWithMultiClient(t *testing.T, numGoodUrls int, numBadUrls int, expectedE blockNumber := int64(2) // Check the caff node can/cannot make progress - _, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), caffClient, 30*time.Second) + _, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), caffClient, 60*time.Second) if expectedError { require.Error(t, err, "The L2 should not be progressing") @@ -66,7 +78,7 @@ func runWithMultiClient(t *testing.T, numGoodUrls int, numBadUrls int, expectedE } // Check the l2Verif node can/cannot make progress - _, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), l2Verif, 30*time.Second) + _, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), l2Verif, 60*time.Second) if expectedError { require.Error(t, err, "The L2 should not be progressing") } else { diff --git a/espresso/environment/optitmism_espresso_test_helpers.go b/espresso/environment/optitmism_espresso_test_helpers.go index 4fd9f2393a2..c139917f5bf 100644 --- a/espresso/environment/optitmism_espresso_test_helpers.go +++ b/espresso/environment/optitmism_espresso_test_helpers.go @@ -56,7 +56,7 @@ func init() { const ESPRESSO_LIGHT_CLIENT_ADDRESS = "0x703848f4c85f18e3acd8196c8ec91eb0b7bd0797" -const ESPRESSO_DEV_NODE_DOCKER_IMAGE = "ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-colorful-snake" +const ESPRESSO_DEV_NODE_DOCKER_IMAGE = "ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-fix-cors" // This is the mnemonic that we use to create the private key for deploying // contacts on the L1 diff --git a/espresso/environment/query_service_intercept.go b/espresso/environment/query_service_intercept.go index 56114ecc775..14e907ae963 100644 --- a/espresso/environment/query_service_intercept.go +++ b/espresso/environment/query_service_intercept.go @@ -2,14 +2,18 @@ package environment import ( "bytes" + "context" "encoding/json" "io" "net/http" "net/http/httptest" "net/url" + "regexp" + "time" espressoTaggedBase64 "github.com/EspressoSystems/espresso-network/sdks/go/tagged-base64" espressoCommon "github.com/EspressoSystems/espresso-network/sdks/go/types" + "github.com/coder/websocket" "github.com/ethereum-optimism/optimism/op-batcher/batcher" "github.com/ethereum-optimism/optimism/op-e2e/system/e2esys" ) @@ -88,6 +92,13 @@ type proxyRequest struct { // ServeHTTP implements http.Handler func (p *proxyRequest) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Check if this is a websocket stream request + if isWebSocketStreamRequest(r) { + p.proxyWebSocket(w, r) + return + } + + // Handle regular HTTP requests defer r.Body.Close() buf := new(bytes.Buffer) if _, err := io.Copy(buf, r.Body); err != nil && err != io.EOF { @@ -135,6 +146,82 @@ func (p *proxyRequest) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } +// proxyWebSocket handles websocket upgrade and proxying +func (p *proxyRequest) proxyWebSocket(w http.ResponseWriter, r *http.Request) { + // Accept the websocket connection from the client + clientConn, err := websocket.Accept(w, r, &websocket.AcceptOptions{ + InsecureSkipVerify: true, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer clientConn.Close(websocket.StatusInternalError, "proxy error") + + // Create websocket URL for the backend + backendURL := p.baseURL + if backendURL.Scheme == "https" { + backendURL.Scheme = "wss" + } else { + backendURL.Scheme = "ws" + } + backendURL.Path = r.URL.Path + backendURL.RawQuery = r.URL.RawQuery + + // Connect to the backend websocket + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + //nolint:bodyclose // Not applicable to coder/websocket. From the websocket.Dial docs: "You never need to close resp.Body yourself." + backendConn, _, err := websocket.Dial(ctx, backendURL.String(), &websocket.DialOptions{}) + if err != nil { + clientConn.Close(websocket.StatusInternalError, "backend connection failed") + return + } + defer backendConn.Close(websocket.StatusNormalClosure, "") + + // Proxy messages bidirectionally + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + // Client to backend + go func() { + defer cancel() + for { + msgType, data, err := clientConn.Read(ctx) + if err != nil { + return + } + err = backendConn.Write(ctx, msgType, data) + if err != nil { + return + } + } + }() + + // Backend to client + go func() { + defer cancel() + for { + msgType, data, err := backendConn.Read(ctx) + if err != nil { + return + } + err = clientConn.Write(ctx, msgType, data) + if err != nil { + return + } + } + }() + + // Wait until context is cancelled (one of the goroutines finished) + <-ctx.Done() + + // Close connections gracefully + clientConn.Close(websocket.StatusNormalClosure, "") + backendConn.Close(websocket.StatusNormalClosure, "") +} + // fakeSubmitTransactionSuccess is a simple HTTP handler that simulates a // successful transaction submission by returning a fake commit hash. type fakeSubmitTransactionSuccess struct{} @@ -304,6 +391,13 @@ func isSubmitTransactionRequest(r *http.Request) bool { requestMatchesPath(r, http.MethodPost, stringEquals("/v0/submit/submit")) } +// isWebSocketStreamRequest checks if the request is a websocket request +// matching the pattern "vN/stream/*" where N is an integer. +func isWebSocketStreamRequest(r *http.Request) bool { + matched, _ := regexp.MatchString(`/stream/`, r.URL.Path) + return matched +} + // DecideHowToHandleRequest implements InterceptHandlerDecider func (d *randomRollFakeSubmitTransactionSuccess) DecideHowToHandleRequest(w http.ResponseWriter, r *http.Request) InterceptHandleDecision { if isSubmitTransactionRequest(r) { diff --git a/espresso/streamer.go b/espresso/streamer.go index 504f7ca798f..d73e611edda 100644 --- a/espresso/streamer.go +++ b/espresso/streamer.go @@ -41,6 +41,7 @@ type LightClientCallerInterface interface { // for modification / wrapping. type EspressoClient interface { FetchLatestBlockHeight(ctx context.Context) (uint64, error) + StreamTransactionsInNamespace(ctx context.Context, height uint64, namespace uint64) (espressoClient.Stream[espressoCommon.TransactionQueryData], error) FetchTransactionsInBlock(ctx context.Context, blockHeight uint64, namespace uint64) (espressoClient.TransactionsInBlock, error) } @@ -97,6 +98,9 @@ type BatchStreamer[B Batch] struct { RemainingBatches map[common.Hash]B unmarshalBatch func([]byte) (*B, error) + + // Use the polling API to fetch transactions + UseFetchApi bool } // Compile time assertion to ensure EspressoStreamer implements @@ -146,8 +150,7 @@ func (s *BatchStreamer[B]) RefreshSafeL1Origin(safeL1Origin eth.BlockID) error { return err } -// Handle both L1 reorgs and batcher restarts by updating our state in case it is -// not consistent with what's on the L1. +// Update streamer state based on L1 and L2 sync status func (s *BatchStreamer[B]) Refresh(ctx context.Context, finalizedL1 eth.L1BlockRef, safeBatchNumber uint64, safeL1Origin eth.BlockID) error { s.FinalizedL1 = finalizedL1 @@ -212,23 +215,30 @@ func (s *BatchStreamer[B]) CheckBatch(ctx context.Context, batch B) (BatchValidi return BatchAccept, i } -// HOTSHOT_BLOCK_LOAD_LIMIT is the maximum number of blocks to attempt to -// load from Espresso in a single process. This helps to limit our block -// polling to a limited number of blocks within a single batched attempt. -const HOTSHOT_BLOCK_LOAD_LIMIT = 100 +// HOTSHOT_BLOCK_STREAM_LIMIT is the maximum number of blocks to attempt to +// load from Espresso in a single process using streaming API. +// This helps to limit our block polling to a limited number of blocks within +// a single batched attempt. +const HOTSHOT_BLOCK_STREAM_LIMIT = 500 + +// HOTSHOT_BLOCK_FETCH_LIMIT is the maximum number of blocks to attempt to +// load from Espresso in a single process using fetch API. +// This helps to limit our block polling to a limited number of blocks within +// a single batched attempt. +const HOTSHOT_BLOCK_FETCH_LIMIT = 100 // computeEspressoBlockHeightsRange computes the range of block heights to fetch // from Espresso. It starts from the last processed block and goes up to -// HOTSHOT_BLOCK_LOAD_LIMIT blocks ahead or the current block height, whichever +// `limit` blocks ahead or the current block height, whichever // is smaller. -func (s *BatchStreamer[B]) computeEspressoBlockHeightsRange(currentBlockHeight uint64) (start uint64, finish uint64) { +func (s *BatchStreamer[B]) computeEspressoBlockHeightsRange(currentBlockHeight uint64, limit uint64) (start uint64, finish uint64) { start = s.hotShotPos if start > 0 { // We've already processed the block in hotShotPos. In order to avoid // reprocessing the same block, we want to start from the next block. start++ } - finish = min(start+HOTSHOT_BLOCK_LOAD_LIMIT, currentBlockHeight) + finish = min(start+limit, currentBlockHeight) return start, finish } @@ -256,9 +266,29 @@ func (s *BatchStreamer[B]) Update(ctx context.Context) error { return err } + // Streaming API implementation + if !s.UseFetchApi { + // Process the remaining batches + s.processRemainingBatches(ctx) + + // We limit the number of blocks to process in a single Update call + start, finish := s.computeEspressoBlockHeightsRange(currentBlockHeight, HOTSHOT_BLOCK_STREAM_LIMIT) + + s.Log.Info("Streaming hotshot blocks", "from", start, "upTo", finish) + + // Process the new batches fetched from Espresso + if err := s.streamHotShotRange(ctx, start, finish); err != nil { + return fmt.Errorf("failed to process hotshot range: %w", err) + } + + return nil + } + + // Fetch API implementation for i := 0; ; i++ { // Fetch more batches from HotShot if available. - start, finish := s.computeEspressoBlockHeightsRange(currentBlockHeight) + start, finish := s.computeEspressoBlockHeightsRange(currentBlockHeight, HOTSHOT_BLOCK_FETCH_LIMIT) + if start > finish || (start == finish && i > 0) { // If start is equal to our finish, then that means we have // already processed all of the blocks available to us. We @@ -280,8 +310,9 @@ func (s *BatchStreamer[B]) Update(ctx context.Context) error { s.processRemainingBatches(ctx) s.Log.Info("Fetching hotshot blocks", "from", start, "upTo", finish) + // Process the new batches fetched from Espresso - if err := s.processHotShotRange(ctx, start, finish); err != nil { + if err := s.fetchHotShotRange(ctx, start, finish); err != nil { return fmt.Errorf("failed to process hotshot range: %w", err) } @@ -302,13 +333,13 @@ func (s *BatchStreamer[B]) Update(ctx context.Context) error { return nil } -// processHotShotRange is a helper method that will load all of the blocks from +// fetchHotShotRange is a helper method that will load all of the blocks from // Hotshot from start to finish, inclusive. It will process each block and // update the batch buffer with any batches found in the block. // It will also update the hotShotPos to the last block processed, in order // to effectively keep track of the last block we have successfully fetched, // and therefore processed from Hotshot. -func (s *BatchStreamer[B]) processHotShotRange(ctx context.Context, start, finish uint64) error { +func (s *BatchStreamer[B]) fetchHotShotRange(ctx context.Context, start, finish uint64) error { // Process the new batches fetched from Espresso for height := start; height <= finish; height++ { s.Log.Trace("Fetching HotShot block", "block", height) @@ -331,8 +362,72 @@ func (s *BatchStreamer[B]) processHotShotRange(ctx context.Context, start, finis continue } - s.processEspressoTransactions(ctx, height, txns) + for _, txn := range txns.Transactions { + s.processEspressoTransaction(ctx, txn) + } + } + + return nil +} + +// streamHotShotRange is a helper method that will load all transactions from +// Hotshot from start to finish, inclusive. It will process each transaction and +// update the batch buffer with any valid batches. +// It will also update the hotShotPos to the last block processed, in order +// to effectively keep track of the last block we have successfully fetched, +// and therefore processed from Hotshot. +func (s *BatchStreamer[B]) streamHotShotRange(ctx context.Context, start, finish uint64) error { + stream, err := s.EspressoClient.StreamTransactionsInNamespace(ctx, start, s.Namespace) + if err != nil { + return fmt.Errorf("failed to stream transactions: %w", err) + } + + defer func() { + go func() { + err := stream.Close() + if err != nil { + s.Log.Error("Failed to close stream", "err", err) + } + }() + }() + + // We give query service a bigger timeout on stream initialisation, as it may take awhile + timeoutCtx, cancel := context.WithTimeout(ctx, 1*time.Second) + + // Process the new batches fetched from Espresso + for { + txn, err := stream.Next(timeoutCtx) + cancel() + + if err != nil { + // Don't error out on timeout, most likely it just indicates that + // next transaction isn't available yet + if timeoutCtx.Err() != nil { + s.Log.Info("Stream timed out") + return nil + } + return fmt.Errorf("failed to fetch next transaction: %w", err) + } + + s.Log.Warn("Fetched Transaction", "block", txn.BlockHeight, "hash", txn.Hash) + + if txn.BlockHeight > finish { + break + } + + // We want to keep track of the latest block we have fully processed. + // This is essential for ensuring we don't unnecessarily keep + // refetching the same blocks that we have already processed. + // This should ensure that we keep moving forward and consuming + // from the Espresso Blocks without missing any blocks. + s.hotShotPos = txn.BlockHeight - 1 + + s.processEspressoTransaction(ctx, txn.Transaction.Payload) + + // Set up smaller timeout for subsequent iterations + timeoutCtx, cancel = context.WithTimeout(ctx, 300*time.Millisecond) } + cancel() return nil } @@ -382,48 +477,40 @@ func (s *BatchStreamer[B]) processRemainingBatches(ctx context.Context) { } } -// processEspressoTransactions is a helper method that encapsulates the logic of +// processEspressoTransaction is a helper method that encapsulates the logic of // processing batches from the transactions in a block fetched from Espresso. -func (s *BatchStreamer[B]) processEspressoTransactions(ctx context.Context, i uint64, txns espressoClient.TransactionsInBlock) { - for _, transaction := range txns.Transactions { - batch, err := s.UnmarshalBatch(transaction) - if err != nil { - s.Log.Warn("Dropping batch with invalid transaction data", "error", err) - continue - } - - validity, pos := s.CheckBatch(ctx, *batch) - - switch validity { +func (s *BatchStreamer[B]) processEspressoTransaction(ctx context.Context, transaction espressoCommon.Bytes) { + batch, err := s.UnmarshalBatch(transaction) + if err != nil { + s.Log.Warn("Dropping batch with invalid transaction data", "error", err) + return + } - case BatchDrop: - s.Log.Info("Dropping batch", batch) - continue + validity, pos := s.CheckBatch(ctx, *batch) - case BatchPast: - s.Log.Info("Batch already processed. Skipping", "batch", (*batch).Number()) - continue + switch validity { + case BatchDrop: + s.Log.Info("Dropping batch", batch) - case BatchUndecided: - hash := (*batch).Hash() - if existingBatch, ok := s.RemainingBatches[hash]; ok { - s.Log.Warn("Batch already in buffer", "batch", existingBatch) - } - s.RemainingBatches[hash] = *batch - continue + case BatchPast: + s.Log.Info("Batch already processed. Skipping", "batch", (*batch).Number()) - case BatchAccept: - s.Log.Info("Inserting accepted batch") - - case BatchFuture: - // The function CheckBatch is not expected to return BatchFuture so if we enter this case there is a problem. - s.Log.Error("Remaining list", "BatchFuture validity not expected for batch", batch) - continue + case BatchUndecided: + hash := (*batch).Hash() + if existingBatch, ok := s.RemainingBatches[hash]; ok { + s.Log.Warn("Batch already in buffer", "batch", existingBatch) } + s.RemainingBatches[hash] = *batch - s.Log.Trace("Inserting batch into buffer", "batch", batch) + case BatchAccept: + s.Log.Info("Inserting batch into buffer", "batch", batch) s.BatchBuffer.Insert(*batch, pos) + + case BatchFuture: + // The function CheckBatch is not expected to return BatchFuture so if we enter this case there is a problem. + s.Log.Error("Remaining list", "BatchFuture validity not expected for batch", batch) } + } // UnmarshalBatch implements EspressoStreamerIFace diff --git a/espresso/streamer_test.go b/espresso/streamer_test.go index ba9902e39e2..d92ce1a3399 100644 --- a/espresso/streamer_test.go +++ b/espresso/streamer_test.go @@ -3,6 +3,7 @@ package espresso_test import ( "context" "encoding/binary" + "encoding/json" "errors" "log/slog" "math/big" @@ -177,6 +178,73 @@ func (ErrorNotFound) Error() string { // that a requested resource was not found. var ErrNotFound error = ErrorNotFound{} +type MockTransactionStream struct { + pos uint64 + subPos uint64 + end uint64 + namespace uint64 + source *MockStreamerSource +} + +func (ms *MockTransactionStream) Next(ctx context.Context) (*espressoCommon.TransactionQueryData, error) { + raw, err := ms.NextRaw(ctx) + if err != nil { + return nil, err + } + var transaction espressoCommon.TransactionQueryData + if err := json.Unmarshal(raw, &transaction); err != nil { + return nil, err + } + return &transaction, nil +} + +func (ms *MockTransactionStream) NextRaw(ctx context.Context) (json.RawMessage, error) { + for { + transactions, err := ms.source.FetchTransactionsInBlock(ctx, ms.pos, ms.namespace) + if err != nil { + // We will return error on NotFound as well to speed up tests. + // More faithful imitation of HotShot streaming API would be to hang + // until we receive new transactions, but that would slow down some + // tests significantly, because streamer would wait for full timeout + // threshold here before finishing update. + return nil, err + } + if len(transactions.Transactions) > int(ms.subPos) { + transaction := &espressoCommon.TransactionQueryData{ + BlockHeight: ms.pos, + Index: ms.subPos, + Transaction: espressoCommon.Transaction{ + Payload: transactions.Transactions[int(ms.subPos)], + Namespace: ms.namespace, + }, + } + ms.subPos += 1 + return json.Marshal(transaction) + } else { + ms.subPos = 0 + ms.pos += 1 + } + } +} + +func (ms *MockTransactionStream) Close() error { + return nil +} + +func (m *MockStreamerSource) StreamTransactionsInNamespace(ctx context.Context, height uint64, namespace uint64) (espressoClient.Stream[espressoCommon.TransactionQueryData], error) { + if m.LatestEspHeight < height { + return nil, ErrNotFound + } + + return &MockTransactionStream{ + pos: height, + subPos: 0, + end: m.LatestEspHeight, + namespace: namespace, + source: m, + }, nil +} + func (m *MockStreamerSource) FetchTransactionsInBlock(ctx context.Context, blockHeight uint64, namespace uint64) (espressoClient.TransactionsInBlock, error) { if m.LatestEspHeight < blockHeight { return espressoClient.TransactionsInBlock{}, ErrNotFound diff --git a/go.mod b/go.mod index 79c37299b40..448bc6cca84 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.7 require ( github.com/BurntSushi/toml v1.5.0 - github.com/EspressoSystems/espresso-network/sdks/go v0.2.1 + github.com/EspressoSystems/espresso-network/sdks/go v0.3.2 github.com/Masterminds/semver/v3 v3.3.1 github.com/andybalholm/brotli v1.1.0 github.com/bmatcuk/doublestar/v4 v4.8.1 @@ -62,9 +62,9 @@ require ( golang.org/x/crypto v0.32.0 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c golang.org/x/mod v0.22.0 - golang.org/x/sync v0.10.0 - golang.org/x/term v0.28.0 - golang.org/x/text v0.21.0 + golang.org/x/sync v0.11.0 + golang.org/x/term v0.29.0 + golang.org/x/text v0.22.0 golang.org/x/time v0.10.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -90,6 +90,7 @@ require ( github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/coder/websocket v1.8.13 // indirect github.com/consensys/bavard v0.1.27 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -293,3 +294,5 @@ exclude ( github.com/kataras/iris/v12 v12.2.0 github.com/kataras/iris/v12 v12.2.11 ) + +replace github.com/EspressoSystems/espresso-network/sdks/go => github.com/EspressoSystems/espresso-network/sdks/go v0.3.2-0.20251007163344-504ab95333c0 diff --git a/go.sum b/go.sum index c5e7fbb5c53..9fbb282d0a5 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/EspressoSystems/espresso-network/sdks/go v0.2.1 h1:lE+2kUIQhKAw78jlTz5L92gYywRD5uydWskwlLn3YwA= -github.com/EspressoSystems/espresso-network/sdks/go v0.2.1/go.mod h1:aJX3rhV7d3QQ3dvmEFIKDfQvSFP9aUnFNENGpXPwELM= +github.com/EspressoSystems/espresso-network/sdks/go v0.3.2-0.20251007163344-504ab95333c0 h1:eHvi82K+tvqH3IQhikusQAiM/N7Rx3dVs8D8CFHlClQ= +github.com/EspressoSystems/espresso-network/sdks/go v0.3.2-0.20251007163344-504ab95333c0/go.mod h1:kaxR08mJb5Mijy7a2RhWCIWOevFI4PcXwDkzoEbsVTk= github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -132,6 +132,8 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE= +github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/consensys/bavard v0.1.27 h1:j6hKUrGAy/H+gpNrpLU3I26n1yc+VMGmd6ID5+gAhOs= github.com/consensys/bavard v0.1.27/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/gnark-crypto v0.16.0 h1:8Dl4eYmUWK9WmlP1Bj6je688gBRJCJbT8Mw4KoTAawo= @@ -1027,8 +1029,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1100,8 +1102,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -1114,8 +1116,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/justfile b/justfile index 7c843b43042..53f20fc02eb 100644 --- a/justfile +++ b/justfile @@ -44,7 +44,7 @@ espresso-enclave-tests: ESPRESSO_RUN_ENCLAVE_TESTS=true go test -timeout={{espresso_tests_timeout}} -p=1 -count=1 ./espresso/enclave-tests/... -IMAGE_NAME := "ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-colorful-snake" +IMAGE_NAME := "ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-fix-cors" remove-espresso-containers: docker remove --force $(docker ps -q --filter ancestor={{IMAGE_NAME}}) diff --git a/kurtosis-devnet/enclaver/Dockerfile b/kurtosis-devnet/enclaver/Dockerfile index 57e6e36a70d..028fe613aaa 100644 --- a/kurtosis-devnet/enclaver/Dockerfile +++ b/kurtosis-devnet/enclaver/Dockerfile @@ -75,26 +75,6 @@ COPY --from=cannon-builder-v1-2-0 /usr/local/bin/cannon-3 ./cannon/multicannon/e RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build cd cannon && make cannon \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$CANNON_VERSION" -# Download and build the espresso-network go crypto helper library -FROM --platform=$BUILDPLATFORM rust:1.84.1-alpine3.20 AS rust-builder -ARG ESPRESSO_SDK_VER=v0.2.1 -# Download the prebuilt static libraries for both archs (change arch as needed) -RUN apk add --no-cache curl -RUN set -e; \ - mkdir -p /libespresso; \ - cd /libespresso; \ - # Download .so and .sha256 for aarch64 - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so; \ - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so.sha256; \ - cat libespresso_crypto_helper-aarch64-unknown-linux-gnu.so.sha256 | \ - awk '{print $1 " libespresso_crypto_helper-aarch64-unknown-linux-gnu.so"}' | \ - sha256sum -c - ; \ - # Download .so and .sha256 for x86_64 - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so; \ - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so.sha256; \ - cat libespresso_crypto_helper-x86_64-unknown-linux-gnu.so.sha256 | \ - awk '{print $1 " libespresso_crypto_helper-x86_64-unknown-linux-gnu.so"}' | \ - sha256sum -c - # We don't use the golang image for batcher because it doesn't play well with CGO FROM --platform=$BUILDPLATFORM alpine:3.20 AS op-batcher-builder @@ -108,8 +88,16 @@ RUN curl -L https://github.com/casey/just/releases/download/$(yq '.tools.just' m # Go sources COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum -# Copy rust libs for dynamic linking -COPY --from=rust-builder /libespresso/* /lib +# Fetch rust libs for dynamic linking +ARG ESPRESSO_SDK_VER=0.3.2 +ARG ESPRESSO_SDK_HELPER_HASH_AARCH64=ec6ce7b37edd173206ad338c84a6a771a0e9dc8b184081af7440ebfc0c531a71 +ARG ESPRESSO_SDK_HELPER_HASH_X86_64=49c50949ec1acf52107cb190c90911e05cc9c4e9d72dd7455496163443760b92 +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_AARCH64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so \ + /lib/ +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_X86_64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so \ + /lib/ # Warm-up the cache WORKDIR /app RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download diff --git a/kurtosis-devnet/enclaver/Dockerfile.nonEnclave b/kurtosis-devnet/enclaver/Dockerfile.nonEnclave index 3ed65a7aa0f..72f140db01c 100644 --- a/kurtosis-devnet/enclaver/Dockerfile.nonEnclave +++ b/kurtosis-devnet/enclaver/Dockerfile.nonEnclave @@ -75,27 +75,6 @@ COPY --from=cannon-builder-v1-2-0 /usr/local/bin/cannon-3 ./cannon/multicannon/e RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build cd cannon && make cannon \ GOOS=$TARGETOS GOARCH=$TARGETARCH GITCOMMIT=$GIT_COMMIT GITDATE=$GIT_DATE VERSION="$CANNON_VERSION" -# Download and build the espresso-network go crypto helper library -FROM --platform=$BUILDPLATFORM rust:1.84.1-alpine3.20 AS rust-builder -ARG ESPRESSO_SDK_VER=v0.2.1 -# Download the prebuilt static libraries for both archs (change arch as needed) -RUN apk add --no-cache curl -RUN set -e; \ - mkdir -p /libespresso; \ - cd /libespresso; \ - # Download .so and .sha256 for aarch64 - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so; \ - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so.sha256; \ - cat libespresso_crypto_helper-aarch64-unknown-linux-gnu.so.sha256 | \ - awk '{print $1 " libespresso_crypto_helper-aarch64-unknown-linux-gnu.so"}' | \ - sha256sum -c - ; \ - # Download .so and .sha256 for x86_64 - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so; \ - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so.sha256; \ - cat libespresso_crypto_helper-x86_64-unknown-linux-gnu.so.sha256 | \ - awk '{print $1 " libespresso_crypto_helper-x86_64-unknown-linux-gnu.so"}' | \ - sha256sum -c - - # We don't use the golang image for batcher because it doesn't play well with CGO FROM --platform=$BUILDPLATFORM alpine:3.20 AS op-batcher-builder ARG OP_BATCHER_VERSION=v0.0.0 @@ -108,8 +87,16 @@ RUN curl -L https://github.com/casey/just/releases/download/$(yq '.tools.just' m # Go sources COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum -# Copy rust libs for dynamic linking -COPY --from=rust-builder /libespresso/* /lib +# Fetch rust libs for dynamic linking +ARG ESPRESSO_SDK_VER=0.3.2 +ARG ESPRESSO_SDK_HELPER_HASH_AARCH64=ec6ce7b37edd173206ad338c84a6a771a0e9dc8b184081af7440ebfc0c531a71 +ARG ESPRESSO_SDK_HELPER_HASH_X86_64=49c50949ec1acf52107cb190c90911e05cc9c4e9d72dd7455496163443760b92 +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_AARCH64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so \ + /lib/ +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_X86_64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so \ + /lib/ # Warm-up the cache WORKDIR /app RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download diff --git a/op-batcher/batcher/espresso.go b/op-batcher/batcher/espresso.go index fc0f9bec107..1b71a91cb97 100644 --- a/op-batcher/batcher/espresso.go +++ b/op-batcher/batcher/espresso.go @@ -398,6 +398,7 @@ func (s *espressoTransactionSubmitter) handleVerifyReceiptJobResponse() { // We're done with this job and transaction, we have successfully // confirmed that the transaction was submitted to Espresso + log.Info("Transaction confirmed on Espresso", "hash", jobResp.job.transaction.transaction.Commit()) } } @@ -774,7 +775,7 @@ func (l *BatchSubmitter) espressoBatchLoadingLoop(ctx context.Context, wg *sync. continue } - l.Log.Trace( + l.Log.Info( "Received block from Espresso", "blockNr", block.NumberU64(), "blockHash", block.Hash(), diff --git a/ops/docker/op-stack-go/Dockerfile b/ops/docker/op-stack-go/Dockerfile index 2ea1ea9fc57..ea0f94e8db3 100644 --- a/ops/docker/op-stack-go/Dockerfile +++ b/ops/docker/op-stack-go/Dockerfile @@ -32,7 +32,7 @@ ARG TARGETARCH # Install yq RUN wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$TARGETARCH -O /usr/local/bin/yq && \ - chmod +x /usr/local/bin/yq + chmod +x /usr/local/bin/yq # Install versioned toolchain COPY ./mise.toml . @@ -74,28 +74,6 @@ FROM --platform=$BUILDPLATFORM us-docker.pkg.dev/oplabs-tools-artifacts/images/c FROM --platform=$BUILDPLATFORM us-docker.pkg.dev/oplabs-tools-artifacts/images/cannon:v1.2.0 AS cannon-builder-v1-2-0 FROM --platform=$BUILDPLATFORM us-docker.pkg.dev/oplabs-tools-artifacts/images/cannon:v1.3.0 AS cannon-builder-v1-3-0 - -# Download and build the espresso-network go crypto helper library -FROM --platform=$BUILDPLATFORM rust:1.84.1-alpine3.20 AS rust-builder -ARG ESPRESSO_SDK_VER=v0.2.1 -# Download the prebuilt static libraries for both archs (change arch as needed) -RUN apk add --no-cache curl -RUN set -e; \ - mkdir -p /libespresso; \ - cd /libespresso; \ - # Download .so and .sha256 for aarch64 - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so; \ - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so.sha256; \ - cat libespresso_crypto_helper-aarch64-unknown-linux-gnu.so.sha256 | \ - awk '{print $1 " libespresso_crypto_helper-aarch64-unknown-linux-gnu.so"}' | \ - sha256sum -c - ; \ - # Download .so and .sha256 for x86_64 - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so; \ - curl -L -O https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so.sha256; \ - cat libespresso_crypto_helper-x86_64-unknown-linux-gnu.so.sha256 | \ - awk '{print $1 " libespresso_crypto_helper-x86_64-unknown-linux-gnu.so"}' | \ - sha256sum -c - - # We don't use the golang image for batcher & op-node because it doesn't play well with CGO FROM --platform=$BUILDPLATFORM alpine:3.20 AS op-cgo-builder ARG OP_BATCHER_VERSION=v0.0.0 @@ -108,8 +86,16 @@ RUN curl -L https://github.com/casey/just/releases/download/$(yq '.tools.just' m # Go sources COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum -# Copy rust libs for dynamic linking -COPY --from=rust-builder /libespresso/* /lib +# Fetch rust libs for dynamic linking +ARG ESPRESSO_SDK_VER=0.3.2 +ARG ESPRESSO_SDK_HELPER_HASH_AARCH64=ec6ce7b37edd173206ad338c84a6a771a0e9dc8b184081af7440ebfc0c531a71 +ARG ESPRESSO_SDK_HELPER_HASH_X86_64=49c50949ec1acf52107cb190c90911e05cc9c4e9d72dd7455496163443760b92 +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_AARCH64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so \ + /lib/ +ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_X86_64} \ + https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so \ + /lib/ # Warm-up the cache WORKDIR /app RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download