Streaming streamer

.circleci/config.yml

@@ -993,7 +993,7 @@ jobs:
       - run:
           name: download espresso-network go sdk
           command: |
-            ver=$(grep "github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}')
+            ver=$(grep "^\s*github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}')
             url="https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ver}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so"
             mkdir -p /home/circleci/local-lib
             wget $url -O /home/circleci/local-lib/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so
@@ -1066,7 +1066,7 @@ jobs:
       - run:
           name: download espresso-network go sdk
           command: |
-            ver=$(grep "github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}')
+            ver=$(grep "^\s*github.com/EspressoSystems/espresso-network/sdks/go" go.mod | awk '{print $2}')
             url="https://github.com/EspressoSystems/espresso-network/releases/download/sdks%2Fgo%2F${ver}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so"
             mkdir -p /home/circleci/local-lib
             wget $url -O /home/circleci/local-lib/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so

espresso/docker-compose.yml

@@ -493,7 +493,7 @@ services:
       OP_CHALLENGER_TRACE_TYPE: permissioned
 
   espresso-dev-node:
-    image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-colorful-snake
+    image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-fix-cors
     depends_on:
       l1-geth:
         condition: service_healthy

espresso/docker/op-geth/Dockerfile

@@ -6,25 +6,6 @@ ARG TARGETARCH
 ARG GIT_COMMIT
 ARG GIT_DATE
 
-# Rust builder for Espresso crypto libraries
-FROM rust:1.88.0-alpine3.22 AS rust-builder
-# TODO: Check the hash of the Espresso GO library when switch to the new one.
-# <https://app.asana.com/1/1208976916964769/project/1209392461754458/task/1210755885228839?focus=true>
-ARG ESPRESSO_NETWORK_GO_VER=0.0.34
-RUN apk add perl make openssl-dev musl-dev gcc
-ADD https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$ESPRESSO_NETWORK_GO_VER.tar.gz /source.tgz
-RUN tar -oxzf /source.tgz
-WORKDIR /espresso-network-go-$ESPRESSO_NETWORK_GO_VER
-RUN --mount=type=cache,target=/usr/local/cargo/registry \
-  --mount=type=cache,target=/usr/local/cargo/git/db \
-  --mount=type=cache,target=/espresso-network-go/verification/rust/target \
-  cargo build --release  --locked  --manifest-path ./verification/rust/Cargo.toml
-RUN mkdir -p /libespresso
-RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \
-  /libespresso/libespresso_crypto_helper-aarch64-unknown-linux-gnu.a
-RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \
-  /libespresso/libespresso_crypto_helper-x86_64-unknown-linux-gnu.a
-
 # CGO builder for components that need Espresso crypto linking
 FROM alpine:3.22 AS op-cgo-builder
 # Install dependencies
@@ -33,11 +14,19 @@ RUN apk add musl-dev gcc go g++ curl tar gzip make gcc linux-headers git jq bash
 COPY ./mise.toml .
 RUN curl -L https://github.com/casey/just/releases/download/$(yq '.tools.just' mise.toml)/just-$(yq '.tools.just' mise.toml)-x86_64-unknown-linux-musl.tar.gz | \
   tar xz -C /usr/local/bin just
+# Fetch rust libs for dynamic linking
+ARG ESPRESSO_SDK_VER=0.3.2
+ARG ESPRESSO_SDK_HELPER_HASH_AARCH64=ec6ce7b37edd173206ad338c84a6a771a0e9dc8b184081af7440ebfc0c531a71
+ARG ESPRESSO_SDK_HELPER_HASH_X86_64=49c50949ec1acf52107cb190c90911e05cc9c4e9d72dd7455496163443760b92
+ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_AARCH64} \
+  https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so \
+  /lib/
+ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_X86_64} \
+  https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so \
+  /lib/
 # Go sources
 COPY ./go.mod /app/go.mod
 COPY ./go.sum /app/go.sum
-# Copy rust libs for dynamic linking
-COPY --from=rust-builder /libespresso/* /lib
 # Warm-up the cache
 WORKDIR /app
 RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download

espresso/docker/op-stack/Dockerfile

@@ -39,25 +39,6 @@ COPY . /app
 ARG GIT_COMMIT
 ARG GIT_DATE
 
-# Rust builder for Espresso crypto libraries
-FROM rust:1.88.0-alpine3.22 AS rust-builder
-# TODO: Check the hash of the Espresso GO library when switch to the new one.
-# <https://app.asana.com/1/1208976916964769/project/1209392461754458/task/1210755885228839?focus=true>
-ARG ESPRESSO_NETWORK_GO_VER=0.0.34
-RUN apk add perl make openssl-dev musl-dev gcc
-ADD https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$ESPRESSO_NETWORK_GO_VER.tar.gz /source.tgz
-RUN tar -oxzf /source.tgz
-WORKDIR /espresso-network-go-$ESPRESSO_NETWORK_GO_VER
-RUN --mount=type=cache,target=/usr/local/cargo/registry \
-  --mount=type=cache,target=/usr/local/cargo/git/db \
-  --mount=type=cache,target=/espresso-network-go/verification/rust/target \
-  cargo build --release  --locked  --manifest-path ./verification/rust/Cargo.toml
-RUN mkdir -p /libespresso
-RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \
-  /libespresso/libespresso_crypto_helper-aarch64-unknown-linux-gnu.a
-RUN cp ./verification/rust/target/release/libespresso_crypto_helper.a \
-  /libespresso/libespresso_crypto_helper-x86_64-unknown-linux-gnu.a
-
 # CGO builder for components that need Espresso crypto linking
 FROM alpine:3.22 AS op-cgo-builder
 # Install dependencies
@@ -70,11 +51,19 @@ RUN case $(uname -m) in \
   esac && \
   curl -L https://github.com/casey/just/releases/download/$(yq '.tools.just' mise.toml)/just-$(yq '.tools.just' mise.toml)-$JUST_ARCH-unknown-linux-musl.tar.gz | \
   tar xz -C /usr/local/bin just
+# Fetch rust libs for dynamic linking
+ARG ESPRESSO_SDK_VER=0.3.2
+ARG ESPRESSO_SDK_HELPER_HASH_AARCH64=ec6ce7b37edd173206ad338c84a6a771a0e9dc8b184081af7440ebfc0c531a71
+ARG ESPRESSO_SDK_HELPER_HASH_X86_64=49c50949ec1acf52107cb190c90911e05cc9c4e9d72dd7455496163443760b92
+ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_AARCH64} \
+  https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-aarch64-unknown-linux-gnu.so \
+  /lib/
+ADD --checksum=sha256:${ESPRESSO_SDK_HELPER_HASH_X86_64} \
+  https://github.com/EspressoSystems/espresso-network/releases/download/sdks/go/v${ESPRESSO_SDK_VER}/libespresso_crypto_helper-x86_64-unknown-linux-gnu.so \
+  /lib/
 # Go sources
 COPY ./go.mod /app/go.mod
 COPY ./go.sum /app/go.sum
-# Copy rust libs for dynamic linking
-COPY --from=rust-builder /libespresso/* /lib
 # Warm-up the cache
 WORKDIR /app
 RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go mod download

espresso/environment/12_enforce_majority_rule_test.go

@@ -5,9 +5,11 @@ import (
 	"math/big"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/coder/websocket"
 	env "github.com/ethereum-optimism/optimism/espresso/environment"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth"
 	"github.com/ethereum-optimism/optimism/op-e2e/system/e2esys"
@@ -24,16 +26,26 @@ const NO_ERROR_EXPECTED = false
 // @param numBadUrls N as mentioned in the above description
 // @param expectedError if set to true, we expect a timeout error as the L2 cannot make progress. Otherwise, we expect no error at all.
 func runWithMultiClient(t *testing.T, numGoodUrls int, numBadUrls int, expectedError bool) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		http.Error(w, "Hello", http.StatusOK)
+		if strings.Contains(r.URL.Path, "stream") {
+			conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{})
+			require.NoError(t, err)
+
+			defer conn.Close(websocket.StatusGoingAway, "Bye")
+
+			err = conn.Write(ctx, websocket.MessageText, []byte("Hello"))
+			require.NoError(t, err)
+
+		} else {
+			http.Error(w, "Hello", http.StatusOK)
+		}
 	}))
 
 	badServerUrl := server.URL
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	launcher := new(env.EspressoDevNodeLauncherDocker)
 
 	system, devNode, err := launcher.StartE2eDevnet(ctx, t, env.SetEspressoUrls(numGoodUrls, numBadUrls, badServerUrl))
@@ -57,7 +69,7 @@ func runWithMultiClient(t *testing.T, numGoodUrls int, numBadUrls int, expectedE
 	blockNumber := int64(2)
 
 	// Check the caff node can/cannot make progress
-	_, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), caffClient, 30*time.Second)
+	_, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), caffClient, 60*time.Second)
 
 	if expectedError {
 		require.Error(t, err, "The L2 should not be progressing")
@@ -66,7 +78,7 @@ func runWithMultiClient(t *testing.T, numGoodUrls int, numBadUrls int, expectedE
 	}
 
 	// Check the l2Verif node can/cannot make progress
-	_, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), l2Verif, 30*time.Second)
+	_, err = geth.WaitForBlockToBeSafe(big.NewInt(blockNumber), l2Verif, 60*time.Second)
 	if expectedError {
 		require.Error(t, err, "The L2 should not be progressing")
 	} else {

espresso/environment/optitmism_espresso_test_helpers.go

@@ -56,7 +56,7 @@ func init() {
 
 const ESPRESSO_LIGHT_CLIENT_ADDRESS = "0x703848f4c85f18e3acd8196c8ec91eb0b7bd0797"
 
-const ESPRESSO_DEV_NODE_DOCKER_IMAGE = "ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-colorful-snake"
+const ESPRESSO_DEV_NODE_DOCKER_IMAGE = "ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:release-fix-cors"
 
 // This is the mnemonic that we use to create the private key for deploying
 // contacts on the L1

espresso/environment/query_service_intercept.go

@@ -2,14 +2,18 @@ package environment
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"regexp"
+	"time"
 
 	espressoTaggedBase64 "github.com/EspressoSystems/espresso-network/sdks/go/tagged-base64"
 	espressoCommon "github.com/EspressoSystems/espresso-network/sdks/go/types"
+	"github.com/coder/websocket"
 	"github.com/ethereum-optimism/optimism/op-batcher/batcher"
 	"github.com/ethereum-optimism/optimism/op-e2e/system/e2esys"
 )
@@ -88,6 +92,13 @@ type proxyRequest struct {
 
 // ServeHTTP implements http.Handler
 func (p *proxyRequest) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Check if this is a websocket stream request
+	if isWebSocketStreamRequest(r) {
+		p.proxyWebSocket(w, r)
+		return
+	}
+
+	// Handle regular HTTP requests
 	defer r.Body.Close()
 	buf := new(bytes.Buffer)
 	if _, err := io.Copy(buf, r.Body); err != nil && err != io.EOF {
@@ -135,6 +146,82 @@ func (p *proxyRequest) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// proxyWebSocket handles websocket upgrade and proxying
+func (p *proxyRequest) proxyWebSocket(w http.ResponseWriter, r *http.Request) {
+	// Accept the websocket connection from the client
+	clientConn, err := websocket.Accept(w, r, &websocket.AcceptOptions{
+		InsecureSkipVerify: true,
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer clientConn.Close(websocket.StatusInternalError, "proxy error")
+
+	// Create websocket URL for the backend
+	backendURL := p.baseURL
+	if backendURL.Scheme == "https" {
+		backendURL.Scheme = "wss"
+	} else {
+		backendURL.Scheme = "ws"
+	}
+	backendURL.Path = r.URL.Path
+	backendURL.RawQuery = r.URL.RawQuery
+
+	// Connect to the backend websocket
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	//nolint:bodyclose // Not applicable to coder/websocket. From the websocket.Dial docs: "You never need to close resp.Body yourself."
+	backendConn, _, err := websocket.Dial(ctx, backendURL.String(), &websocket.DialOptions{})
+	if err != nil {
+		clientConn.Close(websocket.StatusInternalError, "backend connection failed")
+		return
+	}
+	defer backendConn.Close(websocket.StatusNormalClosure, "")
+
+	// Proxy messages bidirectionally
+	ctx, cancel = context.WithCancel(context.Background())
+	defer cancel()
+
+	// Client to backend
+	go func() {
+		defer cancel()
+		for {
+			msgType, data, err := clientConn.Read(ctx)
+			if err != nil {
+				return
+			}
+			err = backendConn.Write(ctx, msgType, data)
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	// Backend to client
+	go func() {
+		defer cancel()
+		for {
+			msgType, data, err := backendConn.Read(ctx)
+			if err != nil {
+				return
+			}
+			err = clientConn.Write(ctx, msgType, data)
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	// Wait until context is cancelled (one of the goroutines finished)
+	<-ctx.Done()
+
+	// Close connections gracefully
+	clientConn.Close(websocket.StatusNormalClosure, "")
+	backendConn.Close(websocket.StatusNormalClosure, "")
+}
+
 // fakeSubmitTransactionSuccess is a simple HTTP handler that simulates a
 // successful transaction submission by returning a fake commit hash.
 type fakeSubmitTransactionSuccess struct{}
@@ -304,6 +391,13 @@ func isSubmitTransactionRequest(r *http.Request) bool {
 		requestMatchesPath(r, http.MethodPost, stringEquals("/v0/submit/submit"))
 }
 
+// isWebSocketStreamRequest checks if the request is a websocket request
+// matching the pattern "vN/stream/*" where N is an integer.
+func isWebSocketStreamRequest(r *http.Request) bool {
+	matched, _ := regexp.MatchString(`/stream/`, r.URL.Path)
+	return matched
+}
+
 // DecideHowToHandleRequest implements InterceptHandlerDecider
 func (d *randomRollFakeSubmitTransactionSuccess) DecideHowToHandleRequest(w http.ResponseWriter, r *http.Request) InterceptHandleDecision {
 	if isSubmitTransactionRequest(r) {