From 9fce58276637e71d611371163b040823ba1b4462 Mon Sep 17 00:00:00 2001
From: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Date: Wed, 22 Apr 2026 09:43:56 +0000
Subject: [PATCH 1/6] fix(bench): require local benchmark data

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019db48c-8b95-744f-aad9-140ad31275fc
Co-authored-by: Amp <amp@ampcode.com>
---
 .github/scripts/bench-reth-build.sh    | 121 +++------------
 .github/scripts/bench-reth-local.sh    |  37 ++---
 .github/scripts/bench-reth-snapshot.sh | 197 ++++---------------------
 .github/workflows/bench-scheduled.yml  |  33 +----
 .github/workflows/bench.yml            | 132 +++++------------
 5 files changed, 101 insertions(+), 419 deletions(-)

diff --git a/.github/scripts/bench-reth-build.sh b/.github/scripts/bench-reth-build.sh
index 56a389a9f8d..5fa9a76522b 100755
--- a/.github/scripts/bench-reth-build.sh
+++ b/.github/scripts/bench-reth-build.sh
@@ -1,24 +1,21 @@
 #!/usr/bin/env bash
 #
-# Builds (or fetches from cache) reth binaries for benchmarking.
+# Builds reth binaries for benchmarking from local source only.
 #
-# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit> [branch-sha]
+# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit>
 #
-#   baseline — build/fetch the baseline binary at <commit> (merge-base)
+#   baseline — build the baseline binary at <commit> (merge-base)
 #              source-dir must be checked out at <commit>
-#   feature  — build/fetch the candidate binary + reth-bench at <commit>
+#   feature  — build the candidate binary + reth-bench at <commit>
 #              source-dir must be checked out at <commit>
-#              optional branch-sha is the PR head commit for cache key
 #
 # Outputs:
 #   baseline: <source-dir>/target/profiling/reth (or reth-bb if BENCH_BIG_BLOCKS=true)
 #   feature:  <source-dir>/target/profiling/reth (or reth-bb), reth-bench installed to cargo bin
 #
-# Required: mc (MinIO client) with a configured alias
 # Optional env: BENCH_BIG_BLOCKS (true/false) — build reth-bb instead of reth
 set -euxo pipefail
 
-MC="mc"
 MODE="$1"
 SOURCE_DIR="$2"
 COMMIT="$3"
@@ -42,110 +39,38 @@ if [ "${BENCH_TRACY:-off}" != "off" ]; then
   EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes"
 fi
 
-# Cache suffix: hash of features+rustflags so different build configs get separate cache entries
-if [ -n "$EXTRA_FEATURES" ] || [ -n "$EXTRA_RUSTFLAGS" ]; then
-  BUILD_SUFFIX="-$(echo "${EXTRA_FEATURES}${EXTRA_RUSTFLAGS}" | sha256sum | cut -c1-12)"
-else
-  BUILD_SUFFIX=""
-fi
+# Build the requested node binary with the benchmark profile.
+build_node_binary() {
+  local features_arg=""
+  local workspace_arg=""
 
-# Verify a cached reth binary was built from the expected commit.
-# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
-verify_binary() {
-  local binary="$1" expected_commit="$2"
-  local version binary_sha
-  version=$("$binary" --version 2>/dev/null) || return 1
-  binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
-  if [ -z "$binary_sha" ]; then
-    echo "Warning: could not extract commit SHA from version output"
-    return 1
+  cd "$SOURCE_DIR"
+  if [ -n "$EXTRA_FEATURES" ]; then
+    # --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
+    features_arg="--features ${EXTRA_FEATURES}"
+    workspace_arg="--workspace"
   fi
-  if [ "$binary_sha" = "$expected_commit" ]; then
-    return 0
-  fi
-  echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
-  return 1
-}
 
-upload_cache_artifact() {
-  local source="$1" destination="$2"
-  if ! $MC cp "$source" "$destination"; then
-    echo "::warning::Failed to upload binary cache artifact to ${destination}; continuing without remote cache write"
-  fi
+  # shellcheck disable=SC2086
+  RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
+    cargo build --profile profiling $NODE_PKG $workspace_arg $features_arg
 }
 
 case "$MODE" in
   baseline|main)
-    BUCKET="minio/reth-binaries/${COMMIT}${BUILD_SUFFIX}"
-    mkdir -p "${SOURCE_DIR}/target/profiling"
-
-    CACHE_VALID=false
-    if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null; then
-      echo "Cache hit for baseline (${COMMIT}), downloading ${NODE_BIN}..."
-      if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
-         chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
-         verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
-        CACHE_VALID=true
-      else
-        echo "Cached baseline binary is stale or download failed, rebuilding..."
-      fi
-    fi
-    if [ "$CACHE_VALID" = false ]; then
-      echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
-      cd "${SOURCE_DIR}"
-      FEATURES_ARG=""
-      WORKSPACE_ARG=""
-      if [ -n "$EXTRA_FEATURES" ]; then
-        # --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
-        FEATURES_ARG="--features ${EXTRA_FEATURES}"
-        WORKSPACE_ARG="--workspace"
-      fi
-      # shellcheck disable=SC2086
-      RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
-        cargo build --profile profiling $NODE_PKG $WORKSPACE_ARG $FEATURES_ARG
-      upload_cache_artifact "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
-    fi
+    echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
+    build_node_binary
     ;;
 
   feature|branch)
-    BRANCH_SHA="${4:-$COMMIT}"
-    BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
-
-    CACHE_VALID=false
-    if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null && $MC stat --no-list "${BUCKET}/reth-bench" &>/dev/null; then
-      echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
-      mkdir -p "${SOURCE_DIR}/target/profiling"
-      if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
-         $MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench && \
-         chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" /home/ubuntu/.cargo/bin/reth-bench && \
-         verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
-        CACHE_VALID=true
-      else
-        echo "Cached feature binary is stale or download failed, rebuilding..."
-      fi
-    fi
-    if [ "$CACHE_VALID" = false ]; then
-      echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
-      cd "${SOURCE_DIR}"
-      rustup show active-toolchain || rustup default stable
-      if [ -n "$EXTRA_FEATURES" ]; then
-        # Can't use `make profiling` when adding features; build explicitly
-        # --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
-        RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
-          cargo build --profile profiling --workspace $NODE_PKG --features "${EXTRA_FEATURES}"
-      else
-        # shellcheck disable=SC2086
-        RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
-          cargo build --profile profiling $NODE_PKG
-      fi
-      make install-reth-bench
-      upload_cache_artifact "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
-      upload_cache_artifact "$(which reth-bench)" "${BUCKET}/reth-bench"
-    fi
+    echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
+    rustup show active-toolchain || rustup default stable
+    build_node_binary
+    make -C "$SOURCE_DIR" install-reth-bench
     ;;
 
   *)
-    echo "Usage: $0 <baseline|feature> <source-dir> <commit> [branch-sha]"
+    echo "Usage: $0 <baseline|feature> <source-dir> <commit>"
     exit 1
     ;;
 esac
diff --git a/.github/scripts/bench-reth-local.sh b/.github/scripts/bench-reth-local.sh
index 1f75498265e..4d2565592d8 100755
--- a/.github/scripts/bench-reth-local.sh
+++ b/.github/scripts/bench-reth-local.sh
@@ -2,7 +2,7 @@
 #
 # local-reth-bench.sh — Run the reth Engine API benchmark locally.
 #
-# Replicates the CI bench.yml workflow (build, snapshot, system tuning,
+# Replicates the CI bench.yml workflow (build, local snapshot validation, system tuning,
 # interleaved B-F-F-B execution, summary, charts) without any GitHub
 # Actions glue (no PR comments, no artifact upload, no Slack).
 #
@@ -21,15 +21,17 @@
 # Requires: the reth repo at RETH_REPO (default: ~/reth)
 #
 # Dependencies (install before first run):
-#   mc (MinIO client), schelk, cpupower, taskset, stdbuf, python3, curl,
-#   make, uv, pzstd, jq, Rust toolchain (cargo/rustup)
+#   schelk, cpupower, taskset, stdbuf, python3, curl,
+#   make, uv, jq, Rust toolchain (cargo/rustup)
+# Optional:
+#   mc for Tracy profile upload
 #
 # The script delegates to the existing bench-reth-*.sh scripts in the reth
 # repo for the actual build, snapshot, and run steps.
 set -euxo pipefail
 
 # ── PATH ──────────────────────────────────────────────────────────────
-# Ensure cargo and user-local bins (mc, uv) are visible
+# Ensure cargo and user-local bins (uv) are visible
 export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
 
 # ── Defaults ──────────────────────────────────────────────────────────
@@ -106,7 +108,7 @@ fi
 
 # ── Check dependencies ───────────────────────────────────────────────
 missing=()
-for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq cargo; do
+for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq cargo; do
   command -v "$cmd" &>/dev/null || missing+=("$cmd")
 done
 if [ ${#missing[@]} -gt 0 ]; then
@@ -238,19 +240,14 @@ echo "  Baseline src : $BASELINE_SRC"
 echo "  Feature src  : $FEATURE_SRC"
 echo
 
-# ── Step 3: Check / download snapshot ────────────────────────────────
-echo "▸ Checking snapshot..."
+# ── Step 3: Validate local snapshot ──────────────────────────────────
+echo "▸ Validating local snapshot..."
 cd "$RETH_REPO"
-SNAPSHOT_NEEDED=false
-if ! "${SCRIPTS_DIR}/bench-reth-snapshot.sh" --check; then
-  SNAPSHOT_NEEDED=true
-  echo "  Snapshot needs update."
-else
-  echo "  Snapshot is up-to-date."
-fi
+"${SCRIPTS_DIR}/bench-reth-snapshot.sh"
+echo "  Snapshot is ready."
 echo
 
-# ── Step 4: Build binaries (+ snapshot download) in parallel ─────────
+# ── Step 4: Build binaries in parallel ───────────────────────────────
 echo "▸ Building binaries (parallel)..."
 cd "$RETH_REPO"
 
@@ -262,19 +259,11 @@ PID_BASELINE=$!
 "${SCRIPTS_DIR}/bench-reth-build.sh" feature "$FEATURE_SRC" "$FEATURE_SHA" &
 PID_FEATURE=$!
 
-PID_SNAPSHOT=
-if [ "$SNAPSHOT_NEEDED" = "true" ]; then
-  echo "  Also downloading snapshot in parallel..."
-  "${SCRIPTS_DIR}/bench-reth-snapshot.sh" &
-  PID_SNAPSHOT=$!
-fi
-
 wait $PID_BASELINE || FAIL=1
 wait $PID_FEATURE  || FAIL=1
-[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
 
 if [ $FAIL -ne 0 ]; then
-  echo "Error: one or more parallel tasks failed (builds / snapshot)"
+  echo "Error: one or more build tasks failed"
   exit 1
 fi
 echo "  Binaries built successfully."
diff --git a/.github/scripts/bench-reth-snapshot.sh b/.github/scripts/bench-reth-snapshot.sh
index 96b515235fd..2f7c73b01e9 100755
--- a/.github/scripts/bench-reth-snapshot.sh
+++ b/.github/scripts/bench-reth-snapshot.sh
@@ -1,197 +1,56 @@
 #!/usr/bin/env bash
 #
-# Downloads the latest snapshot into the schelk volume using
-# `reth download` with progress reporting to the GitHub PR comment.
-#
-# Skips the download if the manifest content hasn't changed since
-# the last successful download (checked via SHA-256 of the manifest).
+# Validates that the benchmark snapshot has already been populated into the
+# local schelk volume.
 #
 # Usage: bench-reth-snapshot.sh [--check]
-#   --check   Only check if a download is needed; exits 0 if up-to-date, 10 if not.
+#   --check   Exit 0 if the local snapshot is ready, 10 if it is missing.
 #
 # Required env:
-#   SCHELK_MOUNT       – schelk mount point (e.g. /reth-bench)
-#   BENCH_RETH_BINARY  – path to the reth binary
-#   GITHUB_TOKEN       – token for GitHub API calls (only for download)
-#   BENCH_COMMENT_ID   – PR comment ID to update (optional)
-#   BENCH_REPO         – owner/repo (e.g. paradigmxyz/reth)
-#   BENCH_JOB_URL      – link to the Actions job
-#   BENCH_ACTOR        – user who triggered the benchmark
-#   BENCH_CONFIG       – config summary line
+#   SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
+# Optional env:
+#   BENCH_BIG_BLOCKS   – true when validating the big-blocks snapshot datadir
+#   BENCH_SNAPSHOT_NAME – expected snapshot label for log/error output
 set -euxo pipefail
 
-MC="mc"
+: "${SCHELK_MOUNT:?SCHELK_MOUNT must be set}"
+
 DATADIR_NAME="datadir"
-HASH_MODE_SUFFIX=""
 if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
   DATADIR_NAME="datadir-big-blocks"
-  HASH_MODE_SUFFIX="-big-blocks"
 fi
 DATADIR="$SCHELK_MOUNT/$DATADIR_NAME"
-HASH_FILE="$HOME/.reth-bench-snapshot-hash${HASH_MODE_SUFFIX}"
-
-resolve_bucket() {
-  local candidate
-
-  for candidate in \
-    "${BENCH_SNAPSHOT_BUCKET:-}" \
-    "r2-reth-snapshots/reth-snapshots" \
-    "minio/reth-snapshots"; do
-    if [ -n "$candidate" ] && $MC ls "$candidate" >/dev/null 2>&1; then
-      echo "$candidate"
-      return 0
-    fi
-  done
-
-  echo "::error::Failed to find a readable snapshot bucket via mc ls"
-  exit 2
-}
-
-resolve_snapshot_name() {
-  local bucket="$1"
 
-  # Big-block benchmarks pin the base snapshot via BENCH_SNAPSHOT_NAME.
+describe_snapshot() {
   if [ -n "${BENCH_SNAPSHOT_NAME:-}" ]; then
-    echo "$BENCH_SNAPSHOT_NAME"
-    return 0
-  fi
-
-  mapfile -t snapshots < <(
-    $MC ls "$bucket" 2>/dev/null \
-      | awk '$NF ~ /\/$/ { print $NF }' \
-      | sed 's:/$::'
-  )
-
-  if [ "${#snapshots[@]}" -lt 2 ]; then
-    echo "::error::Need at least two snapshots in ${bucket} to resolve the previous snapshot"
-    exit 2
+    printf '%s' "${BENCH_SNAPSHOT_NAME}"
+  elif [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
+    printf '%s' 'big-block weekly snapshot'
+  else
+    printf '%s' 'benchmark snapshot'
   fi
-
-  echo "${snapshots[$(( ${#snapshots[@]} - 2 ))]}"
 }
 
-BUCKET="$(resolve_bucket)"
-SNAPSHOT_NAME="$(resolve_snapshot_name "$BUCKET")"
-MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
-BUCKET_ALIAS="${BUCKET%%/*}"
-BUCKET_PATH="${BUCKET#*/}"
-
-echo "Using snapshot bucket: ${BUCKET}"
-echo "Using snapshot: ${SNAPSHOT_NAME}"
-
-# Fetch manifest and compute content hash for reliable freshness check
-XTRACE_WAS_ENABLED=false
-if [[ $- == *x* ]]; then
-  XTRACE_WAS_ENABLED=true
-  set +x
-fi
+snapshot_ready() {
+  [ -d "$DATADIR/db" ] && [ -d "$DATADIR/static_files" ] && [ -f "$DATADIR/jwt.hex" ]
+}
 
-if ! MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null); then
-  echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
-  exit 2
-fi
-ORIGINAL_BASE_URL=$(printf '%s' "$MANIFEST_CONTENT" | jq -r '.base_url // empty')
-REMOTE_HASH=$(printf '%s' "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
+EXPECTED_SNAPSHOT="$(describe_snapshot)"
 
-LOCAL_HASH=""
-[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
+sudo schelk recover -y --kill || true
+sudo schelk mount -y
 
-if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
-  echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
+if snapshot_ready; then
+  echo "Found local ${EXPECTED_SNAPSHOT} at ${DATADIR}"
   exit 0
 fi
 
-echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
+echo "::error::Missing local ${EXPECTED_SNAPSHOT} at ${DATADIR}. Benchmarks no longer download snapshots; pre-populate the local schelk data first."
+ls -la "$SCHELK_MOUNT" || true
+ls -la "$DATADIR" || true
+
 if [ "${1:-}" = "--check" ]; then
   exit 10
 fi
 
-RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
-if [ ! -x "$RETH" ]; then
-  echo "::error::reth binary not found or not executable at $RETH"
-  exit 1
-fi
-
-# Resolve the MinIO HTTP endpoint from the mc alias so reth can
-# fetch archives over HTTP (the manifest's embedded base_url points
-# to the cluster-internal address which is unreachable from runners).
-MINIO_ENDPOINT=$($MC alias list "$BUCKET_ALIAS" --json 2>/dev/null | jq -r '.URL // empty') || true
-if [ -z "$MINIO_ENDPOINT" ]; then
-  echo "::error::Failed to resolve snapshot endpoint from mc alias '${BUCKET_ALIAS}'"
-  exit 1
-fi
-
-MANIFEST_TMP=$(mktemp --suffix=.json)
-trap 'rm -f -- "$MANIFEST_TMP"' EXIT
-if [[ "$MINIO_ENDPOINT" == *".r2.cloudflarestorage.com" ]]; then
-  # R2's S3 endpoint is not the public object-download URL. Keep the manifest's
-  # published base_url (for example r2.dev or a custom domain) for anonymous GETs.
-  printf '%s' "$MANIFEST_CONTENT" > "$MANIFEST_TMP"
-else
-  BASE_URL="${MINIO_ENDPOINT%/}/${BUCKET_PATH}/${SNAPSHOT_NAME}"
-  # Rewrite manifest's base_url with the runner-reachable endpoint.
-  printf '%s' "$MANIFEST_CONTENT" \
-    | jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
-fi
-EFFECTIVE_BASE_URL=$(jq -r '.base_url // empty' "$MANIFEST_TMP")
-
-if [ "$XTRACE_WAS_ENABLED" = true ]; then
-  set -x
-fi
-
-echo "Manifest base_url before rewrite: ${ORIGINAL_BASE_URL:-<empty>}"
-if [[ "$MINIO_ENDPOINT" == *".r2.cloudflarestorage.com" ]]; then
-  echo "Using manifest base_url for R2 snapshot downloads"
-fi
-echo "Manifest base_url after rewrite: ${EFFECTIVE_BASE_URL:-<empty>}"
-
-# Prepare mount. If a previous run left the volume mounted, recover first.
-sudo schelk recover -y --kill || true
-sudo schelk mount -y
-sudo rm -rf "$DATADIR"
-sudo mkdir -p "$DATADIR"
-# reth download runs as current user (not root), needs write access
-sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
-
-update_comment() {
-  local status="$1"
-  [ -z "${BENCH_COMMENT_ID:-}" ] && return 0
-  local body
-  body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
-    "$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
-  curl -sf -X PATCH \
-    -H "Authorization: token ${GITHUB_TOKEN}" \
-    -H "Accept: application/vnd.github.v3+json" \
-    "https://api.github.com/repos/${BENCH_REPO}/issues/comments/${BENCH_COMMENT_ID}" \
-    -d "$(jq -nc --arg body "$body" '{body: $body}')" \
-    > /dev/null 2>&1 || true
-}
-
-update_comment "Downloading snapshot…"
-
-# Download using reth download (manifest-path with rewritten base_url)
-"$RETH" download \
-  --manifest-path "$MANIFEST_TMP" \
-  -y \
-  --minimal \
-  --datadir "$DATADIR"
-
-update_comment "Downloading snapshot… done"
-echo "Snapshot download complete"
-
-# Sanity check: verify expected directories exist
-if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
-  echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
-  ls -la "$DATADIR" || true
-  exit 1
-fi
-
-# Promote the new snapshot to become the schelk baseline (virgin volume).
-# This copies changed blocks from scratch → virgin so that future
-# `schelk recover` calls restore to this new state.
-sync
-sudo schelk promote -y
-
-# Save manifest hash
-echo "$REMOTE_HASH" > "$HASH_FILE"
-echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"
+exit 1
diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml
index d5031a54ec5..8ade5e9fea2 100644
--- a/.github/workflows/bench-scheduled.yml
+++ b/.github/workflows/bench-scheduled.yml
@@ -307,12 +307,6 @@ jobs:
             linux-tools-"$(uname -r)" || \
             sudo apt-get install -y --no-install-recommends linux-tools-generic
 
-          # mc (MinIO client)
-          if ! command -v mc &>/dev/null; then
-            curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
-            chmod +x "$HOME/.local/bin/mc"
-          fi
-
           # uv (Python package manager)
           if ! command -v uv &>/dev/null; then
             curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
@@ -340,7 +334,7 @@ jobs:
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
           echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
           missing=()
-          for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
+          for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
             command -v "$cmd" &>/dev/null || missing+=("$cmd")
           done
           if [ ${#missing[@]} -gt 0 ]; then
@@ -366,19 +360,9 @@ jobs:
           echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
           echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
 
-      - name: Check if snapshot needs update
+      - name: Validate local snapshot
         id: snapshot-check
-        run: |
-          set +e
-          .github/scripts/bench-reth-snapshot.sh --check
-          rc=$?
-          set -e
-          case "$rc" in
-            0)  echo "needed=false" >> "$GITHUB_OUTPUT" ;;
-            10) echo "needed=true"  >> "$GITHUB_OUTPUT" ;;
-            *)  echo "::error::Snapshot check failed (exit $rc)"
-                exit "$rc" ;;
-          esac
+        run: .github/scripts/bench-reth-snapshot.sh
 
       - name: Prepare source dirs
         run: |
@@ -418,15 +402,6 @@ jobs:
             exit 1
           fi
 
-      - name: Download snapshot
-        id: snapshot-download
-        if: steps.snapshot-check.outputs.needed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BENCH_REPO: ${{ github.repository }}
-          BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
-        run: .github/scripts/bench-reth-snapshot.sh
-
       # System tuning for reproducible benchmarks
       - name: System setup
         run: |
@@ -925,8 +900,8 @@ jobs:
             if (!token || !channel) return;
 
             const steps_status = [
+              ['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
               ['building binaries', '${{ steps.build.outcome }}'],
-              ['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
               ['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
               ['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
               ['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index ddcb4b88220..2428dc68c29 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -650,12 +650,6 @@ jobs:
             linux-tools-"$(uname -r)" || \
             sudo apt-get install -y --no-install-recommends linux-tools-generic
 
-          # mc (MinIO client)
-          if ! command -v mc &>/dev/null; then
-            curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
-            chmod +x "$HOME/.local/bin/mc"
-          fi
-
           # uv (Python package manager)
           if ! command -v uv &>/dev/null; then
             curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
@@ -690,7 +684,7 @@ jobs:
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
           echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
           missing=()
-          for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
+          for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
             command -v "$cmd" &>/dev/null || missing+=("$cmd")
           done
           if [ ${#missing[@]} -gt 0 ]; then
@@ -766,61 +760,45 @@ jobs:
             core.setOutput('feature-ref', featureRef);
             core.setOutput('feature-name', featureName);
 
-      - name: Check big-blocks freshness
+      - name: Validate local big blocks
         if: env.BENCH_BIG_BLOCKS == 'true'
         id: big-blocks-check
         run: |
           set -euo pipefail
-          MC="mc --config-dir /home/ubuntu/.mc"
-          MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
-          HASH_FILE="$HOME/.reth-bench-big-blocks-hash"
-          echo "Fetching big-blocks manifest from $MANIFEST..."
-          BB_MANIFEST=$($MC cat "$MANIFEST" 2>/dev/null) || {
-            echo "::error::Failed to fetch big-blocks manifest from $MANIFEST"
+          BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
+          PAYLOAD_DIR="$BIG_BLOCKS_DIR/payloads"
+          MANIFEST="$BIG_BLOCKS_DIR/manifest.json"
+          echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
+
+          if [ ! -f "$MANIFEST" ]; then
+            echo "::error::Missing local big-blocks manifest at $MANIFEST"
             exit 1
-          }
-          BASE_SNAPSHOT=$(echo "$BB_MANIFEST" | jq -r '.base_snapshot // empty')
+          fi
+
+          BASE_SNAPSHOT=$(jq -r '.base_snapshot // empty' "$MANIFEST")
           if [ -z "$BASE_SNAPSHOT" ]; then
             echo "::error::Big-blocks manifest missing base_snapshot field"
             exit 1
           fi
-          echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
-          echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
 
-          REMOTE_HASH=$(echo "$BB_MANIFEST" | sha256sum | awk '{print $1}')
-          LOCAL_HASH=""
-          [ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
-          if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
-            echo "Big blocks up-to-date (hash: ${REMOTE_HASH:0:16}…)"
-            echo "needed=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "Big blocks need update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
-            echo "needed=true" >> "$GITHUB_OUTPUT"
-            echo "remote-hash=${REMOTE_HASH}" >> "$GITHUB_OUTPUT"
+          if [ ! -d "$PAYLOAD_DIR" ]; then
+            echo "::error::Missing local big-block payload directory at $PAYLOAD_DIR"
+            exit 1
           fi
 
-      - name: Check if snapshot needs update
+          PAYLOAD_COUNT=$(find "$PAYLOAD_DIR" -name '*.json' | wc -l)
+          if [ "$PAYLOAD_COUNT" -eq 0 ]; then
+            echo "::error::No payload files found in $PAYLOAD_DIR"
+            exit 1
+          fi
+
+          echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
+          echo "Payload files: $PAYLOAD_COUNT"
+          echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
+
+      - name: Validate local snapshot
         id: snapshot-check
-        run: |
-          set +e
-          .github/scripts/bench-reth-snapshot.sh --check
-          rc=$?
-          set -e
-          case "$rc" in
-            0)  echo "needed=false" >> "$GITHUB_OUTPUT" ;;
-            10) echo "needed=true"  >> "$GITHUB_OUTPUT" ;;
-            *)  echo "::error::Snapshot check failed (exit $rc)"
-                exit "$rc" ;;
-          esac
-
-      - name: Update status (snapshot needed)
-        if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
-        uses: actions/github-script@v9
-        with:
-          github-token: ${{ secrets.DEREK_PAT }}
-          script: |
-            const s = require('./.github/scripts/bench-update-status.js');
-            await s({github, context, status: 'Building binaries (snapshot update pending)...'});
+        run: .github/scripts/bench-reth-snapshot.sh
 
       - name: Prepare source dirs
         run: |
@@ -862,15 +840,6 @@ jobs:
             exit 1
           fi
 
-      - name: Download snapshot
-        id: snapshot-download
-        if: steps.snapshot-check.outputs.needed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BENCH_REPO: ${{ github.repository }}
-          BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
-        run: .github/scripts/bench-reth-snapshot.sh
-
       # System tuning for reproducible benchmarks
       - name: System setup
         run: |
@@ -941,45 +910,6 @@ jobs:
           rm -rf "$BENCH_WORK_DIR"
           mkdir -p "$BENCH_WORK_DIR"
 
-      - name: Download big blocks
-        if: env.BENCH_BIG_BLOCKS == 'true'
-        run: |
-          set -euo pipefail
-          BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
-          echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
-          if [ "${{ steps.big-blocks-check.outputs.needed }}" = "false" ]; then
-            echo "Big blocks cached at $BIG_BLOCKS_DIR, skipping download"
-            echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
-            exit 0
-          fi
-          MC="mc --config-dir /home/ubuntu/.mc"
-          MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
-          rm -rf "$BIG_BLOCKS_DIR"; mkdir -p "$BIG_BLOCKS_DIR"
-
-          # Download and parse manifest
-          echo "Downloading manifest from $MANIFEST..."
-          $MC cat "$MANIFEST" > "$BIG_BLOCKS_DIR/manifest.json"
-          UPLOAD_PATH=$(jq -r '.upload_path' "$BIG_BLOCKS_DIR/manifest.json")
-          COUNT=$(jq -r '.count' "$BIG_BLOCKS_DIR/manifest.json")
-          TARGET_GAS=$(jq -r '.target_gas' "$BIG_BLOCKS_DIR/manifest.json")
-          echo "Manifest: count=$COUNT, target_gas=$TARGET_GAS, archive=$UPLOAD_PATH"
-
-          # Download and extract archive
-          ARCHIVE="minio/$UPLOAD_PATH"
-          echo "Downloading big blocks from $ARCHIVE..."
-          $MC cat "$ARCHIVE" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
-          echo "Big blocks downloaded to $BIG_BLOCKS_DIR"
-
-          # Verify expected directory structure
-          if [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
-            echo "::error::Big blocks archive missing expected payloads/ directory"
-            ls -laR "$BIG_BLOCKS_DIR"
-            exit 1
-          fi
-          echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
-          # Save manifest hash for freshness check on next run
-          echo "${{ steps.big-blocks-check.outputs.remote-hash }}" > "$HOME/.reth-bench-big-blocks-hash"
-
       - name: Start metrics proxy
         run: |
           BENCH_ID="ci-${{ github.run_id }}"
@@ -1374,9 +1304,11 @@ jobs:
           github-token: ${{ secrets.DEREK_PAT }}
           script: |
             const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
+            const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
             const steps_status = [
+              ...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
+              ['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
               ['building binaries', '${{ steps.build.outcome }}'],
-              ['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
               ['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
               ['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
               ...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
@@ -1410,9 +1342,11 @@ jobs:
         with:
           script: |
             const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
+            const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
             const steps_status = [
+              ...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
+              ['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
               ['building binaries', '${{ steps.build.outcome }}'],
-              ['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
               ['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
               ['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
               ...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),

From b31d1bc6f8f129f7129b0036a6303e999826476e Mon Sep 17 00:00:00 2001
From: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Date: Wed, 22 Apr 2026 10:00:34 +0000
Subject: [PATCH 2/6] fix(bench): stop requiring jwt in local snapshots

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019db48c-8b95-744f-aad9-140ad31275fc
Co-authored-by: Amp <amp@ampcode.com>
---
 .github/scripts/bench-reth-snapshot.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/bench-reth-snapshot.sh b/.github/scripts/bench-reth-snapshot.sh
index 2f7c73b01e9..4c295462dbe 100755
--- a/.github/scripts/bench-reth-snapshot.sh
+++ b/.github/scripts/bench-reth-snapshot.sh
@@ -32,7 +32,7 @@ describe_snapshot() {
 }
 
 snapshot_ready() {
-  [ -d "$DATADIR/db" ] && [ -d "$DATADIR/static_files" ] && [ -f "$DATADIR/jwt.hex" ]
+  [ -d "$DATADIR/db" ] && [ -d "$DATADIR/static_files" ]
 }
 
 EXPECTED_SNAPSHOT="$(describe_snapshot)"

From 62d12acc64b895d7161f0762381e1f1747bc35c3 Mon Sep 17 00:00:00 2001
From: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Date: Wed, 22 Apr 2026 10:14:51 +0000
Subject: [PATCH 3/6] fix(ci): drop stale cargo-deny ignores

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019db48c-8b95-744f-aad9-140ad31275fc
Co-authored-by: Amp <amp@ampcode.com>
---
 deny.toml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/deny.toml b/deny.toml
index e54240b4936..2ccfcc4ec21 100644
--- a/deny.toml
+++ b/deny.toml
@@ -4,18 +4,12 @@
 [advisories]
 yanked = "warn"
 ignore = [
-    # https://rustsec.org/advisories/RUSTSEC-2024-0384 used by sse example
-    "RUSTSEC-2024-0384",
     # https://rustsec.org/advisories/RUSTSEC-2024-0436 paste! is unmaintained
     "RUSTSEC-2024-0436",
     # https://rustsec.org/advisories/RUSTSEC-2025-0141 bincode is unmaintained, need to transition all deps to wincode first
     "RUSTSEC-2025-0141",
     # https://rustsec.org/advisories/RUSTSEC-2023-0089 atomic-polyfill is unmaintained, transitive dep via test-fuzz
     "RUSTSEC-2023-0089",
-    # https://rustsec.org/advisories/RUSTSEC-2026-0002 lru upgrade requires a discv5 bump first
-    "RUSTSEC-2026-0002",
-    # https://rustsec.org/advisories/RUSTSEC-2026-0097 rand is unsound with a custom logger
-    "RUSTSEC-2026-0097",
 ]
 
 # This section is considered when running `cargo deny check bans`.

From c8a3f8f427aed40cacb8fbda6f122f5617bf330a Mon Sep 17 00:00:00 2001
From: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Date: Wed, 22 Apr 2026 11:05:43 +0000
Subject: [PATCH 4/6] fix(ci): ignore rustls-webpki CRL advisory

RUSTSEC-2026-0104 only affects CRL parsing before signature
verification. reth does not parse CRLs, so ignore it to unblock
cargo deny on this benchmark workflow PR.

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019db4da-109e-74b8-938e-c5fb27b8f275
Co-authored-by: Amp <amp@ampcode.com>
---
 deny.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/deny.toml b/deny.toml
index 2ccfcc4ec21..ffc0f85dc6e 100644
--- a/deny.toml
+++ b/deny.toml
@@ -10,6 +10,8 @@ ignore = [
     "RUSTSEC-2025-0141",
     # https://rustsec.org/advisories/RUSTSEC-2023-0089 atomic-polyfill is unmaintained, transitive dep via test-fuzz
     "RUSTSEC-2023-0089",
+    # https://rustsec.org/advisories/RUSTSEC-2026-0104 rustls-webpki only affects CRL parsing, which reth does not use
+    "RUSTSEC-2026-0104",
 ]
 
 # This section is considered when running `cargo deny check bans`.

From 1d21fd560a93afdb2fb50b5073ac17174cca964c Mon Sep 17 00:00:00 2001
From: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Date: Wed, 22 Apr 2026 11:07:37 +0000
Subject: [PATCH 5/6] fix(bench): tolerate stale schelk mount state

Big-bench jobs can hit a stale schelk state after host reboots where
incremental recover fails and mount reports the volume is already
mounted. Fall back to full recovery and validate the mounted datadir
instead of failing on that mount exit code alone.

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019db4da-109e-74b8-938e-c5fb27b8f275
Co-authored-by: Amp <amp@ampcode.com>
---
 .github/scripts/bench-reth-run.sh      | 10 ++++++++--
 .github/scripts/bench-reth-snapshot.sh |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/scripts/bench-reth-run.sh b/.github/scripts/bench-reth-run.sh
index 9a757274d93..94dd512f2ee 100755
--- a/.github/scripts/bench-reth-run.sh
+++ b/.github/scripts/bench-reth-run.sh
@@ -88,10 +88,16 @@ trap cleanup EXIT
 # Stop any leftover reth process in the scope, then recover schelk state.
 sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
 sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
-sudo schelk recover -y --kill || true
+sudo schelk recover -y --kill || sudo schelk full-recover -y || true
 
 # Mount
-sudo schelk mount -y
+sudo schelk mount -y || true
+if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
+  echo "::error::Failed to mount benchmark datadir at ${DATADIR}"
+  ls -la "$SCHELK_MOUNT" || true
+  ls -la "$DATADIR" || true
+  exit 1
+fi
 sync
 sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
 echo "=== Cache state after drop ==="
diff --git a/.github/scripts/bench-reth-snapshot.sh b/.github/scripts/bench-reth-snapshot.sh
index 4c295462dbe..5d8b883e544 100755
--- a/.github/scripts/bench-reth-snapshot.sh
+++ b/.github/scripts/bench-reth-snapshot.sh
@@ -37,8 +37,8 @@ snapshot_ready() {
 
 EXPECTED_SNAPSHOT="$(describe_snapshot)"
 
-sudo schelk recover -y --kill || true
-sudo schelk mount -y
+sudo schelk recover -y --kill || sudo schelk full-recover -y || true
+sudo schelk mount -y || true
 
 if snapshot_ready; then
   echo "Found local ${EXPECTED_SNAPSHOT} at ${DATADIR}"

From 05e2545f5a65e5b69c3d6f7c9492a080c63c2400 Mon Sep 17 00:00:00 2001
From: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Date: Wed, 22 Apr 2026 12:07:05 +0000
Subject: [PATCH 6/6] test(ci): restore deny.toml from origin/main

Restore the advisory ignore list to match origin/main exactly so we can
verify whether cargo deny still passes on this branch after the merge.

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019db4da-109e-74b8-938e-c5fb27b8f275
Co-authored-by: Amp <amp@ampcode.com>
---
 deny.toml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/deny.toml b/deny.toml
index 2ccfcc4ec21..e54240b4936 100644
--- a/deny.toml
+++ b/deny.toml
@@ -4,12 +4,18 @@
 [advisories]
 yanked = "warn"
 ignore = [
+    # https://rustsec.org/advisories/RUSTSEC-2024-0384 used by sse example
+    "RUSTSEC-2024-0384",
     # https://rustsec.org/advisories/RUSTSEC-2024-0436 paste! is unmaintained
     "RUSTSEC-2024-0436",
     # https://rustsec.org/advisories/RUSTSEC-2025-0141 bincode is unmaintained, need to transition all deps to wincode first
     "RUSTSEC-2025-0141",
     # https://rustsec.org/advisories/RUSTSEC-2023-0089 atomic-polyfill is unmaintained, transitive dep via test-fuzz
     "RUSTSEC-2023-0089",
+    # https://rustsec.org/advisories/RUSTSEC-2026-0002 lru upgrade requires a discv5 bump first
+    "RUSTSEC-2026-0002",
+    # https://rustsec.org/advisories/RUSTSEC-2026-0097 rand is unsound with a custom logger
+    "RUSTSEC-2026-0097",
 ]
 
 # This section is considered when running `cargo deny check bans`.