diff --git a/Cargo.lock b/Cargo.lock index 0f7d0711255..5a213b0adf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9483,6 +9483,7 @@ dependencies = [ "reth-optimism-evm", "reth-optimism-node", "reth-optimism-primitives", + "reth-optimism-trie", "reth-primitives-traits", "reth-provider", "reth-prune", @@ -9567,7 +9568,6 @@ dependencies = [ "eyre", "futures", "futures-util", - "reth-chainspec", "reth-db", "reth-exex", "reth-node-api", diff --git a/DockerfileOpProof b/DockerfileOpProof new file mode 100644 index 00000000000..0fedaff5ab7 --- /dev/null +++ b/DockerfileOpProof @@ -0,0 +1,48 @@ +FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef +WORKDIR /app + +LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" + +RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config + +# Builds a cargo-chef plan +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +COPY --from=planner /app/recipe.json recipe.json + +ARG BUILD_PROFILE=release +ENV BUILD_PROFILE=$BUILD_PROFILE + +ARG RUSTFLAGS="" +ENV RUSTFLAGS="$RUSTFLAGS" + +ARG FEATURES="" +ENV FEATURES=$FEATURES + +RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json --manifest-path /app/crates/optimism/bin/Cargo.toml + +COPY . . +RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --bin op-reth --manifest-path /app/crates/optimism/bin/Cargo.toml + +RUN ls -la /app/target/$BUILD_PROFILE/op-reth +RUN cp /app/target/$BUILD_PROFILE/op-reth /app/op-reth + +FROM ubuntu AS runtime + +RUN apt-get update && \ + apt-get install -y ca-certificates libssl-dev pkg-config strace && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY --from=builder /app/op-reth /usr/local/bin/ +RUN chmod +x /usr/local/bin/op-reth +COPY crates/optimism/tests/scripts/op-reth-entrypoint.sh /usr/local/bin/op-reth-entrypoint.sh +RUN chmod +x /usr/local/bin/op-reth-entrypoint.sh +COPY LICENSE-* ./ + +EXPOSE 30303 30303/udp 9001 8545 8546 7545 8551 +ENTRYPOINT ["/usr/local/bin/op-reth-entrypoint.sh"] diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index bdb680b2062..bfc5a81c813 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -37,6 +37,7 @@ reth-node-metrics.workspace = true reth-optimism-primitives.workspace = true reth-optimism-chainspec = { workspace = true, features = ["superchain-configs"] } reth-optimism-consensus.workspace = true +reth-optimism-trie.workspace = true reth-chainspec.workspace = true reth-node-events.workspace = true @@ -86,15 +87,9 @@ asm-keccak = [ ] # Jemalloc feature for vergen to generate correct env vars -jemalloc = [ - "reth-node-core/jemalloc", - "reth-node-metrics/jemalloc", -] +jemalloc = ["reth-node-core/jemalloc", "reth-node-metrics/jemalloc"] -dev = [ - "dep:proptest", - "reth-cli-commands/arbitrary", -] +dev = ["dep:proptest", "reth-cli-commands/arbitrary"] serde = [ "alloy-consensus/serde", diff --git a/crates/optimism/cli/src/app.rs b/crates/optimism/cli/src/app.rs index 34048b3d943..e0c31995955 100644 --- a/crates/optimism/cli/src/app.rs +++ b/crates/optimism/cli/src/app.rs @@ -110,6 +110,9 @@ where Commands::ReExecute(command) => { runner.run_until_ctrl_c(command.execute::(components)) } + Commands::InitializeOpProofs(command) => { + runner.run_blocking_until_ctrl_c(command.execute::()) + } } } diff --git a/crates/optimism/cli/src/commands/initialize_proofs.rs b/crates/optimism/cli/src/commands/initialize_proofs.rs new file mode 100644 index 00000000000..e81e84f9dcc --- /dev/null +++ b/crates/optimism/cli/src/commands/initialize_proofs.rs @@ -0,0 +1,100 @@ +//! Command that initializes the OP proofs storage with the current state of the chain. + +use clap::Parser; +use reth_chainspec::ChainInfo; +use reth_cli::chainspec::ChainSpecParser; +use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; +use reth_node_core::version::version_metadata; +use reth_optimism_chainspec::OpChainSpec; +use reth_optimism_primitives::OpPrimitives; +use reth_optimism_trie::{db::MdbxProofsStorage, BackfillJob, OpProofsStorage, OpProofsStore}; +use reth_provider::{BlockNumReader, DBProvider, DatabaseProviderFactory}; +use std::{path::PathBuf, sync::Arc}; +use tracing::info; + +/// Initializes the proofs storage with the current state of the chain. +/// +/// This command must be run before starting the node with proofs history enabled. +/// It backfills the proofs storage with trie nodes from the current chain state. +#[derive(Debug, Parser)] +pub struct InitializeOpProofsCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// The path to the storage DB for proofs history. + /// + /// This should match the path used when starting the node with + /// `--proofs-history.storage-path`. + #[arg( + long = "proofs-history.storage-path", + value_name = "PROOFS_HISTORY_STORAGE_PATH", + required = true + )] + pub storage_path: PathBuf, +} + +impl> InitializeOpProofsCommand { + /// Execute `initialize-op-proofs` command + pub async fn execute>( + self, + ) -> eyre::Result<()> { + info!(target: "reth::cli", "reth {} starting", version_metadata().short_version); + info!(target: "reth::cli", "Initializing OP proofs storage at: {:?}", self.storage_path); + + // Initialize the environment with read-only access + let Environment { provider_factory, .. } = self.env.init::(AccessRights::RO)?; + + // Create the proofs storage + let storage: OpProofsStorage> = Arc::new( + MdbxProofsStorage::new(&self.storage_path) + .map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?, + ) + .into(); + + // Check if already initialized + if let Some((block_number, block_hash)) = storage.get_earliest_block_number().await? { + info!( + target: "reth::cli", + block_number = block_number, + block_hash = ?block_hash, + "Proofs storage already initialized" + ); + return Ok(()); + } + + // Get the current chain state + let ChainInfo { best_number, best_hash, .. } = provider_factory.chain_info()?; + + info!( + target: "reth::cli", + best_number = best_number, + best_hash = ?best_hash, + "Starting backfill job for current chain state" + ); + + // Run the backfill job + { + let db_provider = + provider_factory.database_provider_ro()?.disable_long_read_transaction_safety(); + let db_tx = db_provider.into_tx(); + + BackfillJob::new(storage.clone(), &db_tx).run(best_number, best_hash).await?; + } + + info!( + target: "reth::cli", + best_number = best_number, + best_hash = ?best_hash, + "Proofs storage initialized successfully" + ); + + Ok(()) + } +} + +impl InitializeOpProofsCommand { + /// Returns the underlying chain being used to run this command + pub const fn chain_spec(&self) -> Option<&Arc> { + Some(&self.env.chain) + } +} diff --git a/crates/optimism/cli/src/commands/mod.rs b/crates/optimism/cli/src/commands/mod.rs index 5edd55b0ccb..c3958a18b19 100644 --- a/crates/optimism/cli/src/commands/mod.rs +++ b/crates/optimism/cli/src/commands/mod.rs @@ -14,6 +14,7 @@ use std::{fmt, sync::Arc}; pub mod import; pub mod import_receipts; pub mod init_state; +pub mod initialize_proofs; #[cfg(feature = "dev")] pub mod test_vectors; @@ -61,6 +62,9 @@ pub enum Commands), + /// Initializes the proofs storage with the current state of the chain. + #[command(name = "initialize-op-proofs")] + InitializeOpProofs(initialize_proofs::InitializeOpProofsCommand), } impl< @@ -85,6 +89,7 @@ impl< #[cfg(feature = "dev")] Self::TestVectors(_) => None, Self::ReExecute(cmd) => cmd.chain_spec(), + Self::InitializeOpProofs(cmd) => cmd.chain_spec(), } } } diff --git a/crates/optimism/exex/Cargo.toml b/crates/optimism/exex/Cargo.toml index deaaeb2f5a1..12a967f37a2 100644 --- a/crates/optimism/exex/Cargo.toml +++ b/crates/optimism/exex/Cargo.toml @@ -16,9 +16,8 @@ workspace = true reth-exex.workspace = true reth-node-types.workspace = true reth-node-api.workspace = true -reth-provider.workspace = true -reth-chainspec.workspace = true reth-primitives-traits.workspace = true +reth-provider.workspace = true # op-reth # proofs exex handles `TrieUpdates` in notifications @@ -44,12 +43,11 @@ tempfile.workspace = true [features] test-utils = [ - "reth-provider/test-utils", "reth-db/test-utils", - "reth-chainspec/test-utils", "reth-node-builder/test-utils", "reth-optimism-node/test-utils", "reth-primitives-traits/test-utils", + "reth-provider/test-utils", ] [package.metadata.cargo-udeps.ignore] diff --git a/crates/optimism/exex/src/lib.rs b/crates/optimism/exex/src/lib.rs index 123b5726d9c..164ac1a98e9 100644 --- a/crates/optimism/exex/src/lib.rs +++ b/crates/optimism/exex/src/lib.rs @@ -11,15 +11,12 @@ use alloy_consensus::BlockHeader; use derive_more::Constructor; use futures_util::TryStreamExt; -use reth_chainspec::ChainInfo; use reth_exex::{ExExContext, ExExEvent, ExExNotification}; use reth_node_api::{FullNodeComponents, NodePrimitives}; use reth_node_types::NodeTypes; -use reth_optimism_trie::{live::LiveTrieCollector, BackfillJob, OpProofsStorage, OpProofsStore}; +use reth_optimism_trie::{live::LiveTrieCollector, OpProofsStorage, OpProofsStore}; use reth_primitives_traits::{BlockTy, RecoveredBlock}; -use reth_provider::{ - BlockNumReader, BlockReader, DBProvider, DatabaseProviderFactory, TransactionVariant, -}; +use reth_provider::{BlockReader, TransactionVariant}; use tracing::{debug, error, info}; /// OP Proofs ExEx - processes blocks and tracks state changes within fault proof window. @@ -101,12 +98,11 @@ where { /// Main execution loop for the ExEx pub async fn run(mut self) -> eyre::Result<()> { - { - let db_provider = - self.ctx.provider().database_provider_ro()?.disable_long_read_transaction_safety(); - let db_tx = db_provider.into_tx(); - let ChainInfo { best_number, best_hash } = self.ctx.provider().chain_info()?; - BackfillJob::new(self.storage.clone(), &db_tx).run(best_number, best_hash).await?; + // Check if proofs storage is initialized + if self.storage.get_earliest_block_number().await?.is_none() { + return Err(eyre::eyre!( + "Proofs storage not initialized. Please run 'op-reth initialize-op-proofs --proofs-history.storage-path ' first." + )); } let collector = LiveTrieCollector::new( diff --git a/crates/optimism/tests/Makefile b/crates/optimism/tests/Makefile index 3122062be93..b6419f9cfb1 100644 --- a/crates/optimism/tests/Makefile +++ b/crates/optimism/tests/Makefile @@ -1,7 +1,7 @@ # Variables DOCKER_IMAGE_NAME := op-reth DOCKER_TAG := local -DOCKERFILE_PATH := ../../../DockerfileOp +DOCKERFILE_PATH := ../../../DockerfileOpProof KURTOSIS_PACKAGE := github.com/ethpandaops/optimism-package@998796c0f3bb478d63d729e65f0b76e24112e00d DEVNET ?= opgeth-seq-opreth-val GO_PKG_NAME ?= proofs/core diff --git a/crates/optimism/tests/scripts/op-reth-entrypoint.sh b/crates/optimism/tests/scripts/op-reth-entrypoint.sh new file mode 100644 index 00000000000..b316244595d --- /dev/null +++ b/crates/optimism/tests/scripts/op-reth-entrypoint.sh @@ -0,0 +1,74 @@ +#!/bin/sh +set -e + +# Variables to extract +DATADIR="" +PROOFS_PATH="" +CHAIN="" + +# Helper: require a value after flag +require_value() { + if [ -z "$2" ] || printf "%s" "$2" | grep -q "^--"; then + echo "ERROR: Missing value for $1" >&2 + exit 1 + fi +} + +# Parse arguments +i=1 +while [ "$i" -le "$#" ]; do + eval arg="\${$i}" + + case "$arg" in + --datadir=*) + DATADIR="${arg#*=}" + ;; + + --datadir) + eval next="\${$((i+1))}" + require_value "$arg" "$next" + DATADIR="$next" + i=$((i+1)) + ;; + + --proofs-history.storage-path=*) + PROOFS_PATH="${arg#*=}" + ;; + + --proofs-history.storage-path) + eval next="\${$((i+1))}" + require_value "$arg" "$next" + PROOFS_PATH="$next" + i=$((i+1)) + ;; + + --chain=*) + CHAIN="${arg#*=}" + ;; + + --chain) + eval next="\${$((i+1))}" + require_value "$arg" "$next" + CHAIN="$next" + i=$((i+1)) + ;; + + *) + # ignore unknown args—OR log them + ;; + esac + + i=$((i+1)) +done + +# Log extracted values +echo "extracted --datadir: ${DATADIR:-}" +echo "extracted --proofs-history.storage-path: ${PROOFS_PATH:-}" +echo "extracted --chain: ${CHAIN:-}" + +echo "Initializing op-reth" +op-reth init --datadir="$DATADIR" --chain="$CHAIN" +echo "Initializing op-reth proofs" +op-reth initialize-op-proofs --datadir="$DATADIR" --proofs-history.storage-path="$PROOFS_PATH" +echo "Starting op-reth with args: $*" +op-reth "$@"