Optimization for prover (override #1761)

Cargo.toml

@@ -17,9 +17,10 @@ repository = "https://github.com/scroll-tech/scroll"
 version = "4.7.1"
 
 [workspace.dependencies]
-scroll-zkvm-prover = { git = "https://github.com/scroll-tech/zkvm-prover", tag = "v0.7.1" }
-scroll-zkvm-verifier = { git = "https://github.com/scroll-tech/zkvm-prover", tag = "v0.7.1" }
-scroll-zkvm-types = { git = "https://github.com/scroll-tech/zkvm-prover", tag = "v0.7.1" }
+# with openvm 1.4.2
+scroll-zkvm-prover = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "2e8e29f" }
+scroll-zkvm-verifier = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "2e8e29f" }
+scroll-zkvm-types = { git = "https://github.com/scroll-tech/zkvm-prover", rev = "2e8e29f" }
 
 sbv-primitives = { git = "https://github.com/scroll-tech/stateless-block-verifier", tag = "scroll-v91.2", features = ["scroll", "rkyv"] }
 sbv-utils = { git = "https://github.com/scroll-tech/stateless-block-verifier", tag = "scroll-v91.2" }

crates/libzkp/src/lib.rs

@@ -13,22 +13,8 @@ use serde_json::value::RawValue;
 use std::{collections::HashMap, path::Path, sync::OnceLock};
 use tasks::chunk_interpreter::{ChunkInterpreter, TryFromWithInterpreter};
 
-pub(crate) fn witness_use_legacy_mode(fork_name: &str) -> eyre::Result<bool> {
-    ADDITIONAL_FEATURES
-        .get()
-        .and_then(|features| features.get(fork_name))
-        .map(|cfg| cfg.legacy_witness_encoding)
-        .ok_or_else(|| {
-            eyre::eyre!(
-                "can not find features setting for unrecognized fork {}",
-                fork_name
-            )
-        })
-}
-
 #[derive(Debug, Default, Clone)]
 struct FeatureOptions {
-    legacy_witness_encoding: bool,
     for_openvm_13_prover: bool,
 }
 
@@ -41,11 +27,10 @@ impl FeatureOptions {
         for feat_s in feats.split(':') {
             match feat_s.trim().to_lowercase().as_str() {
                 "legacy_witness" => {
-                    tracing::info!("set witness encoding for legacy mode");
-                    ret.legacy_witness_encoding = true;
+                    tracing::warn!("legacy witness is no longer supported");
                 }
                 "openvm_13" => {
-                    tracing::info!("set prover should use openvm 13");
+                    tracing::warn!("set prover should use openvm 13");
                     ret.for_openvm_13_prover = true;
                 }
                 s => tracing::warn!("unrecognized dynamic feature: {s}"),

crates/libzkp/src/tasks/batch.rs

@@ -4,13 +4,11 @@ use sbv_primitives::{B256, U256};
 use scroll_zkvm_types::{
     batch::{
         build_point_eval_witness, BatchHeader, BatchHeaderV6, BatchHeaderV7, BatchHeaderValidium,
-        BatchInfo, BatchWitness, Envelope, EnvelopeV6, EnvelopeV7, LegacyBatchWitness,
-        ReferenceHeader, N_BLOB_BYTES,
+        BatchInfo, BatchWitness, Envelope, EnvelopeV6, EnvelopeV7, ReferenceHeader, N_BLOB_BYTES,
     },
     chunk::ChunkInfo,
     public_inputs::{ForkName, MultiVersionPublicInputs, Version},
     task::ProvingTask,
-    utils::{to_rkyv_bytes, RancorError},
     version::{Codec, Domain, STFVersion},
 };
 
@@ -118,12 +116,7 @@ pub struct BatchProvingTask {
 impl BatchProvingTask {
     pub fn into_proving_task_with_precheck(self) -> Result<(ProvingTask, BatchInfo, B256)> {
         let (witness, metadata, batch_pi_hash) = self.precheck()?;
-        let serialized_witness = if crate::witness_use_legacy_mode(&self.fork_name)? {
-            let legacy_witness = LegacyBatchWitness::from(witness);
-            to_rkyv_bytes::<RancorError>(&legacy_witness)?.into_vec()
-        } else {
-            super::encode_task_to_witness(&witness)?
-        };
+        let serialized_witness = super::encode_task_to_witness(&witness)?;
 
         let proving_task = ProvingTask {
             identifier: self.batch_header.batch_hash().to_string(),

crates/libzkp/src/tasks/bundle.rs

@@ -1,10 +1,9 @@
 use eyre::Result;
 use sbv_primitives::B256;
 use scroll_zkvm_types::{
-    bundle::{BundleInfo, BundleWitness, LegacyBundleWitness},
+    bundle::{BundleInfo, BundleWitness},
     public_inputs::{MultiVersionPublicInputs, Version},
     task::ProvingTask,
-    utils::{to_rkyv_bytes, RancorError},
 };
 
 use crate::proofs::BatchProof;
@@ -27,12 +26,7 @@ pub struct BundleProvingTask {
 impl BundleProvingTask {
     pub fn into_proving_task_with_precheck(self) -> Result<(ProvingTask, BundleInfo, B256)> {
         let (witness, bundle_info, bundle_pi_hash) = self.precheck()?;
-        let serialized_witness = if crate::witness_use_legacy_mode(&self.fork_name)? {
-            let legacy = LegacyBundleWitness::from(witness);
-            to_rkyv_bytes::<RancorError>(&legacy)?.into_vec()
-        } else {
-            super::encode_task_to_witness(&witness)?
-        };
+        let serialized_witness = super::encode_task_to_witness(&witness)?;
 
         let proving_task = ProvingTask {
             identifier: self.identifier(),

crates/libzkp/src/tasks/chunk.rs

@@ -2,10 +2,9 @@ use eyre::Result;
 use sbv_core::BlockWitness;
 use sbv_primitives::{types::consensus::BlockHeader, B256};
 use scroll_zkvm_types::{
-    chunk::{execute, ChunkInfo, ChunkWitness, LegacyChunkWitness, ValidiumInputs},
+    chunk::{execute, ChunkInfo, ChunkWitness, ValidiumInputs},
     public_inputs::{MultiVersionPublicInputs, Version},
     task::ProvingTask,
-    utils::{to_rkyv_bytes, RancorError},
 };
 
 use super::chunk_interpreter::*;
@@ -117,12 +116,7 @@ impl ChunkProvingTask {
 
     pub fn into_proving_task_with_precheck(self) -> Result<(ProvingTask, ChunkInfo, B256)> {
         let (witness, chunk_info, chunk_pi_hash) = self.precheck()?;
-        let serialized_witness = if crate::witness_use_legacy_mode(&self.fork_name)? {
-            let legacy_witness = LegacyChunkWitness::from(witness);
-            to_rkyv_bytes::<RancorError>(&legacy_witness)?.into_vec()
-        } else {
-            super::encode_task_to_witness(&witness)?
-        };
+        let serialized_witness = super::encode_task_to_witness(&witness)?;
 
         let proving_task = ProvingTask {
             identifier: self.identifier(),

crates/prover-bin/Cargo.toml

@@ -9,7 +9,7 @@ edition.workspace = true
 scroll-zkvm-types.workspace = true
 scroll-zkvm-prover.workspace = true
 libzkp = { path = "../libzkp"}
-scroll-proving-sdk = { git = "https://github.com/scroll-tech/scroll-proving-sdk.git", rev = "05648db" }
+scroll-proving-sdk = { git = "https://github.com/scroll-tech/scroll-proving-sdk.git", rev = "22ad34e" }
 serde.workspace = true
 serde_json.workspace = true
 once_cell.workspace =true
@@ -34,6 +34,7 @@ clap = { version = "4.5", features = ["derive"] }
 ctor = "0.2.8"
 url = { version = "2.5.4", features = ["serde",] }
 serde_bytes = "0.11.15"
+bincode = { version = "2.0" }
 
 [features]
 default = []

crates/prover-bin/src/dumper.rs

@@ -0,0 +1,74 @@
+use async_trait::async_trait;
+use libzkp::ProvingTaskExt;
+use scroll_proving_sdk::prover::{
+    proving_service::{
+        GetVkRequest, GetVkResponse, ProveRequest, ProveResponse, QueryTaskRequest,
+        QueryTaskResponse, TaskStatus,
+    },
+    ProvingService,
+};
+use scroll_zkvm_types::ProvingTask;
+
+#[derive(Default)]
+pub struct Dumper {
+    #[allow(dead_code)]
+    target_path: String,
+}
+
+impl Dumper {
+    fn dump(&self, input_string: &str) -> eyre::Result<()> {
+        let task: ProvingTaskExt = serde_json::from_str(input_string)?;
+        let task = ProvingTask::from(task);
+
+        // stream-encode serialized_witness to input_task.bin using bincode 2.0
+        let input_file = std::fs::File::create("input_task.bin")?;
+        let mut input_writer = std::io::BufWriter::new(input_file);
+        bincode::serde::encode_into_std_write(
+            &task.serialized_witness,
+            &mut input_writer,
+            bincode::config::standard(),
+        )?;
+
+        // stream-encode aggregated_proofs to agg_proofs.bin using bincode 2.0
+        let agg_file = std::fs::File::create("agg_proofs.bin")?;
+        let mut agg_writer = std::io::BufWriter::new(agg_file);
+        bincode::serde::encode_into_std_write(
+            &task.aggregated_proofs,
+            &mut agg_writer,
+            bincode::config::standard(),
+        )?;
+
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl ProvingService for Dumper {
+    fn is_local(&self) -> bool {
+        true
+    }
+    async fn get_vks(&self, _: GetVkRequest) -> GetVkResponse {
+        // get vk has been deprecated in new prover with dynamic asset loading scheme
+        GetVkResponse {
+            vks: vec![],
+            error: None,
+        }
+    }
+    async fn prove(&mut self, req: ProveRequest) -> ProveResponse {
+        let error = if let Err(e) = self.dump(&req.input) {
+            Some(format!("failed to dump: {}", e))
+        } else {
+            None
+        };
+
+        ProveResponse {
+            status: TaskStatus::Failed,
+            error,
+            ..Default::default()
+        }
+    }
+
+    async fn query_task(&mut self, _: QueryTaskRequest) -> QueryTaskResponse {
+        unreachable!("for one_shot routine, we should be returned in prove call");
+    }
+}

crates/prover-bin/src/main.rs

@@ -1,8 +1,9 @@
+mod dumper;
 mod prover;
 mod types;
 mod zk_circuits_handler;
 
-use clap::{ArgAction, Parser, Subcommand};
+use clap::{ArgAction, Parser, Subcommand, ValueEnum};
 use prover::{LocalProver, LocalProverConfig};
 use scroll_proving_sdk::{
     prover::{types::ProofType, ProverBuilder},
@@ -32,12 +33,33 @@ struct Args {
     command: Option<Commands>,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, ValueEnum)]
+enum TaskType {
+    Chunk,
+    Batch,
+    Bundle,
+}
+
+impl From<TaskType> for ProofType {
+    fn from(value: TaskType) -> Self {
+        match value {
+            TaskType::Chunk => ProofType::Chunk,
+            TaskType::Batch => ProofType::Batch,
+            TaskType::Bundle => ProofType::Bundle,
+        }
+    }
+}
+
 #[derive(Subcommand, Debug)]
 enum Commands {
     Handle {
         /// path to save the verifier's asset
         task_path: String,
     },
+    Dump {
+        task_type: TaskType,
+        task_id: String,
+    },
 }
 
 #[derive(Debug, serde::Deserialize)]
@@ -63,6 +85,16 @@ async fn main() -> eyre::Result<()> {
     let local_prover = LocalProver::new(cfg.clone());
 
     match args.command {
+        Some(Commands::Dump { task_type, task_id }) => {
+            let prover = ProverBuilder::new(sdk_config, dumper::Dumper::default())
+                .build()
+                .await
+                .map_err(|e| eyre::eyre!("build prover fail: {e}"))?;
+
+            std::sync::Arc::new(prover)
+                .one_shot(&[task_id], task_type.into())
+                .await;
+        }
         Some(Commands::Handle { task_path }) => {
             let file = File::open(Path::new(&task_path))?;
             let reader = BufReader::new(file);

crates/prover-bin/src/prover.rs

@@ -30,6 +30,9 @@ pub struct AssetsLocationData {
     #[serde(default)]
     /// a altered url for specififed vk
     pub asset_detours: HashMap<String, url::Url>,
+    /// when asset file existed, do not verify from network, help for debugging stuffs
+    #[serde(default)]
+    pub debug_mode: bool,
 }
 
 impl AssetsLocationData {
@@ -79,6 +82,13 @@ impl AssetsLocationData {
                 // Get file metadata to check size
                 if let Ok(metadata) = std::fs::metadata(&local_file_path) {
                     // Make a HEAD request to get remote file size
+                    if self.debug_mode {
+                        println!(
+                            "File {} already exists, skipping download under debugmode",
+                            filename
+                        );
+                        continue;
+                    }
 
                     if let Ok(head_resp) = client.head(download_url.clone()).send().await {
                         if let Some(content_length) = head_resp.headers().get("content-length") {
@@ -201,12 +211,20 @@ impl ProvingService for LocalProver {
                         error: Some(format!("proving task failed: {}", e)),
                         ..Default::default()
                     },
-                    Err(e) => QueryTaskResponse {
-                        task_id: req.task_id,
-                        status: TaskStatus::Failed,
-                        error: Some(format!("proving task panicked: {}", e)),
-                        ..Default::default()
-                    },
+                    Err(e) => {
+                        if e.is_panic() {
+                            // simply re-throw panic for any panicking in proving process,
+                            // cause worker loop and the whole prover exit
+                            std::panic::resume_unwind(e.into_panic());
+                        }
+
+                        QueryTaskResponse {
+                            task_id: req.task_id,
+                            status: TaskStatus::Failed,
+                            error: Some(format!("proving task failed: {}", e)),
+                            ..Default::default()
+                        }
+                    }
                 };
             } else {
                 return QueryTaskResponse {
@@ -273,7 +291,9 @@ impl LocalProver {
         let created_at = duration.as_secs() as f64 + duration.subsec_nanos() as f64 * 1e-9;
 
         let prover_task = UniversalHandler::get_task_from_input(&req.input)?;
-        let is_openvm_13 = prover_task.use_openvm_13;
+        if prover_task.use_openvm_13 {
+            eyre::bail!("prover do not support snark params base on openvm 13");
+        }
         let prover_task: ProvingTask = prover_task.into();
         let vk = hex::encode(&prover_task.vk);
         let handler = if let Some(handler) = self.handlers.get(&vk) {
@@ -300,10 +320,7 @@ impl LocalProver {
                 .location_data
                 .get_asset(&vk, &url_base, &base_config.workspace_path)
                 .await?;
-            let circuits_handler = Arc::new(Mutex::new(UniversalHandler::new(
-                &asset_path,
-                is_openvm_13,
-            )?));
+            let circuits_handler = Arc::new(Mutex::new(UniversalHandler::new(&asset_path)?));
             self.handlers.insert(vk, circuits_handler.clone());
             circuits_handler
         };

crates/prover-bin/src/zk_circuits_handler/universal.rs

@@ -16,15 +16,14 @@ pub struct UniversalHandler {
 unsafe impl Send for UniversalHandler {}
 
 impl UniversalHandler {
-    pub fn new(workspace_path: impl AsRef<Path>, is_openvm_v13: bool) -> Result<Self> {
+    pub fn new(workspace_path: impl AsRef<Path>) -> Result<Self> {
         let path_app_exe = workspace_path.as_ref().join("app.vmexe");
         let path_app_config = workspace_path.as_ref().join("openvm.toml");
-        let segment_len = Some((1 << 21) - 100);
+        let segment_len = Some((1 << 22) - 100);
         let config = ProverConfig {
             path_app_config,
             path_app_exe,
             segment_len,
-            is_openvm_v13,
         };
 
         let prover = Prover::setup(config, None)?;

tests/prover-e2e/mainnet-galileo/.make.env

@@ -0,0 +1,3 @@
+BEGIN_BLOCK?=26653680
+END_BLOCK?=26653686
+SCROLL_FORK_NAME=galileo

tests/prover-e2e/sepolia-feynman/config.json → tests/prover-e2e/mainnet-galileo/config.json

@@ -5,6 +5,10 @@
     "maxOpenNum": 5,
     "maxIdleNum": 1
   },
+  "fetch_config": {
+    "endpoint": "https://mainnet-rpc.scroll.io",
+    "l2_message_queue_address": "0x5300000000000000000000000000000000000000"
+  },  
   "validium_mode": false,
-  "codec_version": 8
+  "codec_version": 9
 }