cherry pick trie node fix

Author: lispc

Cargo.toml.gpu

@@ -63,6 +63,7 @@ metrics-util = "0.17"
 metrics-tracing-context = "0.16.0"
 p3-field = { git = "ssh://[email protected]/scroll-tech/plonky3-gpu.git", tag = "v0.2.0" }
 rayon = "1.10"
+reqwest = "0.12"
 rkyv = "0.8"
 serde = { version = "1", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0" }

crates/integration/tests/chunk_circuit.rs

@@ -23,7 +23,9 @@ fn exec_chunk(task: &ChunkProvingTask) -> eyre::Result<(ExecutionResult, u64)> {
     );
     let stats = task.stats();
     println!("chunk stats {:#?}", stats);
-    ChunkProverType::metadata_with_prechecks(task)?;
+    let mut task = task.clone();
+    let task = &mut task;
+    scroll_zkvm_prover::ChunkProver::metadata_with_prechecks(task)?;
     let stdin = task.build_guest_input()?;
     let exec_result = utils::vm::execute_guest(config, app_exe, &stdin, &Default::default())?;
     let cycle_count = exec_result.total_cycle as u64;

crates/prover/Cargo.toml

@@ -26,18 +26,21 @@ openvm-native-recursion = { workspace = true }
 openvm-sdk = { workspace = true }
 openvm-stark-sdk = { workspace = true, default-features = false }
 
+backon = { version = "1.5", features = ["std-blocking-sleep"] }
 base64 = "0.22"
 git-version = "0.3.5"
 hex = "0.4"
 munge = "=0.4.1"
 once_cell = "1.20"
+reqwest = { workspace = true, features = ["json", "blocking"] }
 serde = "1.0"
 serde_json = "1.0"
 serde_stacker = "0.1"
 thiserror = "2.0"
 toml = "0.8"
 revm = "19.0"
 c-kzg = { version = "1.0", features = ["serde"] }
+regex = "1.11.1"
 
 [dev-dependencies]
 eyre = "0.6"

crates/prover/src/prover/chunk.rs

@@ -1,5 +1,3 @@
-use scroll_zkvm_circuit_input_types::chunk::{ArchivedChunkWitness, ChunkWitness, execute};
-
 use crate::{
     Error, Prover, ProverType,
     commitments::{chunk, chunk_rv32},
@@ -8,7 +6,9 @@ use crate::{
 };
 
 use super::Commitments;
-
+use alloy_primitives::B256;
+use scroll_zkvm_circuit_input_types::chunk::{ArchivedChunkWitness, ChunkWitness, execute};
+use std::{sync::LazyLock, time::Duration};
 pub struct ChunkCircuit;
 pub struct ChunkCircuitRv32;
 
@@ -32,6 +32,13 @@ pub type ChunkProverRv32 = Prover<ChunkProverTypeRv32>;
 
 pub struct GenericChunkProverType<C: Commitments>(std::marker::PhantomData<C>);
 
+static PROVER_DB_GET_ENDPOINT: LazyLock<Option<String>> = LazyLock::new(|| {
+    std::env::var("PROVER_DB_GET_ENDPOINT")
+        .as_deref()
+        .ok()
+        .map(|s| s.trim_end_matches("/").to_string())
+});
+
 impl<C: Commitments> ProverType for GenericChunkProverType<C> {
     const NAME: &'static str = "chunk";
 
@@ -82,9 +89,100 @@ impl<C: Commitments> ProverType for GenericChunkProverType<C> {
             ))
         })?;
 
-        let chunk_info = execute(chunk_witness)
-            .map_err(|e| Error::GenProof(format!("{}: {}", err_prefix, e)))?;
+        let chunk_info = execute(chunk_witness).map_err(|e| {
+            println!("display {}", e);
+            println!("debug {:?}", e);
+            Error::GenProof(format!("{}: {}", err_prefix, e))
+        })?;
 
         Ok(ChunkProofMetadata { chunk_info })
     }
 }
+
+#[tracing::instrument]
+pub(crate) fn fetch_missing_node(hash: B256) -> Result<sbv_primitives::Bytes, String> {
+    use backon::{BlockingRetryableWithContext, ExponentialBuilder};
+
+    const RETRY_POLICY: ExponentialBuilder = ExponentialBuilder::new()
+        .with_min_delay(Duration::from_millis(100))
+        .with_max_delay(Duration::from_secs(10))
+        .with_max_times(3);
+
+    static CLIENT: LazyLock<reqwest::blocking::Client> = LazyLock::new(|| {
+        reqwest::blocking::ClientBuilder::new()
+            .timeout(Duration::from_secs(10))
+            .build()
+            .expect("Failed to create reqwest client")
+    });
+
+    let body = serde_json::json!({
+        "jsonrpc": "2.0",
+        "id": 1,
+        "method": "debug_dbGet",
+        "params": [hash],
+    });
+    tracing::debug!("request_body: {body:?}");
+
+    #[derive(serde::Deserialize)]
+    //#[serde(untagged)]
+    enum Response {
+        result(sbv_primitives::Bytes),
+        error { code: i64, message: String },
+    }
+    // Define the JSON-RPC response structure
+    #[derive(serde::Deserialize)]
+    struct JsonRpcResponse {
+        jsonrpc: String,
+        id: serde_json::Value, // Use Value to handle numeric or string IDs
+        #[serde(flatten)]
+        result: Response, // Use flatten to handle result or error
+    }
+
+    // struct  Response {
+    //    jsonrpc: String,
+    //    id: u32,
+    //    result: sbv_primitives::Bytes,
+    //}
+
+    let client = reqwest::blocking::ClientBuilder::new()
+        .timeout(Duration::from_secs(10))
+        .build()
+        .expect("Failed to create reqwest client");
+    let endpoint = "https://ancient-smart-sunset.scroll-mainnet.quiknode.pro/310e39a5b18fbd648f52b6e7f763fd40ec94d93f";
+    let (_, result) = {
+        |body| {
+            let result = client
+                .post(endpoint)
+                .json(body)
+                .send()
+                .and_then(|r| r.error_for_status())
+                .and_then(|r| {
+                    // println!("r: {:?}", r.text());
+                    r.json::<JsonRpcResponse>()
+                    // panic!();
+                });
+            (body, result)
+        }
+    }
+    .retry(RETRY_POLICY)
+    .notify(|err, dur| {
+        tracing::debug!("retrying {err:?} after {dur:?}");
+    })
+    .context(&body)
+    .call();
+
+    match result {
+        Ok(r) => {
+            match r.result {
+                // 1. success: {"jsonrpc":"2.0","id":1,"result":"0xe19f3c8dfeea98e16e7fcafcce43929240f3ff23ad27fc7a81d5368b0e9eb6876a32"}
+                Response::result(bytes) => Ok(bytes),
+                // 2. method not supported: {"jsonrpc":"2.0","id":1,"error":{"code":-32604,"message":"this request method is not supported"}}
+                // 3. node not found: {"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"leveldb: not found"}}
+                Response::error { code, message } => {
+                    Err(format!("error from RPC: {code} {message}"))
+                }
+            }
+        }
+        Err(e) => Err(format!("failed after max retries, last err: {e}")),
+    }
+}

crates/prover/src/prover/mod.rs

@@ -210,6 +210,8 @@ impl<Type: ProverType> Prover<Type> {
         &self,
         task: &Type::ProvingTask,
     ) -> Result<WrappedProof<Type::ProofMetadata>, Error> {
+        let mut task = task.clone();
+        let task = &mut task;
         let task_id = task.identifier();
 
         // Try reading proof from cache if available, and early return in that case.
@@ -225,6 +227,7 @@ impl<Type: ProverType> Prover<Type> {
 
         // Generate a new proof.
         assert!(!Type::EVM, "Prover::gen_proof not for EVM-prover");
+
         let metadata = Self::metadata_with_prechecks(task)?;
         let proof = self.gen_proof_stark(task)?;
         let wrapped_proof = WrappedProof::new(metadata, proof, Some(self.get_app_vk().as_slice()));
@@ -249,6 +252,8 @@ impl<Type: ProverType> Prover<Type> {
         &self,
         task: &Type::ProvingTask,
     ) -> Result<WrappedProof<Type::ProofMetadata>, Error> {
+        let mut task = task.clone();
+        let task = &mut task;
         let task_id = task.identifier();
 
         // Try reading proof from cache if available, and early return in that case.
@@ -283,8 +288,47 @@ impl<Type: ProverType> Prover<Type> {
 
     /// Validate some pre-checks on the proving task and construct proof metadata.
     #[instrument("Prover::metadata_with_prechecks", skip_all, fields(?task_id = task.identifier()))]
-    pub fn metadata_with_prechecks(task: &Type::ProvingTask) -> Result<Type::ProofMetadata, Error> {
-        Type::metadata_with_prechecks(task)
+    pub fn metadata_with_prechecks(
+        task: &mut Type::ProvingTask,
+    ) -> Result<Type::ProofMetadata, Error> {
+        const MAX_FETCH_NODES_ATTEMPTS: usize = 15;
+        let mut attempts = 0;
+        loop {
+            let result = Type::metadata_with_prechecks(task);
+            match result {
+                Ok(metadata) => {
+                    return Ok(metadata);
+                }
+                Err(e) => {
+                    let pattern = r"SparseTrieError\(BlindedNode \{ path: Nibbles\((0x[0-9a-fA-F]+)\), hash: (0x[0-9a-fA-F]+) \}\)";
+                    let re = regex::Regex::new(pattern).unwrap();
+                    match re.captures(format!("{e}").as_str()) {
+                        None => {
+                            return Err(e);
+                        }
+                        Some(caps) => {
+                            let hash = caps[2].to_string();
+                            println!("missing trie hash {hash}");
+
+                            attempts += 1;
+                            if attempts >= MAX_FETCH_NODES_ATTEMPTS {
+                                return Err(Error::GenProof(format!(
+                                    "failed to fetch nodes after {MAX_FETCH_NODES_ATTEMPTS} attempts: {e}"
+                                )));
+                            }
+
+                            let b256 = hash.parse::<sbv_primitives::B256>().expect("should be hex");
+                            let node =
+                                crate::prover::chunk::fetch_missing_node(b256).map_err(|e| {
+                                    Error::GenProof(format!("failed to fetch missing node: {e}",))
+                                })?;
+                            tracing::warn!("missing node fetched: {node}");
+                            task.insert_state(node);
+                        }
+                    }
+                }
+            }
+        }
     }
 
     /// Verify a [root proof][root_proof].

crates/prover/src/task/chunk.rs

@@ -75,6 +75,10 @@ impl ProvingTask for ChunkProvingTask {
         ForkName::from(self.fork_name.as_str())
     }
 
+    fn insert_state(&mut self, node: alloy_primitives::Bytes) {
+        self.block_witnesses[0].states.push(node);
+    }
+
     fn build_guest_input(&self) -> Result<StdIn, rkyv::rancor::Error> {
         let witness = ChunkWitness {
             blocks: self.block_witnesses.to_vec(),

crates/prover/src/task/mod.rs

@@ -15,12 +15,15 @@ pub mod bundle;
 
 /// Every proving task must have an identifier. The identifier will be appended to a prefix while
 /// storing/reading proof to/from disc.
-pub trait ProvingTask: serde::de::DeserializeOwned {
+pub trait ProvingTask: serde::de::DeserializeOwned + Clone {
     fn identifier(&self) -> String;
 
     fn build_guest_input(&self) -> Result<StdIn, rkyv::rancor::Error>;
 
     fn fork_name(&self) -> ForkName;
+
+    // adhoc
+    fn insert_state(&mut self, witness: sbv_primitives::Bytes) {}
 }
 
 /// Flatten a [`WrappedProof`] and split the proof from the public values. We also split out the