Refactor backend

* refactor: extract TraceBackend as a trait

* remove Iterator implementation for Slice Backend

* fix clippy

* fix examples compiling error

* add (De)Serialize for Tables

* Add Serialize, Deserialize for Slice

crates/cli/src/command.rs

@@ -23,8 +23,8 @@ use halo2_proofs::plonk::Circuit;
 use halo2_proofs::plonk::CircuitData;
 use halo2_proofs::poly::commitment::Params;
 use specs::slice::Slice;
+use specs::slice_backend::InMemoryBackendBuilder;
 use specs::CompilationTable;
-use specs::TraceBackend;
 
 use crate::args::HostMode;
 use crate::config::Config;
@@ -159,9 +159,9 @@ impl SetupArg {
         let env = env_builder.create_env_without_value();
         let mut monitor = TableMonitor::new(
             self.k,
+            InMemoryBackendBuilder,
             env_builder.create_flush_strategy(),
             &self.phantom_functions,
-            TraceBackend::Memory,
             &env,
         );
 

crates/cli/src/config.rs

@@ -28,10 +28,12 @@ use halo2_proofs::poly::commitment::Params;
 use indicatif::ProgressBar;
 use serde::Deserialize;
 use serde::Serialize;
-use specs::TraceBackend;
+use specs::slice_backend::SliceBackendBuilder;
 
 use crate::args::HostMode;
 use crate::names::name_of_circuit_data;
+use crate::names::name_of_etable_slice;
+use crate::names::name_of_external_host_call_table_slice;
 use crate::names::name_of_frame_table_slice;
 use crate::names::name_of_instance;
 use crate::names::name_of_loadinfo;
@@ -235,16 +237,16 @@ impl Config {
         Ok(())
     }
 
-    pub(crate) fn prove(
+    pub(crate) fn prove<B: SliceBackendBuilder>(
         self,
+        slice_backend_builder: B,
         env_builder: &dyn HostEnvBuilder,
         wasm_image: &Path,
         params_dir: &Path,
         output_dir: &Path,
         arg: ExecutionArg,
         context_output_filename: Option<String>,
         mock_test: bool,
-        table_backend: TraceBackend,
         skip: usize,
         padding: Option<usize>,
     ) -> anyhow::Result<()> {
@@ -260,9 +262,9 @@ impl Config {
 
         let mut monitor = TableMonitor::new(
             self.k,
+            slice_backend_builder,
             env_builder.create_flush_strategy(),
             &self.phantom_functions,
-            table_backend,
             &env,
         );
 
@@ -308,7 +310,12 @@ impl Config {
                 style("[5/8]").bold().dim(),
                 dir
             );
-            tables.write(&dir, |slice| name_of_frame_table_slice(&self.name, slice));
+            tables.write(
+                &dir,
+                |index| name_of_frame_table_slice(&self.name, index),
+                |index| name_of_etable_slice(&self.name, index),
+                |index| name_of_external_host_call_table_slice(&self.name, index),
+            )?;
         }
 
         println!("{} Build circuit(s)...", style("[6/8]").bold().dim(),);
@@ -324,9 +331,9 @@ impl Config {
             ProofGenerationInfo::new(&self.name, self.k as usize, HashType::Poseidon);
 
         let progress_bar = ProgressBar::new(if let Some(padding) = padding {
-            usize::max(tables.execution_tables.etable.len(), padding) as u64
+            usize::max(tables.execution_tables.slice_backend.len(), padding) as u64
         } else {
-            tables.execution_tables.etable.len() as u64
+            tables.execution_tables.slice_backend.len() as u64
         });
 
         if skip != 0 {
@@ -335,12 +342,11 @@ impl Config {
         }
 
         let mut slices = Slices::new(self.k, tables, padding)?
+            .into_iter()
             .enumerate()
             .skip(skip)
             .peekable();
         while let Some((index, circuit)) = slices.next() {
-            let circuit = circuit?;
-
             let _is_finalized_circuit = slices.peek().is_none();
 
             if mock_test {

crates/cli/src/file_backend.rs

@@ -0,0 +1,134 @@
+use std::fs;
+use std::io;
+use std::path::Path;
+use std::path::PathBuf;
+
+use serde::Deserialize;
+use serde::Serialize;
+use specs::etable::EventTable;
+use specs::external_host_call_table::ExternalHostCallTable;
+use specs::jtable::FrameTable;
+use specs::slice_backend::Slice;
+use specs::slice_backend::SliceBackend;
+use specs::slice_backend::SliceBackendBuilder;
+
+use crate::names::name_of_etable_slice;
+use crate::names::name_of_external_host_call_table_slice;
+use crate::names::name_of_frame_table_slice;
+
+struct SlicePath {
+    event_table: PathBuf,
+    frame_table: PathBuf,
+    external_host_call_table: PathBuf,
+}
+
+impl From<&SlicePath> for Slice {
+    fn from(val: &SlicePath) -> Self {
+        Slice {
+            etable: EventTable::read(&val.event_table).unwrap(),
+            frame_table: FrameTable::read(&val.frame_table).unwrap(),
+            external_host_call_table: ExternalHostCallTable::read(&val.external_host_call_table)
+                .unwrap(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub(crate) struct FileBackendSlice {
+    event_table: PathBuf,
+    frame_table: PathBuf,
+    external_host_call_table: PathBuf,
+}
+
+impl From<FileBackendSlice> for Slice {
+    fn from(value: FileBackendSlice) -> Self {
+        Slice {
+            etable: EventTable::read(&value.event_table).unwrap(),
+            frame_table: FrameTable::read(&value.frame_table).unwrap(),
+            external_host_call_table: ExternalHostCallTable::read(&value.external_host_call_table)
+                .unwrap(),
+        }
+    }
+}
+
+impl SliceBackend for FileBackendSlice {
+    fn write(
+        &self,
+        path_of_event_table: &Path,
+        path_of_frame_table: &Path,
+        path_of_external_host_call_table: &Path,
+    ) -> io::Result<()> {
+        if self.event_table.as_path().canonicalize()? != path_of_event_table.canonicalize()? {
+            fs::copy(self.event_table.as_path(), path_of_event_table)?;
+        }
+        if self.frame_table.as_path().canonicalize()? != path_of_frame_table.canonicalize()? {
+            fs::copy(self.frame_table.as_path(), path_of_frame_table)?;
+        }
+        if self.external_host_call_table.as_path().canonicalize()?
+            != path_of_external_host_call_table.canonicalize()?
+        {
+            fs::copy(
+                self.external_host_call_table.as_path(),
+                path_of_external_host_call_table,
+            )?;
+        }
+
+        Ok(())
+    }
+}
+
+pub(crate) struct FileBackendBuilder {
+    name: String,
+    dir: PathBuf,
+    index: usize,
+}
+
+impl FileBackendBuilder {
+    pub(crate) fn new(name: String, dir: PathBuf) -> Self {
+        Self {
+            name,
+            dir,
+            index: 0,
+        }
+    }
+}
+
+impl SliceBackendBuilder for FileBackendBuilder {
+    type Output = FileBackendSlice;
+
+    fn build(&mut self, slice: Slice) -> Self::Output {
+        let event_table = {
+            let path = self
+                .dir
+                .join(PathBuf::from(name_of_etable_slice(&self.name, self.index)));
+            slice.etable.write(&path).unwrap();
+            path
+        };
+
+        let frame_table = {
+            let path = self.dir.join(PathBuf::from(name_of_frame_table_slice(
+                &self.name, self.index,
+            )));
+            slice.frame_table.write(&path).unwrap();
+            path
+        };
+
+        let external_host_call_table = {
+            let path = self
+                .dir
+                .join(PathBuf::from(name_of_external_host_call_table_slice(
+                    &self.name, self.index,
+                )));
+            slice.external_host_call_table.write(&path).unwrap();
+            path
+        };
+
+        self.index += 1;
+
+        FileBackendSlice {
+            event_table,
+            frame_table,
+            external_host_call_table,
+        }
+    }
+}

crates/cli/src/main.rs

@@ -17,18 +17,16 @@ use delphinus_zkwasm::runtime::host::default_env::ExecutionArg;
 use args::HostMode;
 use config::Config;
 use delphinus_zkwasm::runtime::host::HostEnvBuilder;
+use file_backend::FileBackendBuilder;
 use names::name_of_config;
-use names::name_of_etable_slice;
-use names::name_of_frame_table_slice;
 use specs::args::parse_args;
-use specs::etable::EventTable;
-use specs::jtable::FrameTable;
-use specs::TraceBackend;
+use specs::slice_backend::InMemoryBackendBuilder;
 
 mod app_builder;
 mod args;
 mod command;
 mod config;
+mod file_backend;
 mod names;
 
 const TRIVIAL_WASM: &str = r#"
@@ -104,68 +102,54 @@ fn main() -> Result<()> {
             let private_inputs = parse_args(&arg.running_arg.private_inputs);
             let context_inputs = parse_args(&arg.running_arg.context_inputs);
 
-            let trace_backend: TraceBackend = if arg.file_backend {
-                let event_table_writer = {
-                    let name = cli.name.clone();
-                    let trace_dir = trace_dir.clone();
-
-                    Box::new(move |slice, etable: &EventTable| {
-                        let filename_of_etable_slice =
-                            PathBuf::from(name_of_etable_slice(&name, slice));
-                        let path = trace_dir.join(filename_of_etable_slice);
-
-                        etable.write(&path).unwrap();
-
-                        path
-                    })
-                };
-
-                let frame_table_writer = {
-                    let name = cli.name.clone();
-                    let trace_dir = trace_dir;
-
-                    Box::new(move |slice, frame_table: &FrameTable| {
-                        let filename_of_frame_table_slice =
-                            PathBuf::from(name_of_frame_table_slice(&name, slice));
-                        let path = trace_dir.join(filename_of_frame_table_slice);
-
-                        frame_table.write(&path).unwrap();
-
-                        path
-                    })
-                };
-
-                TraceBackend::File {
-                    event_table_writer,
-                    frame_table_writer,
-                }
-            } else {
-                TraceBackend::Memory
-            };
-
             let env_builder: Box<dyn HostEnvBuilder> = match config.host_mode {
                 HostMode::Default => Box::new(DefaultHostEnvBuilder::new(config.k)),
                 HostMode::Standard => Box::new(StandardHostEnvBuilder::new(config.k)),
             };
 
-            config.prove(
-                &*env_builder,
-                &arg.wasm_image,
-                &cli.params_dir,
-                &arg.output_dir,
-                ExecutionArg {
-                    public_inputs,
-                    private_inputs,
-                    context_inputs,
-                    indexed_witness: Rc::new(RefCell::new(HashMap::default())),
-                    tree_db: None,
-                },
-                arg.running_arg.context_output,
-                arg.mock_test,
-                trace_backend,
-                arg.skip,
-                arg.padding,
-            )?;
+            if arg.file_backend {
+                let backend_builder = FileBackendBuilder::new(cli.name.clone(), trace_dir);
+
+                config.prove(
+                    backend_builder,
+                    &*env_builder,
+                    &arg.wasm_image,
+                    &cli.params_dir,
+                    &arg.output_dir,
+                    ExecutionArg {
+                        public_inputs,
+                        private_inputs,
+                        context_inputs,
+                        indexed_witness: Rc::new(RefCell::new(HashMap::default())),
+                        tree_db: None,
+                    },
+                    arg.running_arg.context_output,
+                    arg.mock_test,
+                    arg.skip,
+                    arg.padding,
+                )?;
+            } else {
+                let backend_builder = InMemoryBackendBuilder;
+
+                config.prove(
+                    backend_builder,
+                    &*env_builder,
+                    &arg.wasm_image,
+                    &cli.params_dir,
+                    &arg.output_dir,
+                    ExecutionArg {
+                        public_inputs,
+                        private_inputs,
+                        context_inputs,
+                        indexed_witness: Rc::new(RefCell::new(HashMap::default())),
+                        tree_db: None,
+                    },
+                    arg.running_arg.context_output,
+                    arg.mock_test,
+                    arg.skip,
+                    arg.padding,
+                )?;
+            }
         }
         Subcommands::Verify(arg) => {
             let config = Config::read(&mut fs::File::open(

crates/cli/src/names.rs

@@ -47,3 +47,8 @@ pub(crate) fn name_of_etable_slice(name: &str, index: usize) -> String {
 pub(crate) fn name_of_frame_table_slice(name: &str, index: usize) -> String {
     format!("{}.frame_table.{}.data", name, index)
 }
+
+#[inline(always)]
+pub(crate) fn name_of_external_host_call_table_slice(_name: &str, index: usize) -> String {
+    format!("external_host_table.{}.json", index)
+}