Add CompactBinary proof serialization

Cargo.toml

@@ -3,6 +3,8 @@ members = [
     "crates/stwo",
     "crates/air-utils",
     "crates/air-utils-derive",
+    "crates/compact-binary",
+    "crates/compact-binary-derive",
     "crates/constraint-framework",
     "crates/examples",
     "crates/std-shims",
@@ -32,6 +34,7 @@ rand = { version = "0.8.5", default-features = false, features = ["small_rng"] }
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 hashbrown = ">=0.15.2"
 std-shims = { path = "crates/std-shims", default-features = false }
+unsigned-varint = "0.8"
 
 [profile.bench]
 codegen-units = 1

crates/compact-binary-derive/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "stwo-compact-binary-derive"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1.0.37"
+syn = "2.0.90"

crates/compact-binary-derive/src/lib.rs

@@ -0,0 +1,161 @@
+use proc_macro::TokenStream;
+use quote::{quote, ToTokens};
+use syn::{parse_macro_input, parse_quote, Data, DeriveInput, Fields, Type};
+
+/// Proc macro to automatically derive `CompactBinary` trait for structs.
+#[proc_macro_derive(CompactBinary, attributes(zipped))]
+pub fn derive_compact_binary(input: TokenStream) -> TokenStream {
+    // Parse the input tokens into a syntax tree.
+    let input = parse_macro_input!(input as DeriveInput);
+
+    let struct_name = input.ident;
+    let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+
+    // Extract the fields of the struct.
+    let fields = match input.data {
+        Data::Struct(ref data_struct) => match &data_struct.fields {
+            Fields::Named(ref fields_named) => &fields_named.named,
+            Fields::Unnamed(_) | Fields::Unit => {
+                return syn::Error::new_spanned(
+                    struct_name,
+                    "CompactBinary can only be derived for structs with named fields.",
+                )
+                .to_compile_error()
+                .into();
+            }
+        },
+        _ => {
+            return syn::Error::new_spanned(
+                struct_name,
+                "CompactBinary can only be derived for structs.",
+            )
+            .to_compile_error()
+            .into();
+        }
+    };
+
+    // Check if MerkleHasher is present in the where clause or generics
+    let h_is_merklehasher = input
+        .generics
+        .where_clause
+        .as_ref()
+        .map(|wc| {
+            wc.predicates.iter().any(|pred| {
+                pred.to_token_stream()
+                    .to_string()
+                    .contains("H: MerkleHasher")
+            })
+        })
+        .unwrap_or(false)
+        || input.generics.params.iter().any(|param| {
+            if let syn::GenericParam::Type(ty) = param {
+                ty.bounds
+                    .iter()
+                    .any(|b| b.to_token_stream().to_string().contains("MerkleHasher"))
+            } else {
+                false
+            }
+        });
+
+    // Check if any field requires H bounds
+    let needs_h_bounds = fields.iter().any(|f| {
+        if let Type::Path(type_path) = &f.ty {
+            if let Some(seg) = type_path.path.segments.last() {
+                if let syn::PathArguments::AngleBracketed(ref args) = seg.arguments {
+                    return args.args.iter().any(|arg| {
+                        if let syn::GenericArgument::Type(Type::Path(type_path)) = arg {
+                            type_path
+                                .path
+                                .segments
+                                .last()
+                                .is_some_and(|s| s.ident == "H")
+                        } else {
+                            false
+                        }
+                    });
+                }
+            }
+        }
+        false
+    });
+
+    let mut where_clause = where_clause.cloned();
+    // If MerkleHasher is present and H bounds are needed, add the necessary bounds.
+    if h_is_merklehasher && needs_h_bounds {
+        let pred: syn::WherePredicate =
+            parse_quote! { H::Hash: stwo::core::compact_binary::CompactBinary };
+        if let Some(ref mut wc) = where_clause {
+            wc.predicates.push(pred);
+        } else {
+            where_clause = Some::<syn::WhereClause>(
+                parse_quote! { where H::Hash: stwo::core::compact_binary::CompactBinary },
+            );
+        }
+    }
+
+    // Generate code to serialize each field in the order they appear.
+    let compact_serialize_body = fields.iter().enumerate().map(|(i, f)| {
+        let field_name = &f.ident;
+        let field_type = &f.ty;
+        let is_zipped = f.attrs.iter().any(|attr| attr.path().is_ident("zipped"));
+        match is_zipped {
+            true => {
+                quote! {
+                    usize::compact_serialize(&#i, output)?;
+                    let #field_name = stwo::core::compact_binary::ZippedCompactBinary(&self.#field_name);
+                    stwo::core::compact_binary::ZippedCompactBinary::<&#field_type>::compact_serialize(&#field_name, output)?;
+                }
+            }
+            false => {
+                quote! {
+                    usize::compact_serialize(&#i, output)?;
+                    stwo::core::compact_binary::CompactBinary::compact_serialize(&self.#field_name, output)?;
+                }
+            }
+        }
+    });
+
+    // Generate code to deserialize each field in the order they appear.
+    let compact_deserialize_let_bindings = fields.iter().enumerate().map(|(i, f)| {
+        let field_name = &f.ident;
+        let field_type = &f.ty;
+        let is_zipped = f.attrs.iter().any(|attr| attr.path().is_ident("zipped"));
+        match is_zipped {
+            true => {
+                quote! {
+                    let input = stwo::core::compact_binary::strip_expected_tag(input, #i)?;
+                    let (input, #field_name) = stwo::core::compact_binary::ZippedCompactBinary::<&#field_type>::compact_deserialize(input)?;
+                }
+            }
+            false => {
+                quote! {
+                    let input = stwo::core::compact_binary::strip_expected_tag(input, #i)?;
+                    let (input, #field_name) = stwo::core::compact_binary::CompactBinary::compact_deserialize(input)?;
+                }
+            }
+        }
+    });
+    let compact_deserialize_struct_fields = fields.iter().map(|f| {
+        let field_name = &f.ident;
+        quote! { #field_name }
+    });
+
+    // Implement `CompactBinary` for the type.
+    let expanded = quote! {
+        impl #impl_generics stwo::core::compact_binary::CompactBinary for #struct_name #ty_generics #where_clause {
+            fn compact_serialize(&self, output: &mut Vec<u8>) -> Result<(), stwo::core::compact_binary::CompactSerializeError> {
+                u32::compact_serialize(&0, output)?;
+                #(#compact_serialize_body)*
+                Ok(())
+            }
+
+            fn compact_deserialize<'a>(mut input: &'a [u8]) -> Result<(&'a [u8], Self), stwo::core::compact_binary::CompactDeserializeError> {
+                let input = stwo::core::compact_binary::strip_expected_version(input, 0)?;
+                #(#compact_deserialize_let_bindings)*
+                Ok((input, Self { #(#compact_deserialize_struct_fields),* }))
+            }
+        }
+    };
+
+    TokenStream::from(expanded)
+}

crates/compact-binary/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "stwo-compact-binary"
+version.workspace = true
+edition.workspace = true
+
+[features]
+default = ["std"]
+std = [
+    "std-shims/std",
+]
+
+[dependencies]
+starknet-ff = { version = "0.3.7", default-features = false, features = [
+    "alloc",
+    "serde",
+] }
+unsigned-varint.workspace = true
+lz4_flex = { version = "0.11", default-features = false }
+std-shims.workspace = true
+
+[lib]
+bench = false

crates/compact-binary/README.md

@@ -0,0 +1,61 @@
+# Compact binary format specs
+
+## Needs
+
+Stwo proofs can be serialized in two formats:
+
+- `json`: each field of each struct is serialized as base json format.
+- `cairo-serde`: the proof is first converted into a `Vec<FieldElement>`, that is then serialized in a serde json format.
+
+This crate implements a third proof serialization format, `compact-binary`, where the proof is serialized as a `Vec<u8>` in a compact way.
+
+## Format description
+
+- Integers (`u32`, `u64` and `usize`) should be handled as VarInts
+- Relevant fields should be compactified if possible or compressed.
+- Structured data should have:
+  - versions numbers, to be able to update the structure
+  - tags for each field, to be able to add new fields easily
+
+## Versioning description
+
+If we want to add or change a field of a struct `StructA`, while still being able to deserialize previous versions of this struct, we should:
+
+- Update `compact_serialize()` to serialize a new version number, and serialize the new struct
+- Update `compact_deserialize()` to:
+  - Get the version of the deserialized struct
+  - Match on it and dispatch to the deserialization logic corresponding to this version
+
+## Implementation
+
+The current implementation consists of the following elements:
+
+- A `CompactBinary` trait in `crates/compact-binary/src/lib.rs`, along with helper functions and implementations for base structures.
+- A `#[derive(CompactBinary)]` proc macro to implement the trait for structures composed of fields implementing it. Note that the proc macro is only expected to produce a `0` version, if a given structure is to be updated it's implementation should be done manually, while keeping back-compatibility of all previous serialization versions for this structure. See `crates/compact-binary-derive/src/lib.rs`
+Note that the proc macro supports the `#[zipped]` attribute to specify that a given field should be zipped (compressed with LZ4 compression).
+- Error handling through `CompactDeserializeError` enum and `CompactSerializeError` struct
+- Implementations of the `CompactBinary` the trait for structures in `stwo` crate used for CairoProofs.
+
+In the [stwo-cairo repository](https://github.com/starkware-libs/stwo-cairo):
+
+- A `CompactBinary` implementation for `CairoProof`
+- Argument handling for proof and verification in the CLI (added `--proof-format compact-binary`). See `cairo-prove/src/main.rs` and `cairo-prove/src/args.rs`.
+
+## Tests and benchmarks
+
+We execute the example proof in the [stwo-cairo repository](https://github.com/starkware-libs/stwo-cairo):
+```bash
+cairo-prove/target/release/cairo-prove prove cairo-prove/example/target/dev/example.executable.json ./example_proof.compact_bin --arguments 10000 --proof-format compact-binary
+cairo-prove/target/release/cairo-prove verify ./example_proof.compact_bin --proof-format compact-binary
+```
+
+**Note that we've adapted the serialize_proof_to_file() function to use serde_json without a JSON prettier to have more accurate results** 
+
+For this example proof, here are the results:
+
+| File                               | Format            | Size on disk (bytes) | Gain     |
+|------------------------------------|-------------------|---------------------:|---------:|
+| example_proof.base_json            | json              |           2 528 114  |    --    |
+| example_proof.cairo_serde          | cairo-serde       |           2 448 494  |  - 3.1 % |
+| example_proof.compact_bin_unzipped | compact-binary    |             834 606  | - 67.0 % |
+| example_proof.compact_bin_zipped   | compact-binary    |             582 932  | - 76.9 % |