Add dynamic layout basic support (#1824)

* Fix Zero segment location.

* Fixed has_zero_segment naming

* Fix prover input.

* Fixed version reading when no version is supplied

* Added change to changelog.

* fix test_from_serializable()

* fix panic_impl error

* fix cairo version

* Add dummy changelog

* Pin wasm-bindgen

* Register change in CHANGELOG

* Update Cargo.lock

* Remove changes from CHANGELOG

* Add argument parsing for layout params file

* Add dynamic support (no implement)

* Add cairo_layout_params_file.example.json

* Implement dynamic layout creation

* Update CHANGELOG

* Add cli dynamic support for cairo 1

* Make wasm compatible

* Use public_memory_fraction = 4 vy default

* Deserialize bool from int

* Add comparison with python-vm (failing)

* Rebuild .rs files in makefile

* Use 8 as dynamic public_memory_fraction

The same value is used in python-vm

* Use None ratio for dynamic unused builtins

* Add rangecheck96 to private inputs

* Make dyn py files depend on params_file

* Use cpu_component_step=1 by default

* Fix typo in private inputs

* Add range check value to air private input test

* Fix zero segment location

* Use zero builtin instead of None

* Add debug scripts

* Remove dup makefile recipes

* remove outdated test

* Enable ensure-no_std on test

* Fix tests

* Add correct test

* Rename tset

* Add comment

* Add debugging document

* Update cairo layout params file

* Remove duplicated range check

* Remove dup

* Remove debugging and scrippts (moveed to another branch)

* Add comment

* Add tests

* Add dynamic test to cairo-vm-cli

* Add parse test

* Remove compare all dynamic

* Add script for comparing with dynamic layouts

* Add tests to workflow

* Delete logic changes

They are going to be moved to another branch

* Delete more logic changes

* Update rust.yml

* Rename compare_outputs_dynamic_layout.sh to compare_outputs_dynamic_layouts.sh

* Update test script

* Add more tests

* Rename parameter for clarity

* Enable mod builtin only on feature with dynamic layout

* Remove debug assert

* Refactor errors into variants

* Fix failing test

* Move cairo_layout_params_file to test folder

* Document cairo_layout_param_file in the README.md

* Fix clippy warning

* Use mod_builtin feature in tests

---------

Co-authored-by: Alon Titelman <[email protected]>
Co-authored-by: Yuval Goldberg <[email protected]>
Co-authored-by: Omri Eshhar <[email protected]>

Author: 4 people

.github/workflows/rust.yml

@@ -725,6 +725,45 @@ jobs:
     - name: Run script
       run: ./vm/src/tests/compare_factorial_outputs_all_layouts.sh
 
+  compare-outputs-dynamic-layouts:
+    name: Compare outputs with dynamic layouts
+    needs: [ build-programs, build-release ]
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Python3 Build
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        cache: 'pip'
+
+    - name: Install cairo-lang and deps
+      run: pip install -r requirements.txt
+
+    - name: Fetch release binary
+      uses: actions/cache/restore@v3
+      with:
+        key: cli-bin-rel-${{ github.sha }}
+        path: target/release/cairo-vm-cli
+        fail-on-cache-miss: true
+
+    - uses: actions/download-artifact@master
+      with:
+        name: proof_programs
+        path: cairo_programs/proof_programs/
+
+    - name: Fetch programs
+      uses: actions/cache/restore@v3
+      with:
+        path: ${{ env.CAIRO_PROGRAMS_PATH }}
+        key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'examples/wasm-demo/src/array_sum.cairo') }}
+        fail-on-cache-miss: true
+
+    - name: Run script
+      run: ./vm/src/tests/compare_outputs_dynamic_layouts.sh
+
   compare-run-from-cairo-pie-all-outputs:
     name: Compare all outputs from running Cairo PIEs
     needs: [ build-programs, build-release, run-cairo-release ]

.gitignore

@@ -26,4 +26,5 @@ cairo-vm-cli/air_input.pub
 ensure-no_std/Cargo.lock
 cairo_programs/proof_programs/*.cairo
 
+!cairo_layout_params_file.json
 !vm/src/tests/cairo_pie_test_output.json

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 #### Upcoming Changes
 
+* feat(BREAKING): [#1824](https://github.com/lambdaclass/cairo-vm/pull/1824):
+    * Add support for dynamic layout
+    * CLI change(BREAKING): The flag `cairo_layout_params_file` must be specified when using dynamic layout.
+    * Signature change(BREAKING): Both `CairoRunner::new` and `CairoRunner::new_v2` now receive an `Option<CairoLayoutParams>`, used only with dynamic layout.
+
 * chore: bump pip `cairo-lang` 0.13.2 [#1827](https://github.com/lambdaclass/cairo-vm/pull/1827)
 
 * chore: bump `cairo-lang-` dependencies to 2.8.0 [#1833](https://github.com/lambdaclass/cairo-vm/pull/1833/files)

README.md

@@ -182,6 +182,8 @@ The cairo-vm-cli supports the following optional arguments:
 
 - `run_from_cairo_pie`: Runs a Cairo PIE instead of a compiled json file. The name of the file will be the first argument received by the CLI (as if it were to run a normal compiled program). Can only be used if proof_mode is not enabled.
 
+- `cairo_layout_params_file`: Only used with dynamic layout. Receives the name of a json file with the dynamic layout parameters.
+
 For example, to obtain the air public inputs from a fibonacci program run, we can run :
 
 ```bash

bench/criterion_benchmark.rs

@@ -30,6 +30,7 @@ fn build_many_runners(c: &mut Criterion) {
                 CairoRunner::new(
                     black_box(&program),
                     black_box(LayoutName::starknet_with_keccak),
+                    black_box(None),
                     black_box(false),
                     black_box(false),
                 )
@@ -45,7 +46,16 @@ fn load_program_data(c: &mut Criterion) {
     let program = Program::from_bytes(program.as_slice(), Some("main")).unwrap();
     c.bench_function("initialize", |b| {
         b.iter_batched(
-            || CairoRunner::new(&program, LayoutName::starknet_with_keccak, false, false).unwrap(),
+            || {
+                CairoRunner::new(
+                    &program,
+                    LayoutName::starknet_with_keccak,
+                    None,
+                    false,
+                    false,
+                )
+                .unwrap()
+            },
             |mut runner| _ = black_box(runner.initialize(false).unwrap()),
             BatchSize::SmallInput,
         )

bench/iai_benchmark.rs

@@ -34,6 +34,7 @@ fn build_runner() {
     let runner = CairoRunner::new(
         black_box(&program),
         LayoutName::starknet_with_keccak,
+        None,
         false,
         false,
     )
@@ -47,7 +48,14 @@ fn build_runner_helper() -> CairoRunner {
     //Picked the biggest one at the time of writing
     let program = include_bytes!("../cairo_programs/benchmarks/keccak_integration_benchmark.json");
     let program = Program::from_bytes(program.as_slice(), Some("main")).unwrap();
-    CairoRunner::new(&program, LayoutName::starknet_with_keccak, false, false).unwrap()
+    CairoRunner::new(
+        &program,
+        LayoutName::starknet_with_keccak,
+        None,
+        false,
+        false,
+    )
+    .unwrap()
 }
 
 #[inline(never)]

cairo-vm-cli/src/main.rs

@@ -6,6 +6,7 @@ use cairo_vm::cairo_run::{self, EncodeTraceError};
 use cairo_vm::hint_processor::builtin_hint_processor::builtin_hint_processor_definition::BuiltinHintProcessor;
 #[cfg(feature = "with_tracer")]
 use cairo_vm::serde::deserialize_program::DebugInfo;
+use cairo_vm::types::layout::CairoLayoutParams;
 use cairo_vm::types::layout_name::LayoutName;
 use cairo_vm::vm::errors::cairo_run_errors::CairoRunError;
 use cairo_vm::vm::errors::trace_errors::TraceError;
@@ -43,8 +44,13 @@ struct Args {
     entrypoint: String,
     #[structopt(long = "memory_file")]
     memory_file: Option<PathBuf>,
+    /// When using dynamic layout, it's parameters must be specified through a layout params file.
     #[clap(long = "layout", default_value = "plain", value_enum)]
     layout: LayoutName,
+    /// Required when using with dynamic layout.
+    /// Ignored otherwise.
+    #[clap(long = "cairo_layout_params_file", required_if_eq("layout", "dynamic"))]
+    cairo_layout_params_file: Option<PathBuf>,
     #[structopt(long = "proof_mode")]
     proof_mode: bool,
     #[structopt(long = "secure_run")]
@@ -162,6 +168,11 @@ fn run(args: impl Iterator<Item = String>) -> Result<(), Error> {
 
     let trace_enabled = args.trace_file.is_some() || args.air_public_input.is_some();
 
+    let cairo_layout_params = match args.cairo_layout_params_file {
+        Some(file) => Some(CairoLayoutParams::from_file(&file)?),
+        None => None,
+    };
+
     let cairo_run_config = cairo_run::CairoRunConfig {
         entrypoint: &args.entrypoint,
         trace_enabled,
@@ -170,6 +181,7 @@ fn run(args: impl Iterator<Item = String>) -> Result<(), Error> {
         proof_mode: args.proof_mode,
         secure_run: args.secure_run,
         allow_missing_builtins: args.allow_missing_builtins,
+        dynamic_layout_params: cairo_layout_params,
         ..Default::default()
     };
 
@@ -405,6 +417,19 @@ mod tests {
         assert_matches!(run(args), Err(Error::Runner(_)));
     }
 
+    #[test]
+    fn test_run_dynamic_params() {
+        let mut args = vec!["cairo-vm-cli".to_string()];
+        args.extend_from_slice(&["--layout".to_string(), "dynamic".to_string()]);
+        args.extend_from_slice(&[
+            "--cairo_layout_params_file".to_string(),
+            "../vm/src/tests/cairo_layout_params_file.json".to_string(),
+        ]);
+        args.push("../cairo_programs/proof_programs/fibonacci.json".to_string());
+
+        assert_matches!(run(args.into_iter()), Ok(_));
+    }
+
     //Since the functionality here is trivial, I just call the function
     //to fool Codecov.
     #[test]

cairo1-run/src/cairo_run.rs

@@ -38,8 +38,8 @@ use cairo_vm::{
     math_utils::signed_felt,
     serde::deserialize_program::{ApTracking, FlowTrackingData, HintParams, ReferenceManager},
     types::{
-        builtin_name::BuiltinName, layout_name::LayoutName, program::Program,
-        relocatable::MaybeRelocatable,
+        builtin_name::BuiltinName, layout::CairoLayoutParams, layout_name::LayoutName,
+        program::Program, relocatable::MaybeRelocatable,
     },
     vm::{
         errors::{runner_errors::RunnerError, vm_errors::VirtualMachineError},
@@ -86,6 +86,7 @@ pub struct Cairo1RunConfig<'a> {
     pub relocate_mem: bool,
     /// Cairo layout chosen for the run
     pub layout: LayoutName,
+    pub dynamic_layout_params: Option<CairoLayoutParams>,
     /// Run in proof_mode
     pub proof_mode: bool,
     /// Should be true if either air_public_input or cairo_pie_output are needed
@@ -106,6 +107,7 @@ impl Default for Cairo1RunConfig<'_> {
             proof_mode: false,
             finalize_builtins: false,
             append_return_values: false,
+            dynamic_layout_params: None,
         }
     }
 }
@@ -248,6 +250,7 @@ pub fn cairo_run_program(
     let mut runner = CairoRunner::new_v2(
         &program,
         cairo_run_config.layout,
+        cairo_run_config.dynamic_layout_params.clone(),
         runner_mode,
         cairo_run_config.trace_enabled,
     )?;

cairo1-run/src/main.rs

@@ -4,6 +4,7 @@ use cairo1_run::{cairo_run_program, Cairo1RunConfig, FuncArg};
 use cairo_lang_compiler::{
     compile_prepared_db, db::RootDatabase, project::setup_project, CompilerConfig,
 };
+use cairo_vm::types::layout::CairoLayoutParams;
 use cairo_vm::{
     air_public_input::PublicInputError, types::layout_name::LayoutName,
     vm::errors::trace_errors::TraceError, Felt252,
@@ -24,8 +25,13 @@ struct Args {
     trace_file: Option<PathBuf>,
     #[structopt(long = "memory_file")]
     memory_file: Option<PathBuf>,
+    /// When using dynamic layout, it's parameters must be specified through a layout params file.
     #[clap(long = "layout", default_value = "plain", value_enum)]
     layout: LayoutName,
+    /// Required when using with dynamic layout.
+    /// Ignored otherwise.
+    #[clap(long = "cairo_layout_params_file", required_if_eq("layout", "dynamic"))]
+    cairo_layout_params_file: Option<PathBuf>,
     #[clap(long = "proof_mode", value_parser)]
     proof_mode: bool,
     #[clap(long = "air_public_input", requires = "proof_mode")]
@@ -153,6 +159,11 @@ fn run(args: impl Iterator<Item = String>) -> Result<Option<String>, Error> {
         args.args = process_args(&std::fs::read_to_string(filename)?).unwrap();
     }
 
+    let cairo_layout_params = match args.cairo_layout_params_file {
+        Some(file) => Some(CairoLayoutParams::from_file(&file)?),
+        None => None,
+    };
+
     let cairo_run_config = Cairo1RunConfig {
         proof_mode: args.proof_mode,
         serialize_output: args.print_output,
@@ -162,6 +173,7 @@ fn run(args: impl Iterator<Item = String>) -> Result<Option<String>, Error> {
         args: &args.args.0,
         finalize_builtins: args.air_public_input.is_some() || args.cairo_pie_output.is_some(),
         append_return_values: args.append_return_values,
+        dynamic_layout_params: cairo_layout_params,
     };
 
     // Try to parse the file as a sierra program
@@ -478,6 +490,19 @@ mod tests {
         assert_matches!(run(args), Ok(Some(res)) if res == expected_output, "Program {} failed with flags {}", program, extra_flags.concat());
     }
 
+    #[test]
+    fn test_run_dynamic_params() {
+        let mut args = vec!["cairo1-run".to_string()];
+        args.extend_from_slice(&["--layout".to_string(), "dynamic".to_string()]);
+        args.extend_from_slice(&[
+            "--cairo_layout_params_file".to_string(),
+            "../vm/src/tests/cairo_layout_params_file.json".to_string(),
+        ]);
+        args.push("../cairo_programs/cairo-1-programs/fibonacci.cairo".to_string());
+
+        assert_matches!(run(args.into_iter()), Ok(_));
+    }
+
     // these tests are separated so as to run them without --append_return_values and --proof_mode options
     // since they require to use the squashed version of felt252
     #[rstest]