From b9852b5871be0b2e08523aa2352f1dcc61fb490c Mon Sep 17 00:00:00 2001 From: Jonathan Wang <31040440+jonathanpwang@users.noreply.github.com> Date: Fri, 8 Sep 2023 07:41:16 -0700 Subject: [PATCH 1/2] chore: move `OptimizedPoseidonSpec` to `halo2-base` --- snark-verifier-sdk/src/halo2.rs | 2 +- snark-verifier/src/util/hash.rs | 4 +- snark-verifier/src/util/hash/poseidon.rs | 343 +----------------- .../src/util/hash/poseidon/tests.rs | 2 +- 4 files changed, 22 insertions(+), 329 deletions(-) diff --git a/snark-verifier-sdk/src/halo2.rs b/snark-verifier-sdk/src/halo2.rs index 06d5e622..aa5f8810 100644 --- a/snark-verifier-sdk/src/halo2.rs +++ b/snark-verifier-sdk/src/halo2.rs @@ -2,6 +2,7 @@ use super::{read_instances, write_instances, CircuitExt, PlonkSuccinctVerifier, #[cfg(feature = "display")] use ark_std::{end_timer, start_timer}; use halo2_base::halo2_proofs; +pub use halo2_base::poseidon::hasher::spec::OptimizedPoseidonSpec; use halo2_proofs::{ circuit::Layouter, halo2curves::{ @@ -26,7 +27,6 @@ use halo2_proofs::{ use itertools::Itertools; use lazy_static::lazy_static; use rand::{rngs::StdRng, SeedableRng}; -pub use snark_verifier::util::hash::OptimizedPoseidonSpec; use snark_verifier::{ cost::CostEstimation, loader::native::NativeLoader, diff --git a/snark-verifier/src/util/hash.rs b/snark-verifier/src/util/hash.rs index a8fe168c..dfc02cf0 100644 --- a/snark-verifier/src/util/hash.rs +++ b/snark-verifier/src/util/hash.rs @@ -4,7 +4,9 @@ mod poseidon; #[cfg(feature = "loader_halo2")] -pub use crate::util::hash::poseidon::{OptimizedPoseidonSpec, Poseidon}; +pub use crate::util::hash::poseidon::Poseidon; +#[cfg(feature = "loader_halo2")] +pub(crate) use halo2_base::poseidon::hasher::spec::OptimizedPoseidonSpec; #[cfg(feature = "loader_evm")] pub use sha3::{Digest, Keccak256}; diff --git a/snark-verifier/src/util/hash/poseidon.rs b/snark-verifier/src/util/hash/poseidon.rs index e826ac52..740f4c72 100644 --- a/snark-verifier/src/util/hash/poseidon.rs +++ b/snark-verifier/src/util/hash/poseidon.rs @@ -1,4 +1,7 @@ -#![allow(clippy::needless_range_loop)] // for clarity of matrix operations +//! Trait based implementation of Poseidon permutation + +use halo2_base::poseidon::hasher::{mds::SparseMDSMatrix, spec::OptimizedPoseidonSpec}; + use crate::{ loader::{LoadedScalar, ScalarLoader}, util::{ @@ -6,323 +9,11 @@ use crate::{ Itertools, }, }; -use poseidon_rs::poseidon::primitives::Spec as PoseidonSpec; // trait use std::{iter, marker::PhantomData, mem}; #[cfg(test)] mod tests; -// struct so we can use PoseidonSpec trait to generate round constants and MDS matrix -#[derive(Debug)] -pub struct Poseidon128Pow5Gen< - F: PrimeField, - const T: usize, - const RATE: usize, - const R_F: usize, - const R_P: usize, - const SECURE_MDS: usize, -> { - _marker: PhantomData, -} - -impl< - F: PrimeField, - const T: usize, - const RATE: usize, - const R_F: usize, - const R_P: usize, - const SECURE_MDS: usize, - > PoseidonSpec for Poseidon128Pow5Gen -{ - fn full_rounds() -> usize { - R_F - } - - fn partial_rounds() -> usize { - R_P - } - - fn sbox(val: F) -> F { - val.pow_vartime([5]) - } - - // see "Avoiding insecure matrices" in Section 2.3 of https://eprint.iacr.org/2019/458.pdf - // most Specs used in practice have SECURE_MDS = 0 - fn secure_mds() -> usize { - SECURE_MDS - } -} - -// We use the optimized Poseidon implementation described in Supplementary Material Section B of https://eprint.iacr.org/2019/458.pdf -// This involves some further computation of optimized constants and sparse MDS matrices beyond what the Scroll PoseidonSpec generates -// The implementation below is adapted from https://github.com/privacy-scaling-explorations/poseidon - -/// `OptimizedPoseidonSpec` holds construction parameters as well as constants that are used in -/// permutation step. -#[derive(Debug, Clone)] -pub struct OptimizedPoseidonSpec { - pub(crate) r_f: usize, - pub(crate) mds_matrices: MDSMatrices, - pub(crate) constants: OptimizedConstants, -} - -/// `OptimizedConstants` has round constants that are added each round. While -/// full rounds has T sized constants there is a single constant for each -/// partial round -#[derive(Debug, Clone)] -pub struct OptimizedConstants { - pub(crate) start: Vec<[F; T]>, - pub(crate) partial: Vec, - pub(crate) end: Vec<[F; T]>, -} - -/// The type used to hold the MDS matrix -pub(crate) type Mds = [[F; T]; T]; - -/// `MDSMatrices` holds the MDS matrix as well as transition matrix which is -/// also called `pre_sparse_mds` and sparse matrices that enables us to reduce -/// number of multiplications in apply MDS step -#[derive(Debug, Clone)] -pub struct MDSMatrices { - pub(crate) mds: MDSMatrix, - pub(crate) pre_sparse_mds: MDSMatrix, - pub(crate) sparse_matrices: Vec>, -} - -/// `SparseMDSMatrix` are in `[row], [hat | identity]` form and used in linear -/// layer of partial rounds instead of the original MDS -#[derive(Debug, Clone)] -pub struct SparseMDSMatrix { - pub(crate) row: [F; T], - pub(crate) col_hat: [F; RATE], -} - -/// `MDSMatrix` is applied to `State` to achive linear layer of Poseidon -#[derive(Clone, Debug)] -pub struct MDSMatrix(pub(crate) Mds); - -impl MDSMatrix { - pub(crate) fn mul_vector(&self, v: &[F; T]) -> [F; T] { - let mut res = [F::ZERO; T]; - for i in 0..T { - for j in 0..T { - res[i] += self.0[i][j] * v[j]; - } - } - res - } - - fn identity() -> Mds { - let mut mds = [[F::ZERO; T]; T]; - for i in 0..T { - mds[i][i] = F::ONE; - } - mds - } - - /// Multiplies two MDS matrices. Used in sparse matrix calculations - fn mul(&self, other: &Self) -> Self { - let mut res = [[F::ZERO; T]; T]; - for i in 0..T { - for j in 0..T { - for k in 0..T { - res[i][j] += self.0[i][k] * other.0[k][j]; - } - } - } - Self(res) - } - - fn transpose(&self) -> Self { - let mut res = [[F::ZERO; T]; T]; - for i in 0..T { - for j in 0..T { - res[i][j] = self.0[j][i]; - } - } - Self(res) - } - - fn determinant(m: [[F; N]; N]) -> F { - let mut res = F::ONE; - let mut m = m; - for i in 0..N { - let mut pivot = i; - while m[pivot][i] == F::ZERO { - pivot += 1; - assert!(pivot < N, "matrix is not invertible"); - } - if pivot != i { - res = -res; - m.swap(pivot, i); - } - res *= m[i][i]; - let inv = m[i][i].invert().unwrap(); - for j in i + 1..N { - let factor = m[j][i] * inv; - for k in i + 1..N { - m[j][k] -= m[i][k] * factor; - } - } - } - res - } - - /// See Section B in Supplementary Material https://eprint.iacr.org/2019/458.pdf - /// Factorises an MDS matrix `M` into `M'` and `M''` where `M = M' * M''`. - /// Resulted `M''` matrices are the sparse ones while `M'` will contribute - /// to the accumulator of the process - fn factorise(&self) -> (Self, SparseMDSMatrix) { - assert_eq!(RATE + 1, T); - // Given `(t-1 * t-1)` MDS matrix called `hat` constructs the `t * t` matrix in - // form `[[1 | 0], [0 | m]]`, ie `hat` is the right bottom sub-matrix - let prime = |hat: Mds| -> Self { - let mut prime = Self::identity(); - for (prime_row, hat_row) in prime.iter_mut().skip(1).zip(hat.iter()) { - for (el_prime, el_hat) in prime_row.iter_mut().skip(1).zip(hat_row.iter()) { - *el_prime = *el_hat; - } - } - Self(prime) - }; - - // Given `(t-1)` sized `w_hat` vector constructs the matrix in form - // `[[m_0_0 | m_0_i], [w_hat | identity]]` - let prime_prime = |w_hat: [F; RATE]| -> Mds { - let mut prime_prime = Self::identity(); - prime_prime[0] = self.0[0]; - for (row, w) in prime_prime.iter_mut().skip(1).zip(w_hat.iter()) { - row[0] = *w - } - prime_prime - }; - - let w = self.0.iter().skip(1).map(|row| row[0]).collect::>(); - // m_hat is the `(t-1 * t-1)` right bottom sub-matrix of m := self.0 - let mut m_hat = [[F::ZERO; RATE]; RATE]; - for i in 0..RATE { - for j in 0..RATE { - m_hat[i][j] = self.0[i + 1][j + 1]; - } - } - // w_hat = m_hat^{-1} * w, where m_hat^{-1} is matrix inverse and * is matrix mult - // we avoid computing m_hat^{-1} explicitly by using Cramer's rule: https://en.wikipedia.org/wiki/Cramer%27s_rule - let mut w_hat = [F::ZERO; RATE]; - let det = Self::determinant(m_hat); - let det_inv = Option::::from(det.invert()).expect("matrix is not invertible"); - for j in 0..RATE { - let mut m_hat_j = m_hat; - for i in 0..RATE { - m_hat_j[i][j] = w[i]; - } - w_hat[j] = Self::determinant(m_hat_j) * det_inv; - } - let m_prime = prime(m_hat); - let m_prime_prime = prime_prime(w_hat); - // row = first row of m_prime_prime.transpose() = first column of m_prime_prime - let row: [F; T] = - m_prime_prime.iter().map(|row| row[0]).collect::>().try_into().unwrap(); - // col_hat = first column of m_prime_prime.transpose() without first element = first row of m_prime_prime without first element - let col_hat: [F; RATE] = m_prime_prime[0][1..].try_into().unwrap(); - (m_prime, SparseMDSMatrix { row, col_hat }) - } -} - -impl OptimizedPoseidonSpec { - /// Generate new spec with specific number of full and partial rounds. `SECURE_MDS` is usually 0, but may need to be specified because insecure matrices may sometimes be generated - pub fn new() -> Self - where - F: FieldExt, - { - let (round_constants, mds, mds_inv) = - Poseidon128Pow5Gen::::constants(); - let mds = MDSMatrix(mds); - let inverse_mds = MDSMatrix(mds_inv); - - let constants = - Self::calculate_optimized_constants(R_F, R_P, round_constants, &inverse_mds); - let (sparse_matrices, pre_sparse_mds) = Self::calculate_sparse_matrices(R_P, &mds); - - Self { - r_f: R_F, - constants, - mds_matrices: MDSMatrices { mds, sparse_matrices, pre_sparse_mds }, - } - } - - fn calculate_optimized_constants( - r_f: usize, - r_p: usize, - constants: Vec<[F; T]>, - inverse_mds: &MDSMatrix, - ) -> OptimizedConstants { - let (number_of_rounds, r_f_half) = (r_f + r_p, r_f / 2); - assert_eq!(constants.len(), number_of_rounds); - - // Calculate optimized constants for first half of the full rounds - let mut constants_start: Vec<[F; T]> = vec![[F::ZERO; T]; r_f_half]; - constants_start[0] = constants[0]; - for (optimized, constants) in - constants_start.iter_mut().skip(1).zip(constants.iter().skip(1)) - { - *optimized = inverse_mds.mul_vector(constants); - } - - // Calculate constants for partial rounds - let mut acc = constants[r_f_half + r_p]; - let mut constants_partial = vec![F::ZERO; r_p]; - for (optimized, constants) in constants_partial - .iter_mut() - .rev() - .zip(constants.iter().skip(r_f_half).rev().skip(r_f_half)) - { - let mut tmp = inverse_mds.mul_vector(&acc); - *optimized = tmp[0]; - - tmp[0] = F::ZERO; - for ((acc, tmp), constant) in acc.iter_mut().zip(tmp).zip(constants.iter()) { - *acc = tmp + constant - } - } - constants_start.push(inverse_mds.mul_vector(&acc)); - - // Calculate optimized constants for ending half of the full rounds - let mut constants_end: Vec<[F; T]> = vec![[F::ZERO; T]; r_f_half - 1]; - for (optimized, constants) in - constants_end.iter_mut().zip(constants.iter().skip(r_f_half + r_p + 1)) - { - *optimized = inverse_mds.mul_vector(constants); - } - - OptimizedConstants { - start: constants_start, - partial: constants_partial, - end: constants_end, - } - } - - fn calculate_sparse_matrices( - r_p: usize, - mds: &MDSMatrix, - ) -> (Vec>, MDSMatrix) { - let mds = mds.transpose(); - let mut acc = mds.clone(); - let mut sparse_matrices = (0..r_p) - .map(|_| { - let (m_prime, m_prime_prime) = acc.factorise(); - acc = mds.mul(&m_prime); - m_prime_prime - }) - .collect::>>(); - - sparse_matrices.reverse(); - (sparse_matrices, acc.transpose()) - } -} - -// ================ END OF CONSTRUCTION OF POSEIDON SPEC ==================== - -// now we get to actual trait based implementation of Poseidon permutation // this works for any loader, where the two loaders used are NativeLoader (native rust) and Halo2Loader (ZK circuit) #[derive(Clone, Debug)] struct State { @@ -403,9 +94,9 @@ impl, const T: usize, const RATE: usize> State fn apply_sparse_mds(&mut self, mds: &SparseMDSMatrix) { self.inner = iter::once( self.loader() - .sum_with_coeff(&mds.row.iter().cloned().zip(self.inner.iter()).collect_vec()), + .sum_with_coeff(&mds.row().iter().cloned().zip(self.inner.iter()).collect_vec()), ) - .chain(mds.col_hat.iter().zip(self.inner.iter().skip(1)).map(|(coeff, state)| { + .chain(mds.col_hat().iter().zip(self.inner.iter().skip(1)).map(|(coeff, state)| { self.loader().sum_with_coeff(&[(*coeff, &self.inner[0]), (F::ONE, state)]) })) .collect_vec() @@ -475,35 +166,35 @@ impl, const T: usize, const RATE: usize> Posei } fn permutation(&mut self, inputs: &[L]) { - let r_f = self.spec.r_f / 2; - let mds = self.spec.mds_matrices.mds.0; - let pre_sparse_mds = self.spec.mds_matrices.pre_sparse_mds.0; - let sparse_matrices = &self.spec.mds_matrices.sparse_matrices; + let r_f = self.spec.r_f() / 2; + let mds = self.spec.mds_matrices().mds().as_ref(); + let pre_sparse_mds = self.spec.mds_matrices().pre_sparse_mds().as_ref(); + let sparse_matrices = &self.spec.mds_matrices().sparse_matrices(); // First half of the full rounds - let constants = &self.spec.constants.start; + let constants = self.spec.constants().start(); self.state.absorb_with_pre_constants(inputs, &constants[0]); for constants in constants.iter().skip(1).take(r_f - 1) { self.state.sbox_full(constants); - self.state.apply_mds(&mds); + self.state.apply_mds(mds); } self.state.sbox_full(constants.last().unwrap()); - self.state.apply_mds(&pre_sparse_mds); + self.state.apply_mds(pre_sparse_mds); // Partial rounds - let constants = &self.spec.constants.partial; + let constants = self.spec.constants().partial(); for (constant, sparse_mds) in constants.iter().zip(sparse_matrices.iter()) { self.state.sbox_part(constant); self.state.apply_sparse_mds(sparse_mds); } // Second half of the full rounds - let constants = &self.spec.constants.end; + let constants = self.spec.constants().end(); for constants in constants.iter() { self.state.sbox_full(constants); - self.state.apply_mds(&mds); + self.state.apply_mds(mds); } self.state.sbox_full(&[F::ZERO; T]); - self.state.apply_mds(&mds); + self.state.apply_mds(mds); } } diff --git a/snark-verifier/src/util/hash/poseidon/tests.rs b/snark-verifier/src/util/hash/poseidon/tests.rs index 76793c91..507b1d7c 100644 --- a/snark-verifier/src/util/hash/poseidon/tests.rs +++ b/snark-verifier/src/util/hash/poseidon/tests.rs @@ -24,7 +24,7 @@ fn test_mds() { "11597556804922396090267472882856054602429588299176362916247939723151043581408", ], ]; - for (row1, row2) in mds.iter().zip_eq(spec.mds_matrices.mds.0.iter()) { + for (row1, row2) in mds.iter().zip_eq(spec.mds_matrices().mds().as_ref().iter()) { for (e1, e2) in row1.iter().zip_eq(row2.iter()) { assert_eq!(Fr::from_str_vartime(e1).unwrap(), *e2); } From b9877ad8db867fa93574223c107d4bc310c13db4 Mon Sep 17 00:00:00 2001 From: Jonathan Wang <31040440+jonathanpwang@users.noreply.github.com> Date: Fri, 8 Sep 2023 07:43:52 -0700 Subject: [PATCH 2/2] Bump version to 0.1.5 and remove poseidon-rs dep --- snark-verifier-sdk/Cargo.toml | 24 +++++++++++++++--------- snark-verifier/Cargo.toml | 6 +----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/snark-verifier-sdk/Cargo.toml b/snark-verifier-sdk/Cargo.toml index a62b3b0c..681f9121 100644 --- a/snark-verifier-sdk/Cargo.toml +++ b/snark-verifier-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "snark-verifier-sdk" -version = "0.1.4" +version = "0.1.5" edition = "2021" [dependencies] @@ -22,14 +22,14 @@ snark-verifier = { path = "../snark-verifier", default-features = false } getset = "0.1.2" # loader_evm -ethereum-types = { version = "0.14.1", default-features = false, features = ["std"], optional = true } -# sha3 = { version = "0.10", optional = true } -# revm = { version = "2.3.1", optional = true } -# bytes = { version = "1.2", optional = true } -# rlp = { version = "0.5", default-features = false, features = ["std"], optional = true } +ethereum-types = { version = "0.14.1", default-features = false, features = [ + "std", +], optional = true } # zkevm benchmarks -zkevm-circuits = { git = "https://github.com/privacy-scaling-explorations/zkevm-circuits.git", rev = "f834e61", features = ["test"], optional = true } +zkevm-circuits = { git = "https://github.com/privacy-scaling-explorations/zkevm-circuits.git", rev = "f834e61", features = [ + "test", +], optional = true } bus-mapping = { git = "https://github.com/privacy-scaling-explorations/zkevm-circuits.git", rev = "f834e61", optional = true } eth-types = { git = "https://github.com/privacy-scaling-explorations/zkevm-circuits.git", rev = "f834e61", optional = true } mock = { git = "https://github.com/privacy-scaling-explorations/zkevm-circuits.git", rev = "f834e61", optional = true } @@ -45,7 +45,13 @@ crossterm = { version = "0.25" } tui = { version = "0.19", default-features = false, features = ["crossterm"] } [features] -default = ["loader_halo2", "loader_evm", "halo2-axiom", "halo2-base/jemallocator", "display"] +default = [ + "loader_halo2", + "loader_evm", + "halo2-axiom", + "halo2-base/jemallocator", + "display", +] display = ["snark-verifier/display", "dep:ark-std"] loader_evm = ["snark-verifier/loader_evm", "dep:ethereum-types"] loader_halo2 = ["snark-verifier/loader_halo2"] @@ -74,4 +80,4 @@ harness = false [[bench]] name = "read_pk" required-features = ["loader_halo2"] -harness = false \ No newline at end of file +harness = false diff --git a/snark-verifier/Cargo.toml b/snark-verifier/Cargo.toml index fbc40c85..2fbe6682 100644 --- a/snark-verifier/Cargo.toml +++ b/snark-verifier/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "snark-verifier" -version = "0.1.4" +version = "0.1.5" edition = "2021" [dependencies] @@ -16,10 +16,6 @@ pairing = { version = "0.23" } # Use halo2-base as non-optional dependency because it re-exports halo2_proofs, halo2curves, and poseidon, using different repos based on feature flag "halo2-axiom" or "halo2-pse" halo2-base = { git = "https://github.com/axiom-crypto/halo2-lib.git", branch = "develop", default-features = false } -# This is Scroll's audited poseidon circuit. We only use it for the Native Poseidon spec. We do not use the halo2 circuit at all (and it wouldn't even work because the halo2_proofs tag is not compatbile). -# We forked it to upgrade to ff v0.13 and removed the circuit module -poseidon-rs = { git = "https://github.com/axiom-crypto/poseidon-circuit.git", rev = "1aee4a1" } -# poseidon-circuit = { git = "https://github.com/scroll-tech/poseidon-circuit.git", rev = "50015b7" } # parallel rayon = { version = "1.7", optional = true }