From 79da6f7238605e4e8702cdcc1968ab571d6fa951 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 2 May 2024 09:53:31 +0100 Subject: [PATCH 1/3] fix: reduce 2 columns for rom fse order --- aggregator/src/aggregation/decoder.rs | 7 +- .../decoder/tables/rom_fse_order.rs | 67 +++++++------------ 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/aggregator/src/aggregation/decoder.rs b/aggregator/src/aggregation/decoder.rs index 64e7785f1a..30cbff775f 100644 --- a/aggregator/src/aggregation/decoder.rs +++ b/aggregator/src/aggregation/decoder.rs @@ -2047,10 +2047,15 @@ impl DecoderConfig { meta.query_advice(config.tag_config.is_change, Rotation::cur()), ]); - [ + let (cmode_llt, cmode_mot, cmode_mlt) = ( meta.query_advice(config.block_config.compression_modes[0], Rotation::cur()), meta.query_advice(config.block_config.compression_modes[1], Rotation::cur()), meta.query_advice(config.block_config.compression_modes[2], Rotation::cur()), + ); + + let cmodes_lc = (4.expr() * cmode_llt) + (2.expr() * cmode_mot) + cmode_mlt; + [ + cmodes_lc, meta.query_advice(config.tag_config.tag, Rotation::prev()), // tag_prev meta.query_advice(config.tag_config.tag, Rotation::cur()), // tag_cur meta.query_advice(config.tag_config.tag_next, Rotation::cur()), // tag_next diff --git a/aggregator/src/aggregation/decoder/tables/rom_fse_order.rs b/aggregator/src/aggregation/decoder/tables/rom_fse_order.rs index d9b7698013..173ec8ada1 100644 --- a/aggregator/src/aggregation/decoder/tables/rom_fse_order.rs +++ b/aggregator/src/aggregation/decoder/tables/rom_fse_order.rs @@ -58,7 +58,10 @@ pub struct RomFseOrderTable { /// Compression mode boolean flags for LLT, MOT and MLT respectively. /// - Predefined_Mode > 0 /// - Fse_Compressed_Mode > 1 - compression_modes: [Column; 3], + /// + /// We use a linear combination of the 3 flags: + /// - 4 * llt_flag + 2 * mot_flag + mlt_flag + compression_modes_lc: Column, /// The tag that occurred previously. tag_prev: Column, /// The current tag, expected to be ZstdBlockSequenceFseCode. @@ -72,11 +75,7 @@ pub struct RomFseOrderTable { impl RomFseOrderTable { pub fn construct(meta: &mut ConstraintSystem) -> Self { Self { - compression_modes: [ - meta.fixed_column(), - meta.fixed_column(), - meta.fixed_column(), - ], + compression_modes_lc: meta.fixed_column(), tag_prev: meta.fixed_column(), tag_cur: meta.fixed_column(), tag_next: meta.fixed_column(), @@ -97,69 +96,57 @@ impl RomFseOrderTable { |mut region| { for (offset, row) in [ // (1, 1, 1) - (1, 1, 1, SeqHeader, FseCode, FseCode, LLT), - (1, 1, 1, FseCode, FseCode, FseCode, MOT), - (1, 1, 1, FseCode, FseCode, SeqData, MLT), + (7, SeqHeader, FseCode, FseCode, LLT), + (7, FseCode, FseCode, FseCode, MOT), + (7, FseCode, FseCode, SeqData, MLT), // (1, 1, 0) - (1, 1, 0, SeqHeader, FseCode, FseCode, LLT), - (1, 1, 0, FseCode, FseCode, SeqData, MOT), + (6, SeqHeader, FseCode, FseCode, LLT), + (6, FseCode, FseCode, SeqData, MOT), // (1, 0, 1) - (1, 0, 1, SeqHeader, FseCode, FseCode, LLT), - (1, 0, 1, FseCode, FseCode, SeqData, MLT), + (5, SeqHeader, FseCode, FseCode, LLT), + (5, FseCode, FseCode, SeqData, MLT), // (0, 1, 1) - (0, 1, 1, SeqHeader, FseCode, FseCode, MOT), - (0, 1, 1, FseCode, FseCode, SeqData, MLT), + (3, SeqHeader, FseCode, FseCode, MOT), + (3, FseCode, FseCode, SeqData, MLT), // (1, 0, 0) - (1, 0, 0, SeqHeader, FseCode, SeqData, LLT), + (4, SeqHeader, FseCode, SeqData, LLT), // (0, 1, 0) - (0, 1, 0, SeqHeader, FseCode, SeqData, MOT), + (2, SeqHeader, FseCode, SeqData, MOT), // (0, 0, 1) - (0, 0, 1, SeqHeader, FseCode, SeqData, MLT), + (1, SeqHeader, FseCode, SeqData, MLT), ] .iter() .enumerate() { region.assign_fixed( - || format!("llt:compr_mode at offset={offset}"), - self.compression_modes[0], + || format!("compression_modes_lc at offset={offset}"), + self.compression_modes_lc, offset, || Value::known(Fr::from(row.0 as u64)), )?; - region.assign_fixed( - || format!("mot:compr_mode at offset={offset}"), - self.compression_modes[1], - offset, - || Value::known(Fr::from(row.1 as u64)), - )?; - region.assign_fixed( - || format!("mlt:compr_mode at offset={offset}"), - self.compression_modes[2], - offset, - || Value::known(Fr::from(row.2 as u64)), - )?; region.assign_fixed( || format!("tag_prev at offset={offset}"), self.tag_prev, offset, - || Value::known(Fr::from(row.3 as u64)), + || Value::known(Fr::from(row.1 as u64)), )?; region.assign_fixed( || format!("tag_cur at offset={offset}"), self.tag_cur, offset, - || Value::known(Fr::from(row.4 as u64)), + || Value::known(Fr::from(row.2 as u64)), )?; region.assign_fixed( || format!("tag_next at offset={offset}"), self.tag_next, offset, - || Value::known(Fr::from(row.5 as u64)), + || Value::known(Fr::from(row.3 as u64)), )?; region.assign_fixed( || format!("table_kind at offset={offset}"), self.table_kind, offset, - || Value::known(Fr::from(row.6 as u64)), + || Value::known(Fr::from(row.4 as u64)), )?; } @@ -172,9 +159,7 @@ impl RomFseOrderTable { impl LookupTable for RomFseOrderTable { fn columns(&self) -> Vec> { vec![ - self.compression_modes[0].into(), - self.compression_modes[1].into(), - self.compression_modes[2].into(), + self.compression_modes_lc.into(), self.tag_prev.into(), self.tag_cur.into(), self.tag_next.into(), @@ -184,9 +169,7 @@ impl LookupTable for RomFseOrderTable { fn annotations(&self) -> Vec { vec![ - String::from("llt:compression_mode"), - String::from("mot:compression_mode"), - String::from("mlt:compression_mode"), + String::from("compression_modes_lc"), String::from("tag_prev"), String::from("tag_cur"), String::from("tag_next"), From cc118256a4a4bba38b36e37399ca454a853fd0da Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 2 May 2024 11:38:02 +0100 Subject: [PATCH 2/3] feat: add the fixed table module --- aggregator/src/aggregation/decoder/tables.rs | 2 + .../src/aggregation/decoder/tables/fixed.rs | 148 +++++++++ .../tables/fixed/fse_table_transition.rs | 52 ++++ .../decoder/tables/fixed/predefined_fse.rs | 290 ++++++++++++++++++ .../decoder/tables/fixed/seq_code_to_value.rs | 129 ++++++++ .../fixed/seq_data_interleaved_order.rs | 61 ++++ .../decoder/tables/fixed/seq_tag_order.rs | 93 ++++++ .../decoder/tables/fixed/tag_transition.rs | 49 +++ 8 files changed, 824 insertions(+) create mode 100644 aggregator/src/aggregation/decoder/tables/fixed.rs create mode 100644 aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs create mode 100644 aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs create mode 100644 aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs create mode 100644 aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs create mode 100644 aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs create mode 100644 aggregator/src/aggregation/decoder/tables/fixed/tag_transition.rs diff --git a/aggregator/src/aggregation/decoder/tables.rs b/aggregator/src/aggregation/decoder/tables.rs index 01a0a0f6f2..4b4a356f38 100644 --- a/aggregator/src/aggregation/decoder/tables.rs +++ b/aggregator/src/aggregation/decoder/tables.rs @@ -26,3 +26,5 @@ pub use rom_sequence_codes::RomSequenceCodes; /// Validate the following tag given the tag currently being processed. mod rom_tag; pub use rom_tag::RomTagTable; + +mod fixed; diff --git a/aggregator/src/aggregation/decoder/tables/fixed.rs b/aggregator/src/aggregation/decoder/tables/fixed.rs new file mode 100644 index 0000000000..33770485c4 --- /dev/null +++ b/aggregator/src/aggregation/decoder/tables/fixed.rs @@ -0,0 +1,148 @@ +use eth_types::Field; +use gadgets::impl_expr; +use halo2_proofs::{ + circuit::{Layouter, Value}, + halo2curves::bn256::Fr, + plonk::{Column, ConstraintSystem, Error, Expression, Fixed}, +}; +use itertools::Itertools; +use strum::IntoEnumIterator; +use strum_macros::EnumIter; +use zkevm_circuits::table::LookupTable; + +mod fse_table_transition; +use fse_table_transition::RomFseTableTransition; + +mod predefined_fse; +use predefined_fse::RomPredefinedFse; + +mod seq_code_to_value; +use seq_code_to_value::RomSeqCodeToValue; + +mod seq_data_interleaved_order; +use seq_data_interleaved_order::RomSeqDataInterleavedOrder; + +mod seq_tag_order; +use seq_tag_order::RomSeqTagOrder; + +mod tag_transition; +use tag_transition::RomTagTransition; + +pub trait FixedLookupValues { + fn values() -> Vec<[Value; 7]>; +} + +#[derive(Clone, Copy, Debug, EnumIter)] +pub enum FixedLookupTag { + /// Properties used to describe the ZstdTag of the chunk of bytes. + TagTransition = 1, + /// Depending on the compression modes used in the sequences header, we experience different + /// tag transitions and an FSE table for each tag=ZstdTagSequencesFseCode. + SeqTagOrder, + /// The bitstream which sequences are decoded from is interleaved with multiple variants. All + /// those variants are handled here. + SeqDataInterleavedOrder, + /// The predefined code-to-value table that allows us to compute "value" from the "code" + /// decoded. + SeqCodeToValue, + /// The FSE table's layout is assigned such that tables from different blocks appear in a + /// specific order, which is handled here. + FseTableTransition, + /// Represents the FSE table reconstructed from the default distributions, i.e. Predefined FSE + /// table. + PredefinedFse, +} + +impl_expr!(FixedLookupTag); + +impl FixedLookupTag { + fn values(&self) -> Vec<[Value; 7]> { + match self { + Self::TagTransition => RomTagTransition::values(), + Self::SeqTagOrder => RomSeqTagOrder::values(), + Self::SeqDataInterleavedOrder => RomSeqDataInterleavedOrder::values(), + Self::SeqCodeToValue => RomSeqCodeToValue::values(), + Self::FseTableTransition => RomFseTableTransition::values(), + Self::PredefinedFse => RomPredefinedFse::values(), + } + } +} + +#[derive(Clone, Debug)] +pub struct FixedTable { + lookup_tag: Column, + fixed1: Column, + fixed2: Column, + fixed3: Column, + fixed4: Column, + fixed5: Column, + fixed6: Column, +} + +impl FixedTable { + pub fn construct(meta: &mut ConstraintSystem) -> Self { + Self { + lookup_tag: meta.fixed_column(), + fixed1: meta.fixed_column(), + fixed2: meta.fixed_column(), + fixed3: meta.fixed_column(), + fixed4: meta.fixed_column(), + fixed5: meta.fixed_column(), + fixed6: meta.fixed_column(), + } + } + + pub fn load(&self, layouter: &mut impl Layouter) -> Result<(), Error> { + layouter.assign_region( + || "Fixed lookup table", + |mut region| { + for (i, row) in FixedLookupTag::iter() + .flat_map(|lookup_tag| lookup_tag.values()) + .enumerate() + { + for ((&column, annotation), &value) in self + .fixed_columns() + .iter() + .zip_eq(self.annotations()) + .zip_eq(row.iter()) + { + region.assign_fixed( + || format!("{} at offset={i}", annotation), + column, + i, + || value, + )?; + } + } + + Ok(()) + }, + ) + } +} + +impl LookupTable for FixedTable { + fn columns(&self) -> Vec> { + vec![ + self.lookup_tag.into(), + self.fixed1.into(), + self.fixed2.into(), + self.fixed3.into(), + self.fixed4.into(), + self.fixed5.into(), + self.fixed6.into(), + ] + } + + fn annotations(&self) -> Vec { + vec![ + String::from("lookup_tag"), + String::from("fixed1"), + String::from("fixed2"), + String::from("fixed3"), + String::from("fixed4"), + String::from("fixed5"), + String::from("fixed6"), + ] + } +} diff --git a/aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs b/aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs new file mode 100644 index 0000000000..e671a559a3 --- /dev/null +++ b/aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs @@ -0,0 +1,52 @@ +use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; + +use crate::aggregation::decoder::tables::FseTableKind; + +use super::{FixedLookupTag, FixedLookupValues}; + +pub struct RomFseTableTransition { + /// The block index on the previous FSE table. + block_idx_prev: u64, + /// The block index on the current FSE table. + block_idx_curr: u64, + /// The FSE table previously decoded. + table_kind_prev: u64, + /// The FSE table currently decoded. + table_kind_curr: u64, +} + +impl FixedLookupValues for RomFseTableTransition { + fn values() -> Vec<[Value; 7]> { + [ + vec![[ + Value::known(Fr::from(FixedLookupTag::FseTableTransition as u64)), + Value::known(Fr::zero()), // block_idx_prev + Value::known(Fr::one()), // block_idx_curr + Value::known(Fr::zero()), // table_kind_prev + Value::known(Fr::from(FseTableKind::LLT as u64)), + Value::known(Fr::zero()), + Value::known(Fr::zero()), + ]], + [ + (1, 1, FseTableKind::LLT, FseTableKind::MOT), + (1, 1, FseTableKind::MOT, FseTableKind::MLT), + // TODO: add more for multi-block scenario + ] + .map( + |(block_idx_prev, block_idx_curr, table_kind_prev, table_kind_curr)| { + [ + Value::known(Fr::from(FixedLookupTag::FseTableTransition as u64)), + Value::known(Fr::from(block_idx_prev)), + Value::known(Fr::from(block_idx_curr)), + Value::known(Fr::from(table_kind_prev as u64)), + Value::known(Fr::from(table_kind_curr as u64)), + Value::known(Fr::zero()), + Value::known(Fr::zero()), + ] + }, + ) + .to_vec(), + ] + .concat() + } +} diff --git a/aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs b/aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs new file mode 100644 index 0000000000..dc288aeff7 --- /dev/null +++ b/aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs @@ -0,0 +1,290 @@ +use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; + +use crate::aggregation::decoder::tables::FseTableKind; + +use super::{FixedLookupTag, FixedLookupValues}; + +pub struct RomPredefinedFse { + table_kind: FseTableKind, + table_size: u64, + state: u64, + symbol: u64, + baseline: u64, + nb: u64, +} + +pub trait PredefinedFse { + /// Get the accuracy log of the predefined table. + fn accuracy_log(&self) -> u8; + /// Get the number of states in the FSE table. + fn table_size(&self) -> u64 { + 1 << self.accuracy_log() + } + /// Get the symbol in the FSE table for the given state. + fn symbol(&self, state: u64) -> u64; + /// Get the baseline in the FSE table for the given state. + fn baseline(&self, state: u64) -> u64; + /// Get the number of bits (nb) to read from bitstream in the FSE table for the given state. + fn nb(&self, state: u64) -> u64; +} + +impl PredefinedFse for FseTableKind { + fn accuracy_log(&self) -> u8 { + match self { + Self::LLT => 6, + Self::MOT => 5, + Self::MLT => 6, + } + } + + fn symbol(&self, state: u64) -> u64 { + match self { + Self::LLT => match state { + 0..=1 => 0, + 2 => 1, + 3 => 3, + 4 => 4, + 5 => 6, + 6 => 7, + 7 => 9, + 8 => 10, + 9 => 12, + 10 => 14, + 11 => 16, + 12 => 18, + 13 => 19, + 14 => 21, + 15 => 22, + 16 => 24, + 17 => 25, + 18 => 26, + 19 => 27, + 20 => 29, + 21 => 31, + 22 => 0, + 23 => 1, + 24 => 2, + 25 => 4, + 26 => 5, + 27 => 7, + 28 => 8, + 29 => 10, + 30 => 11, + 31 => 13, + 32 => 16, + 33 => 17, + 34 => 19, + 35 => 20, + 36 => 22, + 37 => 23, + 38 => 25, + 39 => 25, + 40 => 26, + 41 => 28, + 42 => 30, + 43 => 0, + 44 => 1, + 45 => 2, + 46 => 3, + 47 => 5, + 48 => 6, + 49 => 8, + 50 => 9, + 51 => 11, + 52 => 12, + 53 => 15, + 54 => 17, + 55 => 18, + 56 => 20, + 57 => 21, + 58 => 23, + 59 => 24, + 60 => 35, + 61 => 34, + 62 => 33, + 63 => 32, + _ => unreachable!(), + }, + Self::MOT => match state { + 0 => 0, + 1 => 6, + 2 => 9, + 3 => 15, + 4 => 21, + 5 => 3, + 6 => 7, + 7 => 12, + 8 => 18, + 9 => 23, + 10 => 5, + 11 => 8, + 12 => 14, + 13 => 20, + 14 => 2, + 15 => 7, + 16 => 11, + 17 => 17, + 18 => 22, + 19 => 4, + 20 => 8, + 21 => 13, + 22 => 19, + 23 => 1, + 24 => 6, + 25 => 10, + 26 => 16, + 27 => 28, + 28 => 27, + 29 => 26, + 30 => 25, + 31 => 24, + _ => unreachable!(), + }, + Self::MLT => match state { + 0..=3 => state, + 4..=5 => state + 1, + 6 => 8, + 7 => 10, + 8 => 13, + 9 => 16, + 10 => 19, + 11 => 22, + 12 => 25, + 13 => 28, + 14 => 31, + 15 => 33, + 16 => 35, + 17 => 37, + 18 => 39, + 19 => 41, + 20 => 43, + 21 => 45, + 22..=25 => state - 21, + 26..=27 => state - 20, + 28 => 9, + 29 => 12, + 30 => 15, + 31 => 18, + 32 => 21, + 33 => 24, + 34 => 27, + 35 => 30, + 36 => 32, + 37 => 34, + 38 => 36, + 39 => 38, + 40 => 40, + 41 => 42, + 42 => 44, + 43..=44 => 1, + 45 => 2, + 46..=47 => state - 42, + 48 => 7, + 49 => 8, + 50 => 11, + 51 => 14, + 52 => 17, + 53 => 20, + 54 => 23, + 55 => 26, + 56 => 29, + 57 => 52, + 58 => 51, + 59 => 50, + 60 => 49, + 61 => 48, + 62 => 47, + 63 => 46, + _ => unreachable!(), + }, + } + } + + fn baseline(&self, state: u64) -> u64 { + match self { + Self::LLT => match state { + 0 => 0, + 1 => 16, + 2 => 32, + 3..=16 => 0, + 17 => 32, + 18..=21 | 23..=24 => 0, + 22 | 25 | 27 | 29 | 32 | 34 | 36 | 40 => 32, + 26 | 28 | 30..=31 | 33 | 35 | 37..=38 | 41..=42 | 53 | 60..=63 => 0, + 39 | 44 => 16, + 43 => 48, + 45..=52 | 54..=59 => 32, + _ => unreachable!(), + }, + Self::MOT => match state { + 0..=14 | 16..=19 | 21..=23 | 25..=31 => 0, + 15 | 20 | 24 => 16, + _ => unreachable!(), + }, + Self::MLT => match state { + 0..=1 | 3..=21 | 23 | 25 | 27..=42 | 50..=63 => 0, + 2 | 24 | 26 | 43 | 46..=49 => 32, + 22 | 45 => 16, + 44 => 48, + _ => unreachable!(), + }, + } + } + + fn nb(&self, state: u64) -> u64 { + match self { + Self::LLT => match state { + 0..=1 | 22..=23 | 38..=39 | 43..=44 => 4, + 2..=9 | 11..=18 | 24..=30 | 32..=37 | 40 | 45..=52 | 54..=59 => 5, + 10 | 19..=21 | 31 | 41..=42 | 53 | 60..=63 => 6, + _ => unreachable!(), + }, + Self::MOT => match state { + 0 | 2..=5 | 7..=10 | 12..=14 | 16..=19 | 21..=23 | 25..=31 => 5, + 1 | 6 | 11 | 15 | 20 | 24 => 4, + _ => unreachable!(), + }, + Self::MLT => match state { + 0 | 7..=21 | 28..=42 | 50..=63 => 6, + 1 | 22..=23 | 43..=45 => 4, + 2..=6 | 24..=27 | 46..=49 => 5, + _ => unreachable!(), + }, + } + } +} + +pub fn predefined_fse(table_kind: FseTableKind) -> Vec { + let table_size = table_kind.table_size(); + (0..table_size) + .map(|state| RomPredefinedFse { + table_kind, + table_size, + state, + symbol: table_kind.symbol(state), + baseline: table_kind.baseline(state), + nb: table_kind.nb(state), + }) + .collect() +} + +impl FixedLookupValues for RomPredefinedFse { + fn values() -> Vec<[Value; 7]> { + [FseTableKind::LLT, FseTableKind::MOT, FseTableKind::MLT] + .map(predefined_fse) + .iter() + .flatten() + .map(|row| { + [ + Value::known(Fr::from(FixedLookupTag::PredefinedFse as u64)), + Value::known(Fr::from(row.table_kind as u64)), + Value::known(Fr::from(row.table_size)), + Value::known(Fr::from(row.state)), + Value::known(Fr::from(row.symbol)), + Value::known(Fr::from(row.baseline)), + Value::known(Fr::from(row.nb)), + ] + }) + .collect() + } +} diff --git a/aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs b/aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs new file mode 100644 index 0000000000..9996c01ae4 --- /dev/null +++ b/aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs @@ -0,0 +1,129 @@ +use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; + +use crate::aggregation::decoder::tables::FseTableKind; + +use super::{FixedLookupTag, FixedLookupValues}; + +#[derive(Clone, Debug)] +pub struct RomSeqCodeToValue { + /// The FSE table kind (LLT, MOT, MLT) + pub table_kind: FseTableKind, + /// The code decoded from the FSE table before. + pub code: u64, + /// The baseline for code-to-value. + pub baseline: u64, + /// The number of bits to read now for code-to-value. + pub nb: u64, +} + +impl From<(FseTableKind, u64, u64, u64)> for RomSeqCodeToValue { + fn from(v: (FseTableKind, u64, u64, u64)) -> Self { + Self { + table_kind: v.0, + code: v.1, + baseline: v.2, + nb: v.3, + } + } +} + +pub trait CodeToValue { + fn code_to_value() -> Vec; +} + +pub struct LiteralLengthCodes; +pub struct MatchLengthCodes; +pub struct MatchOffsetCodes; + +impl CodeToValue for LiteralLengthCodes { + fn code_to_value() -> Vec { + (0..16) + .map(|i| (i, i, 0)) + .chain([ + (16, 16, 1), + (17, 18, 1), + (18, 20, 1), + (19, 22, 1), + (20, 24, 2), + (21, 28, 2), + (22, 32, 3), + (23, 40, 3), + (24, 48, 4), + (25, 64, 6), + (26, 128, 7), + (27, 256, 8), + (28, 512, 9), + (29, 1024, 10), + (30, 2048, 11), + (31, 4096, 12), + (32, 8192, 13), + (33, 16384, 14), + (34, 32768, 15), + (35, 65536, 16), + ]) + .map(|tuple| (FseTableKind::LLT, tuple.0, tuple.1, tuple.2).into()) + .collect() + } +} + +impl CodeToValue for MatchLengthCodes { + fn code_to_value() -> Vec { + (0..32) + .map(|i| (i, i + 3, 0)) + .chain([ + (32, 35, 1), + (33, 37, 1), + (34, 39, 1), + (35, 41, 1), + (36, 43, 2), + (37, 47, 2), + (38, 51, 3), + (39, 59, 3), + (40, 67, 4), + (41, 83, 4), + (42, 99, 5), + (43, 131, 7), + (44, 259, 8), + (45, 515, 9), + (46, 1027, 10), + (47, 2051, 11), + (48, 4099, 12), + (49, 8195, 13), + (50, 16387, 14), + (51, 32771, 15), + (52, 65539, 16), + ]) + .map(|tuple| (FseTableKind::MLT, tuple.0, tuple.1, tuple.2).into()) + .collect() + } +} + +impl CodeToValue for MatchOffsetCodes { + // N <- 31 for Match Offset Codes. + fn code_to_value() -> Vec { + (0..32) + .map(|i| (FseTableKind::MOT, i, 1 << i, i).into()) + .collect() + } +} + +impl FixedLookupValues for RomSeqCodeToValue { + fn values() -> Vec<[Value; 7]> { + std::iter::empty() + .chain(LiteralLengthCodes::code_to_value()) + .chain(MatchOffsetCodes::code_to_value()) + .chain(MatchLengthCodes::code_to_value()) + .map(|row| { + [ + Value::known(Fr::from(FixedLookupTag::SeqCodeToValue as u64)), + Value::known(Fr::from(row.table_kind as u64)), + Value::known(Fr::from(row.code)), + Value::known(Fr::from(row.baseline)), + Value::known(Fr::from(row.nb)), + Value::known(Fr::zero()), + Value::known(Fr::zero()), + ] + }) + .collect() + } +} diff --git a/aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs b/aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs new file mode 100644 index 0000000000..10a5db8004 --- /dev/null +++ b/aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs @@ -0,0 +1,61 @@ +use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; + +use crate::aggregation::decoder::tables::{fixed::FixedLookupTag, FseTableKind}; + +use super::FixedLookupValues; + +pub struct RomSeqDataInterleavedOrder { + /// FSE table used in the previous bitstring. + table_kind_prev: FseTableKind, + /// FSE table used in the current bitstring. + table_kind_curr: FseTableKind, + /// Boolean flag to indicate whether we are initialising the FSE state. + is_init_state: bool, + /// Boolean flag to indicate whether we are updating the FSE state. + is_update_state: bool, +} + +impl FixedLookupValues for RomSeqDataInterleavedOrder { + fn values() -> Vec<[Value; 7]> { + use FseTableKind::{LLT, MLT, MOT}; + + [ + // init stat (LLT) + vec![[ + Value::known(Fr::from(FixedLookupTag::SeqDataInterleavedOrder as u64)), + Value::known(Fr::zero()), // table_kind_prev + Value::known(Fr::from(LLT as u64)), // table_kind_curr + Value::known(Fr::one()), // is_init_state + Value::known(Fr::zero()), // is_update_state + Value::known(Fr::zero()), + Value::known(Fr::zero()), + ]], + [ + (LLT, MOT, true, false), // init state (MOT) + (MOT, MLT, true, false), // init state (MLT) + (MLT, MOT, false, false), + (MOT, MLT, false, false), + (MLT, LLT, false, false), + (LLT, LLT, false, true), + (LLT, MLT, false, true), + (MLT, MOT, false, true), + (MOT, MOT, false, false), + ] + .map( + |(table_kind_prev, table_kind_curr, is_init_state, is_update_state)| { + [ + Value::known(Fr::from(FixedLookupTag::SeqDataInterleavedOrder as u64)), + Value::known(Fr::from(table_kind_prev as u64)), + Value::known(Fr::from(table_kind_curr as u64)), + Value::known(Fr::from(is_init_state as u64)), + Value::known(Fr::from(is_update_state as u64)), + Value::known(Fr::zero()), + Value::known(Fr::zero()), + ] + }, + ) + .to_vec(), + ] + .concat() + } +} diff --git a/aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs b/aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs new file mode 100644 index 0000000000..da0015d316 --- /dev/null +++ b/aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs @@ -0,0 +1,93 @@ +use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; + +use crate::aggregation::decoder::{ + tables::{fixed::FixedLookupTag, FseTableKind}, + witgen::ZstdTag, +}; + +use super::FixedLookupValues; + +/// The possible orders are: +/// +/// - (1, 1, 1): +/// - SequenceHeader > FseCode > FseCode (LLT) +/// - FseCode > FseCode > FseCode (MOT) +/// - FseCode > FseCode > SequenceData (MLT) +/// - (1, 1, 0): +/// - SequenceHeader > FseCode > FseCode (LLT) +/// - FseCode > FseCode > SequenceData (MOT) +/// - (1, 0, 1): +/// - SequenceHeader > FseCode > FseCode (LLT) +/// - FseCode > FseCode > SequenceData (MLT) +/// - (0, 1, 1): +/// - SequenceHeader > FseCode > FseCode (MOT) +/// - FseCode > FseCode > SequenceData (MLT) +/// - (1, 0, 0): +/// - SequenceHeader > FseCode > SequenceData (LLT) +/// - (0, 1, 0): +/// - SequenceHeader > FseCode > SequenceData (MOT) +/// - (0, 0, 1): +/// - SequenceHeader > FseCode > SequenceData (MLT) +pub struct RomSeqTagOrder { + /// Boolean flag to mark if LLT is Fse_Compressed_Mode or Predefined_Mode. + cmode_llt: bool, + /// Boolean flag to mark if MOT is Fse_Compressed_Mode or Predefined_Mode. + cmode_mot: bool, + /// Boolean flag to mark if MLT is Fse_Compressed_Mode or Predefined_Mode. + cmode_mlt: bool, + /// Tag that was handled before the current tag. + tag_prev: ZstdTag, + /// Tag currently being handled. + tag_curr: ZstdTag, + /// Tag that will be handled after the current tag. + tag_next: ZstdTag, + /// The FSE table that we expect with the current tag. + fse_table: FseTableKind, +} + +impl FixedLookupValues for RomSeqTagOrder { + fn values() -> Vec<[Value; 7]> { + use FseTableKind::{LLT, MLT, MOT}; + use ZstdTag::{ + ZstdBlockFseCode as FseCode, ZstdBlockLstream as SeqData, + ZstdBlockSequenceHeader as SeqHeader, + }; + + [ + // (1, 1, 1) + (1, 1, 1, SeqHeader, FseCode, FseCode, LLT), + (1, 1, 1, FseCode, FseCode, FseCode, MOT), + (1, 1, 1, FseCode, FseCode, SeqData, MLT), + // (1, 1, 0) + (1, 1, 0, SeqHeader, FseCode, FseCode, LLT), + (1, 1, 0, FseCode, FseCode, SeqData, MOT), + // (1, 0, 1) + (1, 0, 1, SeqHeader, FseCode, FseCode, LLT), + (1, 0, 1, FseCode, FseCode, SeqData, MLT), + // (0, 1, 1) + (0, 1, 1, SeqHeader, FseCode, FseCode, MOT), + (0, 1, 1, FseCode, FseCode, SeqData, MLT), + // (1, 0, 0) + (1, 0, 0, SeqHeader, FseCode, SeqData, LLT), + // (0, 1, 0) + (0, 1, 0, SeqHeader, FseCode, SeqData, MOT), + // (0, 0, 1) + (0, 0, 1, SeqHeader, FseCode, SeqData, MLT), + ] + .map( + |(cmode_llt, cmode_mot, cmode_mlt, tag_prev, tag_curr, tag_next, table_kind)| { + let cmode_lc = 4 * cmode_llt + 2 * cmode_mot + cmode_mlt; + [ + Value::known(Fr::from(FixedLookupTag::SeqTagOrder as u64)), + Value::known(Fr::from(cmode_lc)), + Value::known(Fr::from(tag_prev as u64)), + Value::known(Fr::from(tag_curr as u64)), + Value::known(Fr::from(tag_next as u64)), + Value::known(Fr::from(table_kind as u64)), + Value::known(Fr::zero()), + ] + }, + ) + .to_vec() + } +} diff --git a/aggregator/src/aggregation/decoder/tables/fixed/tag_transition.rs b/aggregator/src/aggregation/decoder/tables/fixed/tag_transition.rs new file mode 100644 index 0000000000..67dd3f8ea1 --- /dev/null +++ b/aggregator/src/aggregation/decoder/tables/fixed/tag_transition.rs @@ -0,0 +1,49 @@ +use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; + +use crate::aggregation::decoder::{tables::fixed::FixedLookupTag, witgen::ZstdTag}; + +use super::FixedLookupValues; + +pub struct RomTagTransition { + /// The current tag. + tag: ZstdTag, + /// The tag that will be processed after the current tag is finished processing. + tag_next: ZstdTag, + /// The maximum number of bytes that are needed to represent the current tag. + max_len: u64, + /// Whether this tag outputs a decoded byte or not. + is_output: bool, + /// Whether this tag is processed from back-to-front or not. + is_reverse: bool, + /// Whether this tag belongs to a ``block`` in zstd or not. + is_block: bool, +} + +impl FixedLookupValues for RomTagTransition { + fn values() -> Vec<[Value; 7]> { + use ZstdTag::{ + BlockHeader, FrameContentSize, FrameHeaderDescriptor, ZstdBlockLiteralsHeader, + ZstdBlockLiteralsRawBytes, ZstdBlockSequenceHeader, + }; + + [ + (FrameHeaderDescriptor, FrameContentSize, 1), + (FrameContentSize, BlockHeader, 8), + (BlockHeader, ZstdBlockLiteralsHeader, 3), + (ZstdBlockLiteralsHeader, ZstdBlockLiteralsRawBytes, 5), + (ZstdBlockLiteralsRawBytes, ZstdBlockSequenceHeader, 1048575), // (1 << 20) - 1 + ] + .map(|(tag, tag_next, max_len)| { + [ + Value::known(Fr::from(FixedLookupTag::TagTransition as u64)), + Value::known(Fr::from(tag as u64)), + Value::known(Fr::from(tag_next as u64)), + Value::known(Fr::from(max_len)), + Value::known(Fr::from(tag.is_output())), + Value::known(Fr::from(tag.is_reverse())), + Value::known(Fr::from(tag.is_block())), + ] + }) + .to_vec() + } +} From 6bfc3b378d9124adafc546ef45ebc5d9b0998268 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 2 May 2024 12:07:27 +0100 Subject: [PATCH 3/3] chore(refactor): remove rom tables and use fixed table lookup --- aggregator/src/aggregation/decoder.rs | 64 +- aggregator/src/aggregation/decoder/tables.rs | 17 +- .../src/aggregation/decoder/tables/fixed.rs | 1 + .../tables/fixed/fse_table_transition.rs | 2 +- .../decoder/tables/fixed/predefined_fse.rs | 14 +- .../decoder/tables/fixed/seq_code_to_value.rs | 2 +- .../fixed/seq_data_interleaved_order.rs | 7 +- .../decoder/tables/fixed/seq_tag_order.rs | 4 +- .../src/aggregation/decoder/tables/fse.rs | 40 +- .../decoder/tables/rom_fse_order.rs | 784 ------------------ .../decoder/tables/rom_sequence_codes.rs | 189 ----- .../src/aggregation/decoder/tables/rom_tag.rs | 93 --- aggregator/src/aggregation/decoder/witgen.rs | 2 - .../src/aggregation/decoder/witgen/types.rs | 32 +- 14 files changed, 100 insertions(+), 1151 deletions(-) delete mode 100644 aggregator/src/aggregation/decoder/tables/rom_fse_order.rs delete mode 100644 aggregator/src/aggregation/decoder/tables/rom_sequence_codes.rs delete mode 100644 aggregator/src/aggregation/decoder/tables/rom_tag.rs diff --git a/aggregator/src/aggregation/decoder.rs b/aggregator/src/aggregation/decoder.rs index 30cbff775f..219bd81a7d 100644 --- a/aggregator/src/aggregation/decoder.rs +++ b/aggregator/src/aggregation/decoder.rs @@ -24,12 +24,14 @@ use zkevm_circuits::{ util::Challenges, }; +use crate::aggregation::decoder::tables::FixedLookupTag; + use self::{ - tables::{ - BitstringTable, FseTable, FseTableKind, LiteralsHeaderTable, RomFseOrderTable, - RomSequenceCodes, RomSequencesDataInterleavedOrder, RomTagTable, + tables::{BitstringTable, FixedTable, FseTable, LiteralsHeaderTable}, + witgen::{ + FseTableKind, ZstdTag, N_BITS_PER_BYTE, N_BITS_REPEAT_FLAG, N_BITS_ZSTD_TAG, + N_BLOCK_HEADER_BYTES, }, - witgen::{ZstdTag, N_BITS_PER_BYTE, N_BITS_REPEAT_FLAG, N_BITS_ZSTD_TAG, N_BLOCK_HEADER_BYTES}, }; #[derive(Clone, Debug)] @@ -82,14 +84,8 @@ pub struct DecoderConfig { fse_table: FseTable, /// Helper table for sequences as instructions. /// TODO(enable): sequence_instruction_table: SequenceInstructionTable, - /// ROM table for validating tag transition. - rom_tag_table: RomTagTable, - /// ROM table for the correct order in which FSE tables are described in the sequences section. - rom_fse_order_table: RomFseOrderTable, - /// ROM table for the correct interleaved order while processing tag=ZstdBlockSequencesData. - rom_interleaved_order_table: RomSequencesDataInterleavedOrder, - /// ROM table for sequence codes to value. LLC, MOC and MLC. - rom_sequence_codes_table: RomSequenceCodes, + /// Fixed lookups table. + fixed_table: FixedTable, } #[derive(Clone, Debug)] @@ -970,11 +966,8 @@ impl DecoderConfig { bitwise_op_table, }: DecoderConfigArgs, ) -> Self { - // Fixed tables - let rom_tag_table = RomTagTable::construct(meta); - let rom_fse_order_table = RomFseOrderTable::construct(meta); - let rom_interleaved_order_table = RomSequencesDataInterleavedOrder::construct(meta); - let rom_sequence_codes_table = RomSequenceCodes::construct(meta); + // Fixed table + let fixed_table = FixedTable::construct(meta); // Helper tables let literals_header_table = LiteralsHeaderTable::configure(meta, range8, range16); @@ -1021,10 +1014,7 @@ impl DecoderConfig { bitstring_table, fse_table, // TODO(enable): sequence_instruction_table, - rom_tag_table, - rom_fse_order_table, - rom_interleaved_order_table, - rom_sequence_codes_table, + fixed_table, }; macro_rules! is_tag { @@ -1279,11 +1269,12 @@ impl DecoderConfig { cb.gate(condition) }); - meta.lookup_any("DecoderConfig: lookup RomTagTable", |meta| { + meta.lookup_any("DecoderConfig: fixed lookup (tag transition)", |meta| { let condition = meta.query_fixed(config.q_first, Rotation::cur()) + meta.query_advice(config.tag_config.is_change, Rotation::cur()); [ + FixedLookupTag::TagTransition.expr(), meta.query_advice(config.tag_config.tag, Rotation::cur()), meta.query_advice(config.tag_config.tag_next, Rotation::cur()), meta.query_advice(config.tag_config.max_len, Rotation::cur()), @@ -1292,7 +1283,7 @@ impl DecoderConfig { meta.query_advice(config.block_config.is_block, Rotation::cur()), ] .into_iter() - .zip_eq(config.rom_tag_table.table_exprs(meta)) + .zip_eq(config.fixed_table.table_exprs(meta)) .map(|(value, table)| (condition.expr() * value, table)) .collect() }); @@ -2055,14 +2046,16 @@ impl DecoderConfig { let cmodes_lc = (4.expr() * cmode_llt) + (2.expr() * cmode_mot) + cmode_mlt; [ + FixedLookupTag::SeqTagOrder.expr(), cmodes_lc, meta.query_advice(config.tag_config.tag, Rotation::prev()), // tag_prev meta.query_advice(config.tag_config.tag, Rotation::cur()), // tag_cur meta.query_advice(config.tag_config.tag_next, Rotation::cur()), // tag_next meta.query_advice(config.fse_decoder.table_kind, Rotation::cur()), // table_kind + 0.expr(), // unused ] .into_iter() - .zip_eq(config.rom_fse_order_table.table_exprs(meta)) + .zip_eq(config.fixed_table.table_exprs(meta)) .map(|(arg, table)| (condition.expr() * arg, table)) .collect() }, @@ -2412,13 +2405,16 @@ impl DecoderConfig { ); [ + FixedLookupTag::SeqDataInterleavedOrder.expr(), table_kind_prev, table_kind_curr, is_init_state, is_update_state, + 0.expr(), // unused + 0.expr(), // unused ] .into_iter() - .zip_eq(config.rom_interleaved_order_table.table_exprs(meta)) + .zip_eq(config.fixed_table.table_exprs(meta)) .map(|(arg, table)| (condition.expr() * arg, table)) .collect() }, @@ -2761,11 +2757,19 @@ impl DecoderConfig { .bitstring_len(meta, Rotation::cur()), ); - [table_kind, code, baseline, nb] - .into_iter() - .zip_eq(config.rom_sequence_codes_table.table_exprs(meta)) - .map(|(arg, table)| (condition.expr() * arg, table)) - .collect() + [ + FixedLookupTag::SeqCodeToValue.expr(), + table_kind, + code, + baseline, + nb, + 0.expr(), // unused + 0.expr(), // unused + ] + .into_iter() + .zip_eq(config.fixed_table.table_exprs(meta)) + .map(|(arg, table)| (condition.expr() * arg, table)) + .collect() }, ); diff --git a/aggregator/src/aggregation/decoder/tables.rs b/aggregator/src/aggregation/decoder/tables.rs index 4b4a356f38..dbc08b22f4 100644 --- a/aggregator/src/aggregation/decoder/tables.rs +++ b/aggregator/src/aggregation/decoder/tables.rs @@ -12,19 +12,6 @@ pub use fse::FseTable; mod literals_header; pub use literals_header::LiteralsHeaderTable; -/// Validate the assignment of FSE table kind while decoding FSE tables in the sequences section. -mod rom_fse_order; -pub use rom_fse_order::{ - predefined_table, predefined_table_values, FsePredefinedTable, FseTableKind, RomFseOrderTable, - RomSequencesDataInterleavedOrder, -}; - -/// The fixed code to Baseline/NumBits for Literal Length. -mod rom_sequence_codes; -pub use rom_sequence_codes::RomSequenceCodes; - -/// Validate the following tag given the tag currently being processed. -mod rom_tag; -pub use rom_tag::RomTagTable; - +/// Fixed lookup table and its variants. mod fixed; +pub use fixed::{predefined_fse, FixedLookupTag, FixedTable, PredefinedFse}; diff --git a/aggregator/src/aggregation/decoder/tables/fixed.rs b/aggregator/src/aggregation/decoder/tables/fixed.rs index 33770485c4..7edd6ed5ff 100644 --- a/aggregator/src/aggregation/decoder/tables/fixed.rs +++ b/aggregator/src/aggregation/decoder/tables/fixed.rs @@ -15,6 +15,7 @@ use fse_table_transition::RomFseTableTransition; mod predefined_fse; use predefined_fse::RomPredefinedFse; +pub use predefined_fse::{predefined_fse, PredefinedFse}; mod seq_code_to_value; use seq_code_to_value::RomSeqCodeToValue; diff --git a/aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs b/aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs index e671a559a3..d11f8f45d2 100644 --- a/aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs +++ b/aggregator/src/aggregation/decoder/tables/fixed/fse_table_transition.rs @@ -1,6 +1,6 @@ use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; -use crate::aggregation::decoder::tables::FseTableKind; +use crate::aggregation::decoder::witgen::FseTableKind; use super::{FixedLookupTag, FixedLookupValues}; diff --git a/aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs b/aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs index dc288aeff7..12b7837db9 100644 --- a/aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs +++ b/aggregator/src/aggregation/decoder/tables/fixed/predefined_fse.rs @@ -1,16 +1,16 @@ use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; -use crate::aggregation::decoder::tables::FseTableKind; +use crate::aggregation::decoder::witgen::FseTableKind; use super::{FixedLookupTag, FixedLookupValues}; pub struct RomPredefinedFse { - table_kind: FseTableKind, - table_size: u64, - state: u64, - symbol: u64, - baseline: u64, - nb: u64, + pub table_kind: FseTableKind, + pub table_size: u64, + pub state: u64, + pub symbol: u64, + pub baseline: u64, + pub nb: u64, } pub trait PredefinedFse { diff --git a/aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs b/aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs index 9996c01ae4..819511f7ca 100644 --- a/aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs +++ b/aggregator/src/aggregation/decoder/tables/fixed/seq_code_to_value.rs @@ -1,6 +1,6 @@ use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; -use crate::aggregation::decoder::tables::FseTableKind; +use crate::aggregation::decoder::witgen::FseTableKind; use super::{FixedLookupTag, FixedLookupValues}; diff --git a/aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs b/aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs index 10a5db8004..4e79fdb227 100644 --- a/aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs +++ b/aggregator/src/aggregation/decoder/tables/fixed/seq_data_interleaved_order.rs @@ -1,8 +1,9 @@ use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; -use crate::aggregation::decoder::tables::{fixed::FixedLookupTag, FseTableKind}; - -use super::FixedLookupValues; +use crate::aggregation::decoder::{ + tables::fixed::{FixedLookupTag, FixedLookupValues}, + witgen::FseTableKind, +}; pub struct RomSeqDataInterleavedOrder { /// FSE table used in the previous bitstring. diff --git a/aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs b/aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs index da0015d316..a63620b958 100644 --- a/aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs +++ b/aggregator/src/aggregation/decoder/tables/fixed/seq_tag_order.rs @@ -1,8 +1,8 @@ use halo2_proofs::{circuit::Value, halo2curves::bn256::Fr}; use crate::aggregation::decoder::{ - tables::{fixed::FixedLookupTag, FseTableKind}, - witgen::ZstdTag, + tables::fixed::FixedLookupTag, + witgen::{FseTableKind, ZstdTag}, }; use super::FixedLookupValues; diff --git a/aggregator/src/aggregation/decoder/tables/fse.rs b/aggregator/src/aggregation/decoder/tables/fse.rs index a00897590b..e02e157db2 100644 --- a/aggregator/src/aggregation/decoder/tables/fse.rs +++ b/aggregator/src/aggregation/decoder/tables/fse.rs @@ -13,8 +13,9 @@ use zkevm_circuits::{ table::{BitwiseOp, BitwiseOpTable, LookupTable, Pow2Table, RangeTable, U8Table}, }; -use crate::aggregation::decoder::tables::rom_fse_order::{ - FseTableKind, RomFsePredefinedTable, RomFseTableTransition, +use crate::aggregation::decoder::{ + tables::{FixedLookupTag, FixedTable}, + witgen::FseTableKind, }; /// The FSE table verifies that given the symbols and the states allocated to those symbols, the @@ -119,10 +120,8 @@ pub struct FseTable { baseline: Column, /// The number of bits to read from bitstream when at this state. nb: Column, - /// ROM table for verifying FSE table kind and block_idx transition. - rom_fse_transition: RomFseTableTransition, - /// ROM table for FSE predefined tables. - rom_fse_predefined: RomFsePredefinedTable, + /// Fixed lookup table. + fixed_table: FixedTable, } impl FseTable { @@ -135,8 +134,7 @@ impl FseTable { bitwise_op_table: BitwiseOpTable, ) -> Self { // Fixed table to check the transition of table kinds and block idx. - let rom_fse_transition = RomFseTableTransition::construct(meta); - let rom_fse_predefined = RomFsePredefinedTable::construct(meta); + let fixed_table = FixedTable::construct(meta); // Auxiliary table to validate that (baseline, nb) were assigned correctly to the states // allocated to a symbol. @@ -157,8 +155,7 @@ impl FseTable { is_skipped_state: meta.advice_column(), baseline: meta.advice_column(), nb: meta.advice_column(), - rom_fse_transition, - rom_fse_predefined, + fixed_table, }; // Check that on the starting row of each FSE table, i.e. q_start=true: @@ -229,13 +226,16 @@ impl FseTable { ); [ + FixedLookupTag::FseTableTransition.expr(), block_idx_prev, block_idx_curr, table_kind_prev, table_kind_curr, + 0.expr(), // unused + 0.expr(), // unused ] .into_iter() - .zip_eq(config.rom_fse_transition.table_exprs(meta)) + .zip_eq(config.fixed_table.table_exprs(meta)) .map(|(arg, table)| (condition.expr() * arg, table)) .collect() }, @@ -595,11 +595,19 @@ impl FseTable { meta.query_advice(config.nb, Rotation::cur()), ); - [table_kind, table_size, state, symbol, baseline, nb] - .into_iter() - .zip_eq(config.rom_fse_predefined.table_exprs(meta)) - .map(|(arg, table)| (condition.expr() * arg, table)) - .collect() + [ + FixedLookupTag::PredefinedFse.expr(), + table_kind, + table_size, + state, + symbol, + baseline, + nb, + ] + .into_iter() + .zip_eq(config.fixed_table.table_exprs(meta)) + .map(|(arg, table)| (condition.expr() * arg, table)) + .collect() }); // For every new symbol detected. diff --git a/aggregator/src/aggregation/decoder/tables/rom_fse_order.rs b/aggregator/src/aggregation/decoder/tables/rom_fse_order.rs deleted file mode 100644 index 173ec8ada1..0000000000 --- a/aggregator/src/aggregation/decoder/tables/rom_fse_order.rs +++ /dev/null @@ -1,784 +0,0 @@ -use eth_types::Field; -use gadgets::impl_expr; -use halo2_proofs::{ - circuit::{Layouter, Value}, - halo2curves::bn256::Fr, - plonk::{Column, ConstraintSystem, Error, Expression, Fixed}, -}; -use itertools::Itertools; -use once_cell::sync::Lazy; -use zkevm_circuits::table::LookupTable; - -use crate::aggregation::decoder::witgen::ZstdTag::{ - // TODO: update to the correct tags once witgen code is merged. - ZstdBlockFseCode as ZstdBlockSequenceFseCode, - ZstdBlockLstream as ZstdBlockSequenceData, - ZstdBlockSequenceHeader, -}; - -/// FSE table variants that we observe in the sequences section. -#[derive(Clone, Copy, Debug)] -#[allow(clippy::upper_case_acronyms)] -pub enum FseTableKind { - /// Literal length FSE table. - LLT = 1, - /// Match offset FSE table. - MOT, - /// Match length FSE table. - MLT, -} - -impl_expr!(FseTableKind); - -/// Read-only table that allows us to check the correct assignment of FSE table kind. -/// -/// The possible orders are: -/// -/// - (1, 1, 1): -/// - SequenceHeader > FseCode > FseCode (LLT) -/// - FseCode > FseCode > FseCode (MOT) -/// - FseCode > FseCode > SequenceData (MLT) -/// - (1, 1, 0): -/// - SequenceHeader > FseCode > FseCode (LLT) -/// - FseCode > FseCode > SequenceData (MOT) -/// - (1, 0, 1): -/// - SequenceHeader > FseCode > FseCode (LLT) -/// - FseCode > FseCode > SequenceData (MLT) -/// - (0, 1, 1): -/// - SequenceHeader > FseCode > FseCode (MOT) -/// - FseCode > FseCode > SequenceData (MLT) -/// - (1, 0, 0): -/// - SequenceHeader > FseCode > SequenceData (LLT) -/// - (0, 1, 0): -/// - SequenceHeader > FseCode > SequenceData (MOT) -/// - (0, 0, 1): -/// - SequenceHeader > FseCode > SequenceData (MLT) -#[derive(Clone, Debug)] -pub struct RomFseOrderTable { - /// Compression mode boolean flags for LLT, MOT and MLT respectively. - /// - Predefined_Mode > 0 - /// - Fse_Compressed_Mode > 1 - /// - /// We use a linear combination of the 3 flags: - /// - 4 * llt_flag + 2 * mot_flag + mlt_flag - compression_modes_lc: Column, - /// The tag that occurred previously. - tag_prev: Column, - /// The current tag, expected to be ZstdBlockSequenceFseCode. - tag_cur: Column, - /// The tag that follows the current tag. - tag_next: Column, - /// The FSE table kind, possible values: LLT=1, MOT=2, MLT=3. - table_kind: Column, -} - -impl RomFseOrderTable { - pub fn construct(meta: &mut ConstraintSystem) -> Self { - Self { - compression_modes_lc: meta.fixed_column(), - tag_prev: meta.fixed_column(), - tag_cur: meta.fixed_column(), - tag_next: meta.fixed_column(), - table_kind: meta.fixed_column(), - } - } - - /// Load the FSE order ROM table. - pub fn load(&self, layouter: &mut impl Layouter) -> Result<(), Error> { - use crate::aggregation::decoder::witgen::ZstdTag::{ - ZstdBlockFseCode as FseCode, ZstdBlockLstream as SeqData, - ZstdBlockSequenceHeader as SeqHeader, - }; - use FseTableKind::{LLT, MLT, MOT}; - - layouter.assign_region( - || "(ROM): FSE order table", - |mut region| { - for (offset, row) in [ - // (1, 1, 1) - (7, SeqHeader, FseCode, FseCode, LLT), - (7, FseCode, FseCode, FseCode, MOT), - (7, FseCode, FseCode, SeqData, MLT), - // (1, 1, 0) - (6, SeqHeader, FseCode, FseCode, LLT), - (6, FseCode, FseCode, SeqData, MOT), - // (1, 0, 1) - (5, SeqHeader, FseCode, FseCode, LLT), - (5, FseCode, FseCode, SeqData, MLT), - // (0, 1, 1) - (3, SeqHeader, FseCode, FseCode, MOT), - (3, FseCode, FseCode, SeqData, MLT), - // (1, 0, 0) - (4, SeqHeader, FseCode, SeqData, LLT), - // (0, 1, 0) - (2, SeqHeader, FseCode, SeqData, MOT), - // (0, 0, 1) - (1, SeqHeader, FseCode, SeqData, MLT), - ] - .iter() - .enumerate() - { - region.assign_fixed( - || format!("compression_modes_lc at offset={offset}"), - self.compression_modes_lc, - offset, - || Value::known(Fr::from(row.0 as u64)), - )?; - region.assign_fixed( - || format!("tag_prev at offset={offset}"), - self.tag_prev, - offset, - || Value::known(Fr::from(row.1 as u64)), - )?; - region.assign_fixed( - || format!("tag_cur at offset={offset}"), - self.tag_cur, - offset, - || Value::known(Fr::from(row.2 as u64)), - )?; - region.assign_fixed( - || format!("tag_next at offset={offset}"), - self.tag_next, - offset, - || Value::known(Fr::from(row.3 as u64)), - )?; - region.assign_fixed( - || format!("table_kind at offset={offset}"), - self.table_kind, - offset, - || Value::known(Fr::from(row.4 as u64)), - )?; - } - - Ok(()) - }, - ) - } -} - -impl LookupTable for RomFseOrderTable { - fn columns(&self) -> Vec> { - vec![ - self.compression_modes_lc.into(), - self.tag_prev.into(), - self.tag_cur.into(), - self.tag_next.into(), - self.table_kind.into(), - ] - } - - fn annotations(&self) -> Vec { - vec![ - String::from("compression_modes_lc"), - String::from("tag_prev"), - String::from("tag_cur"), - String::from("tag_next"), - String::from("table_kind"), - ] - } -} - -#[derive(Clone, Debug)] -pub struct RomFseTableTransition { - /// The block index on the previous FSE table. - block_idx_prev: Column, - /// The block index on the current FSE table. - block_idx_curr: Column, - /// The FSE table previously decoded. - table_kind_prev: Column, - /// The FSE table currently decoded. - table_kind_curr: Column, -} - -impl RomFseTableTransition { - pub fn construct(meta: &mut ConstraintSystem) -> Self { - Self { - block_idx_prev: meta.fixed_column(), - block_idx_curr: meta.fixed_column(), - table_kind_prev: meta.fixed_column(), - table_kind_curr: meta.fixed_column(), - } - } - - pub fn load(&self, layouter: &mut impl Layouter) -> Result<(), Error> { - layouter.assign_region( - || "ROM: fse table transition", - |mut region| { - // assign for the preliminary transition. - region.assign_fixed( - || "block_idx_prev", - self.block_idx_prev, - 0, - || Value::known(Fr::zero()), - )?; - region.assign_fixed( - || "block_idx_curr", - self.block_idx_curr, - 0, - || Value::known(Fr::one()), - )?; - region.assign_fixed( - || "table_kind_prev", - self.table_kind_prev, - 0, - || Value::known(Fr::zero()), - )?; - region.assign_fixed( - || "table_kind_curr", - self.table_kind_curr, - 0, - || Value::known(Fr::from(FseTableKind::LLT as u64)), - )?; - - // assign for the other transitons. - for (i, &(block_idx_prev, block_idx_curr, table_kind_prev, table_kind_curr)) in [ - (1, 1, FseTableKind::LLT, FseTableKind::MOT), - (1, 1, FseTableKind::MOT, FseTableKind::MLT), - // TODO: add more for multi-block scenario - ] - .iter() - .enumerate() - { - region.assign_fixed( - || "block_idx_prev", - self.block_idx_prev, - i + 1, - || Value::known(Fr::from(block_idx_prev)), - )?; - region.assign_fixed( - || "block_idx_curr", - self.block_idx_curr, - i + 1, - || Value::known(Fr::from(block_idx_curr)), - )?; - region.assign_fixed( - || "table_kind_prev", - self.table_kind_prev, - i + 1, - || Value::known(Fr::from(table_kind_prev as u64)), - )?; - region.assign_fixed( - || "table_kind_curr", - self.table_kind_curr, - i + 1, - || Value::known(Fr::from(table_kind_curr as u64)), - )?; - } - - Ok(()) - }, - ) - } -} - -impl LookupTable for RomFseTableTransition { - fn columns(&self) -> Vec> { - vec![ - self.block_idx_prev.into(), - self.block_idx_curr.into(), - self.table_kind_prev.into(), - self.table_kind_curr.into(), - ] - } - - fn annotations(&self) -> Vec { - vec![ - String::from("block_idx_prev"), - String::from("block_idx_curr"), - String::from("table_kind_prev"), - String::from("table_kind_curr"), - ] - } -} - -#[derive(Clone, Debug)] -pub struct RomSequencesDataInterleavedOrder { - /// FSE table used in the previous bitstring. - table_kind_prev: Column, - /// FSE table used in the current bitstring. - table_kind_curr: Column, - /// Boolean flag to indicate whether we are initialising the FSE state. - is_init_state: Column, - /// Boolean flag to indicate whether we are updating the FSE state. - is_update_state: Column, -} - -impl RomSequencesDataInterleavedOrder { - pub fn construct(meta: &mut ConstraintSystem) -> Self { - Self { - table_kind_prev: meta.fixed_column(), - table_kind_curr: meta.fixed_column(), - is_init_state: meta.fixed_column(), - is_update_state: meta.fixed_column(), - } - } - - pub fn load(&self, layouter: &mut impl Layouter) -> Result<(), Error> { - layouter.assign_region( - || "(ROM): sequences data interleaved order", - |mut region| { - // handle the first row, i.e. (None, LLT, init_state=true, update_state=false). - region.assign_fixed( - || "table_kind_prev", - self.table_kind_prev, - 0, - || Value::known(Fr::zero()), - )?; - region.assign_fixed( - || "table_kind_curr", - self.table_kind_curr, - 0, - || Value::known(Fr::from(FseTableKind::LLT as u64)), - )?; - region.assign_fixed( - || "is_init_state", - self.is_init_state, - 0, - || Value::known(Fr::one()), - )?; - region.assign_fixed( - || "is_update_state", - self.is_update_state, - 0, - || Value::known(Fr::zero()), - )?; - - for (i, &(table_kind_prev, table_kind_curr, is_init_state, is_update_state)) in [ - (FseTableKind::LLT, FseTableKind::MOT, true, false), // init state (MOT) - (FseTableKind::MOT, FseTableKind::MLT, true, false), // init state (MLT) - (FseTableKind::MLT, FseTableKind::MOT, false, false), - (FseTableKind::MOT, FseTableKind::MLT, false, false), - (FseTableKind::MLT, FseTableKind::LLT, false, false), - (FseTableKind::LLT, FseTableKind::LLT, false, true), - (FseTableKind::LLT, FseTableKind::MLT, false, true), - (FseTableKind::MLT, FseTableKind::MOT, false, true), - (FseTableKind::MOT, FseTableKind::MOT, false, false), - ] - .iter() - .enumerate() - { - region.assign_fixed( - || "table_kind_prev", - self.table_kind_prev, - i + 1, - || Value::known(Fr::from(table_kind_prev as u64)), - )?; - region.assign_fixed( - || "table_kind_curr", - self.table_kind_curr, - i + 1, - || Value::known(Fr::from(table_kind_curr as u64)), - )?; - region.assign_fixed( - || "is_init_state", - self.is_init_state, - i + 1, - || Value::known(Fr::from(is_init_state as u64)), - )?; - region.assign_fixed( - || "is_update_state", - self.is_update_state, - i + 1, - || Value::known(Fr::from(is_update_state as u64)), - )?; - } - - Ok(()) - }, - ) - } -} - -impl LookupTable for RomSequencesDataInterleavedOrder { - fn columns(&self) -> Vec> { - vec![ - self.table_kind_prev.into(), - self.table_kind_curr.into(), - self.is_init_state.into(), - self.is_update_state.into(), - ] - } - - fn annotations(&self) -> Vec { - vec![ - String::from("table_kind_prev"), - String::from("table_kind_curr"), - String::from("is_init_state"), - String::from("is_update_state"), - ] - } -} - -pub trait FsePredefinedTable { - /// Get the accuracy log of the predefined table. - fn accuracy_log(&self) -> u8; - /// Get the number of states in the FSE table. - fn table_size(&self) -> u64 { - 1 << self.accuracy_log() - } - /// Get the symbol in the FSE table for the given state. - fn symbol(&self, state: u64) -> u64; - /// Get the baseline in the FSE table for the given state. - fn baseline(&self, state: u64) -> u64; - /// Get the number of bits (nb) to read from bitstream in the FSE table for the given state. - fn nb(&self, state: u64) -> u64; -} - -impl FsePredefinedTable for FseTableKind { - fn accuracy_log(&self) -> u8 { - match self { - Self::LLT => 6, - Self::MOT => 5, - Self::MLT => 6, - } - } - - fn symbol(&self, state: u64) -> u64 { - match self { - Self::LLT => match state { - 0..=1 => 0, - 2 => 1, - 3 => 3, - 4 => 4, - 5 => 6, - 6 => 7, - 7 => 9, - 8 => 10, - 9 => 12, - 10 => 14, - 11 => 16, - 12 => 18, - 13 => 19, - 14 => 21, - 15 => 22, - 16 => 24, - 17 => 25, - 18 => 26, - 19 => 27, - 20 => 29, - 21 => 31, - 22 => 0, - 23 => 1, - 24 => 2, - 25 => 4, - 26 => 5, - 27 => 7, - 28 => 8, - 29 => 10, - 30 => 11, - 31 => 13, - 32 => 16, - 33 => 17, - 34 => 19, - 35 => 20, - 36 => 22, - 37 => 23, - 38 => 25, - 39 => 25, - 40 => 26, - 41 => 28, - 42 => 30, - 43 => 0, - 44 => 1, - 45 => 2, - 46 => 3, - 47 => 5, - 48 => 6, - 49 => 8, - 50 => 9, - 51 => 11, - 52 => 12, - 53 => 15, - 54 => 17, - 55 => 18, - 56 => 20, - 57 => 21, - 58 => 23, - 59 => 24, - 60 => 35, - 61 => 34, - 62 => 33, - 63 => 32, - _ => unreachable!(), - }, - Self::MOT => match state { - 0 => 0, - 1 => 6, - 2 => 9, - 3 => 15, - 4 => 21, - 5 => 3, - 6 => 7, - 7 => 12, - 8 => 18, - 9 => 23, - 10 => 5, - 11 => 8, - 12 => 14, - 13 => 20, - 14 => 2, - 15 => 7, - 16 => 11, - 17 => 17, - 18 => 22, - 19 => 4, - 20 => 8, - 21 => 13, - 22 => 19, - 23 => 1, - 24 => 6, - 25 => 10, - 26 => 16, - 27 => 28, - 28 => 27, - 29 => 26, - 30 => 25, - 31 => 24, - _ => unreachable!(), - }, - Self::MLT => match state { - 0..=3 => state, - 4..=5 => state + 1, - 6 => 8, - 7 => 10, - 8 => 13, - 9 => 16, - 10 => 19, - 11 => 22, - 12 => 25, - 13 => 28, - 14 => 31, - 15 => 33, - 16 => 35, - 17 => 37, - 18 => 39, - 19 => 41, - 20 => 43, - 21 => 45, - 22..=25 => state - 21, - 26..=27 => state - 20, - 28 => 9, - 29 => 12, - 30 => 15, - 31 => 18, - 32 => 21, - 33 => 24, - 34 => 27, - 35 => 30, - 36 => 32, - 37 => 34, - 38 => 36, - 39 => 38, - 40 => 40, - 41 => 42, - 42 => 44, - 43..=44 => 1, - 45 => 2, - 46..=47 => state - 42, - 48 => 7, - 49 => 8, - 50 => 11, - 51 => 14, - 52 => 17, - 53 => 20, - 54 => 23, - 55 => 26, - 56 => 29, - 57 => 52, - 58 => 51, - 59 => 50, - 60 => 49, - 61 => 48, - 62 => 47, - 63 => 46, - _ => unreachable!(), - }, - } - } - - fn baseline(&self, state: u64) -> u64 { - match self { - Self::LLT => match state { - 0 => 0, - 1 => 16, - 2 => 32, - 3..=16 => 0, - 17 => 32, - 18..=21 | 23..=24 => 0, - 22 | 25 | 27 | 29 | 32 | 34 | 36 | 40 => 32, - 26 | 28 | 30..=31 | 33 | 35 | 37..=38 | 41..=42 | 53 | 60..=63 => 0, - 39 | 44 => 16, - 43 => 48, - 45..=52 | 54..=59 => 32, - _ => unreachable!(), - }, - Self::MOT => match state { - 0..=14 | 16..=19 | 21..=23 | 25..=31 => 0, - 15 | 20 | 24 => 16, - _ => unreachable!(), - }, - Self::MLT => match state { - 0..=1 | 3..=21 | 23 | 25 | 27..=42 | 50..=63 => 0, - 2 | 24 | 26 | 43 | 46..=49 => 32, - 22 | 45 => 16, - 44 => 48, - _ => unreachable!(), - }, - } - } - - fn nb(&self, state: u64) -> u64 { - match self { - Self::LLT => match state { - 0..=1 | 22..=23 | 38..=39 | 43..=44 => 4, - 2..=9 | 11..=18 | 24..=30 | 32..=37 | 40 | 45..=52 | 54..=59 => 5, - 10 | 19..=21 | 31 | 41..=42 | 53 | 60..=63 => 6, - _ => unreachable!(), - }, - Self::MOT => match state { - 0 | 2..=5 | 7..=10 | 12..=14 | 16..=19 | 21..=23 | 25..=31 => 5, - 1 | 6 | 11 | 15 | 20 | 24 => 4, - _ => unreachable!(), - }, - Self::MLT => match state { - 0 | 7..=21 | 28..=42 | 50..=63 => 6, - 1 | 22..=23 | 43..=45 => 4, - 2..=6 | 24..=27 | 46..=49 => 5, - _ => unreachable!(), - }, - } - } -} - -pub fn predefined_table(table_kind: FseTableKind) -> Vec<(u64, u64, u64, u64)> { - let table_size = table_kind.table_size(); - (0..table_size) - .map(|state| { - ( - state, - table_kind.symbol(state), - table_kind.baseline(state), - table_kind.nb(state), - ) - }) - .collect() -} - -pub fn predefined_table_values(table_kind: FseTableKind) -> Vec<[Value; 6]> { - let table_size = table_kind.table_size(); - (0..table_size) - .map(|state| { - let symbol = table_kind.symbol(state); - let baseline = table_kind.baseline(state); - let nb = table_kind.nb(state); - [ - Value::known(Fr::from(table_kind as u64)), - Value::known(Fr::from(table_size)), - Value::known(Fr::from(state)), - Value::known(Fr::from(symbol)), - Value::known(Fr::from(baseline)), - Value::known(Fr::from(nb)), - ] - }) - .collect() -} - -/// The Predefined Literal Length FSE table, as per default distributions. -/// -/// - https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#literals-length -/// - https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#literal-length-code -pub static FSE_PREDEFINED_LLT: Lazy; 6]>> = - Lazy::new(|| predefined_table_values(FseTableKind::LLT)); - -/// The Predefined Match Length FSE table, as per default distributions. -/// -/// - https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#match-length -/// - https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#match-length-code -pub static FSE_PREDEFINED_MLT: Lazy; 6]>> = - Lazy::new(|| predefined_table_values(FseTableKind::MLT)); - -/// The Predefined Match Offset FSE table, as per default distributions. -/// -/// - https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#offset-codes-1 -/// - https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#offset-code -pub static FSE_PREDEFINED_MOT: Lazy; 6]>> = - Lazy::new(|| predefined_table_values(FseTableKind::MOT)); - -#[derive(Clone, Debug)] -pub struct RomFsePredefinedTable { - table_kind: Column, - table_size: Column, - state: Column, - symbol: Column, - baseline: Column, - nb: Column, -} - -impl RomFsePredefinedTable { - pub fn construct(meta: &mut ConstraintSystem) -> Self { - Self { - table_kind: meta.fixed_column(), - table_size: meta.fixed_column(), - state: meta.fixed_column(), - symbol: meta.fixed_column(), - baseline: meta.fixed_column(), - nb: meta.fixed_column(), - } - } - - pub fn load(&self, layouter: &mut impl Layouter) -> Result<(), Error> { - layouter.assign_region( - || "ROM: fse predefined", - |mut region| { - for (offset, row) in [ - FSE_PREDEFINED_LLT.as_slice(), - FSE_PREDEFINED_MLT.as_slice(), - FSE_PREDEFINED_MOT.as_slice(), - ] - .concat() - .iter() - .enumerate() - { - for ((&column, annotation), &value) in Self::fixed_columns(self) - .iter() - .zip_eq(Self::annotations(self)) - .zip_eq(row.iter()) - { - region.assign_fixed( - || format!("{annotation} at offset={offset}"), - column, - offset, - || value, - )?; - } - } - - Ok(()) - }, - ) - } -} - -impl LookupTable for RomFsePredefinedTable { - fn columns(&self) -> Vec> { - vec![ - self.table_kind.into(), - self.table_size.into(), - self.state.into(), - self.symbol.into(), - self.baseline.into(), - self.nb.into(), - ] - } - - fn annotations(&self) -> Vec { - vec![ - String::from("table_kind"), - String::from("table_size"), - String::from("state"), - String::from("symbol"), - String::from("baseline"), - String::from("nb"), - ] - } -} diff --git a/aggregator/src/aggregation/decoder/tables/rom_sequence_codes.rs b/aggregator/src/aggregation/decoder/tables/rom_sequence_codes.rs deleted file mode 100644 index 4e4903315f..0000000000 --- a/aggregator/src/aggregation/decoder/tables/rom_sequence_codes.rs +++ /dev/null @@ -1,189 +0,0 @@ -use halo2_proofs::{ - circuit::{Layouter, Value}, - halo2curves::bn256::Fr, - plonk::{Any, Column, ConstraintSystem, Error, Fixed}, -}; -use zkevm_circuits::table::LookupTable; - -use super::rom_fse_order::FseTableKind; - -pub struct CodeFseRow { - pub table_kind: FseTableKind, - pub code: u64, - pub baseline: u64, - pub nb: u64, -} - -impl From<(FseTableKind, u64, u64, u64)> for CodeFseRow { - fn from(v: (FseTableKind, u64, u64, u64)) -> Self { - Self { - table_kind: v.0, - code: v.1, - baseline: v.2, - nb: v.3, - } - } -} - -pub trait SequenceCodeTable { - fn code_table() -> Vec; -} - -#[derive(Clone, Debug)] -pub struct RomSequenceCodes { - table_kind: Column, - code: Column, - baseline: Column, - nb: Column, -} - -impl RomSequenceCodes { - pub fn construct(meta: &mut ConstraintSystem) -> Self { - Self { - table_kind: meta.fixed_column(), - code: meta.fixed_column(), - baseline: meta.fixed_column(), - nb: meta.fixed_column(), - } - } - - pub fn load(&self, layouter: &mut impl Layouter) -> Result<(), Error> { - layouter.assign_region( - || "(ROM): Sequence Codes (code to value)", - |mut region| { - for (offset, row) in std::iter::empty() - .chain(LiteralLengthCodes::code_table()) - .chain(MatchLengthCodes::code_table()) - .chain(MatchOffsetCodes::code_table()) - .enumerate() - { - region.assign_fixed( - || "table_kind", - self.table_kind, - offset, - || Value::known(Fr::from(row.table_kind as u64)), - )?; - region.assign_fixed( - || "code", - self.code, - offset, - || Value::known(Fr::from(row.code)), - )?; - region.assign_fixed( - || "baseline", - self.baseline, - offset, - || Value::known(Fr::from(row.baseline)), - )?; - region.assign_fixed( - || "nb", - self.nb, - offset, - || Value::known(Fr::from(row.nb)), - )?; - } - - Ok(()) - }, - ) - } -} - -impl LookupTable for RomSequenceCodes { - fn columns(&self) -> Vec> { - vec![ - self.table_kind.into(), - self.code.into(), - self.baseline.into(), - self.nb.into(), - ] - } - - fn annotations(&self) -> Vec { - vec![ - String::from("table_kind"), - String::from("code"), - String::from("baseline"), - String::from("nb"), - ] - } -} - -#[derive(Clone, Debug)] -pub struct LiteralLengthCodes; -#[derive(Clone, Debug)] -pub struct MatchLengthCodes; -#[derive(Clone, Debug)] -pub struct MatchOffsetCodes; - -impl SequenceCodeTable for LiteralLengthCodes { - fn code_table() -> Vec { - (0..16) - .map(|i| (i, i, 0)) - .chain([ - (16, 16, 1), - (17, 18, 1), - (18, 20, 1), - (19, 22, 1), - (20, 24, 2), - (21, 28, 2), - (22, 32, 3), - (23, 40, 3), - (24, 48, 4), - (25, 64, 6), - (26, 128, 7), - (27, 256, 8), - (28, 512, 9), - (29, 1024, 10), - (30, 2048, 11), - (31, 4096, 12), - (32, 8192, 13), - (33, 16384, 14), - (34, 32768, 15), - (35, 65536, 16), - ]) - .map(|tuple| (FseTableKind::LLT, tuple.0, tuple.1, tuple.2).into()) - .collect() - } -} - -impl SequenceCodeTable for MatchLengthCodes { - fn code_table() -> Vec { - (0..32) - .map(|i| (i, i + 3, 0)) - .chain([ - (32, 35, 1), - (33, 37, 1), - (34, 39, 1), - (35, 41, 1), - (36, 43, 2), - (37, 47, 2), - (38, 51, 3), - (39, 59, 3), - (40, 67, 4), - (41, 83, 4), - (42, 99, 5), - (43, 131, 7), - (44, 259, 8), - (45, 515, 9), - (46, 1027, 10), - (47, 2051, 11), - (48, 4099, 12), - (49, 8195, 13), - (50, 16387, 14), - (51, 32771, 15), - (52, 65539, 16), - ]) - .map(|tuple| (FseTableKind::MLT, tuple.0, tuple.1, tuple.2).into()) - .collect() - } -} - -impl SequenceCodeTable for MatchOffsetCodes { - // N <- 31 for Match Offset Codes. - fn code_table() -> Vec { - (0..32) - .map(|i| (FseTableKind::MOT, i, 1 << i, i).into()) - .collect() - } -} diff --git a/aggregator/src/aggregation/decoder/tables/rom_tag.rs b/aggregator/src/aggregation/decoder/tables/rom_tag.rs deleted file mode 100644 index 25fdceb97d..0000000000 --- a/aggregator/src/aggregation/decoder/tables/rom_tag.rs +++ /dev/null @@ -1,93 +0,0 @@ -use halo2_proofs::{ - circuit::Layouter, - halo2curves::bn256::Fr, - plonk::{Any, Column, ConstraintSystem, Error, Fixed}, -}; -use itertools::Itertools; -use zkevm_circuits::table::LookupTable; - -use crate::aggregation::decoder::witgen::RomTagTableRow; - -/// Read-only Memory table for the Decompression circuit. This table allows us a lookup argument -/// from the Decompression circuit to check if a given row can occur depending on the row's tag, -/// next tag and tag length. -#[derive(Clone, Copy, Debug)] -pub struct RomTagTable { - /// Tag of the current field being decoded. - pub tag: Column, - /// Tag of the following field when the current field is finished decoding. - pub tag_next: Column, - /// The maximum length in terms of number of bytes that the current tag can take up. - pub max_len: Column, - /// Whether this tag outputs a decoded byte or not. - pub is_output: Column, - /// Whether this tag is processed back-to-front, i.e. in reverse order. - pub is_reverse: Column, - /// Whether this tag belongs to a ``block`` in zstd or not. - pub is_block: Column, -} - -impl LookupTable for RomTagTable { - fn columns(&self) -> Vec> { - vec![ - self.tag.into(), - self.tag_next.into(), - self.max_len.into(), - self.is_output.into(), - self.is_reverse.into(), - self.is_block.into(), - ] - } - - fn annotations(&self) -> Vec { - vec![ - String::from("tag"), - String::from("tag_next"), - String::from("max_len"), - String::from("is_output"), - String::from("is_reverse"), - String::from("is_block"), - ] - } -} - -impl RomTagTable { - /// Construct the Tag ROM table. - pub fn construct(meta: &mut ConstraintSystem) -> Self { - Self { - tag: meta.fixed_column(), - tag_next: meta.fixed_column(), - max_len: meta.fixed_column(), - is_output: meta.fixed_column(), - is_reverse: meta.fixed_column(), - is_block: meta.fixed_column(), - } - } - - /// Load the Tag ROM table. - pub fn load(&self, layouter: &mut impl Layouter) -> Result<(), Error> { - layouter.assign_region( - || "(ROM): Zstd tag table", - |mut region| { - for (offset, row) in RomTagTableRow::rows().iter().enumerate() { - for (&column, (value, annotation)) in - >::fixed_columns(self).iter().zip( - row.values::() - .into_iter() - .zip_eq(>::annotations(self).iter()), - ) - { - region.assign_fixed( - || format!("{annotation} at offset={offset}"), - column, - offset, - || value, - )?; - } - } - - Ok(()) - }, - ) - } -} diff --git a/aggregator/src/aggregation/decoder/witgen.rs b/aggregator/src/aggregation/decoder/witgen.rs index 376456a4da..e25b1744e3 100644 --- a/aggregator/src/aggregation/decoder/witgen.rs +++ b/aggregator/src/aggregation/decoder/witgen.rs @@ -12,8 +12,6 @@ pub use types::{ZstdTag::*, *}; pub mod util; use util::{be_bits_to_value, increment_idx, le_bits_to_value, value_bits_le}; -use crate::aggregation::decoder::tables::FseTableKind; - const TAG_MAX_LEN: [(ZstdTag, u64); 13] = [ (FrameHeaderDescriptor, 1), (FrameContentSize, 8), diff --git a/aggregator/src/aggregation/decoder/witgen/types.rs b/aggregator/src/aggregation/decoder/witgen/types.rs index fb7afd742d..71a8c7d25f 100644 --- a/aggregator/src/aggregation/decoder/witgen/types.rs +++ b/aggregator/src/aggregation/decoder/witgen/types.rs @@ -10,8 +10,6 @@ use halo2_proofs::{circuit::Value, plonk::Expression}; use itertools::Itertools; use strum_macros::EnumIter; -use crate::aggregation::decoder::tables::FseTableKind; - use super::{ params::N_BITS_PER_BYTE, util::{bit_length, read_variable_bit_packing, smaller_powers_of_two, value_bits_le}, @@ -281,6 +279,20 @@ impl From for usize { } } +/// FSE table variants that we observe in the sequences section. +#[derive(Clone, Copy, Debug)] +#[allow(clippy::upper_case_acronyms)] +pub enum FseTableKind { + /// Literal length FSE table. + LLT = 1, + /// Match offset FSE table. + MOT, + /// Match length FSE table. + MLT, +} + +impl_expr!(FseTableKind); + impl ToString for ZstdTag { fn to_string(&self) -> String { String::from(match self { @@ -891,7 +903,7 @@ impl ZstdWitnessRow { #[cfg(test)] mod tests { - use crate::aggregation::decoder::tables::{predefined_table, FsePredefinedTable}; + use crate::aggregation::decoder::tables::{predefined_fse, PredefinedFse}; use super::*; @@ -987,7 +999,7 @@ mod tests { &normalised_probs, table_kind.accuracy_log(), ); - let expected_predefined_table = predefined_table(table_kind); + let expected_predefined_table = predefined_fse(table_kind); let mut computed_predefined_table = sym_to_states .values() @@ -1001,14 +1013,18 @@ mod tests { .zip_eq(computed_predefined_table.iter()) .enumerate() { - assert_eq!(computed.state, expected.0, "state mismatch at i={}", i); - assert_eq!(computed.symbol, expected.1, "symbol mismatch at i={}", i); + assert_eq!(computed.state, expected.state, "state mismatch at i={}", i); + assert_eq!( + computed.symbol, expected.symbol, + "symbol mismatch at i={}", + i + ); assert_eq!( - computed.baseline, expected.2, + computed.baseline, expected.baseline, "baseline mismatch at i={}", i ); - assert_eq!(computed.num_bits, expected.3, "nb mismatch at i={}", i); + assert_eq!(computed.num_bits, expected.nb, "nb mismatch at i={}", i); } } }