From ec3d49d20e276025a8a97ce6c3e08ae79aaa3e68 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Thu, 16 Jan 2025 17:03:22 +0100 Subject: [PATCH 01/63] feat(conglomeration): implements the context translation --- prover/protocol/compiler/vortex/compiler.go | 10 +- prover/protocol/compiler/vortex/option.go | 2 +- prover/protocol/compiler/vortex/prover.go | 2 +- .../conglomeration/conglomeration.go | 202 ++++++++++++++++++ .../distributed/conglomeration/translator.go | 103 +++++++++ 5 files changed, 312 insertions(+), 7 deletions(-) create mode 100644 prover/protocol/distributed/conglomeration/conglomeration.go create mode 100644 prover/protocol/distributed/conglomeration/translator.go diff --git a/prover/protocol/compiler/vortex/compiler.go b/prover/protocol/compiler/vortex/compiler.go index 7aa626bd381..5eadb65a98e 100644 --- a/prover/protocol/compiler/vortex/compiler.go +++ b/prover/protocol/compiler/vortex/compiler.go @@ -119,7 +119,7 @@ type Ctx struct { VortexParams *vortex.Params SisParams *ringsis.Params // Optional parameter - numOpenedCol int + NumOpenedCol int // By rounds commitments : if a round is dried we make an empty sublist. // Inversely, for the `driedByRounds` which track the dried commitments. @@ -140,7 +140,7 @@ type Ctx struct { // Committed matrix (rs encoded) of the precomputed columns CommittedMatrix vortex.EncodedMatrix // Tree in case of Merkle mode - tree *smt.Tree + Tree *smt.Tree // colHashes used in self recursion DhWithMerkle []field.Element } @@ -437,8 +437,8 @@ func (ctx *Ctx) NbColsToOpen() int { // If the context was created with the relevant option, // we return the instructed value - if ctx.numOpenedCol > 0 { - return ctx.numOpenedCol + if ctx.NumOpenedCol > 0 { + return ctx.NumOpenedCol } if !utils.IsPowerOfTwo(ctx.BlowUpFactor) { @@ -710,7 +710,7 @@ func (ctx *Ctx) commitPrecomputeds() { committedMatrix, tree, colHashes := ctx.VortexParams.CommitMerkle(pols) ctx.Items.Precomputeds.DhWithMerkle = colHashes ctx.Items.Precomputeds.CommittedMatrix = committedMatrix - ctx.Items.Precomputeds.tree = tree + ctx.Items.Precomputeds.Tree = tree // And assign the 1-sized column to contain the root var root field.Element diff --git a/prover/protocol/compiler/vortex/option.go b/prover/protocol/compiler/vortex/option.go index 668e8ec758e..b673a574b74 100644 --- a/prover/protocol/compiler/vortex/option.go +++ b/prover/protocol/compiler/vortex/option.go @@ -11,7 +11,7 @@ type VortexOp func(ctx *Ctx) // not be used in production) func ForceNumOpenedColumns(nbCol int) VortexOp { return func(ctx *Ctx) { - ctx.numOpenedCol = nbCol + ctx.NumOpenedCol = nbCol } } diff --git a/prover/protocol/compiler/vortex/prover.go b/prover/protocol/compiler/vortex/prover.go index bad0c99a18d..c2e8d6d68d6 100644 --- a/prover/protocol/compiler/vortex/prover.go +++ b/prover/protocol/compiler/vortex/prover.go @@ -85,7 +85,7 @@ func (ctx *Ctx) OpenSelectedColumns(pr *wizard.ProverRuntime) { // Append the precomputed committedMatrices and trees when IsCommitToPrecomputed is true if ctx.IsCommitToPrecomputed() { committedMatrices = append(committedMatrices, ctx.Items.Precomputeds.CommittedMatrix) - trees = append(trees, ctx.Items.Precomputeds.tree) + trees = append(trees, ctx.Items.Precomputeds.Tree) } for round := 0; round <= ctx.MaxCommittedRound; round++ { diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go new file mode 100644 index 00000000000..ce4ab61a549 --- /dev/null +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -0,0 +1,202 @@ +package conglomeration + +import ( + "github.com/consensys/linea-monorepo/prover/protocol/coin" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" + "github.com/consensys/linea-monorepo/prover/protocol/ifaces" + "github.com/consensys/linea-monorepo/prover/protocol/query" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" +) + +// recursionCtx holds compilation context informations about the wizard +type recursionCtx struct { + // A pointer to the compiled-IOP over which the compilation step has run + Translator *compTranslator + // The Vortex compilation context + PcsCtx *vortex.Ctx + PublicInputs []wizard.PublicInput + NonEmptyMerkleRootPositions []int + FirstRound, LastRound int + Columns [][]ifaces.Column + QueryParams [][]ifaces.Query + VerifierActions [][]wizard.VerifierAction + Coins [][]coin.Info + FsHooks [][]wizard.VerifierAction + LocalOpenings []query.LocalOpening +} + +func Conglomerate( + tmpl *wizard.CompiledIOP, + maxNumSegment int, +) (comp *wizard.CompiledIOP) { + return nil +} + +func addVerifierToComp( + comp *wizard.CompiledIOP, + tmpl *wizard.CompiledIOP, +) { +} + +// initCtx initializes a new context +func initRecursionCtx( + id string, + target *wizard.CompiledIOP, +) *recursionCtx { + return &recursionCtx{ + Translator: &compTranslator{Prefix: id, Target: target}, + } +} + +// captureCompPreVortex scans the content of tmpl to store the compilation infos of the +// CompiledIOP at the beginning of the compilation. The scanned wizard items are +// inserted into `comp` with a prefix `id` and recorded within the context. +func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { + + var ( + polyQuery = tmpl.PcsCtxs.(*vortex.Ctx).Query + lastRound = tmpl.QueriesParams.Round(polyQuery.QueryID) + ) + + ctx.LastRound = lastRound + + for round := 0; round <= lastRound; round++ { + + ctx.Columns = append(ctx.Columns, []ifaces.Column{}) + ctx.QueryParams = append(ctx.QueryParams, []ifaces.Query{}) + ctx.VerifierActions = append(ctx.VerifierActions, []wizard.VerifierAction{}) + ctx.Coins = append(ctx.Coins, []coin.Info{}) + ctx.FsHooks = append(ctx.FsHooks, []wizard.VerifierAction{}) + + for _, colName := range tmpl.Columns.AllKeysAt(round) { + + // filter the columns by status + var ( + status = tmpl.Columns.Status(colName) + size = tmpl.Columns.GetSize(colName) + ) + + if !status.IsPublic() { + // the column is not public so it is not part of the proof + continue + } + + newCol := ctx.Translator.InsertColumn(round, colName, size, status) + ctx.Columns[round] = append(ctx.Columns[round], newCol) + ctx.Translator.Target.Columns.IgnoreButKeepInProverTranscript(newCol.GetColID()) + } + + for _, qName := range tmpl.QueriesParams.AllKeysAt(round) { + + if tmpl.QueriesParams.IsSkippedFromVerifierTranscript(qName) { + continue + } + + // Importantly, the queries that we port should be already + // compiled in the tmpl. + if !tmpl.QueriesParams.IsIgnored(qName) { + panic("the template is invalid, all its queries should be compiled") + } + + // The uni-eval query is directly handled in a different section + // of the compilation. + if qName == polyQuery.QueryID { + continue + } + + // Not that we do not filter the already compiled queries + qInfo := tmpl.QueriesParams.Data(qName) + qInfo = ctx.Translator.InsertQueryParams(round, qInfo) + ctx.QueryParams[round] = append(ctx.QueryParams[round], qInfo) + ctx.Translator.Target.QueriesParams.MarkAsSkippedFromVerifierTranscript(qInfo.Name()) + } + + for _, cName := range tmpl.Coins.AllKeysAt(round) { + + if tmpl.Coins.IsSkippedFromVerifierTranscript(cName) { + continue + } + + coin := tmpl.Coins.Data(cName) + coin = ctx.Translator.InsertCoin(round, cName, coin.Type, coin.Size) + ctx.Coins[round] = append(ctx.Coins[round], coin) + ctx.Translator.Target.Coins.MarkAsSkippedFromVerifierTranscript(coin.Name) + } + + verifierActions := tmpl.SubVerifiers.Inner() + + for i := range verifierActions[round] { + + va := verifierActions[round][i] + if va.IsSkipped() { + continue + } + + ctx.VerifierActions[round] = append(ctx.VerifierActions[round], va) + } + + resetFs := tmpl.FiatShamirHooks.Inner() + + for _, fsHook := range resetFs[round] { + + if fsHook.IsSkipped() { + continue + } + + ctx.FsHooks[round] = append(ctx.VerifierActions[round], fsHook) + } + } +} + +func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { + + var ( + srcVortexCtx = tmpl.PcsCtxs.(*vortex.Ctx) + dstVortexCtx = &vortex.Ctx{ + BlowUpFactor: srcVortexCtx.BlowUpFactor, + DryTreshold: srcVortexCtx.DryTreshold, + CommittedRowsCount: srcVortexCtx.CommittedRowsCount, + NumCols: srcVortexCtx.NumCols, + MaxCommittedRound: srcVortexCtx.MaxCommittedRound, + NumOpenedCol: srcVortexCtx.NumOpenedCol, + CommitmentsByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.CommitmentsByRounds), + DriedByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.DriedByRounds), + PolynomialsTouchedByTheQuery: ctx.Translator.TranslateColumnSet(srcVortexCtx.PolynomialsTouchedByTheQuery), + ShadowCols: ctx.Translator.TranslateColumnSet(srcVortexCtx.ShadowCols), + Query: ctx.Translator.TranslateUniEval(ctx.LastRound, srcVortexCtx.Query), + } + ) + + if srcVortexCtx.ReplaceSisByMimc { + panic("it should not replace by MiMC") + } + + dstVortexCtx.Items.Precomputeds.PrecomputedColums = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Precomputeds.PrecomputedColums) + dstVortexCtx.Items.Precomputeds.MerkleRoot = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.MerkleRoot.GetColID()) + dstVortexCtx.Items.Precomputeds.Dh = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.Dh.GetColID()) + dstVortexCtx.Items.Precomputeds.CommittedMatrix = srcVortexCtx.Items.Precomputeds.CommittedMatrix + dstVortexCtx.Items.Precomputeds.Tree = srcVortexCtx.Items.Precomputeds.Tree + dstVortexCtx.Items.Precomputeds.DhWithMerkle = srcVortexCtx.Items.Precomputeds.DhWithMerkle + + dstVortexCtx.Items.Dh = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Dh) + dstVortexCtx.Items.Alpha = ctx.Translator.GetCoin(srcVortexCtx.Items.Alpha.Name) + dstVortexCtx.Items.Ualpha = ctx.Translator.GetColumn(srcVortexCtx.Items.Ualpha.GetColID()) + dstVortexCtx.Items.Q = ctx.Translator.GetCoin(srcVortexCtx.Items.Q.Name) + dstVortexCtx.Items.OpenedColumns = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.OpenedColumns) + dstVortexCtx.Items.MerkleProofs = ctx.Translator.GetColumn(srcVortexCtx.Items.MerkleProofs.GetColID()) + dstVortexCtx.Items.MerkleRoots = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.MerkleRoots) +} + +// TranslateUniEval returns a copied UnivariateEval query with the columns translated +// and the names translated. The returned query is registered in the translator comp. +func (comp *compTranslator) TranslateUniEval(round int, q query.UnivariateEval) query.UnivariateEval { + var ( + res = query.NewUnivariateEval(q.QueryID, q.Pols...) + ) + + for i := range res.Pols { + res.Pols[i] = comp.GetColumn(res.Pols[i].GetColID()) + } + + return comp.InsertQueryParams(round, res).(query.UnivariateEval) +} diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go new file mode 100644 index 00000000000..6b33783b897 --- /dev/null +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -0,0 +1,103 @@ +package conglomeration + +import ( + "github.com/consensys/linea-monorepo/prover/protocol/coin" + "github.com/consensys/linea-monorepo/prover/protocol/column" + "github.com/consensys/linea-monorepo/prover/protocol/ifaces" + "github.com/consensys/linea-monorepo/prover/protocol/query" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/consensys/linea-monorepo/prover/utils/collection" +) + +// compTranslator is a builder struct for building a target [wizard.CompiledIOP] +// instances from another source [wizard.CompiledIOP]. All items in the built +// compiled IOP are prefixed with an identifier. +type compTranslator struct { + Prefix string + Target *wizard.CompiledIOP +} + +// InsertColumn inserts a new column in the target compiled IOP. The column name +// is prefixed with comp.Prefix. +func (comp *compTranslator) InsertColumn(round int, name ifaces.ColID, size int, status column.Status) ifaces.Column { + name = ifaces.ColID(comp.Prefix) + "." + name + return comp.Target.InsertColumn(round, name, size, status) +} + +// GetColumn returns a column from the target compiled IOP. +func (comp *compTranslator) GetColumn(name ifaces.ColID) ifaces.Column { + name = ifaces.ColID(comp.Prefix) + "." + name + return comp.Target.Columns.GetHandle(name) +} + +// InsertCoin inserts a new coin in the target compiled IOP. The coin name +// is prefixed with the comp.Prefix. +func (comp *compTranslator) InsertCoin(round int, name coin.Name, type_ coin.Type, size ...int) coin.Info { + name = coin.Name(comp.Prefix) + "." + name + return comp.Target.InsertCoin(round, name, type_, size...) +} + +// GetCoin returns a coin with the prefixed name in the target compiled IOP. +// It panics if the prefixed coin is not found. +func (comp *compTranslator) GetCoin(name coin.Name) coin.Info { + name = coin.Name(comp.Prefix) + "." + name + return comp.Target.Coins.Data(name) +} + +// InsertQueryParams inserts a new query in the target compiled IOP prefixing the +// query name however the inner-fields of the query are not prefixed or translated. +// So it should be preferrably applied only over "Ignored" queries as the content of +// the inserted query will be invalid. +func (comp *compTranslator) InsertQueryParams(round int, q ifaces.Query) ifaces.Query { + name := ifaces.QueryID(comp.Prefix) + "." + q.Name() + q = copyQueryWithName(name, q) + comp.Target.QueriesParams.AddToRound(round, name, q) + return q +} + +// copyQueryWithName returns a copy of the query with a new name. +func copyQueryWithName(name ifaces.QueryID, q ifaces.Query) ifaces.Query { + switch q := q.(type) { + case query.UnivariateEval: + return query.NewUnivariateEval(name, q.Pols...) + case query.LocalOpening: + return query.NewLocalOpening(name, q.Pol) + case query.InnerProduct: + return query.NewInnerProduct(name, q.A, q.Bs...) + case query.GrandProduct: + return query.NewGrandProduct(q.Round, q.Inputs, name) + case query.LogDerivativeSum: + return query.NewLogDerivativeSum(q.Round, q.Inputs, name) + default: + panic("unknown query type") + } +} + +// TranslateColumnList translates a collection of pre-inserted columns +func (comp *compTranslator) TranslateColumnList(cols []ifaces.Column) []ifaces.Column { + var res []ifaces.Column + for _, col := range cols { + res = append(res, comp.GetColumn(col.GetColID())) + } + return res +} + +// TranslateColumnVecVec translates a collection of pre-inserted columns +func (comp *compTranslator) TranslateColumnVecVec(cols collection.VecVec[ifaces.ColID]) collection.VecVec[ifaces.ColID] { + var res = collection.NewVecVec[ifaces.ColID]() + for r, vec := range cols.Inner() { + for _, c := range vec { + res.AppendToInner(r, comp.GetColumn(c).GetColID()) + } + } + return res +} + +// TranslateColumnSet translates a set of pre-inserted columns +func (comp *compTranslator) TranslateColumnSet(cols map[ifaces.ColID]struct{}) map[ifaces.ColID]struct{} { + var res = make(map[ifaces.ColID]struct{}) + for col := range cols { + res[comp.GetColumn(col).GetColID()] = struct{}{} + } + return res +} From 2a8afe276c2c6a77308a3ae73858e65ffea5f279 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Thu, 16 Jan 2025 21:47:54 +0100 Subject: [PATCH 02/63] feat(runtime): hides the verifier and gnark verifier runtime under an interface --- prover/protocol/compiler/dummy/dummy.go | 4 +- .../compiler/fullrecursion/actions.go | 16 +++--- .../compiler/fullrecursion/circuit.go | 6 +-- .../protocol/compiler/globalcs/evaluation.go | 8 +-- .../compiler/grandproduct/compiler.go | 4 +- .../compiler/innerproduct/verifier.go | 4 +- .../compiler/logderivativesum/compile.go | 4 +- prover/protocol/compiler/lookup/verifier.go | 4 +- .../protocol/compiler/permutation/verifier.go | 4 +- .../compiler/selfrecursion/column_opening.go | 8 +-- .../compiler/selfrecursion/lincomb_phase.go | 4 +- prover/protocol/compiler/splitter/splitter.go | 16 +++--- .../compiler/splitter/sticker/sticker.go | 4 +- .../stitch_split/stitcher/constraints.go | 4 +- .../univariates/local_opening_point.go | 4 +- .../univariates/multi_to_single_point.go | 11 ++-- .../compiler/univariates/naturalize.go | 4 +- prover/protocol/compiler/vortex/compiler.go | 4 +- .../compiler/vortex/gnark_verifier.go | 10 ++-- prover/protocol/compiler/vortex/prover.go | 2 +- prover/protocol/compiler/vortex/verifier.go | 8 +-- prover/protocol/dedicated/functionals/fold.go | 4 +- .../dedicated/functionals/foldouter.go | 4 +- prover/protocol/dedicated/plonk/alignment.go | 4 +- prover/protocol/dedicated/plonk/compile.go | 4 +- .../dedicated/projection/projection.go | 4 +- .../dedicated/reedsolomon/reedsolomon.go | 4 +- .../protocol/dedicated/selector/subsample.go | 4 +- prover/protocol/wizard/actions.go | 12 ++--- prover/protocol/wizard/gnark_verifier.go | 50 ++++++++++++++++++- prover/protocol/wizard/verifier.go | 34 ++++++++++++- 31 files changed, 169 insertions(+), 88 deletions(-) diff --git a/prover/protocol/compiler/dummy/dummy.go b/prover/protocol/compiler/dummy/dummy.go index c0b3415e11b..e28f0e856fe 100644 --- a/prover/protocol/compiler/dummy/dummy.go +++ b/prover/protocol/compiler/dummy/dummy.go @@ -84,7 +84,7 @@ func Compile(comp *wizard.CompiledIOP) { One step to be run at the end, by verifying every constraint "a la mano" */ - verifier := func(run *wizard.VerifierRuntime) error { + verifier := func(run wizard.Runtime) error { logrus.Infof("started to run the dummy verifier") @@ -137,6 +137,6 @@ func Compile(comp *wizard.CompiledIOP) { } logrus.Debugf("NB: The gnark circuit does not check the verifier of the dummy reduction\n") - comp.InsertVerifier(numRounds-1, verifier, func(frontend.API, *wizard.WizardVerifierCircuit) {}) + comp.InsertVerifier(numRounds-1, verifier, func(frontend.API, wizard.GnarkRuntime) {}) } diff --git a/prover/protocol/compiler/fullrecursion/actions.go b/prover/protocol/compiler/fullrecursion/actions.go index 44927aa99a9..228287cd309 100644 --- a/prover/protocol/compiler/fullrecursion/actions.go +++ b/prover/protocol/compiler/fullrecursion/actions.go @@ -73,11 +73,11 @@ func (c LocalOpeningAssignment) Run(run *wizard.ProverRuntime) { } } -func (c *ConsistencyCheck) Run(run *wizard.VerifierRuntime) error { +func (c *ConsistencyCheck) Run(run wizard.Runtime) error { var ( initialFsCirc = run.GetLocalPointEvalParams(c.LocalOpenings[0].ID).Y - initialFsRt = run.FiatShamirHistory[c.FirstRound+1][0][0] + initialFsRt = run.FsHistory()[c.FirstRound+1][0][0] piCursor = 2 ) @@ -131,11 +131,11 @@ func (c *ConsistencyCheck) Run(run *wizard.VerifierRuntime) error { return nil } -func (c *ConsistencyCheck) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (c *ConsistencyCheck) RunGnark(api frontend.API, run wizard.GnarkRuntime) { var ( initialFsCirc = run.GetLocalPointEvalParams(c.LocalOpenings[0].ID).Y - initialFsRt = run.FiatShamirHistory[c.FirstRound+1][0][0] + initialFsRt = run.FsHistory()[c.FirstRound+1][0][0] piCursor = 2 ) @@ -187,15 +187,15 @@ func (c *ConsistencyCheck) IsSkipped() bool { return c.isSkipped } -func (r *ResetFsActions) Run(run *wizard.VerifierRuntime) error { +func (r *ResetFsActions) Run(run wizard.Runtime) error { finalFsCirc := run.GetLocalPointEvalParams(r.LocalOpenings[1].ID).Y - run.FS.SetState([]field.Element{finalFsCirc}) + run.Fs().SetState([]field.Element{finalFsCirc}) return nil } -func (r *ResetFsActions) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (r *ResetFsActions) RunGnark(api frontend.API, run wizard.GnarkRuntime) { finalFsCirc := run.GetLocalPointEvalParams(r.LocalOpenings[1].ID).Y - run.FS.SetState([]frontend.Variable{finalFsCirc}) + run.Fs().SetState([]frontend.Variable{finalFsCirc}) } func (r *ResetFsActions) Skip() { diff --git a/prover/protocol/compiler/fullrecursion/circuit.go b/prover/protocol/compiler/fullrecursion/circuit.go index 2151dfa7f33..6524e8fd79c 100644 --- a/prover/protocol/compiler/fullrecursion/circuit.go +++ b/prover/protocol/compiler/fullrecursion/circuit.go @@ -20,7 +20,7 @@ type gnarkCircuit struct { X frontend.Variable `gnark:",public"` Ys []frontend.Variable `gnark:",public"` Pubs []frontend.Variable `gnark:",public"` - WizardVerifier *wizard.WizardVerifierCircuit + WizardVerifier wizard.GnarkRuntime comp *wizard.CompiledIOP `gnark:"-"` ctx *fullRecursionCtx `gnark:"-"` withoutGkr bool `gnark:"-"` @@ -66,7 +66,7 @@ func allocateGnarkCircuit(comp *wizard.CompiledIOP, ctx *fullRecursionCtx) *gnar func (c *gnarkCircuit) Define(api frontend.API) error { - w := c.WizardVerifier + w := c.WizardVerifier.(*wizard.WizardVerifierCircuit) if c.withoutGkr { w.FS = fiatshamir.NewGnarkFiatShamir(api, nil) @@ -116,7 +116,7 @@ func (c *gnarkCircuit) generateAllRandomCoins(api frontend.API) { var ( ctx = c.ctx - w = c.WizardVerifier + w = c.WizardVerifier.(*wizard.WizardVerifierCircuit) ) w.FS.SetState([]frontend.Variable{c.InitialFsState}) diff --git a/prover/protocol/compiler/globalcs/evaluation.go b/prover/protocol/compiler/globalcs/evaluation.go index b612b78ec30..2fe167f50d1 100644 --- a/prover/protocol/compiler/globalcs/evaluation.go +++ b/prover/protocol/compiler/globalcs/evaluation.go @@ -165,7 +165,7 @@ func (pa evaluationProver) Run(run *wizard.ProverRuntime) { } // Run evaluate the constraint and checks that -func (ctx *evaluationVerifier) Run(run *wizard.VerifierRuntime) error { +func (ctx *evaluationVerifier) Run(run wizard.Runtime) error { var ( // Will be assigned to "X", the random point at which we check the constraint. @@ -239,7 +239,7 @@ func (ctx *evaluationVerifier) Run(run *wizard.VerifierRuntime) error { } // Verifier step, evaluate the constraint and checks that -func (ctx *evaluationVerifier) RunGnark(api frontend.API, c *wizard.WizardVerifierCircuit) { +func (ctx *evaluationVerifier) RunGnark(api frontend.API, c wizard.GnarkRuntime) { // Will be assigned to "X", the random point at which we check the constraint. r := c.GetRandomCoinField(ctx.EvalCoin.Name) @@ -299,7 +299,7 @@ func (ctx *evaluationVerifier) RunGnark(api frontend.API, c *wizard.WizardVerifi // recombineQuotientSharesEvaluation returns the evaluations of the quotients // on point r -func (ctx evaluationVerifier) recombineQuotientSharesEvaluation(run *wizard.VerifierRuntime, r field.Element) ([]field.Element, error) { +func (ctx evaluationVerifier) recombineQuotientSharesEvaluation(run wizard.Runtime, r field.Element) ([]field.Element, error) { var ( // res stores the list of the recombined quotient evaluations for each @@ -386,7 +386,7 @@ func (ctx evaluationVerifier) recombineQuotientSharesEvaluation(run *wizard.Veri // recombineQuotientSharesEvaluation returns the evaluations of the quotients // on point r -func (ctx evaluationVerifier) recombineQuotientSharesEvaluationGnark(api frontend.API, run *wizard.WizardVerifierCircuit, r frontend.Variable) []frontend.Variable { +func (ctx evaluationVerifier) recombineQuotientSharesEvaluationGnark(api frontend.API, run wizard.GnarkRuntime, r frontend.Variable) []frontend.Variable { var ( // res stores the list of the recombined quotient evaluations for each diff --git a/prover/protocol/compiler/grandproduct/compiler.go b/prover/protocol/compiler/grandproduct/compiler.go index 9e017a6b14c..6219a85e800 100644 --- a/prover/protocol/compiler/grandproduct/compiler.go +++ b/prover/protocol/compiler/grandproduct/compiler.go @@ -99,7 +99,7 @@ type FinalProductCheck struct { } // Run implements the [wizard.VerifierAction] -func (f *FinalProductCheck) Run(run *wizard.VerifierRuntime) error { +func (f *FinalProductCheck) Run(run wizard.Runtime) error { // zProd stores the product of the ending values of the zs as queried // in the protocol via the local opening queries. @@ -120,7 +120,7 @@ func (f *FinalProductCheck) Run(run *wizard.VerifierRuntime) error { } // RunGnark implements the [wizard.VerifierAction] -func (f *FinalProductCheck) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (f *FinalProductCheck) RunGnark(api frontend.API, run wizard.GnarkRuntime) { claimedProd := run.GetGrandProductParams(f.GrandProductID).Prod // zProd stores the product of the ending values of the z columns diff --git a/prover/protocol/compiler/innerproduct/verifier.go b/prover/protocol/compiler/innerproduct/verifier.go index accc5acd0e9..cb0c21e06c3 100644 --- a/prover/protocol/compiler/innerproduct/verifier.go +++ b/prover/protocol/compiler/innerproduct/verifier.go @@ -24,7 +24,7 @@ type verifierForSize struct { } // Run implements [wizard.VerifierAction] -func (v *verifierForSize) Run(run *wizard.VerifierRuntime) error { +func (v *verifierForSize) Run(run wizard.Runtime) error { var ( // ys stores the list of all the inner-product openings @@ -59,7 +59,7 @@ func (v *verifierForSize) Run(run *wizard.VerifierRuntime) error { } // RunGnark implements the [wizard.VerifierAction] interface -func (v *verifierForSize) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (v *verifierForSize) RunGnark(api frontend.API, run wizard.GnarkRuntime) { var ( // ys stores the list of all the inner-product openings diff --git a/prover/protocol/compiler/logderivativesum/compile.go b/prover/protocol/compiler/logderivativesum/compile.go index bc506f4f254..a4819c55b5d 100644 --- a/prover/protocol/compiler/logderivativesum/compile.go +++ b/prover/protocol/compiler/logderivativesum/compile.go @@ -73,7 +73,7 @@ type FinalEvaluationCheck struct { } // Run implements the [wizard.VerifierAction] -func (f *FinalEvaluationCheck) Run(run *wizard.VerifierRuntime) error { +func (f *FinalEvaluationCheck) Run(run wizard.Runtime) error { // zSum stores the sum of the ending values of the zs as queried // in the protocol via the local opening queries. @@ -94,7 +94,7 @@ func (f *FinalEvaluationCheck) Run(run *wizard.VerifierRuntime) error { } // RunGnark implements the [wizard.VerifierAction] -func (f *FinalEvaluationCheck) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (f *FinalEvaluationCheck) RunGnark(api frontend.API, run wizard.GnarkRuntime) { claimedSum := run.GetLogDerivSumParams(f.LogDerivSumID).Sum // SigmaSKSum stores the sum of the ending values of the SigmaSs as queried diff --git a/prover/protocol/compiler/lookup/verifier.go b/prover/protocol/compiler/lookup/verifier.go index be3219d085d..042c6262cd9 100644 --- a/prover/protocol/compiler/lookup/verifier.go +++ b/prover/protocol/compiler/lookup/verifier.go @@ -25,7 +25,7 @@ type finalEvaluationCheck struct { } // Run implements the [wizard.VerifierAction] -func (f *finalEvaluationCheck) Run(run *wizard.VerifierRuntime) error { +func (f *finalEvaluationCheck) Run(run wizard.Runtime) error { // zSum stores the sum of the ending values of the zs as queried // in the protocol via the local opening queries. @@ -43,7 +43,7 @@ func (f *finalEvaluationCheck) Run(run *wizard.VerifierRuntime) error { } // RunGnark implements the [wizard.VerifierAction] -func (f *finalEvaluationCheck) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (f *finalEvaluationCheck) RunGnark(api frontend.API, run wizard.GnarkRuntime) { // SigmaSKSum stores the sum of the ending values of the SigmaSs as queried // in the protocol via the diff --git a/prover/protocol/compiler/permutation/verifier.go b/prover/protocol/compiler/permutation/verifier.go index d076e643ba1..5b685d304d0 100644 --- a/prover/protocol/compiler/permutation/verifier.go +++ b/prover/protocol/compiler/permutation/verifier.go @@ -18,7 +18,7 @@ type VerifierCtx struct { // Run implements the [wizard.VerifierAction] interface and checks that the // product of the products given by the ZCtx is equal to one. -func (v *VerifierCtx) Run(run *wizard.VerifierRuntime) error { +func (v *VerifierCtx) Run(run wizard.Runtime) error { mustBeOne := field.One() @@ -38,7 +38,7 @@ func (v *VerifierCtx) Run(run *wizard.VerifierRuntime) error { // Run implements the [wizard.VerifierAction] interface and is as // [VerifierCtx.Run] but in the context of a gnark circuit. -func (v *VerifierCtx) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (v *VerifierCtx) RunGnark(api frontend.API, run wizard.GnarkRuntime) { mustBeOne := frontend.Variable(1) diff --git a/prover/protocol/compiler/selfrecursion/column_opening.go b/prover/protocol/compiler/selfrecursion/column_opening.go index 1091010eec8..04d442e847d 100644 --- a/prover/protocol/compiler/selfrecursion/column_opening.go +++ b/prover/protocol/compiler/selfrecursion/column_opening.go @@ -435,7 +435,7 @@ func (ctx *SelfRecursionCtx) collapsingPhase() { ctx.comp.InsertVerifier( left.Round(), - func(run *wizard.VerifierRuntime) error { + func(run wizard.Runtime) error { if left.GetVal(run) != right.GetVal(run) { l, r := left.GetVal(run), right.GetVal(run) return fmt.Errorf("consistency between u_alpha and the preimage: "+ @@ -445,7 +445,7 @@ func (ctx *SelfRecursionCtx) collapsingPhase() { } return nil }, - func(api frontend.API, run *wizard.WizardVerifierCircuit) { + func(api frontend.API, run wizard.GnarkRuntime) { api.AssertIsEqual( left.GetFrontendVariable(api, run), right.GetFrontendVariable(api, run), @@ -621,7 +621,7 @@ func (ctx *SelfRecursionCtx) foldPhase() { // And the final check // check the folding of the polynomial is correct - ctx.comp.InsertVerifier(round, func(run *wizard.VerifierRuntime) error { + ctx.comp.InsertVerifier(round, func(run wizard.Runtime) error { // fetch the assignments to edual and dcollapse edual := ctx.Columns.Edual.GetColAssignment(run) @@ -666,7 +666,7 @@ func (ctx *SelfRecursionCtx) foldPhase() { } return nil - }, func(api frontend.API, run *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, run wizard.GnarkRuntime) { // fetch the assignments to edual and dcollapse edual := ctx.Columns.Edual.GetColAssignmentGnark(run) diff --git a/prover/protocol/compiler/selfrecursion/lincomb_phase.go b/prover/protocol/compiler/selfrecursion/lincomb_phase.go index 7c9a6dbc0d8..50f48db0feb 100644 --- a/prover/protocol/compiler/selfrecursion/lincomb_phase.go +++ b/prover/protocol/compiler/selfrecursion/lincomb_phase.go @@ -70,7 +70,7 @@ func (ctx *SelfRecursionCtx) consistencyBetweenYsAndUalpha() { // And let the verifier check that they should be both equal ctx.comp.InsertVerifier( round, - func(run *wizard.VerifierRuntime) error { + func(run wizard.Runtime) error { ys := ctx.Columns.Ys.GetColAssignment(run) alpha := run.GetRandomCoinField(ctx.Coins.Alpha.Name) @@ -81,7 +81,7 @@ func (ctx *SelfRecursionCtx) consistencyBetweenYsAndUalpha() { } return nil }, - func(api frontend.API, run *wizard.WizardVerifierCircuit) { + func(api frontend.API, run wizard.GnarkRuntime) { ys := ctx.Columns.Ys.GetColAssignmentGnark(run) alpha := run.GetRandomCoinField(ctx.Coins.Alpha.Name) uAlphaX := ctx.Accessors.InterpolateUalphaX.GetFrontendVariable(api, run) diff --git a/prover/protocol/compiler/splitter/splitter.go b/prover/protocol/compiler/splitter/splitter.go index 062fb8ebc4c..c44de922d71 100644 --- a/prover/protocol/compiler/splitter/splitter.go +++ b/prover/protocol/compiler/splitter/splitter.go @@ -250,13 +250,13 @@ func (ctx splitterCtx) compileGlobal(comp *wizard.CompiledIOP, q query.GlobalCon } // Requires the verifier to verify the query itself - comp.InsertVerifier(round, func(vr *wizard.VerifierRuntime) error { + comp.InsertVerifier(round, func(vr wizard.Runtime) error { err := q.Check(vr) if err != nil { return fmt.Errorf("failure for query %v, here is why %v", q.ID, err) } return nil - }, func(api frontend.API, wvc *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, wvc wizard.GnarkRuntime) { q.CheckGnark(api, wvc) }) @@ -416,13 +416,13 @@ func (ctx splitterCtx) compileLocal(comp *wizard.CompiledIOP, q query.LocalConst } // Requires the verifier to verify the query itself - comp.InsertVerifier(round, func(vr *wizard.VerifierRuntime) error { + comp.InsertVerifier(round, func(vr wizard.Runtime) error { err := q.Check(vr) if err != nil { return fmt.Errorf("failure for query %v, here is why %v", q.ID, err) } return nil - }, func(api frontend.API, wvc *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, wvc wizard.GnarkRuntime) { q.CheckGnark(api, wvc) }) @@ -488,9 +488,9 @@ func (ctx splitterCtx) compileLocalOpening(comp *wizard.CompiledIOP, q query.Loc verifiercol.AssertIsPublicCol(comp, q.Pol) // Requires the verifier to verify the query itself - comp.InsertVerifier(round, func(vr *wizard.VerifierRuntime) error { + comp.InsertVerifier(round, func(vr wizard.Runtime) error { return q.Check(vr) - }, func(api frontend.API, wvc *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, wvc wizard.GnarkRuntime) { q.CheckGnark(api, wvc) }) @@ -522,7 +522,7 @@ func (ctx splitterCtx) compileLocalOpening(comp *wizard.CompiledIOP, q query.Loc }) // The verifier ensures that the old and new queries have the same assignement - comp.InsertVerifier(round, func(run *wizard.VerifierRuntime) error { + comp.InsertVerifier(round, func(run wizard.Runtime) error { oldParams := run.GetLocalPointEvalParams(q.ID) newParams := run.GetLocalPointEvalParams(newQName) @@ -531,7 +531,7 @@ func (ctx splitterCtx) compileLocalOpening(comp *wizard.CompiledIOP, q query.Loc } return nil - }, func(api frontend.API, run *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, run wizard.GnarkRuntime) { oldParams := run.GetLocalPointEvalParams(q.ID) newParams := run.GetLocalPointEvalParams(newQName) api.AssertIsEqual(oldParams.Y, newParams.Y) diff --git a/prover/protocol/compiler/splitter/sticker/sticker.go b/prover/protocol/compiler/splitter/sticker/sticker.go index e90ec7ff48d..b03ccaf2075 100644 --- a/prover/protocol/compiler/splitter/sticker/sticker.go +++ b/prover/protocol/compiler/splitter/sticker/sticker.go @@ -438,7 +438,7 @@ func (ctx *stickContext) compileFixedEvaluation() { }) // The verifier ensures that the old and new queries have the same assignement - ctx.comp.InsertVerifier(round, func(run *wizard.VerifierRuntime) error { + ctx.comp.InsertVerifier(round, func(run wizard.Runtime) error { oldParams := run.GetLocalPointEvalParams(q.ID) newParams := run.GetLocalPointEvalParams(queryName(q.ID)) @@ -447,7 +447,7 @@ func (ctx *stickContext) compileFixedEvaluation() { } return nil - }, func(api frontend.API, run *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, run wizard.GnarkRuntime) { oldParams := run.GetLocalPointEvalParams(q.ID) newParams := run.GetLocalPointEvalParams(queryName(q.ID)) api.AssertIsEqual(oldParams.Y, newParams.Y) diff --git a/prover/protocol/compiler/stitch_split/stitcher/constraints.go b/prover/protocol/compiler/stitch_split/stitcher/constraints.go index 7d269a2e7e4..25d778a3954 100644 --- a/prover/protocol/compiler/stitch_split/stitcher/constraints.go +++ b/prover/protocol/compiler/stitch_split/stitcher/constraints.go @@ -248,9 +248,9 @@ func insertVerifier( ) { // Requires the verifier to verify the query itself - comp.InsertVerifier(round, func(vr *wizard.VerifierRuntime) error { + comp.InsertVerifier(round, func(vr wizard.Runtime) error { return q.Check(vr) - }, func(api frontend.API, wvc *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, wvc wizard.GnarkRuntime) { q.CheckGnark(api, wvc) }) diff --git a/prover/protocol/compiler/univariates/local_opening_point.go b/prover/protocol/compiler/univariates/local_opening_point.go index e3896cdd0c3..c055f62a697 100644 --- a/prover/protocol/compiler/univariates/local_opening_point.go +++ b/prover/protocol/compiler/univariates/local_opening_point.go @@ -96,7 +96,7 @@ func (ctx *localOpeningCtx) prover(assi *wizard.ProverRuntime) { assi.AssignUnivariate(ctx.fixedToVariable(), field.One(), ys...) } -func (ctx localOpeningCtx) verifier(assi *wizard.VerifierRuntime) error { +func (ctx localOpeningCtx) verifier(assi wizard.Runtime) error { ys := []field.Element{} // Collect the evaluation from the assigned compiled queries @@ -131,7 +131,7 @@ func (ctx localOpeningCtx) verifier(assi *wizard.VerifierRuntime) error { return nil } -func (ctx localOpeningCtx) gnarkVerifier(api frontend.API, c *wizard.WizardVerifierCircuit) { +func (ctx localOpeningCtx) gnarkVerifier(api frontend.API, c wizard.GnarkRuntime) { ys := []frontend.Variable{} // Collect the evaluation from the assigned compiled queries diff --git a/prover/protocol/compiler/univariates/multi_to_single_point.go b/prover/protocol/compiler/univariates/multi_to_single_point.go index 27020fbe82b..ea82e4fff85 100644 --- a/prover/protocol/compiler/univariates/multi_to_single_point.go +++ b/prover/protocol/compiler/univariates/multi_to_single_point.go @@ -2,12 +2,13 @@ package univariates import ( "fmt" - ppool "github.com/consensys/linea-monorepo/prover/utils/parallel/pool" "math/big" "reflect" "runtime" "sync" + ppool "github.com/consensys/linea-monorepo/prover/utils/parallel/pool" + "github.com/consensys/gnark/frontend" "github.com/sirupsen/logrus" @@ -396,12 +397,12 @@ func (ctx mptsCtx) claimEvaluation(run *wizard.ProverRuntime) { } // verifier of the evaluation -func (ctx mptsCtx) verifier(run *wizard.VerifierRuntime) error { +func (ctx mptsCtx) verifier(run wizard.Runtime) error { ys, hs := ctx.getYsHs( run.GetUnivariateParams, func(qName ifaces.QueryID) query.UnivariateEval { - return run.Spec.QueriesParams.Data(qName).(query.UnivariateEval) + return run.GetSpec().QueriesParams.Data(qName).(query.UnivariateEval) }, ) @@ -513,7 +514,7 @@ func (ctx mptsCtx) verifier(run *wizard.VerifierRuntime) error { Gnark function generating constraints to mirror the verification of the evaluation step. */ -func (ctx mptsCtx) gnarkVerify(api frontend.API, c *wizard.WizardVerifierCircuit) { +func (ctx mptsCtx) gnarkVerify(api frontend.API, c wizard.GnarkRuntime) { logrus.Infof("Start verifying MPTS reduction") @@ -696,7 +697,7 @@ func getLagrangesPolys(domain []field.Element) (lagranges [][]field.Element) { // Mirrrors `getYsHs` to build a gnark circuit func (ctx mptsCtx) getYsHsGnark( - c *wizard.WizardVerifierCircuit, + c wizard.GnarkRuntime, ) ( ys map[ifaces.ColID][]frontend.Variable, hs []frontend.Variable, diff --git a/prover/protocol/compiler/univariates/naturalize.go b/prover/protocol/compiler/univariates/naturalize.go index 658591a8ee3..abc6e06e375 100644 --- a/prover/protocol/compiler/univariates/naturalize.go +++ b/prover/protocol/compiler/univariates/naturalize.go @@ -275,7 +275,7 @@ func (ctx *naturalizationCtx) prove(run *wizard.ProverRuntime) { } } -func (ctx naturalizationCtx) Verify(run *wizard.VerifierRuntime) error { +func (ctx naturalizationCtx) Verify(run wizard.Runtime) error { // Get the original query originalQuery := run.GetUnivariateEval(ctx.q.QueryID) @@ -346,7 +346,7 @@ func (ctx naturalizationCtx) Verify(run *wizard.VerifierRuntime) error { } -func (ctx naturalizationCtx) GnarkVerify(api frontend.API, c *wizard.WizardVerifierCircuit) { +func (ctx naturalizationCtx) GnarkVerify(api frontend.API, c wizard.GnarkRuntime) { logrus.Tracef("verifying naturalization") diff --git a/prover/protocol/compiler/vortex/compiler.go b/prover/protocol/compiler/vortex/compiler.go index 5eadb65a98e..7e45a2e617b 100644 --- a/prover/protocol/compiler/vortex/compiler.go +++ b/prover/protocol/compiler/vortex/compiler.go @@ -140,7 +140,7 @@ type Ctx struct { // Committed matrix (rs encoded) of the precomputed columns CommittedMatrix vortex.EncodedMatrix // Tree in case of Merkle mode - Tree *smt.Tree + tree *smt.Tree // colHashes used in self recursion DhWithMerkle []field.Element } @@ -710,7 +710,7 @@ func (ctx *Ctx) commitPrecomputeds() { committedMatrix, tree, colHashes := ctx.VortexParams.CommitMerkle(pols) ctx.Items.Precomputeds.DhWithMerkle = colHashes ctx.Items.Precomputeds.CommittedMatrix = committedMatrix - ctx.Items.Precomputeds.Tree = tree + ctx.Items.Precomputeds.tree = tree // And assign the 1-sized column to contain the root var root field.Element diff --git a/prover/protocol/compiler/vortex/gnark_verifier.go b/prover/protocol/compiler/vortex/gnark_verifier.go index 430a1560408..3fd7795ae8d 100644 --- a/prover/protocol/compiler/vortex/gnark_verifier.go +++ b/prover/protocol/compiler/vortex/gnark_verifier.go @@ -14,7 +14,7 @@ import ( "github.com/consensys/linea-monorepo/prover/utils" ) -func (ctx *Ctx) GnarkVerify(api frontend.API, vr *wizard.WizardVerifierCircuit) { +func (ctx *Ctx) GnarkVerify(api frontend.API, vr wizard.GnarkRuntime) { // The skip verification flag may be on, if the current vortex // context get self-recursed. In this case, the verifier does @@ -61,7 +61,7 @@ func (ctx *Ctx) GnarkVerify(api frontend.API, vr *wizard.WizardVerifierCircuit) // function that will defer the hashing to gkr factoryHasherFunc := func(_ frontend.API) (hash.FieldHasher, error) { - h := vr.HasherFactory.NewHasher() + h := vr.GetHasherFactory().NewHasher() return h, nil } @@ -91,7 +91,7 @@ func (ctx *Ctx) GnarkVerify(api frontend.API, vr *wizard.WizardVerifierCircuit) } // returns the Ys as a vector -func (ctx *Ctx) gnarkGetYs(_ frontend.API, vr *wizard.WizardVerifierCircuit) (ys [][]frontend.Variable) { +func (ctx *Ctx) gnarkGetYs(_ frontend.API, vr wizard.GnarkRuntime) (ys [][]frontend.Variable) { query := ctx.Query params := vr.GetUnivariateParams(ctx.Query.QueryID) @@ -161,7 +161,7 @@ func (ctx *Ctx) gnarkGetYs(_ frontend.API, vr *wizard.WizardVerifierCircuit) (ys // Returns the opened columns from the messages. The returned columns are // split "by-commitment-round". -func (ctx *Ctx) GnarkRecoverSelectedColumns(api frontend.API, vr *wizard.WizardVerifierCircuit) [][][]frontend.Variable { +func (ctx *Ctx) GnarkRecoverSelectedColumns(api frontend.API, vr wizard.GnarkRuntime) [][][]frontend.Variable { // Collect the columns : first extract the full columns // Bear in mind that the prover messages are zero-padded @@ -213,7 +213,7 @@ func (ctx *Ctx) GnarkRecoverSelectedColumns(api frontend.API, vr *wizard.WizardV } // Evaluates explicitly the public polynomials (proof, vk, public inputs) -func (ctx *Ctx) gnarkExplicitPublicEvaluation(api frontend.API, vr *wizard.WizardVerifierCircuit) { +func (ctx *Ctx) gnarkExplicitPublicEvaluation(api frontend.API, vr wizard.GnarkRuntime) { params := vr.GetUnivariateParams(ctx.Query.QueryID) diff --git a/prover/protocol/compiler/vortex/prover.go b/prover/protocol/compiler/vortex/prover.go index c2e8d6d68d6..bad0c99a18d 100644 --- a/prover/protocol/compiler/vortex/prover.go +++ b/prover/protocol/compiler/vortex/prover.go @@ -85,7 +85,7 @@ func (ctx *Ctx) OpenSelectedColumns(pr *wizard.ProverRuntime) { // Append the precomputed committedMatrices and trees when IsCommitToPrecomputed is true if ctx.IsCommitToPrecomputed() { committedMatrices = append(committedMatrices, ctx.Items.Precomputeds.CommittedMatrix) - trees = append(trees, ctx.Items.Precomputeds.Tree) + trees = append(trees, ctx.Items.Precomputeds.tree) } for round := 0; round <= ctx.MaxCommittedRound; round++ { diff --git a/prover/protocol/compiler/vortex/verifier.go b/prover/protocol/compiler/vortex/verifier.go index 1b27c1779b6..e27e7d48f75 100644 --- a/prover/protocol/compiler/vortex/verifier.go +++ b/prover/protocol/compiler/vortex/verifier.go @@ -13,7 +13,7 @@ import ( "github.com/consensys/linea-monorepo/prover/utils/types" ) -func (ctx *Ctx) Verify(vr *wizard.VerifierRuntime) error { +func (ctx *Ctx) Verify(vr wizard.Runtime) error { // The skip verification flag may be on, if the current vortex // context get self-recursed. In this case, the verifier does @@ -77,7 +77,7 @@ func (ctx *Ctx) getNbCommittedRows(round int) int { } // returns the Ys as a vector -func (ctx *Ctx) getYs(vr *wizard.VerifierRuntime) (ys [][]field.Element) { +func (ctx *Ctx) getYs(vr wizard.Runtime) (ys [][]field.Element) { query := ctx.Query params := vr.GetUnivariateParams(ctx.Query.QueryID) @@ -130,7 +130,7 @@ func (ctx *Ctx) getYs(vr *wizard.VerifierRuntime) (ys [][]field.Element) { // Returns the opened columns from the messages. The returned columns are // split "by-commitment-round". -func (ctx *Ctx) RecoverSelectedColumns(vr *wizard.VerifierRuntime, entryList []int) [][][]field.Element { +func (ctx *Ctx) RecoverSelectedColumns(vr wizard.Runtime, entryList []int) [][][]field.Element { // Collect the columns : first extract the full columns // Bear in mind that the prover messages are zero-padded @@ -184,7 +184,7 @@ func (ctx *Ctx) RecoverSelectedColumns(vr *wizard.VerifierRuntime, entryList []i } // Evaluates explicitly the public polynomials (proof, vk, public inputs) -func (ctx *Ctx) explicitPublicEvaluation(vr *wizard.VerifierRuntime) error { +func (ctx *Ctx) explicitPublicEvaluation(vr wizard.Runtime) error { params := vr.GetUnivariateParams(ctx.Query.QueryID) diff --git a/prover/protocol/dedicated/functionals/fold.go b/prover/protocol/dedicated/functionals/fold.go index a78df5436bd..a7b572a4f44 100644 --- a/prover/protocol/dedicated/functionals/fold.go +++ b/prover/protocol/dedicated/functionals/fold.go @@ -50,12 +50,12 @@ func Fold(comp *wizard.CompiledIOP, h ifaces.Column, x ifaces.Accessor, innerDeg verRound := utils.Max(outerCoinAcc.Round(), foldedEvalAcc.Round()) // Check that the two evaluations yield the same result - comp.InsertVerifier(verRound, func(a *wizard.VerifierRuntime) error { + comp.InsertVerifier(verRound, func(a wizard.Runtime) error { if foldedEvalAcc.GetVal(a) != hEvalAcc.GetVal(a) { return fmt.Errorf("verifier of folding failed %v", foldedName) } return nil - }, func(api frontend.API, wvc *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, wvc wizard.GnarkRuntime) { c := foldedEvalAcc.GetFrontendVariable(api, wvc) c_ := hEvalAcc.GetFrontendVariable(api, wvc) api.AssertIsEqual(c, c_) diff --git a/prover/protocol/dedicated/functionals/foldouter.go b/prover/protocol/dedicated/functionals/foldouter.go index 98340869ebc..c4fcaa53314 100644 --- a/prover/protocol/dedicated/functionals/foldouter.go +++ b/prover/protocol/dedicated/functionals/foldouter.go @@ -53,12 +53,12 @@ func FoldOuter(comp *wizard.CompiledIOP, h ifaces.Column, x ifaces.Accessor, out verRound := utils.Max(innerCoinAcc.Round(), foldedEvalAcc.Round()) // Check that the two evaluations yield the same result - comp.InsertVerifier(verRound, func(run *wizard.VerifierRuntime) error { + comp.InsertVerifier(verRound, func(run wizard.Runtime) error { if foldedEvalAcc.GetVal(run) != hEvalAcc.GetVal(run) { return fmt.Errorf("verifier of folding failed %v", foldedName) } return nil - }, func(api frontend.API, run *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, run wizard.GnarkRuntime) { c := foldedEvalAcc.GetFrontendVariable(api, run) c_ := hEvalAcc.GetFrontendVariable(api, run) api.AssertIsEqual(c, c_) diff --git a/prover/protocol/dedicated/plonk/alignment.go b/prover/protocol/dedicated/plonk/alignment.go index 59ef0deae8e..68298ac58a9 100644 --- a/prover/protocol/dedicated/plonk/alignment.go +++ b/prover/protocol/dedicated/plonk/alignment.go @@ -391,7 +391,7 @@ type checkActivatorAndMask struct { skipped bool } -func (c *checkActivatorAndMask) Run(run *wizard.VerifierRuntime) error { +func (c *checkActivatorAndMask) Run(run wizard.Runtime) error { for i := range c.circMaskOpenings { var ( localOpening = run.GetLocalPointEvalParams(c.circMaskOpenings[i].ID) @@ -410,7 +410,7 @@ func (c *checkActivatorAndMask) Run(run *wizard.VerifierRuntime) error { return nil } -func (c *checkActivatorAndMask) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (c *checkActivatorAndMask) RunGnark(api frontend.API, run wizard.GnarkRuntime) { for i := range c.circMaskOpenings { var ( valOpened = run.GetLocalPointEvalParams(c.circMaskOpenings[i].ID).Y diff --git a/prover/protocol/dedicated/plonk/compile.go b/prover/protocol/dedicated/plonk/compile.go index 3e9182fbe64..9b429781a6f 100644 --- a/prover/protocol/dedicated/plonk/compile.go +++ b/prover/protocol/dedicated/plonk/compile.go @@ -306,7 +306,7 @@ type checkingActivators struct { var _ wizard.VerifierAction = &checkingActivators{} -func (ca *checkingActivators) Run(run *wizard.VerifierRuntime) error { +func (ca *checkingActivators) Run(run wizard.Runtime) error { for i := range ca.Cols { curr := ca.Cols[i].GetColAssignmentAt(run, 0) @@ -325,7 +325,7 @@ func (ca *checkingActivators) Run(run *wizard.VerifierRuntime) error { return nil } -func (ca *checkingActivators) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (ca *checkingActivators) RunGnark(api frontend.API, run wizard.GnarkRuntime) { for i := range ca.Cols { curr := ca.Cols[i].GetColAssignmentGnarkAt(run, 0) diff --git a/prover/protocol/dedicated/projection/projection.go b/prover/protocol/dedicated/projection/projection.go index 4edd80fa377..71651627fb1 100644 --- a/prover/protocol/dedicated/projection/projection.go +++ b/prover/protocol/dedicated/projection/projection.go @@ -315,7 +315,7 @@ func (pa projectionProverAction) Run(run *wizard.ProverRuntime) { } // Run implements the [wizard.VerifierAction] interface. -func (va *projectionVerifierAction) Run(run *wizard.VerifierRuntime) error { +func (va *projectionVerifierAction) Run(run wizard.Runtime) error { var ( a = run.GetLocalPointEvalParams(va.HornerA0.ID).Y @@ -330,7 +330,7 @@ func (va *projectionVerifierAction) Run(run *wizard.VerifierRuntime) error { } // RunGnark implements the [wizard.VerifierAction] interface. -func (va *projectionVerifierAction) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (va *projectionVerifierAction) RunGnark(api frontend.API, run wizard.GnarkRuntime) { var ( a = run.GetLocalPointEvalParams(va.HornerA0.ID).Y diff --git a/prover/protocol/dedicated/reedsolomon/reedsolomon.go b/prover/protocol/dedicated/reedsolomon/reedsolomon.go index 02f656305f5..e6753c655ae 100644 --- a/prover/protocol/dedicated/reedsolomon/reedsolomon.go +++ b/prover/protocol/dedicated/reedsolomon/reedsolomon.go @@ -60,14 +60,14 @@ func CheckReedSolomon(comp *wizard.CompiledIOP, rate int, h ifaces.Column) { h, ) - comp.InsertVerifier(round+1, func(a *wizard.VerifierRuntime) error { + comp.InsertVerifier(round+1, func(a wizard.Runtime) error { y := coeffCheck.GetVal(a) y_ := evalCheck.GetVal(a) if y != y_ { return fmt.Errorf("reed-solomon check failed - %v is not a codeword", h.GetColID()) } return nil - }, func(api frontend.API, wvc *wizard.WizardVerifierCircuit) { + }, func(api frontend.API, wvc wizard.GnarkRuntime) { y := coeffCheck.GetFrontendVariable(api, wvc) y_ := evalCheck.GetFrontendVariable(api, wvc) api.AssertIsEqual(y, y_) diff --git a/prover/protocol/dedicated/selector/subsample.go b/prover/protocol/dedicated/selector/subsample.go index 2b7dbecad31..1ac6f0bcb10 100644 --- a/prover/protocol/dedicated/selector/subsample.go +++ b/prover/protocol/dedicated/selector/subsample.go @@ -233,7 +233,7 @@ func CheckSubsample(comp *wizard.CompiledIOP, name string, large, small []ifaces comp.InsertVerifier( round+1, - func(run *wizard.VerifierRuntime) error { + func(run wizard.Runtime) error { resAccLast := run.GetLocalPointEvalParams(accLargeLast.ID) expectedResAccLast := run.GetLocalPointEvalParams(accSmallLast.ID) if resAccLast.Y != expectedResAccLast.Y { @@ -241,7 +241,7 @@ func CheckSubsample(comp *wizard.CompiledIOP, name string, large, small []ifaces } return nil }, - func(a frontend.API, run *wizard.WizardVerifierCircuit) { + func(a frontend.API, run wizard.GnarkRuntime) { resAccLast := run.GetLocalPointEvalParams(accLargeLast.ID) expectedResAccLast := run.GetLocalPointEvalParams(accSmallLast.ID) a.AssertIsEqual(resAccLast.Y, expectedResAccLast.Y) diff --git a/prover/protocol/wizard/actions.go b/prover/protocol/wizard/actions.go index 398017fba90..bf59a2e0c80 100644 --- a/prover/protocol/wizard/actions.go +++ b/prover/protocol/wizard/actions.go @@ -20,24 +20,24 @@ type VerifierAction interface { IsSkipped() bool // Run executes the VerifierAction over a [VerifierRuntime] it returns an // error. - Run(*VerifierRuntime) error + Run(Runtime) error // RunGnark is as Run but in a gnark circuit. Instead, of the returning an // error the function enforces the passing of the verifier's checks. - RunGnark(frontend.API, *WizardVerifierCircuit) + RunGnark(frontend.API, GnarkRuntime) } // genVerifierAction represents a verifier action represented by closures type genVerifierAction struct { skipped bool - run func(*VerifierRuntime) error - runGnark func(frontend.API, *WizardVerifierCircuit) + run func(Runtime) error + runGnark func(frontend.API, GnarkRuntime) } -func (gva *genVerifierAction) Run(run *VerifierRuntime) error { +func (gva *genVerifierAction) Run(run Runtime) error { return gva.run(run) } -func (gva *genVerifierAction) RunGnark(api frontend.API, run *WizardVerifierCircuit) { +func (gva *genVerifierAction) RunGnark(api frontend.API, run GnarkRuntime) { gva.runGnark(api, run) } diff --git a/prover/protocol/wizard/gnark_verifier.go b/prover/protocol/wizard/gnark_verifier.go index a07ce74cb58..fe5acc9c8c2 100644 --- a/prover/protocol/wizard/gnark_verifier.go +++ b/prover/protocol/wizard/gnark_verifier.go @@ -16,9 +16,26 @@ import ( "github.com/sirupsen/logrus" ) +// GnarkRuntime is the interface implemented by the struct [WizardVerifierCircuit] +// and is used to interact with the GnarkVerifierStep. +type GnarkRuntime interface { + ifaces.GnarkRuntime + GetSpec() *CompiledIOP + GetPublicInput(api frontend.API, name string) frontend.Variable + GetGrandProductParams(name ifaces.QueryID) query.GnarkGrandProductParams + GetLogDerivSumParams(name ifaces.QueryID) query.GnarkLogDerivSumParams + GetLocalPointEvalParams(name ifaces.QueryID) query.GnarkLocalOpeningParams + GetInnerProductParams(name ifaces.QueryID) query.GnarkInnerProductParams + GetUnivariateEval(name ifaces.QueryID) query.UnivariateEval + GetUnivariateParams(name ifaces.QueryID) query.GnarkUnivariateEvalParams + Fs() *fiatshamir.GnarkFiatShamir + FsHistory() [][2][]frontend.Variable + GetHasherFactory() *gkrmimc.HasherFactory +} + // GnarkVerifierStep functions that can be registered in the CompiledIOP by the successive // compilation steps. They correspond to "precompiled" verification steps. -type GnarkVerifierStep func(frontend.API, *WizardVerifierCircuit) +type GnarkVerifierStep func(frontend.API, GnarkRuntime) // WizardVerifierCircuit the [VerifierRuntime] in a gnark circuit. The complete // implementation follows this mirror logic. @@ -554,3 +571,34 @@ func (c *WizardVerifierCircuit) GetPublicInput(api frontend.API, name string) fr utils.Panic("could not find public input nb %v", name) return field.Element{} } + +// Fs returns the Fiat-Shamir state of the verifier circuit +func (c *WizardVerifierCircuit) Fs() *fiatshamir.GnarkFiatShamir { + return c.FS +} + +// FsHistory returns the Fiat-Shamir state history of the verifier circuit +func (c *WizardVerifierCircuit) FsHistory() [][2][]frontend.Variable { + return c.FiatShamirHistory +} + +// SetFs sets the Fiat-Shamir state of the verifier circuit +func (c *WizardVerifierCircuit) SetFs(fs *fiatshamir.GnarkFiatShamir) { + c.FS = fs +} + +// GetHasherFactory returns the hasher factory of the verifier circuit; nil +// if none is set. +func (c *WizardVerifierCircuit) GetHasherFactory() *gkrmimc.HasherFactory { + return c.HasherFactory +} + +// SetHasherFactory sets the hasher factory of the verifier circuit +func (c *WizardVerifierCircuit) SetHasherFactory(hf *gkrmimc.HasherFactory) { + c.HasherFactory = hf +} + +// GetSpec returns the compiled IOP of the verifier circuit +func (c *WizardVerifierCircuit) GetSpec() *CompiledIOP { + return c.Spec +} diff --git a/prover/protocol/wizard/verifier.go b/prover/protocol/wizard/verifier.go index 934565719cc..b121e01d40c 100644 --- a/prover/protocol/wizard/verifier.go +++ b/prover/protocol/wizard/verifier.go @@ -33,11 +33,28 @@ type Proof struct { QueriesParams collection.Mapping[ifaces.QueryID, ifaces.QueryParams] } +// Runtime is a generic interface extending the [ifaces.Runtime] interface +// with all methods of [wizard.VerifierRuntime]. This is used to allow the +// writing of adapters for the verifier runtime. +type Runtime interface { + ifaces.Runtime + GetSpec() *CompiledIOP + GetPublicInput(name string) field.Element + GetGrandProductParams(name ifaces.QueryID) query.GrandProductParams + GetLogDerivSumParams(name ifaces.QueryID) query.LogDerivSumParams + GetLocalPointEvalParams(name ifaces.QueryID) query.LocalOpeningParams + GetInnerProductParams(name ifaces.QueryID) query.InnerProductParams + GetUnivariateEval(name ifaces.QueryID) query.UnivariateEval + GetUnivariateParams(name ifaces.QueryID) query.UnivariateEvalParams + Fs() *fiatshamir.State + FsHistory() [][2][]field.Element +} + // VerifierStep specifies a single step of verifier for a single subprotocol. // This can be used to specify verifier checks involving user-provided // columns for relations that cannot be automatically enforced via a // [ifaces.Query] -type VerifierStep func(a *VerifierRuntime) error +type VerifierStep func(a Runtime) error // VerifierRuntime runtime collects all data that visible or computed by the // verifier of the wizard protocol. This includes the prover's messages, the @@ -401,3 +418,18 @@ func (run *VerifierRuntime) GetPublicInput(name string) field.Element { utils.Panic("could not find public input nb %v", name) return field.Element{} } + +// Fs returns the Fiat-Shamir state +func (run *VerifierRuntime) Fs() *fiatshamir.State { + return run.FS +} + +// FsHistory returns the Fiat-Shamir state history +func (run *VerifierRuntime) FsHistory() [][2][]field.Element { + return run.FiatShamirHistory +} + +// GetSpec returns the compiled IOP +func (run *VerifierRuntime) GetSpec() *CompiledIOP { + return run.Spec +} From 67e6acd4e3e58fb479fc9d57fb2bb3012c4f6c85 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 00:51:37 +0100 Subject: [PATCH 03/63] feat(verifier): adds the require functionalities to the verifier runtime --- prover/protocol/wizard/gnark_verifier.go | 24 +++++++++++++++++++++++ prover/protocol/wizard/verifier.go | 25 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/prover/protocol/wizard/gnark_verifier.go b/prover/protocol/wizard/gnark_verifier.go index fe5acc9c8c2..eb861a4a703 100644 --- a/prover/protocol/wizard/gnark_verifier.go +++ b/prover/protocol/wizard/gnark_verifier.go @@ -31,6 +31,9 @@ type GnarkRuntime interface { Fs() *fiatshamir.GnarkFiatShamir FsHistory() [][2][]frontend.Variable GetHasherFactory() *gkrmimc.HasherFactory + InsertCoin(name coin.Name, value interface{}) + GetState(name string) (any, bool) + SetState(name string, value any) } // GnarkVerifierStep functions that can be registered in the CompiledIOP by the successive @@ -113,6 +116,10 @@ type WizardVerifierCircuit struct { // round. The first entry is the initial state, the final entry is the final // state. FiatShamirHistory [][2][]frontend.Variable `gnark:"-"` + + // State is a generic-purpose data store that the verifier steps can use to + // communicate with each other across rounds. + State map[string]interface{} `gnark:"-"` } // AllocateWizardCircuit allocates the inner-slices of the verifier struct from a precompiled IOP. It @@ -602,3 +609,20 @@ func (c *WizardVerifierCircuit) SetHasherFactory(hf *gkrmimc.HasherFactory) { func (c *WizardVerifierCircuit) GetSpec() *CompiledIOP { return c.Spec } + +// InsertCoin inserts a coin in the verifier circuit. This has +// a use for implementing recursive application. +func (c *WizardVerifierCircuit) InsertCoin(name coin.Name, value interface{}) { + c.Coins.InsertNew(name, value) +} + +// GetState returns the value of a state variable in the verifier circuit +func (c *WizardVerifierCircuit) GetState(name string) (any, bool) { + res, ok := c.State[name] + return res, ok +} + +// SetState sets the value of a state variable in the verifier circuit +func (c *WizardVerifierCircuit) SetState(name string, value any) { + c.State[name] = value +} diff --git a/prover/protocol/wizard/verifier.go b/prover/protocol/wizard/verifier.go index b121e01d40c..5f3271e751d 100644 --- a/prover/protocol/wizard/verifier.go +++ b/prover/protocol/wizard/verifier.go @@ -48,6 +48,9 @@ type Runtime interface { GetUnivariateParams(name ifaces.QueryID) query.UnivariateEvalParams Fs() *fiatshamir.State FsHistory() [][2][]field.Element + InsertCoin(name coin.Name, value any) + GetState(name string) (any, bool) + SetState(name string, value any) } // VerifierStep specifies a single step of verifier for a single subprotocol. @@ -90,6 +93,10 @@ type VerifierRuntime struct { // round. The first entry is the initial state, the final entry is the final // state. FiatShamirHistory [][2][]field.Element + + // State stores arbitrary data that can be used by the verifier. This + // can be used to communicate values between verifier states. + State map[string]interface{} } // Verify verifies a wizard proof. The caller specifies a [CompiledIOP] that @@ -433,3 +440,21 @@ func (run *VerifierRuntime) FsHistory() [][2][]field.Element { func (run *VerifierRuntime) GetSpec() *CompiledIOP { return run.Spec } + +// InsertCoin inserts a coin into the runtime. It should not be +// used by usual verifier action but is useful when implementing +// recursion utilities. +func (run *VerifierRuntime) InsertCoin(name coin.Name, value any) { + run.Coins.InsertNew(name, value) +} + +// GetState returns an arbitrary value stored in the runtime +func (run *VerifierRuntime) GetState(name string) (any, bool) { + res, ok := run.State[name] + return res, ok +} + +// SetState sets an arbitrary value in the runtime +func (run *VerifierRuntime) SetState(name string, value any) { + run.State[name] = value +} From 4daaed83cf36bf7719f781a2d0ad9e29685efcb0 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 00:52:34 +0100 Subject: [PATCH 04/63] feat(conglomeration): implements the pre-vortex verifier step --- .../conglomeration/conglomeration.go | 21 +- .../distributed/conglomeration/translator.go | 210 +++++++++++++++ .../distributed/conglomeration/verifier.go | 244 ++++++++++++++++++ 3 files changed, 461 insertions(+), 14 deletions(-) create mode 100644 prover/protocol/distributed/conglomeration/verifier.go diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index ce4ab61a549..f808aa238a1 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -1,6 +1,8 @@ package conglomeration import ( + "fmt" + "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" @@ -33,9 +35,13 @@ func Conglomerate( } func addVerifierToComp( + id int, comp *wizard.CompiledIOP, tmpl *wizard.CompiledIOP, ) { + ctx := initRecursionCtx(fmt.Sprintf("verifier-%v", id), comp) + ctx.captureCompPreVortex(tmpl) + ctx.captureVortexCtx(tmpl) } // initCtx initializes a new context @@ -175,7 +181,6 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { dstVortexCtx.Items.Precomputeds.MerkleRoot = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.MerkleRoot.GetColID()) dstVortexCtx.Items.Precomputeds.Dh = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.Dh.GetColID()) dstVortexCtx.Items.Precomputeds.CommittedMatrix = srcVortexCtx.Items.Precomputeds.CommittedMatrix - dstVortexCtx.Items.Precomputeds.Tree = srcVortexCtx.Items.Precomputeds.Tree dstVortexCtx.Items.Precomputeds.DhWithMerkle = srcVortexCtx.Items.Precomputeds.DhWithMerkle dstVortexCtx.Items.Dh = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Dh) @@ -185,18 +190,6 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { dstVortexCtx.Items.OpenedColumns = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.OpenedColumns) dstVortexCtx.Items.MerkleProofs = ctx.Translator.GetColumn(srcVortexCtx.Items.MerkleProofs.GetColID()) dstVortexCtx.Items.MerkleRoots = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.MerkleRoots) -} - -// TranslateUniEval returns a copied UnivariateEval query with the columns translated -// and the names translated. The returned query is registered in the translator comp. -func (comp *compTranslator) TranslateUniEval(round int, q query.UnivariateEval) query.UnivariateEval { - var ( - res = query.NewUnivariateEval(q.QueryID, q.Pols...) - ) - - for i := range res.Pols { - res.Pols[i] = comp.GetColumn(res.Pols[i].GetColID()) - } - return comp.InsertQueryParams(round, res).(query.UnivariateEval) + ctx.PcsCtx = dstVortexCtx } diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 6b33783b897..5ae00f0f94e 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -1,6 +1,10 @@ package conglomeration import ( + "github.com/consensys/gnark/frontend" + "github.com/consensys/linea-monorepo/prover/crypto/fiatshamir" + "github.com/consensys/linea-monorepo/prover/crypto/mimc/gkrmimc" + "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/column" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" @@ -9,6 +13,11 @@ import ( "github.com/consensys/linea-monorepo/prover/utils/collection" ) +var ( + _ wizard.Runtime = &runtimeTranslator{} + _ wizard.GnarkRuntime = &gnarkRuntimeTranslator{} +) + // compTranslator is a builder struct for building a target [wizard.CompiledIOP] // instances from another source [wizard.CompiledIOP]. All items in the built // compiled IOP are prefixed with an identifier. @@ -17,6 +26,19 @@ type compTranslator struct { Target *wizard.CompiledIOP } +// runtimeTranslator is an adapter structure prefixing every ColID and QueryID and +// coin.Name with a prefix string. +type runtimeTranslator struct { + Prefix string + Rt wizard.Runtime +} + +// gnarkRuntimeTranslator is as [runtimeTranslator] but for [wizard.GnarkRuntime] +type gnarkRuntimeTranslator struct { + Prefix string + Rt wizard.GnarkRuntime +} + // InsertColumn inserts a new column in the target compiled IOP. The column name // is prefixed with comp.Prefix. func (comp *compTranslator) InsertColumn(round int, name ifaces.ColID, size int, status column.Status) ifaces.Column { @@ -101,3 +123,191 @@ func (comp *compTranslator) TranslateColumnSet(cols map[ifaces.ColID]struct{}) m } return res } + +// TranslateUniEval returns a copied UnivariateEval query with the columns translated +// and the names translated. The returned query is registered in the translator comp. +func (comp *compTranslator) TranslateUniEval(round int, q query.UnivariateEval) query.UnivariateEval { + var res = query.NewUnivariateEval(q.QueryID, q.Pols...) + for i := range res.Pols { + res.Pols[i] = comp.GetColumn(res.Pols[i].GetColID()) + } + return comp.InsertQueryParams(round, res).(query.UnivariateEval) +} + +func (run *runtimeTranslator) GetColumn(name ifaces.ColID) ifaces.ColAssignment { + name = ifaces.ColID(run.Prefix) + "." + name + return run.Rt.GetColumn(name) +} + +func (run *runtimeTranslator) GetColumnAt(name ifaces.ColID, pos int) field.Element { + name = ifaces.ColID(run.Prefix) + "." + name + return run.Rt.GetColumnAt(name, pos) +} + +func (run *runtimeTranslator) GetRandomCoinField(name coin.Name) field.Element { + name = coin.Name(run.Prefix) + "." + name + return run.Rt.GetRandomCoinField(name) +} + +func (run *runtimeTranslator) GetRandomCoinIntegerVec(name coin.Name) []int { + name = coin.Name(run.Prefix) + "." + name + return run.Rt.GetRandomCoinIntegerVec(name) +} + +func (run *runtimeTranslator) GetParams(id ifaces.QueryID) ifaces.QueryParams { + id = ifaces.QueryID(run.Prefix) + "." + id + return run.Rt.GetParams(id) +} + +func (run *runtimeTranslator) GetSpec() *wizard.CompiledIOP { + return run.Rt.GetSpec() +} + +func (run *runtimeTranslator) GetPublicInput(name string) field.Element { + name = run.Prefix + "." + name + return run.Rt.GetPublicInput(name) +} + +func (run *runtimeTranslator) GetGrandProductParams(name ifaces.QueryID) query.GrandProductParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetGrandProductParams(name) +} + +func (run *runtimeTranslator) GetLogDerivSumParams(name ifaces.QueryID) query.LogDerivSumParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetLogDerivSumParams(name) +} + +func (run *runtimeTranslator) GetLocalPointEvalParams(name ifaces.QueryID) query.LocalOpeningParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetLocalPointEvalParams(name) +} + +func (run *runtimeTranslator) GetInnerProductParams(name ifaces.QueryID) query.InnerProductParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetInnerProductParams(name) +} + +func (run *runtimeTranslator) GetUnivariateEval(name ifaces.QueryID) query.UnivariateEval { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetUnivariateEval(name) +} + +func (run *runtimeTranslator) GetUnivariateParams(name ifaces.QueryID) query.UnivariateEvalParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetUnivariateParams(name) +} + +func (run *runtimeTranslator) Fs() *fiatshamir.State { + return run.Rt.Fs() +} + +func (run *runtimeTranslator) FsHistory() [][2][]field.Element { + return run.Rt.FsHistory() +} + +func (run *runtimeTranslator) InsertCoin(name coin.Name, value any) { + name = coin.Name(run.Prefix) + "." + name + run.Rt.InsertCoin(name, value) +} + +func (run *runtimeTranslator) GetState(name string) (any, bool) { + name = run.Prefix + "." + name + return run.Rt.GetState(name) +} + +func (run *runtimeTranslator) SetState(name string, value any) { + name = run.Prefix + "." + name + run.Rt.SetState(name, value) +} + +func (run *gnarkRuntimeTranslator) GetColumn(name ifaces.ColID) []frontend.Variable { + name = ifaces.ColID(run.Prefix) + "." + name + return run.Rt.GetColumn(name) +} + +func (run *gnarkRuntimeTranslator) GetColumnAt(name ifaces.ColID, at int) frontend.Variable { + name = ifaces.ColID(run.Prefix) + "." + name + return run.Rt.GetColumnAt(name, at) +} + +func (run *gnarkRuntimeTranslator) GetRandomCoinField(name coin.Name) frontend.Variable { + name = coin.Name(run.Prefix) + "." + name + return run.Rt.GetRandomCoinField(name) +} + +func (run *gnarkRuntimeTranslator) GetRandomCoinIntegerVec(name coin.Name) []frontend.Variable { + name = coin.Name(run.Prefix) + "." + name + return run.Rt.GetRandomCoinIntegerVec(name) +} + +func (run *gnarkRuntimeTranslator) GetParams(id ifaces.QueryID) ifaces.GnarkQueryParams { + id = ifaces.QueryID(run.Prefix) + "." + id + return run.Rt.GetParams(id) +} + +func (run *gnarkRuntimeTranslator) GetSpec() *wizard.CompiledIOP { + return run.Rt.GetSpec() +} + +func (run *gnarkRuntimeTranslator) GetPublicInput(api frontend.API, name string) frontend.Variable { + name = run.Prefix + "." + name + return run.Rt.GetPublicInput(api, name) +} + +func (run *gnarkRuntimeTranslator) GetGrandProductParams(name ifaces.QueryID) query.GnarkGrandProductParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetGrandProductParams(name) +} + +func (run *gnarkRuntimeTranslator) GetLogDerivSumParams(name ifaces.QueryID) query.GnarkLogDerivSumParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetLogDerivSumParams(name) +} + +func (run *gnarkRuntimeTranslator) GetLocalPointEvalParams(name ifaces.QueryID) query.GnarkLocalOpeningParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetLocalPointEvalParams(name) +} + +func (run *gnarkRuntimeTranslator) GetInnerProductParams(name ifaces.QueryID) query.GnarkInnerProductParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetInnerProductParams(name) +} + +func (run *gnarkRuntimeTranslator) GetUnivariateEval(name ifaces.QueryID) query.UnivariateEval { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetUnivariateEval(name) +} + +func (run *gnarkRuntimeTranslator) GetUnivariateParams(name ifaces.QueryID) query.GnarkUnivariateEvalParams { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetUnivariateParams(name) +} + +func (run *gnarkRuntimeTranslator) Fs() *fiatshamir.GnarkFiatShamir { + return run.Rt.Fs() +} + +func (run *gnarkRuntimeTranslator) FsHistory() [][2][]frontend.Variable { + return run.Rt.FsHistory() +} + +func (run *gnarkRuntimeTranslator) GetHasherFactory() *gkrmimc.HasherFactory { + return run.Rt.GetHasherFactory() +} + +func (run *gnarkRuntimeTranslator) InsertCoin(name coin.Name, value any) { + name = coin.Name(run.Prefix) + "." + name + run.Rt.InsertCoin(name, value) +} + +func (run *gnarkRuntimeTranslator) GetState(name string) (any, bool) { + name = run.Prefix + "." + name + return run.Rt.GetState(name) +} + +func (run *gnarkRuntimeTranslator) SetState(name string, value any) { + name = run.Prefix + "." + name + run.Rt.SetState(name, value) +} diff --git a/prover/protocol/distributed/conglomeration/verifier.go b/prover/protocol/distributed/conglomeration/verifier.go new file mode 100644 index 00000000000..9e7957b4791 --- /dev/null +++ b/prover/protocol/distributed/conglomeration/verifier.go @@ -0,0 +1,244 @@ +package conglomeration + +import ( + "errors" + + "github.com/consensys/gnark/frontend" + "github.com/consensys/linea-monorepo/prover/crypto/fiatshamir" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/coin" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" +) + +var _ wizard.Runtime = &RuntimeWithReplacedFS{} + +// PreVortexVerifierStep is a step replicating the verifier of the tmpl at round +// `Round` before the Vortex compilation step. +type PreVortexVerifierStep struct { + Ctx *recursionCtx + Round int +} + +// RuntimeWithReplacedFS is a runtime that wraps another runtime and replaces +// the returns of [Fs] and [FsHistory] with the ones provided in the struct. +type RuntimeWithReplacedFS struct { + wizard.Runtime + FS *fiatshamir.State + FiatShamirHistory [][2][]field.Element +} + +// GnarkRuntimeWithReplacedFS is a GnarkRuntime that wraps another runtime and +// replaces the returns of [Fs] and [FsHistory] with the ones provided in the struct. +type GnarkRuntimeWithReplacedFS struct { + wizard.GnarkRuntime + FS *fiatshamir.GnarkFiatShamir + FiatShamirHistory [][2][]frontend.Variable +} + +func (pa PreVortexVerifierStep) Run(run wizard.Runtime) error { + + pa.generateRandomCoins(run) + + // Wraps the runtime into a translation adapter + var ( + err error + wrappedRun = &runtimeTranslator{Prefix: pa.Ctx.Translator.Prefix, Rt: run} + ) + + // Copy the verifier actions of the template into the target + for _, va := range pa.Ctx.VerifierActions[pa.Round] { + err = errors.Join(err, va.Run(wrappedRun)) + } + + return err +} + +// generateRandomCoins generates all the coins for the current round +// so that they are made available to the forthcoming verifier actions. +func (pa PreVortexVerifierStep) generateRandomCoins(run wizard.Runtime) { + + const ( + fiatShamirHistoryStr = "fiat-shamir-history" + fiatShamirTranscriptStr = "fiat-shamir-transcript" + ) + + var ( + currRound = pa.Round + initialState = run.Fs().State() + ctx = pa.Ctx + spec = run.GetSpec() + // Wraps the runtime into a translation adapter, first to get the FS state + // and history. + wrappedRun wizard.Runtime = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + fsAny, _ = wrappedRun.GetState(fiatShamirTranscriptStr) + fs = fsAny.(*fiatshamir.State) + fsHistoryAny, _ = wrappedRun.GetState(fiatShamirHistoryStr) + fsHistory = fsHistoryAny.([][2][]field.Element) + ) + + if fs == nil { + fs = fiatshamir.NewMiMCFiatShamir() + } + + if fsHistory == nil { + fsHistory = make([][2][]field.Element, ctx.LastRound+1) + } + + // Wraps it a second time + wrappedRun = &RuntimeWithReplacedFS{ + Runtime: wrappedRun, + FS: fs, + FiatShamirHistory: fsHistory, + } + + if currRound > 0 { + + cols := ctx.Columns[currRound-1] + for _, col := range cols { + instance := run.GetColumn(col.GetColID()) + fs.UpdateSV(instance) + } + + queries := ctx.QueryParams[currRound-1] + for _, q := range queries { + params := run.GetParams(q.Name()) + params.UpdateFS(fs) + } + } + + toCompute := ctx.Coins[currRound] + for _, coin := range toCompute { + info := spec.Coins.Data(coin.Name) + value := info.Sample(fs) + run.InsertCoin(coin.Name, value) + } + + for _, fsHook := range ctx.FsHooks[currRound] { + fsHook.Run(wrappedRun) + } + + fsHistory[currRound] = [2][]field.Element{ + initialState, + fs.State(), + } + + run.SetState(fiatShamirHistoryStr, fsHistory) + run.SetState(fiatShamirTranscriptStr, fs) +} + +// Fs returns the Fiat-Shamir state +func (run *RuntimeWithReplacedFS) Fs() *fiatshamir.State { + return run.FS +} + +// FsHistory returns the Fiat-Shamir state history +func (run *RuntimeWithReplacedFS) FsHistory() [][2][]field.Element { + return run.FiatShamirHistory +} + +func (pa PreVortexVerifierStep) RunGnark(api frontend.API, run wizard.GnarkRuntime) error { + + pa.generateRandomCoinsGnark(api, run) + + // Wraps the runtime into a translation adapter + var ( + err error + wrappedRun = &gnarkRuntimeTranslator{Prefix: pa.Ctx.Translator.Prefix, Rt: run} + ) + + // Copy the verifier actions of the template into the target + for _, va := range pa.Ctx.VerifierActions[pa.Round] { + va.RunGnark(api, wrappedRun) + } + + return err +} + +// generateRandomCoinsGnark generates all the coins for the current round +// so that they are made available to the forthcoming verifier actions. +func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run wizard.GnarkRuntime) { + + const ( + fiatShamirHistoryStr = "fiat-shamir-history" + fiatShamirTranscriptStr = "fiat-shamir-transcript" + ) + + var ( + currRound = pa.Round + initialState = run.Fs().State() + ctx = pa.Ctx + spec = run.GetSpec() + // Wraps the runtime into a translation adapter, first to get the FS state + // and history. + wrappedRun wizard.GnarkRuntime = &gnarkRuntimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + fsAny, _ = wrappedRun.GetState(fiatShamirTranscriptStr) + fs = fsAny.(*fiatshamir.GnarkFiatShamir) + fsHistoryAny, _ = wrappedRun.GetState(fiatShamirHistoryStr) + fsHistory = fsHistoryAny.([][2][]frontend.Variable) + ) + + if fs == nil && run.GetHasherFactory() != nil { + fs = fiatshamir.NewGnarkFiatShamir(api, run.GetHasherFactory()) + } + + if fsHistory == nil { + fsHistory = make([][2][]frontend.Variable, ctx.LastRound+1) + } + + // Wraps it a second time + wrappedRun = &GnarkRuntimeWithReplacedFS{ + GnarkRuntime: wrappedRun, + FS: fs, + FiatShamirHistory: fsHistory, + } + + if currRound > 0 { + + cols := ctx.Columns[currRound-1] + for _, col := range cols { + instance := run.GetColumn(col.GetColID()) + fs.UpdateVec(instance) + } + + queries := ctx.QueryParams[currRound-1] + for _, q := range queries { + params := run.GetParams(q.Name()) + params.UpdateFS(fs) + } + } + + toCompute := ctx.Coins[currRound] + for _, c := range toCompute { + info := spec.Coins.Data(c.Name) + switch info.Type { + case coin.Field: + value := fs.RandomField() + run.InsertCoin(c.Name, value) + case coin.IntegerVec: + value := fs.RandomManyIntegers(info.Size, info.UpperBound) + run.InsertCoin(c.Name, value) + } + } + + for _, fsHook := range ctx.FsHooks[currRound] { + fsHook.RunGnark(api, wrappedRun) + } + + fsHistory[currRound] = [2][]frontend.Variable{ + initialState, + fs.State(), + } + + run.SetState(fiatShamirHistoryStr, fsHistory) + run.SetState(fiatShamirTranscriptStr, fs) +} + +// Fs returns the Fiat-Shamir state +func (run *GnarkRuntimeWithReplacedFS) Fs() *fiatshamir.GnarkFiatShamir { + return run.FS +} + +// FsHistory returns the Fiat-Shamir state history +func (run *GnarkRuntimeWithReplacedFS) FsHistory() [][2][]frontend.Variable { + return run.FiatShamirHistory +} From 2554acf935bdd6bf6ad12700a0564ee5feef0e1a Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 15:46:25 +0100 Subject: [PATCH 05/63] feat(conglomeration): implements the pre-vortex prover step --- .../protocol/distributed/conglomeration/conglomeration.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index f808aa238a1..aa129df4862 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -31,7 +31,13 @@ func Conglomerate( tmpl *wizard.CompiledIOP, maxNumSegment int, ) (comp *wizard.CompiledIOP) { - return nil + + comp = wizard.NewCompiledIOP() + + for id := 0; id < maxNumSegment; id++ { + addVerifierToComp(id, comp, tmpl) + } + } func addVerifierToComp( From 27a2dc617835888f1b4a15bb9e3ad7a20a506251 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 15:46:31 +0100 Subject: [PATCH 06/63] fixup --- .../distributed/conglomeration/prover.go | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 prover/protocol/distributed/conglomeration/prover.go diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go new file mode 100644 index 00000000000..ee4b844a949 --- /dev/null +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -0,0 +1,43 @@ +package conglomeration + +import ( + "strings" + + "github.com/consensys/linea-monorepo/prover/protocol/wizard" +) + +// PreVortexProverStep is a step replicating the prover of the tmpl at round +// `Round` before the Vortex compilation step. It works by adding columns from +// a wizard proof stored in the prover runtime. The proof is fetched from the +// runtime state. That means the prover step should be run after the proof has +// been attached to the runtime. +type PreVortexProverStep struct { + Ctx *recursionCtx + Round int +} + +func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { + + var ( + prefix = pa.Ctx.Translator.Prefix + proof = run.State.MustGet(prefix + ".subproof").(wizard.Proof) + queriesParams = pa.Ctx.QueryParams[pa.Round] + colums = pa.Ctx.Columns[pa.Round] + ) + + for _, col := range colums { + name := unprefix(prefix, col.GetColID()) + run.AssignColumn(col.GetColID(), proof.Messages.MustGet(name)) + } + + for _, param := range queriesParams { + name := unprefix(prefix, param.Name()) + run.QueriesParams.InsertNew(param.Name(), proof.QueriesParams.MustGet(name)) + } +} + +func unprefix[T ~string](prefix string, name T) T { + p, n := string(prefix), string(name) + r := strings.TrimPrefix(n, p) + return T(r) +} From d6f96252a9718e4e93129ccbe1f47485f04c6836 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 17:19:06 +0100 Subject: [PATCH 07/63] clean(vortex): remove the Dh field --- prover/protocol/compiler/vortex/compiler.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/prover/protocol/compiler/vortex/compiler.go b/prover/protocol/compiler/vortex/compiler.go index 7e45a2e617b..714ea6b74bb 100644 --- a/prover/protocol/compiler/vortex/compiler.go +++ b/prover/protocol/compiler/vortex/compiler.go @@ -135,8 +135,6 @@ type Ctx struct { PrecomputedColums []ifaces.Column // Merkle Root of the precomputeds columns MerkleRoot ifaces.Column - // List of the column hashes for the precomputed columns - Dh ifaces.Column // Committed matrix (rs encoded) of the precomputed columns CommittedMatrix vortex.EncodedMatrix // Tree in case of Merkle mode @@ -144,8 +142,6 @@ type Ctx struct { // colHashes used in self recursion DhWithMerkle []field.Element } - // (not used in the Merkle proof version) - Dh []ifaces.Column // Alpha is a random combination linear coin Alpha coin.Info // Linear combination of the row-encoded matrix @@ -185,12 +181,10 @@ func newCtx(comp *wizard.CompiledIOP, univQ query.UnivariateEval, blowUpFactor i Precomputeds struct { PrecomputedColums []ifaces.Column MerkleRoot ifaces.Column - Dh ifaces.Column CommittedMatrix vortex.EncodedMatrix tree *smt.Tree DhWithMerkle []field.Element } - Dh []ifaces.Column Alpha coin.Info Ualpha ifaces.Column Q coin.Info From ed3cc3ace5baf86cf65371ecda96b799efef24c2 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 17:36:10 +0100 Subject: [PATCH 08/63] feat(conglomeration): unifies the prover actions and verifier actions for every context --- .../distributed/conglomeration/prover.go | 35 ++++---- .../distributed/conglomeration/verifier.go | 82 ++++++++++--------- 2 files changed, 62 insertions(+), 55 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index ee4b844a949..9541c6489d4 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -12,27 +12,28 @@ import ( // runtime state. That means the prover step should be run after the proof has // been attached to the runtime. type PreVortexProverStep struct { - Ctx *recursionCtx + Ctxs []*recursionCtx Round int } func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { - - var ( - prefix = pa.Ctx.Translator.Prefix - proof = run.State.MustGet(prefix + ".subproof").(wizard.Proof) - queriesParams = pa.Ctx.QueryParams[pa.Round] - colums = pa.Ctx.Columns[pa.Round] - ) - - for _, col := range colums { - name := unprefix(prefix, col.GetColID()) - run.AssignColumn(col.GetColID(), proof.Messages.MustGet(name)) - } - - for _, param := range queriesParams { - name := unprefix(prefix, param.Name()) - run.QueriesParams.InsertNew(param.Name(), proof.QueriesParams.MustGet(name)) + for _, ctx := range pa.Ctxs { + var ( + prefix = ctx.Translator.Prefix + proof = run.State.MustGet(prefix + ".subproof").(wizard.Proof) + queriesParams = ctx.QueryParams[pa.Round] + colums = ctx.Columns[pa.Round] + ) + + for _, col := range colums { + name := unprefix(prefix, col.GetColID()) + run.AssignColumn(col.GetColID(), proof.Messages.MustGet(name)) + } + + for _, param := range queriesParams { + name := unprefix(prefix, param.Name()) + run.QueriesParams.InsertNew(param.Name(), proof.QueriesParams.MustGet(name)) + } } } diff --git a/prover/protocol/distributed/conglomeration/verifier.go b/prover/protocol/distributed/conglomeration/verifier.go index 9e7957b4791..44a474998c9 100644 --- a/prover/protocol/distributed/conglomeration/verifier.go +++ b/prover/protocol/distributed/conglomeration/verifier.go @@ -10,13 +10,19 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/wizard" ) +const ( + fiatShamirHistoryStr = "fiat-shamir-history" + fiatShamirTranscriptStr = "fiat-shamir-transcript" +) + var _ wizard.Runtime = &RuntimeWithReplacedFS{} // PreVortexVerifierStep is a step replicating the verifier of the tmpl at round // `Round` before the Vortex compilation step. type PreVortexVerifierStep struct { - Ctx *recursionCtx - Round int + Ctxs []*recursionCtx + Round int + isSkipped bool } // RuntimeWithReplacedFS is a runtime that wraps another runtime and replaces @@ -37,17 +43,20 @@ type GnarkRuntimeWithReplacedFS struct { func (pa PreVortexVerifierStep) Run(run wizard.Runtime) error { - pa.generateRandomCoins(run) + var err error - // Wraps the runtime into a translation adapter - var ( - err error - wrappedRun = &runtimeTranslator{Prefix: pa.Ctx.Translator.Prefix, Rt: run} - ) + for _, ctx := range pa.Ctxs { + generateRandomCoins(run, ctx, pa.Round) + + // Wraps the runtime into a translation adapter + var ( + wrappedRun = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + ) - // Copy the verifier actions of the template into the target - for _, va := range pa.Ctx.VerifierActions[pa.Round] { - err = errors.Join(err, va.Run(wrappedRun)) + // Copy the verifier actions of the template into the target + for _, va := range ctx.VerifierActions[pa.Round] { + err = errors.Join(err, va.Run(wrappedRun)) + } } return err @@ -55,18 +64,10 @@ func (pa PreVortexVerifierStep) Run(run wizard.Runtime) error { // generateRandomCoins generates all the coins for the current round // so that they are made available to the forthcoming verifier actions. -func (pa PreVortexVerifierStep) generateRandomCoins(run wizard.Runtime) { - - const ( - fiatShamirHistoryStr = "fiat-shamir-history" - fiatShamirTranscriptStr = "fiat-shamir-transcript" - ) +func generateRandomCoins(run wizard.Runtime, ctx *recursionCtx, currRound int) { var ( - currRound = pa.Round - initialState = run.Fs().State() - ctx = pa.Ctx - spec = run.GetSpec() + spec = run.GetSpec() // Wraps the runtime into a translation adapter, first to get the FS state // and history. wrappedRun wizard.Runtime = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} @@ -74,6 +75,7 @@ func (pa PreVortexVerifierStep) generateRandomCoins(run wizard.Runtime) { fs = fsAny.(*fiatshamir.State) fsHistoryAny, _ = wrappedRun.GetState(fiatShamirHistoryStr) fsHistory = fsHistoryAny.([][2][]field.Element) + initialState = fs.State() ) if fs == nil { @@ -136,27 +138,25 @@ func (run *RuntimeWithReplacedFS) FsHistory() [][2][]field.Element { return run.FiatShamirHistory } -func (pa PreVortexVerifierStep) RunGnark(api frontend.API, run wizard.GnarkRuntime) error { +func (pa PreVortexVerifierStep) RunGnark(api frontend.API, run wizard.GnarkRuntime) { - pa.generateRandomCoinsGnark(api, run) + for _, ctx := range pa.Ctxs { - // Wraps the runtime into a translation adapter - var ( - err error - wrappedRun = &gnarkRuntimeTranslator{Prefix: pa.Ctx.Translator.Prefix, Rt: run} - ) + pa.generateRandomCoinsGnark(api, run, ctx, pa.Round) - // Copy the verifier actions of the template into the target - for _, va := range pa.Ctx.VerifierActions[pa.Round] { - va.RunGnark(api, wrappedRun) - } + // Wraps the runtime into a translation adapter + var wrappedRun = &gnarkRuntimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} - return err + // Copy the verifier actions of the template into the target + for _, va := range ctx.VerifierActions[pa.Round] { + va.RunGnark(api, wrappedRun) + } + } } // generateRandomCoinsGnark generates all the coins for the current round // so that they are made available to the forthcoming verifier actions. -func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run wizard.GnarkRuntime) { +func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run wizard.GnarkRuntime, ctx *recursionCtx, currRound int) { const ( fiatShamirHistoryStr = "fiat-shamir-history" @@ -164,10 +164,7 @@ func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run w ) var ( - currRound = pa.Round - initialState = run.Fs().State() - ctx = pa.Ctx - spec = run.GetSpec() + spec = run.GetSpec() // Wraps the runtime into a translation adapter, first to get the FS state // and history. wrappedRun wizard.GnarkRuntime = &gnarkRuntimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} @@ -175,6 +172,7 @@ func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run w fs = fsAny.(*fiatshamir.GnarkFiatShamir) fsHistoryAny, _ = wrappedRun.GetState(fiatShamirHistoryStr) fsHistory = fsHistoryAny.([][2][]frontend.Variable) + initialState = fs.State() ) if fs == nil && run.GetHasherFactory() != nil { @@ -242,3 +240,11 @@ func (run *GnarkRuntimeWithReplacedFS) Fs() *fiatshamir.GnarkFiatShamir { func (run *GnarkRuntimeWithReplacedFS) FsHistory() [][2][]frontend.Variable { return run.FiatShamirHistory } + +func (pa *PreVortexVerifierStep) IsSkipped() bool { + return pa.isSkipped +} + +func (pa *PreVortexVerifierStep) Skip() { + pa.isSkipped = true +} From 9d16c6fb5cd1e9f67113baaaf9c89c3c4b0d7f69 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 17:37:17 +0100 Subject: [PATCH 09/63] feat(conglomeration): fix the vortex capture --- .../conglomeration/conglomeration.go | 51 ++++++++++++------- .../distributed/conglomeration/translator.go | 18 +++++-- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index aa129df4862..640420714c0 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/consensys/linea-monorepo/prover/protocol/coin" + "github.com/consensys/linea-monorepo/prover/protocol/column" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" @@ -33,21 +34,37 @@ func Conglomerate( ) (comp *wizard.CompiledIOP) { comp = wizard.NewCompiledIOP() + var ctxs []*recursionCtx for id := 0; id < maxNumSegment; id++ { - addVerifierToComp(id, comp, tmpl) + prefix := fmt.Sprintf("verifier-%v", id) + ctx := initRecursionCtx(prefix, comp) + ctx.captureCompPreVortex(tmpl) + ctx.captureVortexCtx(tmpl) + ctxs = append(ctxs, ctx) } -} + for round := 0; round <= ctxs[0].LastRound; round++ { + + var ( + hasCoin = len(ctxs[0].Coins[round]) > 0 + hasVAction = len(ctxs[0].VerifierActions[round]) > 0 + hasFsHook = len(ctxs[0].FsHooks[round]) > 0 + hasColumn = len(ctxs[0].Columns[round]) > 0 + hasQParams = len(ctxs[0].QueryParams[round]) > 0 + ) + + if hasCoin || hasVAction || hasFsHook { + comp.RegisterVerifierAction(round, &PreVortexVerifierStep{Ctxs: ctxs, Round: round}) + } + + if hasColumn || hasQParams { + comp.RegisterProverAction(round, &PreVortexProverStep{Ctxs: ctxs, Round: round}) + } + } + + return comp -func addVerifierToComp( - id int, - comp *wizard.CompiledIOP, - tmpl *wizard.CompiledIOP, -) { - ctx := initRecursionCtx(fmt.Sprintf("verifier-%v", id), comp) - ctx.captureCompPreVortex(tmpl) - ctx.captureVortexCtx(tmpl) } // initCtx initializes a new context @@ -84,8 +101,8 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { // filter the columns by status var ( - status = tmpl.Columns.Status(colName) - size = tmpl.Columns.GetSize(colName) + col = tmpl.Columns.GetHandle(colName).(column.Natural) + status = col.Status() ) if !status.IsPublic() { @@ -93,7 +110,7 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { continue } - newCol := ctx.Translator.InsertColumn(round, colName, size, status) + newCol := ctx.Translator.InsertColumn(col) ctx.Columns[round] = append(ctx.Columns[round], newCol) ctx.Translator.Target.Columns.IgnoreButKeepInProverTranscript(newCol.GetColID()) } @@ -185,16 +202,14 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { dstVortexCtx.Items.Precomputeds.PrecomputedColums = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Precomputeds.PrecomputedColums) dstVortexCtx.Items.Precomputeds.MerkleRoot = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.MerkleRoot.GetColID()) - dstVortexCtx.Items.Precomputeds.Dh = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.Dh.GetColID()) dstVortexCtx.Items.Precomputeds.CommittedMatrix = srcVortexCtx.Items.Precomputeds.CommittedMatrix dstVortexCtx.Items.Precomputeds.DhWithMerkle = srcVortexCtx.Items.Precomputeds.DhWithMerkle - dstVortexCtx.Items.Dh = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Dh) dstVortexCtx.Items.Alpha = ctx.Translator.GetCoin(srcVortexCtx.Items.Alpha.Name) - dstVortexCtx.Items.Ualpha = ctx.Translator.GetColumn(srcVortexCtx.Items.Ualpha.GetColID()) + dstVortexCtx.Items.Ualpha = ctx.Translator.InsertColumn(srcVortexCtx.Items.Ualpha.(column.Natural)) dstVortexCtx.Items.Q = ctx.Translator.GetCoin(srcVortexCtx.Items.Q.Name) - dstVortexCtx.Items.OpenedColumns = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.OpenedColumns) - dstVortexCtx.Items.MerkleProofs = ctx.Translator.GetColumn(srcVortexCtx.Items.MerkleProofs.GetColID()) + dstVortexCtx.Items.OpenedColumns = ctx.Translator.InsertColumns(srcVortexCtx.Items.OpenedColumns) + dstVortexCtx.Items.MerkleProofs = ctx.Translator.InsertColumn(srcVortexCtx.Items.MerkleProofs.(column.Natural)) dstVortexCtx.Items.MerkleRoots = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.MerkleRoots) ctx.PcsCtx = dstVortexCtx diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 5ae00f0f94e..014c47c2620 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -41,9 +41,21 @@ type gnarkRuntimeTranslator struct { // InsertColumn inserts a new column in the target compiled IOP. The column name // is prefixed with comp.Prefix. -func (comp *compTranslator) InsertColumn(round int, name ifaces.ColID, size int, status column.Status) ifaces.Column { - name = ifaces.ColID(comp.Prefix) + "." + name - return comp.Target.InsertColumn(round, name, size, status) +func (comp *compTranslator) InsertColumn(col column.Natural) ifaces.Column { + name := ifaces.ColID(comp.Prefix) + "." + col.ID + return comp.Target.InsertColumn(col.Round(), name, col.Size(), col.Status()) +} + +// InsertColumns inserts a list of columns in the target compiled IOP by adding +// a prefix to their names. The inputs columns are expected to be of type +// Natural or this will lead to a panic. +func (comp *compTranslator) InsertColumns(cols []ifaces.Column) []ifaces.Column { + res := make([]ifaces.Column, 0, len(cols)) + for i := range cols { + r := comp.InsertColumn(cols[i].(column.Natural)) + res = append(res, r) + } + return res } // GetColumn returns a column from the target compiled IOP. From b553efea8b9a6d5d69acdcd202d52f30515f782e Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 17:43:36 +0100 Subject: [PATCH 10/63] fixup(translator): make InsertCoin more helpful --- .../distributed/conglomeration/conglomeration.go | 12 ++++++------ .../distributed/conglomeration/translator.go | 13 ++++++++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 640420714c0..6947bab1bfa 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -146,10 +146,10 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { continue } - coin := tmpl.Coins.Data(cName) - coin = ctx.Translator.InsertCoin(round, cName, coin.Type, coin.Size) - ctx.Coins[round] = append(ctx.Coins[round], coin) - ctx.Translator.Target.Coins.MarkAsSkippedFromVerifierTranscript(coin.Name) + coinInfo := tmpl.Coins.Data(cName) + coinInfo = ctx.Translator.InsertCoin(coinInfo) + ctx.Coins[round] = append(ctx.Coins[round], coinInfo) + ctx.Translator.Target.Coins.MarkAsSkippedFromVerifierTranscript(coinInfo.Name) } verifierActions := tmpl.SubVerifiers.Inner() @@ -205,9 +205,9 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { dstVortexCtx.Items.Precomputeds.CommittedMatrix = srcVortexCtx.Items.Precomputeds.CommittedMatrix dstVortexCtx.Items.Precomputeds.DhWithMerkle = srcVortexCtx.Items.Precomputeds.DhWithMerkle - dstVortexCtx.Items.Alpha = ctx.Translator.GetCoin(srcVortexCtx.Items.Alpha.Name) + dstVortexCtx.Items.Alpha = ctx.Translator.InsertCoin(srcVortexCtx.Items.Alpha) dstVortexCtx.Items.Ualpha = ctx.Translator.InsertColumn(srcVortexCtx.Items.Ualpha.(column.Natural)) - dstVortexCtx.Items.Q = ctx.Translator.GetCoin(srcVortexCtx.Items.Q.Name) + dstVortexCtx.Items.Q = ctx.Translator.InsertCoin(srcVortexCtx.Items.Q) dstVortexCtx.Items.OpenedColumns = ctx.Translator.InsertColumns(srcVortexCtx.Items.OpenedColumns) dstVortexCtx.Items.MerkleProofs = ctx.Translator.InsertColumn(srcVortexCtx.Items.MerkleProofs.(column.Natural)) dstVortexCtx.Items.MerkleRoots = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.MerkleRoots) diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 014c47c2620..5a89e585f50 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -66,9 +66,16 @@ func (comp *compTranslator) GetColumn(name ifaces.ColID) ifaces.Column { // InsertCoin inserts a new coin in the target compiled IOP. The coin name // is prefixed with the comp.Prefix. -func (comp *compTranslator) InsertCoin(round int, name coin.Name, type_ coin.Type, size ...int) coin.Info { - name = coin.Name(comp.Prefix) + "." + name - return comp.Target.InsertCoin(round, name, type_, size...) +func (comp *compTranslator) InsertCoin(info coin.Info) coin.Info { + name := coin.Name(comp.Prefix) + "." + info.Name + switch info.Type { + case coin.IntegerVec: + return comp.Target.InsertCoin(info.Round, name, info.Type, info.Size, info.UpperBound) + case coin.Field: + return comp.Target.InsertCoin(info.Round, name, info.Type) + default: + panic("unknown coin type") + } } // GetCoin returns a coin with the prefixed name in the target compiled IOP. From 8ecf79229ddc4eadab4beeb1fa0e20f284d3b5c4 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 22:02:11 +0100 Subject: [PATCH 11/63] feat(vortex): adds a prefix system for run.State objects --- .../compiler/selfrecursion/column_opening.go | 6 +-- prover/protocol/compiler/vortex/compiler.go | 4 ++ prover/protocol/compiler/vortex/names.go | 22 ++++++++++- prover/protocol/compiler/vortex/prover.go | 2 +- .../conglomeration/conglomeration.go | 38 +++++++++++------- .../distributed/conglomeration/prover.go | 39 ++++++++++++++++++- 6 files changed, 91 insertions(+), 20 deletions(-) diff --git a/prover/protocol/compiler/selfrecursion/column_opening.go b/prover/protocol/compiler/selfrecursion/column_opening.go index 04d442e847d..e856b68ce33 100644 --- a/prover/protocol/compiler/selfrecursion/column_opening.go +++ b/prover/protocol/compiler/selfrecursion/column_opening.go @@ -255,8 +255,8 @@ func (ctx *SelfRecursionCtx) linearHashAndMerkle() { } for round := 0; round <= totalNumRounds; round++ { - colSisHashName := ctx.VortexCtx.CommitmentName(round) - colSisHashSV, found := run.State.TryGet(string(colSisHashName)) + colSisHashName := ctx.VortexCtx.SisHashName(round) + colSisHashSV, found := run.State.TryGet(colSisHashName) if !found { // continue with the same committedRound until we meet a non-dry // round or we reach the total number of committed rounds @@ -291,7 +291,7 @@ func (ctx *SelfRecursionCtx) linearHashAndMerkle() { } // Frees the colSisHash - run.State.TryDel(string(colSisHashName)) + run.State.TryDel(colSisHashName) // Increment only if the committedRound is non-dry committedRound++ diff --git a/prover/protocol/compiler/vortex/compiler.go b/prover/protocol/compiler/vortex/compiler.go index 714ea6b74bb..34d6d251048 100644 --- a/prover/protocol/compiler/vortex/compiler.go +++ b/prover/protocol/compiler/vortex/compiler.go @@ -126,6 +126,10 @@ type Ctx struct { CommitmentsByRounds collection.VecVec[ifaces.ColID] DriedByRounds collection.VecVec[ifaces.ColID] + // RunStateNamePrefix is used to prefix some of the names of components of the + // compilation context. Mainly state objects. + RunStateNamePrefix string + // Items created by Vortex, includes the proof message and the coins Items struct { // List of items used only if the CommitPrecomputed flag is set diff --git a/prover/protocol/compiler/vortex/names.go b/prover/protocol/compiler/vortex/names.go index 7536bcf2970..cd407cd9250 100644 --- a/prover/protocol/compiler/vortex/names.go +++ b/prover/protocol/compiler/vortex/names.go @@ -32,14 +32,32 @@ func (ctx *Ctx) CommitmentName(round int) ifaces.ColID { return ifaces.ColIDf("VORTEX_%v_COMMITMENT_ROUND_%v", ctx.SelfRecursionCount, round) } +// SisHashName returns a preformatted message representing the Sis hash digests +// for each round that we store in the state. +func (ctx *Ctx) SisHashName(round int) string { + name := fmt.Sprintf("VORTEX_%v_SIS_HASH_%v", ctx.SelfRecursionCount, round) + if len(ctx.RunStateNamePrefix) > 0 { + return name + } + return ctx.RunStateNamePrefix + "." + name +} + // returns the name of a prover state for a given round of Vortex func (ctx *Ctx) VortexProverStateName(round int) string { - return fmt.Sprintf("VORTEX_%v_PROVER_STATE_%v", ctx.SelfRecursionCount, round) + name := fmt.Sprintf("VORTEX_%v_PROVER_STATE_%v", ctx.SelfRecursionCount, round) + if len(ctx.RunStateNamePrefix) > 0 { + return name + } + return ctx.RunStateNamePrefix + "." + name } // returns the name of a prover state for a given round of Vortex func (ctx *Ctx) MerkleTreeName(round int) string { - return fmt.Sprintf("VORTEX_%v_MERKLE_TREE_%v", ctx.SelfRecursionCount, round) + name := fmt.Sprintf("VORTEX_%v_MERKLE_TREE_%v", ctx.SelfRecursionCount, round) + if len(ctx.RunStateNamePrefix) > 0 { + return name + } + return ctx.RunStateNamePrefix + "." + name } // returns the name of the vector containing all the Merkle proofs diff --git a/prover/protocol/compiler/vortex/prover.go b/prover/protocol/compiler/vortex/prover.go index bad0c99a18d..2c0879b7fbf 100644 --- a/prover/protocol/compiler/vortex/prover.go +++ b/prover/protocol/compiler/vortex/prover.go @@ -33,7 +33,7 @@ func (ctx *Ctx) AssignColumn(round int) func(*wizard.ProverRuntime) { // Only to be read by the self-recursion compiler. if ctx.IsSelfrecursed { - pr.State.InsertNew(string(ctx.CommitmentName(round)), sisDigest) + pr.State.InsertNew(ctx.SisHashName(round), sisDigest) } // And assign the 1-sized column to contain the root diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 6947bab1bfa..e79c2134879 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -21,6 +21,7 @@ type recursionCtx struct { NonEmptyMerkleRootPositions []int FirstRound, LastRound int Columns [][]ifaces.Column + ColumnsIgnored [][]ifaces.Column QueryParams [][]ifaces.Query VerifierActions [][]wizard.VerifierAction Coins [][]coin.Info @@ -181,21 +182,32 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { var ( srcVortexCtx = tmpl.PcsCtxs.(*vortex.Ctx) - dstVortexCtx = &vortex.Ctx{ - BlowUpFactor: srcVortexCtx.BlowUpFactor, - DryTreshold: srcVortexCtx.DryTreshold, - CommittedRowsCount: srcVortexCtx.CommittedRowsCount, - NumCols: srcVortexCtx.NumCols, - MaxCommittedRound: srcVortexCtx.MaxCommittedRound, - NumOpenedCol: srcVortexCtx.NumOpenedCol, - CommitmentsByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.CommitmentsByRounds), - DriedByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.DriedByRounds), - PolynomialsTouchedByTheQuery: ctx.Translator.TranslateColumnSet(srcVortexCtx.PolynomialsTouchedByTheQuery), - ShadowCols: ctx.Translator.TranslateColumnSet(srcVortexCtx.ShadowCols), - Query: ctx.Translator.TranslateUniEval(ctx.LastRound, srcVortexCtx.Query), - } + comsByRound = srcVortexCtx.CommitmentsByRounds.Inner() ) + for _, coms := range comsByRound { + ctx.ColumnsIgnored = append(ctx.ColumnsIgnored, nil) + for _, comID := range coms { + com := tmpl.Columns.GetHandle(comID) + ctx.Translator.InsertColumn(com.(column.Natural)) + } + } + + dstVortexCtx := &vortex.Ctx{ + RunStateNamePrefix: ctx.Translator.Prefix, + BlowUpFactor: srcVortexCtx.BlowUpFactor, + DryTreshold: srcVortexCtx.DryTreshold, + CommittedRowsCount: srcVortexCtx.CommittedRowsCount, + NumCols: srcVortexCtx.NumCols, + MaxCommittedRound: srcVortexCtx.MaxCommittedRound, + NumOpenedCol: srcVortexCtx.NumOpenedCol, + CommitmentsByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.CommitmentsByRounds), + DriedByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.DriedByRounds), + PolynomialsTouchedByTheQuery: ctx.Translator.TranslateColumnSet(srcVortexCtx.PolynomialsTouchedByTheQuery), + ShadowCols: ctx.Translator.TranslateColumnSet(srcVortexCtx.ShadowCols), + Query: ctx.Translator.TranslateUniEval(ctx.LastRound, srcVortexCtx.Query), + } + if srcVortexCtx.ReplaceSisByMimc { panic("it should not replace by MiMC") } diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 9541c6489d4..05da13a1557 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -6,6 +6,10 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/wizard" ) +const ( + subProofInStatePrefixStr = ".subProof" +) + // PreVortexProverStep is a step replicating the prover of the tmpl at round // `Round` before the Vortex compilation step. It works by adding columns from // a wizard proof stored in the prover runtime. The proof is fetched from the @@ -16,11 +20,28 @@ type PreVortexProverStep struct { Round int } +// AssignVortexUAlpha assigns the UAlpha column for all the subproofs. As +// for [PreVortexVerifierStep], this step should be run after the corresponding +// proofs have been added to the runtime states. +type AssignVortexUAlpha struct { + Ctxs []*recursionCtx + Round int +} + +// AssignVortexOpenedCols assigns the OpenedCols for all the subproofs. As +// for [PreVortexVerifierStep], this step should be run after the corresponding +// proofs have been added to the runtime states. +type AssignVortexOpenedCols struct { + Ctxs []*recursionCtx + Round int +} + func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { for _, ctx := range pa.Ctxs { + var ( prefix = ctx.Translator.Prefix - proof = run.State.MustGet(prefix + ".subproof").(wizard.Proof) + proof = run.State.MustGet(prefix + subProofInStatePrefixStr).(wizard.Proof) queriesParams = ctx.QueryParams[pa.Round] colums = ctx.Columns[pa.Round] ) @@ -37,6 +58,22 @@ func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { } } +func (pa AssignVortexUAlpha) Run(run *wizard.ProverRuntime) { + for _, ctx := range pa.Ctxs { + // Since all the context of the pcs is translated, this does not + // need to run over a translated prover runtime. + ctx.PcsCtx.ComputeLinearComb(run) + } +} + +func (pa AssignVortexOpenedCols) Run(run *wizard.ProverRuntime) { + for _, ctx := range pa.Ctxs { + // Since all the context of the pcs is translated, this does not + // need to run over a translated prover runtime. + ctx.PcsCtx.OpenSelectedColumns(run) + } +} + func unprefix[T ~string](prefix string, name T) T { p, n := string(prefix), string(name) r := strings.TrimPrefix(n, p) From 2da9f6795278d5dc5659f61a9f213904c7f645f6 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 17 Jan 2025 22:20:41 +0100 Subject: [PATCH 12/63] feat(conglomeration): adds the context recursion into the conglomeration --- .../protocol/compiler/selfrecursion/context.go | 17 +++++++++++------ .../compiler/selfrecursion/selfrecursion.go | 10 ++++++++++ .../conglomeration/conglomeration.go | 3 ++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/prover/protocol/compiler/selfrecursion/context.go b/prover/protocol/compiler/selfrecursion/context.go index 5226926f964..3ad408628ad 100644 --- a/prover/protocol/compiler/selfrecursion/context.go +++ b/prover/protocol/compiler/selfrecursion/context.go @@ -197,12 +197,10 @@ type SelfRecursionCtx struct { } } -// Initializes a context for the self recursion -func NewSelfRecursionCxt(comp *wizard.CompiledIOP) SelfRecursionCtx { - - // Extract the vortex context from the compiledIOP though - // the "CryptographicCompilerCtx" - vortexCtx := assertVortexCompiled(comp) +// NewRecursionCtx returns a new recursion context taking a specified +// [vortex.Ctx] and SelfRecursionCount. It can be used for custom use +// of the recursion wizard. +func NewRecursionCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx) SelfRecursionCtx { ctx := SelfRecursionCtx{ comp: comp, @@ -280,6 +278,13 @@ func NewSelfRecursionCxt(comp *wizard.CompiledIOP) SelfRecursionCtx { return ctx } +// Initializes a context for the self recursion +func NewSelfRecursionCxt(comp *wizard.CompiledIOP) SelfRecursionCtx { + // Extract the vortex context from the compiledIOP though the "Pcs" + vortexCtx := assertVortexCompiled(comp) + return NewRecursionCtx(comp, vortexCtx) +} + // Asserts that the compiled IOP has the appropriate cryptographic context func assertVortexCompiled(comp *wizard.CompiledIOP) *vortex.Ctx { // When we compiled using Vortex, we annotated the compiledIOP diff --git a/prover/protocol/compiler/selfrecursion/selfrecursion.go b/prover/protocol/compiler/selfrecursion/selfrecursion.go index edb1189e3fb..e4805601ad8 100644 --- a/prover/protocol/compiler/selfrecursion/selfrecursion.go +++ b/prover/protocol/compiler/selfrecursion/selfrecursion.go @@ -1,6 +1,7 @@ package selfrecursion import ( + "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/wizard" "github.com/sirupsen/logrus" ) @@ -19,3 +20,12 @@ func SelfRecurse(comp *wizard.CompiledIOP) { // Update the self-recursion counter comp.SelfRecursionCount++ } + +// RecurseOverCustomCtx applies the same compilation steps as [SelfRecurse] +// over a specified vortex compilation context. +func RecurseOverCustomCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx) { + ctx := NewRecursionCtx(comp, vortexCtx) + ctx.Precomputations() + ctx.RowLinearCombinationPhase() + ctx.ColumnOpeningPhase() +} diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index e79c2134879..8ee8d814ae6 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -5,6 +5,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/column" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/selfrecursion" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" @@ -42,6 +43,7 @@ func Conglomerate( ctx := initRecursionCtx(prefix, comp) ctx.captureCompPreVortex(tmpl) ctx.captureVortexCtx(tmpl) + selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx) ctxs = append(ctxs, ctx) } @@ -65,7 +67,6 @@ func Conglomerate( } return comp - } // initCtx initializes a new context From 041240183c8bc883b355e805db5777c466cf1566 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 01:18:29 +0100 Subject: [PATCH 13/63] fix(fsHook): adds fsHook in the equalizer --- prover/protocol/wizard/builder.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/prover/protocol/wizard/builder.go b/prover/protocol/wizard/builder.go index 3a52c709e94..937ca54b43b 100644 --- a/prover/protocol/wizard/builder.go +++ b/prover/protocol/wizard/builder.go @@ -310,4 +310,12 @@ func (b *Builder) equalizeRounds(numRounds int) { utils.Panic("Bug : numRounds is %v but %v rounds are registered for the verifier. %v", numRounds, comp.SubVerifiers.Len(), helpMsg) } comp.SubVerifiers.Reserve(numRounds) + + /* + Check and reserve for the FiatShamirHooks + */ + if comp.FiatShamirHooks.Len() > numRounds { + utils.Panic("Bug : numRounds is %v but %v rounds are registered for the FiatShamirHooks. %v", numRounds, comp.FiatShamirHooks.Len(), helpMsg) + } + comp.FiatShamirHooks.Reserve(numRounds) } From 71d3280541a37ba5cfcf864ed98272c28fc786f8 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 01:19:05 +0100 Subject: [PATCH 14/63] feat(wizard): adds ExtractProof and RunProverUntilRound --- prover/protocol/wizard/prover.go | 63 ++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index e52c80c6f18..7d77e9bebb5 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -125,7 +125,7 @@ type ProverRuntime struct { // in the distributed prover by the module runtimes to access the initial // wizard runtime. ParentRuntime *ProverRuntime - + // FiatShamirHistory tracks the fiat-shamir state at the beginning of every // round. The first entry is the initial state, the final entry is the final // state. @@ -153,29 +153,7 @@ type ProverRuntime struct { // when the specified protocol is complicated and involves multiple multi-rounds // sub-protocols that runs independently. func Prove(c *CompiledIOP, highLevelprover ProverStep) Proof { - - runtime := RunProver(c, highLevelprover) - - /* - Pass all the prover message columns as part of the proof - */ - messages := collection.NewMapping[ifaces.ColID, ifaces.ColAssignment]() - - for _, name := range runtime.Spec.Columns.AllKeysProof() { - messageValue := runtime.Columns.MustGet(name) - messages.InsertNew(name, messageValue) - } - - // And also the public inputs - for _, name := range runtime.Spec.Columns.AllKeysPublicInput() { - messageValue := runtime.Columns.MustGet(name) - messages.InsertNew(name, messageValue) - } - - return Proof{ - Messages: messages, - QueriesParams: runtime.QueriesParams, - } + return RunProver(c, highLevelprover).ExtractProof() } // RunProver initializes a [ProverRuntime], runs the prover and returns the final @@ -202,6 +180,43 @@ func RunProver(c *CompiledIOP, highLevelprover ProverStep) *ProverRuntime { return &runtime } +// RunProverUntilRound runs the prover until the specified round +func RunProverUntilRound(c *CompiledIOP, highLevelprover ProverStep, round int) *ProverRuntime { + + runtime := c.createProver() + + highLevelprover(&runtime) + runtime.runProverSteps() + + for runtime.currRound+1 < round { + runtime.goNextRound() + runtime.runProverSteps() + } + + return &runtime +} + +// ExtractProof extracts the proof from a [ProverRuntime] +func (run *ProverRuntime) ExtractProof() Proof { + messages := collection.NewMapping[ifaces.ColID, ifaces.ColAssignment]() + + for _, name := range run.Spec.Columns.AllKeysProof() { + messageValue := run.Columns.MustGet(name) + messages.InsertNew(name, messageValue) + } + + // And also the public inputs + for _, name := range run.Spec.Columns.AllKeysPublicInput() { + messageValue := run.Columns.MustGet(name) + messages.InsertNew(name, messageValue) + } + + return Proof{ + Messages: messages, + QueriesParams: run.QueriesParams, + } +} + // NumRounds returns the total number of rounds in the corresponding WizardIOP. // // Deprecated: this method does not bring anything useful as its already easy From ed6b657d2d2d2a355c0f67ca95652e8ca65528e2 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 01:24:00 +0100 Subject: [PATCH 15/63] fix(typo): dh to rooth in the error message --- prover/protocol/compiler/selfrecursion/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/compiler/selfrecursion/context.go b/prover/protocol/compiler/selfrecursion/context.go index 3ad408628ad..10d1fcb4e8e 100644 --- a/prover/protocol/compiler/selfrecursion/context.go +++ b/prover/protocol/compiler/selfrecursion/context.go @@ -236,7 +236,7 @@ func NewRecursionCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx) SelfRecurs // Assume that the rounds commitments have a `Proof` status if comp.Columns.Status(rooth.GetColID()) != column.Proof { utils.Panic( - "Assumed the Dh to be %v but status is %v", + "Assumed the rootH to be %v but status is %v", column.Proof.String(), comp.Columns.Status(rooth.GetColID()), ) From e94f9cdfb6fb01fb5c8375f70e43aef409f1a650 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 01:38:06 +0100 Subject: [PATCH 16/63] rename(fs): rename long method to AllKeysInProverTranscript --- prover/protocol/column/store.go | 6 +++--- prover/protocol/wizard/prover.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/prover/protocol/column/store.go b/prover/protocol/column/store.go index 7e669f86f2a..2b04d0f9968 100644 --- a/prover/protocol/column/store.go +++ b/prover/protocol/column/store.go @@ -467,9 +467,9 @@ func (s *Store) IsIgnoredAndNotKeptInTranscript(colName ifaces.ColID) bool { return in.Status == Ignored && !in.IncludeInProverFS } -// AllKeysProofsOrIgnoredButKeptInProverTranscript returns the list of the -// columns to be used as part of the FS transcript. -func (s *Store) AllKeysProofsOrIgnoredButKeptInProverTranscript(round int) []ifaces.ColID { +// AllKeysInProverTranscript returns the list of the columns to +// be used as part of the FS transcript. +func (s *Store) AllKeysInProverTranscript(round int) []ifaces.ColID { res := []ifaces.ColID{} rnd := s.byRounds.MustGet(round) // precomputed are always at round zero diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index 7d77e9bebb5..a6a07fef3ae 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -531,7 +531,7 @@ func (run *ProverRuntime) goNextRound() { FS using the last round of the prover because he is always the last one to "talk" in the protocol. */ - msgsToFS := run.Spec.Columns.AllKeysProofsOrIgnoredButKeptInProverTranscript(run.currRound) + msgsToFS := run.Spec.Columns.AllKeysInProverTranscript(run.currRound) for _, msgName := range msgsToFS { instance := run.GetMessage(msgName) run.FS.UpdateSV(instance) From 27fa36e9153604aeac275319f05e5836387dfeac Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 12:45:17 +0100 Subject: [PATCH 17/63] feat(fs): adds a possibility to explicitly ignore FS columns --- prover/protocol/column/store.go | 41 ++++++++++++++++--- .../compiler/fullrecursion/circuit.go | 5 +++ .../distributed/conglomeration/verifier.go | 12 ++++++ prover/protocol/wizard/gnark_verifier.go | 5 +++ prover/protocol/wizard/prover.go | 5 +++ prover/protocol/wizard/verifier.go | 5 +++ 6 files changed, 67 insertions(+), 6 deletions(-) diff --git a/prover/protocol/column/store.go b/prover/protocol/column/store.go index 2b04d0f9968..e848e6c769d 100644 --- a/prover/protocol/column/store.go +++ b/prover/protocol/column/store.go @@ -49,6 +49,12 @@ type storedColumnInfo struct { // FullRecursion. This field is only meaningfull for [Ignored] columns as // they are excluded by default. IncludeInProverFS bool + // ExcludeFromProverFS states the prover should not include the column in + // his FS transcript. This overrides [IncludeInProverFS], meaning that if + // [IncludeInProverFS] is true but ExcludeFromProverFS is true, the column + // will still be excluded from the transcript. This is used explicit FS + // compilation. + ExcludeFromProverFS bool } // AddToRound constructs a [Natural], registers it in the [Store] and returns @@ -460,11 +466,35 @@ func (s *Store) IgnoreButKeepInProverTranscript(colName ifaces.ColID) { in.IncludeInProverFS = true } -// IsIgnoredAndNotKeptInTranscript indicates whether the column can be ignored -// from the transcript and is used during the Fiat-Shamir randomness generation. -func (s *Store) IsIgnoredAndNotKeptInTranscript(colName ifaces.ColID) bool { +// ExcludeFromProverFS marks a column as excluded from the FS transcript but +// without changing its status. This is used as part of the conglomeration +// where the imported columns take part in a separate FS transcript from the +// canonical of the host wizard. +func (s *Store) ExcludeFromProverFS(colName ifaces.ColID) { in := s.info(colName) - return in.Status == Ignored && !in.IncludeInProverFS + in.ExcludeFromProverFS = true +} + +// isExcludedFromProverFS returns true if the passed column ID relates to a column +// that does not take part in the FS transcript. +func (in *storedColumnInfo) isExcludedFromProverFS() bool { + + if in.ExcludeFromProverFS { + return true + } + + if in.Status == Ignored && in.IncludeInProverFS { + return true + } + + return in.Status.IsPublic() +} + +// IsExplicitlyExcludedFromProverFS returns true if the passed column ID relates to +// a column explicitly marked as excluded from the FS transcript. +func (s *Store) IsExplicitlyExcludedFromProverFS(colName ifaces.ColID) bool { + info := s.info(colName) + return info.ExcludeFromProverFS } // AllKeysInProverTranscript returns the list of the columns to @@ -475,8 +505,7 @@ func (s *Store) AllKeysInProverTranscript(round int) []ifaces.ColID { for i, info := range rnd { - ok := (info.Status == Proof) || (info.Status == Ignored && info.IncludeInProverFS) - if !ok { + if info.isExcludedFromProverFS() { continue } diff --git a/prover/protocol/compiler/fullrecursion/circuit.go b/prover/protocol/compiler/fullrecursion/circuit.go index 6524e8fd79c..9e92c4515f6 100644 --- a/prover/protocol/compiler/fullrecursion/circuit.go +++ b/prover/protocol/compiler/fullrecursion/circuit.go @@ -129,6 +129,11 @@ func (c *gnarkCircuit) generateAllRandomCoins(api frontend.API) { toUpdateFS := ctx.Columns[currRound-1] for _, msg := range toUpdateFS { + + if c.comp.Columns.IsExplicitlyExcludedFromProverFS(msg.GetColID()) { + continue + } + val := w.GetColumn(msg.GetColID()) w.FS.UpdateVec(val) } diff --git a/prover/protocol/distributed/conglomeration/verifier.go b/prover/protocol/distributed/conglomeration/verifier.go index 44a474998c9..b1894f9c507 100644 --- a/prover/protocol/distributed/conglomeration/verifier.go +++ b/prover/protocol/distributed/conglomeration/verifier.go @@ -97,6 +97,12 @@ func generateRandomCoins(run wizard.Runtime, ctx *recursionCtx, currRound int) { cols := ctx.Columns[currRound-1] for _, col := range cols { + + name := unprefix(ctx.Translator.Prefix, col.GetColID()) + if ctx.Tmpl.Columns.IsExplicitlyExcludedFromProverFS(name) { + continue + } + instance := run.GetColumn(col.GetColID()) fs.UpdateSV(instance) } @@ -194,6 +200,12 @@ func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run w cols := ctx.Columns[currRound-1] for _, col := range cols { + + name := unprefix(ctx.Translator.Prefix, col.GetColID()) + if ctx.Tmpl.Columns.IsExplicitlyExcludedFromProverFS(name) { + continue + } + instance := run.GetColumn(col.GetColID()) fs.UpdateVec(instance) } diff --git a/prover/protocol/wizard/gnark_verifier.go b/prover/protocol/wizard/gnark_verifier.go index eb861a4a703..4311e9b3e6e 100644 --- a/prover/protocol/wizard/gnark_verifier.go +++ b/prover/protocol/wizard/gnark_verifier.go @@ -214,6 +214,11 @@ func (c *WizardVerifierCircuit) generateAllRandomCoins(api frontend.API) { // the last one to "talk" in the protocol. toUpdateFS := c.Spec.Columns.AllKeysProofAt(currRound - 1) for _, msg := range toUpdateFS { + + if c.Spec.Columns.IsExplicitlyExcludedFromProverFS(msg) { + continue + } + msgContent := c.GetColumn(msg) c.FS.UpdateVec(msgContent) } diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index a6a07fef3ae..5e948753474 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -533,6 +533,11 @@ func (run *ProverRuntime) goNextRound() { */ msgsToFS := run.Spec.Columns.AllKeysInProverTranscript(run.currRound) for _, msgName := range msgsToFS { + + if run.Spec.Columns.IsExplicitlyExcludedFromProverFS(msgName) { + continue + } + instance := run.GetMessage(msgName) run.FS.UpdateSV(instance) } diff --git a/prover/protocol/wizard/verifier.go b/prover/protocol/wizard/verifier.go index 5f3271e751d..dbe79a36da2 100644 --- a/prover/protocol/wizard/verifier.go +++ b/prover/protocol/wizard/verifier.go @@ -192,6 +192,11 @@ func (run *VerifierRuntime) generateAllRandomCoins() { */ msgsToFS := run.Spec.Columns.AllKeysProofAt(currRound - 1) for _, msgName := range msgsToFS { + + if run.Spec.Columns.IsExplicitlyExcludedFromProverFS(msgName) { + continue + } + instance := run.GetColumn(msgName) run.FS.UpdateSV(instance) } From c6a4801e1f08f499d02a7e36f59b42397fced888 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 12:45:42 +0100 Subject: [PATCH 18/63] fix(typo): improves the doc for IsCommitToPrecomputed --- prover/protocol/compiler/vortex/compiler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prover/protocol/compiler/vortex/compiler.go b/prover/protocol/compiler/vortex/compiler.go index 34d6d251048..55aeba54f60 100644 --- a/prover/protocol/compiler/vortex/compiler.go +++ b/prover/protocol/compiler/vortex/compiler.go @@ -528,7 +528,9 @@ func (ctx *Ctx) NumEncodedCols() int { return res } -// Create a method to decide when to commit to the precomputed +// IsCommitToPrecomputed returns true if the current compilation step +// commits to the precomputed columns. This is detected by checking if +// the number of precomputed columns is greater than the dry treshold. func (ctx *Ctx) IsCommitToPrecomputed() bool { return len(ctx.Items.Precomputeds.PrecomputedColums) > ctx.DryTreshold } From 9e48b6aff0f99d3b0bef46c9188dbb6df789f6cd Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 12:47:02 +0100 Subject: [PATCH 19/63] fix(translator): fix TranslateUniEval --- prover/protocol/distributed/conglomeration/translator.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 5a89e585f50..5d478ded485 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -146,10 +146,11 @@ func (comp *compTranslator) TranslateColumnSet(cols map[ifaces.ColID]struct{}) m // TranslateUniEval returns a copied UnivariateEval query with the columns translated // and the names translated. The returned query is registered in the translator comp. func (comp *compTranslator) TranslateUniEval(round int, q query.UnivariateEval) query.UnivariateEval { - var res = query.NewUnivariateEval(q.QueryID, q.Pols...) - for i := range res.Pols { - res.Pols[i] = comp.GetColumn(res.Pols[i].GetColID()) + newPols := make([]ifaces.Column, len(q.Pols)) + for i := range newPols { + newPols[i] = comp.GetColumn(q.Pols[i].GetColID()) } + var res = query.NewUnivariateEval(q.QueryID, newPols...) return comp.InsertQueryParams(round, res).(query.UnivariateEval) } From 6351de1a5b8372650d01f00a2228ead846dca914 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 12:50:25 +0100 Subject: [PATCH 20/63] fixup --- .../distributed/conglomeration/conglomeration.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 8ee8d814ae6..00e98e53ddf 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -16,6 +16,7 @@ import ( type recursionCtx struct { // A pointer to the compiled-IOP over which the compilation step has run Translator *compTranslator + Tmpl *wizard.CompiledIOP // The Vortex compilation context PcsCtx *vortex.Ctx PublicInputs []wizard.PublicInput @@ -40,7 +41,7 @@ func Conglomerate( for id := 0; id < maxNumSegment; id++ { prefix := fmt.Sprintf("verifier-%v", id) - ctx := initRecursionCtx(prefix, comp) + ctx := initRecursionCtx(prefix, comp, tmpl) ctx.captureCompPreVortex(tmpl) ctx.captureVortexCtx(tmpl) selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx) @@ -69,13 +70,11 @@ func Conglomerate( return comp } -// initCtx initializes a new context -func initRecursionCtx( - id string, - target *wizard.CompiledIOP, -) *recursionCtx { +// initRecursionCtx initializes a new context +func initRecursionCtx(id string, target *wizard.CompiledIOP, tmpl *wizard.CompiledIOP) *recursionCtx { return &recursionCtx{ Translator: &compTranslator{Prefix: id, Target: target}, + Tmpl: tmpl, } } @@ -114,7 +113,7 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { newCol := ctx.Translator.InsertColumn(col) ctx.Columns[round] = append(ctx.Columns[round], newCol) - ctx.Translator.Target.Columns.IgnoreButKeepInProverTranscript(newCol.GetColID()) + ctx.Translator.Target.Columns.ExcludeFromProverFS(newCol.GetColID()) } for _, qName := range tmpl.QueriesParams.AllKeysAt(round) { @@ -139,7 +138,7 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { qInfo := tmpl.QueriesParams.Data(qName) qInfo = ctx.Translator.InsertQueryParams(round, qInfo) ctx.QueryParams[round] = append(ctx.QueryParams[round], qInfo) - ctx.Translator.Target.QueriesParams.MarkAsSkippedFromVerifierTranscript(qInfo.Name()) + ctx.Translator.Target.QueriesParams.MarkAsSkippedFromProverTranscript(qInfo.Name()) } for _, cName := range tmpl.Coins.AllKeysAt(round) { From 696e978bd31ca67852057785954a1a646ef90e75 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 12:52:16 +0100 Subject: [PATCH 21/63] feat(conglomeration): implements the main top-level function --- .../conglomeration/conglomeration.go | 16 ++-- .../distributed/conglomeration/prover.go | 92 +++++++++++++++++++ 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 00e98e53ddf..4723f61cbbd 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -31,13 +31,16 @@ type recursionCtx struct { LocalOpenings []query.LocalOpening } -func Conglomerate( - tmpl *wizard.CompiledIOP, - maxNumSegment int, -) (comp *wizard.CompiledIOP) { +// ConglomerateDefineFunc returns a function that defines a conglomerate +// comp and a placeholder pointer for the recursion context. On return +// of the function, the pointer points to an empty slice and is populated +// once [wizard.Compile] has been called with def. +func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def func(*wizard.Builder), ctxsPlaceHolder *[]*recursionCtx) { - comp = wizard.NewCompiledIOP() var ctxs []*recursionCtx + def = func(b *wizard.Builder) { + + comp := b.CompiledIOP for id := 0; id < maxNumSegment; id++ { prefix := fmt.Sprintf("verifier-%v", id) @@ -66,8 +69,9 @@ func Conglomerate( comp.RegisterProverAction(round, &PreVortexProverStep{Ctxs: ctxs, Round: round}) } } + } - return comp + return def, &ctxs } // initRecursionCtx initializes a new context diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 05da13a1557..714a6786e25 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -3,13 +3,27 @@ package conglomeration import ( "strings" + "github.com/consensys/linea-monorepo/prover/crypto/state-management/smt" + vCom "github.com/consensys/linea-monorepo/prover/crypto/vortex" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/consensys/linea-monorepo/prover/utils" ) const ( subProofInStatePrefixStr = ".subProof" ) +// Witness is a collection of inputs corresponding to a segment proof to provide +// to the main prover of a conglomerate comp. +type Witness struct { + Proof wizard.Proof + CommittedMatrices []vCom.EncodedMatrix + SisHashes [][]field.Element + Trees []*smt.Tree +} + // PreVortexProverStep is a step replicating the prover of the tmpl at round // `Round` before the Vortex compilation step. It works by adding columns from // a wizard proof stored in the prover runtime. The proof is fetched from the @@ -36,6 +50,84 @@ type AssignVortexOpenedCols struct { Round int } +// ProveConglomeration returns the main prover step of the conglomeration wizard. +// It takes a list of [Witness] as input and complete the list with the last +// value. +func ProveConglomeration(ctxs []*recursionCtx, witnesses []Witness) wizard.ProverStep { + + if len(witnesses) > len(ctxs) { + utils.Panic("More witnesses than ctxs, numWitnesses: %v, numCtxs: %v", len(witnesses), len(ctxs)) + } + + return func(run *wizard.ProverRuntime) { + + for i, witness := range witnesses { + storeWitnessInState(run, ctxs[i], witness) + } + + for i := len(witnesses); i < len(ctxs); i++ { + storeWitnessInState(run, ctxs[i], witnesses[len(witnesses)-1]) + } + } +} + +func storeWitnessInState(run *wizard.ProverRuntime, ctx *recursionCtx, witness Witness) { + + var ( + prefix = ctx.Translator.Prefix + lastRound = ctx.LastRound + ) + + run.State.InsertNew(prefix+subProofInStatePrefixStr, witness.Proof) + + for round := 0; round <= lastRound; round++ { + + if witness.CommittedMatrices[round] != nil { + run.State.InsertNew(ctx.PcsCtx.VortexProverStateName(round), witness.CommittedMatrices[round]) + } + + if witness.SisHashes[round] != nil { + run.State.InsertNew(ctx.PcsCtx.SisHashName(round), witness.SisHashes[round]) + } + + if witness.Trees[round] != nil { + run.State.InsertNew(ctx.PcsCtx.MerkleTreeName(round), witness.Trees[round]) + } + } +} + +// ExtractWitness extracts a [Witness] from a prover runtime toward being conglomerated. +func ExtractWitness(run *wizard.ProverRuntime) Witness { + + var ( + pcs = run.Spec.PcsCtxs.(*vortex.Ctx) + committedMatrices []vCom.EncodedMatrix + sisHashes [][]field.Element + trees []*smt.Tree + lastRound = run.Spec.QueriesParams.Round(pcs.Query.QueryID) + ) + + for round := 0; round <= lastRound; round++ { + + var ( + committedMatrix, _ = run.State.TryGet(pcs.VortexProverStateName(round)) + sisHash, _ = run.State.TryGet(pcs.SisHashName(round)) + tree, _ = run.State.TryGet(pcs.MerkleTreeName(round)) + ) + + committedMatrices = append(committedMatrices, committedMatrix.(vCom.EncodedMatrix)) + sisHashes = append(sisHashes, sisHash.([]field.Element)) + trees = append(trees, tree.(*smt.Tree)) + } + + return Witness{ + Proof: run.ExtractProof(), + CommittedMatrices: committedMatrices, + SisHashes: sisHashes, + Trees: trees, + } +} + func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { for _, ctx := range pa.Ctxs { From 68d5d62379d98c8743653c54ae9ac45bd3df48e7 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 12:52:46 +0100 Subject: [PATCH 22/63] various fixup while testing --- .../conglomeration/conglomeration.go | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 4723f61cbbd..d77c8449a0e 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -42,34 +42,34 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu comp := b.CompiledIOP - for id := 0; id < maxNumSegment; id++ { - prefix := fmt.Sprintf("verifier-%v", id) + for id := 0; id < maxNumSegment; id++ { + prefix := fmt.Sprintf("verifier-%v", id) ctx := initRecursionCtx(prefix, comp, tmpl) - ctx.captureCompPreVortex(tmpl) - ctx.captureVortexCtx(tmpl) - selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx) - ctxs = append(ctxs, ctx) - } + ctx.captureCompPreVortex(tmpl) + ctx.captureVortexCtx(tmpl) + selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx) + ctxs = append(ctxs, ctx) + } - for round := 0; round <= ctxs[0].LastRound; round++ { + for round := 0; round <= ctxs[0].LastRound; round++ { - var ( - hasCoin = len(ctxs[0].Coins[round]) > 0 - hasVAction = len(ctxs[0].VerifierActions[round]) > 0 - hasFsHook = len(ctxs[0].FsHooks[round]) > 0 - hasColumn = len(ctxs[0].Columns[round]) > 0 - hasQParams = len(ctxs[0].QueryParams[round]) > 0 - ) + var ( + hasCoin = len(ctxs[0].Coins[round]) > 0 + hasVAction = len(ctxs[0].VerifierActions[round]) > 0 + hasFsHook = len(ctxs[0].FsHooks[round]) > 0 + hasColumn = len(ctxs[0].Columns[round]) > 0 + hasQParams = len(ctxs[0].QueryParams[round]) > 0 + ) - if hasCoin || hasVAction || hasFsHook { - comp.RegisterVerifierAction(round, &PreVortexVerifierStep{Ctxs: ctxs, Round: round}) - } + if hasCoin || hasVAction || hasFsHook { + comp.RegisterVerifierAction(round, &PreVortexVerifierStep{Ctxs: ctxs, Round: round}) + } - if hasColumn || hasQParams { - comp.RegisterProverAction(round, &PreVortexProverStep{Ctxs: ctxs, Round: round}) + if hasColumn || hasQParams { + comp.RegisterProverAction(round, &PreVortexProverStep{Ctxs: ctxs, Round: round}) + } } } - } return def, &ctxs } @@ -205,6 +205,8 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { NumCols: srcVortexCtx.NumCols, MaxCommittedRound: srcVortexCtx.MaxCommittedRound, NumOpenedCol: srcVortexCtx.NumOpenedCol, + VortexParams: srcVortexCtx.VortexParams, + SisParams: srcVortexCtx.SisParams, CommitmentsByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.CommitmentsByRounds), DriedByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.DriedByRounds), PolynomialsTouchedByTheQuery: ctx.Translator.TranslateColumnSet(srcVortexCtx.PolynomialsTouchedByTheQuery), @@ -216,10 +218,12 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { panic("it should not replace by MiMC") } - dstVortexCtx.Items.Precomputeds.PrecomputedColums = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Precomputeds.PrecomputedColums) - dstVortexCtx.Items.Precomputeds.MerkleRoot = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.MerkleRoot.GetColID()) - dstVortexCtx.Items.Precomputeds.CommittedMatrix = srcVortexCtx.Items.Precomputeds.CommittedMatrix - dstVortexCtx.Items.Precomputeds.DhWithMerkle = srcVortexCtx.Items.Precomputeds.DhWithMerkle + if srcVortexCtx.IsCommitToPrecomputed() { + dstVortexCtx.Items.Precomputeds.PrecomputedColums = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Precomputeds.PrecomputedColums) + dstVortexCtx.Items.Precomputeds.MerkleRoot = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.MerkleRoot.GetColID()) + dstVortexCtx.Items.Precomputeds.CommittedMatrix = srcVortexCtx.Items.Precomputeds.CommittedMatrix + dstVortexCtx.Items.Precomputeds.DhWithMerkle = srcVortexCtx.Items.Precomputeds.DhWithMerkle + } dstVortexCtx.Items.Alpha = ctx.Translator.InsertCoin(srcVortexCtx.Items.Alpha) dstVortexCtx.Items.Ualpha = ctx.Translator.InsertColumn(srcVortexCtx.Items.Ualpha.(column.Natural)) From d041d3ad576a3ec161652d520d2f229e1b7162ab Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 12:53:02 +0100 Subject: [PATCH 23/63] implements a tests for pure vortex protocol --- .../conglomeration/conglomeration_test.go | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 prover/protocol/distributed/conglomeration/conglomeration_test.go diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go new file mode 100644 index 00000000000..25cf659ecf6 --- /dev/null +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -0,0 +1,67 @@ +package conglomeration + +import ( + "testing" + + "github.com/consensys/linea-monorepo/prover/crypto/ringsis" + "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" + "github.com/consensys/linea-monorepo/prover/protocol/ifaces" + "github.com/consensys/linea-monorepo/prover/protocol/query" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/stretchr/testify/require" +) + +func TestConglomerationPureVortex(t *testing.T) { + + var ( + numCol = 16 + numRow = 16 + numProof = 16 + a []ifaces.Column + u query.UnivariateEval + sisParams = &ringsis.Params{LogTwoBound: 16, LogTwoDegree: 1} + vortexCompFunc = vortex.Compile(2, vortex.WithSISParams(sisParams), vortex.ForceNumOpenedColumns(2)) + ) + + define := func(builder *wizard.Builder) { + for i := 0; i < numCol; i++ { + a = append(a, builder.RegisterCommit(ifaces.ColIDf("a-%v", i), numRow)) + } + u = builder.CompiledIOP.InsertUnivariate(0, "u", a) + } + + prover := func(k int) func(run *wizard.ProverRuntime) { + return func(run *wizard.ProverRuntime) { + ys := make([]field.Element, len(a)) + for i := range a { + y := field.NewElement(uint64(i + k)) + run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numCol)) + ys = append(ys, y) + } + run.AssignUnivariate(u.QueryID, field.NewElement(0), ys...) + } + } + + var ( + tmpl = wizard.Compile(define, vortexCompFunc) + congDef, ctxsPHolder = ConglomerateDefineFunc(tmpl, numProof) + cong = wizard.Compile(congDef, dummy.CompileAtProverLvl) + ctxs = *ctxsPHolder + lastRound = ctxs[0].LastRound + ) + + witnesses := make([]Witness, numProof) + for i := range witnesses { + runtime := wizard.RunProverUntilRound(tmpl, prover(i), lastRound+1) + witnesses[i] = ExtractWitness(runtime) + } + + proof := wizard.Prove(cong, ProveConglomeration(ctxs, witnesses)) + err := wizard.Verify(cong, proof) + + require.NoError(t, err) + +} From 01b335e9a94d2e65e5a8e62010e93b28765afca4 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 13:11:16 +0100 Subject: [PATCH 24/63] feat(recursion): prefixs all the items of the self-recursion --- .../compiler/selfrecursion/column_opening.go | 1 + .../compiler/selfrecursion/context.go | 17 ++++- .../protocol/compiler/selfrecursion/names.go | 72 +++++++++++++------ .../compiler/selfrecursion/selfrecursion.go | 4 +- prover/protocol/dedicated/mimc/linear_hash.go | 32 +++++---- .../dedicated/mimc/linear_hash_test.go | 2 +- .../conglomeration/conglomeration.go | 2 +- 7 files changed, 89 insertions(+), 41 deletions(-) diff --git a/prover/protocol/compiler/selfrecursion/column_opening.go b/prover/protocol/compiler/selfrecursion/column_opening.go index e856b68ce33..8c90a1d0963 100644 --- a/prover/protocol/compiler/selfrecursion/column_opening.go +++ b/prover/protocol/compiler/selfrecursion/column_opening.go @@ -366,6 +366,7 @@ func (ctx *SelfRecursionCtx) linearHashAndMerkle() { // And the linear hashing mimcW.CheckLinearHash( ctx.comp, + ctx.linearHashVerificationName(), ctx.Columns.ConcatenatedDhQ, ctx.VortexCtx.SisParams.OutputSize(), leavesSizeUnpadded, diff --git a/prover/protocol/compiler/selfrecursion/context.go b/prover/protocol/compiler/selfrecursion/context.go index 10d1fcb4e8e..f0652fc917b 100644 --- a/prover/protocol/compiler/selfrecursion/context.go +++ b/prover/protocol/compiler/selfrecursion/context.go @@ -25,6 +25,11 @@ type SelfRecursionCtx struct { // step. SelfRecursionCnt int + // NamePrefix is a prefix for the names of the items generated by + // the current self-recursion compilation context. Leaving it as + // empty means that no prefix is used. + NamePrefix string + // Accessors Accessors struct { // (EvalBivariate) @@ -200,12 +205,13 @@ type SelfRecursionCtx struct { // NewRecursionCtx returns a new recursion context taking a specified // [vortex.Ctx] and SelfRecursionCount. It can be used for custom use // of the recursion wizard. -func NewRecursionCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx) SelfRecursionCtx { +func NewRecursionCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx, prefix string) SelfRecursionCtx { ctx := SelfRecursionCtx{ comp: comp, VortexCtx: vortexCtx, SelfRecursionCnt: comp.SelfRecursionCount, + NamePrefix: prefix, } // Transport the compilation items of the vortex context into @@ -282,7 +288,7 @@ func NewRecursionCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx) SelfRecurs func NewSelfRecursionCxt(comp *wizard.CompiledIOP) SelfRecursionCtx { // Extract the vortex context from the compiledIOP though the "Pcs" vortexCtx := assertVortexCompiled(comp) - return NewRecursionCtx(comp, vortexCtx) + return NewRecursionCtx(comp, vortexCtx, "") } // Asserts that the compiled IOP has the appropriate cryptographic context @@ -318,3 +324,10 @@ func assertVortexCompiled(comp *wizard.CompiledIOP) *vortex.Ctx { func (ctx *SelfRecursionCtx) SisKey() *ringsis.Key { return &ctx.VortexCtx.VortexParams.Key } + +// isPrefixed returns whether a prefix is attached to the inner +// vortex Ctx. If defined, the prefix is used to prefix the +// column names. +func (ctx *SelfRecursionCtx) isPrefixed() bool { + return len(ctx.VortexCtx.RunStateNamePrefix) > 0 +} diff --git a/prover/protocol/compiler/selfrecursion/names.go b/prover/protocol/compiler/selfrecursion/names.go index ffb8bcba709..8fcc66112d3 100644 --- a/prover/protocol/compiler/selfrecursion/names.go +++ b/prover/protocol/compiler/selfrecursion/names.go @@ -12,7 +12,8 @@ import ( // Name of the polynomial I(x) func (ctx *SelfRecursionCtx) iName(length int) ifaces.ColID { - return ifaces.ColIDf("PRECOMPUTED_%v_I_%v", ctx.SelfRecursionCnt, length) + name := ifaces.ColIDf("PRECOMPUTED_%v_I_%v", ctx.SelfRecursionCnt, length) + return maybePrefix(ctx, name) } // Name of the aH polynomials @@ -23,98 +24,129 @@ func (ctx *SelfRecursionCtx) ahName(key *ringsis.Key, start, length, maxSize int } subName := ifaces.ColIDf("SISKEY_%v_%v_%v", key.LogTwoBound, key.LogTwoDegree, key.MaxNumFieldHashable()) - return ifaces.ColIDf("%v_%v_%v_%v", subName, start, length, maxSize) + name := ifaces.ColIDf("%v_%v_%v_%v", subName, start, length, maxSize) + return maybePrefix(ctx, name) } // Name of the preimage in limb expanded. nameWhole is the name of the // associated column without limb expansion. func (ctx *SelfRecursionCtx) limbExpandedPreimageName(nameWhole ifaces.ColID) ifaces.ColID { - return ifaces.ColIDf("%v_LIMB_EXPANDED_%v", nameWhole, ctx.SelfRecursionCnt) + name := ifaces.ColIDf("%v_LIMB_EXPANDED_%v", nameWhole, ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the UalphaQ column func (ctx *SelfRecursionCtx) uAlphaQName() ifaces.ColID { - return ifaces.ColIDf("SELFRECURSION_U_ALPHA_Q_%v", ctx.SelfRecursionCnt) + name := ifaces.ColIDf("SELFRECURSION_U_ALPHA_Q_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the self-recursed inclusion query func (ctx *SelfRecursionCtx) selectQInclusion() ifaces.QueryID { - return ifaces.QueryIDf("SELFRECURSION_SELECT_Q_INCLUSION_%v", ctx.SelfRecursionCnt) + name := ifaces.QueryIDf("SELFRECURSION_SELECT_Q_INCLUSION_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the collapse coin func (ctx *SelfRecursionCtx) collapseCoin() coin.Name { - return coin.Namef("SELFRECURSION_COLLAPSE_COIN_%v", ctx.SelfRecursionCnt) + name := coin.Namef("SELFRECURSION_COLLAPSE_COIN_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name for the coeff eval for consistency check between Ualphaq // and the preimage (left-side, over UalphaA) func (ctx *SelfRecursionCtx) constencyUalphaQPreimageLeft() string { - return fmt.Sprintf("SELFRECURSION_CONSISTENCY_UALPHA_PREIMAGE_LEFT_%v", ctx.SelfRecursionCnt) + name := fmt.Sprintf("SELFRECURSION_CONSISTENCY_UALPHA_PREIMAGE_LEFT_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name for the coeff eval for consistency check between Ualphaq // and the preimage (right-side, over preiimages) func (ctx *SelfRecursionCtx) constencyUalphaQPreimageRight() string { - return fmt.Sprintf("SELFRECURSION_CONSISTENCY_UALPHA_PREIMAGE_RIGHT_%v", ctx.SelfRecursionCnt) + name := fmt.Sprintf("SELFRECURSION_CONSISTENCY_UALPHA_PREIMAGE_RIGHT_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of Edual func (ctx *SelfRecursionCtx) eDual() ifaces.ColID { - return ifaces.ColIDf("SELFRECURSION_E_DUAL_%v", ctx.SelfRecursionCnt) + name := ifaces.ColIDf("SELFRECURSION_E_DUAL_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name for the fold coin func (ctx *SelfRecursionCtx) foldCoinName() coin.Name { - return coin.Namef("SELFRECURSION_FOLD_COIN_%v", ctx.SelfRecursionCnt) + name := coin.Namef("SELFRECURSION_FOLD_COIN_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the inner product between between PreimageCollapseFold and ACollapseFold func (ctx *SelfRecursionCtx) preimagesAndAmergeIP() ifaces.QueryID { - return ifaces.QueryIDf("SELFRECURSION_PREIMAGE_A_IP_%v", ctx.SelfRecursionCnt) + name := ifaces.QueryIDf("SELFRECURSION_PREIMAGE_A_IP_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the interpolation context for Ualpha func (ctx *SelfRecursionCtx) interpolateUAlphaX() string { - return fmt.Sprintf("SELFRECURSION_INTERPOLATE_UALPHA_X_%v", ctx.SelfRecursionCnt) + name := fmt.Sprintf("SELFRECURSION_INTERPOLATE_UALPHA_X_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the concatenation of the DhQs func (ctx *SelfRecursionCtx) concatenatedDhQ() ifaces.ColID { - return ifaces.ColIDf("SELFRECURSION_CONCAT_DHQ_%v", ctx.SelfRecursionCnt) + name := ifaces.ColIDf("SELFRECURSION_CONCAT_DHQ_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the MerkleLeaves func (ctx *SelfRecursionCtx) merkleLeavesName() ifaces.ColID { - return ifaces.ColIDf("SELFRECURSION_MERKLE_LEAVES_%v", ctx.SelfRecursionCnt) + name := ifaces.ColIDf("SELFRECURSION_MERKLE_LEAVES_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the MerklePositions func (ctx *SelfRecursionCtx) merklePositionssName() ifaces.ColID { - return ifaces.ColIDf("SELFRECURSION_MERKLE_POSITIONS_%v", ctx.SelfRecursionCnt) + name := ifaces.ColIDf("SELFRECURSION_MERKLE_POSITIONS_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the MerkleRoots func (ctx *SelfRecursionCtx) merkleRootsName() ifaces.ColID { - return ifaces.ColIDf("SELFRECURSION_MERKLE_ROOTS_%v", ctx.SelfRecursionCnt) + name := ifaces.ColIDf("SELFRECURSION_MERKLE_ROOTS_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the Merkle proof verification func (ctx *SelfRecursionCtx) merkleProofVerificationName() string { - return fmt.Sprintf("SELFRECURSION_MERKLE_%v", ctx.SelfRecursionCnt) + name := fmt.Sprintf("SELFRECURSION_MERKLE_%v", ctx.SelfRecursionCnt) + return maybePrefix(ctx, name) } // Name of the collapsed key func (ctx *SelfRecursionCtx) aCollapsedName() string { - return fmt.Sprintf("SELFRECURSION_ACOLLAPSE_%v", ctx.comp.SelfRecursionCount) + name := fmt.Sprintf("SELFRECURSION_ACOLLAPSE_%v", ctx.comp.SelfRecursionCount) + return maybePrefix(ctx, name) } // Name of the collapsed key func (ctx *SelfRecursionCtx) rootHasGlue() ifaces.QueryID { - return ifaces.QueryIDf("SELFRECURSION_ROOT_HASH_GLUE_%v", ctx.comp.SelfRecursionCount) + name := ifaces.QueryIDf("SELFRECURSION_ROOT_HASH_GLUE_%v", ctx.comp.SelfRecursionCount) + return maybePrefix(ctx, name) } // Positions glue func (ctx *SelfRecursionCtx) positionGlue() ifaces.QueryID { - return ifaces.QueryIDf("SELFRECURSION_POSITION_GLUE_%v", ctx.comp.SelfRecursionCount) + name := ifaces.QueryIDf("SELFRECURSION_POSITION_GLUE_%v", ctx.comp.SelfRecursionCount) + return maybePrefix(ctx, name) +} + +// linearHashVerificatioName returns the name passed to the wizard helper building the +// linear hash verifier. +func (ctx *SelfRecursionCtx) linearHashVerificationName() string { + name := fmt.Sprintf("SELFRECURSION_LINEAR_HASH_VERIFICATION_%v", ctx.comp.SelfRecursionCount) + return maybePrefix(ctx, name) +} + +// maybePrefix adds the prefix if defined in the context +func maybePrefix[T ~string](ctx *SelfRecursionCtx, name T) T { + return T(ctx.NamePrefix+".") + name } diff --git a/prover/protocol/compiler/selfrecursion/selfrecursion.go b/prover/protocol/compiler/selfrecursion/selfrecursion.go index e4805601ad8..bc38923b527 100644 --- a/prover/protocol/compiler/selfrecursion/selfrecursion.go +++ b/prover/protocol/compiler/selfrecursion/selfrecursion.go @@ -23,8 +23,8 @@ func SelfRecurse(comp *wizard.CompiledIOP) { // RecurseOverCustomCtx applies the same compilation steps as [SelfRecurse] // over a specified vortex compilation context. -func RecurseOverCustomCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx) { - ctx := NewRecursionCtx(comp, vortexCtx) +func RecurseOverCustomCtx(comp *wizard.CompiledIOP, vortexCtx *vortex.Ctx, prefix string) { + ctx := NewRecursionCtx(comp, vortexCtx, prefix) ctx.Precomputations() ctx.RowLinearCombinationPhase() ctx.ColumnOpeningPhase() diff --git a/prover/protocol/dedicated/mimc/linear_hash.go b/prover/protocol/dedicated/mimc/linear_hash.go index 24b01abd3b5..3d1d75cfd8b 100644 --- a/prover/protocol/dedicated/mimc/linear_hash.go +++ b/prover/protocol/dedicated/mimc/linear_hash.go @@ -24,7 +24,7 @@ type linearHashCtx struct { // The compiled IOP comp *wizard.CompiledIOP - // Names of the "data" columns + name string // Output column, which containing the result each // individual hash. @@ -60,6 +60,7 @@ Check a linear hashby chunk of columns */ func CheckLinearHash( comp *wizard.CompiledIOP, + name string, tohash ifaces.Column, period int, numHash int, expectedHashes ifaces.Column, @@ -68,6 +69,7 @@ func CheckLinearHash( // Initialize the context ctx := linearHashCtx{ comp: comp, + name: name, ToHash: tohash, Period: period, NumHash: numHash, @@ -83,7 +85,7 @@ func CheckLinearHash( if ctx.IsFullyActive { selector.CheckSubsample( comp, - prefixWithLinearHash(comp, "RES_EXTRACTION"), + prefixWithLinearHash(comp, name, "RES_EXTRACTION"), []ifaces.Column{ctx.NewStateClean}, []ifaces.Column{ctx.ExpectedHash}, period-1, @@ -91,7 +93,7 @@ func CheckLinearHash( } else { ctx.comp.InsertInclusion( ctx.Round, - ifaces.QueryID(prefixWithLinearHash(comp, "RESULT_CHECK_%v", tohash.GetColID())), + ifaces.QueryID(prefixWithLinearHash(comp, name, "RESULT_CHECK_%v", tohash.GetColID())), []ifaces.Column{ctx.IsEndOfHash, ctx.NewStateClean}, []ifaces.Column{ctx.IsActiveExpected(), ctx.ExpectedHash}, ) @@ -99,9 +101,9 @@ func CheckLinearHash( } -func prefixWithLinearHash(comp *wizard.CompiledIOP, msg string, args ...any) string { - args = append([]any{comp.SelfRecursionCount}, args...) - return fmt.Sprintf("LINEAR_HASH_%v_"+msg, args...) +func prefixWithLinearHash(comp *wizard.CompiledIOP, name, msg string, args ...any) string { + args = append([]any{name, comp.SelfRecursionCount}, args...) + return fmt.Sprintf("%v.LINEAR_HASH_%v_"+msg, args...) } // Declares assign and constraints the columns OldStates and NewStates @@ -110,19 +112,19 @@ func (ctx *linearHashCtx) HashingCols() { // Registers the old states columns ctx.OldState = ctx.comp.InsertCommit( ctx.Round, - ifaces.ColID(prefixWithLinearHash(ctx.comp, "OLD_STATE_%v", ctx.ToHash.GetColID())), + ifaces.ColID(prefixWithLinearHash(ctx.comp, ctx.name, "OLD_STATE_%v", ctx.ToHash.GetColID())), ctx.ToHash.Size(), ) ctx.NewState = ctx.comp.InsertCommit( ctx.Round, - ifaces.ColID(prefixWithLinearHash(ctx.comp, "NEW_STATE_%v", ctx.ToHash.GetColID())), + ifaces.ColID(prefixWithLinearHash(ctx.comp, ctx.name, "NEW_STATE_%v", ctx.ToHash.GetColID())), ctx.ToHash.Size(), ) ctx.NewStateClean = ctx.comp.InsertCommit( ctx.Round, - ifaces.ColIDf(prefixWithLinearHash(ctx.comp, "NEW_STATE_CLEAN_%v", ctx.ToHash.GetColID())), + ifaces.ColIDf(prefixWithLinearHash(ctx.comp, ctx.name, "NEW_STATE_CLEAN_%v", ctx.ToHash.GetColID())), ctx.ToHash.Size(), ) @@ -179,7 +181,7 @@ func (ctx *linearHashCtx) HashingCols() { ctx.comp.InsertGlobal( ctx.Round, - ifaces.QueryID(prefixWithLinearHash(ctx.comp, "STATE_PROPAGATION_%v", ctx.ToHash.GetColID())), + ifaces.QueryID(prefixWithLinearHash(ctx.comp, ctx.name, "STATE_PROPAGATION_%v", ctx.ToHash.GetColID())), expr, true, // no bound cancel to also enforce the first value of old state to be zero ) @@ -189,7 +191,7 @@ func (ctx *linearHashCtx) HashingCols() { // ctx.comp.InsertGlobal( ctx.Round, - ifaces.QueryIDf(prefixWithLinearHash(ctx.comp, "CLEAN_NEW_STATE_%v", ctx.ToHash.GetColID())), + ifaces.QueryIDf(prefixWithLinearHash(ctx.comp, ctx.name, "CLEAN_NEW_STATE_%v", ctx.ToHash.GetColID())), ctx.IsActiveVar(). Mul(ifaces.ColumnAsVariable(ctx.NewState)). Sub(ifaces.ColumnAsVariable(ctx.NewStateClean)), @@ -200,7 +202,7 @@ func (ctx *linearHashCtx) HashingCols() { // ctx.comp.InsertMiMC( ctx.Round, - ifaces.QueryID(prefixWithLinearHash(ctx.comp, "BLOCKS_COMPRESSION_%v", ctx.ToHash.GetColID())), + ifaces.QueryID(prefixWithLinearHash(ctx.comp, ctx.name, "BLOCKS_COMPRESSION_%v", ctx.ToHash.GetColID())), ctx.ToHash, ctx.OldState, ctx.NewState, ) @@ -227,7 +229,7 @@ func (ctx *linearHashCtx) IsActiveExpected() ifaces.Column { } ctx.isActiveExpected = ctx.comp.InsertPrecomputed( - ifaces.ColIDf(prefixWithLinearHash(ctx.comp, "IS_ACTIVE_EXPECTED_%v", ctx.ToHash.GetColID())), + ifaces.ColIDf(prefixWithLinearHash(ctx.comp, ctx.name, "IS_ACTIVE_EXPECTED_%v", ctx.ToHash.GetColID())), assignment, ) } @@ -244,7 +246,7 @@ func (ctx *linearHashCtx) IsActiveVar() *symbolic.Expression { // Lazily registers the columns if ctx.IsActiveLarge == nil { ctx.IsActiveLarge = ctx.comp.InsertPrecomputed( - ifaces.ColIDf(prefixWithLinearHash(ctx.comp, "IS_ACTIVE_%v", ctx.ToHash.GetColID())), + ifaces.ColIDf(prefixWithLinearHash(ctx.comp, ctx.name, "IS_ACTIVE_%v", ctx.ToHash.GetColID())), smartvectors.RightZeroPadded( vector.Repeat(field.One(), ctx.NumHash*ctx.Period), ctx.ToHash.Size(), @@ -272,7 +274,7 @@ func (ctx *linearHashCtx) IsEndOfHashVar() *symbolic.Expression { } ctx.IsEndOfHash = ctx.comp.InsertPrecomputed( - ifaces.ColIDf(prefixWithLinearHash(ctx.comp, "IS_END_OF_HASH_%v", ctx.ToHash.GetColID())), + ifaces.ColIDf(prefixWithLinearHash(ctx.comp, ctx.name, "IS_END_OF_HASH_%v", ctx.ToHash.GetColID())), smartvectors.RightZeroPadded(window, ctx.ToHash.Size()), ) } diff --git a/prover/protocol/dedicated/mimc/linear_hash_test.go b/prover/protocol/dedicated/mimc/linear_hash_test.go index 95bb2dc8124..47ab9b31103 100644 --- a/prover/protocol/dedicated/mimc/linear_hash_test.go +++ b/prover/protocol/dedicated/mimc/linear_hash_test.go @@ -38,7 +38,7 @@ func TestLinearHash(t *testing.T) { define := func(b *wizard.Builder) { tohash = b.RegisterCommit("TOHASH", numRowLarge) expectedhash = b.RegisterCommit("HASHED", numRowSmall) - linhash.CheckLinearHash(b.CompiledIOP, tohash, period, numhash, expectedhash) + linhash.CheckLinearHash(b.CompiledIOP, "test", tohash, period, numhash, expectedhash) } prove := func(run *wizard.ProverRuntime) { diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index d77c8449a0e..2b55156c607 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -47,7 +47,7 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu ctx := initRecursionCtx(prefix, comp, tmpl) ctx.captureCompPreVortex(tmpl) ctx.captureVortexCtx(tmpl) - selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx) + selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx, ctx.Translator.Prefix) ctxs = append(ctxs, ctx) } From 819e3903c6df91b55daf759b7f3f227f694a4926 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 13:26:24 +0100 Subject: [PATCH 25/63] fixup(test): fix appending to presized slice instead of len 0 --- .../protocol/distributed/conglomeration/conglomeration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index 25cf659ecf6..5f7c9c14319 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -35,7 +35,7 @@ func TestConglomerationPureVortex(t *testing.T) { prover := func(k int) func(run *wizard.ProverRuntime) { return func(run *wizard.ProverRuntime) { - ys := make([]field.Element, len(a)) + ys := make([]field.Element, 0, len(a)) for i := range a { y := field.NewElement(uint64(i + k)) run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numCol)) From 3e08235a63a7618500057b18f956a7920350527e Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 13:27:54 +0100 Subject: [PATCH 26/63] fixup(extract): check for nilness before casting to type --- .../protocol/distributed/conglomeration/prover.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 714a6786e25..9a8422a9e43 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -115,9 +115,17 @@ func ExtractWitness(run *wizard.ProverRuntime) Witness { tree, _ = run.State.TryGet(pcs.MerkleTreeName(round)) ) - committedMatrices = append(committedMatrices, committedMatrix.(vCom.EncodedMatrix)) - sisHashes = append(sisHashes, sisHash.([]field.Element)) - trees = append(trees, tree.(*smt.Tree)) + if committedMatrix != nil { + committedMatrices = append(committedMatrices, committedMatrix.(vCom.EncodedMatrix)) + } + + if sisHash != nil { + sisHashes = append(sisHashes, sisHash.([]field.Element)) + } + + if tree != nil { + trees = append(trees, tree.(*smt.Tree)) + } } return Witness{ From 16d8e288fb080dc4fb3d64b6e13e6bb72909176f Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:41:52 +0100 Subject: [PATCH 27/63] clean(vortex): make the vortex compiler rely on items.Name instead of regenerating the names --- prover/protocol/compiler/vortex/prover.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prover/protocol/compiler/vortex/prover.go b/prover/protocol/compiler/vortex/prover.go index 2c0879b7fbf..f897ff3e33b 100644 --- a/prover/protocol/compiler/vortex/prover.go +++ b/prover/protocol/compiler/vortex/prover.go @@ -67,11 +67,11 @@ func (ctx *Ctx) ComputeLinearComb(pr *wizard.ProverRuntime) { } // And get the randomness - randomCoinLC := pr.GetRandomCoinField(ctx.LinCombRandCoinName()) + randomCoinLC := pr.GetRandomCoinField(ctx.Items.Alpha.Name) // and compute and assign the random linear combination of the rows proof := ctx.VortexParams.InitOpeningWithLC(committedSV, randomCoinLC) - pr.AssignColumn(ctx.LinCombName(), proof.LinearCombination) + pr.AssignColumn(ctx.Items.Ualpha.GetColID(), proof.LinearCombination) } // Prover steps of Vortex where he opens the columns selected by the verifier @@ -107,7 +107,7 @@ func (ctx *Ctx) OpenSelectedColumns(pr *wizard.ProverRuntime) { trees = append(trees, tree) } - entryList := pr.GetRandomCoinIntegerVec(ctx.RandColSelectionName()) + entryList := pr.GetRandomCoinIntegerVec(ctx.Items.Q.Name) proof := vortex.OpeningProof{} // Merkle mode only: @@ -131,7 +131,7 @@ func (ctx *Ctx) OpenSelectedColumns(pr *wizard.ProverRuntime) { assignable = smartvectors.RightZeroPadded(fullCol, utils.NextPowerOfTwo(len(fullCol))) } - pr.AssignColumn(ctx.SelectedColName(j), assignable) + pr.AssignColumn(ctx.Items.OpenedColumns[j].GetColID(), assignable) } packedMProofs := ctx.packMerkleProofs(proof.MerkleProofs) From 0b410bbeaf472db7ecd82885757c8eddcf59547b Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:43:05 +0100 Subject: [PATCH 28/63] fix(vortex): make the prefix be used when len>0 and not len==0 --- prover/protocol/compiler/vortex/names.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prover/protocol/compiler/vortex/names.go b/prover/protocol/compiler/vortex/names.go index cd407cd9250..ced90f744b7 100644 --- a/prover/protocol/compiler/vortex/names.go +++ b/prover/protocol/compiler/vortex/names.go @@ -36,7 +36,7 @@ func (ctx *Ctx) CommitmentName(round int) ifaces.ColID { // for each round that we store in the state. func (ctx *Ctx) SisHashName(round int) string { name := fmt.Sprintf("VORTEX_%v_SIS_HASH_%v", ctx.SelfRecursionCount, round) - if len(ctx.RunStateNamePrefix) > 0 { + if len(ctx.RunStateNamePrefix) == 0 { return name } return ctx.RunStateNamePrefix + "." + name @@ -45,7 +45,7 @@ func (ctx *Ctx) SisHashName(round int) string { // returns the name of a prover state for a given round of Vortex func (ctx *Ctx) VortexProverStateName(round int) string { name := fmt.Sprintf("VORTEX_%v_PROVER_STATE_%v", ctx.SelfRecursionCount, round) - if len(ctx.RunStateNamePrefix) > 0 { + if len(ctx.RunStateNamePrefix) == 0 { return name } return ctx.RunStateNamePrefix + "." + name @@ -54,7 +54,7 @@ func (ctx *Ctx) VortexProverStateName(round int) string { // returns the name of a prover state for a given round of Vortex func (ctx *Ctx) MerkleTreeName(round int) string { name := fmt.Sprintf("VORTEX_%v_MERKLE_TREE_%v", ctx.SelfRecursionCount, round) - if len(ctx.RunStateNamePrefix) > 0 { + if len(ctx.RunStateNamePrefix) == 0 { return name } return ctx.RunStateNamePrefix + "." + name From f893be90a7ce638eacd6deedefe5a30ccef33f64 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:43:54 +0100 Subject: [PATCH 29/63] doc(conglo): adds doc for ColumnIgnored --- .../distributed/conglomeration/conglomeration.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 2b55156c607..488f2e5c6e7 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -23,12 +23,11 @@ type recursionCtx struct { NonEmptyMerkleRootPositions []int FirstRound, LastRound int Columns [][]ifaces.Column - ColumnsIgnored [][]ifaces.Column - QueryParams [][]ifaces.Query - VerifierActions [][]wizard.VerifierAction - Coins [][]coin.Info - FsHooks [][]wizard.VerifierAction - LocalOpenings []query.LocalOpening + // The columns ignored are the one that are compiled by the vortex context. + // They are added in the target 'comp' but are assigned to zero. Although, + // they do not directly play a role in the protocol anymore, they are still + // referenced by the self-recursion compiler. + ColumnsIgnored [][]ifaces.Column } // ConglomerateDefineFunc returns a function that defines a conglomerate From 89e9854b7eb92b9ea41838fc82294cfbf7e43e4e Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:45:06 +0100 Subject: [PATCH 30/63] fixup(conglo): adds the recursive compilation at the end --- .../conglomeration/conglomeration.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 488f2e5c6e7..9489c0cbcd7 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -28,6 +28,11 @@ type recursionCtx struct { // they do not directly play a role in the protocol anymore, they are still // referenced by the self-recursion compiler. ColumnsIgnored [][]ifaces.Column + QueryParams [][]ifaces.Query + VerifierActions [][]wizard.VerifierAction + Coins [][]coin.Info + FsHooks [][]wizard.VerifierAction + LocalOpenings []query.LocalOpening } // ConglomerateDefineFunc returns a function that defines a conglomerate @@ -46,7 +51,6 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu ctx := initRecursionCtx(prefix, comp, tmpl) ctx.captureCompPreVortex(tmpl) ctx.captureVortexCtx(tmpl) - selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx, ctx.Translator.Prefix) ctxs = append(ctxs, ctx) } @@ -68,6 +72,16 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu comp.RegisterProverAction(round, &PreVortexProverStep{Ctxs: ctxs, Round: round}) } } + + comp.RegisterProverAction(ctxs[0].LastRound, &AssignVortexQuery{Ctxs: ctxs}) + comp.RegisterProverAction(ctxs[0].LastRound+1, &AssignVortexUAlpha{Ctxs: ctxs}) + comp.RegisterProverAction(ctxs[0].LastRound+2, &AssignVortexOpenedCols{Ctxs: ctxs}) + + // Importantly, the recursion compilation should happen after we added the vortex + // columns as they depends on the later. + for _, ctx := range ctxs { + selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx, ctx.Translator.Prefix) + } } return def, &ctxs @@ -192,7 +206,8 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { ctx.ColumnsIgnored = append(ctx.ColumnsIgnored, nil) for _, comID := range coms { com := tmpl.Columns.GetHandle(comID) - ctx.Translator.InsertColumn(com.(column.Natural)) + com = ctx.Translator.InsertColumn(com.(column.Natural)) + ctx.ColumnsIgnored[len(ctx.ColumnsIgnored)-1] = append(ctx.ColumnsIgnored[len(ctx.ColumnsIgnored)-1], com) } } From 4b2f06bcabc5a4c7d18d3591db6dea71a425ce73 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:45:51 +0100 Subject: [PATCH 31/63] fixup(conglo): assigns the ignored columns --- prover/protocol/distributed/conglomeration/prover.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 9a8422a9e43..d01c55baa13 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -144,6 +144,7 @@ func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { proof = run.State.MustGet(prefix + subProofInStatePrefixStr).(wizard.Proof) queriesParams = ctx.QueryParams[pa.Round] colums = ctx.Columns[pa.Round] + columnIgnored = ctx.ColumnsIgnored[pa.Round] ) for _, col := range colums { @@ -151,6 +152,10 @@ func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { run.AssignColumn(col.GetColID(), proof.Messages.MustGet(name)) } + for _, col := range columnIgnored { + run.AssignColumn(col.GetColID(), smartvectors.NewConstant(field.Zero(), col.Size())) + } + for _, param := range queriesParams { name := unprefix(prefix, param.Name()) run.QueriesParams.InsertNew(param.Name(), proof.QueriesParams.MustGet(name)) From 167a51bcb7a60e66658604845d0e15f0e633d090 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:46:27 +0100 Subject: [PATCH 32/63] fixup(prover): safeguard check for the witness assignment --- prover/protocol/distributed/conglomeration/prover.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index d01c55baa13..01d95bf52d1 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -82,15 +82,15 @@ func storeWitnessInState(run *wizard.ProverRuntime, ctx *recursionCtx, witness W for round := 0; round <= lastRound; round++ { - if witness.CommittedMatrices[round] != nil { + if len(witness.CommittedMatrices) > round && witness.CommittedMatrices[round] != nil { run.State.InsertNew(ctx.PcsCtx.VortexProverStateName(round), witness.CommittedMatrices[round]) } - if witness.SisHashes[round] != nil { + if len(witness.SisHashes) > round && witness.SisHashes[round] != nil { run.State.InsertNew(ctx.PcsCtx.SisHashName(round), witness.SisHashes[round]) } - if witness.Trees[round] != nil { + if len(witness.Trees) > round && witness.Trees[round] != nil { run.State.InsertNew(ctx.PcsCtx.MerkleTreeName(round), witness.Trees[round]) } } From aec9f9f3ab435bf93186b8cb96ef5009b8054fce Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:47:14 +0100 Subject: [PATCH 33/63] fixup(conglo): adds a prover step to assign the vortex query --- .../distributed/conglomeration/prover.go | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 01d95bf52d1..57d36d5a665 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -5,6 +5,7 @@ import ( "github.com/consensys/linea-monorepo/prover/crypto/state-management/smt" vCom "github.com/consensys/linea-monorepo/prover/crypto/vortex" + "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -34,20 +35,23 @@ type PreVortexProverStep struct { Round int } +// AssignVortexQuery assigns the query for all the subproofs. +type AssignVortexQuery struct { + Ctxs []*recursionCtx +} + // AssignVortexUAlpha assigns the UAlpha column for all the subproofs. As // for [PreVortexVerifierStep], this step should be run after the corresponding // proofs have been added to the runtime states. type AssignVortexUAlpha struct { - Ctxs []*recursionCtx - Round int + Ctxs []*recursionCtx } // AssignVortexOpenedCols assigns the OpenedCols for all the subproofs. As // for [PreVortexVerifierStep], this step should be run after the corresponding // proofs have been added to the runtime states. type AssignVortexOpenedCols struct { - Ctxs []*recursionCtx - Round int + Ctxs []*recursionCtx } // ProveConglomeration returns the main prover step of the conglomeration wizard. @@ -163,6 +167,19 @@ func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { } } +func (pa AssignVortexQuery) Run(run *wizard.ProverRuntime) { + for _, ctx := range pa.Ctxs { + + var ( + prefix = ctx.Translator.Prefix + proof = run.State.MustGet(prefix + subProofInStatePrefixStr).(wizard.Proof) + name = unprefix(prefix, ctx.PcsCtx.Query.QueryID) + ) + + run.QueriesParams.InsertNew(ctx.PcsCtx.Query.QueryID, proof.QueriesParams.MustGet(name)) + } +} + func (pa AssignVortexUAlpha) Run(run *wizard.ProverRuntime) { for _, ctx := range pa.Ctxs { // Since all the context of the pcs is translated, this does not @@ -180,7 +197,7 @@ func (pa AssignVortexOpenedCols) Run(run *wizard.ProverRuntime) { } func unprefix[T ~string](prefix string, name T) T { - p, n := string(prefix), string(name) + p, n := string(prefix)+".", string(name) r := strings.TrimPrefix(n, p) return T(r) } From 2c9c48128ac0108b43bd8738bf64d61bcbda967e Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:47:44 +0100 Subject: [PATCH 34/63] fixup(extractProof); fix the extract proof function for conglomeration --- prover/protocol/wizard/prover.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index 5e948753474..ffeb156d590 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -196,24 +196,39 @@ func RunProverUntilRound(c *CompiledIOP, highLevelprover ProverStep, round int) return &runtime } -// ExtractProof extracts the proof from a [ProverRuntime] +// ExtractProof extracts the proof from a [ProverRuntime]. If the runtime has +// been obtained via a [RunProverUntilRound], then it may be the case that +// some columns have not been assigned at all. Those won't be included in the +// returned proof. func (run *ProverRuntime) ExtractProof() Proof { messages := collection.NewMapping[ifaces.ColID, ifaces.ColAssignment]() for _, name := range run.Spec.Columns.AllKeysProof() { + + cols := run.Spec.Columns.GetHandle(name) + if run.currRound < cols.Round() { + continue + } + messageValue := run.Columns.MustGet(name) messages.InsertNew(name, messageValue) } - // And also the public inputs for _, name := range run.Spec.Columns.AllKeysPublicInput() { messageValue := run.Columns.MustGet(name) messages.InsertNew(name, messageValue) } + queriesParams := collection.NewMapping[ifaces.QueryID, ifaces.QueryParams]() + for round := 0; round <= run.currRound; round++ { + for _, name := range run.Spec.QueriesParams.AllKeysAt(round) { + queriesParams.InsertNew(name, run.QueriesParams.MustGet(name)) + } + } + return Proof{ Messages: messages, - QueriesParams: run.QueriesParams, + QueriesParams: queriesParams, } } From cf2c4671cf5d1f3a8c8b547d8d0a57532358fde0 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 15:49:01 +0100 Subject: [PATCH 35/63] fixup(vortex): use the Merkle proof column name directly instead of regenerating it --- prover/protocol/compiler/vortex/prover.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/compiler/vortex/prover.go b/prover/protocol/compiler/vortex/prover.go index f897ff3e33b..6bcd1261df8 100644 --- a/prover/protocol/compiler/vortex/prover.go +++ b/prover/protocol/compiler/vortex/prover.go @@ -135,7 +135,7 @@ func (ctx *Ctx) OpenSelectedColumns(pr *wizard.ProverRuntime) { } packedMProofs := ctx.packMerkleProofs(proof.MerkleProofs) - pr.AssignColumn(ctx.MerkleProofName(), packedMProofs) + pr.AssignColumn(ctx.Items.MerkleProofs.GetColID(), packedMProofs) } // returns true if the round is dry (i.e, there is nothing to commit to) From ec7cf1e4e3ddb1e9f366d3875a29279a439afe88 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 17:34:04 +0100 Subject: [PATCH 36/63] rename(srec): left, right into more explicit names --- .../compiler/selfrecursion/column_opening.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/prover/protocol/compiler/selfrecursion/column_opening.go b/prover/protocol/compiler/selfrecursion/column_opening.go index 8c90a1d0963..a7eca71020e 100644 --- a/prover/protocol/compiler/selfrecursion/column_opening.go +++ b/prover/protocol/compiler/selfrecursion/column_opening.go @@ -405,7 +405,7 @@ func (ctx *SelfRecursionCtx) collapsingPhase() { // Consistency check between the collapsed preimage and UalphaQ { - left := functionals.CoeffEval( + uAlphaQEval := functionals.CoeffEval( ctx.comp, ctx.constencyUalphaQPreimageLeft(), ctx.Coins.Collapse, @@ -424,7 +424,7 @@ func (ctx *SelfRecursionCtx) collapsingPhase() { evaluation point, we get a bivariate polynomial evaluation */ - right := functionals.EvalCoeffBivariate( + preImageEval := functionals.EvalCoeffBivariate( ctx.comp, ctx.constencyUalphaQPreimageRight(), ctx.Columns.PreimagesCollapse, @@ -435,12 +435,12 @@ func (ctx *SelfRecursionCtx) collapsingPhase() { ) ctx.comp.InsertVerifier( - left.Round(), + uAlphaQEval.Round(), func(run wizard.Runtime) error { - if left.GetVal(run) != right.GetVal(run) { - l, r := left.GetVal(run), right.GetVal(run) + if uAlphaQEval.GetVal(run) != preImageEval.GetVal(run) { + l, r := uAlphaQEval.GetVal(run), preImageEval.GetVal(run) return fmt.Errorf("consistency between u_alpha and the preimage: "+ - "mismatch between left and right %v != %v", + "mismatch between uAlphaQEval=%v preimages=%v", l.String(), r.String(), ) } @@ -448,8 +448,8 @@ func (ctx *SelfRecursionCtx) collapsingPhase() { }, func(api frontend.API, run wizard.GnarkRuntime) { api.AssertIsEqual( - left.GetFrontendVariable(api, run), - right.GetFrontendVariable(api, run), + uAlphaQEval.GetFrontendVariable(api, run), + preImageEval.GetFrontendVariable(api, run), ) }, ) From f77958cacad6b0ff8447d007e8747105c2442a08 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 17:34:29 +0100 Subject: [PATCH 37/63] fixup(conglo): remove unused method isPrefixed --- prover/protocol/compiler/selfrecursion/context.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/prover/protocol/compiler/selfrecursion/context.go b/prover/protocol/compiler/selfrecursion/context.go index f0652fc917b..ae226a8c906 100644 --- a/prover/protocol/compiler/selfrecursion/context.go +++ b/prover/protocol/compiler/selfrecursion/context.go @@ -324,10 +324,3 @@ func assertVortexCompiled(comp *wizard.CompiledIOP) *vortex.Ctx { func (ctx *SelfRecursionCtx) SisKey() *ringsis.Key { return &ctx.VortexCtx.VortexParams.Key } - -// isPrefixed returns whether a prefix is attached to the inner -// vortex Ctx. If defined, the prefix is used to prefix the -// column names. -func (ctx *SelfRecursionCtx) isPrefixed() bool { - return len(ctx.VortexCtx.RunStateNamePrefix) > 0 -} From e3746aeec7d65f4c7be23665734390069fe17acb Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sat, 18 Jan 2025 22:01:28 +0100 Subject: [PATCH 38/63] feat(conglo): adds an option in vortex to pre-announce that the context is self-recursed --- prover/protocol/compiler/vortex/option.go | 9 +++++++ .../conglomeration/conglomeration.go | 27 ++++++++++++------- .../conglomeration/conglomeration_test.go | 2 +- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/prover/protocol/compiler/vortex/option.go b/prover/protocol/compiler/vortex/option.go index b673a574b74..be66d3cbd3a 100644 --- a/prover/protocol/compiler/vortex/option.go +++ b/prover/protocol/compiler/vortex/option.go @@ -36,3 +36,12 @@ func ReplaceSisByMimc() VortexOp { ctx.SisParams = nil } } + +// PremarkAsSelfRecursed marks the ctx as selfrecursed. This is useful +// toward conglomerating the receiver comp but is not needed for +// self-recursion or full-recursion. +func PremarkAsSelfRecursed() VortexOp { + return func(ctx *Ctx) { + ctx.IsSelfrecursed = true + } +} diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 9489c0cbcd7..dcf6d2e0428 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -10,6 +10,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/consensys/linea-monorepo/prover/utils" ) // recursionCtx holds compilation context informations about the wizard @@ -211,16 +212,22 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { } } + if !srcVortexCtx.IsSelfrecursed || srcVortexCtx.ReplaceSisByMimc { + utils.Panic("the input vortex ctx is expected to be selfrecursed or having SIS replaced by MiMC. Please sure the input comp has been last compiled by Vortex with the option [vortex.MarkAsSelfRecursed]") + } + dstVortexCtx := &vortex.Ctx{ - RunStateNamePrefix: ctx.Translator.Prefix, - BlowUpFactor: srcVortexCtx.BlowUpFactor, - DryTreshold: srcVortexCtx.DryTreshold, - CommittedRowsCount: srcVortexCtx.CommittedRowsCount, - NumCols: srcVortexCtx.NumCols, - MaxCommittedRound: srcVortexCtx.MaxCommittedRound, - NumOpenedCol: srcVortexCtx.NumOpenedCol, - VortexParams: srcVortexCtx.VortexParams, - SisParams: srcVortexCtx.SisParams, + RunStateNamePrefix: ctx.Translator.Prefix, + BlowUpFactor: srcVortexCtx.BlowUpFactor, + DryTreshold: srcVortexCtx.DryTreshold, + CommittedRowsCount: srcVortexCtx.CommittedRowsCount, + NumCols: srcVortexCtx.NumCols, + MaxCommittedRound: srcVortexCtx.MaxCommittedRound, + NumOpenedCol: srcVortexCtx.NumOpenedCol, + VortexParams: srcVortexCtx.VortexParams, + SisParams: srcVortexCtx.SisParams, + // Although the srcVor + IsSelfrecursed: true, CommitmentsByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.CommitmentsByRounds), DriedByRounds: ctx.Translator.TranslateColumnVecVec(srcVortexCtx.DriedByRounds), PolynomialsTouchedByTheQuery: ctx.Translator.TranslateColumnSet(srcVortexCtx.PolynomialsTouchedByTheQuery), @@ -232,6 +239,8 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { panic("it should not replace by MiMC") } + ctx.Translator.Target.QueriesParams.MarkAsIgnored(dstVortexCtx.Query.QueryID) + if srcVortexCtx.IsCommitToPrecomputed() { dstVortexCtx.Items.Precomputeds.PrecomputedColums = ctx.Translator.TranslateColumnList(srcVortexCtx.Items.Precomputeds.PrecomputedColums) dstVortexCtx.Items.Precomputeds.MerkleRoot = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.MerkleRoot.GetColID()) diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index 5f7c9c14319..cea7556b6af 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -23,7 +23,7 @@ func TestConglomerationPureVortex(t *testing.T) { a []ifaces.Column u query.UnivariateEval sisParams = &ringsis.Params{LogTwoBound: 16, LogTwoDegree: 1} - vortexCompFunc = vortex.Compile(2, vortex.WithSISParams(sisParams), vortex.ForceNumOpenedColumns(2)) + vortexCompFunc = vortex.Compile(2, vortex.WithSISParams(sisParams), vortex.ForceNumOpenedColumns(2), vortex.PremarkAsSelfRecursed()) ) define := func(builder *wizard.Builder) { From 23ee220226a350e0de019d55e9a420bd59b76f52 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sun, 19 Jan 2025 01:54:25 +0100 Subject: [PATCH 39/63] feat(vortex): various features to fixup bugs in conglo * better error message in the verifier of self-recursion * bad condition in AllKeysInProverTranscript * vortex proving from rs encoded matrix * fix how the fs is extracted from the state in conglo * make FiatShamirSetup public * initializes the state in verifier runtime --- prover/crypto/vortex/opening_prover.go | 40 +++++++++++++++++-- prover/protocol/column/store.go | 4 +- .../compiler/selfrecursion/lincomb_phase.go | 2 +- prover/protocol/compiler/vortex/prover.go | 33 +++++++++++++++ .../distributed/conglomeration/prover.go | 2 +- .../distributed/conglomeration/verifier.go | 19 +++++---- prover/protocol/wizard/compiled.go | 6 +-- prover/protocol/wizard/fiatshamir.go | 2 +- prover/protocol/wizard/gnark_verifier.go | 2 +- prover/protocol/wizard/prover.go | 7 +++- prover/protocol/wizard/verifier.go | 4 +- 11 files changed, 100 insertions(+), 21 deletions(-) diff --git a/prover/crypto/vortex/opening_prover.go b/prover/crypto/vortex/opening_prover.go index 03e8c9c24ec..64f48a70a21 100644 --- a/prover/crypto/vortex/opening_prover.go +++ b/prover/crypto/vortex/opening_prover.go @@ -41,7 +41,6 @@ type OpeningProof struct { // functions and is motivated by the fact that this is simpler to construct in // our settings. func (params *Params) InitOpeningWithLC(committedSV []smartvectors.SmartVector, randomCoin field.Element) *OpeningProof { - proof := OpeningProof{} if len(committedSV) == 0 { utils.Panic("attempted to open an empty witness") @@ -55,15 +54,48 @@ func (params *Params) InitOpeningWithLC(committedSV []smartvectors.SmartVector, for i := range committedSV { subTask = append(subTask, committedSV[i].SubVector(start, stop)) } - // Collect the result in the larger slice at the end + // Collect the result in the larger slice at the end subResult := smartvectors.PolyEval(subTask, randomCoin) subResult.WriteInSlice(linComb[start:stop]) }) linCombSV := smartvectors.NewRegular(linComb) - proof.LinearCombination = params.rsEncode(linCombSV, nil) - return &proof + + return &OpeningProof{ + LinearCombination: params.rsEncode(linCombSV, nil), + } +} + +// InitOpeningFromAlreadyEncodedLC initiates the construction of a Vortex proof +// by returning the encoding of the linear combinations of the committed +// row-vectors contained in committedSV by the successive powers of randomCoin. +// +// The returned proof is partially assigned and must be completed using +// [WithEntryList] to conclude the opening protocol. +func (params *Params) InitOpeningFromAlreadyEncodedLC(rsCommittedSV EncodedMatrix, randomCoin field.Element) *OpeningProof { + + if len(rsCommittedSV) == 0 { + utils.Panic("attempted to open an empty witness") + } + + // Compute the linear combination + linComb := make([]field.Element, params.NumEncodedCols()) + + parallel.ExecuteChunky(len(linComb), func(start, stop int) { + subTask := make([]smartvectors.SmartVector, 0, len(rsCommittedSV)) + for i := range rsCommittedSV { + subTask = append(subTask, rsCommittedSV[i].SubVector(start, stop)) + } + + // Collect the result in the larger slice at the end + subResult := smartvectors.PolyEval(subTask, randomCoin) + subResult.WriteInSlice(linComb[start:stop]) + }) + + return &OpeningProof{ + LinearCombination: smartvectors.NewRegular(linComb), + } } // Complete completes the proof adding the columns pointed by entryList diff --git a/prover/protocol/column/store.go b/prover/protocol/column/store.go index e848e6c769d..92a4f10917d 100644 --- a/prover/protocol/column/store.go +++ b/prover/protocol/column/store.go @@ -509,7 +509,9 @@ func (s *Store) AllKeysInProverTranscript(round int) []ifaces.ColID { continue } - res = append(res, rnd[i].ID) + if info.Status.IsPublic() || info.IncludeInProverFS { + res = append(res, rnd[i].ID) + } } return res diff --git a/prover/protocol/compiler/selfrecursion/lincomb_phase.go b/prover/protocol/compiler/selfrecursion/lincomb_phase.go index 50f48db0feb..ea60c55bb7c 100644 --- a/prover/protocol/compiler/selfrecursion/lincomb_phase.go +++ b/prover/protocol/compiler/selfrecursion/lincomb_phase.go @@ -77,7 +77,7 @@ func (ctx *SelfRecursionCtx) consistencyBetweenYsAndUalpha() { ysAlpha := smartvectors.EvalCoeff(ys, alpha) uAlphaX := ctx.Accessors.InterpolateUalphaX.GetVal(run) if uAlphaX != ysAlpha { - return fmt.Errorf("ConsistencyBetweenYsAndUalpha did not pass") + return fmt.Errorf("ConsistencyBetweenYsAndUalpha did not pass, ysAlphaX=%v uAlphaX=%v", ysAlpha.String(), uAlphaX.String()) } return nil }, diff --git a/prover/protocol/compiler/vortex/prover.go b/prover/protocol/compiler/vortex/prover.go index 6bcd1261df8..65db1105111 100644 --- a/prover/protocol/compiler/vortex/prover.go +++ b/prover/protocol/compiler/vortex/prover.go @@ -74,6 +74,39 @@ func (ctx *Ctx) ComputeLinearComb(pr *wizard.ProverRuntime) { pr.AssignColumn(ctx.Items.Ualpha.GetColID(), proof.LinearCombination) } +// ComputeLinearCombFromRsMatrix is the same as ComputeLinearComb but uses +// the RS encoded matrix instead of using the basic one. It is slower than +// the later but is recommended. +func (ctx *Ctx) ComputeLinearCombFromRsMatrix(pr *wizard.ProverRuntime) { + + committedSV := []smartvectors.SmartVector{} + + // Add the precomputed columns to commitedSV if IsCommitToPrecomputed is true + if ctx.IsCommitToPrecomputed() { + committedSV = append(committedSV, ctx.Items.Precomputeds.CommittedMatrix...) + } + + // Collect all the committed polynomials : round by round + for round := 0; round <= ctx.MaxCommittedRound; round++ { + // There are not included in the commitments so there + // is no need to compute their linear combination. + if ctx.isDry(round) { + continue + } + + committedMatrix := pr.State.MustGet(ctx.VortexProverStateName(round)).(vortex.EncodedMatrix) + committedSV = append(committedSV, committedMatrix...) + } + + // And get the randomness + randomCoinLC := pr.GetRandomCoinField(ctx.Items.Alpha.Name) + + // and compute and assign the random linear combination of the rows + proof := ctx.VortexParams.InitOpeningFromAlreadyEncodedLC(committedSV, randomCoinLC) + + pr.AssignColumn(ctx.Items.Ualpha.GetColID(), proof.LinearCombination) +} + // Prover steps of Vortex where he opens the columns selected by the verifier func (ctx *Ctx) OpenSelectedColumns(pr *wizard.ProverRuntime) { diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 57d36d5a665..17a959e176a 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -184,7 +184,7 @@ func (pa AssignVortexUAlpha) Run(run *wizard.ProverRuntime) { for _, ctx := range pa.Ctxs { // Since all the context of the pcs is translated, this does not // need to run over a translated prover runtime. - ctx.PcsCtx.ComputeLinearComb(run) + ctx.PcsCtx.ComputeLinearCombFromRsMatrix(run) } } diff --git a/prover/protocol/distributed/conglomeration/verifier.go b/prover/protocol/distributed/conglomeration/verifier.go index b1894f9c507..aa74a14eb8b 100644 --- a/prover/protocol/distributed/conglomeration/verifier.go +++ b/prover/protocol/distributed/conglomeration/verifier.go @@ -72,20 +72,25 @@ func generateRandomCoins(run wizard.Runtime, ctx *recursionCtx, currRound int) { // and history. wrappedRun wizard.Runtime = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} fsAny, _ = wrappedRun.GetState(fiatShamirTranscriptStr) - fs = fsAny.(*fiatshamir.State) fsHistoryAny, _ = wrappedRun.GetState(fiatShamirHistoryStr) - fsHistory = fsHistoryAny.([][2][]field.Element) - initialState = fs.State() + fs *fiatshamir.State + fsHistory [][2][]field.Element ) - if fs == nil { + if fsAny == nil { fs = fiatshamir.NewMiMCFiatShamir() + } else { + fs = fsAny.(*fiatshamir.State) } - if fsHistory == nil { + if fsHistoryAny == nil { fsHistory = make([][2][]field.Element, ctx.LastRound+1) + } else { + fsHistory = fsHistoryAny.([][2][]field.Element) } + initialState := fs.State() + // Wraps it a second time wrappedRun = &RuntimeWithReplacedFS{ Runtime: wrappedRun, @@ -239,8 +244,8 @@ func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run w fs.State(), } - run.SetState(fiatShamirHistoryStr, fsHistory) - run.SetState(fiatShamirTranscriptStr, fs) + wrappedRun.SetState(fiatShamirHistoryStr, fsHistory) + wrappedRun.SetState(fiatShamirTranscriptStr, fs) } // Fs returns the Fiat-Shamir state diff --git a/prover/protocol/wizard/compiled.go b/prover/protocol/wizard/compiled.go index af3c80f74fe..c68f567e6e4 100644 --- a/prover/protocol/wizard/compiled.go +++ b/prover/protocol/wizard/compiled.go @@ -111,13 +111,13 @@ type CompiledIOP struct { // process. An artefact must satisfy the io.ReadWriteTo interface. Artefacts artefactCache - // fiatShamirSetup stores an initial value to use to bootstrap the Fiat-Shamir + // FiatShamirSetup stores an initial value to use to bootstrap the Fiat-Shamir // transcript. This value is obtained by hashing diverse meta-data of the // describing the wizard: a version number, the description of the field, // a description of all the columns and all the queries etc... // - // For efficiency reasons, the fiatShamirSetup is derived using SHA2. - fiatShamirSetup field.Element + // For efficiency reasons, the FiatShamirSetup is derived using SHA2. + FiatShamirSetup field.Element // FunctionalPublic inputs lists the queries representing a public inputs // and their identifiers diff --git a/prover/protocol/wizard/fiatshamir.go b/prover/protocol/wizard/fiatshamir.go index 59ecbaa7e72..109d92438f5 100644 --- a/prover/protocol/wizard/fiatshamir.go +++ b/prover/protocol/wizard/fiatshamir.go @@ -34,7 +34,7 @@ func (comp *CompiledIOP) BootstrapFiatShamir(vm VersionMetadata, ser CompiledIOP // hasher.Write(compBlob) digest := hasher.Sum(nil) digest[0] = 0 // This is to prevent potential errors due to overflowing the field - comp.fiatShamirSetup.SetBytes(digest) + comp.FiatShamirSetup.SetBytes(digest) return comp } diff --git a/prover/protocol/wizard/gnark_verifier.go b/prover/protocol/wizard/gnark_verifier.go index 4311e9b3e6e..345fd3f7b8d 100644 --- a/prover/protocol/wizard/gnark_verifier.go +++ b/prover/protocol/wizard/gnark_verifier.go @@ -185,7 +185,7 @@ func AllocateWizardCircuit(comp *CompiledIOP) (*WizardVerifierCircuit, error) { func (c *WizardVerifierCircuit) Verify(api frontend.API) { c.HasherFactory = gkrmimc.NewHasherFactory(api) c.FS = fiatshamir.NewGnarkFiatShamir(api, c.HasherFactory) - c.FS.Update(c.Spec.fiatShamirSetup) + c.FS.Update(c.Spec.FiatShamirSetup) c.FiatShamirHistory = make([][2][]frontend.Variable, c.Spec.NumRounds()) c.generateAllRandomCoins(api) diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index ffeb156d590..176c078493c 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -250,7 +250,7 @@ func (c *CompiledIOP) createProver() ProverRuntime { // Create a new fresh FS state and bootstrap it fs := fiatshamir.NewMiMCFiatShamir() - fs.Update(c.fiatShamirSetup) + fs.Update(c.FiatShamirSetup) // Instantiates an empty Assignment (but link it to the CompiledIOP) runtime := ProverRuntime{ @@ -595,6 +595,11 @@ func (run *ProverRuntime) goNextRound() { */ toCompute := run.Spec.Coins.AllKeysAt(run.currRound) for _, coin := range toCompute { + + if run.Spec.Coins.IsSkippedFromVerifierTranscript(coin) { + continue + } + info := run.Spec.Coins.Data(coin) value := info.Sample(run.FS) run.Coins.InsertNew(coin, value) diff --git a/prover/protocol/wizard/verifier.go b/prover/protocol/wizard/verifier.go index dbe79a36da2..1513143adf6 100644 --- a/prover/protocol/wizard/verifier.go +++ b/prover/protocol/wizard/verifier.go @@ -153,9 +153,10 @@ func (c *CompiledIOP) createVerifier(proof Proof) VerifierRuntime { QueriesParams: proof.QueriesParams, FS: fiatshamir.NewMiMCFiatShamir(), FiatShamirHistory: make([][2][]field.Element, c.NumRounds()), + State: make(map[string]interface{}), } - runtime.FS.Update(c.fiatShamirSetup) + runtime.FS.Update(c.FiatShamirSetup) /* Insert the verifying key into the messages @@ -228,6 +229,7 @@ func (run *VerifierRuntime) generateAllRandomCoins() { */ toCompute := run.Spec.Coins.AllKeysAt(currRound) for _, coin := range toCompute { + if run.Spec.Coins.IsSkippedFromVerifierTranscript(coin) { continue } From 2cad230b9a6716111d6eea8bd9d966ccfec946ab Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sun, 19 Jan 2025 15:44:18 +0100 Subject: [PATCH 40/63] fixup(fs): fix the implementation of isExcludedFromProverFS --- prover/protocol/column/store.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/prover/protocol/column/store.go b/prover/protocol/column/store.go index 92a4f10917d..a32de623a3c 100644 --- a/prover/protocol/column/store.go +++ b/prover/protocol/column/store.go @@ -483,11 +483,15 @@ func (in *storedColumnInfo) isExcludedFromProverFS() bool { return true } - if in.Status == Ignored && in.IncludeInProverFS { - return true + if in.Status.IsPublic() { + return false + } + + if in.IncludeInProverFS { + return false } - return in.Status.IsPublic() + return true } // IsExplicitlyExcludedFromProverFS returns true if the passed column ID relates to @@ -509,9 +513,7 @@ func (s *Store) AllKeysInProverTranscript(round int) []ifaces.ColID { continue } - if info.Status.IsPublic() || info.IncludeInProverFS { - res = append(res, rnd[i].ID) - } + res = append(res, rnd[i].ID) } return res From cc58d1daaa58af4c5c6e546ea8525b81f2aa531b Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Sun, 19 Jan 2025 15:56:26 +0100 Subject: [PATCH 41/63] fixup(typo): not -> note --- prover/protocol/distributed/conglomeration/conglomeration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index dcf6d2e0428..510a84f04b1 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -152,7 +152,7 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { continue } - // Not that we do not filter the already compiled queries + // Note that we do not filter the already compiled queries qInfo := tmpl.QueriesParams.Data(qName) qInfo = ctx.Translator.InsertQueryParams(round, qInfo) ctx.QueryParams[round] = append(ctx.QueryParams[round], qInfo) From 1838328a5d7dcfe89369a49cc0d1280043d3b7ec Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 12:55:05 +0000 Subject: [PATCH 42/63] feat(conglo): implements the fs joint --- .../conglomeration/conglomeration.go | 3 + .../distributed/conglomeration/prover.go | 31 ++++ .../distributed/conglomeration/verifier.go | 145 ++++++++++++++---- 3 files changed, 152 insertions(+), 27 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 510a84f04b1..f8fc0958b22 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -74,6 +74,9 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu } } + comp.FiatShamirHooks.AppendToInner(0, &SubFsInitialize{Ctxs: ctxs}) + comp.FiatShamirHooks.AppendToInner(ctxs[0].LastRound, &FsJoinHook{Ctxs: ctxs}) + comp.RegisterProverAction(ctxs[0].LastRound+1, &FsJoinProverStep{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound, &AssignVortexQuery{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound+1, &AssignVortexUAlpha{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound+2, &AssignVortexOpenedCols{Ctxs: ctxs}) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 17a959e176a..19762d112e2 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -1,8 +1,10 @@ package conglomeration import ( + "fmt" "strings" + "github.com/consensys/linea-monorepo/prover/crypto/mimc" "github.com/consensys/linea-monorepo/prover/crypto/state-management/smt" vCom "github.com/consensys/linea-monorepo/prover/crypto/vortex" "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" @@ -14,6 +16,7 @@ import ( const ( subProofInStatePrefixStr = ".subProof" + finalFsStateInStateStr = ".finalFsState" ) // Witness is a collection of inputs corresponding to a segment proof to provide @@ -23,6 +26,7 @@ type Witness struct { CommittedMatrices []vCom.EncodedMatrix SisHashes [][]field.Element Trees []*smt.Tree + FinalFsState []field.Element } // PreVortexProverStep is a step replicating the prover of the tmpl at round @@ -35,6 +39,12 @@ type PreVortexProverStep struct { Round int } +// FsJoinProverStep is prover step setting the fiat-shamir state of the main +// transcript to the hash of the final states of the subproofs. +type FsJoinProverStep struct { + Ctxs []*recursionCtx +} + // AssignVortexQuery assigns the query for all the subproofs. type AssignVortexQuery struct { Ctxs []*recursionCtx @@ -83,6 +93,7 @@ func storeWitnessInState(run *wizard.ProverRuntime, ctx *recursionCtx, witness W ) run.State.InsertNew(prefix+subProofInStatePrefixStr, witness.Proof) + run.State.InsertNew(prefix+finalFsStateInStateStr, witness.FinalFsState) for round := 0; round <= lastRound; round++ { @@ -137,6 +148,7 @@ func ExtractWitness(run *wizard.ProverRuntime) Witness { CommittedMatrices: committedMatrices, SisHashes: sisHashes, Trees: trees, + FinalFsState: run.FS.State(), } } @@ -201,3 +213,22 @@ func unprefix[T ~string](prefix string, name T) T { r := strings.TrimPrefix(n, p) return T(r) } + +func (pa *FsJoinProverStep) Run(run *wizard.ProverRuntime) { + + mainState := field.NewElement(0) + + for _, ctx := range pa.Ctxs { + + var ( + prefix = ctx.Translator.Prefix + fsState = run.State.MustGet(prefix + finalFsStateInStateStr).([]field.Element) + ) + + mainState = mimc.BlockCompression(mainState, fsState[0]) + } + + fmt.Printf("[join-fs-prover] mainState: %v\n", mainState.String()) + + run.FS.SetState([]field.Element{mainState}) +} diff --git a/prover/protocol/distributed/conglomeration/verifier.go b/prover/protocol/distributed/conglomeration/verifier.go index aa74a14eb8b..6b7de6513c2 100644 --- a/prover/protocol/distributed/conglomeration/verifier.go +++ b/prover/protocol/distributed/conglomeration/verifier.go @@ -2,9 +2,11 @@ package conglomeration import ( "errors" + "fmt" "github.com/consensys/gnark/frontend" "github.com/consensys/linea-monorepo/prover/crypto/fiatshamir" + "github.com/consensys/linea-monorepo/prover/crypto/mimc" "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -41,11 +43,29 @@ type GnarkRuntimeWithReplacedFS struct { FiatShamirHistory [][2][]frontend.Variable } +// FsJoinHook is a fiat-shamir hook whose purpose is to join the interal +// fiat-shamir states of the segment proofs into the main one. It works +// by setting the main fs-state as the hash of the internal states. +type FsJoinHook struct { + Ctxs []*recursionCtx + isSkipped bool +} + +// SubFsInitialize is a fiat-shamir hook whose purpose is to initialize the +// internal fiat-shamir states of the segment proofs. It is set as a FS hook +// as it is guaranteed to be run before any prover/verifier step, ensuring the +// fs states are available at the beginning. +type SubFsInitialize struct { + Ctxs []*recursionCtx + isSkipped bool +} + func (pa PreVortexVerifierStep) Run(run wizard.Runtime) error { var err error for _, ctx := range pa.Ctxs { + generateRandomCoins(run, ctx, pa.Round) // Wraps the runtime into a translation adapter @@ -73,24 +93,11 @@ func generateRandomCoins(run wizard.Runtime, ctx *recursionCtx, currRound int) { wrappedRun wizard.Runtime = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} fsAny, _ = wrappedRun.GetState(fiatShamirTranscriptStr) fsHistoryAny, _ = wrappedRun.GetState(fiatShamirHistoryStr) - fs *fiatshamir.State - fsHistory [][2][]field.Element + fs = fsAny.(*fiatshamir.State) + fsHistory = fsHistoryAny.([][2][]field.Element) + initialState = fs.State() ) - if fsAny == nil { - fs = fiatshamir.NewMiMCFiatShamir() - } else { - fs = fsAny.(*fiatshamir.State) - } - - if fsHistoryAny == nil { - fsHistory = make([][2][]field.Element, ctx.LastRound+1) - } else { - fsHistory = fsHistoryAny.([][2][]field.Element) - } - - initialState := fs.State() - // Wraps it a second time wrappedRun = &RuntimeWithReplacedFS{ Runtime: wrappedRun, @@ -135,8 +142,8 @@ func generateRandomCoins(run wizard.Runtime, ctx *recursionCtx, currRound int) { fs.State(), } - run.SetState(fiatShamirHistoryStr, fsHistory) - run.SetState(fiatShamirTranscriptStr, fs) + wrappedRun.SetState(fiatShamirHistoryStr, fsHistory) + wrappedRun.SetState(fiatShamirTranscriptStr, fs) } // Fs returns the Fiat-Shamir state @@ -180,20 +187,12 @@ func (pa PreVortexVerifierStep) generateRandomCoinsGnark(api frontend.API, run w // and history. wrappedRun wizard.GnarkRuntime = &gnarkRuntimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} fsAny, _ = wrappedRun.GetState(fiatShamirTranscriptStr) - fs = fsAny.(*fiatshamir.GnarkFiatShamir) fsHistoryAny, _ = wrappedRun.GetState(fiatShamirHistoryStr) + fs = fsAny.(*fiatshamir.GnarkFiatShamir) fsHistory = fsHistoryAny.([][2][]frontend.Variable) initialState = fs.State() ) - if fs == nil && run.GetHasherFactory() != nil { - fs = fiatshamir.NewGnarkFiatShamir(api, run.GetHasherFactory()) - } - - if fsHistory == nil { - fsHistory = make([][2][]frontend.Variable, ctx.LastRound+1) - } - // Wraps it a second time wrappedRun = &GnarkRuntimeWithReplacedFS{ GnarkRuntime: wrappedRun, @@ -265,3 +264,95 @@ func (pa *PreVortexVerifierStep) IsSkipped() bool { func (pa *PreVortexVerifierStep) Skip() { pa.isSkipped = true } + +func (fs *FsJoinHook) Run(run wizard.Runtime) error { + + mainState := field.NewElement(0) + + for _, ctx := range fs.Ctxs { + + var ( + wrappedRun wizard.Runtime = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + fsAny, _ = wrappedRun.GetState(fiatShamirTranscriptStr) + fs = fsAny.(*fiatshamir.State) + ) + + mainState = mimc.BlockCompression(mainState, fs.State()[0]) + } + + fmt.Printf("[join-fs-hook] mainState: %v\n", mainState.String()) + + run.Fs().SetState([]field.Element{mainState}) + + return nil +} + +func (fs *FsJoinHook) RunGnark(api frontend.API, run wizard.GnarkRuntime) { + + mainState := frontend.Variable(0) + + for _, ctx := range fs.Ctxs { + + var ( + wrappedRun wizard.GnarkRuntime = &gnarkRuntimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + fsAny, _ = wrappedRun.GetState(fiatShamirTranscriptStr) + fs = fsAny.(*fiatshamir.GnarkFiatShamir) + ) + + mainState = mimc.GnarkBlockCompression(api, mainState, fs.State()[0]) + } + + run.Fs().SetState([]frontend.Variable{mainState}) +} + +func (fs *FsJoinHook) Skip() { + fs.isSkipped = true +} + +func (fs *FsJoinHook) IsSkipped() bool { + return fs.isSkipped +} + +func (fs *SubFsInitialize) Run(run wizard.Runtime) error { + + for _, ctx := range fs.Ctxs { + + var ( + wrappedRun wizard.Runtime = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + fs = fiatshamir.NewMiMCFiatShamir() + fsHistory = make([][2][]field.Element, ctx.LastRound+1) + ) + + fs.Update(ctx.Tmpl.FiatShamirSetup) + + wrappedRun.SetState(fiatShamirHistoryStr, fsHistory) + wrappedRun.SetState(fiatShamirTranscriptStr, fs) + } + + return nil +} + +func (fs *SubFsInitialize) RunGnark(api frontend.API, run wizard.GnarkRuntime) { + + for _, ctx := range fs.Ctxs { + + var ( + wrappedRun wizard.GnarkRuntime = &gnarkRuntimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + fs = fiatshamir.NewGnarkFiatShamir(api, run.GetHasherFactory()) + fsHistory = make([][2][]frontend.Variable, ctx.LastRound+1) + ) + + fs.Update(api, ctx.Tmpl.FiatShamirSetup) + + wrappedRun.SetState(fiatShamirHistoryStr, fsHistory) + wrappedRun.SetState(fiatShamirTranscriptStr, fs) + } +} + +func (fs *SubFsInitialize) Skip() { + fs.isSkipped = true +} + +func (fs *SubFsInitialize) IsSkipped() bool { + return fs.isSkipped +} From a96e03880b8579f9595c436133971b160f3b6f15 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 13:13:11 +0000 Subject: [PATCH 43/63] fixup: assigns the correct round for the fsProverJoinStep --- prover/protocol/distributed/conglomeration/conglomeration.go | 2 +- prover/protocol/distributed/conglomeration/prover.go | 3 --- prover/protocol/distributed/conglomeration/verifier.go | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index f8fc0958b22..7c4c70cdde8 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -76,7 +76,7 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu comp.FiatShamirHooks.AppendToInner(0, &SubFsInitialize{Ctxs: ctxs}) comp.FiatShamirHooks.AppendToInner(ctxs[0].LastRound, &FsJoinHook{Ctxs: ctxs}) - comp.RegisterProverAction(ctxs[0].LastRound+1, &FsJoinProverStep{Ctxs: ctxs}) + comp.RegisterProverAction(ctxs[0].LastRound, &FsJoinProverStep{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound, &AssignVortexQuery{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound+1, &AssignVortexUAlpha{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound+2, &AssignVortexOpenedCols{Ctxs: ctxs}) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 19762d112e2..9fa6db30542 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -1,7 +1,6 @@ package conglomeration import ( - "fmt" "strings" "github.com/consensys/linea-monorepo/prover/crypto/mimc" @@ -228,7 +227,5 @@ func (pa *FsJoinProverStep) Run(run *wizard.ProverRuntime) { mainState = mimc.BlockCompression(mainState, fsState[0]) } - fmt.Printf("[join-fs-prover] mainState: %v\n", mainState.String()) - run.FS.SetState([]field.Element{mainState}) } diff --git a/prover/protocol/distributed/conglomeration/verifier.go b/prover/protocol/distributed/conglomeration/verifier.go index 6b7de6513c2..cbc3f8fb025 100644 --- a/prover/protocol/distributed/conglomeration/verifier.go +++ b/prover/protocol/distributed/conglomeration/verifier.go @@ -2,7 +2,6 @@ package conglomeration import ( "errors" - "fmt" "github.com/consensys/gnark/frontend" "github.com/consensys/linea-monorepo/prover/crypto/fiatshamir" @@ -280,8 +279,6 @@ func (fs *FsJoinHook) Run(run wizard.Runtime) error { mainState = mimc.BlockCompression(mainState, fs.State()[0]) } - fmt.Printf("[join-fs-hook] mainState: %v\n", mainState.String()) - run.Fs().SetState([]field.Element{mainState}) return nil From f6309ac63b440b89ba176d686124d1890f5337b4 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 13:16:15 +0000 Subject: [PATCH 44/63] cleanup: clean the fsHistory in-line with the fs.state in prover-join-fs --- prover/protocol/distributed/conglomeration/prover.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 9fa6db30542..94338709245 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -41,7 +41,8 @@ type PreVortexProverStep struct { // FsJoinProverStep is prover step setting the fiat-shamir state of the main // transcript to the hash of the final states of the subproofs. type FsJoinProverStep struct { - Ctxs []*recursionCtx + Ctxs []*recursionCtx + Round int } // AssignVortexQuery assigns the query for all the subproofs. @@ -227,5 +228,8 @@ func (pa *FsJoinProverStep) Run(run *wizard.ProverRuntime) { mainState = mimc.BlockCompression(mainState, fsState[0]) } - run.FS.SetState([]field.Element{mainState}) + newState := []field.Element{mainState} + + run.FS.SetState(newState) + run.FiatShamirHistory[pa.Round][1] = newState } From e8134fa1626b0ad6db33f915f7a030304d03d000 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 15:06:09 +0000 Subject: [PATCH 45/63] code(fs): remove redundant sanity-check for prover-fs --- prover/protocol/wizard/prover.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index dba812e561c..0ae1759735e 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -541,27 +541,6 @@ func (run *ProverRuntime) goNextRound() { initialState := run.FS.State() - /* - Make sure all issued random coin have been "consumed" by all the prover - steps, in the round we are closing. An error occuring here is more likely - an error in the compiler than an error from the user because it is not - responsible for setting the coin. Thus, this is more a sanity check. - */ - toBeConsumed := run.Spec.Coins.AllKeysAt(run.currRound) - run.Coins.MustExists(toBeConsumed...) - - /* - We do not make this check for the columns, the reason is that we delete - the columns that we do not use anymore. - */ - - /* - Then, make sure all the query parameters have been set - during the rounds we are closing - */ - toBeParametrized := run.Spec.QueriesParams.AllKeysAt(run.currRound) - run.QueriesParams.MustExists(toBeParametrized...) - if !run.Spec.DummyCompiled { /* From d6ada8803eab3e230e8999d9f08e61fe6d99ab6a Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 17:12:35 +0000 Subject: [PATCH 46/63] fixup(fs): declare pre-vortex-verifier-steps as a fs-hook --- .../conglomeration/conglomeration.go | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 7c4c70cdde8..53c05c04ddb 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -66,7 +66,12 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu ) if hasCoin || hasVAction || hasFsHook { - comp.RegisterVerifierAction(round, &PreVortexVerifierStep{Ctxs: ctxs, Round: round}) + // The way the verifier runtime is that it will generate all the random coins at once and + // then, it runs all the verifier actions in parallel. What this action from the verifier + // is trying to do is to prepare a ctx-local FS state that can be later used in a join to + // derive a sound global FS state. Thus, we need it to run along side the "main" fs random + // coin generation. This is why this is declared as an FS hook and not as a VerifierAction. + comp.FiatShamirHooks.AppendToInner(round, &PreVortexVerifierStep{Ctxs: ctxs, Round: round}) } if hasColumn || hasQParams { @@ -119,6 +124,20 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { ctx.Coins = append(ctx.Coins, []coin.Info{}) ctx.FsHooks = append(ctx.FsHooks, []wizard.VerifierAction{}) + // Importantly, the coins are added before. Otherwise the 'assertConsistentRound' + // clause would not accept inserting columns or queries. + for _, cName := range tmpl.Coins.AllKeysAt(round) { + + if tmpl.Coins.IsSkippedFromVerifierTranscript(cName) { + continue + } + + coinInfo := tmpl.Coins.Data(cName) + coinInfo = ctx.Translator.InsertCoin(coinInfo) + ctx.Coins[round] = append(ctx.Coins[round], coinInfo) + ctx.Translator.Target.Coins.MarkAsSkippedFromVerifierTranscript(coinInfo.Name) + } + for _, colName := range tmpl.Columns.AllKeysAt(round) { // filter the columns by status @@ -162,18 +181,6 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { ctx.Translator.Target.QueriesParams.MarkAsSkippedFromProverTranscript(qInfo.Name()) } - for _, cName := range tmpl.Coins.AllKeysAt(round) { - - if tmpl.Coins.IsSkippedFromVerifierTranscript(cName) { - continue - } - - coinInfo := tmpl.Coins.Data(cName) - coinInfo = ctx.Translator.InsertCoin(coinInfo) - ctx.Coins[round] = append(ctx.Coins[round], coinInfo) - ctx.Translator.Target.Coins.MarkAsSkippedFromVerifierTranscript(coinInfo.Name) - } - verifierActions := tmpl.SubVerifiers.Inner() for i := range verifierActions[round] { From f4c92160ddd71ffe3b44ffd26e4cb8f17f5b3e0c Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 17:12:55 +0000 Subject: [PATCH 47/63] test(conglomeration): adds a multi-round test --- .../conglomeration/conglomeration_test.go | 122 ++++++++++++++++-- 1 file changed, 110 insertions(+), 12 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index cea7556b6af..39ab34c914c 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -6,6 +6,7 @@ import ( "github.com/consensys/linea-monorepo/prover/crypto/ringsis" "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" @@ -14,16 +15,25 @@ import ( "github.com/stretchr/testify/require" ) -func TestConglomerationPureVortex(t *testing.T) { +type ( + defineFuncType func(*wizard.Builder) + proverFuncType func(int) func(*wizard.ProverRuntime) +) + +type conglomerationTestCase struct { + define defineFuncType + prove proverFuncType + numProof int +} + +func TestConglomerationPureVortexSingleRound(t *testing.T) { var ( - numCol = 16 - numRow = 16 - numProof = 16 - a []ifaces.Column - u query.UnivariateEval - sisParams = &ringsis.Params{LogTwoBound: 16, LogTwoDegree: 1} - vortexCompFunc = vortex.Compile(2, vortex.WithSISParams(sisParams), vortex.ForceNumOpenedColumns(2), vortex.PremarkAsSelfRecursed()) + numCol = 16 + numRow = 16 + numProof = 16 + a []ifaces.Column + u query.UnivariateEval ) define := func(builder *wizard.Builder) { @@ -38,15 +48,104 @@ func TestConglomerationPureVortex(t *testing.T) { ys := make([]field.Element, 0, len(a)) for i := range a { y := field.NewElement(uint64(i + k)) - run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numCol)) + run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numRow)) ys = append(ys, y) } run.AssignUnivariate(u.QueryID, field.NewElement(0), ys...) } } + runConglomerationTestCase(t, conglomerationTestCase{ + define: define, + prove: prover, + numProof: numProof, + }) +} + +func TestConglomerationPureVortexMultiRound(t *testing.T) { + var ( - tmpl = wizard.Compile(define, vortexCompFunc) + numRound = 4 + numCol = 4 + numRow = 16 + numProof = 16 + a []ifaces.Column + ) + + define := func(builder *wizard.Builder) { + + allCols := make([]ifaces.Column, 0, numCol*numRound) + + for round := 0; round < numRound; round++ { + + if round > 0 { + _ = builder.InsertCoin(round, coin.Namef("c-%v", round), coin.Field) + } + + roundCols := make([]ifaces.Column, 0, numCol) + + for i := 0; i < numCol; i++ { + newCol := builder.InsertCommit(round, ifaces.ColIDf("a-%v-%v", round, i), numRow) + roundCols = append(roundCols, newCol) + } + + if round > 0 { + builder.SubProvers.AppendToInner(round, func(run *wizard.ProverRuntime) { + for i := range roundCols { + x := field.NewElement(uint64(round*numCol + i)) + run.AssignColumn(roundCols[i].GetColID(), smartvectors.NewConstant(x, numRow)) + } + + if round == numRound-1 { + + } + }) + } + + if round == 0 { + a = roundCols + } + + allCols = append(allCols, roundCols...) + } + + u := builder.CompiledIOP.InsertUnivariate(numRound-1, "u", allCols) + + builder.CompiledIOP.SubProvers.AppendToInner(numRound-1, func(run *wizard.ProverRuntime) { + ys := make([]field.Element, 0, len(allCols)) + for _, col := range allCols { + ys = append(ys, col.GetColAssignmentAt(run, 0)) + } + run.AssignUnivariate(u.QueryID, field.NewElement(0), ys...) + }) + + } + + prover := func(k int) func(run *wizard.ProverRuntime) { + return func(run *wizard.ProverRuntime) { + ys := make([]field.Element, 0, len(a)) + for i := range a { + y := field.NewElement(uint64(i + k)) + run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numRow)) + ys = append(ys, y) + } + } + } + + runConglomerationTestCase(t, conglomerationTestCase{ + define: define, + prove: prover, + numProof: numProof, + }) +} + +func runConglomerationTestCase(t *testing.T, tc conglomerationTestCase) { + + var ( + sisParams = &ringsis.Params{LogTwoBound: 16, LogTwoDegree: 1} + vortexCompFunc = vortex.Compile(2, vortex.WithSISParams(sisParams), vortex.ForceNumOpenedColumns(2), vortex.PremarkAsSelfRecursed()) + numProof = tc.numProof + tmpl = wizard.Compile(wizard.DefineFunc(tc.define), vortexCompFunc) congDef, ctxsPHolder = ConglomerateDefineFunc(tmpl, numProof) cong = wizard.Compile(congDef, dummy.CompileAtProverLvl) ctxs = *ctxsPHolder @@ -55,7 +154,7 @@ func TestConglomerationPureVortex(t *testing.T) { witnesses := make([]Witness, numProof) for i := range witnesses { - runtime := wizard.RunProverUntilRound(tmpl, prover(i), lastRound+1) + runtime := wizard.RunProverUntilRound(tmpl, tc.prove(i), lastRound+1) witnesses[i] = ExtractWitness(runtime) } @@ -63,5 +162,4 @@ func TestConglomerationPureVortex(t *testing.T) { err := wizard.Verify(cong, proof) require.NoError(t, err) - } From 1e59cf7845bee6949a1aede166a42cfe9a0bf779 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 17:17:09 +0000 Subject: [PATCH 48/63] fix(lint): skip math.Rand error for the encoding --- prover/backend/execution/prove.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/backend/execution/prove.go b/prover/backend/execution/prove.go index 4970e344fbf..7a719192bd5 100644 --- a/prover/backend/execution/prove.go +++ b/prover/backend/execution/prove.go @@ -190,7 +190,7 @@ func mustProveAndPass( case config.ProverModeEncodeOnly: profiling.ProfileTrace("encode-decode-no-circuit", true, false, func() { - filepath := "/tmp/wizard-assignment/blob-" + strconv.Itoa(rand.Int()) + ".bin" + filepath := "/tmp/wizard-assignment/blob-" + strconv.Itoa(rand.Int()) + ".bin" //nolint encodeOnlyZkEvm := zkevm.EncodeOnlyZkEvm(traces) numChunks := runtime.GOMAXPROCS(0) From 2c495adbc19f209e83b4b3180f1ec0fa1400927d Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 20 Jan 2025 17:17:26 +0000 Subject: [PATCH 49/63] fixup(lint): preallocating slice before appending --- prover/protocol/distributed/conglomeration/translator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 5d478ded485..373409ef83a 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -116,7 +116,7 @@ func copyQueryWithName(name ifaces.QueryID, q ifaces.Query) ifaces.Query { // TranslateColumnList translates a collection of pre-inserted columns func (comp *compTranslator) TranslateColumnList(cols []ifaces.Column) []ifaces.Column { - var res []ifaces.Column + res := make([]ifaces.Column, 0, len(cols)) for _, col := range cols { res = append(res, comp.GetColumn(col.GetColID())) } From 25585b1e84859984a9d50467153300d45f720d2b Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 21 Jan 2025 17:49:08 +0000 Subject: [PATCH 50/63] fixup(init-fs): ensure the sub-fs-initialize is run before the pre-vortex-verifier-steps --- .../protocol/distributed/conglomeration/conglomeration.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 53c05c04ddb..e433d1a9ac0 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -55,6 +55,11 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu ctxs = append(ctxs, ctx) } + // This FS hook has to be defined before we add the pre-vortex verifier + // hooks to ensure that the FS state is properly initialize the verifier + // runtime. + comp.FiatShamirHooks.AppendToInner(0, &SubFsInitialize{Ctxs: ctxs}) + for round := 0; round <= ctxs[0].LastRound; round++ { var ( @@ -79,7 +84,6 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu } } - comp.FiatShamirHooks.AppendToInner(0, &SubFsInitialize{Ctxs: ctxs}) comp.FiatShamirHooks.AppendToInner(ctxs[0].LastRound, &FsJoinHook{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound, &FsJoinProverStep{Ctxs: ctxs}) comp.RegisterProverAction(ctxs[0].LastRound, &AssignVortexQuery{Ctxs: ctxs}) From 9314ec7ac93dc835db8f23a6a420407798436192 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Wed, 22 Jan 2025 14:53:44 +0000 Subject: [PATCH 51/63] * minor(vortex): exporting the Tree field for the precomputed * minor(wizard): adding GetQuery to the Runtime and GnarkRuntime interface * fixup(conglo): support the case where there are dry rounds in vortex * minor(mpts): using the run GetQuery method * minor(selfrec): better error message * test(conglo): adds a test case for arcane + lookup --- .../compiler/selfrecursion/precomputations.go | 2 +- .../univariates/multi_to_single_point.go | 2 +- prover/protocol/compiler/vortex/compiler.go | 6 +- prover/protocol/compiler/vortex/prover.go | 2 +- .../conglomeration/conglomeration.go | 36 +++++++--- .../distributed/conglomeration/prover.go | 18 +++-- .../distributed/conglomeration/translator.go | 71 +++++++++++++++---- prover/protocol/wizard/gnark_verifier.go | 13 ++++ prover/protocol/wizard/verifier.go | 16 +++++ 9 files changed, 132 insertions(+), 34 deletions(-) diff --git a/prover/protocol/compiler/selfrecursion/precomputations.go b/prover/protocol/compiler/selfrecursion/precomputations.go index e1fd9589734..159aba4d3be 100644 --- a/prover/protocol/compiler/selfrecursion/precomputations.go +++ b/prover/protocol/compiler/selfrecursion/precomputations.go @@ -95,7 +95,7 @@ func (ctx *SelfRecursionCtx) registersAh() { // associated Dh should be nil. That happens when the examinated round // is a "dry" round or when it has been self-recursed already. if (len(comsInRoundsI) == 0) != (ctx.Columns.Rooth[i] == nil) { - panic("nilness mismatch") + utils.Panic("nilness mismatch for round=%v #coms-in-round=%v vs root-is-nil=%v", i, len(comsInRoundsI), ctx.Columns.Rooth[i] == nil) } // Check if there is no rows to commit diff --git a/prover/protocol/compiler/univariates/multi_to_single_point.go b/prover/protocol/compiler/univariates/multi_to_single_point.go index ea82e4fff85..f9c560cb302 100644 --- a/prover/protocol/compiler/univariates/multi_to_single_point.go +++ b/prover/protocol/compiler/univariates/multi_to_single_point.go @@ -402,7 +402,7 @@ func (ctx mptsCtx) verifier(run wizard.Runtime) error { ys, hs := ctx.getYsHs( run.GetUnivariateParams, func(qName ifaces.QueryID) query.UnivariateEval { - return run.GetSpec().QueriesParams.Data(qName).(query.UnivariateEval) + return run.GetQuery(qName).(query.UnivariateEval) }, ) diff --git a/prover/protocol/compiler/vortex/compiler.go b/prover/protocol/compiler/vortex/compiler.go index 55aeba54f60..9f690d7cba8 100644 --- a/prover/protocol/compiler/vortex/compiler.go +++ b/prover/protocol/compiler/vortex/compiler.go @@ -142,7 +142,7 @@ type Ctx struct { // Committed matrix (rs encoded) of the precomputed columns CommittedMatrix vortex.EncodedMatrix // Tree in case of Merkle mode - tree *smt.Tree + Tree *smt.Tree // colHashes used in self recursion DhWithMerkle []field.Element } @@ -186,7 +186,7 @@ func newCtx(comp *wizard.CompiledIOP, univQ query.UnivariateEval, blowUpFactor i PrecomputedColums []ifaces.Column MerkleRoot ifaces.Column CommittedMatrix vortex.EncodedMatrix - tree *smt.Tree + Tree *smt.Tree DhWithMerkle []field.Element } Alpha coin.Info @@ -710,7 +710,7 @@ func (ctx *Ctx) commitPrecomputeds() { committedMatrix, tree, colHashes := ctx.VortexParams.CommitMerkle(pols) ctx.Items.Precomputeds.DhWithMerkle = colHashes ctx.Items.Precomputeds.CommittedMatrix = committedMatrix - ctx.Items.Precomputeds.tree = tree + ctx.Items.Precomputeds.Tree = tree // And assign the 1-sized column to contain the root var root field.Element diff --git a/prover/protocol/compiler/vortex/prover.go b/prover/protocol/compiler/vortex/prover.go index 65db1105111..4b81787d670 100644 --- a/prover/protocol/compiler/vortex/prover.go +++ b/prover/protocol/compiler/vortex/prover.go @@ -118,7 +118,7 @@ func (ctx *Ctx) OpenSelectedColumns(pr *wizard.ProverRuntime) { // Append the precomputed committedMatrices and trees when IsCommitToPrecomputed is true if ctx.IsCommitToPrecomputed() { committedMatrices = append(committedMatrices, ctx.Items.Precomputeds.CommittedMatrix) - trees = append(trees, ctx.Items.Precomputeds.tree) + trees = append(trees, ctx.Items.Precomputeds.Tree) } for round := 0; round <= ctx.MaxCommittedRound; round++ { diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index e433d1a9ac0..e56c03993f0 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -28,12 +28,13 @@ type recursionCtx struct { // They are added in the target 'comp' but are assigned to zero. Although, // they do not directly play a role in the protocol anymore, they are still // referenced by the self-recursion compiler. - ColumnsIgnored [][]ifaces.Column - QueryParams [][]ifaces.Query - VerifierActions [][]wizard.VerifierAction - Coins [][]coin.Info - FsHooks [][]wizard.VerifierAction - LocalOpenings []query.LocalOpening + ColumnIgnoredPrecomputed []ifaces.Column + ColumnsIgnored [][]ifaces.Column + QueryParams [][]ifaces.Query + VerifierActions [][]wizard.VerifierAction + Coins [][]coin.Info + FsHooks [][]wizard.VerifierAction + LocalOpenings []query.LocalOpening } // ConglomerateDefineFunc returns a function that defines a conglomerate @@ -155,7 +156,14 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { continue } - newCol := ctx.Translator.InsertColumn(col) + var newCol ifaces.Column + + if tmpl.Precomputed.Exists(colName) { + newCol = ctx.Translator.InsertPrecomputed(col, tmpl.Precomputed.MustGet(colName)) + } else { + newCol = ctx.Translator.InsertColumn(col) + } + ctx.Columns[round] = append(ctx.Columns[round], newCol) ctx.Translator.Target.Columns.ExcludeFromProverFS(newCol.GetColID()) } @@ -213,10 +221,19 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { var ( - srcVortexCtx = tmpl.PcsCtxs.(*vortex.Ctx) - comsByRound = srcVortexCtx.CommitmentsByRounds.Inner() + srcVortexCtx = tmpl.PcsCtxs.(*vortex.Ctx) + comsByRound = srcVortexCtx.CommitmentsByRounds.Inner() + srcPrecomputed = srcVortexCtx.Items.Precomputeds.PrecomputedColums ) + ctx.ColumnIgnoredPrecomputed = make([]ifaces.Column, len(srcPrecomputed)) + for i := range srcPrecomputed { + ctx.ColumnIgnoredPrecomputed[i] = ctx.Translator.InsertPrecomputed( + srcPrecomputed[i].(column.Natural), + tmpl.Precomputed.MustGet(srcPrecomputed[i].GetColID()), + ) + } + for _, coms := range comsByRound { ctx.ColumnsIgnored = append(ctx.ColumnsIgnored, nil) for _, comID := range coms { @@ -260,6 +277,7 @@ func (ctx *recursionCtx) captureVortexCtx(tmpl *wizard.CompiledIOP) { dstVortexCtx.Items.Precomputeds.MerkleRoot = ctx.Translator.GetColumn(srcVortexCtx.Items.Precomputeds.MerkleRoot.GetColID()) dstVortexCtx.Items.Precomputeds.CommittedMatrix = srcVortexCtx.Items.Precomputeds.CommittedMatrix dstVortexCtx.Items.Precomputeds.DhWithMerkle = srcVortexCtx.Items.Precomputeds.DhWithMerkle + dstVortexCtx.Items.Precomputeds.Tree = srcVortexCtx.Items.Precomputeds.Tree } dstVortexCtx.Items.Alpha = ctx.Translator.InsertCoin(srcVortexCtx.Items.Alpha) diff --git a/prover/protocol/distributed/conglomeration/prover.go b/prover/protocol/distributed/conglomeration/prover.go index 94338709245..42c935395e5 100644 --- a/prover/protocol/distributed/conglomeration/prover.go +++ b/prover/protocol/distributed/conglomeration/prover.go @@ -8,6 +8,7 @@ import ( vCom "github.com/consensys/linea-monorepo/prover/crypto/vortex" "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/column" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/wizard" "github.com/consensys/linea-monorepo/prover/utils" @@ -132,14 +133,12 @@ func ExtractWitness(run *wizard.ProverRuntime) Witness { if committedMatrix != nil { committedMatrices = append(committedMatrices, committedMatrix.(vCom.EncodedMatrix)) - } - - if sisHash != nil { sisHashes = append(sisHashes, sisHash.([]field.Element)) - } - - if tree != nil { trees = append(trees, tree.(*smt.Tree)) + } else { + committedMatrices = append(committedMatrices, nil) + sisHashes = append(sisHashes, nil) + trees = append(trees, nil) } } @@ -165,6 +164,13 @@ func (pa PreVortexProverStep) Run(run *wizard.ProverRuntime) { for _, col := range colums { name := unprefix(prefix, col.GetColID()) + + if col.(column.Natural).Status() == column.VerifyingKey { + // those don't need to be assigned and not included in the + // proof either. + continue + } + run.AssignColumn(col.GetColID(), proof.Messages.MustGet(name)) } diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 373409ef83a..5765a49a01f 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -1,6 +1,8 @@ package conglomeration import ( + "fmt" + "github.com/consensys/gnark/frontend" "github.com/consensys/linea-monorepo/prover/crypto/fiatshamir" "github.com/consensys/linea-monorepo/prover/crypto/mimc/gkrmimc" @@ -40,12 +42,37 @@ type gnarkRuntimeTranslator struct { } // InsertColumn inserts a new column in the target compiled IOP. The column name -// is prefixed with comp.Prefix. +// is prefixed with comp.Prefix. The function checks that the passed column does +// not have a precomputed status (e.g. either precomputed or verifying key). func (comp *compTranslator) InsertColumn(col column.Natural) ifaces.Column { + + switch col.Status() { + case column.Precomputed, column.VerifyingKey: + panic("cannot insert a precomputed or verifying key column as normal column. Use [InsertPrecomputed] instead") + } name := ifaces.ColID(comp.Prefix) + "." + col.ID return comp.Target.InsertColumn(col.Round(), name, col.Size(), col.Status()) } +// InsertPrecomputed inserts a new column as a precomputed column to the target +// compiled IOP. To differ with [InsertColumn], this method does also add the +// column to the list of precomputed columns. +func (comp *compTranslator) InsertPrecomputed(col column.Natural, ass ifaces.ColAssignment) ifaces.Column { + name := ifaces.ColID(comp.Prefix) + "." + col.ID + + switch col.Status() { + case column.VerifyingKey: + // assertedly, the round of a precomputed column is always 0 + col := comp.Target.InsertColumn(0, name, col.Size(), col.Status()) + comp.Target.Precomputed.InsertNew(name, ass) + return col + case column.Precomputed, column.Ignored: + return comp.Target.InsertPrecomputed(name, ass) + default: + panic(fmt.Sprintf("not a precomputed column: status=%v name=%v", col.Status().String(), col.ID)) + } +} + // InsertColumns inserts a list of columns in the target compiled IOP by adding // a prefix to their names. The inputs columns are expected to be of type // Natural or this will lead to a panic. @@ -91,33 +118,41 @@ func (comp *compTranslator) GetCoin(name coin.Name) coin.Info { // the inserted query will be invalid. func (comp *compTranslator) InsertQueryParams(round int, q ifaces.Query) ifaces.Query { name := ifaces.QueryID(comp.Prefix) + "." + q.Name() - q = copyQueryWithName(name, q) - comp.Target.QueriesParams.AddToRound(round, name, q) - return q -} -// copyQueryWithName returns a copy of the query with a new name. -func copyQueryWithName(name ifaces.QueryID, q ifaces.Query) ifaces.Query { + var q2 ifaces.Query switch q := q.(type) { case query.UnivariateEval: - return query.NewUnivariateEval(name, q.Pols...) + q2 = query.NewUnivariateEval(name, q.Pols...) case query.LocalOpening: - return query.NewLocalOpening(name, q.Pol) + q2 = query.NewLocalOpening(name, q.Pol) case query.InnerProduct: - return query.NewInnerProduct(name, q.A, q.Bs...) + q2 = query.NewInnerProduct(name, q.A, q.Bs...) case query.GrandProduct: - return query.NewGrandProduct(q.Round, q.Inputs, name) + q2 = query.NewGrandProduct(q.Round, q.Inputs, name) case query.LogDerivativeSum: - return query.NewLogDerivativeSum(q.Round, q.Inputs, name) + q2 = query.NewLogDerivativeSum(q.Round, q.Inputs, name) default: panic("unknown query type") } + + comp.Target.QueriesParams.AddToRound(round, name, q2) + comp.Target.QueriesParams.MarkAsIgnored(q2.Name()) + return q2 } -// TranslateColumnList translates a collection of pre-inserted columns +// TranslateColumnList translates a collection of pre-inserted columns. +// If one of the columns provided in the list is nil, it will be ignored +// and the function will return nil at the same position in the returned +// list of column. func (comp *compTranslator) TranslateColumnList(cols []ifaces.Column) []ifaces.Column { res := make([]ifaces.Column, 0, len(cols)) for _, col := range cols { + + if col == nil { + res = append(res, nil) + continue + } + res = append(res, comp.GetColumn(col.GetColID())) } return res @@ -241,6 +276,11 @@ func (run *runtimeTranslator) SetState(name string, value any) { run.Rt.SetState(name, value) } +func (run *runtimeTranslator) GetQuery(name ifaces.QueryID) ifaces.Query { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetQuery(name) +} + func (run *gnarkRuntimeTranslator) GetColumn(name ifaces.ColID) []frontend.Variable { name = ifaces.ColID(run.Prefix) + "." + name return run.Rt.GetColumn(name) @@ -331,3 +371,8 @@ func (run *gnarkRuntimeTranslator) SetState(name string, value any) { name = run.Prefix + "." + name run.Rt.SetState(name, value) } + +func (run *gnarkRuntimeTranslator) GetQuery(name ifaces.QueryID) ifaces.Query { + name = ifaces.QueryID(run.Prefix) + "." + name + return run.Rt.GetQuery(name) +} diff --git a/prover/protocol/wizard/gnark_verifier.go b/prover/protocol/wizard/gnark_verifier.go index 345fd3f7b8d..c8ad4c179ff 100644 --- a/prover/protocol/wizard/gnark_verifier.go +++ b/prover/protocol/wizard/gnark_verifier.go @@ -34,6 +34,7 @@ type GnarkRuntime interface { InsertCoin(name coin.Name, value interface{}) GetState(name string) (any, bool) SetState(name string, value any) + GetQuery(name ifaces.QueryID) ifaces.Query } // GnarkVerifierStep functions that can be registered in the CompiledIOP by the successive @@ -631,3 +632,15 @@ func (c *WizardVerifierCircuit) GetState(name string) (any, bool) { func (c *WizardVerifierCircuit) SetState(name string, value any) { c.State[name] = value } + +// GetQuery returns a query from its name +func (c *WizardVerifierCircuit) GetQuery(name ifaces.QueryID) ifaces.Query { + if c.Spec.QueriesParams.Exists(name) { + return c.Spec.QueriesParams.Data(name) + } + if c.Spec.QueriesNoParams.Exists(name) { + return c.Spec.QueriesNoParams.Data(name) + } + utils.Panic("could not find query nb %v", name) + return nil +} diff --git a/prover/protocol/wizard/verifier.go b/prover/protocol/wizard/verifier.go index 1513143adf6..b6a9da08637 100644 --- a/prover/protocol/wizard/verifier.go +++ b/prover/protocol/wizard/verifier.go @@ -46,6 +46,7 @@ type Runtime interface { GetInnerProductParams(name ifaces.QueryID) query.InnerProductParams GetUnivariateEval(name ifaces.QueryID) query.UnivariateEval GetUnivariateParams(name ifaces.QueryID) query.UnivariateEvalParams + GetQuery(name ifaces.QueryID) ifaces.Query Fs() *fiatshamir.State FsHistory() [][2][]field.Element InsertCoin(name coin.Name, value any) @@ -465,3 +466,18 @@ func (run *VerifierRuntime) GetState(name string) (any, bool) { func (run *VerifierRuntime) SetState(name string, value any) { run.State[name] = value } + +// GetQuery returns a query from its name +func (run *VerifierRuntime) GetQuery(name ifaces.QueryID) ifaces.Query { + + if run.Spec.QueriesParams.Exists(name) { + return run.Spec.QueriesParams.Data(name) + } + + if run.Spec.QueriesNoParams.Exists(name) { + return run.Spec.QueriesNoParams.Data(name) + } + + utils.Panic("could not find query nb %v", name) + return nil +} From f4930c4dc7c5eaaa06930e8dbe79b3fe56b81905 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Wed, 22 Jan 2025 14:53:56 +0000 Subject: [PATCH 52/63] fixup --- .../conglomeration/conglomeration_test.go | 72 ++++++++++++++++--- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index 39ab34c914c..b8f4edb228b 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -1,4 +1,4 @@ -package conglomeration +package conglomeration_test import ( "testing" @@ -7,8 +7,10 @@ import ( "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/coin" + "github.com/consensys/linea-monorepo/prover/protocol/compiler" "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" + "github.com/consensys/linea-monorepo/prover/protocol/distributed/conglomeration" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -18,12 +20,31 @@ import ( type ( defineFuncType func(*wizard.Builder) proverFuncType func(int) func(*wizard.ProverRuntime) + compilerSuite []func(*wizard.CompiledIOP) +) + +var ( + commonSisParams = &ringsis.Params{LogTwoBound: 16, LogTwoDegree: 1} + commonVortexStep = vortex.Compile( + 2, // the inverse-rate of the RS code + vortex.WithSISParams(commonSisParams), + vortex.ForceNumOpenedColumns(2), + vortex.PremarkAsSelfRecursed(), + ) + vortexOnlyCompilationSuite = []func(*wizard.CompiledIOP){ + commonVortexStep, + } + arcaneCompilationSuite = []func(*wizard.CompiledIOP){ + compiler.Arcane(1<<8, 1<<10, false), + commonVortexStep, + } ) type conglomerationTestCase struct { define defineFuncType prove proverFuncType numProof int + suite compilerSuite } func TestConglomerationPureVortexSingleRound(t *testing.T) { @@ -59,6 +80,7 @@ func TestConglomerationPureVortexSingleRound(t *testing.T) { define: define, prove: prover, numProof: numProof, + suite: vortexOnlyCompilationSuite, }) } @@ -123,11 +145,9 @@ func TestConglomerationPureVortexMultiRound(t *testing.T) { prover := func(k int) func(run *wizard.ProverRuntime) { return func(run *wizard.ProverRuntime) { - ys := make([]field.Element, 0, len(a)) for i := range a { y := field.NewElement(uint64(i + k)) run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numRow)) - ys = append(ys, y) } } } @@ -136,29 +156,61 @@ func TestConglomerationPureVortexMultiRound(t *testing.T) { define: define, prove: prover, numProof: numProof, + suite: vortexOnlyCompilationSuite, + }) +} + +func TestConglomerationLookup(t *testing.T) { + + var ( + numCol = 16 + numRow = 16 + numProof = 16 + a []ifaces.Column + ) + + define := func(builder *wizard.Builder) { + for i := 0; i < numCol; i++ { + a = append(a, builder.RegisterCommit(ifaces.ColIDf("a-%v", i), numRow)) + builder.Range(ifaces.QueryIDf("range-%v", i), a[i], 1<<16) + } + } + + prover := func(k int) func(run *wizard.ProverRuntime) { + return func(run *wizard.ProverRuntime) { + for i := range a { + y := field.NewElement(uint64(i + k)) + run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numRow)) + } + } + } + + runConglomerationTestCase(t, conglomerationTestCase{ + define: define, + prove: prover, + numProof: numProof, + suite: arcaneCompilationSuite, }) } func runConglomerationTestCase(t *testing.T, tc conglomerationTestCase) { var ( - sisParams = &ringsis.Params{LogTwoBound: 16, LogTwoDegree: 1} - vortexCompFunc = vortex.Compile(2, vortex.WithSISParams(sisParams), vortex.ForceNumOpenedColumns(2), vortex.PremarkAsSelfRecursed()) numProof = tc.numProof - tmpl = wizard.Compile(wizard.DefineFunc(tc.define), vortexCompFunc) - congDef, ctxsPHolder = ConglomerateDefineFunc(tmpl, numProof) + tmpl = wizard.Compile(wizard.DefineFunc(tc.define), tc.suite...) + congDef, ctxsPHolder = conglomeration.ConglomerateDefineFunc(tmpl, numProof) cong = wizard.Compile(congDef, dummy.CompileAtProverLvl) ctxs = *ctxsPHolder lastRound = ctxs[0].LastRound ) - witnesses := make([]Witness, numProof) + witnesses := make([]conglomeration.Witness, numProof) for i := range witnesses { runtime := wizard.RunProverUntilRound(tmpl, tc.prove(i), lastRound+1) - witnesses[i] = ExtractWitness(runtime) + witnesses[i] = conglomeration.ExtractWitness(runtime) } - proof := wizard.Prove(cong, ProveConglomeration(ctxs, witnesses)) + proof := wizard.Prove(cong, conglomeration.ProveConglomeration(ctxs, witnesses)) err := wizard.Verify(cong, proof) require.NoError(t, err) From 9a30baa647113e6a9be570b3e2167d632c75c219 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 24 Jan 2025 14:25:12 +0100 Subject: [PATCH 53/63] testing(conglo): complete testing using sophisticated suites as input protocols --- .../conglomeration/conglomeration.go | 2 +- .../conglomeration/conglomeration_test.go | 96 ++++++++++++++----- .../distributed/conglomeration/translator.go | 20 ++++ prover/protocol/wizard/compiled.go | 6 +- prover/protocol/wizard/prover.go | 8 +- 5 files changed, 102 insertions(+), 30 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index e56c03993f0..b30d2519446 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -162,9 +162,9 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { newCol = ctx.Translator.InsertPrecomputed(col, tmpl.Precomputed.MustGet(colName)) } else { newCol = ctx.Translator.InsertColumn(col) + ctx.Columns[round] = append(ctx.Columns[round], newCol) } - ctx.Columns[round] = append(ctx.Columns[round], newCol) ctx.Translator.Target.Columns.ExcludeFromProverFS(newCol.GetColID()) } diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index b8f4edb228b..f489b6665a2 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -9,6 +9,9 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/compiler" "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/fullrecursion" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/mimc" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/selfrecursion" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/distributed/conglomeration" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" @@ -38,6 +41,26 @@ var ( compiler.Arcane(1<<8, 1<<10, false), commonVortexStep, } + arcaneAndSelfRecCompilationSuite = []func(*wizard.CompiledIOP){ + compiler.Arcane(1<<8, 1<<10, false), + commonVortexStep, + selfrecursion.SelfRecurse, + mimc.CompileMiMC, + compiler.Arcane(1<<8, 1<<10, false), + commonVortexStep, + } + arcaneFullRecSelfRecCompilationSuite = []func(*wizard.CompiledIOP){ + compiler.Arcane(1<<8, 1<<10, false), + commonVortexStep, + selfrecursion.SelfRecurse, + mimc.CompileMiMC, + compiler.Arcane(1<<8, 1<<10, false), + commonVortexStep, + fullrecursion.FullRecursion(true), + mimc.CompileMiMC, + compiler.Arcane(1<<8, 1<<10, false), + commonVortexStep, + } ) type conglomerationTestCase struct { @@ -162,35 +185,58 @@ func TestConglomerationPureVortexMultiRound(t *testing.T) { func TestConglomerationLookup(t *testing.T) { - var ( - numCol = 16 - numRow = 16 - numProof = 16 - a []ifaces.Column - ) - - define := func(builder *wizard.Builder) { - for i := 0; i < numCol; i++ { - a = append(a, builder.RegisterCommit(ifaces.ColIDf("a-%v", i), numRow)) - builder.Range(ifaces.QueryIDf("range-%v", i), a[i], 1<<16) - } + tcs := []struct { + name string + suite compilerSuite + }{ + { + name: "arcane", + suite: arcaneCompilationSuite, + }, + { + name: "arcane/self-recursion", + suite: arcaneAndSelfRecCompilationSuite, + }, + { + name: "arcane/full-recursion/self-recursion", + suite: arcaneFullRecSelfRecCompilationSuite, + }, } - prover := func(k int) func(run *wizard.ProverRuntime) { - return func(run *wizard.ProverRuntime) { - for i := range a { - y := field.NewElement(uint64(i + k)) - run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numRow)) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + + var ( + numCol = 16 + numRow = 16 + numProof = 16 + a []ifaces.Column + ) + + define := func(builder *wizard.Builder) { + for i := 0; i < numCol; i++ { + a = append(a, builder.RegisterCommit(ifaces.ColIDf("a-%v", i), numRow)) + builder.Range(ifaces.QueryIDf("range-%v", i), a[i], 1<<16) + } } - } - } - runConglomerationTestCase(t, conglomerationTestCase{ - define: define, - prove: prover, - numProof: numProof, - suite: arcaneCompilationSuite, - }) + prover := func(k int) func(run *wizard.ProverRuntime) { + return func(run *wizard.ProverRuntime) { + for i := range a { + y := field.NewElement(uint64(i + k)) + run.AssignColumn(a[i].GetColID(), smartvectors.NewConstant(y, numRow)) + } + } + } + + runConglomerationTestCase(t, conglomerationTestCase{ + define: define, + prove: prover, + numProof: numProof, + suite: tc.suite, + }) + }) + } } func runConglomerationTestCase(t *testing.T, tc conglomerationTestCase) { diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 5765a49a01f..7e3ec4c3a63 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -9,6 +9,7 @@ import ( "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/column" + "github.com/consensys/linea-monorepo/prover/protocol/column/verifiercol" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -163,6 +164,13 @@ func (comp *compTranslator) TranslateColumnVecVec(cols collection.VecVec[ifaces. var res = collection.NewVecVec[ifaces.ColID]() for r, vec := range cols.Inner() { for _, c := range vec { + + // If it does not exists, then it is a verifier column + if !comp.Target.Columns.Exists(c) { + res.AppendToInner(r, c) + continue + } + res.AppendToInner(r, comp.GetColumn(c).GetColID()) } } @@ -173,6 +181,13 @@ func (comp *compTranslator) TranslateColumnVecVec(cols collection.VecVec[ifaces. func (comp *compTranslator) TranslateColumnSet(cols map[ifaces.ColID]struct{}) map[ifaces.ColID]struct{} { var res = make(map[ifaces.ColID]struct{}) for col := range cols { + + // If it does not exists, then it is a verifier column + if !comp.Target.Columns.Exists(col) { + res[col] = struct{}{} + continue + } + res[comp.GetColumn(col).GetColID()] = struct{}{} } return res @@ -183,6 +198,11 @@ func (comp *compTranslator) TranslateColumnSet(cols map[ifaces.ColID]struct{}) m func (comp *compTranslator) TranslateUniEval(round int, q query.UnivariateEval) query.UnivariateEval { newPols := make([]ifaces.Column, len(q.Pols)) for i := range newPols { + if _, ok := q.Pols[i].(verifiercol.VerifierCol); ok { + newPols[i] = q.Pols[i] + continue + } + newPols[i] = comp.GetColumn(q.Pols[i].GetColID()) } var res = query.NewUnivariateEval(q.QueryID, newPols...) diff --git a/prover/protocol/wizard/compiled.go b/prover/protocol/wizard/compiled.go index c68f567e6e4..38b107754e4 100644 --- a/prover/protocol/wizard/compiled.go +++ b/prover/protocol/wizard/compiled.go @@ -597,9 +597,9 @@ func (c *CompiledIOP) InsertLogDerivativeSum(lastRound int, id ifaces.QueryID, i // assertConsistentRound compares the round passed as an argument and panic if it greater than // coin.Round. This helps ensuring that we do not have "useless" rounds. func (c *CompiledIOP) assertConsistentRound(round int) { - if round > c.Coins.NumRounds() { - utils.Panic("Inserted at round %v, but the max should be %v", round, c.Coins.NumRounds()) - } + // if round > c.Coins.NumRounds() { + // utils.Panic("Inserted at round %v, but the max should be %v", round, c.Coins.NumRounds()) + // } } // InsertMiMC declares a MiMC constraints query; a constraint that all the diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index 0ae1759735e..0214ae6b2c5 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -1,6 +1,7 @@ package wizard import ( + "fmt" "sync" "github.com/consensys/linea-monorepo/prover/crypto/fiatshamir" @@ -540,6 +541,7 @@ func (run *ProverRuntime) getRandomCoinGeneric(name coin.Name, requestedType coi func (run *ProverRuntime) goNextRound() { initialState := run.FS.State() + fmt.Printf("[prover-fs] round=%v initial-state=%v\n", run.currRound+1, initialState[0].String()) if !run.Spec.DummyCompiled { @@ -556,6 +558,10 @@ func (run *ProverRuntime) goNextRound() { continue } + if run.Spec.Precomputed.Exists(msgName) { + continue + } + instance := run.GetMessage(msgName) run.FS.UpdateSV(instance) } @@ -599,7 +605,7 @@ func (run *ProverRuntime) goNextRound() { toCompute := run.Spec.Coins.AllKeysAt(run.currRound) for _, coin := range toCompute { - if run.Spec.Coins.IsSkippedFromVerifierTranscript(coin) { + if run.Spec.Coins.IsSkippedFromProverTranscript(coin) { continue } From 1fbf002b67b8d1858e0dcbd99d46e702f73078d5 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 24 Jan 2025 14:27:15 +0100 Subject: [PATCH 54/63] fixup(printf): remove trailing printf --- prover/protocol/wizard/prover.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index 0214ae6b2c5..17115ae1e97 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -1,7 +1,6 @@ package wizard import ( - "fmt" "sync" "github.com/consensys/linea-monorepo/prover/crypto/fiatshamir" @@ -541,7 +540,6 @@ func (run *ProverRuntime) getRandomCoinGeneric(name coin.Name, requestedType coi func (run *ProverRuntime) goNextRound() { initialState := run.FS.State() - fmt.Printf("[prover-fs] round=%v initial-state=%v\n", run.currRound+1, initialState[0].String()) if !run.Spec.DummyCompiled { From 9f51da0d19a941e9f67fd34532b706221dee4d2c Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 24 Jan 2025 14:28:38 +0100 Subject: [PATCH 55/63] simplify(wizard): remove the assert consistent round sanity-check function --- prover/protocol/wizard/compiled.go | 31 ------------------------------ 1 file changed, 31 deletions(-) diff --git a/prover/protocol/wizard/compiled.go b/prover/protocol/wizard/compiled.go index 38b107754e4..c9c82a29880 100644 --- a/prover/protocol/wizard/compiled.go +++ b/prover/protocol/wizard/compiled.go @@ -174,8 +174,6 @@ func (c *CompiledIOP) InsertColumn(round int, name ifaces.ColID, size int, statu utils.Panic("column %v has size %v", name, size) } - c.assertConsistentRound(round) - if len(name) == 0 { panic("Column with an empty name") } @@ -220,8 +218,6 @@ func (c *CompiledIOP) InsertCoin(round int, name coin.Name, type_ coin.Type, siz // - the definition round is inconsistent with the expression func (c *CompiledIOP) InsertGlobal(round int, name ifaces.QueryID, expr *symbolic.Expression, noBoundCancel ...bool) query.GlobalConstraint { - c.assertConsistentRound(round) - // The constructor of the global constraint is assumed to perform all the // well-formation checks of the constraint. cs := query.NewGlobalConstraint(name, expr, noBoundCancel...) @@ -264,8 +260,6 @@ func (c *CompiledIOP) InsertGlobal(round int, name ifaces.QueryID, expr *symboli // - the definition round is inconsistent with the expression func (c *CompiledIOP) InsertLocal(round int, name ifaces.QueryID, cs_ *symbolic.Expression) query.LocalConstraint { - c.assertConsistentRound(round) - cs := query.NewLocalConstraint(name, cs_) boarded := cs.Board() metadatas := boarded.ListVariableMetadata() @@ -300,7 +294,6 @@ func (c *CompiledIOP) InsertLocal(round int, name ifaces.QueryID, cs_ *symbolic. // - any column in `a` or `b“ is a not registered columns // - a constraint with the same name already exists in the CompiledIOP func (c *CompiledIOP) InsertPermutation(round int, name ifaces.QueryID, a, b []ifaces.Column) query.Permutation { - c.assertConsistentRound(round) query_ := query.NewPermutation(name, [][]ifaces.Column{a}, [][]ifaces.Column{b}) c.QueriesNoParams.AddToRound(round, name, query_) return query_ @@ -310,7 +303,6 @@ func (c *CompiledIOP) InsertPermutation(round int, name ifaces.QueryID, a, b []i // fragmented tables. Meanining that permutation operates over the union of // the rows of multiple tables. func (c *CompiledIOP) InsertFragmentedPermutation(round int, name ifaces.QueryID, a, b [][]ifaces.Column) query.Permutation { - c.assertConsistentRound(round) query_ := query.NewPermutation(name, a, b) c.QueriesNoParams.AddToRound(round, name, query_) return query_ @@ -343,7 +335,6 @@ func (c *CompiledIOP) InsertFixedPermutation(round int, name ifaces.QueryID, p [ // - the columns in `included` do not all have the same size // - a constraint with the same name already exists in the CompiledIOP func (c *CompiledIOP) InsertInclusion(round int, name ifaces.QueryID, including, included []ifaces.Column) { - c.assertConsistentRound(round) query := query.NewInclusion(name, included, [][]ifaces.Column{including}, nil, nil) c.QueriesNoParams.AddToRound(round, name, query) } @@ -353,7 +344,6 @@ Creates an inclusion query. Both the including and the included tables are filte the filters should be columns containing only field elements for 0 and 1 */ func (c *CompiledIOP) InsertInclusionDoubleConditional(round int, name ifaces.QueryID, including, included []ifaces.Column, includingFilter, includedFilter ifaces.Column) { - c.assertConsistentRound(round) query := query.NewInclusion(name, included, [][]ifaces.Column{including}, includedFilter, []ifaces.Column{includingFilter}) c.QueriesNoParams.AddToRound(round, name, query) } @@ -363,7 +353,6 @@ Creates an inclusion query. Only the including table is filtered the filters should be columns containing only field elements for 0 and 1 */ func (c *CompiledIOP) InsertInclusionConditionalOnIncluding(round int, name ifaces.QueryID, including, included []ifaces.Column, includingFilter ifaces.Column) { - c.assertConsistentRound(round) query := query.NewInclusion(name, included, [][]ifaces.Column{including}, nil, []ifaces.Column{includingFilter}) c.QueriesNoParams.AddToRound(round, name, query) } @@ -373,7 +362,6 @@ Creates an inclusion query. Only the included table is filtered the filters should be columns containing only field elements for 0 and 1 */ func (c *CompiledIOP) InsertInclusionConditionalOnIncluded(round int, name ifaces.QueryID, including, included []ifaces.Column, includedFilter ifaces.Column) { - c.assertConsistentRound(round) query := query.NewInclusion(name, included, [][]ifaces.Column{including}, includedFilter, nil) c.QueriesNoParams.AddToRound(round, name, query) } @@ -394,7 +382,6 @@ func (c *CompiledIOP) GenericFragmentedConditionalInclusion( includingFilter []ifaces.Column, includedFilter ifaces.Column, ) { - c.assertConsistentRound(round) query := query.NewInclusion(name, included, including, includedFilter, includingFilter) c.QueriesNoParams.AddToRound(round, name, query) } @@ -437,7 +424,6 @@ func (c *CompiledIOP) InsertPrecomputed(name ifaces.ColID, v smartvectors.SmartV // // The name must be non-empty and unique and the size must be a power of 2. func (c *CompiledIOP) InsertProof(round int, name ifaces.ColID, size int) (msg ifaces.Column) { - c.assertConsistentRound(round) // Common : No zero length if size == 0 { @@ -452,7 +438,6 @@ func (c *CompiledIOP) InsertProof(round int, name ifaces.ColID, size int) (msg i // Deprecated: we never really use this type of column to denote actual public // inputs. The plan is to resort to using [query.LocalOpeningParams] instead. func (c *CompiledIOP) InsertPublicInput(round int, name ifaces.ColID, size int) (msg ifaces.Column) { - c.assertConsistentRound(round) // Common : No zero length if size == 0 { @@ -479,7 +464,6 @@ func (c *CompiledIOP) InsertPublicInput(round int, name ifaces.ColID, size int) // not intend to run the verifier of the Wizard protocol in a gnark circuit, // passing `nil` is fine. func (c *CompiledIOP) InsertVerifier(round int, ver VerifierStep, gnarkVer GnarkVerifierStep) { - c.assertConsistentRound(round) c.SubVerifiers.AppendToInner(round, &genVerifierAction{ run: ver, runGnark: gnarkVer, @@ -510,7 +494,6 @@ func (c *CompiledIOP) InsertRange(round int, name ifaces.QueryID, h ifaces.Colum panic("max is zero : perhaps an overflow") } - c.assertConsistentRound(round) /* In case the range is applied over a composite handle. We apply the range over each natural component of the handle. @@ -531,7 +514,6 @@ func (c *CompiledIOP) InsertRange(round int, name ifaces.QueryID, h ifaces.Colum // - a query with the same name has already been registered in the Wizard // - the provided columns `a` and `bs` do not all have the same size func (c *CompiledIOP) InsertInnerProduct(round int, name ifaces.QueryID, a ifaces.Column, bs []ifaces.Column) query.InnerProduct { - c.assertConsistentRound(round) // Also ensures that the query round does not predates the columns rounds maxComRound := a.Round() @@ -565,7 +547,6 @@ func (run *CompiledIOP) GetInnerProduct(name ifaces.QueryID) query.InnerProduct // - the name is the empty string // - a query with the same name has already been registered in the Wizard func (c *CompiledIOP) InsertUnivariate(round int, name ifaces.QueryID, pols []ifaces.Column) query.UnivariateEval { - c.assertConsistentRound(round) q := query.NewUnivariateEval(name, pols...) // Finally registers the query c.QueriesParams.AddToRound(round, name, q) @@ -576,7 +557,6 @@ func (c *CompiledIOP) InsertUnivariate(round int, name ifaces.QueryID, pols []if // in the current CompiledIOP. A local opening query requires the prover of the // protocol to "open" the first position of the vector. func (c *CompiledIOP) InsertLocalOpening(round int, name ifaces.QueryID, pol ifaces.Column) query.LocalOpening { - c.assertConsistentRound(round) q := query.NewLocalOpening(name, pol) // Finally registers the query c.QueriesParams.AddToRound(round, name, q) @@ -587,21 +567,12 @@ func (c *CompiledIOP) InsertLocalOpening(round int, name ifaces.QueryID, pol ifa // It generates a single global summation for many Sigma Columns from Lookup compilation. // The sigma columns are categorized by [round,size]. func (c *CompiledIOP) InsertLogDerivativeSum(lastRound int, id ifaces.QueryID, in map[int]*query.LogDerivativeSumInput) query.LogDerivativeSum { - c.assertConsistentRound(lastRound) q := query.NewLogDerivativeSum(lastRound, in, id) // Finally registers the query c.QueriesParams.AddToRound(lastRound, id, q) return q } -// assertConsistentRound compares the round passed as an argument and panic if it greater than -// coin.Round. This helps ensuring that we do not have "useless" rounds. -func (c *CompiledIOP) assertConsistentRound(round int) { - // if round > c.Coins.NumRounds() { - // utils.Panic("Inserted at round %v, but the max should be %v", round, c.Coins.NumRounds()) - // } -} - // InsertMiMC declares a MiMC constraints query; a constraint that all the // entries of new are obtained by running the compression function of MiMC over // the entries of block and old, row-by-row. @@ -611,7 +582,6 @@ func (c *CompiledIOP) assertConsistentRound(round int) { // - the declaration round is anterior to the declaration round of the // provided input columns. func (c *CompiledIOP) InsertMiMC(round int, id ifaces.QueryID, block, old, new ifaces.Column) query.MiMC { - c.assertConsistentRound(round) q := query.NewMiMC(id, block, old, new) c.QueriesNoParams.AddToRound(round, id, q) return q @@ -647,7 +617,6 @@ func (c *CompiledIOP) RegisterVerifierAction(round int, action VerifierAction) { // Register a GrandProduct query func (c *CompiledIOP) InsertGrandProduct(round int, id ifaces.QueryID, in map[int]*query.GrandProductInput) query.GrandProduct { - c.assertConsistentRound(round) q := query.NewGrandProduct(round, in, id) // Finally registers the query c.QueriesParams.AddToRound(round, q.Name(), q) From ac8d5f388b5d01c463fe548f070cacd81ed32221 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 24 Jan 2025 14:34:04 +0100 Subject: [PATCH 56/63] ci(test): skip the very heavy conglomeration test --- .../protocol/distributed/conglomeration/conglomeration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index f489b6665a2..d791703ac95 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -185,6 +185,8 @@ func TestConglomerationPureVortexMultiRound(t *testing.T) { func TestConglomerationLookup(t *testing.T) { + t.Skip() + tcs := []struct { name string suite compilerSuite From a6a465eba5820070d30b2c5cd81dee06ceb14c8a Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 3 Feb 2025 07:00:26 +0100 Subject: [PATCH 57/63] fixup(translator): the existence check for the translated column was buggy. --- .../distributed/conglomeration/translator.go | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/translator.go b/prover/protocol/distributed/conglomeration/translator.go index 7e3ec4c3a63..4123cf7647e 100644 --- a/prover/protocol/distributed/conglomeration/translator.go +++ b/prover/protocol/distributed/conglomeration/translator.go @@ -13,6 +13,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/consensys/linea-monorepo/prover/utils" "github.com/consensys/linea-monorepo/prover/utils/collection" ) @@ -92,6 +93,13 @@ func (comp *compTranslator) GetColumn(name ifaces.ColID) ifaces.Column { return comp.Target.Columns.GetHandle(name) } +// ColumnExists returns a boolean indicating of the column is already +// registered in the translator. +func (comp *compTranslator) ColumnExists(name ifaces.ColID) bool { + name = ifaces.ColID(comp.Prefix) + "." + name + return comp.Target.Columns.Exists(name) +} + // InsertCoin inserts a new coin in the target compiled IOP. The coin name // is prefixed with the comp.Prefix. func (comp *compTranslator) InsertCoin(info coin.Info) coin.Info { @@ -144,7 +152,9 @@ func (comp *compTranslator) InsertQueryParams(round int, q ifaces.Query) ifaces. // TranslateColumnList translates a collection of pre-inserted columns. // If one of the columns provided in the list is nil, it will be ignored // and the function will return nil at the same position in the returned -// list of column. +// list of column. If the column is non-nil but not found in the translated +// comp, then it is assumed the column is a verifier col and the column is +// returned as is. func (comp *compTranslator) TranslateColumnList(cols []ifaces.Column) []ifaces.Column { res := make([]ifaces.Column, 0, len(cols)) for _, col := range cols { @@ -154,6 +164,12 @@ func (comp *compTranslator) TranslateColumnList(cols []ifaces.Column) []ifaces.C continue } + if !comp.ColumnExists(col.GetColID()) { + if _, ok := col.(verifiercol.VerifierCol); !ok { + utils.Panic("expects all the unregistered methods to be verifiercol, but got type=%T for column name=%v", col, col.GetColID()) + } + } + res = append(res, comp.GetColumn(col.GetColID())) } return res @@ -165,8 +181,8 @@ func (comp *compTranslator) TranslateColumnVecVec(cols collection.VecVec[ifaces. for r, vec := range cols.Inner() { for _, c := range vec { - // If it does not exists, then it is a verifier column - if !comp.Target.Columns.Exists(c) { + // If it does not exists, then it is assumed to be a verifier column + if !comp.ColumnExists(c) { res.AppendToInner(r, c) continue } @@ -182,8 +198,8 @@ func (comp *compTranslator) TranslateColumnSet(cols map[ifaces.ColID]struct{}) m var res = make(map[ifaces.ColID]struct{}) for col := range cols { - // If it does not exists, then it is a verifier column - if !comp.Target.Columns.Exists(col) { + // If it does not exists, then it is assumed to be a verifier column + if !comp.ColumnExists(col) { res[col] = struct{}{} continue } From b04df265702b3a5460de51909648b401589259af Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 3 Feb 2025 07:16:24 +0100 Subject: [PATCH 58/63] testing(conglo): reduces the computation footprint --- .../conglomeration/conglomeration_test.go | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index d791703ac95..1d72764d931 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -17,6 +17,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -42,11 +43,11 @@ var ( commonVortexStep, } arcaneAndSelfRecCompilationSuite = []func(*wizard.CompiledIOP){ - compiler.Arcane(1<<8, 1<<10, false), + compiler.Arcane(1, 1<<10, false), commonVortexStep, selfrecursion.SelfRecurse, mimc.CompileMiMC, - compiler.Arcane(1<<8, 1<<10, false), + compiler.Arcane(1, 1<<10, false), commonVortexStep, } arcaneFullRecSelfRecCompilationSuite = []func(*wizard.CompiledIOP){ @@ -75,7 +76,7 @@ func TestConglomerationPureVortexSingleRound(t *testing.T) { var ( numCol = 16 numRow = 16 - numProof = 16 + numProof = 2 a []ifaces.Column u query.UnivariateEval ) @@ -185,7 +186,7 @@ func TestConglomerationPureVortexMultiRound(t *testing.T) { func TestConglomerationLookup(t *testing.T) { - t.Skip() + logrus.SetLevel(logrus.FatalLevel) tcs := []struct { name string @@ -199,10 +200,10 @@ func TestConglomerationLookup(t *testing.T) { name: "arcane/self-recursion", suite: arcaneAndSelfRecCompilationSuite, }, - { - name: "arcane/full-recursion/self-recursion", - suite: arcaneFullRecSelfRecCompilationSuite, - }, + // { + // name: "arcane/full-recursion/self-recursion", + // suite: arcaneFullRecSelfRecCompilationSuite, + // }, } for _, tc := range tcs { @@ -211,14 +212,14 @@ func TestConglomerationLookup(t *testing.T) { var ( numCol = 16 numRow = 16 - numProof = 16 + numProof = 4 a []ifaces.Column ) define := func(builder *wizard.Builder) { for i := 0; i < numCol; i++ { a = append(a, builder.RegisterCommit(ifaces.ColIDf("a-%v", i), numRow)) - builder.Range(ifaces.QueryIDf("range-%v", i), a[i], 1<<16) + builder.Range(ifaces.QueryIDf("range-%v", i), a[i], 1<<8) } } From a2c25b561453da4e2fe1441ce83af6b1fbf95b3d Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Mon, 3 Feb 2025 07:16:55 +0100 Subject: [PATCH 59/63] clean(from_alleged_ys): adds a panic clause if a column is not found. --- prover/protocol/column/verifiercol/from_alleged_ys.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/prover/protocol/column/verifiercol/from_alleged_ys.go b/prover/protocol/column/verifiercol/from_alleged_ys.go index 6d332fffb23..8a9cce0bc5c 100644 --- a/prover/protocol/column/verifiercol/from_alleged_ys.go +++ b/prover/protocol/column/verifiercol/from_alleged_ys.go @@ -9,7 +9,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" - "github.com/sirupsen/logrus" + "github.com/consensys/linea-monorepo/prover/utils" ) // compile check to enforce the struct to belong to the corresponding interface @@ -39,10 +39,9 @@ func NewFromYs(comp *wizard.CompiledIOP, q query.UnivariateEval, ranges []ifaces nameMap[polName.GetColID()] = struct{}{} } - // No make the explicit check for _, rangeName := range ranges { if _, ok := nameMap[rangeName]; !ok && !strings.Contains(string(rangeName), "SHADOW") { - logrus.Debugf("NewFromYs : %v is not part of the query %v. It will be zeroized", rangeName, q.QueryID) + utils.Panic("NewFromYs : %v is not part of the query %v", rangeName, q.QueryID) } } From 7e97447836ead3cbad09e035f68331d589aa837d Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 4 Feb 2025 07:41:33 +0100 Subject: [PATCH 60/63] fixup: missed merge change --- prover/protocol/compiler/projection/verifier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prover/protocol/compiler/projection/verifier.go b/prover/protocol/compiler/projection/verifier.go index 59b4f4d0d5d..7ac500068af 100644 --- a/prover/protocol/compiler/projection/verifier.go +++ b/prover/protocol/compiler/projection/verifier.go @@ -20,7 +20,7 @@ type projectionVerifierAction struct { } // Run implements the [wizard.VerifierAction] interface. -func (va *projectionVerifierAction) Run(run *wizard.VerifierRuntime) error { +func (va *projectionVerifierAction) Run(run wizard.Runtime) error { var ( a = run.GetLocalPointEvalParams(va.HornerA0.ID).Y @@ -35,7 +35,7 @@ func (va *projectionVerifierAction) Run(run *wizard.VerifierRuntime) error { } // RunGnark implements the [wizard.VerifierAction] interface. -func (va *projectionVerifierAction) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { +func (va *projectionVerifierAction) RunGnark(api frontend.API, run wizard.GnarkRuntime) { var ( a = run.GetLocalPointEvalParams(va.HornerA0.ID).Y From 9b60dc544ee2cb91eed5f20a2910151126fed1f6 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 4 Feb 2025 07:43:22 +0100 Subject: [PATCH 61/63] clean(legacy): remove deprecated PublicInput column status --- prover/protocol/column/status.go | 6 ++-- prover/protocol/column/store.go | 37 ------------------------ prover/protocol/wizard/compiled.go | 27 ++++++++--------- prover/protocol/wizard/gnark_verifier.go | 6 ---- prover/protocol/wizard/prover.go | 17 ----------- prover/protocol/wizard/verifier.go | 6 ---- 6 files changed, 14 insertions(+), 85 deletions(-) diff --git a/prover/protocol/column/status.go b/prover/protocol/column/status.go index 2e0838c9e22..f8d372be101 100644 --- a/prover/protocol/column/status.go +++ b/prover/protocol/column/status.go @@ -55,7 +55,7 @@ const ( // protocol. Meaning that this is not part of the proof. // // Deprecated: we don't really use this to create public inputs. - PublicInput + _ // VerifyingKey indicates the column is defined offline during the definition // of the protocol or the compilation and that the column is directly // available to the verifier. It is preferable to avoid tagging large @@ -82,8 +82,6 @@ func (s Status) String() string { return "PROOF" case Precomputed: return "PRECOMPUTED" - case PublicInput: - return "PUBLIC_INPUT" case VerifyingKey: return "VERIFYING_KEY" case VerifierDefined: @@ -95,7 +93,7 @@ func (s Status) String() string { // IsPublic returns true if the column is visible to the verifier func (s Status) IsPublic() bool { switch s { - case Proof, PublicInput, VerifyingKey, VerifierDefined: + case Proof, VerifyingKey, VerifierDefined: return true default: return false diff --git a/prover/protocol/column/store.go b/prover/protocol/column/store.go index e49dc1b6b0e..0a7a8f18bb4 100644 --- a/prover/protocol/column/store.go +++ b/prover/protocol/column/store.go @@ -171,19 +171,6 @@ func (r *Store) AllKeysProof() []ifaces.ColID { return res } -// AllKeysPublicInput returns the list of the [PublicInput] column's ID ordered -// by rounds and then by order ot insertion. -func (r *Store) AllKeysPublicInput() []ifaces.ColID { - res := []ifaces.ColID{} - - for round := 0; round < r.NumRounds(); round++ { - proof := r.AllKeysPublicInputAt(round) - res = append(res, proof...) - } - - return res -} - // AllKeysCommitted returns the list of all the IDs of the all the [Committed] // columns ordered by rounds and then by IDs. func (r *Store) AllKeysCommitted() []ifaces.ColID { @@ -233,22 +220,6 @@ func (r *Store) AllKeysProofAt(round int) []ifaces.ColID { return res } -// AllKeysPublicInputAt returns the list of all the prover messages in a given -// round. The resulting slice is ordered by order of insertion. -func (r *Store) AllKeysPublicInputAt(round int) []ifaces.ColID { - res := []ifaces.ColID{} - rnd := r.byRounds.MustGet(round) - - for i, info := range rnd { - if info.Status != PublicInput { - continue - } - res = append(res, rnd[i].ID) - } - - return res -} - // Returns the list of all the [Precomputed] columns' ID. The returned slice is // ordered by rounds and then by order of insertion. func (r *Store) AllPrecomputed() []ifaces.ColID { @@ -448,10 +419,6 @@ func assertCorrectStatusTransition(old, new Status) { // If it's ignored, it's ignored case old == Ignored && new != Ignored: forbiddenTransition = true - // You can't change the status of the public inputs because that would - // change the statement of the zkEVM. - case old == PublicInput && new != PublicInput: - forbiddenTransition = true // It's a special status and cannot be changed. case old == VerifierDefined && new != VerifierDefined: forbiddenTransition = true @@ -489,10 +456,6 @@ func (in *storedColumnInfo) isExcludedFromProverFS() bool { return true } - if in.Status.IsPublic() { - return false - } - if in.IncludeInProverFS { return false } diff --git a/prover/protocol/wizard/compiled.go b/prover/protocol/wizard/compiled.go index 5ceb4cc02db..0b2738918fa 100644 --- a/prover/protocol/wizard/compiled.go +++ b/prover/protocol/wizard/compiled.go @@ -433,20 +433,6 @@ func (c *CompiledIOP) InsertProof(round int, name ifaces.ColID, size int) (msg i return c.Columns.AddToRound(round, name, size, column.Proof) } -// InsertPublicInput registers a public input column, and specifies static information regarding it - -// Deprecated: we never really use this type of column to denote actual public -// inputs. The plan is to resort to using [query.LocalOpeningParams] instead. -func (c *CompiledIOP) InsertPublicInput(round int, name ifaces.ColID, size int) (msg ifaces.Column) { - - // Common : No zero length - if size == 0 { - utils.Panic("when registering %v, VecType with length zero", name) - } - - return c.Columns.AddToRound(round, name, size, column.PublicInput) -} - // InsertVerifier registers a verifier steps into the current CompiledIOP; // meaning a "native" Go function that performs one or more checks involving // wizard items that are accessible to the verifier of the specified protocol. @@ -654,9 +640,20 @@ func (c *CompiledIOP) InsertProjection(id ifaces.QueryID, in query.ProjectionInp in.FilterA.Round(), in.FilterB.Round()) ) - c.assertConsistentRound(round) q := query.NewProjection(round, id, in) // Finally registers the query c.QueriesNoParams.AddToRound(round, q.Name(), q) return q } + +// AddPublicInput inserts a public-input in the compiled-IOP +func (c *CompiledIOP) InsertPublicInput(name string, acc ifaces.Accessor) PublicInput { + + res := PublicInput{ + Name: name, + Acc: acc, + } + + c.PublicInputs = append(c.PublicInputs, res) + return res +} diff --git a/prover/protocol/wizard/gnark_verifier.go b/prover/protocol/wizard/gnark_verifier.go index c8ad4c179ff..fc2c1292f4e 100644 --- a/prover/protocol/wizard/gnark_verifier.go +++ b/prover/protocol/wizard/gnark_verifier.go @@ -224,12 +224,6 @@ func (c *WizardVerifierCircuit) generateAllRandomCoins(api frontend.API) { c.FS.UpdateVec(msgContent) } - toUpdateFS = c.Spec.Columns.AllKeysPublicInputAt(currRound - 1) - for _, msg := range toUpdateFS { - msgContent := c.GetColumn(msg) - c.FS.UpdateVec(msgContent) - } - /* Also include the prover's allegations for all evaluations */ diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index 86a0b965f6f..5b1aa151e50 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -238,11 +238,6 @@ func (run *ProverRuntime) ExtractProof() Proof { messages.InsertNew(name, messageValue) } - for _, name := range run.Spec.Columns.AllKeysPublicInput() { - messageValue := run.Columns.MustGet(name) - messages.InsertNew(name, messageValue) - } - queriesParams := collection.NewMapping[ifaces.QueryID, ifaces.QueryParams]() for round := 0; round <= run.currRound; round++ { for _, name := range run.Spec.QueriesParams.AllKeysAt(round) { @@ -569,18 +564,6 @@ func (run *ProverRuntime) goNextRound() { run.FS.UpdateSV(instance) } - /* - Make sure that all messages have been written and use them - to update the FS state. Note that we do not need to update - FS using the last round of the prover because he is always - the last one to "talk" in the protocol. - */ - msgsToFS = run.Spec.Columns.AllKeysPublicInputAt(run.currRound) - for _, msgName := range msgsToFS { - instance := run.GetMessage(msgName) - run.FS.UpdateSV(instance) - } - /* Also include the prover's allegations for all evaluations */ diff --git a/prover/protocol/wizard/verifier.go b/prover/protocol/wizard/verifier.go index bd62ef924f9..aaacfe165fb 100644 --- a/prover/protocol/wizard/verifier.go +++ b/prover/protocol/wizard/verifier.go @@ -245,12 +245,6 @@ func (run *VerifierRuntime) generateAllRandomCoins() { run.FS.UpdateSV(instance) } - msgsToFS = run.Spec.Columns.AllKeysPublicInputAt(currRound - 1) - for _, msgName := range msgsToFS { - instance := run.GetColumn(msgName) - run.FS.UpdateSV(instance) - } - /* Also include the prover's allegations for all evaluations */ From f22c034bf06f3c8d874f0f5a958ce4b34dd8cd6a Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 4 Feb 2025 08:10:09 +0100 Subject: [PATCH 62/63] feat(conglo): integrates the cross-consistency checking in the public inputs --- .../compiler/inclusion/inclusion.go | 3 +- .../compiler/inclusion/inclusion_test.go | 23 ++- .../conglomeration/conglomeration.go | 8 + .../conglomeration/conglomeration_test.go | 37 +++-- .../cross_segment_consistency.go | 105 +++++++++++++ .../distributed/constants/constant.go | 7 + prover/protocol/distributed/xcomp/xcomp.go | 144 ------------------ prover/protocol/wizard/compiled.go | 13 ++ 8 files changed, 178 insertions(+), 162 deletions(-) create mode 100644 prover/protocol/distributed/conglomeration/cross_segment_consistency.go create mode 100644 prover/protocol/distributed/constants/constant.go delete mode 100644 prover/protocol/distributed/xcomp/xcomp.go diff --git a/prover/protocol/distributed/compiler/inclusion/inclusion.go b/prover/protocol/distributed/compiler/inclusion/inclusion.go index d14f9f5033e..855e25b2790 100644 --- a/prover/protocol/distributed/compiler/inclusion/inclusion.go +++ b/prover/protocol/distributed/compiler/inclusion/inclusion.go @@ -6,6 +6,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/accessors" "github.com/consensys/linea-monorepo/prover/protocol/column" "github.com/consensys/linea-monorepo/prover/protocol/distributed" + "github.com/consensys/linea-monorepo/prover/protocol/distributed/constants" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -162,7 +163,7 @@ func GetShareOfLogDerivativeSum(in DistributionInputs) { // declare [query.LogDerivSumParams] as [wizard.PublicInput] moduleComp.PublicInputs = append(moduleComp.PublicInputs, wizard.PublicInput{ - Name: accessors.LOGDERIVSUM_ACCESSOR, + Name: constants.LogDerivativeSumPublicInput, Acc: accessors.NewLogDerivSumAccessor(logDerivQuery), }) diff --git a/prover/protocol/distributed/compiler/inclusion/inclusion_test.go b/prover/protocol/distributed/compiler/inclusion/inclusion_test.go index 654d5589d30..e62939f5892 100644 --- a/prover/protocol/distributed/compiler/inclusion/inclusion_test.go +++ b/prover/protocol/distributed/compiler/inclusion/inclusion_test.go @@ -1,16 +1,18 @@ package inclusion_test import ( + "errors" "testing" "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" logderiv "github.com/consensys/linea-monorepo/prover/protocol/compiler/logderivativesum" "github.com/consensys/linea-monorepo/prover/protocol/distributed" "github.com/consensys/linea-monorepo/prover/protocol/distributed/compiler/inclusion" + "github.com/consensys/linea-monorepo/prover/protocol/distributed/constants" "github.com/consensys/linea-monorepo/prover/protocol/distributed/lpp" md "github.com/consensys/linea-monorepo/prover/protocol/distributed/namebaseddiscoverer" - "github.com/consensys/linea-monorepo/prover/protocol/distributed/xcomp" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/wizard" "github.com/stretchr/testify/require" @@ -25,7 +27,7 @@ func TestSeedGeneration(t *testing.T) { ) var ( - allVerfiers = []*wizard.VerifierRuntime{} + allVerfiers = []wizard.Runtime{} ) //initialComp define := func(b *wizard.Builder) { @@ -176,7 +178,20 @@ func TestSeedGeneration(t *testing.T) { } // apply the crosse checks over the public inputs. - xComp := xcomp.GetCrossComp(allVerfiers) - wizard.Verify(xComp, wizard.Proof{}) + require.NoError(t, checkConsistency(allVerfiers)) +} + +func checkConsistency(runs []wizard.Runtime) error { + + var res field.Element + for _, run := range runs { + logderiv := run.GetPublicInput(constants.LogDerivativeSumPublicInput) + res.Add(&res, &logderiv) + } + + if !res.IsZero() { + return errors.New("the logderiv sums do not cancel each others") + } + return nil } diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index b30d2519446..94e3824d0d4 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -7,6 +7,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/column" "github.com/consensys/linea-monorepo/prover/protocol/compiler/selfrecursion" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" + "github.com/consensys/linea-monorepo/prover/protocol/distributed/constants" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -96,6 +97,8 @@ func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def fu for _, ctx := range ctxs { selfrecursion.RecurseOverCustomCtx(comp, ctx.PcsCtx, ctx.Translator.Prefix) } + + comp.RegisterVerifierAction(ctxs[0].LastRound, &CrossSegmentCheck{Ctxs: ctxs}) } return def, &ctxs @@ -117,6 +120,11 @@ func (ctx *recursionCtx) captureCompPreVortex(tmpl *wizard.CompiledIOP) { var ( polyQuery = tmpl.PcsCtxs.(*vortex.Ctx).Query lastRound = tmpl.QueriesParams.Round(polyQuery.QueryID) + + // This sanity-check ensures that the template has the right public inputs + _ = tmpl.GetPublicInputAccessor(constants.GrandProductPublicInput) + _ = tmpl.GetPublicInputAccessor(constants.GrandSumPublicInput) + _ = tmpl.GetPublicInputAccessor(constants.LogDerivativeSumPublicInput) ) ctx.LastRound = lastRound diff --git a/prover/protocol/distributed/conglomeration/conglomeration_test.go b/prover/protocol/distributed/conglomeration/conglomeration_test.go index 1d72764d931..d40bc10c6ca 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration_test.go +++ b/prover/protocol/distributed/conglomeration/conglomeration_test.go @@ -6,14 +6,15 @@ import ( "github.com/consensys/linea-monorepo/prover/crypto/ringsis" "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/accessors" "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/compiler" "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" - "github.com/consensys/linea-monorepo/prover/protocol/compiler/fullrecursion" "github.com/consensys/linea-monorepo/prover/protocol/compiler/mimc" "github.com/consensys/linea-monorepo/prover/protocol/compiler/selfrecursion" "github.com/consensys/linea-monorepo/prover/protocol/compiler/vortex" "github.com/consensys/linea-monorepo/prover/protocol/distributed/conglomeration" + "github.com/consensys/linea-monorepo/prover/protocol/distributed/constants" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -50,18 +51,18 @@ var ( compiler.Arcane(1, 1<<10, false), commonVortexStep, } - arcaneFullRecSelfRecCompilationSuite = []func(*wizard.CompiledIOP){ - compiler.Arcane(1<<8, 1<<10, false), - commonVortexStep, - selfrecursion.SelfRecurse, - mimc.CompileMiMC, - compiler.Arcane(1<<8, 1<<10, false), - commonVortexStep, - fullrecursion.FullRecursion(true), - mimc.CompileMiMC, - compiler.Arcane(1<<8, 1<<10, false), - commonVortexStep, - } + // arcaneFullRecSelfRecCompilationSuite = []func(*wizard.CompiledIOP){ + // compiler.Arcane(1<<8, 1<<10, false), + // commonVortexStep, + // selfrecursion.SelfRecurse, + // mimc.CompileMiMC, + // compiler.Arcane(1<<8, 1<<10, false), + // commonVortexStep, + // fullrecursion.FullRecursion(true), + // mimc.CompileMiMC, + // compiler.Arcane(1<<8, 1<<10, false), + // commonVortexStep, + // } ) type conglomerationTestCase struct { @@ -86,6 +87,9 @@ func TestConglomerationPureVortexSingleRound(t *testing.T) { a = append(a, builder.RegisterCommit(ifaces.ColIDf("a-%v", i), numRow)) } u = builder.CompiledIOP.InsertUnivariate(0, "u", a) + builder.InsertPublicInput(constants.GrandProductPublicInput, accessors.NewConstant(field.NewElement(1))) + builder.InsertPublicInput(constants.GrandSumPublicInput, accessors.NewConstant(field.NewElement(0))) + builder.InsertPublicInput(constants.LogDerivativeSumPublicInput, accessors.NewConstant(field.NewElement(0))) } prover := func(k int) func(run *wizard.ProverRuntime) { @@ -165,6 +169,9 @@ func TestConglomerationPureVortexMultiRound(t *testing.T) { run.AssignUnivariate(u.QueryID, field.NewElement(0), ys...) }) + builder.InsertPublicInput(constants.GrandProductPublicInput, accessors.NewConstant(field.NewElement(1))) + builder.InsertPublicInput(constants.GrandSumPublicInput, accessors.NewConstant(field.NewElement(0))) + builder.InsertPublicInput(constants.LogDerivativeSumPublicInput, accessors.NewConstant(field.NewElement(0))) } prover := func(k int) func(run *wizard.ProverRuntime) { @@ -221,6 +228,10 @@ func TestConglomerationLookup(t *testing.T) { a = append(a, builder.RegisterCommit(ifaces.ColIDf("a-%v", i), numRow)) builder.Range(ifaces.QueryIDf("range-%v", i), a[i], 1<<8) } + + builder.InsertPublicInput(constants.GrandProductPublicInput, accessors.NewConstant(field.NewElement(1))) + builder.InsertPublicInput(constants.GrandSumPublicInput, accessors.NewConstant(field.NewElement(0))) + builder.InsertPublicInput(constants.LogDerivativeSumPublicInput, accessors.NewConstant(field.NewElement(0))) } prover := func(k int) func(run *wizard.ProverRuntime) { diff --git a/prover/protocol/distributed/conglomeration/cross_segment_consistency.go b/prover/protocol/distributed/conglomeration/cross_segment_consistency.go new file mode 100644 index 00000000000..431aa978776 --- /dev/null +++ b/prover/protocol/distributed/conglomeration/cross_segment_consistency.go @@ -0,0 +1,105 @@ +package conglomeration + +import ( + "errors" + "fmt" + + "github.com/consensys/gnark/frontend" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/distributed/constants" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" +) + +// crossSegmentCheclk is a verifier action that performs cross-segment checks: +// for instance, it checks that the log-derivative sums all sums to 0 and that +// the grand product is 1. The goal is to ensure that the lookups, permutations +// in the original protocol are satisfied. +type CrossSegmentCheck struct { + Ctxs []*recursionCtx + skip bool +} + +// Run implements the [wizard.VerifierAction], it handles the cross checks over +// the public inputs. for example the global sum over the LogDerivativeSum from +// different segments should be zero. +func (pir *CrossSegmentCheck) Run(run wizard.Runtime) error { + + var ( + logDerivSumAcc, grandSumAcc field.Element + grandProductAcc = field.One() + err error + ) + + for _, ctx := range pir.Ctxs { + + var ( + wrappedRun = &runtimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + tmpl = ctx.Tmpl + logDerivSum = tmpl.GetPublicInputAccessor(constants.LogDerivativeSumPublicInput).GetVal(wrappedRun) + grandProd = tmpl.GetPublicInputAccessor(constants.GrandProductPublicInput).GetVal(wrappedRun) + grandSum = tmpl.GetPublicInputAccessor(constants.GrandSumPublicInput).GetVal(wrappedRun) + ) + + logDerivSumAcc.Add(&logDerivSumAcc, &logDerivSum) + grandSumAcc.Add(&grandSumAcc, &grandSum) + grandProductAcc.Mul(&grandProductAcc, &grandProd) + } + + if logDerivSumAcc != field.Zero() { + err = errors.Join(err, fmt.Errorf("the global sum over LogDerivSumParams is not zero,"+ + " maybe the same coin over different modules has different values")) + } + + if grandProductAcc != field.One() { + err = errors.Join(err, fmt.Errorf("the global product overGrandProductParams is not 1,"+ + " maybe the same coin over different modules has different values")) + } + + if grandSumAcc != field.Zero() { + err = errors.Join(err, fmt.Errorf("the global sum over GrandSumParams is not zero,"+ + " maybe the same coin over different modules has different values")) + } + + if err != nil { + return fmt.Errorf("[conglomeration.crossSegmentConsistency] %w", err) + } + + return nil +} + +// RunGnark implements the [wizard.VerifierAction] +func (pir *CrossSegmentCheck) RunGnark(api frontend.API, run wizard.GnarkRuntime) { + + var ( + logDerivSumAcc = frontend.Variable(0) + grandSumAcc = frontend.Variable(0) + grandProductAcc = frontend.Variable(1) + ) + + for _, ctx := range pir.Ctxs { + + var ( + wrappedRun = &gnarkRuntimeTranslator{Prefix: ctx.Translator.Prefix, Rt: run} + tmpl = ctx.Tmpl + logDerivSum = tmpl.GetPublicInputAccessor(constants.LogDerivativeSumPublicInput).GetFrontendVariable(api, wrappedRun) + grandProd = tmpl.GetPublicInputAccessor(constants.GrandProductPublicInput).GetFrontendVariable(api, wrappedRun) + grandSum = tmpl.GetPublicInputAccessor(constants.GrandSumPublicInput).GetFrontendVariable(api, wrappedRun) + ) + + logDerivSumAcc = api.Add(logDerivSumAcc, logDerivSum) + grandSumAcc = api.Add(grandSumAcc, grandSum) + grandProductAcc = api.Mul(grandProductAcc, grandProd) + } + + api.AssertIsEqual(logDerivSumAcc, field.Zero()) + api.AssertIsEqual(grandProductAcc, field.One()) + api.AssertIsEqual(grandSumAcc, field.Zero()) +} + +func (v *CrossSegmentCheck) Skip() { + v.skip = true +} + +func (v *CrossSegmentCheck) IsSkipped() bool { + return v.skip +} diff --git a/prover/protocol/distributed/constants/constant.go b/prover/protocol/distributed/constants/constant.go new file mode 100644 index 00000000000..0a35a13d59a --- /dev/null +++ b/prover/protocol/distributed/constants/constant.go @@ -0,0 +1,7 @@ +package constants + +const ( + LogDerivativeSumPublicInput = "LOG_DERIVATE_SUM_PUBLIC_INPUT" + GrandProductPublicInput = "GRAND_PRODUCT_PUBLIC_INPUT" + GrandSumPublicInput = "GRAND_SUM_PUBLIC_INPUT" +) diff --git a/prover/protocol/distributed/xcomp/xcomp.go b/prover/protocol/distributed/xcomp/xcomp.go deleted file mode 100644 index 0dde03eeb1b..00000000000 --- a/prover/protocol/distributed/xcomp/xcomp.go +++ /dev/null @@ -1,144 +0,0 @@ -package xcomp - -import ( - "fmt" - - "github.com/consensys/gnark/frontend" - "github.com/consensys/linea-monorepo/prover/maths/field" - "github.com/consensys/linea-monorepo/prover/protocol/accessors" - "github.com/consensys/linea-monorepo/prover/protocol/wizard" - "github.com/consensys/linea-monorepo/prover/utils/collection" -) - -// GetCrossComp generates an (empty) compiledIOP object that is handling the crosse checks -// for example, the global sum over logDerivativeSum is zero. -func GetCrossComp(vRuntimes []*wizard.VerifierRuntime) *wizard.CompiledIOP { - - xComp := wizard.NewCompiledIOP() - - // initialize the PICollector - va := PublicInputCollector{ - PIFromLogDeriv: collection.NewMapping[string, field.Element](), - PIFromGrandProd: collection.NewMapping[string, field.Element](), - PIFromGrandSum: collection.NewMapping[string, field.Element](), - } - // get the publicInputs values from different verifiers. - for i, runtime := range vRuntimes { - va.Index = i - CollectPIfromVerifer(runtime, &va) - } - - // register a verifier action to check the consistency of public inputs - xComp.RegisterVerifierAction(0, &PublicInputChecker{PublicInputCollector: va}) - - return xComp -} - -// PublicInputCollector collects the public input values from different modules/segments. -type PublicInputCollector struct { - // maps for collecting publicInputs from different modules. - PIFromLogDeriv, PIFromGrandProd, PIFromGrandSum collection.Mapping[string, field.Element] - // index for the verifier from which we are receiving the publicInputs - Index int -} - -// CollectPIfromVerifer adds the public inputs of a given verifier to the Collector. -func CollectPIfromVerifer(run *wizard.VerifierRuntime, pic *PublicInputCollector) { - var ( - allPI = run.Spec.PublicInputs - ) - - for _, pi := range allPI { - - name := fmt.Sprintf("%v_%v", pi.Name, pic.Index) - - switch v := pi.Acc.(type) { - - case *accessors.FromLogDerivSumAccessor: - pic.PIFromLogDeriv.InsertNew(name, v.GetVal(run)) - - case *accessors.FromGrandProductAccessor: - pic.PIFromLogDeriv.InsertNew(name, v.GetVal(run)) - - case *accessors.FromGrandSumAccessor: - pic.PIFromLogDeriv.InsertNew(name, v.GetVal(run)) - } - } - -} - -type PublicInputChecker struct { - PublicInputCollector - skip bool -} - -// Run implements the [wizard.VerifierAction], it handles the cross checks over the public inputs. -// for example the global sum over the LogDerivativeSum from different segments should be zero. -func (pir *PublicInputChecker) Run(run *wizard.VerifierRuntime) error { - var ( - logDerivSum, grandSum field.Element - grandProduct = field.One() - ) - - for _, key := range pir.PIFromLogDeriv.ListAllKeys() { - curr := pir.PIFromLogDeriv.MustGet(key) - logDerivSum.Add(&logDerivSum, &curr) - } - for _, key := range pir.PIFromGrandProd.ListAllKeys() { - curr := pir.PIFromGrandProd.MustGet(key) - grandProduct.Add(&grandProduct, &curr) - } - for _, key := range pir.PIFromGrandSum.ListAllKeys() { - curr := pir.PIFromGrandSum.MustGet(key) - grandSum.Add(&grandSum, &curr) - } - - if logDerivSum != field.Zero() { - panic("the global sum over LogDerivSumParams is not zero," + - " maybe the same coin over different modules has different values") - } - - if grandProduct != field.One() { - panic("the global product overGrandProductParams is not 1," + - " maybe the same coin over different modules has different values") - } - - if grandSum != field.Zero() { - panic("the global sum over GrandSumParams is not zero," + - " maybe the same coin over different modules has different values") - } - return nil - -} - -// RunGnark implements the [wizard.VerifierAction] -func (pir *PublicInputChecker) RunGnark(api frontend.API, run *wizard.WizardVerifierCircuit) { - var ( - logDerivSum, grandProduct, grandSum frontend.API - ) - - for _, key := range pir.PIFromLogDeriv.ListAllKeys() { - curr := pir.PIFromLogDeriv.MustGet(key) - api.Add(logDerivSum, curr) - } - for _, key := range pir.PIFromGrandProd.ListAllKeys() { - curr := pir.PIFromGrandProd.MustGet(key) - api.Add(grandProduct, curr) - } - for _, key := range pir.PIFromGrandSum.ListAllKeys() { - curr := pir.PIFromGrandSum.MustGet(key) - api.Add(grandSum, curr) - } - - api.AssertIsEqual(logDerivSum, field.Zero()) - api.AssertIsEqual(grandProduct, field.One()) - api.AssertIsEqual(grandSum, field.Zero()) -} - -func (v *PublicInputChecker) Skip() { - v.skip = true -} - -func (v *PublicInputChecker) IsSkipped() bool { - return v.skip -} diff --git a/prover/protocol/wizard/compiled.go b/prover/protocol/wizard/compiled.go index 0b2738918fa..65f11771a88 100644 --- a/prover/protocol/wizard/compiled.go +++ b/prover/protocol/wizard/compiled.go @@ -657,3 +657,16 @@ func (c *CompiledIOP) InsertPublicInput(name string, acc ifaces.Accessor) Public c.PublicInputs = append(c.PublicInputs, res) return res } + +// GetPublicInputAccessor attempts to find a public input with the provided name +// and panic if it fails to do so. The method returns the accessor in case of +// success. +func (c *CompiledIOP) GetPublicInputAccessor(name string) ifaces.Accessor { + for _, pi := range c.PublicInputs { + if pi.Name == name { + return pi.Acc + } + } + utils.Panic("could not find public input %v", name) + return nil // unreachable +} From 0e300c2e4470a21781df92e8c7c9a19e1c47f0b2 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Thu, 6 Feb 2025 08:51:21 +0100 Subject: [PATCH 63/63] doc(com): adds some doc to explain how the input compiledIOP should be compiled --- prover/protocol/distributed/conglomeration/conglomeration.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/prover/protocol/distributed/conglomeration/conglomeration.go b/prover/protocol/distributed/conglomeration/conglomeration.go index 94e3824d0d4..02a016a8ac3 100644 --- a/prover/protocol/distributed/conglomeration/conglomeration.go +++ b/prover/protocol/distributed/conglomeration/conglomeration.go @@ -42,6 +42,10 @@ type recursionCtx struct { // comp and a placeholder pointer for the recursion context. On return // of the function, the pointer points to an empty slice and is populated // once [wizard.Compile] has been called with def. +// +// To be conglomerable, `tmpl` must be compiled with the [vortex.Compile] +// using the [vortex.PremarkAsSelfRecursed] option and without the +// [vortex.ReplaceByMiMC]. func ConglomerateDefineFunc(tmpl *wizard.CompiledIOP, maxNumSegment int) (def func(*wizard.Builder), ctxsPlaceHolder *[]*recursionCtx) { var ctxs []*recursionCtx