From 7e9c8794601042fa82dd4c06bf2a209881acee46 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 07:35:49 +0200 Subject: [PATCH 01/18] feat(projection): implements the multi-ary projection query --- prover/protocol/query/projection.go | 259 +++++++++++++++++++++------- 1 file changed, 192 insertions(+), 67 deletions(-) diff --git a/prover/protocol/query/projection.go b/prover/protocol/query/projection.go index a165b117178..4ce955eae10 100644 --- a/prover/protocol/query/projection.go +++ b/prover/protocol/query/projection.go @@ -4,45 +4,110 @@ import ( "fmt" "github.com/consensys/gnark/frontend" - "github.com/consensys/linea-monorepo/prover/maths/common/poly" + "github.com/consensys/linea-monorepo/prover/maths/common/vector" "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/utils" ) +// Projection represents a projection query. A projection query enforces that +// two sides A and B contains the same values on the same order at positions +// marked with a 1 in the corresponding filter. The query supports multi-ary +// projections: meaning that the A and B sides can have multiple columns and +// the query will read both sides left-to-right then top-to-bottom (e.g. like +// reading text in english). In that context, the two sides can have different +// "width": this allows the query to be used for flattening an array of columns +// or widening it. +// +// And on top of that, the query can also "tables": the query look over vector +// of values instead of just values. In that case, all parts of the two sides +// of multi-ary projection must have the same number of columns. +type Projection struct { + Round int + ID ifaces.QueryID + Inp ProjectionMultiAryInput +} + +// ProjectionInput is a collection of parameters to provide to a [Projection] +// query. It corresponds to the case where the projection query is "unary". type ProjectionInput struct { + // ColumnA and ColumnB are the columns of the left and right side. Each + // entry of either corresponds to a column in the projected table. ColumnA, ColumnB []ifaces.Column + // FilterA and FilterB are the filters of the ColumnA and ColumnB FilterA, FilterB ifaces.Column } -type Projection struct { - Round int - ID ifaces.QueryID - Inp ProjectionInput + +// ProjectionMultiAryInput is a collection of parameters to provide to a +// [Projection] query in the general case. +type ProjectionMultiAryInput struct { + // ColumnsA and ColumnsB are the columns of the left and right side. + // The lists are structured as list of projected tables, each list + // is processed left-to-right then top-to-bottom. + ColumnsA, ColumnsB [][]ifaces.Column + // FiltersA and FiltersB are the filters of the left-to-right side. + FiltersA, FiltersB []ifaces.Column } // NewProjection constructs a projection. Will panic if it is mal-formed -func NewProjection( +func NewProjection(round int, id ifaces.QueryID, inp ProjectionInput) Projection { + return NewProjectionMultiAry(round, id, ProjectionMultiAryInput{ + ColumnsA: [][]ifaces.Column{inp.ColumnA}, + ColumnsB: [][]ifaces.Column{inp.ColumnB}, + FiltersA: []ifaces.Column{inp.FilterA}, + FiltersB: []ifaces.Column{inp.FilterB}, + }) +} + +// NewProjectionMultiAry returns a new [Projection] query object. +func NewProjectionMultiAry( round int, id ifaces.QueryID, - inp ProjectionInput, + inp ProjectionMultiAryInput, ) Projection { + + if len(inp.ColumnsA) == 0 || len(inp.ColumnsB) == 0 { + utils.Panic("A and B must have at least one table: len(A)=%v, len(B)=%v", len(inp.ColumnsA), len(inp.ColumnsB)) + } + + if len(inp.ColumnsA[0]) == 0 { + utils.Panic("A and B must have at least one column: len(A[0])=%v", len(inp.ColumnsA[0])) + } + var ( - sizeA = inp.FilterA.Size() - sizeB = inp.FilterB.Size() - numCol = len(inp.ColumnA) + numCols = len(inp.ColumnsA[0]) + numPartsA = len(inp.ColumnsA) + numPartsB = len(inp.ColumnsB) ) - if len(inp.ColumnB) != numCol { - utils.Panic("A and B must have the same number of columns") + if len(inp.FiltersA) != numPartsA || len(inp.FiltersB) != numPartsB { + utils.Panic("A and B must have the same number of filters: len(A)=%v, len(B)=%v", len(inp.FiltersA), len(inp.FiltersB)) } - if ifaces.AssertSameLength(inp.ColumnA...) != sizeA { - utils.Panic("A and its filter do not have the same column sizes") + for i := range inp.ColumnsA { + if len(inp.ColumnsA[i]) != numCols { + utils.Panic("All table must have the same number of columns: len(A[%v])=%v, len(A[0])=%v", i, len(inp.ColumnsA), numCols) + } + + size := ifaces.AssertSameLength(inp.ColumnsA[i]...) + + if size != inp.FiltersA[i].Size() { + utils.Panic("A[%v] and its filter do not have the same column sizes", i) + } } - if ifaces.AssertSameLength(inp.ColumnB...) != sizeB { - utils.Panic("B and its filter do not have the same column sizes") + for i := range inp.ColumnsB { + if len(inp.ColumnsB[i]) != numCols { + utils.Panic("All table must have the same number of columns: len(B[%v])=%v, len(A[0])=%v", i, len(inp.ColumnsB), numCols) + } + + size := ifaces.AssertSameLength(inp.ColumnsB[i]...) + + if size != inp.FiltersB[i].Size() { + utils.Panic("B[%v] and its filter do not have the same column sizes", i) + } } + return Projection{Round: round, ID: id, Inp: inp} } @@ -53,52 +118,104 @@ func (p Projection) Name() ifaces.QueryID { // Check implements the [ifaces.Query] interface func (p Projection) Check(run ifaces.Runtime) error { + + // The function is implemented by creating two iterator functions and + // checking that they yield the same values. var ( - numCols = len(p.Inp.ColumnA) - sizeA = p.Inp.ColumnA[0].Size() - sizeB = p.Inp.ColumnB[0].Size() - linCombRand, evalRand field.Element - a = make([]ifaces.ColAssignment, numCols) - b = make([]ifaces.ColAssignment, numCols) - fA = p.Inp.FilterA.GetColAssignment(run).IntoRegVecSaveAlloc() - fB = p.Inp.FilterB.GetColAssignment(run).IntoRegVecSaveAlloc() - aLinComb = make([]field.Element, sizeA) - bLinComb = make([]field.Element, sizeB) + aCurrPart, aCurrRow = 0, 0 + bCurrPart, bCurrRow = 0, 0 ) - _, errAlpha := linCombRand.SetRandom() - _, errBeta := evalRand.SetRandom() - if errAlpha != nil { - // Cannot happen unless the entropy was exhausted - panic(errAlpha) - } - if errBeta != nil { - // Cannot happen unless the entropy was exhausted - panic(errBeta) - } - // Populate a - for colIndex, pol := range p.Inp.ColumnA { - a[colIndex] = pol.GetColAssignment(run) - } - // Populate b - for colIndex, pol := range p.Inp.ColumnB { - b[colIndex] = pol.GetColAssignment(run) - } - // Compute the linear combination of the columns of a and b - for row := 0; row < sizeA; row++ { - aLinComb[row] = rowLinComb(linCombRand, row, a) - } - for row := 0; row < sizeB; row++ { - bLinComb[row] = rowLinComb(linCombRand, row, b) + + nextA := func() ([]field.Element, bool) { + for { + + if aCurrRow >= p.Inp.FiltersA[aCurrPart].Size() { + return nil, false + } + + fa := p.Inp.FiltersA[aCurrPart].GetColAssignmentAt(run, aCurrRow) + + if fa.IsZero() { + aCurrPart++ + if aCurrPart == len(p.Inp.ColumnsA) { + aCurrPart = 0 + aCurrRow++ + } + continue + } + + var res []field.Element + for _, col := range p.Inp.ColumnsA[aCurrPart] { + res = append(res, col.GetColAssignmentAt(run, aCurrRow)) + } + + aCurrPart++ + if aCurrPart == len(p.Inp.ColumnsA) { + aCurrPart = 0 + aCurrRow++ + } + + return res, true + } } - var ( - hornerA = poly.GetHornerTrace(aLinComb, fA, evalRand) - hornerB = poly.GetHornerTrace(bLinComb, fB, evalRand) - ) - if hornerA[0] != hornerB[0] { - return fmt.Errorf("the projection query %v check is not satisfied", p.ID) + + nextB := func() ([]field.Element, bool) { + for { + + if bCurrRow >= p.Inp.FiltersB[bCurrPart].Size() { + return nil, false + } + + fb := p.Inp.FiltersB[bCurrPart].GetColAssignmentAt(run, bCurrRow) + + if fb.IsZero() { + bCurrPart++ + if bCurrPart == len(p.Inp.ColumnsB) { + bCurrPart = 0 + bCurrRow++ + } + continue + } + + var res []field.Element + for _, col := range p.Inp.ColumnsB[bCurrPart] { + res = append(res, col.GetColAssignmentAt(run, bCurrRow)) + } + + bCurrPart++ + if bCurrPart == len(p.Inp.ColumnsB) { + bCurrPart = 0 + bCurrRow++ + } + + return res, true + } } - return nil + for { + a, aOk := nextA() + b, bOk := nextB() + + if !aOk && !bOk { + return nil + } + + if aOk != bOk { + return fmt.Errorf("a and b must yield the same number of rows, a %v b %v", aOk, bOk) + } + + if len(a) != len(b) { + // Note: this is redundant with the constructor's check, this should + // not be a runtime error. + panic("A and B must yield the same number of columns") + } + + for i := range a { + if !a[i].Equal(&b[i]) { + return fmt.Errorf("a and b must yield the same values, a=%v b=%v", vector.Prettify(a), vector.Prettify(b)) + } + } + } } // GnarkCheck implements the [ifaces.Query] interface. It will panic in this @@ -122,23 +239,31 @@ func (p Projection) GetShiftedRelatedColumns() []ifaces.Column { res := []ifaces.Column{} - if p.Inp.FilterA.IsComposite() { - res = append(res, p.Inp.FilterA) + for _, f := range p.Inp.FiltersA { + if f.IsComposite() { + res = append(res, f) + } } - if p.Inp.FilterB.IsComposite() { - res = append(res, p.Inp.FilterB) + for _, f := range p.Inp.FiltersB { + if f.IsComposite() { + res = append(res, f) + } } - for i := range p.Inp.ColumnA { - if p.Inp.ColumnA[i].IsComposite() { - res = append(res, p.Inp.ColumnA[i]) + for i := range p.Inp.ColumnsA { + for _, col := range p.Inp.ColumnsA[i] { + if col.IsComposite() { + res = append(res, col) + } } } - for i := range p.Inp.ColumnB { - if p.Inp.ColumnB[i].IsComposite() { - res = append(res, p.Inp.ColumnB[i]) + for i := range p.Inp.ColumnsB { + for _, col := range p.Inp.ColumnsB[i] { + if col.IsComposite() { + res = append(res, col) + } } } From acfb98f33bb583b0d51e8de62cced932e99b4cfd Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 08:39:28 +0200 Subject: [PATCH 02/18] fixup(projection): fix the size check for the projection query --- prover/protocol/query/projection.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/prover/protocol/query/projection.go b/prover/protocol/query/projection.go index 4ce955eae10..92bfae68a9e 100644 --- a/prover/protocol/query/projection.go +++ b/prover/protocol/query/projection.go @@ -70,14 +70,16 @@ func NewProjectionMultiAry( utils.Panic("A and B must have at least one table: len(A)=%v, len(B)=%v", len(inp.ColumnsA), len(inp.ColumnsB)) } - if len(inp.ColumnsA[0]) == 0 { - utils.Panic("A and B must have at least one column: len(A[0])=%v", len(inp.ColumnsA[0])) + if len(inp.ColumnsA[0]) == 0 || len(inp.ColumnsB[0]) == 0 { + utils.Panic("A and B must have at least one column: len(A[0])=%v, len(B[0])=%v", len(inp.ColumnsA[0]), len(inp.ColumnsB[0])) } var ( numCols = len(inp.ColumnsA[0]) numPartsA = len(inp.ColumnsA) numPartsB = len(inp.ColumnsB) + sizeA = inp.ColumnsA[0][0].Size() + sizeB = inp.ColumnsB[0][0].Size() ) if len(inp.FiltersA) != numPartsA || len(inp.FiltersB) != numPartsB { @@ -91,7 +93,15 @@ func NewProjectionMultiAry( size := ifaces.AssertSameLength(inp.ColumnsA[i]...) - if size != inp.FiltersA[i].Size() { + if i == 0 { + sizeA = size + } + + if size != sizeA { + utils.Panic("All table must have the same number of columns: len(A[%v])=%v, len(A[0])=%v", i, len(inp.ColumnsA), numCols) + } + + if sizeA != inp.FiltersA[i].Size() { utils.Panic("A[%v] and its filter do not have the same column sizes", i) } } @@ -103,7 +113,15 @@ func NewProjectionMultiAry( size := ifaces.AssertSameLength(inp.ColumnsB[i]...) - if size != inp.FiltersB[i].Size() { + if i == 0 { + sizeB = size + } + + if size != sizeB { + utils.Panic("All table must have the same number of columns: len(B[%v])=%v, len(B[0])=%v", i, len(inp.ColumnsB), numCols) + } + + if sizeB != inp.FiltersB[i].Size() { utils.Panic("B[%v] and its filter do not have the same column sizes", i) } } From 0d4b362dac0afe16bd4ab0f1e88132d66efb2de2 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 08:41:13 +0200 Subject: [PATCH 03/18] feat(horner): implements the multi-ary horner query --- prover/protocol/query/horner.go | 109 ++++++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 27 deletions(-) diff --git a/prover/protocol/query/horner.go b/prover/protocol/query/horner.go index aa19ebe017a..7a2e711cf5f 100644 --- a/prover/protocol/query/horner.go +++ b/prover/protocol/query/horner.go @@ -18,11 +18,13 @@ type HornerPart struct { // SignNegative indicates that the result should be negated. SignNegative bool // Coefficient is the coefficient of the term. It may be a - // column or random linear combination of columns. - Coefficient *symbolic.Expression + // column or random linear combination of columns. Each entry + // in the table corresponds to a multi-ary entry. + Coefficients []*symbolic.Expression // Selector is a boolean indicator column telling which terms // or [Coefficients] should be included in the Horner evaluation. - Selector ifaces.Column + // Each entry in the list corresponds to a multi-ary entry. + Selectors []ifaces.Column // X is the "x" value in the Horner evaluation query. Most of the // time, the accessor will be a random coin. The typing to accessor // allows for more flexibility. @@ -53,6 +55,10 @@ type HornerPart struct { // ``` // // where x and n0 are parameters of the query. +// +// As an addtional feature, the Horner query may be "multi-ary" meaning +// that the summation is not just computed vertically but left-to-right +// then top-to-bottom over a range of expressions. type Horner struct { // Round is the round of definition of the query Round int @@ -90,16 +96,26 @@ func NewHorner(round int, id ifaces.QueryID, parts []HornerPart) Horner { for i := range parts { - var ( - board = parts[i].Coefficient.Board() - size = column.ExprIsOnSameLengthHandles(&board) - ) + for j := range parts[i].Selectors { + var ( + board = parts[i].Coefficients[j].Board() + size = column.ExprIsOnSameLengthHandles(&board) + ) - if parts[i].Selector.Size() != size { - utils.Panic("Horner part %v has a selector of size %v and a coefficient of size %v", i, parts[i].Selector.Size(), size) - } + if size == 0 { + utils.Panic("Horner part %v has a coefficient of size 0", i) + } + + if parts[i].Selectors[j].Size() != size { + utils.Panic("Horner part %v has a selector of size %v and a coefficient of size %v", i, parts[i].Selectors[j].Size(), size) + } + + if parts[i].size > 0 && size != parts[i].size { + utils.Panic("Horner part %v has a selector of size %v and a coefficient of size %v", i, parts[i].Selectors[j].Size(), size) + } - parts[i].size = size + parts[i].size = size + } } return Horner{ @@ -151,24 +167,12 @@ func (p *HornerParams) GetResult(run ifaces.Runtime, q Horner) (n1s []int, final for i, part := range q.Parts { var ( - dataBoard = part.Coefficient.Board() - data = column.EvalExprColumn(run, dataBoard).IntoRegVecSaveAlloc() - sel = part.Selector.GetColAssignment(run).IntoRegVecSaveAlloc() - n0 = p.Parts[i].N0 - x = part.X.GetVal(run) - count = 0 - res field.Element - xN0 = new(field.Element).Exp(x, big.NewInt(int64(n0))) + n0 = p.Parts[i].N0 + res, count = getResultOfParts(run, &part) + x = part.X.GetVal(run) + xN0 = new(field.Element).Exp(x, big.NewInt(int64(n0))) ) - for j := len(data) - 1; j >= 0; j-- { - if sel[j].IsOne() { - res.Mul(&res, &x) - res.Add(&res, &data[j]) - count++ - } - } - res.Mul(&res, xN0) if part.SignNegative { @@ -182,6 +186,57 @@ func (p *HornerParams) GetResult(run ifaces.Runtime, q Horner) (n1s []int, final return n1s, finalResult } +// getResultOfParts computes the result of a part i of the [HornerQuery]. It +// returns the result of the evaluation and the selector count. +func getResultOfParts(run ifaces.Runtime, q *HornerPart) (field.Element, int) { + + var ( + datas = [][]field.Element{} + selectors = [][]field.Element{} + count = 0 + x = q.X.GetVal(run) + acc = field.Zero() + size = 0 + ) + + for coor := range q.Coefficients { + + var ( + board = q.Coefficients[coor].Board() + data = column.EvalExprColumn(run, board).IntoRegVecSaveAlloc() + selector = q.Selectors[coor].GetColAssignment(run).IntoRegVecSaveAlloc() + ) + + datas = append(datas, data) + selectors = append(selectors, selector) + + if coor == 0 { + size = len(data) + } + + if size != len(data) { + // Note, this is already check at the constructor level. + utils.Panic("All data must have the same size") + } + + } + + for row := 0; row < size; row++ { + for coor := 0; coor < len(datas); coor++ { + + if selectors[coor][row].IsZero() { + continue + } + + count++ + acc.Mul(&acc, &x) + acc.Add(&acc, &datas[coor][row]) + } + } + + return acc, count +} + // Name implements the [ifaces.Query] interface func (h *Horner) Name() ifaces.QueryID { return h.ID From a550e7138c6368646e3ed4001244542ca2d50af6 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 08:50:33 +0200 Subject: [PATCH 04/18] feat(testtools): adjust the testtools so that we can test retrocompatibility --- prover/protocol/internal/testtools/horner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prover/protocol/internal/testtools/horner.go b/prover/protocol/internal/testtools/horner.go index 330d346325b..b356d4472fa 100644 --- a/prover/protocol/internal/testtools/horner.go +++ b/prover/protocol/internal/testtools/horner.go @@ -234,16 +234,16 @@ func (t *HornerTestcase) Define(comp *wizard.CompiledIOP) { for i := range parts { parts[i] = query.HornerPart{ SignNegative: t.SignNegativeParts[i], - Coefficient: sym.NewVariable(comp.InsertCommit( + Coefficients: []*sym.Expression{sym.NewVariable(comp.InsertCommit( 0, formatName[ifaces.ColID]("Horner", t.NameStr, "Coefficient", i), t.Coefficients[i].Len(), - )), - Selector: comp.InsertCommit( + ))}, + Selectors: []ifaces.Column{comp.InsertCommit( 0, formatName[ifaces.ColID]("Horner", t.NameStr, "Selector", i), t.Selectors[i].Len(), - ), + )}, X: accessors.NewConstant(t.Xs[i]), } } From 3c5ded28ec0396a2e6d75de94de98ab20025ee3b Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 08:50:57 +0200 Subject: [PATCH 05/18] fixup(horner): iterate in reverse order over the rows to pass the retro-tests --- prover/protocol/query/horner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/query/horner.go b/prover/protocol/query/horner.go index 7a2e711cf5f..ca76dd128ad 100644 --- a/prover/protocol/query/horner.go +++ b/prover/protocol/query/horner.go @@ -221,7 +221,7 @@ func getResultOfParts(run ifaces.Runtime, q *HornerPart) (field.Element, int) { } - for row := 0; row < size; row++ { + for row := size - 1; row >= 0; row-- { for coor := 0; coor < len(datas); coor++ { if selectors[coor][row].IsZero() { From 00dbb90b1a06e58c02b7af2241dfcd433d6bcc3f Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 09:18:17 +0200 Subject: [PATCH 06/18] feat(projection): updates the projection query to work with the multi-ary projection --- .../compiler/horner/projection_to_horner.go | 56 ++++++++++++++----- prover/protocol/query/horner.go | 1 - 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/prover/protocol/compiler/horner/projection_to_horner.go b/prover/protocol/compiler/horner/projection_to_horner.go index d43736d215c..bdaec40c926 100644 --- a/prover/protocol/compiler/horner/projection_to_horner.go +++ b/prover/protocol/compiler/horner/projection_to_horner.go @@ -58,32 +58,58 @@ func ProjectionToHorner(comp *wizard.CompiledIOP) { comp.QueriesNoParams.MarkAsIgnored(qName) - qRound := comp.QueriesNoParams.Round(qName) - round = max(round, qRound+1) - var ( - alpha = comp.InsertCoin(qRound+1, coin.Name(qName+"_COIN_ALPHA"), coin.Field) - a = symbolic.NewVariable(projection.Inp.ColumnA[0]) - b = symbolic.NewVariable(projection.Inp.ColumnB[0]) + qRound = comp.QueriesNoParams.Round(qName) + widthA = len(projection.Inp.ColumnsA) + widthB = len(projection.Inp.ColumnsB) + numCols = len(projection.Inp.ColumnsA[0]) + as = make([]*symbolic.Expression, widthA) + bs = make([]*symbolic.Expression, widthB) + selectorsA = make([]ifaces.Column, widthA) + selectorsB = make([]ifaces.Column, widthB) + gamma coin.Info + alpha = comp.InsertCoin(qRound+1, coin.Name(qName+"_COIN_ALPHA"), coin.Field) ) - if len(projection.Inp.ColumnA) > 1 { - gamma := comp.InsertCoin(qRound+1, coin.Name(qName+"_COIN_GAMMA"), coin.Field) - a = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnA) - b = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnB) + round = max(round, qRound+1) + + if numCols > 1 { + gamma = comp.InsertCoin(qRound+1, coin.Name(qName+"_COIN_GAMMA"), coin.Field) + } + + for i := 0; i < widthA; i++ { + a := symbolic.NewVariable(projection.Inp.ColumnsA[i][0]) + + if numCols > 1 { + a = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsA[i]) + } + + as = append(as, a) + selectorsA = append(selectorsA, projection.Inp.FiltersA[i]) + } + + for i := 0; i < widthB; i++ { + b := symbolic.NewVariable(projection.Inp.ColumnsB[i][0]) + + if numCols > 1 { + b = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsB[i]) + } + + bs = append(bs, b) + selectorsB = append(selectorsB, projection.Inp.FiltersB[i]) } parts = append( parts, query.HornerPart{ - Coefficient: a, - Selector: projection.Inp.FilterA, - X: accessors.NewFromCoin(alpha), + Coefficients: as, + Selectors: selectorsA, + X: accessors.NewFromCoin(alpha), }, query.HornerPart{ SignNegative: true, - Coefficient: b, - Selector: projection.Inp.FilterB, + Coefficients: bs, + Selectors: selectorsB, X: accessors.NewFromCoin(alpha), }, ) diff --git a/prover/protocol/query/horner.go b/prover/protocol/query/horner.go index ca76dd128ad..82c9762efe2 100644 --- a/prover/protocol/query/horner.go +++ b/prover/protocol/query/horner.go @@ -218,7 +218,6 @@ func getResultOfParts(run ifaces.Runtime, q *HornerPart) (field.Element, int) { // Note, this is already check at the constructor level. utils.Panic("All data must have the same size") } - } for row := size - 1; row >= 0; row-- { From 6518f5e860da0392d9adcbebf2b9f280e2ed7a4a Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 09:22:18 +0200 Subject: [PATCH 07/18] feat(horner): ensures the changes are retro compatible with the horner compilers --- prover/protocol/compiler/horner/horner.go | 22 +++++++++---------- .../compiler/horner/projection_to_horner.go | 14 +++++------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/prover/protocol/compiler/horner/horner.go b/prover/protocol/compiler/horner/horner.go index 47b4fca0628..2895ff931a9 100644 --- a/prover/protocol/compiler/horner/horner.go +++ b/prover/protocol/compiler/horner/horner.go @@ -114,14 +114,14 @@ func compileHornerQuery(comp *wizard.CompiledIOP, q *query.Horner) { sym.Sub( col, sym.Mul( - sym.Sub(1, q.Parts[i].Selector), + sym.Sub(1, q.Parts[i].Selectors[0]), column.Shift(col, 1), ), sym.Mul( - q.Parts[i].Selector, + q.Parts[i].Selectors[0], sym.Add( sym.Mul(q.Parts[i].X, column.Shift(col, 1)), - q.Parts[i].Coefficient), + q.Parts[i].Coefficients[0]), ), ), ) @@ -133,8 +133,8 @@ func compileHornerQuery(comp *wizard.CompiledIOP, q *query.Horner) { sym.Sub( column.Shift(col, -1), sym.Mul( - column.Shift(q.Parts[i].Selector, -1), - column.ShiftExpr(q.Parts[i].Coefficient, -1), + column.Shift(q.Parts[i].Selectors[0], -1), + column.ShiftExpr(q.Parts[i].Coefficients[0], -1), ), ), ) @@ -146,10 +146,10 @@ func compileHornerQuery(comp *wizard.CompiledIOP, q *query.Horner) { } selectorsForSize := ctx.Selectors[partSize] - *selectorsForSize = append(*selectorsForSize, q.Parts[i].Selector) + *selectorsForSize = append(*selectorsForSize, q.Parts[i].Selectors[0]) ctx.AccumulatingCols = append(ctx.AccumulatingCols, col) ctx.LocOpenings = append(ctx.LocOpenings, loc) - iPRound = max(iPRound, q.Parts[i].Selector.Round()) + iPRound = max(iPRound, q.Parts[i].Selectors[0].Round()) } sizes := utils.SortedKeysOf(ctx.Selectors, func(a, b int) bool { @@ -187,8 +187,8 @@ func (a assignHornerCtx) Run(run *wizard.ProverRuntime) { var ( col = make([]field.Element, a.AccumulatingCols[i].Size()) - coeffBoard = a.Q.Parts[i].Coefficient.Board() - selector = a.Q.Parts[i].Selector.GetColAssignment(run).IntoRegVecSaveAlloc() + coeffBoard = a.Q.Parts[i].Coefficients[0].Board() + selector = a.Q.Parts[i].Selectors[0].GetColAssignment(run).IntoRegVecSaveAlloc() data = column.EvalExprColumn(run, coeffBoard).IntoRegVecSaveAlloc() x = a.Q.Parts[i].X.GetVal(run) n0 = params.Parts[i].N0 @@ -300,7 +300,7 @@ func (c *checkHornerResult) Run(run wizard.Runtime) error { for j, c := range hornerQuery.Parts { - if c.Selector.GetColID() != ipQuery.Bs[k].GetColID() { + if c.Selectors[0].GetColID() != ipQuery.Bs[k].GetColID() { continue } @@ -378,7 +378,7 @@ func (c *checkHornerResult) RunGnark(api frontend.API, run wizard.GnarkRuntime) for j, c := range hornerQuery.Parts { - if c.Selector.GetColID() != ipQuery.Bs[k].GetColID() { + if c.Selectors[0].GetColID() != ipQuery.Bs[k].GetColID() { continue } diff --git a/prover/protocol/compiler/horner/projection_to_horner.go b/prover/protocol/compiler/horner/projection_to_horner.go index bdaec40c926..10d63d6449d 100644 --- a/prover/protocol/compiler/horner/projection_to_horner.go +++ b/prover/protocol/compiler/horner/projection_to_horner.go @@ -78,25 +78,23 @@ func ProjectionToHorner(comp *wizard.CompiledIOP) { } for i := 0; i < widthA; i++ { - a := symbolic.NewVariable(projection.Inp.ColumnsA[i][0]) + as[i] = symbolic.NewVariable(projection.Inp.ColumnsA[i][0]) if numCols > 1 { - a = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsA[i]) + as[i] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsA[i]) } - as = append(as, a) - selectorsA = append(selectorsA, projection.Inp.FiltersA[i]) + selectorsA[i] = projection.Inp.FiltersA[i] } for i := 0; i < widthB; i++ { - b := symbolic.NewVariable(projection.Inp.ColumnsB[i][0]) + bs[i] = symbolic.NewVariable(projection.Inp.ColumnsB[i][0]) if numCols > 1 { - b = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsB[i]) + bs[i] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsB[i]) } - bs = append(bs, b) - selectorsB = append(selectorsB, projection.Inp.FiltersB[i]) + selectorsB[i] = projection.Inp.FiltersB[i] } parts = append( From 83a49c0de8f0dcd1f4ee6137a7f266c1b42a5f27 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 15:53:25 +0200 Subject: [PATCH 08/18] minor(horner): alias the import of symbolic expression to sym --- .../protocol/compiler/horner/projection_to_horner.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/prover/protocol/compiler/horner/projection_to_horner.go b/prover/protocol/compiler/horner/projection_to_horner.go index 10d63d6449d..9faf5f92c9a 100644 --- a/prover/protocol/compiler/horner/projection_to_horner.go +++ b/prover/protocol/compiler/horner/projection_to_horner.go @@ -10,7 +10,7 @@ import ( "github.com/consensys/linea-monorepo/prover/protocol/query" "github.com/consensys/linea-monorepo/prover/protocol/wizard" "github.com/consensys/linea-monorepo/prover/protocol/wizardutils" - "github.com/consensys/linea-monorepo/prover/symbolic" + sym "github.com/consensys/linea-monorepo/prover/symbolic" "github.com/consensys/linea-monorepo/prover/utils" ) @@ -63,8 +63,8 @@ func ProjectionToHorner(comp *wizard.CompiledIOP) { widthA = len(projection.Inp.ColumnsA) widthB = len(projection.Inp.ColumnsB) numCols = len(projection.Inp.ColumnsA[0]) - as = make([]*symbolic.Expression, widthA) - bs = make([]*symbolic.Expression, widthB) + as = make([]*sym.Expression, widthA) + bs = make([]*sym.Expression, widthB) selectorsA = make([]ifaces.Column, widthA) selectorsB = make([]ifaces.Column, widthB) gamma coin.Info @@ -79,7 +79,7 @@ func ProjectionToHorner(comp *wizard.CompiledIOP) { for i := 0; i < widthA; i++ { - as[i] = symbolic.NewVariable(projection.Inp.ColumnsA[i][0]) + as[i] = sym.NewVariable(projection.Inp.ColumnsA[i][0]) if numCols > 1 { as[i] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsA[i]) } @@ -89,7 +89,7 @@ func ProjectionToHorner(comp *wizard.CompiledIOP) { for i := 0; i < widthB; i++ { - bs[i] = symbolic.NewVariable(projection.Inp.ColumnsB[i][0]) + bs[i] = sym.NewVariable(projection.Inp.ColumnsB[i][0]) if numCols > 1 { bs[i] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsB[i]) } From dde52dbe4c294c9c8fc1aebec4b2a5b27b02c40a Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 15:53:45 +0200 Subject: [PATCH 09/18] feat(horner): make the changes to the horner compiler --- prover/protocol/compiler/horner/horner.go | 234 ++++++++++++---------- 1 file changed, 130 insertions(+), 104 deletions(-) diff --git a/prover/protocol/compiler/horner/horner.go b/prover/protocol/compiler/horner/horner.go index 2895ff931a9..b87e97049c9 100644 --- a/prover/protocol/compiler/horner/horner.go +++ b/prover/protocol/compiler/horner/horner.go @@ -23,14 +23,11 @@ import ( type hornerCtx struct { // Column is the accumulating column used to check the computation of a // horner value for one [HornerPart]. - AccumulatingCols []ifaces.Column - - // Selectors are the list of the selectors for each columns, sorted by - // size. - Selectors map[int]*[]ifaces.Column + AccumulatingCols [][]ifaces.Column // CountingInnerProduct is the inner-product query used to check the - // counting for each selector. + // counting for each selector. Each entry of the inner-product query + // maps to a selector. CountingInnerProducts []query.InnerProduct // LocOpenings are the local openings used to check the first value of @@ -88,42 +85,53 @@ func compileHornerQuery(comp *wizard.CompiledIOP, q *query.Horner) { var ( round = q.Round ctx = hornerCtx{ - Selectors: make(map[int]*[]ifaces.Column), - Q: q, + Q: q, } iPRound = 0 ) - for i := range q.Parts { + for i, part := range q.Parts { - col := comp.InsertCommit( - round, - ifaces.ColIDf("HORNER_%v_PART_%v_COLUMN", q.ID, i), - q.Parts[i].Size(), + var ( + numList = len(q.Parts[i].Coefficients) + accumulators = make([]ifaces.Column, numList) ) + for j := 0; j < numList; j++ { + + accumulators[j] = comp.InsertCommit( + round, + ifaces.ColIDf("HORNER_%v_PART_%v_COLUMN_%v", q.ID, i, j), + part.Size(), + ) + } + + for j := 0; j < numList-1; j++ { + + prevAcc := column.Shift(accumulators[numList-1], 1) + if j > 0 { + prevAcc = accumulators[j-1] + } + + comp.InsertGlobal( + round, + ifaces.QueryIDf("HORNER_%v_PART_%v_GLOBAL_%v", q.ID, i, j), + sym.Sub( + accumulators[j], + microAccumulate( + part.Selectors[j], + prevAcc, + part.X, + part.Coefficients[j], + ), + ), + ) + } + loc := comp.InsertLocalOpening( round, ifaces.QueryIDf("HORNER_%v_PART_%v_LOCAL_OPENING", q.ID, i), - col, - ) - - comp.InsertGlobal( - round, - ifaces.QueryIDf("HORNER_%v_PART_%v_GLOBAL", q.ID, i), - sym.Sub( - col, - sym.Mul( - sym.Sub(1, q.Parts[i].Selectors[0]), - column.Shift(col, 1), - ), - sym.Mul( - q.Parts[i].Selectors[0], - sym.Add( - sym.Mul(q.Parts[i].X, column.Shift(col, 1)), - q.Parts[i].Coefficients[0]), - ), - ), + accumulators[numList-1], ) // This query takes care of checking the initial value of the column @@ -131,7 +139,7 @@ func compileHornerQuery(comp *wizard.CompiledIOP, q *query.Horner) { round, ifaces.QueryIDf("HORNER_%v_PART_%v_LOCAL", q.ID, i), sym.Sub( - column.Shift(col, -1), + column.Shift(accumulators[0], -1), sym.Mul( column.Shift(q.Parts[i].Selectors[0], -1), column.ShiftExpr(q.Parts[i].Coefficients[0], -1), @@ -139,36 +147,27 @@ func compileHornerQuery(comp *wizard.CompiledIOP, q *query.Horner) { ), ) - partSize := q.Parts[i].Size() - - if _, ok := ctx.Selectors[partSize]; !ok { - ctx.Selectors[partSize] = &[]ifaces.Column{} - } - - selectorsForSize := ctx.Selectors[partSize] - *selectorsForSize = append(*selectorsForSize, q.Parts[i].Selectors[0]) - ctx.AccumulatingCols = append(ctx.AccumulatingCols, col) + ctx.AccumulatingCols = append(ctx.AccumulatingCols, accumulators) ctx.LocOpenings = append(ctx.LocOpenings, loc) iPRound = max(iPRound, q.Parts[i].Selectors[0].Round()) } - sizes := utils.SortedKeysOf(ctx.Selectors, func(a, b int) bool { - return a < b - }) - - for _, size := range sizes { + for i, part := range q.Parts { - selectors := ctx.Selectors[size] + size := part.Size() - ctx.CountingInnerProducts = append( - ctx.CountingInnerProducts, - comp.InsertInnerProduct( - iPRound, - ifaces.QueryIDf("HORNER_%v_COUNTING_%v_SRCNT_%v", q.ID, size, comp.SelfRecursionCount), - verifiercol.NewConstantCol(field.One(), size), - *selectors, - ), + // In theory, it would be more efficient to batch everything in a single + // inner-product by size. But that would make the code harder to understand + // and would require backtracking which result of the query corresponds to + // which part of the Horner query. + ip := comp.InsertInnerProduct( + iPRound, + ifaces.QueryIDf("HORNER_%v_COUNTING_%v_SRCNT_%v_%v", q.ID, size, comp.SelfRecursionCount, i), + verifiercol.NewConstantCol(field.One(), size), + ctx.Q.Parts[i].Selectors, ) + + ctx.CountingInnerProducts = append(ctx.CountingInnerProducts, ip) } comp.RegisterProverAction(iPRound, assignHornerIP{ctx}) @@ -183,48 +182,51 @@ func (a assignHornerCtx) Run(run *wizard.ProverRuntime) { res = field.Zero() ) - for i, lo := range a.LocOpenings { + for i, part := range a.Q.Parts { var ( - col = make([]field.Element, a.AccumulatingCols[i].Size()) - coeffBoard = a.Q.Parts[i].Coefficients[0].Board() - selector = a.Q.Parts[i].Selectors[0].GetColAssignment(run).IntoRegVecSaveAlloc() - data = column.EvalExprColumn(run, coeffBoard).IntoRegVecSaveAlloc() - x = a.Q.Parts[i].X.GetVal(run) - n0 = params.Parts[i].N0 - count = 0 + arity = len(part.Selectors) + datas = make([]smartvectors.SmartVector, arity) + selectors = make([]smartvectors.SmartVector, arity) + x = part.X.GetVal(run) + n0 = params.Parts[i].N0 + count = 0 + numRow = part.Size() + acc = field.Zero() + accumulators = make([][]field.Element, arity) ) - col[len(col)-1].Mul(&selector[len(col)-1], &data[len(col)-1]) - if selector[len(col)-1].IsOne() { - count++ + for k := 0; k < arity; k++ { + board := part.Coefficients[k].Board() + datas[k] = column.EvalExprColumn(run, board) + selectors[k] = part.Selectors[k].GetColAssignment(run) + accumulators[k] = make([]field.Element, numRow) } - for j := len(col) - 2; j >= 0; j-- { + for row := numRow - 1; row >= 0; row-- { + for k := 0; k < arity; k++ { - if selector[j].IsZero() { - col[j].Set(&col[j+1]) - continue - } + sel := selectors[k].Get(row) + if sel.IsOne() { + count++ + } - if selector[j].IsOne() { - col[j].Mul(&col[j+1], &x) - col[j].Add(&col[j], &data[j]) - count++ - continue + acc = computeMicroAccumulate(selectors[k].Get(row), acc, x, datas[k].Get(row)) + accumulators[k][row] = acc } - - utils.Panic("selector should be a binary column (and this should be enforced by the caller). If this is failing, then the circuit has a soundness error") } if n0+count != params.Parts[i].N1 { + // To update once we merge with the "code 78" branch as it means that a constraint is not satisfied. utils.Panic("the counting of the 1s in the filter does not match the one in the local-opening: (%v-%v) != %v", params.Parts[i].N1, n0, count) } - run.AssignColumn(a.AccumulatingCols[i].GetColID(), smartvectors.NewRegular(col)) - run.AssignLocalPoint(lo.ID, col[0]) + for k := 0; k < arity; k++ { + run.AssignColumn(a.AccumulatingCols[i][k].GetColID(), smartvectors.NewRegular(accumulators[k])) + } - tmp := col[0] + tmp := accumulators[arity-1][0] + run.AssignLocalPoint(a.LocOpenings[i].ID, tmp) if n0 > 0 { xN0 := new(field.Element).Exp(x, big.NewInt(int64(n0))) @@ -245,19 +247,15 @@ func (a assignHornerCtx) Run(run *wizard.ProverRuntime) { func (a assignHornerIP) Run(run *wizard.ProverRuntime) { - sizes := utils.SortedKeysOf(a.Selectors, func(a, b int) bool { - return a < b - }) - - for i, size := range sizes { + for i := range a.Q.Parts { var ( ip = a.CountingInnerProducts[i] - selectors = a.Selectors[size] - res = make([]field.Element, len(*selectors)) + selectors = a.Q.Parts[i].Selectors + res = make([]field.Element, len(selectors)) ) - for i, selector := range *selectors { + for i, selector := range selectors { sel := selector.GetColAssignment(run).IntoRegVecSaveAlloc() for j := range sel { res[i].Add(&res[i], &sel[j]) @@ -266,6 +264,7 @@ func (a assignHornerIP) Run(run *wizard.ProverRuntime) { run.AssignInnerProduct(ip.ID, res...) } + } func (c *checkHornerResult) Run(run wizard.Runtime) error { @@ -276,13 +275,7 @@ func (c *checkHornerResult) Run(run wizard.Runtime) error { res = field.Zero() ) - sizes := utils.SortedKeysOf(c.Selectors, func(a, b int) bool { - return a < b - }) - - // This loop is responsible for checking the consistency of the IP queries - // with the N1-N0 difference from the Horner params. - for i := range sizes { + for i := range c.Q.Parts { var ( ipQuery = c.CountingInnerProducts[i] @@ -358,13 +351,7 @@ func (c *checkHornerResult) RunGnark(api frontend.API, run wizard.GnarkRuntime) res = frontend.Variable(0) ) - sizes := utils.SortedKeysOf(c.Selectors, func(a, b int) bool { - return a < b - }) - - // This loop is responsible for checking the consistency of the IP queries - // with the N1-N0 difference from the Horner params. - for i := range sizes { + for i := range c.Q.Parts { var ( ipQuery = c.CountingInnerProducts[i] @@ -426,3 +413,42 @@ func (c *checkHornerResult) Skip() { func (c *checkHornerResult) IsSkipped() bool { return c.skipped } + +// microAccumulate returns an atomic accumulator update expression. +// +// the returned expression evaluates to: +// +// ``` +// +// sel == 1 => acc.X + p +// sel == 0 => acc +// +// ``` +func microAccumulate(sel, acc, x, p any) *sym.Expression { + return sym.Add( + sym.Mul( + sel, + sym.Add(p, sym.Mul(x, acc)), + ), + sym.Mul( + sym.Sub(1, sel), + acc, + ), + ) +} + +func computeMicroAccumulate(sel, acc, x, p field.Element) field.Element { + + if sel.IsZero() { + return acc + } + + if sel.IsOne() { + var tmp field.Element + tmp.Mul(&x, &acc) + tmp.Add(&tmp, &p) + return tmp + } + + panic("selector is non-binary") +} From f6c94b760896a1a7d21b10fcf3733e0651f2018b Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 19:16:54 +0200 Subject: [PATCH 10/18] fixup(horner): all the tests passes for horner and the other query --- prover/protocol/compiler/horner/horner.go | 58 ++--- .../compiler/horner/projection_to_horner.go | 23 +- prover/protocol/internal/testtools/horner.go | 168 +++++++++------ .../protocol/internal/testtools/projection.go | 200 ++++++++++-------- prover/protocol/wizard/compiled.go | 36 +++- 5 files changed, 276 insertions(+), 209 deletions(-) diff --git a/prover/protocol/compiler/horner/horner.go b/prover/protocol/compiler/horner/horner.go index b87e97049c9..98fa62e6c37 100644 --- a/prover/protocol/compiler/horner/horner.go +++ b/prover/protocol/compiler/horner/horner.go @@ -280,37 +280,26 @@ func (c *checkHornerResult) Run(run wizard.Runtime) error { var ( ipQuery = c.CountingInnerProducts[i] ipParams = run.GetInnerProductParams(c.CountingInnerProducts[i].ID) - found = false + ipCount = 0 ) for k := range ipQuery.Bs { + // Note: this check is not purely necessary from the verifier viewpoint. If + // the result is not a uint64, then it means the query was malformed: the + // result of the inner-product is the inner-product of two binary vectors + // and there is no way they are big enough to overflow the 2**64. + // + // Still, it is useful information as it indicates the protocol is malformed. if !ipParams.Ys[k].IsUint64() { return errors.New("ip result does not fit on a uint64") } - ipCount := int(ipParams.Ys[k].Uint64()) - - for j, c := range hornerQuery.Parts { - - if c.Selectors[0].GetColID() != ipQuery.Bs[k].GetColID() { - continue - } - - found = true - params := hornerParams.Parts[j] - n0, n1 := params.N0, params.N1 - - if n1-n0 != ipCount { - return fmt.Errorf("inner-product and horner params do not match: %v - %v (%v) != %v", n1, n0, n1-n0, ipCount) - } - - break - } + ipCount += int(ipParams.Ys[k].Uint64()) + } - if !found { - utils.Panic("could not find selector %v from the Horner query", ipQuery.Bs[k].String()) - } + if hornerParams.Parts[i].N0+ipCount != hornerParams.Parts[i].N1 { + return fmt.Errorf("the counting of the 1s in the filter does not match the one in the local-opening: (%v-%v) != %v", hornerParams.Parts[i].N1, hornerParams.Parts[i].N0, ipCount) } } @@ -356,31 +345,14 @@ func (c *checkHornerResult) RunGnark(api frontend.API, run wizard.GnarkRuntime) var ( ipQuery = c.CountingInnerProducts[i] ipParams = run.GetInnerProductParams(c.CountingInnerProducts[i].ID) - found = false + ipCount = frontend.Variable(0) ) for k := range ipQuery.Bs { - - ipCount := ipParams.Ys[k] - - for j, c := range hornerQuery.Parts { - - if c.Selectors[0].GetColID() != ipQuery.Bs[k].GetColID() { - continue - } - - found = true - params := hornerParams.Parts[j] - n0, n1 := params.N0, params.N1 - - api.AssertIsEqual(n1, api.Add(n0, ipCount)) - break - } - - if !found { - utils.Panic("could not find selector %v from the Horner query", ipQuery.Bs[k].String()) - } + ipCount = api.Add(ipCount, ipParams.Ys[k]) } + + api.AssertIsEqual(api.Add(hornerParams.Parts[i].N0, ipCount), hornerParams.Parts[i].N1) } // This loop is responsible for checking that the final result is correctly diff --git a/prover/protocol/compiler/horner/projection_to_horner.go b/prover/protocol/compiler/horner/projection_to_horner.go index 9faf5f92c9a..e1db3a486bc 100644 --- a/prover/protocol/compiler/horner/projection_to_horner.go +++ b/prover/protocol/compiler/horner/projection_to_horner.go @@ -17,9 +17,6 @@ import ( // projectionContext is a compilation artefact generated during the execution of // the [InsertProjection] and which is used to instantiate the Horner query. type projectionContext struct { - // Xs are the coins used as X values in the Horner query that compiles the - // projection queries. - Xs []coin.Info // Query is the Horner query generated during the compilation of the projection // queries. Query query.Horner @@ -79,22 +76,28 @@ func ProjectionToHorner(comp *wizard.CompiledIOP) { for i := 0; i < widthA; i++ { - as[i] = sym.NewVariable(projection.Inp.ColumnsA[i][0]) + as[widthA-i-1] = sym.NewVariable(projection.Inp.ColumnsA[i][0]) if numCols > 1 { - as[i] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsA[i]) + as[widthA-i-1] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsA[i]) } - selectorsA[i] = projection.Inp.FiltersA[i] + // The reversal in the assignment is required due to the order + // in which the [Horner] query iterates over the coefficient in + // the multi-ary settings. + selectorsA[widthA-i-1] = projection.Inp.FiltersA[i] } for i := 0; i < widthB; i++ { - bs[i] = sym.NewVariable(projection.Inp.ColumnsB[i][0]) + bs[widthB-i-1] = sym.NewVariable(projection.Inp.ColumnsB[i][0]) if numCols > 1 { - bs[i] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsB[i]) + bs[widthB-i-1] = wizardutils.RandLinCombColSymbolic(gamma, projection.Inp.ColumnsB[i]) } - selectorsB[i] = projection.Inp.FiltersB[i] + // The reversal in the assignment is required due to the order + // in which the [Horner] query iterates over the coefficient in + // the multi-ary settings. + selectorsB[widthB-i-1] = projection.Inp.FiltersB[i] } parts = append( @@ -111,8 +114,6 @@ func ProjectionToHorner(comp *wizard.CompiledIOP) { X: accessors.NewFromCoin(alpha), }, ) - - ctx.Xs = append(ctx.Xs, alpha, alpha) } if len(parts) == 0 { diff --git a/prover/protocol/internal/testtools/horner.go b/prover/protocol/internal/testtools/horner.go index b356d4472fa..904f17c73e4 100644 --- a/prover/protocol/internal/testtools/horner.go +++ b/prover/protocol/internal/testtools/horner.go @@ -21,10 +21,10 @@ type HornerTestcase struct { SignNegativeParts []bool // Coefficients are the coefficients for each parts of the query - Coefficients []smartvectors.SmartVector + Coefficients [][]smartvectors.SmartVector // Selectors are the selectors for each parts of the query - Selectors []smartvectors.SmartVector + Selectors [][]smartvectors.SmartVector // N0s are the N0 values for each parts of the query N0s []int @@ -50,12 +50,12 @@ var ListOfHornerTestcasePositive = []*HornerTestcase{ { NameStr: "positive/none-selected-single", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.Zero(), 8), - }, + }}, N0s: []int{0}, N1s: []int{0}, Xs: []field.Element{field.One()}, @@ -65,13 +65,21 @@ var ListOfHornerTestcasePositive = []*HornerTestcase{ { NameStr: "positive/two-parts-cancelling", SignNegativeParts: []bool{false, true}, - Coefficients: []smartvectors.SmartVector{ - RandomFromSeed(8, 1), - RandomFromSeed(8, 1), + Coefficients: [][]smartvectors.SmartVector{ + { + RandomFromSeed(8, 1), + }, + { + RandomFromSeed(8, 1), + }, }, - Selectors: []smartvectors.SmartVector{ - smartvectors.NewConstant(field.One(), 8), - smartvectors.NewConstant(field.One(), 8), + Selectors: [][]smartvectors.SmartVector{ + { + smartvectors.NewConstant(field.One(), 8), + }, + { + smartvectors.NewConstant(field.One(), 8), + }, }, N0s: []int{0, 0}, N1s: []int{8, 8}, @@ -82,12 +90,12 @@ var ListOfHornerTestcasePositive = []*HornerTestcase{ { NameStr: "positive/just-counting", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, + }}, N0s: []int{0}, N1s: []int{8}, Xs: []field.Element{field.One()}, @@ -97,12 +105,12 @@ var ListOfHornerTestcasePositive = []*HornerTestcase{ { NameStr: "positive/just-counting", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, + }}, N0s: []int{0}, N1s: []int{8}, Xs: []field.Element{field.NewElement(2)}, @@ -112,17 +120,38 @@ var ListOfHornerTestcasePositive = []*HornerTestcase{ { NameStr: "positive/12345..7", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.ForTest(0, 1, 2, 3, 4, 5, 6, 7), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, + }}, N0s: []int{0}, N1s: []int{8}, Xs: []field.Element{field.NewElement(2)}, FinalResult: field.NewElement(1538), }, + + { + NameStr: "positive/multi-ary", + SignNegativeParts: []bool{false}, + Coefficients: [][]smartvectors.SmartVector{ + { + smartvectors.ForTest(1, 3, 5, 7, 9, 11, 13, 15), + smartvectors.ForTest(0, 2, 4, 6, 8, 10, 12, 14), + }, + }, + Selectors: [][]smartvectors.SmartVector{ + { + smartvectors.NewConstant(field.One(), 8), + smartvectors.NewConstant(field.One(), 8), + }, + }, + N0s: []int{0}, + N1s: []int{16}, + Xs: []field.Element{field.NewElement(2)}, + FinalResult: field.NewElement(917506), + }, } var ListOfHornerTestcaseNegative = []*HornerTestcase{ @@ -130,12 +159,12 @@ var ListOfHornerTestcaseNegative = []*HornerTestcase{ { NameStr: "negative/none-selected-single/bad-count", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.Zero(), 8), - }, + }}, N0s: []int{0}, N1s: []int{1}, Xs: []field.Element{field.One()}, @@ -146,12 +175,12 @@ var ListOfHornerTestcaseNegative = []*HornerTestcase{ { NameStr: "negative/none-selected-single/bad-result", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.Zero(), 8), - }, + }}, N0s: []int{0}, N1s: []int{0}, Xs: []field.Element{field.One()}, @@ -162,14 +191,14 @@ var ListOfHornerTestcaseNegative = []*HornerTestcase{ { NameStr: "negative/two-parts-should-be-expected-to-cancel", SignNegativeParts: []bool{false, true}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ RandomFromSeed(8, 1), RandomFromSeed(8, 1), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), smartvectors.NewConstant(field.One(), 8), - }, + }}, N0s: []int{0, 0}, N1s: []int{8, 8}, Xs: []field.Element{field.One(), field.One()}, @@ -180,14 +209,14 @@ var ListOfHornerTestcaseNegative = []*HornerTestcase{ { NameStr: "negative/two-parts-should-be-expected-to-cancel", SignNegativeParts: []bool{false, true}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ RandomFromSeed(8, 1), RandomFromSeed(8, 1), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), smartvectors.NewConstant(field.One(), 8), - }, + }}, N0s: []int{0, 0}, N1s: []int{8, 7}, Xs: []field.Element{field.One(), field.One()}, @@ -198,12 +227,12 @@ var ListOfHornerTestcaseNegative = []*HornerTestcase{ { NameStr: "negative/just-counting/bad-n0", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, + }}, N0s: []int{1}, N1s: []int{8}, Xs: []field.Element{field.One()}, @@ -214,12 +243,12 @@ var ListOfHornerTestcaseNegative = []*HornerTestcase{ { NameStr: "negative/just-counting-x=2/bad-result", SignNegativeParts: []bool{false}, - Coefficients: []smartvectors.SmartVector{ + Coefficients: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, - Selectors: []smartvectors.SmartVector{ + }}, + Selectors: [][]smartvectors.SmartVector{{ smartvectors.NewConstant(field.One(), 8), - }, + }}, N0s: []int{0}, N1s: []int{8}, Xs: []field.Element{field.NewElement(2)}, @@ -234,17 +263,23 @@ func (t *HornerTestcase) Define(comp *wizard.CompiledIOP) { for i := range parts { parts[i] = query.HornerPart{ SignNegative: t.SignNegativeParts[i], - Coefficients: []*sym.Expression{sym.NewVariable(comp.InsertCommit( + Coefficients: make([]*sym.Expression, len(t.Coefficients[i])), + Selectors: make([]ifaces.Column, len(t.Selectors[i])), + X: accessors.NewConstant(t.Xs[i]), + } + + for j := range parts[i].Coefficients { + parts[i].Coefficients[j] = sym.NewVariable(comp.InsertCommit( 0, - formatName[ifaces.ColID]("Horner", t.NameStr, "Coefficient", i), - t.Coefficients[i].Len(), - ))}, - Selectors: []ifaces.Column{comp.InsertCommit( + formatName[ifaces.ColID]("Horner", t.NameStr, "Coefficient", i, j), + t.Coefficients[i][j].Len(), + )) + + parts[i].Selectors[j] = comp.InsertCommit( 0, - formatName[ifaces.ColID]("Horner", t.NameStr, "Selector", i), - t.Selectors[i].Len(), - )}, - X: accessors.NewConstant(t.Xs[i]), + formatName[ifaces.ColID]("Horner", t.NameStr, "Selector", i, j), + t.Selectors[i][j].Len(), + ) } } @@ -266,15 +301,18 @@ func (t *HornerTestcase) Assign(run *wizard.ProverRuntime) { N1: t.N1s[i], } - run.AssignColumn( - formatName[ifaces.ColID]("Horner", t.NameStr, "Coefficient", i), - t.Coefficients[i], - ) + for j := range t.Coefficients[i] { - run.AssignColumn( - formatName[ifaces.ColID]("Horner", t.NameStr, "Selector", i), - t.Selectors[i], - ) + run.AssignColumn( + formatName[ifaces.ColID]("Horner", t.NameStr, "Coefficient", i, j), + t.Coefficients[i][j], + ) + + run.AssignColumn( + formatName[ifaces.ColID]("Horner", t.NameStr, "Selector", i, j), + t.Selectors[i][j], + ) + } } run.AssignHornerParams( diff --git a/prover/protocol/internal/testtools/projection.go b/prover/protocol/internal/testtools/projection.go index e99789334c5..dc3a74dd917 100644 --- a/prover/protocol/internal/testtools/projection.go +++ b/prover/protocol/internal/testtools/projection.go @@ -11,8 +11,8 @@ import ( // ProjectionTestcase represents a test-case for a projection query type ProjectionTestcase struct { NameStr string - FilterA, FilterB smartvectors.SmartVector - As, Bs []smartvectors.SmartVector + FilterA, FilterB []smartvectors.SmartVector + As, Bs [][]smartvectors.SmartVector ShouldFail bool } @@ -20,52 +20,72 @@ var ListOfProjectionTestcasePositive = []*ProjectionTestcase{ { NameStr: "positive/selector-full-zeroes", - FilterA: smartvectors.NewConstant(field.Zero(), 16), - FilterB: smartvectors.NewConstant(field.Zero(), 8), - As: []smartvectors.SmartVector{ + FilterA: []smartvectors.SmartVector{smartvectors.NewConstant(field.Zero(), 16)}, + FilterB: []smartvectors.SmartVector{smartvectors.NewConstant(field.Zero(), 8)}, + As: [][]smartvectors.SmartVector{{ smartvectors.PseudoRand(rng, 16), - }, - Bs: []smartvectors.SmartVector{ + }}, + Bs: [][]smartvectors.SmartVector{{ smartvectors.PseudoRand(rng, 8), - }, + }}, }, { NameStr: "positive/counting-values", - FilterA: OnesAt(16, []int{2, 4, 6, 8, 10}), - FilterB: OnesAt(8, []int{1, 2, 3, 4, 5}), - As: []smartvectors.SmartVector{ + FilterA: []smartvectors.SmartVector{OnesAt(16, []int{2, 4, 6, 8, 10})}, + FilterB: []smartvectors.SmartVector{OnesAt(8, []int{1, 2, 3, 4, 5})}, + As: [][]smartvectors.SmartVector{{ CountingAt(16, 0, []int{2, 4, 6, 8, 10}), - }, - Bs: []smartvectors.SmartVector{ + }}, + Bs: [][]smartvectors.SmartVector{{ CountingAt(8, 0, []int{1, 2, 3, 4, 5}), - }, + }}, }, { NameStr: "positive/selector-full-zeroes-multicolumn", - FilterA: smartvectors.NewConstant(field.Zero(), 16), - FilterB: smartvectors.NewConstant(field.Zero(), 8), - As: []smartvectors.SmartVector{ + FilterA: []smartvectors.SmartVector{smartvectors.NewConstant(field.Zero(), 16)}, + FilterB: []smartvectors.SmartVector{smartvectors.NewConstant(field.Zero(), 8)}, + As: [][]smartvectors.SmartVector{{ smartvectors.PseudoRand(rng, 16), smartvectors.PseudoRand(rng, 16), - }, - Bs: []smartvectors.SmartVector{ + }}, + Bs: [][]smartvectors.SmartVector{{ smartvectors.PseudoRand(rng, 8), smartvectors.PseudoRand(rng, 8), - }, + }}, }, { NameStr: "positive/counting-values-multicolumn", - FilterA: OnesAt(16, []int{2, 4, 6, 8, 10}), - FilterB: OnesAt(8, []int{1, 2, 3, 4, 5}), - As: []smartvectors.SmartVector{ + FilterA: []smartvectors.SmartVector{OnesAt(16, []int{2, 4, 6, 8, 10})}, + FilterB: []smartvectors.SmartVector{OnesAt(8, []int{1, 2, 3, 4, 5})}, + As: [][]smartvectors.SmartVector{{ CountingAt(16, 0, []int{2, 4, 6, 8, 10}), CountingAt(16, 5, []int{2, 4, 6, 8, 10}), - }, - Bs: []smartvectors.SmartVector{ + }}, + Bs: [][]smartvectors.SmartVector{{ CountingAt(8, 0, []int{1, 2, 3, 4, 5}), CountingAt(8, 5, []int{1, 2, 3, 4, 5}), + }}, + }, + + { + NameStr: "positive/spaghettification", + FilterA: []smartvectors.SmartVector{smartvectors.NewConstant(field.One(), 16)}, + FilterB: []smartvectors.SmartVector{ + smartvectors.NewConstant(field.One(), 8), + smartvectors.NewConstant(field.One(), 8), + }, + As: [][]smartvectors.SmartVector{{ + smartvectors.ForTest(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), + }}, + Bs: [][]smartvectors.SmartVector{ + { + smartvectors.ForTest(1, 3, 5, 7, 9, 11, 13, 15), + }, + { + smartvectors.ForTest(2, 4, 6, 8, 10, 12, 14, 16), + }, }, }, } @@ -74,40 +94,40 @@ var ListOfProjectionTestcaseNegative = []*ProjectionTestcase{ { NameStr: "negative/full-random-with-full-ones-selectors", - FilterA: smartvectors.NewConstant(field.One(), 16), - FilterB: smartvectors.NewConstant(field.One(), 8), - As: []smartvectors.SmartVector{ + FilterA: []smartvectors.SmartVector{smartvectors.NewConstant(field.One(), 16)}, + FilterB: []smartvectors.SmartVector{smartvectors.NewConstant(field.One(), 8)}, + As: [][]smartvectors.SmartVector{{ smartvectors.PseudoRand(rng, 16), - }, - Bs: []smartvectors.SmartVector{ + }}, + Bs: [][]smartvectors.SmartVector{{ smartvectors.PseudoRand(rng, 8), - }, + }}, ShouldFail: true, }, { NameStr: "negative/counting-too-many", - FilterA: OnesAt(16, []int{2, 4, 6, 8, 10}), - FilterB: OnesAt(8, []int{1, 2, 3, 4, 5, 6}), - As: []smartvectors.SmartVector{ + FilterA: []smartvectors.SmartVector{OnesAt(16, []int{2, 4, 6, 8, 10})}, + FilterB: []smartvectors.SmartVector{OnesAt(8, []int{1, 2, 3, 4, 5, 6})}, + As: [][]smartvectors.SmartVector{{ CountingAt(16, 0, []int{2, 4, 6, 8, 10}), - }, - Bs: []smartvectors.SmartVector{ + }}, + Bs: [][]smartvectors.SmartVector{{ CountingAt(8, 0, []int{1, 2, 3, 4, 5, 6}), - }, + }}, ShouldFail: true, }, { NameStr: "negative/counting-misaligned", - FilterA: OnesAt(16, []int{2, 4, 6, 8, 10}), - FilterB: OnesAt(8, []int{1, 2, 3, 4, 5}), - As: []smartvectors.SmartVector{ + FilterA: []smartvectors.SmartVector{OnesAt(16, []int{2, 4, 6, 8, 10})}, + FilterB: []smartvectors.SmartVector{OnesAt(8, []int{1, 2, 3, 4, 5})}, + As: [][]smartvectors.SmartVector{{ CountingAt(16, 0, []int{2, 4, 6, 8, 10}), - }, - Bs: []smartvectors.SmartVector{ + }}, + Bs: [][]smartvectors.SmartVector{{ CountingAt(8, 0, []int{1, 2, 3, 4, 6}), - }, + }}, ShouldFail: true, }, } @@ -116,38 +136,46 @@ var ListOfProjectionTestcaseNegative = []*ProjectionTestcase{ // columns and a [query.Projection] as specified by the testcase. func (tc *ProjectionTestcase) Define(comp *wizard.CompiledIOP) { - inp := query.ProjectionInput{ + inp := query.ProjectionMultiAryInput{} - FilterA: comp.InsertCommit( - 0, - formatName[ifaces.ColID]("Projection", tc.Name, "filterA"), - tc.FilterA.Len(), - ), + for i := range tc.FilterA { - FilterB: comp.InsertCommit( + inp.FiltersA = append(inp.FiltersA, comp.InsertCommit( 0, - formatName[ifaces.ColID]("Projection", tc.Name, "filterB"), - tc.FilterB.Len(), - ), - - ColumnA: make([]ifaces.Column, len(tc.As)), - ColumnB: make([]ifaces.Column, len(tc.Bs)), + formatName[ifaces.ColID]("Projection", tc.Name, "filterA", i), + tc.FilterA[i].Len(), + )) + + columnsA := make([]ifaces.Column, len(tc.As[i])) + for j := range columnsA { + columnsA[j] = comp.InsertCommit( + 0, + formatName[ifaces.ColID]("Projection", tc.Name, "A", i, j), + tc.As[i][j].Len(), + ) + } + + inp.ColumnsA = append(inp.ColumnsA, columnsA) } - for i := range inp.ColumnA { - inp.ColumnA[i] = comp.InsertCommit( - 0, - formatName[ifaces.ColID]("Projection", tc.Name, "A", i), - tc.As[i].Len(), - ) - } + for i := range tc.FilterB { - for i := range inp.ColumnB { - inp.ColumnB[i] = comp.InsertCommit( + inp.FiltersB = append(inp.FiltersB, comp.InsertCommit( 0, - formatName[ifaces.ColID]("Projection", tc.Name, "B", i), - tc.Bs[i].Len(), - ) + formatName[ifaces.ColID]("Projection", tc.Name, "filterB", i), + tc.FilterB[i].Len(), + )) + + columnsB := make([]ifaces.Column, len(tc.Bs[i])) + for j := range columnsB { + columnsB[j] = comp.InsertCommit( + 0, + formatName[ifaces.ColID]("Projection", tc.Name, "B", i, j), + tc.Bs[i][j].Len(), + ) + } + + inp.ColumnsB = append(inp.ColumnsB, columnsB) } comp.InsertProjection( @@ -160,28 +188,34 @@ func (tc *ProjectionTestcase) Define(comp *wizard.CompiledIOP) { // columns taking place in the [query.Projection] query. func (tc *ProjectionTestcase) Assign(run *wizard.ProverRuntime) { - run.AssignColumn( - formatName[ifaces.ColID]("Projection", tc.Name, "filterA"), - tc.FilterA, - ) - - run.AssignColumn( - formatName[ifaces.ColID]("Projection", tc.Name, "filterB"), - tc.FilterB, - ) + for i := range tc.FilterA { - for i := range tc.As { run.AssignColumn( - formatName[ifaces.ColID]("Projection", tc.Name, "A", i), - tc.As[i], + formatName[ifaces.ColID]("Projection", tc.Name, "filterA", i), + tc.FilterA[i], ) + + for j := range tc.As[i] { + run.AssignColumn( + formatName[ifaces.ColID]("Projection", tc.Name, "A", i, j), + tc.As[i][j], + ) + } } - for i := range tc.Bs { + for i := range tc.FilterB { + run.AssignColumn( - formatName[ifaces.ColID]("Projection", tc.Name, "B", i), - tc.Bs[i], + formatName[ifaces.ColID]("Projection", tc.Name, "filterB", i), + tc.FilterB[i], ) + + for j := range tc.Bs[i] { + run.AssignColumn( + formatName[ifaces.ColID]("Projection", tc.Name, "B", i, j), + tc.Bs[i][j], + ) + } } } diff --git a/prover/protocol/wizard/compiled.go b/prover/protocol/wizard/compiled.go index 3cbe37c72d7..de25123d6fa 100644 --- a/prover/protocol/wizard/compiled.go +++ b/prover/protocol/wizard/compiled.go @@ -3,6 +3,8 @@ package wizard import ( // "reflect" + "slices" + "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" "github.com/consensys/linea-monorepo/prover/maths/field" "github.com/consensys/linea-monorepo/prover/protocol/coin" @@ -628,18 +630,38 @@ The projection query checks if a0 = b2, a3 = b8, a4 = b9 Note that the query imposes that: - the number of 1 in the filters are equal - the order of filtered elements is preserved + +The "in" argument can be either a [query.ProjectionInput] or a +[query.ProjectionMultiAryInput]. */ -func (c *CompiledIOP) InsertProjection(id ifaces.QueryID, in query.ProjectionInput) query.Projection { - var ( - round = max( +func (c *CompiledIOP) InsertProjection(id ifaces.QueryID, in any) query.Projection { + + var q query.Projection + + switch in := in.(type) { + + case query.ProjectionInput: + round := max( column.MaxRound(in.ColumnA...), column.MaxRound(in.ColumnB...), in.FilterA.Round(), in.FilterB.Round()) - ) - q := query.NewProjection(round, id, in) - // Finally registers the query - c.QueriesNoParams.AddToRound(round, q.Name(), q) + q = query.NewProjection(round, id, in) + + case query.ProjectionMultiAryInput: + round := max( + column.MaxRound(slices.Concat(in.ColumnsA...)...), + column.MaxRound(slices.Concat(in.ColumnsB...)...), + column.MaxRound(in.FiltersA...), + column.MaxRound(in.FiltersB...)) + q = query.NewProjectionMultiAry(round, id, in) + + default: + panic("invalid projection input") + } + + c.QueriesNoParams.AddToRound(q.Round, q.Name(), q) + return q } From 94dc44778758aa845b700099def7eb8152e7476d Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Tue, 13 May 2025 22:27:39 +0200 Subject: [PATCH 11/18] testing(projections): adds a test with selectors --- .../protocol/internal/testtools/projection.go | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/prover/protocol/internal/testtools/projection.go b/prover/protocol/internal/testtools/projection.go index dc3a74dd917..e771def580f 100644 --- a/prover/protocol/internal/testtools/projection.go +++ b/prover/protocol/internal/testtools/projection.go @@ -88,6 +88,28 @@ var ListOfProjectionTestcasePositive = []*ProjectionTestcase{ }, }, }, + + { + NameStr: "positive/spaghettification-with-selectors", + FilterA: []smartvectors.SmartVector{ + smartvectors.NewConstant(field.One(), 8), + }, + FilterB: []smartvectors.SmartVector{ + smartvectors.ForTest(1, 1, 1, 1, 0, 0, 0, 0), + smartvectors.ForTest(1, 1, 1, 1, 0, 0, 0, 0), + }, + As: [][]smartvectors.SmartVector{{ + smartvectors.ForTest(1, 2, 3, 4, 5, 6, 7, 8), + }}, + Bs: [][]smartvectors.SmartVector{ + { + smartvectors.ForTest(1, 3, 5, 7, -1, -1, -1, -1), + }, + { + smartvectors.ForTest(2, 4, 6, 8, -1, -1, -1, -1), + }, + }, + }, } var ListOfProjectionTestcaseNegative = []*ProjectionTestcase{ From 2ce08775a4f4ff557f4d8fe0f69c3538fa77dc3f Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Wed, 14 May 2025 10:00:41 +0200 Subject: [PATCH 12/18] feat(distributed): port the changes to the distributed prover --- .../distributed/module_discovery_standard.go | 6 ++- .../protocol/distributed/module_filtering.go | 19 ++++--- prover/protocol/distributed/module_gl.go | 4 +- prover/protocol/distributed/module_lpp.go | 29 ++++++----- .../distributed/module_translation.go | 43 +++++++++++----- prover/protocol/distributed/module_witness.go | 49 ++++++++++--------- 6 files changed, 91 insertions(+), 59 deletions(-) diff --git a/prover/protocol/distributed/module_discovery_standard.go b/prover/protocol/distributed/module_discovery_standard.go index 8b5c56e0f07..477e1ccd85c 100644 --- a/prover/protocol/distributed/module_discovery_standard.go +++ b/prover/protocol/distributed/module_discovery_standard.go @@ -450,7 +450,11 @@ func (disc *QueryBasedModuleDiscoverer) Analyze(comp *wizard.CompiledIOP) { case *query.Horner: for _, part := range q.Parts { - group := append(column.ColumnsOfExpression(part.Coefficient), part.Selector) + group := []ifaces.Column{} + for k := range part.Coefficients { + group = append(group, column.ColumnsOfExpression(part.Coefficients[k])...) + group = append(group, part.Selectors[k]) + } toGroup = append(toGroup, rootsOfColumns(group)) } } diff --git a/prover/protocol/distributed/module_filtering.go b/prover/protocol/distributed/module_filtering.go index a7a3bcc9e6a..68fe1112111 100644 --- a/prover/protocol/distributed/module_filtering.go +++ b/prover/protocol/distributed/module_filtering.go @@ -190,8 +190,11 @@ func (mf moduleFilter) FilterCompiledIOP(comp *wizard.CompiledIOP) FilteredModul } args := mf.FilterHornerParts(q) for i := range args { - cols := column.ColumnsOfExpression(args[i].Coefficient) - cols = append(cols, args[i].Selector) + cols := []ifaces.Column{} + for k := range args[i].Selectors { + cols = append(cols, column.ColumnsOfExpression(args[i].Coefficients[k])...) + cols = append(cols, args[i].Selectors[k]) + } roots := column.RootsOf(cols, true) for _, root := range roots { fmi.addColumnLPP(root) @@ -432,14 +435,14 @@ func (filter moduleFilter) FilterHornerParts(q *query.Horner) []query.HornerPart for _, part := range q.Parts { - resolvedMod := ModuleOfList( - filter.Disc, - part.Coefficient, - symbolic.NewVariable(part.Selector), - ) + exprList := []*symbolic.Expression{} + for k := range part.Selectors { + exprList = append(exprList, part.Coefficients[k]) + exprList = append(exprList, symbolic.NewVariable(part.Selectors[k])) + } + resolvedMod := ModuleOfList(filter.Disc, exprList...) resolvedMod.MustBeResolved() - if resolvedMod != filter.Module { continue } diff --git a/prover/protocol/distributed/module_gl.go b/prover/protocol/distributed/module_gl.go index f526f51bd30..8c96c509c58 100644 --- a/prover/protocol/distributed/module_gl.go +++ b/prover/protocol/distributed/module_gl.go @@ -193,12 +193,12 @@ func NewModuleGL(builder *wizard.Builder, moduleInput *FilteredModuleInputs) *Mo } for _, rangeCs := range moduleInput.Range { - newCol := moduleGL.TranslateColumn(rangeCs.Handle, 0) + newCol := moduleGL.TranslateColumn(rangeCs.Handle) moduleGL.Wiop.InsertRange(1, rangeCs.ID, newCol, rangeCs.B) } for _, localOpening := range moduleInput.LocalOpenings { - newCol := moduleGL.TranslateColumn(localOpening.Pol, 0) + newCol := moduleGL.TranslateColumn(localOpening.Pol) moduleGL.Wiop.InsertLocalOpening(1, localOpening.ID, newCol) } diff --git a/prover/protocol/distributed/module_lpp.go b/prover/protocol/distributed/module_lpp.go index 0ee6a4b30e4..035c09ed951 100644 --- a/prover/protocol/distributed/module_lpp.go +++ b/prover/protocol/distributed/module_lpp.go @@ -356,26 +356,29 @@ func (a LppWitnessAssignment) Run(run *wizard.ProverRuntime) { // addCoinFromExpression scans the metadata of the expression looking // for coins and adds them to the [ModuleLPP] as [coin.FieldFromSeed]. -func (m *ModuleLPP) addCoinFromExpression(expr *symbolic.Expression) { +func (m *ModuleLPP) addCoinFromExpression(exprs ...*symbolic.Expression) { - var ( - board = expr.Board() - metadata = board.ListVariableMetadata() - ) + for _, expr := range exprs { - for i := range metadata { + var ( + board = expr.Board() + metadata = board.ListVariableMetadata() + ) - switch meta := metadata[i].(type) { + for i := range metadata { - case coin.Info: + switch meta := metadata[i].(type) { - m.InsertCoin(meta.Name, meta.Round) - return + case coin.Info: - case ifaces.Accessor: + m.InsertCoin(meta.Name, meta.Round) + continue - m.addCoinFromAccessor(meta) - return + case ifaces.Accessor: + + m.addCoinFromAccessor(meta) + continue + } } } } diff --git a/prover/protocol/distributed/module_translation.go b/prover/protocol/distributed/module_translation.go index d5a63d28f51..0b0e43c70c8 100644 --- a/prover/protocol/distributed/module_translation.go +++ b/prover/protocol/distributed/module_translation.go @@ -75,18 +75,18 @@ func (mt *moduleTranslator) InsertPrecomputed(col column.Natural, data smartvect // // The sizeHint argument is meant to deduce what the size of a translated // [verifiercol.ConstCol] -func (mt *moduleTranslator) TranslateColumn(col ifaces.Column, sizeHint int) ifaces.Column { +func (mt *moduleTranslator) TranslateColumn(col ifaces.Column) ifaces.Column { switch c := col.(type) { case column.Natural: return mt.Wiop.Columns.GetHandle(c.ID) case column.Shifted: return column.Shifted{ - Parent: mt.TranslateColumn(c.Parent, sizeHint), + Parent: mt.TranslateColumn(c.Parent), Offset: c.Offset, } case verifiercol.ConstCol: - return verifiercol.NewConstantCol(c.F, sizeHint) + return verifiercol.NewConstantCol(c.F, c.Size()) default: utils.Panic("unexpected type of column: type: %T, name: %v", col, col.GetColID()) } @@ -94,6 +94,17 @@ func (mt *moduleTranslator) TranslateColumn(col ifaces.Column, sizeHint int) ifa return nil } +// TranslateColumnList returns a list of equivalent columns from the new module. +// The function panics if the column cannot be resolved. It will happen if the +// column has an expected type or is defined from not resolvable items. +func (mt *moduleTranslator) TranslateColumnList(cols []ifaces.Column) []ifaces.Column { + res := make([]ifaces.Column, len(cols)) + for i := range res { + res[i] = mt.TranslateColumn(cols[i]) + } + return res +} + // TranslateExpression returns an expression corresponding to the provided // expression but in term of the input module. When the function encounters // a [verifiercol.Constcol] as part of the expression, it converts it into @@ -101,8 +112,6 @@ func (mt *moduleTranslator) TranslateColumn(col ifaces.Column, sizeHint int) ifa // process and is strictly equivalent. func (mt *moduleTranslator) TranslateExpression(expr *symbolic.Expression) *symbolic.Expression { - sizeHint := NewSizeOfExpr(mt.Disc, expr) - return expr.ReconstructBottomUpSingleThreaded( func(e *symbolic.Expression, children []*symbolic.Expression) *symbolic.Expression { switch op := e.Operator.(type) { @@ -121,7 +130,7 @@ func (mt *moduleTranslator) TranslateExpression(expr *symbolic.Expression) *symb if constcol, isconst := root.(verifiercol.ConstCol); isconst { return symbolic.NewConstant(constcol.F) } - newCol := mt.TranslateColumn(m, sizeHint) + newCol := mt.TranslateColumn(m) return symbolic.NewVariable(newCol) case coin.Info: newCoin := mt.TranslateCoin(m) @@ -136,6 +145,16 @@ func (mt *moduleTranslator) TranslateExpression(expr *symbolic.Expression) *symb ) } +// TranslateExpressionList returns a list of equivalent expressions from the new +// module. +func (mt *moduleTranslator) TranslateExpressionList(exprs []*symbolic.Expression) []*symbolic.Expression { + res := make([]*symbolic.Expression, len(exprs)) + for i := range res { + res[i] = mt.TranslateExpression(exprs[i]) + } + return res +} + // TranslateCoin returns the equivalent coin from the new module. // The function looks for a coin with the same name and inserts it // as a [coin.FieldFromSeed] if it is not found. @@ -165,7 +184,7 @@ func (mt *moduleTranslator) TranslateAccessor(acc ifaces.Accessor) ifaces.Access return accessors.NewFromCoin(newCoin) case *accessors.FromPublicColumn: - newCol := mt.TranslateColumn(a.Col, 1) + newCol := mt.TranslateColumn(a.Col) return accessors.NewFromPublicColumn(newCol, a.Pos) case *accessors.FromLocalOpeningYAccessor: @@ -191,8 +210,8 @@ func (mt *moduleTranslator) InsertPlonkInWizard(oldQuery *query.PlonkInWizard) * newQuery := &query.PlonkInWizard{ ID: oldQuery.ID, - Data: mt.TranslateColumn(oldQuery.Data, 0), - Selector: mt.TranslateColumn(oldQuery.Selector, 0), + Data: mt.TranslateColumn(oldQuery.Data), + Selector: mt.TranslateColumn(oldQuery.Selector), Circuit: oldQuery.Circuit, PlonkOptions: oldQuery.PlonkOptions, } @@ -295,13 +314,13 @@ func (mt *ModuleLPP) InsertHorner( for _, oldPart := range parts { newPart := query.HornerPart{ - Coefficient: mt.TranslateExpression(oldPart.Coefficient), + Coefficients: mt.TranslateExpressionList(oldPart.Coefficients), SignNegative: oldPart.SignNegative, - Selector: mt.TranslateColumn(oldPart.Selector, 0), + Selectors: mt.TranslateColumnList(oldPart.Selectors), X: mt.TranslateAccessor(oldPart.X), } - mt.addCoinFromExpression(newPart.Coefficient) + mt.addCoinFromExpression(newPart.Coefficients...) mt.addCoinFromAccessor(newPart.X) res.Parts = append(res.Parts, newPart) diff --git a/prover/protocol/distributed/module_witness.go b/prover/protocol/distributed/module_witness.go index 243d0aa9af7..c5a7fec2f40 100644 --- a/prover/protocol/distributed/module_witness.go +++ b/prover/protocol/distributed/module_witness.go @@ -267,37 +267,40 @@ func (mw *ModuleWitnessLPP) NextN0s(moduleLPP *ModuleLPP) []int { for i := range newN0s { - // Note: the selector might be a non-natural column. Possibly a const-col. - selCol := args[i].Selector + for k := range args[i].Selectors { - if constCol, isConstCol := selCol.(verifiercol.ConstCol); isConstCol { + // Note: the selector might be a non-natural column. Possibly a const-col. + selCol := args[i].Selectors[k] - if constCol.F.IsZero() { - continue - } + if constCol, isConstCol := selCol.(verifiercol.ConstCol); isConstCol { - if constCol.F.IsOne() { - newN0s[i] += constCol.Size() - continue - } + if constCol.F.IsZero() { + continue + } - utils.Panic("the selector column has non-zero values: %v", constCol.F.String()) - } + if constCol.F.IsOne() { + newN0s[i] += constCol.Size() + continue + } - // Expectedly, at this point. The column must be a natural column. We can't support - // shifted selector columns. - _ = selCol.(column.Natural) + utils.Panic("the selector column has non-zero values: %v", constCol.F.String()) + } - selSV, ok := mw.Columns[selCol.GetColID()] - if !ok { - utils.Panic("selector: %v is missing from witness columns for module: %v index: %v", selCol, mw.ModuleNames, mw.ModuleIndex) - } + // Expectedly, at this point. The column must be a natural column. We can't support + // shifted selector columns. + _ = selCol.(column.Natural) + + selSV, ok := mw.Columns[selCol.GetColID()] + if !ok { + utils.Panic("selector: %v is missing from witness columns for module: %v index: %v", selCol, mw.ModuleNames, mw.ModuleIndex) + } - sel := selSV.IntoRegVecSaveAlloc() + sel := selSV.IntoRegVecSaveAlloc() - for j := range sel { - if sel[j].IsOne() { - newN0s[i]++ + for j := range sel[k] { + if sel[j].IsOne() { + newN0s[i]++ + } } } } From 0a7631f59bb02733c09baf498ef61871d8f5efcc Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Wed, 14 May 2025 10:03:58 +0200 Subject: [PATCH 13/18] chores(golangci): migrate golangci-lint --- prover/.golangci.yml | 68 +++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/prover/.golangci.yml b/prover/.golangci.yml index d04c530c913..7839ee3501d 100644 --- a/prover/.golangci.yml +++ b/prover/.golangci.yml @@ -1,37 +1,41 @@ +version: "2" +run: + issues-exit-code: 1 linters: - disable-all: true - # @alex: we will need to sort out the linters because they take too much time - # and memory to run practically since we merged go-corset. We shall revise - # our toolset. + default: none enable: - # - gofmt - # - staticcheck - gosec - # - gosimple - # - govet - ineffassign - prealloc - -run: - issues-exit-code: 1 - # List of build tags, all linters use it. - # Default: []. - # build-tags: - -issues: - exclude-dirs: - - compressor - - zkevm/arithmetization - exclude: - # Only appears on CI - - '.*printf: non-constant format string in call to.*' - -linters-settings: - staticcheck: - checks: - - all - - '-SA1019' # disable the rule against deprecated code - - '-SA1006' - gosec: - excludes: - - G115 # Conversions from int -> uint etc + settings: + gosec: + excludes: + - G115 + staticcheck: + checks: + - -SA1006 + - -SA1019 + - all + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + - path: (.+)\.go$ + text: '.*printf: non-constant format string in call to.*' + paths: + - compressor + - zkevm/arithmetization + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ From bf4bcc3c90e6521531116ebd28a177c9593e3223 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Wed, 14 May 2025 10:05:54 +0200 Subject: [PATCH 14/18] fix(lint): fix the missing linting rules --- prover/protocol/compiler/vortex/compiler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/prover/protocol/compiler/vortex/compiler.go b/prover/protocol/compiler/vortex/compiler.go index 5a3ebc11fc1..188dbcc96e6 100644 --- a/prover/protocol/compiler/vortex/compiler.go +++ b/prover/protocol/compiler/vortex/compiler.go @@ -898,7 +898,9 @@ func (ctx *Ctx) startingRound() int { // in the same order of appearance as in the query. The function ignores // the precomputed columns. func (ctx *Ctx) commitmentsAtRoundFromQuery(round int) []ifaces.ColID { - var res []ifaces.ColID + + res := make([]ifaces.ColID, 0, len(ctx.Query.Pols)) + for _, p := range ctx.Query.Pols { if p.Round() != round { @@ -927,7 +929,7 @@ func (ctx *Ctx) commitmentsAtRoundFromQuery(round int) []ifaces.ColID { // the precomputed columns. func (ctx *Ctx) commitmentsAtRoundFromQueryPrecomputed() []ifaces.ColID { - var res []ifaces.ColID + res := make([]ifaces.ColID, 0, len(ctx.Query.Pols)) for _, p := range ctx.Query.Pols { From 2c7d4e8fcfbcf2cdbe94b6e44d4b26773db40047 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Wed, 14 May 2025 10:11:32 +0200 Subject: [PATCH 15/18] ci(lint): update the version of golangci-lint --- .github/workflows/prover-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prover-testing.yml b/.github/workflows/prover-testing.yml index 9d1d6003fde..bd07ce76feb 100644 --- a/.github/workflows/prover-testing.yml +++ b/.github/workflows/prover-testing.yml @@ -39,7 +39,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.61.0 + version: v2.0.2 working-directory: prover args: --timeout=5m - name: generated files should not be modified From abb08ef1d3515ec6c6060c95d9a72b42140d7647 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Wed, 14 May 2025 10:23:37 +0200 Subject: [PATCH 16/18] fixup(ci): updates the golangci-lint runner version from v6 to v7 --- .github/workflows/prover-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prover-testing.yml b/.github/workflows/prover-testing.yml index bd07ce76feb..5c470c3e213 100644 --- a/.github/workflows/prover-testing.yml +++ b/.github/workflows/prover-testing.yml @@ -37,7 +37,7 @@ jobs: working-directory: prover run: if [[ -n $(gofmt -l .) ]]; then echo "please run gofmt"; exit 1; fi - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v7 with: version: v2.0.2 working-directory: prover From 40a6d6c3a9ab869366136e96e84bc1cf94c06d22 Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 16 May 2025 08:01:19 +0200 Subject: [PATCH 17/18] Update prover/protocol/query/projection.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: AlexandreBelling --- prover/protocol/query/projection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/query/projection.go b/prover/protocol/query/projection.go index 92bfae68a9e..c470eb8bcd5 100644 --- a/prover/protocol/query/projection.go +++ b/prover/protocol/query/projection.go @@ -88,7 +88,7 @@ func NewProjectionMultiAry( for i := range inp.ColumnsA { if len(inp.ColumnsA[i]) != numCols { - utils.Panic("All table must have the same number of columns: len(A[%v])=%v, len(A[0])=%v", i, len(inp.ColumnsA), numCols) + utils.Panic("All table must have the same number of columns: len(A[%v])=%v, len(A[0])=%v", i, len(inp.ColumnsA[i]), numCols) } size := ifaces.AssertSameLength(inp.ColumnsA[i]...) From 591406d31ca0b33a5a7b1463002901e1f202a18c Mon Sep 17 00:00:00 2001 From: AlexandreBelling Date: Fri, 16 May 2025 08:01:27 +0200 Subject: [PATCH 18/18] Update prover/protocol/query/projection.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: AlexandreBelling --- prover/protocol/query/projection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/protocol/query/projection.go b/prover/protocol/query/projection.go index c470eb8bcd5..43e33984247 100644 --- a/prover/protocol/query/projection.go +++ b/prover/protocol/query/projection.go @@ -108,7 +108,7 @@ func NewProjectionMultiAry( for i := range inp.ColumnsB { if len(inp.ColumnsB[i]) != numCols { - utils.Panic("All table must have the same number of columns: len(B[%v])=%v, len(A[0])=%v", i, len(inp.ColumnsB), numCols) + utils.Panic("All table must have the same number of columns: len(B[%v])=%v, numCols=%v", i, len(inp.ColumnsB[i]), numCols) } size := ifaces.AssertSameLength(inp.ColumnsB[i]...)