Skip to content

refactor prove/setup args #288

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Tabaie merged 7 commits into from
Nov 19, 2024
Merged

refactor prove/setup args #288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e66f097
refactor prove/setup args
Tabaie Nov 13, 2024
8853fa7
refac contain global variables in the main package
Tabaie Nov 15, 2024
12a7dbd
Merge branch 'main' into refactor/cmd-args
Tabaie Nov 19, 2024
87e3e3b
Merge branch 'main' into refactor/cmd-args
Tabaie Nov 19, 2024
566b23e
refactor eliminate unnecessary arguments
Tabaie Nov 19, 2024
a6df8d8
Merge branch 'main' into refactor/cmd-args
Tabaie Nov 19, 2024
b7d4e47
Merge branch 'main' into refactor/cmd-args
Tabaie Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 23 additions & 38 deletions prover/cmd/prover/cmd/prove.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,90 +12,75 @@ import (
"github.com/consensys/linea-monorepo/prover/backend/execution"
"github.com/consensys/linea-monorepo/prover/backend/files"
"github.com/consensys/linea-monorepo/prover/config"
"github.com/spf13/cobra"
)

var (
fInput string
fOutput string
fLarge bool
)

// proveCmd represents the prove command
var proveCmd = &cobra.Command{
Use: "prove",
Short: "prove process a request, creates a proof with the adequate circuit and writes the proof to a file",
RunE: cmdProve,
}

func init() {
rootCmd.AddCommand(proveCmd)

proveCmd.Flags().StringVar(&fInput, "in", "", "input file")
proveCmd.Flags().StringVar(&fOutput, "out", "", "output file")
proveCmd.Flags().BoolVar(&fLarge, "large", false, "run the large execution circuit")

type ProverArgs struct {
Input string
Output string
Large bool
ConfigFile string
}

func cmdProve(cmd *cobra.Command, args []string) error {
func Prove(args ProverArgs) error {
const cmdName = "prove"
// TODO @gbotrel with a specific flag, we could compile the circuit and compare with the checksum of the
// asset we deserialize, to make sure we are using the circuit associated with the compiled binary and the setup.

// read config
cfg, err := config.NewConfigFromFile(fConfigFile)
cfg, err := config.NewConfigFromFile(args.ConfigFile)
if err != nil {
return fmt.Errorf("%s failed to read config file: %w", cmd.Name(), err)
return fmt.Errorf("%s failed to read config file: %w", cmdName, err)
}

// discover the type of the job from the input file name
jobExecution := strings.Contains(fInput, "getZkProof")
jobBlobDecompression := strings.Contains(fInput, "getZkBlobCompressionProof")
jobAggregation := strings.Contains(fInput, "getZkAggregatedProof")
jobExecution := strings.Contains(args.Input, "getZkProof")
jobBlobDecompression := strings.Contains(args.Input, "getZkBlobCompressionProof")
jobAggregation := strings.Contains(args.Input, "getZkAggregatedProof")

if jobExecution {
req := &execution.Request{}
if err := readRequest(fInput, req); err != nil {
return fmt.Errorf("could not read the input file (%v): %w", fInput, err)
if err := readRequest(args.Input, req); err != nil {
return fmt.Errorf("could not read the input file (%v): %w", args.Input, err)
}
// we use the large traces in 2 cases;
// 1. the user explicitly asked for it (fLarge)
// 1. the user explicitly asked for it (args.Large)
// 2. the job contains the large suffix and we are a large machine (cfg.Execution.CanRunLarge)
large := fLarge || (strings.Contains(fInput, "large") && cfg.Execution.CanRunFullLarge)
large := args.Large || (strings.Contains(args.Input, "large") && cfg.Execution.CanRunFullLarge)

resp, err := execution.Prove(cfg, req, large)
if err != nil {
return fmt.Errorf("could not prove the execution: %w", err)
}

return writeResponse(fOutput, resp)
return writeResponse(args.Output, resp)
}

if jobBlobDecompression {
req := &blobdecompression.Request{}
if err := readRequest(fInput, req); err != nil {
return fmt.Errorf("could not read the input file (%v): %w", fInput, err)
if err := readRequest(args.Input, req); err != nil {
return fmt.Errorf("could not read the input file (%v): %w", args.Input, err)
}

resp, err := blobdecompression.Prove(cfg, req)
if err != nil {
return fmt.Errorf("could not prove the blob decompression: %w", err)
}

return writeResponse(fOutput, resp)
return writeResponse(args.Output, resp)
}

if jobAggregation {
req := &aggregation.Request{}
if err := readRequest(fInput, req); err != nil {
return fmt.Errorf("could not read the input file (%v): %w", fInput, err)
if err := readRequest(args.Input, req); err != nil {
return fmt.Errorf("could not read the input file (%v): %w", args.Input, err)
}

resp, err := aggregation.Prove(cfg, req)
if err != nil {
return fmt.Errorf("could not prove the aggregation: %w", err)
}

return writeResponse(fOutput, resp)
return writeResponse(args.Output, resp)
}

return errors.New("unknown job type")
Expand Down
36 changes: 0 additions & 36 deletions prover/cmd/prover/cmd/root.go
View file
Open in desktop

This file was deleted.

89 changes: 36 additions & 53 deletions prover/cmd/prover/cmd/setup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/sirupsen/logrus"

"github.com/consensys/gnark-crypto/ecc"
"github.com/consensys/gnark/backend/plonk"
"github.com/consensys/linea-monorepo/prover/circuits"
"github.com/consensys/linea-monorepo/prover/circuits/aggregation"
v0 "github.com/consensys/linea-monorepo/prover/circuits/blobdecompression/v0"
Expand All @@ -26,27 +27,17 @@ import (
"github.com/consensys/linea-monorepo/prover/config"
"github.com/consensys/linea-monorepo/prover/utils"
"github.com/consensys/linea-monorepo/prover/zkevm"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/consensys/gnark/backend/plonk"
)

var (
fForce bool
fCircuits string
fDictPath string
fAssetsDir string
)

// setupCmd represents the setup command
var setupCmd = &cobra.Command{
Use: "setup",
Short: "pre compute assets for Linea circuits",
RunE: cmdSetup,
type SetupArgs struct {
Force bool
Circuits string
DictPath string
AssetsDir string
ConfigFile string
}

var allCircuits = []string{
var AllCircuits = []string{
string(circuits.ExecutionCircuitID),
string(circuits.ExecutionLargeCircuitID),
string(circuits.BlobDecompressionV0CircuitID),
Expand All @@ -57,39 +48,30 @@ var allCircuits = []string{
string(circuits.EmulationDummyCircuitID), // we want to generate Verifier.sol for this one
}

func init() {
rootCmd.AddCommand(setupCmd)
setupCmd.Flags().BoolVar(&fForce, "force", false, "overwrites existing files")
setupCmd.Flags().StringVar(&fCircuits, "circuits", strings.Join(allCircuits, ","), "comma separated list of circuits to setup")
setupCmd.Flags().StringVar(&fDictPath, "dict", "", "path to the dictionary file used in blob (de)compression")
setupCmd.Flags().StringVar(&fAssetsDir, "assets-dir", "", "path to the directory where the assets are stored (override conf)")

viper.BindPFlag("assets_dir", setupCmd.Flags().Lookup("assets-dir"))
}

func cmdSetup(cmd *cobra.Command, args []string) error {
func Setup(context context.Context, args SetupArgs) error {
const cmdName = "setup"
// read config
cfg, err := config.NewConfigFromFile(fConfigFile)
cfg, err := config.NewConfigFromFile(args.ConfigFile)
if err != nil {
return fmt.Errorf("%s failed to read config file: %w", cmd.Name(), err)
return fmt.Errorf("%s failed to read config file: %w", cmdName, err)
}

if fDictPath != "" {
if args.DictPath != "" {
// fail early if the dictionary file is not found but was specified.
if _, err := os.Stat(fDictPath); err != nil {
return fmt.Errorf("%s dictionary file not found: %w", cmd.Name(), err)
if _, err := os.Stat(args.DictPath); err != nil {
return fmt.Errorf("%s dictionary file not found: %w", cmdName, err)
}
}

// parse inCircuits
inCircuits := make(map[circuits.CircuitID]bool)
for _, c := range allCircuits {
for _, c := range AllCircuits {
inCircuits[circuits.CircuitID(c)] = false
}
_inCircuits := strings.Split(fCircuits, ",")
_inCircuits := strings.Split(args.Circuits, ",")
for _, c := range _inCircuits {
if _, ok := inCircuits[circuits.CircuitID(c)]; !ok {
return fmt.Errorf("%s unknown circuit: %s", cmd.Name(), c)
return fmt.Errorf("%s unknown circuit: %s", cmdName, c)
}
inCircuits[circuits.CircuitID(c)] = true
}
Expand All @@ -101,7 +83,7 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
var srsProvider circuits.SRSProvider
srsProvider, err = circuits.NewSRSStore(cfg.PathForSRS())
if err != nil {
return fmt.Errorf("%s failed to create SRS provider: %w", cmd.Name(), err)
return fmt.Errorf("%s failed to create SRS provider: %w", cmdName, err)
}

// for each circuit, we start by compiling the circuit
Expand All @@ -128,9 +110,9 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
zkEvm := zkevm.FullZkEvm(&limits)
builder = execution.NewBuilder(zkEvm)
case circuits.BlobDecompressionV0CircuitID, circuits.BlobDecompressionV1CircuitID:
dict, err = os.ReadFile(fDictPath)
dict, err = os.ReadFile(args.DictPath)
if err != nil {
return fmt.Errorf("%s failed to read dictionary file: %w", cmd.Name(), err)
return fmt.Errorf("%s failed to read dictionary file: %w", cmdName, err)
}

if c == circuits.BlobDecompressionV0CircuitID {
Expand All @@ -151,14 +133,14 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
continue // dummy, aggregation, emulation or public input circuits are handled later
}

if err := updateSetup(cmd.Context(), cfg, srsProvider, c, builder, extraFlags); err != nil {
if err := updateSetup(context, cfg, args.Force, srsProvider, c, builder, extraFlags); err != nil {
return err
}
if dict != nil {
// we save the dictionary to disk
dictPath := filepath.Join(cfg.PathForSetup(string(c)), config.DictionaryFileName)
if err := os.WriteFile(dictPath, dict, 0600); err != nil {
return fmt.Errorf("%s failed to write dictionary file: %w", cmd.Name(), err)
return fmt.Errorf("%s failed to write dictionary file: %w", cmdName, err)
}
}

Expand All @@ -172,7 +154,7 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
// get verifying key for public-input circuit
piSetup, err := circuits.LoadSetup(cfg, circuits.PublicInputInterconnectionCircuitID)
if err != nil {
return fmt.Errorf("%s failed to load public input interconnection setup: %w", cmd.Name(), err)
return fmt.Errorf("%s failed to load public input interconnection setup: %w", cmdName, err)
}

// first, we need to collect the verifying keys
Expand All @@ -196,7 +178,7 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
return fmt.Errorf("unknown dummy circuit: %s", allowedInput)
}

vk, err := getDummyCircuitVK(cmd.Context(), cfg, srsProvider, circuits.CircuitID(allowedInput), dummy.NewBuilder(mockID, curveID.ScalarField()))
vk, err := getDummyCircuitVK(context, cfg, srsProvider, circuits.CircuitID(allowedInput), dummy.NewBuilder(mockID, curveID.ScalarField()))
if err != nil {
return err
}
Expand All @@ -209,15 +191,15 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
vkPath := filepath.Join(setupPath, config.VerifyingKeyFileName)
vk := plonk.NewVerifyingKey(ecc.BLS12_377)
if err := circuits.ReadVerifyingKey(vkPath, vk); err != nil {
return fmt.Errorf("%s failed to read verifying key for circuit %s: %w", cmd.Name(), allowedInput, err)
return fmt.Errorf("%s failed to read verifying key for circuit %s: %w", cmdName, allowedInput, err)
}

allowedVkForAggregation = append(allowedVkForAggregation, vk)
}

// we need to compute the digest of the verifying keys & store them in the manifest
// for the aggregation circuits to be able to check compatibility at run time with the proofs
allowedVkForAggregationDigests := listOfCheckum(allowedVkForAggregation)
allowedVkForAggregationDigests := listOfChecksums(allowedVkForAggregation)
extraFlagsForAggregationCircuit := map[string]any{
"allowedVkForAggregationDigests": allowedVkForAggregationDigests,
}
Expand All @@ -229,7 +211,7 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
logrus.Infof("setting up %s (numProofs=%d)", c, numProofs)

builder := aggregation.NewBuilder(numProofs, cfg.Aggregation.AllowedInputs, piSetup, allowedVkForAggregation)
if err := updateSetup(cmd.Context(), cfg, srsProvider, c, builder, extraFlagsForAggregationCircuit); err != nil {
if err := updateSetup(context, cfg, args.Force, srsProvider, c, builder, extraFlagsForAggregationCircuit); err != nil {
return err
}

Expand All @@ -238,7 +220,7 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
vkPath := filepath.Join(setupPath, config.VerifyingKeyFileName)
vk := plonk.NewVerifyingKey(ecc.BW6_761)
if err := circuits.ReadVerifyingKey(vkPath, vk); err != nil {
return fmt.Errorf("%s failed to read verifying key for circuit %s: %w", cmd.Name(), c, err)
return fmt.Errorf("%s failed to read verifying key for circuit %s: %w", cmdName, c, err)
}

allowedVkForEmulation = append(allowedVkForEmulation, vk)
Expand All @@ -248,7 +230,7 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
c := circuits.EmulationCircuitID
logrus.Infof("setting up %s", c)
builder := emulation.NewBuilder(allowedVkForEmulation)
return updateSetup(cmd.Context(), cfg, srsProvider, c, builder, nil)
return updateSetup(context, cfg, args.Force, srsProvider, c, builder, nil)

}

Expand Down Expand Up @@ -281,7 +263,7 @@ func getDummyCircuitVK(ctx context.Context, cfg *config.Config, srsProvider circ
// and if so, if the checksums match.
// if the files already exist and the checksums match, it skips the setup.
// else it does the setup and writes the assets to disk.
func updateSetup(ctx context.Context, cfg *config.Config, srsProvider circuits.SRSProvider, circuit circuits.CircuitID, builder circuits.Builder, extraFlags map[string]any) error {
func updateSetup(ctx context.Context, cfg *config.Config, force bool, srsProvider circuits.SRSProvider, circuit circuits.CircuitID, builder circuits.Builder, extraFlags map[string]any) error {
if extraFlags == nil {
extraFlags = make(map[string]any)
}
Expand All @@ -297,7 +279,7 @@ func updateSetup(ctx context.Context, cfg *config.Config, srsProvider circuits.S
setupPath := cfg.PathForSetup(string(circuit))
manifestPath := filepath.Join(setupPath, config.ManifestFileName)

if !fForce {
if !force {
// we may want to skip setup if the files already exist
// and the checksums match
// read manifest if already exists
Expand Down Expand Up @@ -325,12 +307,13 @@ func updateSetup(ctx context.Context, cfg *config.Config, srsProvider circuits.S
return setup.WriteTo(setupPath)
}

// listOfCheckum Computes a list of SHA256 checksums for a list of assets, the result is given
// listOfChecksums Computes a list of SHA256 checksums for a list of assets, the result is given
// in hexstring.
func listOfCheckum[T io.WriterTo](assets []T) []string {
func listOfChecksums[T io.WriterTo](assets []T) []string {
res := make([]string, len(assets))
h := sha256.New()
for i := range assets {
h := sha256.New()
h.Reset()
_, err := assets[i].WriteTo(h)
if err != nil {
// It is unexpected that writing in a hasher could possibly fail.
Expand Down
Loading