diff --git a/prover/backend/execution/prove.go b/prover/backend/execution/prove.go index 89192ef5259..4970e344fbf 100644 --- a/prover/backend/execution/prove.go +++ b/prover/backend/execution/prove.go @@ -1,11 +1,19 @@ package execution import ( + "fmt" + "math/rand/v2" + "os" + "runtime" + "strconv" + "time" + "github.com/consensys/gnark-crypto/ecc" "github.com/consensys/linea-monorepo/prover/circuits" "github.com/consensys/linea-monorepo/prover/circuits/dummy" "github.com/consensys/linea-monorepo/prover/circuits/execution" "github.com/consensys/linea-monorepo/prover/config" + "github.com/consensys/linea-monorepo/prover/protocol/serialization" public_input "github.com/consensys/linea-monorepo/prover/public-input" "github.com/consensys/linea-monorepo/prover/utils" "github.com/consensys/linea-monorepo/prover/utils/profiling" @@ -179,6 +187,31 @@ func mustProveAndPass( logrus.Infof("Prover checks passed") return "", "" + case config.ProverModeEncodeOnly: + + profiling.ProfileTrace("encode-decode-no-circuit", true, false, func() { + filepath := "/tmp/wizard-assignment/blob-" + strconv.Itoa(rand.Int()) + ".bin" + + encodeOnlyZkEvm := zkevm.EncodeOnlyZkEvm(traces) + numChunks := runtime.GOMAXPROCS(0) + + // Serialize the assignment + encodingDuration := time.Now() + encodeOnlyZkEvm.AssignAndEncodeInChunks(filepath, w.ZkEVM, numChunks) + + // Deserialize the assignment + decodingDuration := time.Now() + _, errDec := serialization.DeserializeAssignment(filepath, numChunks) + if errDec != nil { + panic(fmt.Sprintf("Error during deserialization: %v", errDec)) + } + fmt.Printf("[Encoding Summary] took %v sec to encode an assignmente and write it into the files \n", time.Since(encodingDuration).Seconds()) + fmt.Printf("[Decoding Summary] took %v sec to read the files and decode it into an assignment\n", time.Since(decodingDuration).Seconds()) + }) + + os.Exit(0) + return "", "" + default: panic("not implemented") } diff --git a/prover/config/config.go b/prover/config/config.go index 2a0ee91e350..d13f56b117e 100644 --- a/prover/config/config.go +++ b/prover/config/config.go @@ -203,7 +203,7 @@ type Execution struct { WithRequestDir `mapstructure:",squash"` // ProverMode stores the kind of prover to use. - ProverMode ProverMode `mapstructure:"prover_mode" validate:"required,oneof=dev partial full proofless bench check-only"` + ProverMode ProverMode `mapstructure:"prover_mode" validate:"required,oneof=dev partial full proofless bench check-only encode-only"` // CanRunFullLarge indicates whether the prover is running on a large machine (and can run full large traces). CanRunFullLarge bool `mapstructure:"can_run_full_large"` diff --git a/prover/config/types.go b/prover/config/types.go index 83cfa02a264..56c480fad31 100644 --- a/prover/config/types.go +++ b/prover/config/types.go @@ -26,5 +26,6 @@ const ( // in a context where it is simpler to not have to deal with the setup. ProverModeBench ProverMode = "bench" // ProverModeCheckOnly is used to test the constraints of the whole system - ProverModeCheckOnly ProverMode = "check-only" + ProverModeCheckOnly ProverMode = "check-only" + ProverModeEncodeOnly ProverMode = "encode-only" ) diff --git a/prover/go.mod b/prover/go.mod index 0c8f23232eb..c14ce840867 100644 --- a/prover/go.mod +++ b/prover/go.mod @@ -76,6 +76,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/prover/go.sum b/prover/go.sum index 26ae67ec025..f4e8d6435ac 100644 --- a/prover/go.sum +++ b/prover/go.sum @@ -369,6 +369,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/prover/maths/common/smartvectors/windowed.go b/prover/maths/common/smartvectors/windowed.go index 44fb103e30d..6bf4ffa6b07 100644 --- a/prover/maths/common/smartvectors/windowed.go +++ b/prover/maths/common/smartvectors/windowed.go @@ -47,6 +47,21 @@ func (p *PaddedCircularWindow) Len() int { return p.totLen } +// Offset returns the offset of the PCW +func (p *PaddedCircularWindow) Offset() int { + return p.offset +} + +// Windows returns the length of the window of the PCQ +func (p *PaddedCircularWindow) Window() []field.Element { + return p.window +} + +// PaddingVal returns the value used for padding the window +func (p *PaddedCircularWindow) PaddingVal() field.Element { + return p.paddingVal +} + // Returns a queries position func (p *PaddedCircularWindow) GetBase(n int) (field.Element, error) { // Check if the queried index is in the window diff --git a/prover/protocol/serialization/column_assignment.go b/prover/protocol/serialization/column_assignment.go new file mode 100644 index 00000000000..fa49ba672fd --- /dev/null +++ b/prover/protocol/serialization/column_assignment.go @@ -0,0 +1,380 @@ +package serialization + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "math/big" + "sync" + "unsafe" + + "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/ifaces" + "github.com/consensys/linea-monorepo/prover/utils/collection" + "github.com/consensys/linea-monorepo/prover/utils/parallel" + "github.com/pierrec/lz4/v4" + "github.com/sirupsen/logrus" +) + +// WAssignment is an alias for the mapping type used to represent the assignment +// of a column in a [wizard.ProverRuntime] +type WAssignment = collection.Mapping[ifaces.ColID, smartvectors.SmartVector] + +// hashWAssignment computes a hash of the serialized WAssignment structure. +func hashWAssignment(a WAssignment) string { + serialized, err := json.Marshal(a) + if err != nil { + panic("Failed to serialize WAssignment for hashing: " + err.Error()) + } + h := sha256.New() + h.Write(serialized) + return hex.EncodeToString(h.Sum(nil)) +} + +// SerializeAssignment serializes map representing the column assignment of a +// wizard protocol. +func SerializeAssignment(a WAssignment, numChunks int) []json.RawMessage { + logrus.Infof("Hash of WAssignment before serialization: %s", hashWAssignment(a)) + + var ( + as = a.InnerMap() + ser = map[string]*CompressedSmartVector{} + names = a.ListAllKeys() + lock = &sync.Mutex{} + ) + + parallel.ExecuteChunky(len(names), func(start, stop int) { + for i := start; i < stop; i++ { + v := CompressSmartVector(as[names[i]]) + lock.Lock() + // Convert names[i] to string using fmt.Sprintf or another method + ser[fmt.Sprintf("%v", names[i])] = v + lock.Unlock() + } + }) + + // Calculate the size of `ser` in bytes + var serSizeBytes uintptr + for _, v := range ser { + serSizeBytes += unsafe.Sizeof(*v) + } + + // Convert size to GB + serSizeGB := float64(serSizeBytes) / (1024 * 1024 * 1024) + logrus.Infof("Size of ser : %.6f GB", serSizeGB) + + // Parallelize CBOR serialization by chunking `ser` + chunkSize := (len(ser) + numChunks - 1) / numChunks // Calculate the size of each chunk + var serializedChunks = make([]json.RawMessage, numChunks) + var wg sync.WaitGroup + var m sync.Mutex + for i := 0; i < numChunks; i++ { + wg.Add(1) + go func(chunkIndex int) { + defer wg.Done() + + // Select chunk of data for this goroutine + start := chunkIndex * chunkSize + stop := start + chunkSize + if stop > len(names) { + stop = len(names) + } + + // Prepare chunk map for serialization + chunk := make(map[string]*CompressedSmartVector) + for j := start; j < stop; j++ { + // Convert names[j] to string + key := fmt.Sprintf("%v", names[j]) + chunk[key] = ser[key] + } + + // Serialize the chunk with CBOR + serializedChunk := serializeAnyWithCborPkg(chunk) + logrus.Infof("Serialized chunk %d, size: %d bytes", i, len(serializedChunk)) + + // Store the result in the slice + m.Lock() + serializedChunks[chunkIndex] = serializedChunk + m.Unlock() + }(i) + } + wg.Wait() + + return serializedChunks +} + +// CompressChunks compresses each serialized chunk +func CompressChunks(chunks []json.RawMessage) []json.RawMessage { + compressedChunks := make([]json.RawMessage, len(chunks)) + var wg sync.WaitGroup + + for i, chunk := range chunks { + wg.Add(1) + go func(i int, chunk json.RawMessage) { + defer wg.Done() + var compressedData bytes.Buffer + lz4Writer := lz4.NewWriter(&compressedData) + _, err := lz4Writer.Write(chunk) + if err != nil { + logrus.Errorf("Error compressing chunk %d: %v", i, err) + panic(err) + } + lz4Writer.Close() + compressedChunks[i] = compressedData.Bytes() + logrus.Infof("Compressed chunk %d, size: %d bytes", i, len(compressedChunks[i])) + }(i, chunk) + } + wg.Wait() + + return compressedChunks +} + +// DeserializeAssignment deserializes a blob of bytes into a set of column +// assignments representing assigned columns of a Wizard protocol. +func DeserializeAssignment(filepath string, numChunks int) (WAssignment, error) { + var ( + res = collection.NewMapping[ifaces.ColID, smartvectors.SmartVector]() + lock = &sync.Mutex{} + ) + + logrus.Infof("Reading the assignment files") + + // Read and decompress each chunk individually + var wg sync.WaitGroup + for i := 0; i < numChunks; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + chunkPath := fmt.Sprintf("%s_chunk_%d", filepath, i) + chunkData, err := ioutil.ReadFile(chunkPath) + if err != nil { + logrus.Errorf("Failed to read chunk %d: %v", i, err) + return + } + + logrus.Infof("Reading chunk %d from %s, size: %d bytes", i, chunkPath, len(chunkData)) + lz4Reader := lz4.NewReader(bytes.NewReader(chunkData)) + var decompressedData bytes.Buffer + n, err := decompressedData.ReadFrom(lz4Reader) + if err != nil { + logrus.Errorf("Error decompressing chunk %d: %v", i, err) + return + } + logrus.Infof("Decompressed chunk %d, size: %d bytes", i, n) + + // Deserialize the decompressed chunk + var chunkMap map[string]*CompressedSmartVector + if err := deserializeAnyWithCborPkg(decompressedData.Bytes(), &chunkMap); err != nil { + logrus.Errorf("Error deserializing chunk %d: %v", i, err) + return + } + + // Reconstruct the WAssignment + for k, v := range chunkMap { + decompressed := v.Decompress() + lock.Lock() + res.InsertNew(ifaces.ColID(k), decompressed) + lock.Unlock() + } + }(i) + } + wg.Wait() + + logrus.Infof("Hash of WAssignment after deserialization: %s", hashWAssignment(res)) + + return res, nil +} + +// CompressedSmartVector represents a [smartvectors.SmartVector] in a more +// space-efficient manner. +type CompressedSmartVector struct { + F []CompressedSVFragment +} + +// CompressedSVFragment represent a portion of a SerializableSmartVector +type CompressedSVFragment struct { + // L is the byte length used by the fragment + L uint8 + // X is the value used to represent a single field element + X *big.Int + // V is a byteslice storing the bytes of a vector if the fragment represent + // plain values. + V []byte + // N is the number of repetion used + N int +} + +func CompressSmartVector(sv smartvectors.SmartVector) *CompressedSmartVector { + + switch v := sv.(type) { + case *smartvectors.Constant: + return &CompressedSmartVector{ + F: []CompressedSVFragment{ + newConstantSVFragment(v.Val(), v.Len()), + }, + } + case *smartvectors.Regular: + return &CompressedSmartVector{ + F: []CompressedSVFragment{ + newSliceSVFragment(*v), + }, + } + case *smartvectors.PaddedCircularWindow: + var ( + w = v.Window() + offset = v.Offset() + paddingVal = v.PaddingVal() + fullLen = v.Len() + ) + + // It's a left-padded value + if offset == 0 { + return &CompressedSmartVector{ + F: []CompressedSVFragment{ + newSliceSVFragment(w), + newConstantSVFragment(paddingVal, fullLen-len(w)), + }, + } + } + + // It's a right-padded value + if offset+len(w) == fullLen { + return &CompressedSmartVector{ + F: []CompressedSVFragment{ + newConstantSVFragment(paddingVal, fullLen-len(w)), + newSliceSVFragment(w), + }, + } + } + + } + + // The other cases are not expected, we still support them via a + // suboptimal method. + return &CompressedSmartVector{ + F: []CompressedSVFragment{ + newSliceSVFragment(sv.IntoRegVecSaveAlloc()), + }, + } +} + +func (sv *CompressedSmartVector) Decompress() smartvectors.SmartVector { + + if len(sv.F) == 1 && sv.F[0].isConstant() { + val := new(field.Element).SetBigInt(sv.F[0].X) + return smartvectors.NewConstant(*val, sv.F[0].N) + } + + if len(sv.F) == 1 && sv.F[0].isPlain() { + return smartvectors.NewRegular(sv.F[0].readSlice()) + } + + if len(sv.F) == 2 && sv.F[0].isConstant() && sv.F[1].isPlain() { + + var ( + paddingVal = new(field.Element).SetBigInt(sv.F[0].X) + window = sv.F[1].readSlice() + size = sv.F[0].N + len(window) + ) + + return smartvectors.LeftPadded(window, *paddingVal, size) + } + + if len(sv.F) == 2 && sv.F[1].isConstant() && sv.F[0].isPlain() { + + var ( + paddingVal = new(field.Element).SetBigInt(sv.F[1].X) + window = sv.F[0].readSlice() + size = sv.F[1].N + len(window) + ) + + return smartvectors.RightPadded(window, *paddingVal, size) + } + + panic("unexpected pattern") +} + +func (f *CompressedSVFragment) isConstant() bool { + return f.X != nil +} + +func (f *CompressedSVFragment) isPlain() bool { + return f.V != nil +} + +func (f *CompressedSVFragment) readSlice() []field.Element { + + var ( + l = int(f.L) + buf = bytes.NewBuffer(f.V) + tmp = [32]byte{} + n = f.N + ) + + if l > 0 { + n = len(f.V) / l + } + + var ( + res = make([]field.Element, n) + ) + + for i := range res { + buf.Read(tmp[32-l:]) + res[i].SetBytes(tmp[:]) + } + + return res +} + +func newConstantSVFragment(x field.Element, n int) CompressedSVFragment { + + var ( + f big.Int + _ = x.BigInt(&f) + ) + + return CompressedSVFragment{ + X: &f, + N: n, + } +} + +func newSliceSVFragment(fv []field.Element) CompressedSVFragment { + + var ( + l int + ) + + for i := range fv { + l = max(l, (fv[i].BitLen()+7)/8) + } + + var ( + res = make([]byte, 0, len(fv)*l) + resBuf = bytes.NewBuffer(res) + ) + + for i := range fv { + fbytes := fv[i].Bytes() + resBuf.Write(fbytes[32-l:]) + } + + compressed := CompressedSVFragment{ + L: uint8(l), + V: resBuf.Bytes(), + } + + // Can happen if the caller provides a vector of the form [0, 0, 0, 0]. In + // that case the value of "n" cannot be infered from the slice because the + // slice will be empty. The solution is to provide a length to the vector + if l == 0 { + compressed.N = len(fv) + } + + return compressed +} diff --git a/prover/protocol/serialization/column_assignment_test.go b/prover/protocol/serialization/column_assignment_test.go new file mode 100644 index 00000000000..2fc088d21b4 --- /dev/null +++ b/prover/protocol/serialization/column_assignment_test.go @@ -0,0 +1,79 @@ +package serialization + +import ( + "fmt" + "io/ioutil" + "os" + "strconv" + "testing" + + "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/linea-monorepo/prover/maths/common/vector" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/ifaces" + "github.com/consensys/linea-monorepo/prover/utils/collection" + "github.com/stretchr/testify/assert" +) + +// Global svs array to be reused in both tests +var svs = []smartvectors.SmartVector{ + smartvectors.NewConstant(field.Zero(), 16), + smartvectors.ForTest(0, 0, 0, 0), + smartvectors.NewConstant(field.NewElement(42), 16), + smartvectors.ForTest(1, 2, 3, 4), + smartvectors.LeftPadded(vector.ForTest(1, 2, 3, 4), field.Zero(), 16), + smartvectors.LeftPadded(vector.ForTest(1, 2, 3, 4), field.One(), 16), + smartvectors.RightPadded(vector.ForTest(1, 2, 3, 4), field.Zero(), 16), + smartvectors.RightPadded(vector.ForTest(1, 2, 3, 4), field.One(), 16), +} + +func TestSerializeAndDeserializeAssignment(t *testing.T) { + // Create a sample WAssignment using svs + wAssignment := collection.NewMapping[ifaces.ColID, smartvectors.SmartVector]() + for i, sv := range svs { + wAssignment.InsertNew(ifaces.ColID(fmt.Sprintf("col%d", i+1)), sv) + } + + // Serialize the WAssignment + numChunks := 4 + serializedChunks := SerializeAssignment(wAssignment, numChunks) + + // Compress the serialized chunks + compressedChunks := CompressChunks(serializedChunks) + + // Write compressed chunks to temporary files + tempDir, err := ioutil.TempDir("", "serialization_test") + assert.NoError(t, err) + defer os.RemoveAll(tempDir) + + for i, chunk := range compressedChunks { + chunkPath := tempDir + "/chunk_" + strconv.Itoa(i) + err := ioutil.WriteFile(chunkPath, chunk, 0600) // Use 0600 permissions + assert.NoError(t, err) + } + + // Deserialize the WAssignment from the temporary files + deserializedAssignment, err := DeserializeAssignment(tempDir+"/chunk_", numChunks) + assert.NoError(t, err) + + // Verify that the deserialized WAssignment matches the original WAssignment + assert.Equal(t, hashWAssignment(wAssignment), hashWAssignment(deserializedAssignment)) +} + +func TestSmartVectorCompression(t *testing.T) { + for i := range svs { + t.Run(fmt.Sprintf("testcase-%v", i), func(t *testing.T) { + t.Logf("original smartvector: %v\n", svs[i].Pretty()) + + var ( + compressed = CompressSmartVector(svs[i]) + decompressed = compressed.Decompress() + recompressed = CompressSmartVector(decompressed) + ) + + assert.Equal(t, svs[i], decompressed) + assert.Equal(t, compressed, recompressed) + }) + } + +} diff --git a/prover/protocol/wizard/prover.go b/prover/protocol/wizard/prover.go index 0b3cf4c4165..03466ebcdbd 100644 --- a/prover/protocol/wizard/prover.go +++ b/prover/protocol/wizard/prover.go @@ -207,6 +207,25 @@ func RunProver(c *CompiledIOP, highLevelprover ProverStep) *ProverRuntime { return &runtime } +// ProveOnlyFirstRound computes the first round of the prover and returns the +// resulting ProverRuntime containing all the generated assignments. +func ProverOnlyFirstRound(c *CompiledIOP, highLevelprover ProverStep) *ProverRuntime { + runtime := c.createProver() + + // Run the user provided assignment function. We can't expect it + // to run all the rounds, because the compilation could have added + // extra-rounds. + // + highLevelprover(&runtime) + + // Then, run the compiled prover steps. This will only run thoses of the + // first round. + // + runtime.runProverSteps() + + return &runtime +} + // NumRounds returns the total number of rounds in the corresponding WizardIOP. // // Deprecated: this method does not bring anything useful as its already easy diff --git a/prover/zkevm/encoding_only.go b/prover/zkevm/encoding_only.go new file mode 100644 index 00000000000..6ff279bdf30 --- /dev/null +++ b/prover/zkevm/encoding_only.go @@ -0,0 +1,105 @@ +package zkevm + +import ( + "fmt" + "os" + "sync" + "time" + + "github.com/consensys/linea-monorepo/prover/config" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/lookup" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/mimc" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/specialqueries" + "github.com/consensys/linea-monorepo/prover/protocol/serialization" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/sirupsen/logrus" +) + +var ( + encodeOnlyZkevm *ZkEvm + onceEncodeOnlyZkevm = sync.Once{} + encodeOnlyCompilationSuite = compilationSuite{ + mimc.CompileMiMC, + specialqueries.RangeProof, + lookup.CompileLogDerivative, + } +) + +func EncodeOnlyZkEvm(tl *config.TracesLimits) *ZkEvm { + onceEncodeOnlyZkevm.Do(func() { + encodeOnlyZkevm = fullZKEVMWithSuite(tl, encodeOnlyCompilationSuite) + }) + + return encodeOnlyZkevm +} + +func (z *ZkEvm) AssignAndEncodeInChunks(filepath string, input *Witness, numChunks int) { + // Start encoding and measure time + encodingStart := time.Now() + run := wizard.ProverOnlyFirstRound(z.WizardIOP, z.prove(input)) + firstRoundOnlyDuration := time.Since(encodingStart).Seconds() + logrus.Infof("ProverOnlyFirstRound complete, took %.2f seconds", firstRoundOnlyDuration) + + // Start serialization + serializationStart := time.Now() + serializedChunks := serialization.SerializeAssignment(run.Columns, numChunks) + serializationDuration := time.Since(serializationStart).Seconds() + logrus.Infof("CBOR serialization complete, took %.2f seconds", serializationDuration) + + // Calculate total size of serialized data + totalSerializedSize := 0 + for _, chunk := range serializedChunks { + totalSerializedSize += len(chunk) + } + encodingDuration := time.Since(encodingStart).Seconds() + logrus.Infof("Encoding (ProverOnlyFirstRound + Serialization) complete, total serialized size: %d bytes, took %.2f seconds", totalSerializedSize, encodingDuration) + + // Start compression and measure time + compressionStart := time.Now() + compressedSerializedChunks := serialization.CompressChunks(serializedChunks) + compressionDuration := time.Since(compressionStart).Seconds() + + // Calculate total size of compressed data + totalCompressedSize := 0 + for _, chunk := range compressedSerializedChunks { + totalCompressedSize += len(chunk) + } + logrus.Infof("Compression complete, total compressed size: %d bytes, took %.2f seconds", totalCompressedSize, compressionDuration) + + // Start writing to files + writingStart := time.Now() + var wg sync.WaitGroup + for i := 0; i < numChunks; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + chunk := compressedSerializedChunks[i] + chunkPath := fmt.Sprintf("%s_chunk_%d", filepath, i) + + // Measure writing time for each chunk + writeStart := time.Now() + f, err := os.Create(chunkPath) + if err != nil { + logrus.Errorf("[%v] error creating file %s: %v", time.Now(), chunkPath, err) + return + } + defer f.Close() + + _, err = f.Write(chunk) + if err != nil { + logrus.Errorf("[%v] error writing to file %s: %v", time.Now(), chunkPath, err) + return + } + writeDuration := time.Since(writeStart).Seconds() + logrus.Infof("Completed writing chunk %d to %s, took %.2f seconds", i, chunkPath, writeDuration) + }(i) + } + wg.Wait() + writingDuration := time.Since(writingStart).Seconds() // Total writing time + logrus.Infof("Writing complete, total compressed size: %d bytes, took %.2f seconds", totalCompressedSize, writingDuration) + + // Total summary + totalDuration := encodingDuration + compressionDuration + writingDuration + logrus.Infof("Total serialized size %d bytes, total compressed size %d bytes, took %.2f sec total (encoding + compression + writing)", totalSerializedSize, totalCompressedSize, totalDuration) + logrus.Infof("Compression Ratio: %.2f", float64(totalSerializedSize)/float64(totalCompressedSize)) +}