Epic: DA-Compression

aggregator/Cargo.toml

@@ -4,10 +4,10 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
+
 eth-types = { path = "../eth-types" }
+gadgets = { path = "../gadgets" }
 zkevm-circuits = { path = "../zkevm-circuits" }
 
 ark-std.workspace = true
@@ -25,12 +25,23 @@ halo2-ecc.workspace = true
 halo2_proofs.workspace = true
 snark-verifier.workspace = true
 snark-verifier-sdk.workspace = true
+strum.workspace = true
+strum_macros.workspace = true
 
+# da-4844
 revm-precompile.workspace = true
 revm-primitives.workspace = true
 c-kzg.workspace = true
 num-bigint.workspace = true
 
+# da-compression
+bitstream-io = "2.2.0"
+zstd = { git = "https://github.com/scroll-tech/zstd-rs", branch = "hack/mul-block", features = ["experimental"]}
+
+[dev-dependencies]
+
+csv = "1.1"
+
 [features]
 default = ["revm-precompile/c-kzg"]
 print-trace = ["ark-std/print-trace"]

aggregator/data/test_batches/batch005.hex

@@ -0,0 +1 @@
+000002d0f902cd11841c9c38008302aa6c9480e38291e06339d10aab483c65695d004dbd5c6980b902642cc4081e0000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000001def55c366bc340000000000000000000000000000000000000000000000000000000065b3f17700000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000006000000000000000000000000006efdbff2a14a7c8e15944d1f4a48f9f95f663a40000000000000000000000000000000000000000000000000000000001227a4f00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000814a23b053fd0f102aeeda0459215c2444799c70000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000006000000000000000000000000006efdbff2a14a7c8e15944d1f4a48f9f95f663a400000000000000000000000061ff9ad32e02a3f1104e138f28ec07a7f40205f50000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000083104ec3a018779682e00d75d49ac92d1949f2bfb5571c8c4fa7f8cec1c2f2d83788508ed8a033b8a1ef0f3be65be915e3b8c2afc88c65d2795863449e922e42203ed396cd5c000000adf8ab15841c9c38008255e8940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c8336b83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec3a00546d5f670bacc4e5c05556754be1a8a470c7b9e8592984a965c9b320c6ae913a01c0aa476da91c5cd704a00f5760ee800f264f0f218cfe4746b257bae8e91cf84000000adf8ab16841c9c38008255e8940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c8336b83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec4a06d73fb8a5385fb0417b3a870167dd397e98888d8e86ba5b68f08e9e83d7d34c2a00d4e6ca4f2dd64a0705bdb30d393d086bc117fb93a71c25ea95efb50a782960f00000297f9029406841c9c38008303a7be942db0afd0045f3518c77ec6591a542e326befd3d7872165400ce38000b90224ac9650d800000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001a0000000000000000000000000000000000000000000000000000000000000012475ceafe6000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000004b5951eac3722a54cb839dd0b70355c0969c6be2000000000000000000000000000000000000000000000000002165400ce3800000000000000000000000000000000000000000000000000000000000013fe6820000000000000000000000000000000000000000000000000000000065b3c4ff000000000000000000000000000000000000000000000000000000000000002b5300000000000000000000000000000000000004000bb806efdbff2a14a7c8e15944d1f4a48f9f95f663a400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000412210e8a0000000000000000000000000000000000000000000000000000000083104ec4a0bf112f7fdb0e72f36f5195bf5a1fc41b3a23d3d5dfb7d45dd80b0f16f6beb8b4a057c849ea70712bdb0872a2f0e97099b6589da26e870093ec6a9d40d052a361cb000000adf8ab17841c9c38008255e8940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c8336b83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec3a0b77d9a2776e2141c03dd15cad415f7ab51e52226a86013afeb8814edbfbd5c42a0699182c6777d0c014265cf45594831636749d2f231ff4a34731a35b8dda97fe5000002d6f902d305841c9c380083028cca9480e38291e06339d10aab483c65695d004dbd5c69862d79883d2000b902642cc4081e000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000003ac1ba4258e5d9a0000000000000000000000000000000000000000000000000000000065b3f18a000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002d79883d200000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000518bf6018616617f7ce02882a372556bfe93aaf3000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000006000000000000000000000000053000000000000000000000000000000000000040000000000000000000000005de6409d2921d0f02ca5121a4ad2aff06014f5cf0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000083104ec4a0d17fa93ab25551f2d5e05cfcbecc281bf3ea05ad985c7a7f671d43b0bf401a72a00bd38e1ae5a3b0b710d5547dc3263c2c7007de9d54c47b1cb53fde3d0e2c5d13000000adf8ab18841c9c38008255e8940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c8336b83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec3a072ee3bca3ec50212cc9938a2ea3d2b7566f93337308ae3d3e03a259099540fbca041fed2cdb36a8a6bb86fb92818e6aac6786aab9cc8118a75ac627fd46320f50300000270f9026d80841c9c38008305308894a6b71e26c5e0845f74c812102ca7114b6a896ab280b902041688f0b90000000000000000000000003e5c63644e683549055b9be8653de26e0b4cd36e00000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000018d4630778a0000000000000000000000000000000000000000000000000000000000000164b63e800d0000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000140000000000000000000000000f48f2b2d2a534e402487b3ee7c18c33aec0fe5e400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000001661a21bb1e83519202b1959a0169118772b74bb00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083104ec3a0f765123d2a9a7e9cb2ab4fd43c12e4aeaeaaf2de38d82ab1f353e3a9828d68cea029acc9503ca07208f74f39ecbb9c00ba0266b6fee069a79ee7ca6aaa77f139e5000000eef8ec23841c9c380083016b48945940a60866255031830aa1edfdd8b56ab39765b780b88472f66b670000000000000000000000000000000000000000000000000001de7c927900450000000000000000000000000000000000000000000000000000018d4630fb6400000000000000000000000006efdbff2a14a7c8e15944d1f4a48f9f95f663a40000000000000000000000000000000000000000000000000000000003dd614083104ec4a08bad69616b2c57dcb640a3c0ec0d4575c4b78ac13988cbdca88926afacb30abfa024f85d9436541d3bd5e03feab5861ff49720afa9aaa475003238a64bd4ad4772000000aff8ad82070a841c9c3800826716940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c832ab83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec3a07f9af4f9cd2f0816fba102bc2748174f4cb89a28a85d12b8fdafdf1ed8870397a00bc321e6a78e8c345682244958420b3cd3dde8cb3382b8799d7862b96ad2c839000000adf8ab19841c9c38008255e8940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c8336b83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec3a0fcad68db437b8f9cb5f7adc17a7fba147965cffbbba25a5932db7fb1ad23b805a028b1802a4b38d0aacbfed7034f8f76dd8e3c65758bde8681480583fe41dfaa8e0000006cf86a1c841c9c380082aa4c94c30215fe04bd85fac45e472ef332a0037641b4898084b49004e983104ec4a0cade06a731f537f00f2caebfb176a88491ca69b0617f849812ac9c0699589111a06deb36ea3fd58532e7a65316d642c605447a74467ed36f7e2bddaae83f42fe4b000000adf8ab17841c9c380082a7929406efdbff2a14a7c8e15944d1f4a48f9f95f663a480b844095ea7b300000000000000000000000080e38291e06339d10aab483c65695d004dbd5c6900000000000000000000000000000000000000000000000000000000003d090083104ec3a07c09994dc49214556b3a84c6db0d11fbb56776de5df79b4c6cf55fbb02c0f668a02d30f449390bd5f56fc89bbb66141e6dd10ea6e1be6823fbd3b32841da09abd5000000adf8ab03841c9c380082f2e894fea5157746cdb9c08095878e196748356c948d5e80b844a22cb4650000000000000000000000000cab6977a9c70e04458b740476b498b214019641000000000000000000000000000000000000000000000000000000000000000183104ec3a0a0b33ae475bc63aac2fea0716bc0dc92ed7235aef247b1aadbe8b6582393b1aba05e44df9b544fa4c7e56f11b8fd2646934570f7d90b9a018b72d2edacd08c9a690000008ff88d83030409841c9c380082a4f294530000000000000000000000000000000000000280a4bede39b5000000000000000000000000000000000000000000000000000000056b70646183104ec3a08845b06d9c02324dd8c9b72694c33221b889788f9f32891ff98aa3d07ade6b0da00648a751d578de7e48c426772692aec3a3648a98472b68f59141325d7d9d53cc000000adf8ab29841c9c380082eb929406efdbff2a14a7c8e15944d1f4a48f9f95f663a480b844095ea7b30000000000000000000000006131b5fae19ea4f9d964eac0408e4408b66337b50000000000000000000000000000000000000000000000000000000042c2e41d83104ec4a06d3d218b42f80d8d3492d2e80dbb2f133aac9bde7f14a6ba8ee9ec7b943282dba008d8ae5a6a3cbeea7b0d5f15035dfe33380402ff67d68587d2bd7747deaa51590000016ff9016c0e841c9c3800826e8e9447fbe95e981c0df9737b6971b451fb15fdc989d980b901045b7d7482000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000406230666437646337316637663435383038343834613364313436643763653463326138386331666536346435396665633530616133613535353065353265393600000000000000000000000000000000000000000000000000000000000000406631386634353231393631393864376636373333303539646338386531666433626563383136396432396662373263396539353036346436383833373832353383104ec4a09f878c3fc62e8689d79bae378f3dede6b4406f823c0510b1f83e8f431e02c244a039e15d267fbc58ca28d0cf2e6a3254d7a3a996abe5e2ff67c3b1f57aa7942047000000aff8ad8204c0841c9c3800826716940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c832ab83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec4a05d5e571e16e4260a19b01ae6e4a3d182a6ba4e3e2c946cfff5a741bfca809218a0581d45a59999d0c7a455d6c19b60ac7d4e3ae4886a8a315a28a27be6edb763fe00000156f901531c841c9c38008302f2a59418b71386418a9fca5ae7165e31c385a5130011b6870c3b65da5f82deb8e47ff36ab5000000000000000000000000000000000000000000000000000000000075f9a2000000000000000000000000000000000000000000000000000000000000008000000000000000000000000039bccd4144c414b6b410f67850f330c8934e52360000000000000000000000000000000000000000000000000000000065b3c75f0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000530000000000000000000000000000000000000400000000000000000000000006efdbff2a14a7c8e15944d1f4a48f9f95f663a483104ec4a059c6ffce0b94c2337b52a5905af57f54181e7b3d87c2950ef99f0b1d50c7edb9a03bb6d19ccb030ce85d71c84068bb2c6f5f9e8a9c06a4dd32f42033eceb95881c000000adf8ab80841c9c3800829c40940a88bc5c32b684d467b43c06d9e0899efeaf59df86d12f0c4c832ab83e646174613a2c7b2270223a226c61796572322d3230222c226f70223a22636c61696d222c227469636b223a22244c32222c22616d74223a2231303030227d83104ec3a07a758ebcc36ff8c6793b2aeb31d53a83d178dee8813350ecaa0b0e6e16fd1346a05f1b221ab8178ef1fbb04fb09f9fa77ac6734981252cd638170185606e4392940000006cf86a1d841c9c380082aa4c94c30215fe04bd85fac45e472ef332a0037641b4898084b49004e983104ec4a007fecf5ae2d18f957dee01c03f9f8de7f16d4fff8c201caa3dc95f4adb98d71aa050184ae27523b84616d4788c2f41081591ba068aa752578ef1a7a596adf62f2d00000156f901530a8417d7840083033c209426cb8660eefcb2f7652e7796ed713c9fb8373f8e8701d63571595000b8e47ff36ab500000000000000000000000000000000000000000000000000000000001194200000000000000000000000000000000000000000000000000000000000000080000000000000000000000000a290106b849e0b89b2b988bb5bb0fabe5ca3a8e50000000000000000000000000000000000000000000000000000000065b3c75f0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000530000000000000000000000000000000000000400000000000000000000000006efdbff2a14a7c8e15944d1f4a48f9f95f663a483104ec3a0b858650b5530989699ce1ff48e17a0f917eb725403d4662c4c9a14f833e65f8ea05dc9fac6f73b05f8a8beeb165bd42ddd365469bd1068dad09a063e7d6430246b0000016ff9016c1d841c9c3800826e8e9447fbe95e981c0df9737b6971b451fb15fdc989d980b901045b7d7482000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000403561656634633236646538323766663239626139373136616335303034343266656235386663613163366363383839373537343239643633646539396266356600000000000000000000000000000000000000000000000000000000000000403561656634633236646538323766663239626139373136616335303034343266656235386663613163366363383839373537343239643633646539396266356683104ec3a00dd6c69a321444e81d9892a3c38118e13c77e258f913aaaeaf4b1336e89de918a0149e56769db141eae12efe3ccdf586cf5070377a4a77001a3836bef799f5aef800000070f86e1c841c9c3800825208945e809a85aa182a9921edd10a4163745bb3e3628488025f83981097800a8083104ec3a06896cf307be155baab79cb8499ac5eac1ce780fdd6a8f1332a6fe2b29db90b73a03fa16599679ec377e7cc84f6056294965643fd2303fa08a048a24003154a9a9b