Merge #3684

3684: Improve support for cycles access r=zhangsoledad a=zhangsoledad  ### What problem does this PR solve? Improve ckb's support for recording and accessing cycles as described in [3613](#3673), while implementing the fee_rate estimate short-term scheme. ### What is changed and how it works? * For cycles and tx_size, the appending of records is done in a compatible way, so that when a new version of the node is run, new data is written, cycles and tx_size are appended, and old data is read in a compatible way. RPC returns data that is not recorded as `null`. * `get_block`/`get_block_by_number` adds a parameter `with_cycles` to choose whether to return cycles or not, mainly because some scenarios, such as indexer, do not need cycles and are performance sensitive, so an additional parameter is added to avoid the extra overhead needed to return cycles. ### Check List  Tests  - Unit test - Integration test ### Release note  ```release-note Title Only: Include only the PR title in the release note. ``` Co-authored-by: zhangsoledad <[email protected]>
nervosnetwork · Nov 10, 2022 · 89c5a3f · 89c5a3f
2 parents 335469a + 2d6da08
commit 89c5a3f
Show file tree

Hide file tree

Showing 36 changed files with 1,265 additions and 171 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/chain/src/chain.rs b/chain/src/chain.rs
@@ -22,7 +22,7 @@ use ckb_types::{
             ResolvedTransaction,
         },
         service::{Request, DEFAULT_CHANNEL_SIZE, SIGNAL_CHANNEL_SIZE},
-        BlockExt, BlockNumber, BlockView, Capacity, Cycle, HeaderView,
+        BlockExt, BlockNumber, BlockView, Cycle, HeaderView,
     },
     packed::{Byte32, ProposalShortId},
     utilities::merkle_mountain_range::ChainRootMMR,
@@ -412,6 +412,8 @@ impl ChainService {
             total_uncles_count: parent_ext.total_uncles_count + block.data().uncles().len() as u64,
             verified: None,
             txs_fees: vec![],
+            cycles: None,
+            txs_sizes: None,
         };
 
         db_txn.insert_block_epoch_index(
@@ -755,8 +757,12 @@ impl ChainService {
                             };
                             match verified {
                                 Ok((cycles, cache_entries)) => {
-                                    let txs_fees =
-                                        cache_entries.iter().map(|entry| entry.fee).collect();
+                                    let txs_sizes = resolved
+                                        .iter()
+                                        .map(|rtx| {
+                                            rtx.transaction.data().serialized_size_in_block() as u64
+                                        })
+                                        .collect();
                                     txn.attach_block(b)?;
                                     attach_block_cell(txn, b)?;
                                     mmr.push(b.digest())
@@ -766,7 +772,8 @@ impl ChainService {
                                         txn,
                                         &b.header().hash(),
                                         ext.clone(),
-                                        Some(txs_fees),
+                                        Some(&cache_entries),
+                                        Some(txs_sizes),
                                     )?;
 
                                     if !switch.disable_script() && b.transactions().len() > 1 {
@@ -798,7 +805,7 @@ impl ChainService {
                 attach_block_cell(txn, b)?;
                 mmr.push(b.digest())
                     .map_err(|e| InternalErrorKind::MMR.other(e))?;
-                self.insert_ok_ext(txn, &b.header().hash(), ext.clone(), None)?;
+                self.insert_ok_ext(txn, &b.header().hash(), ext.clone(), None, None)?;
             }
         }
 
@@ -835,12 +842,19 @@ impl ChainService {
         txn: &StoreTransaction,
         hash: &Byte32,
         mut ext: BlockExt,
-        txs_fees: Option<Vec<Capacity>>,
+        cache_entries: Option<&[Completed]>,
+        txs_sizes: Option<Vec<u64>>,
     ) -> Result<(), Error> {
         ext.verified = Some(true);
-        if let Some(txs_fees) = txs_fees {
+        if let Some(entries) = cache_entries {
+            let (txs_fees, cycles) = entries
+                .iter()
+                .map(|entry| (entry.fee, entry.cycles))
+                .unzip();
             ext.txs_fees = txs_fees;
+            ext.cycles = Some(cycles);
         }
+        ext.txs_sizes = txs_sizes;
         txn.insert_block_ext(hash, &ext)
     }
 

diff --git a/chain/src/tests/find_fork.rs b/chain/src/tests/find_fork.rs
@@ -66,6 +66,8 @@ fn test_find_fork_case1() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
 
     let mut fork = ForkChanges::default();
@@ -136,6 +138,8 @@ fn test_find_fork_case2() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
 
     let mut fork = ForkChanges::default();
@@ -207,6 +211,8 @@ fn test_find_fork_case3() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
     let mut fork = ForkChanges::default();
 
@@ -277,6 +283,8 @@ fn test_find_fork_case4() {
         // if txs in parent is invalid, txs in block is also invalid
         verified: None,
         txs_fees: vec![],
+        cycles: None,
+        txs_sizes: None,
     };
 
     let mut fork = ForkChanges::default();

diff --git a/chain/src/tests/load_code_with_snapshot.rs b/chain/src/tests/load_code_with_snapshot.rs
@@ -108,7 +108,10 @@ fn test_load_code() {
     let ret = tx_pool.submit_local_tx(tx.clone()).unwrap();
     assert!(ret.is_ok(), "ret {:?}", ret);
     let tx_status = tx_pool.get_tx_status(tx.hash());
-    assert_eq!(tx_status.unwrap().unwrap(), TxStatus::Pending);
+    assert_eq!(
+        tx_status.unwrap().unwrap(),
+        (TxStatus::Pending, Some(11174))
+    );
 }
 
 #[test]
@@ -177,7 +180,7 @@ fn test_load_code_with_snapshot() {
         let mut counter = 0;
         loop {
             let tx_status = tx_pool.get_tx_status(tx.hash());
-            if let Ok(Ok(status)) = tx_status {
+            if let Ok(Ok((status, _))) = tx_status {
                 if status == TxStatus::Pending {
                     break;
                 }
@@ -265,7 +268,7 @@ fn _test_load_code_with_snapshot_after_hardfork(script_type: ScriptHashType) {
         let mut counter = 0;
         loop {
             let tx_status = tx_pool.get_tx_status(tx.hash());
-            if let Ok(Ok(status)) = tx_status {
+            if let Ok(Ok((status, _))) = tx_status {
                 if status == TxStatus::Pending {
                     break;
                 }

diff --git a/devtools/doc/rpc.py b/devtools/doc/rpc.py
@@ -256,6 +256,8 @@ def handle_starttag(self, tag, attrs):
                 self.ty = '`null`'
             if self.ty == RUST_DOC_PREFIX + '/std/primitive.bool.html':
                 self.ty = '`boolean`'
+            if self.ty == RUST_DOC_PREFIX + '/std/primitive.f64.html':
+                self.ty = '`64-bit floating point`'
             if self.ty == RUST_DOC_PREFIX + '/alloc/string/struct.String.html':
                 self.ty = '`string`'
             elif self.ty == RUST_DOC_PREFIX + '/core/option/enum.Option.html':

diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml
@@ -43,6 +43,7 @@ ckb-tx-pool = { path = "../tx-pool", version = "= 0.106.0-pre" }
 ckb-memory-tracker = { path = "../util/memory-tracker", version = "= 0.106.0-pre" }
 ckb-pow = { path = "../pow", version = "= 0.106.0-pre" }
 ckb-indexer = { path = "../util/indexer", version = "= 0.106.0-pre" }
+itertools = "0.10.5"
 tokio = "1"
 
 [dev-dependencies]

diff --git a/rpc/README.md b/rpc/README.md
@@ -52,6 +52,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.61.0.
         * [Method `get_consensus`](#method-get_consensus)
         * [Method `get_block_median_time`](#method-get_block_median_time)
         * [Method `estimate_cycles`](#method-estimate_cycles)
+        * [Method `get_fee_rate_statics`](#method-get_fee_rate_statics)
     * [Module Experiment](#module-experiment)
         * [Method `dry_run_transaction`](#method-dry_run_transaction)
         * [Method `calculate_dao_maximum_withdraw`](#method-calculate_dao_maximum_withdraw)
@@ -105,6 +106,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.61.0.
     * [Type `BlockEconomicState`](#type-blockeconomicstate)
     * [Type `BlockIssuance`](#type-blockissuance)
     * [Type `BlockNumber`](#type-blocknumber)
+    * [Type `BlockResponse`](#type-blockresponse)
     * [Type `BlockTemplate`](#type-blocktemplate)
     * [Type `BlockView`](#type-blockview)
     * [Type `Byte32`](#type-byte32)
@@ -130,6 +132,7 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.61.0.
     * [Type `EpochNumberWithFraction`](#type-epochnumberwithfraction)
     * [Type `EpochView`](#type-epochview)
     * [Type `EstimateCycles`](#type-estimatecycles)
+    * [Type `FeeRateStatics`](#type-feeratestatics)
     * [Type `H256`](#type-h256)
     * [Type `HardForkFeature`](#type-hardforkfeature)
     * [Type `Header`](#type-header)
@@ -281,10 +284,11 @@ A cell is live if
 *   it is not found as an input in any transaction in the canonical chain.
 
 #### Method `get_block`
-* `get_block(block_hash, verbosity)`
+* `get_block(block_hash, verbosity, with_cycles)`
     * `block_hash`: [`H256`](#type-h256)
     * `verbosity`: [`Uint32`](#type-uint32) `|` `null`
-* result: [`BlockView`](#type-blockview) `|` [`SerializedBlock`](#type-serializedblock) `|` `null`
+    * `with_cycles`: `boolean` `|` `null`
+* result: [`BlockResponse`](#type-blockresponse) `|` `null`
 
 Returns the information about a block by hash.
 
@@ -294,6 +298,8 @@ Returns the information about a block by hash.
 
 *   `verbosity` - result format which allows 0 and 2. (**Optional**, the default is 2.)
 
+*   `with_cycles` - whether the return cycles of block transactions. (**Optional**, default false.)
+
 ###### Returns
 
 The RPC returns a block or null. When the RPC returns a block, the block hash must equal to the parameter `block_hash`.
@@ -315,7 +321,9 @@ Request
   "jsonrpc": "2.0",
   "method": "get_block",
   "params": [
-    "0xa5f5c85987a15de25661e5a214f2c1449cd803f071acc7999820f25246471f40"
+    "0xa5f5c85987a15de25661e5a214f2c1449cd803f071acc7999820f25246471f40",
+     null,
+     true
   ]
 }
 ```
@@ -378,7 +386,8 @@ Response
         ]
       }
     ],
-    "uncles": []
+    "uncles": [],
+    "cycles": []
   }
 }
 ```
@@ -397,10 +406,11 @@ The response looks like below when `verbosity` is 0.
 
 
 #### Method `get_block_by_number`
-* `get_block_by_number(block_number, verbosity)`
+* `get_block_by_number(block_number, verbosity, with_cycles)`
     * `block_number`: [`BlockNumber`](#type-blocknumber)
     * `verbosity`: [`Uint32`](#type-uint32) `|` `null`
-* result: [`BlockView`](#type-blockview) `|` [`SerializedBlock`](#type-serializedblock) `|` `null`
+    * `with_cycles`: `boolean` `|` `null`
+* result: [`BlockResponse`](#type-blockresponse) `|` `null`
 
 Returns the block in the [canonical chain](#canonical-chain) with the specific block number.
 
@@ -410,6 +420,8 @@ Returns the block in the [canonical chain](#canonical-chain) with the specific b
 
 *   `verbosity` - result format which allows 0 and 2. (**Optional**, the default is 2.)
 
+*   `with_cycles` - whether the return cycles of block transactions. (**Optional**, default false.)
+
 ###### Returns
 
 The RPC returns the block when `block_number` is less than or equal to the tip block number returned by [`get_tip_block_number`](#method-get_tip_block_number) and returns null otherwise.
@@ -500,7 +512,8 @@ Response
         ]
       }
     ],
-    "uncles": []
+    "uncles": [],
+    "cycles": null
   }
 }
 ```
@@ -827,6 +840,7 @@ Response
       "version": "0x0",
       "witnesses": []
     },
+    "cycles": "0x219",
     "tx_status": {
       "block_hash": null,
       "status": "pending",
@@ -846,6 +860,7 @@ The response looks like below when `verbosity` is 0.
   "jsonrpc": "2.0",
   "result": {
     "transaction": "0x.....",
+    "cycles": "0x219",
     "tx_status": {
       "block_hash": null,
       "status": "pending",
@@ -1661,6 +1676,51 @@ Response
 ```
 
 
+#### Method `get_fee_rate_statics`
+* `get_fee_rate_statics(target)`
+    * `target`: [`Uint64`](#type-uint64) `|` `null`
+* result: [`FeeRateStatics`](#type-feeratestatics) `|` `null`
+
+Returns the fee_rate statistics of confirmed blocks on the chain
+
+###### Params
+
+*   `target` - Specify the number (1 - 101) of confirmed blocks to be counted. If the number is even, automatically add one. If not specified, defaults to 21
+
+###### Returns
+
+If the query has data records, it returns statistics, if not, it returns null.
+
+###### Examples
+
+Request
+
+
+```
+{
+  "id": 42,
+  "jsonrpc": "2.0",
+  "method": "get_fee_rate_statics",
+  "params": []
+}
+```
+
+
+Response
+
+
+```
+{
+  "id": 42,
+  "jsonrpc": "2.0",
+  "result": {
+    "mean":59.29387293275573,
+    "median":5.288207297726071
+   }
+}
+```
+
+
 ### Module Experiment
 
 RPC Module Experiment for experimenting methods.
@@ -4946,6 +5006,19 @@ Consecutive block number starting from 0.
 
 This is a 64-bit unsigned integer type encoded as the 0x-prefixed hex string in JSON. See examples of [Uint64](#type-uint64).
 
+### Type `BlockResponse`
+
+The wrapper represent response of `get_block` | `get_block_by_number`, return a Block with cycles.
+
+#### Fields
+
+`BlockResponse` is a JSON object with the following fields.
+
+*   `block`: [`ResponseFormat`](#type-responseformat) - The block structure
+
+*   `cycles`: `Array<` [`Cycle`](#type-cycle) `>` `|` `null` - The block transactions consumed cycles.
+
+
 ### Type `BlockTemplate`
 
 A block template for miners.
@@ -5559,6 +5632,19 @@ Response result of the RPC method `estimate_cycles`.
 *   `cycles`: [`Cycle`](#type-cycle) - The count of cycles that the VM has consumed to verify this transaction.
 
 
+### Type `FeeRateStatics`
+
+The fee_rate statistics information, includes mean and median
+
+#### Fields
+
+`FeeRateStatics` is a JSON object with the following fields.
+
+*   `mean`: `64-bit floating point` - mean
+
+*   `median`: `64-bit floating point` - median
+
+
 ### Type `H256`
 
 The 32-byte fixed-length binary data.
@@ -6573,6 +6659,8 @@ The JSON view of a transaction as well as its status.
 
 *   `transaction`: [`ResponseFormat`](#type-responseformat) `|` `null` - The transaction.
 
+*   `cycles`: [`Cycle`](#type-cycle) `|` `null` - The transaction consumed cycles.
+
 *   `tx_status`: [`TxStatus`](#type-txstatus) - The Transaction status.