Merge branch 'main' into vk/assign-optimize

Author: vineethk

.github/workflows/provision-replay-verify-archive-disks.yaml

@@ -46,7 +46,6 @@ jobs:
   provision-testnet:
     if: |
       github.event_name == 'schedule' ||
-      github.event_name == 'push' ||
       github.event_name == 'workflow_dispatch' && (inputs.NETWORK == 'testnet' || inputs.NETWORK == 'all')
     needs: determine-test-metadata
     uses: ./.github/workflows/workflow-run-replay-verify-archive-storage-provision.yaml

.github/workflows/replay-verify-on-archive.yaml

@@ -58,7 +58,6 @@ jobs:
   replay-testnet:
     if: |
       github.event_name == 'schedule' ||
-      github.event_name == 'push' ||
       github.event_name == 'workflow_dispatch' && (inputs.NETWORK == 'testnet' || inputs.NETWORK == 'all')
     needs: determine-test-metadata
     uses: ./.github/workflows/workflow-run-replay-verify-on-archive.yaml
@@ -72,8 +71,6 @@ jobs:
   replay-mainnet:
     if: |
       github.event_name == 'schedule' ||
-      github.event_name == 'push' ||
-      github.event_name == 'pull_request' ||
       github.event_name == 'workflow_dispatch' && (inputs.NETWORK == 'mainnet' || inputs.NETWORK == 'all' )
     needs: determine-test-metadata
     uses: ./.github/workflows/workflow-run-replay-verify-on-archive.yaml

.github/workflows/workflow-run-replay-verify-archive-storage-provision.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.inputs.BRANCH || 'add_replay_verify_workflow' }}      
+          ref: ${{ github.event.pull_request.head.ref || github.ref }}
 
       # Authenticate to Google Cloud the project is aptos-ci
       - name: Authenticate to Google Cloud

.github/workflows/workflow-run-replay-verify-on-archive.yaml

@@ -46,7 +46,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.inputs.BRANCH || 'add_replay_verify_workflow' }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref }}
 
       - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
         id: docker-setup
@@ -113,6 +113,7 @@ jobs:
         run: |
           cd testsuite/replay-verify
           poetry run python main.py  --network ${{ inputs.NETWORK }} --cleanup
+          echo "Cleanup completed"
 
           
           

Cargo.lock

@@ -916,3 +916,12 @@ serde-reflection = { git = "https://github.com/aptos-labs/serde-reflection", rev
 merlin = { git = "https://github.com/aptos-labs/merlin" }
 x25519-dalek = { git = "https://github.com/aptos-labs/x25519-dalek", branch = "zeroize_v1" }
 tonic = { git = "https://github.com/aptos-labs/tonic.git", rev = "0da1ba8b1751d6e19eb55be24cccf9ae933c666e" }
+futures = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-channel = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-executor = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-core = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-util = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-sink = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-io = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-task = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }
+futures-macro = { git = "https://github.com/aptos-labs/futures-rs", branch = "backport" }

Cargo.toml

@@ -25,8 +25,7 @@ use aptos_api_types::{
     AsConverter, EncodeSubmissionRequest, GasEstimation, GasEstimationBcs, HashValue,
     HexEncodedBytes, LedgerInfo, MoveType, PendingTransaction, SubmitTransactionRequest,
     Transaction, TransactionData, TransactionOnChainData, TransactionsBatchSingleSubmissionFailure,
-    TransactionsBatchSubmissionResult, UserTransaction, VerifyInput, VerifyInputWithRecursion,
-    MAX_RECURSIVE_TYPES_ALLOWED, U64,
+    TransactionsBatchSubmissionResult, UserTransaction, VerifyInput, VerifyInputWithRecursion, U64,
 };
 use aptos_crypto::{hash::CryptoHash, signing_message};
 use aptos_types::{
@@ -1041,10 +1040,12 @@ impl TransactionsApi {
         ledger_info: &LedgerInfo,
         data: SubmitTransactionPost,
     ) -> Result<SignedTransaction, SubmitTransactionError> {
+        pub const MAX_SIGNED_TRANSACTION_DEPTH: usize = 16;
+
         match data {
             SubmitTransactionPost::Bcs(data) => {
                 let signed_transaction: SignedTransaction =
-                    bcs::from_bytes_with_limit(&data.0, MAX_RECURSIVE_TYPES_ALLOWED as usize)
+                    bcs::from_bytes_with_limit(&data.0, MAX_SIGNED_TRANSACTION_DEPTH)
                         .context("Failed to deserialize input into SignedTransaction")
                         .map_err(|err| {
                             SubmitTransactionError::bad_request_with_code(

api/src/transactions.rs

@@ -521,11 +521,9 @@ pub enum MoveType {
     Unparsable(String),
 }
 
-/// Maximum number of recursive types
-/// Currently, this is allowed up to the serde limit of 16
-///
-/// TODO: Should this number be re-evaluated
-pub const MAX_RECURSIVE_TYPES_ALLOWED: u8 = 16;
+/// Maximum number of recursive types - Same as (non-public)
+/// move_core_types::safe_serialize::MAX_TYPE_TAG_NESTING
+pub const MAX_RECURSIVE_TYPES_ALLOWED: u8 = 8;
 
 impl VerifyInputWithRecursion for MoveType {
     fn verify(&self, recursion_count: u8) -> anyhow::Result<()> {

api/types/src/move_types.rs

@@ -0,0 +1,26 @@
+Loop { loop_count: Some(100000), loop_type: NoOp }	60	0.960	1.119	42122.6
+Loop { loop_count: Some(10000), loop_type: Arithmetic }	60	0.956	1.074	26240.7
+CreateObjects { num_objects: 10, object_payload_size: 0 }	60	0.938	1.168	156.6
+CreateObjects { num_objects: 10, object_payload_size: 10240 }	60	0.924	1.086	9713.2
+CreateObjects { num_objects: 100, object_payload_size: 0 }	60	0.922	1.275	1577.0
+CreateObjects { num_objects: 100, object_payload_size: 10240 }	60	0.935	1.070	11728.7
+InitializeVectorPicture { length: 128 }	60	0.926	1.069	169.4
+VectorPicture { length: 128 }	60	0.908	1.075	50.2
+VectorPictureRead { length: 128 }	60	0.919	1.059	48.0
+InitializeVectorPicture { length: 30720 }	60	0.939	1.127	28404.4
+VectorPicture { length: 30720 }	60	0.936	1.095	6935.6
+VectorPictureRead { length: 30720 }	60	0.939	1.093	6948.2
+SmartTablePicture { length: 30720, num_points_per_txn: 200 }	60	0.947	1.080	43673.3
+SmartTablePicture { length: 1048576, num_points_per_txn: 300 }	60	0.947	1.111	74145.8
+ResourceGroupsSenderWriteTag { string_length: 1024 }	60	0.918	1.075	15.8
+ResourceGroupsSenderMultiChange { string_length: 1024 }	60	0.909	1.169	32.9
+TokenV1MintAndTransferFT	60	0.953	1.069	384.6
+TokenV1MintAndTransferNFTSequential	60	0.938	1.064	600.3
+TokenV2AmbassadorMint { numbered: true }	60	0.951	1.057	516.6
+LiquidityPoolSwap { is_stable: true }	60	0.961	1.139	582.6
+LiquidityPoolSwap { is_stable: false }	60	0.929	1.099	563.0
+CoinInitAndMint	60	0.928	1.130	205.0
+FungibleAssetMint	60	0.930	1.098	235.8
+IncGlobalMilestoneAggV2 { milestone_every: 1 }	60	0.914	1.051	33.5
+IncGlobalMilestoneAggV2 { milestone_every: 2 }	60	0.914	1.105	19.0
+EmitEvents { count: 1000 }	60	0.937	1.158	8818.7

aptos-move/e2e-benchmark/data/calibration_values.tsv

@@ -15,7 +15,7 @@ use aptos_transaction_generator_lib::{
 use aptos_types::{account_address::AccountAddress, transaction::TransactionPayload};
 use rand::{rngs::StdRng, SeedableRng};
 use serde_json::json;
-use std::{collections::HashMap, process::exit};
+use std::{collections::HashMap, fs, process::exit};
 
 // bump after a bigger test or perf change, so you can easily distinguish runs
 // that are on top of this commit
@@ -85,42 +85,16 @@ const ALLOWED_REGRESSION: f64 = 0.15;
 const ALLOWED_IMPROVEMENT: f64 = 0.15;
 const ABSOLUTE_BUFFER_US: f64 = 2.0;
 
-const CALIBRATION_VALUES: &str = "
-Loop { loop_count: Some(100000), loop_type: NoOp }	60	0.955	1.074	41893.7
-Loop { loop_count: Some(10000), loop_type: Arithmetic }	60	0.965	1.078	25915.0
-CreateObjects { num_objects: 10, object_payload_size: 0 }	60	0.924	1.082	158.1
-CreateObjects { num_objects: 10, object_payload_size: 10240 }	60	0.951	1.118	9356.2
-CreateObjects { num_objects: 100, object_payload_size: 0 }	60	0.926	1.082	1574.2
-CreateObjects { num_objects: 100, object_payload_size: 10240 }	60	0.952	1.092	11541.9
-InitializeVectorPicture { length: 128 }	10	0.965	1.038	163.3
-VectorPicture { length: 128 }	10	0.938	1.060	48.8
-VectorPictureRead { length: 128 }	10	0.977	1.077	46.4
-InitializeVectorPicture { length: 30720 }	60	0.948	1.123	27893.4
-VectorPicture { length: 30720 }	60	0.931	1.125	6923.1
-VectorPictureRead { length: 30720 }	60	0.934	1.102	6923.1
-SmartTablePicture { length: 30720, num_points_per_txn: 200 }	60	0.952	1.109	43594.7
-SmartTablePicture { length: 1048576, num_points_per_txn: 300 }	60	0.957	1.120	73865.4
-ResourceGroupsSenderWriteTag { string_length: 1024 }	60	0.934	1.134	15.0
-ResourceGroupsSenderMultiChange { string_length: 1024 }	60	0.929	1.122	32.3
-TokenV1MintAndTransferFT	60	0.958	1.093	385.2
-TokenV1MintAndTransferNFTSequential	60	0.973	1.139	588.1
-TokenV2AmbassadorMint { numbered: true }	60	0.960	1.141	512.5
-LiquidityPoolSwap { is_stable: true }	60	0.961	1.103	590.3
-LiquidityPoolSwap { is_stable: false }	60	0.954	1.134	552.2
-CoinInitAndMint	10	0.975	1.043	199.6
-FungibleAssetMint	10	0.954	1.038	236.3
-IncGlobalMilestoneAggV2 { milestone_every: 1 }	10	0.960	1.047	32.9
-IncGlobalMilestoneAggV2 { milestone_every: 2 }	10	0.971	1.066	18.1
-EmitEvents { count: 1000 }	10	0.969	1.052	8615.5
-";
-
 struct CalibrationInfo {
     // count: usize,
     expected_time_micros: f64,
 }
 
 fn get_parsed_calibration_values() -> HashMap<String, CalibrationInfo> {
-    CALIBRATION_VALUES
+    let calibration_values =
+        fs::read_to_string("aptos-move/e2e-benchmark/data/calibration_values.tsv")
+            .expect("Unable to read file");
+    calibration_values
         .trim()
         .split('\n')
         .map(|line| {

aptos-move/e2e-benchmark/src/main.rs

@@ -1590,7 +1590,6 @@ module aptos_framework::account {
         create_account_unchecked(addr);
         register_coin<FakeCoin>(addr);
 
-        let eventhandle = &borrow_global<Account>(addr).coin_register_events;
         let event = CoinRegister { account: addr, type_info: type_info::type_of<FakeCoin>() };
 
         let events = event::emitted_events<CoinRegister>();

aptos-move/framework/aptos-framework/sources/account.move

@@ -322,7 +322,7 @@ impl Default for ConsensusConfig {
             num_bounded_executor_tasks: 16,
             enable_pre_commit: true,
             max_pending_rounds_in_commit_vote_cache: 100,
-            optimistic_sig_verification: false,
+            optimistic_sig_verification: true,
             enable_round_timeout_msg: true,
             enable_pipeline: false,
         }

config/src/config/consensus_config.rs

@@ -2,7 +2,7 @@
 // Parts of the project are originally copyright © Meta Platforms, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{order_vote::OrderVote, quorum_cert::QuorumCert};
+use crate::{common::Author, order_vote::OrderVote, quorum_cert::QuorumCert};
 use anyhow::{ensure, Context};
 use aptos_types::validator_verifier::ValidatorVerifier;
 use serde::{Deserialize, Serialize};
@@ -46,7 +46,17 @@ impl OrderVoteMsg {
 
     /// This function verifies the order_vote component in the order_vote_msg.
     /// The quorum cert is verified in the round manager when the quorum certificate is used.
-    pub fn verify_order_vote(&self, validator: &ValidatorVerifier) -> anyhow::Result<()> {
+    pub fn verify_order_vote(
+        &self,
+        sender: Author,
+        validator: &ValidatorVerifier,
+    ) -> anyhow::Result<()> {
+        ensure!(
+            self.order_vote.author() == sender,
+            "Order vote author {:?} is different from the sender {:?}",
+            self.order_vote.author(),
+            sender
+        );
         ensure!(
             self.quorum_cert().certified_block() == self.order_vote().ledger_info().commit_info(),
             "QuorumCert and OrderVote do not match"

consensus/consensus-types/src/order_vote_msg.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::common::{Author, Round};
-use anyhow::Context;
+use anyhow::{ensure, Context};
 use aptos_crypto::{bls12381, CryptoMaterialError};
 use aptos_short_hex_str::AsShortHexStr;
 use aptos_types::{
@@ -101,7 +101,13 @@ impl CommitVote {
 
     /// Verifies that the consensus data hash of LedgerInfo corresponds to the commit proposal,
     /// and then verifies the signature.
-    pub fn verify(&self, validator: &ValidatorVerifier) -> anyhow::Result<()> {
+    pub fn verify(&self, sender: Author, validator: &ValidatorVerifier) -> anyhow::Result<()> {
+        ensure!(
+            self.author() == sender,
+            "Commit vote author {:?} doesn't match with the sender {:?}",
+            self.author(),
+            sender
+        );
         validator
             .optimistic_verify(self.author(), &self.ledger_info, &self.signature)
             .context("Failed to verify Commit Vote")

consensus/consensus-types/src/pipeline/commit_vote.rs

@@ -107,15 +107,15 @@ impl PipelineFutures {
 pub struct PipelineInputTx {
     pub rand_tx: Option<oneshot::Sender<Option<Randomness>>>,
     pub order_vote_tx: Option<oneshot::Sender<()>>,
-    pub order_proof_tx: tokio::sync::broadcast::Sender<()>,
-    pub commit_proof_tx: tokio::sync::broadcast::Sender<LedgerInfoWithSignatures>,
+    pub order_proof_tx: Option<oneshot::Sender<()>>,
+    pub commit_proof_tx: Option<oneshot::Sender<LedgerInfoWithSignatures>>,
 }
 
 pub struct PipelineInputRx {
     pub rand_rx: oneshot::Receiver<Option<Randomness>>,
     pub order_vote_rx: oneshot::Receiver<()>,
-    pub order_proof_rx: tokio::sync::broadcast::Receiver<()>,
-    pub commit_proof_rx: tokio::sync::broadcast::Receiver<LedgerInfoWithSignatures>,
+    pub order_proof_fut: TaskFuture<()>,
+    pub commit_proof_fut: TaskFuture<LedgerInfoWithSignatures>,
 }
 
 /// A representation of a block that has been added to the execution pipeline. It might either be in ordered

consensus/consensus-types/src/pipelined_block.rs

@@ -82,10 +82,19 @@ impl ProposalMsg {
 
     pub fn verify(
         &self,
+        sender: Author,
         validator: &ValidatorVerifier,
         proof_cache: &ProofCache,
         quorum_store_enabled: bool,
     ) -> Result<()> {
+        if let Some(proposal_author) = self.proposal.author() {
+            ensure!(
+                proposal_author == sender,
+                "Proposal author {:?} doesn't match sender {:?}",
+                proposal_author,
+                sender
+            );
+        }
         self.proposal().payload().map_or(Ok(()), |p| {
             p.verify(validator, proof_cache, quorum_store_enabled)
         })?;