diff --git a/Cargo.lock b/Cargo.lock index d5056dbfcc27d..982e6bcc444b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1603,6 +1603,7 @@ version = "0.1.0" dependencies = [ "anyhow", "aptos-executor", + "aptos-executor-types", "aptos-experimental-runtimes", "aptos-infallible", "aptos-logger", diff --git a/execution/executor-benchmark/src/native_executor.rs b/execution/executor-benchmark/src/native_executor.rs index 463777bfed0c2..2eea6d915e11a 100644 --- a/execution/executor-benchmark/src/native_executor.rs +++ b/execution/executor-benchmark/src/native_executor.rs @@ -6,9 +6,8 @@ use crate::{ metrics::TIMER, }; use anyhow::Result; -use aptos_executor::{ - block_executor::TransactionBlockExecutor, components::chunk_output::ChunkOutput, -}; +use aptos_executor::block_executor::TransactionBlockExecutor; +use aptos_executor_types::execution_output::ExecutionOutput; use aptos_storage_interface::cached_state_view::CachedStateView; use aptos_types::{ account_address::AccountAddress, @@ -351,7 +350,7 @@ impl TransactionBlockExecutor for NativeExecutor { transactions: ExecutableTransactions, state_view: CachedStateView, _onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result { + ) -> Result { let transactions = match transactions { ExecutableTransactions::Unsharded(txns) => txns, _ => todo!("sharded execution not yet supported"), @@ -422,7 +421,7 @@ impl TransactionBlockExecutor for NativeExecutor { }) .collect::>>() })?; - Ok(ChunkOutput { + Ok(ExecutionOutput { transactions: transactions.into_iter().map(|t| t.into_inner()).collect(), transaction_outputs, state_cache: state_view.into_state_cache(), diff --git a/execution/executor-types/src/execution_output.rs b/execution/executor-types/src/execution_output.rs new file mode 100644 index 0000000000000..41b20fd1828b2 --- /dev/null +++ b/execution/executor-types/src/execution_output.rs @@ -0,0 +1,21 @@ +// Copyright © Aptos Foundation +// Parts of the project are originally copyright © Meta Platforms, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#![forbid(unsafe_code)] + +use aptos_storage_interface::cached_state_view::StateCache; +use aptos_types::transaction::{block_epilogue::BlockEndInfo, Transaction, TransactionOutput}; + +pub struct ExecutionOutput { + /// Input transactions. + pub transactions: Vec, + /// Raw VM output. + pub transaction_outputs: Vec, + /// Carries the frozen base state view, so all in-mem nodes involved won't drop before the + /// execution result is processed; as well as all the accounts touched during execution, together + /// with their proofs. + pub state_cache: StateCache, + /// Optional StateCheckpoint payload + pub block_end_info: Option, +} diff --git a/execution/executor-types/src/lib.rs b/execution/executor-types/src/lib.rs index b0b1dc81987bf..7f40557d14a90 100644 --- a/execution/executor-types/src/lib.rs +++ b/execution/executor-types/src/lib.rs @@ -36,6 +36,7 @@ use std::{ }; mod error; +pub mod execution_output; mod ledger_update_output; pub mod parsed_transaction_output; pub mod state_checkpoint_output; diff --git a/execution/executor/proptest-regressions/tests/mod.txt b/execution/executor/proptest-regressions/tests/mod.txt deleted file mode 100644 index a10d74db7985b..0000000000000 --- a/execution/executor/proptest-regressions/tests/mod.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 8d8247fbd6da3ebef1e20384b2f0c098c1a6f0edd28041ffdc99afc406654c67 # shrinks to (num_user_txns, reconfig_txn_index) = (10, 0) -cc 06cc3868004fdece03d52e325e1086deaa8eed5686d2490640b52d6b42e8ece8 # shrinks to a_size = 0, b_size = 0, c_size = 0, amount = 1 diff --git a/execution/executor/src/components/block_tree/mod.rs b/execution/executor/src/block_executor/block_tree/mod.rs similarity index 99% rename from execution/executor/src/components/block_tree/mod.rs rename to execution/executor/src/block_executor/block_tree/mod.rs index c8cca189bf267..3967ac1e63120 100644 --- a/execution/executor/src/components/block_tree/mod.rs +++ b/execution/executor/src/block_executor/block_tree/mod.rs @@ -8,8 +8,8 @@ mod test; use crate::{ - components::partial_state_compute_result::PartialStateComputeResult, logging::{LogEntry, LogSchema}, + types::partial_state_compute_result::PartialStateComputeResult, }; use anyhow::{anyhow, ensure, Result}; use aptos_consensus_types::block::Block as ConsensusBlock; diff --git a/execution/executor/src/components/block_tree/test.rs b/execution/executor/src/block_executor/block_tree/test.rs similarity index 96% rename from execution/executor/src/components/block_tree/test.rs rename to execution/executor/src/block_executor/block_tree/test.rs index b18acd7dbda82..3687b8059e52b 100644 --- a/execution/executor/src/components/block_tree/test.rs +++ b/execution/executor/src/block_executor/block_tree/test.rs @@ -2,9 +2,9 @@ // Parts of the project are originally copyright © Meta Platforms, Inc. // SPDX-License-Identifier: Apache-2.0 -use crate::components::{ - block_tree::{epoch_genesis_block_id, BlockLookup, BlockTree}, - partial_state_compute_result::PartialStateComputeResult, +use crate::{ + block_executor::block_tree::{epoch_genesis_block_id, BlockLookup, BlockTree}, + types::partial_state_compute_result::PartialStateComputeResult, }; use aptos_crypto::{hash::PRE_GENESIS_BLOCK_ID, HashValue}; use aptos_infallible::Mutex; diff --git a/execution/executor/src/block_executor.rs b/execution/executor/src/block_executor/mod.rs similarity index 91% rename from execution/executor/src/block_executor.rs rename to execution/executor/src/block_executor/mod.rs index 73ed2811a64d7..1dd9f38d343f9 100644 --- a/execution/executor/src/block_executor.rs +++ b/execution/executor/src/block_executor/mod.rs @@ -5,21 +5,22 @@ #![forbid(unsafe_code)] use crate::{ - components::{ - block_tree::BlockTree, chunk_output::ChunkOutput, do_ledger_update::DoLedgerUpdate, - partial_state_compute_result::PartialStateComputeResult, - }, logging::{LogEntry, LogSchema}, metrics::{ COMMIT_BLOCKS, CONCURRENCY_GAUGE, EXECUTE_BLOCK, OTHER_TIMERS, SAVE_TRANSACTIONS, TRANSACTIONS_SAVED, UPDATE_LEDGER, VM_EXECUTE_BLOCK, }, + types::partial_state_compute_result::PartialStateComputeResult, + workflow::{ + do_get_execution_output::DoGetExecutionOutput, do_ledger_update::DoLedgerUpdate, + do_state_checkpoint::DoStateCheckpoint, + }, }; use anyhow::Result; use aptos_crypto::HashValue; use aptos_executor_types::{ - state_checkpoint_output::StateCheckpointOutput, state_compute_result::StateComputeResult, - BlockExecutorTrait, ExecutorError, ExecutorResult, + execution_output::ExecutionOutput, state_checkpoint_output::StateCheckpointOutput, + state_compute_result::StateComputeResult, BlockExecutorTrait, ExecutorError, ExecutorResult, }; use aptos_experimental_runtimes::thread_manager::THREAD_MANAGER; use aptos_infallible::RwLock; @@ -38,15 +39,18 @@ use aptos_types::{ state_store::{state_value::StateValue, StateViewId}, }; use aptos_vm::AptosVM; +use block_tree::BlockTree; use fail::fail_point; use std::{marker::PhantomData, sync::Arc}; +pub mod block_tree; + pub trait TransactionBlockExecutor: Send + Sync { fn execute_transaction_block( transactions: ExecutableTransactions, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result; + ) -> Result; } impl TransactionBlockExecutor for AptosVM { @@ -54,8 +58,12 @@ impl TransactionBlockExecutor for AptosVM { transactions: ExecutableTransactions, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result { - ChunkOutput::by_transaction_execution::(transactions, state_view, onchain_config) + ) -> Result { + DoGetExecutionOutput::by_transaction_execution::( + transactions, + state_view, + onchain_config, + ) } } @@ -267,7 +275,17 @@ where let _timer = OTHER_TIMERS.timer_with(&["state_checkpoint"]); THREAD_MANAGER.get_exe_cpu_pool().install(|| { - chunk_output.into_state_checkpoint_output(parent_output.state(), block_id) + fail_point!("executor::block_state_checkpoint", |_| { + Err(anyhow::anyhow!("Injected error in block state checkpoint.")) + }); + + DoStateCheckpoint::run( + chunk_output, + parent_output.state(), + Some(block_id), + None, + /*is_block=*/ true, + ) })? }; diff --git a/execution/executor/src/components/chunk_commit_queue.rs b/execution/executor/src/chunk_executor/chunk_commit_queue.rs similarity index 95% rename from execution/executor/src/components/chunk_commit_queue.rs rename to execution/executor/src/chunk_executor/chunk_commit_queue.rs index ddfd14caca790..b14f4853bc3ac 100644 --- a/execution/executor/src/components/chunk_commit_queue.rs +++ b/execution/executor/src/chunk_executor/chunk_commit_queue.rs @@ -4,9 +4,11 @@ #![forbid(unsafe_code)] -use crate::components::{ - chunk_result_verifier::ChunkResultVerifier, executed_chunk::ExecutedChunk, - partial_state_compute_result::PartialStateComputeResult, +use crate::{ + chunk_executor::chunk_result_verifier::ChunkResultVerifier, + types::{ + executed_chunk::ExecutedChunk, partial_state_compute_result::PartialStateComputeResult, + }, }; use anyhow::{anyhow, ensure, Result}; use aptos_executor_types::state_checkpoint_output::StateCheckpointOutput; diff --git a/execution/executor/src/components/chunk_result_verifier.rs b/execution/executor/src/chunk_executor/chunk_result_verifier.rs similarity index 100% rename from execution/executor/src/components/chunk_result_verifier.rs rename to execution/executor/src/chunk_executor/chunk_result_verifier.rs diff --git a/execution/executor/src/chunk_executor.rs b/execution/executor/src/chunk_executor/mod.rs similarity index 96% rename from execution/executor/src/chunk_executor.rs rename to execution/executor/src/chunk_executor/mod.rs index c5854affe53e0..f13a36fc4d624 100644 --- a/execution/executor/src/chunk_executor.rs +++ b/execution/executor/src/chunk_executor/mod.rs @@ -5,18 +5,15 @@ #![forbid(unsafe_code)] use crate::{ - components::{ - apply_chunk_output::ApplyChunkOutput, - chunk_commit_queue::{ChunkCommitQueue, ChunkToUpdateLedger}, - chunk_output::ChunkOutput, - chunk_result_verifier::{ChunkResultVerifier, ReplayChunkVerifier, StateSyncChunkVerifier}, - do_ledger_update::DoLedgerUpdate, - executed_chunk::ExecutedChunk, - partial_state_compute_result::PartialStateComputeResult, - transaction_chunk::{ChunkToApply, ChunkToExecute, TransactionChunk}, - }, logging::{LogEntry, LogSchema}, metrics::{APPLY_CHUNK, CHUNK_OTHER_TIMERS, COMMIT_CHUNK, CONCURRENCY_GAUGE, EXECUTE_CHUNK}, + types::{ + executed_chunk::ExecutedChunk, partial_state_compute_result::PartialStateComputeResult, + }, + workflow::{ + do_get_execution_output::DoGetExecutionOutput, do_ledger_update::DoLedgerUpdate, + do_state_checkpoint::DoStateCheckpoint, + }, }; use anyhow::{anyhow, ensure, Result}; use aptos_executor_types::{ @@ -44,6 +41,8 @@ use aptos_types::{ write_set::WriteSet, }; use aptos_vm::VMExecutor; +use chunk_commit_queue::{ChunkCommitQueue, ChunkToUpdateLedger}; +use chunk_result_verifier::{ChunkResultVerifier, ReplayChunkVerifier, StateSyncChunkVerifier}; use fail::fail_point; use itertools::multizip; use std::{ @@ -55,6 +54,11 @@ use std::{ }, time::Instant, }; +use transaction_chunk::{ChunkToApply, ChunkToExecute, TransactionChunk}; + +pub mod chunk_commit_queue; +pub mod chunk_result_verifier; +pub mod transaction_chunk; pub struct ChunkExecutor { db: DbReaderWriter, @@ -301,14 +305,13 @@ impl ChunkExecutorInner { let chunk_output = chunk.into_output::(state_view)?; // Calculate state snapshot - let (result_state, next_epoch_state, state_checkpoint_output) = - ApplyChunkOutput::calculate_state_checkpoint( - chunk_output, - &self.commit_queue.lock().latest_state(), - None, // append_state_checkpoint_to_block - Some(chunk_verifier.state_checkpoint_hashes()), - false, // is_block - )?; + let (result_state, next_epoch_state, state_checkpoint_output) = DoStateCheckpoint::run( + chunk_output, + &self.commit_queue.lock().latest_state(), + None, // append_state_checkpoint_to_block + Some(chunk_verifier.state_checkpoint_hashes()), + false, // is_block + )?; // Enqueue for next stage. self.commit_queue @@ -597,7 +600,7 @@ impl ChunkExecutorInner { .collect::>(); // State sync executor shouldn't have block gas limit. - let chunk_output = ChunkOutput::by_transaction_execution::( + let chunk_output = DoGetExecutionOutput::by_transaction_execution::( txns.into(), state_view, BlockExecutorConfigFromOnchain::new_no_block_limit(), diff --git a/execution/executor/src/components/transaction_chunk.rs b/execution/executor/src/chunk_executor/transaction_chunk.rs similarity index 88% rename from execution/executor/src/components/transaction_chunk.rs rename to execution/executor/src/chunk_executor/transaction_chunk.rs index 41b995cfea428..b9f3b08205f16 100644 --- a/execution/executor/src/components/transaction_chunk.rs +++ b/execution/executor/src/chunk_executor/transaction_chunk.rs @@ -2,10 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{ - components::chunk_output::ChunkOutput, metrics::{CHUNK_OTHER_TIMERS, VM_EXECUTE_CHUNK}, + workflow::do_get_execution_output::DoGetExecutionOutput, }; use anyhow::Result; +use aptos_executor_types::execution_output::ExecutionOutput; use aptos_experimental_runtimes::thread_manager::optimal_min_len; use aptos_metrics_core::TimerHelper; use aptos_storage_interface::cached_state_view::CachedStateView; @@ -37,7 +38,7 @@ pub trait TransactionChunk { self.len() == 0 } - fn into_output(self, state_view: CachedStateView) -> Result; + fn into_output(self, state_view: CachedStateView) -> Result; } pub struct ChunkToExecute { @@ -54,7 +55,7 @@ impl TransactionChunk for ChunkToExecute { self.transactions.len() } - fn into_output(self, state_view: CachedStateView) -> Result { + fn into_output(self, state_view: CachedStateView) -> Result { let ChunkToExecute { transactions, first_version: _, @@ -76,7 +77,7 @@ impl TransactionChunk for ChunkToExecute { }; let _timer = VM_EXECUTE_CHUNK.start_timer(); - ChunkOutput::by_transaction_execution::( + DoGetExecutionOutput::by_transaction_execution::( sig_verified_txns.into(), state_view, BlockExecutorConfigFromOnchain::new_no_block_limit(), @@ -99,13 +100,13 @@ impl TransactionChunk for ChunkToApply { self.transactions.len() } - fn into_output(self, state_view: CachedStateView) -> Result { + fn into_output(self, state_view: CachedStateView) -> Result { let Self { transactions, transaction_outputs, first_version: _, } = self; - ChunkOutput::by_transaction_output(transactions, transaction_outputs, state_view) + DoGetExecutionOutput::by_transaction_output(transactions, transaction_outputs, state_view) } } diff --git a/execution/executor/src/db_bootstrapper.rs b/execution/executor/src/db_bootstrapper/mod.rs similarity index 92% rename from execution/executor/src/db_bootstrapper.rs rename to execution/executor/src/db_bootstrapper/mod.rs index d4bfeae878975..e7b0922a1a3d2 100644 --- a/execution/executor/src/db_bootstrapper.rs +++ b/execution/executor/src/db_bootstrapper/mod.rs @@ -4,7 +4,10 @@ #![forbid(unsafe_code)] -use crate::components::{chunk_output::ChunkOutput, executed_chunk::ExecutedChunk}; +use crate::{ + types::executed_chunk::ExecutedChunk, + workflow::{do_get_execution_output::DoGetExecutionOutput, ApplyExecutionOutput}, +}; use anyhow::{anyhow, ensure, format_err, Result}; use aptos_crypto::HashValue; use aptos_logger::prelude::*; @@ -128,13 +131,14 @@ pub fn calculate_genesis( get_state_epoch(&base_state_view)? }; - let (mut chunk, _, _) = ChunkOutput::by_transaction_execution::( + let execution_output = DoGetExecutionOutput::by_transaction_execution::( vec![genesis_txn.clone().into()].into(), base_state_view, BlockExecutorConfigFromOnchain::new_no_block_limit(), - )? - .apply_to_ledger(&executed_trees, None)?; - let output = &chunk.output; + )?; + + let output = ApplyExecutionOutput::run(execution_output, &executed_trees, None)?; + ensure!( output.expect_ledger_update_output().num_txns() != 0, "Genesis txn execution failed." @@ -163,7 +167,6 @@ pub fn calculate_genesis( "Genesis txn didn't output reconfig event." ); - let output = output.expect_complete_result(); let ledger_info_with_sigs = LedgerInfoWithSignatures::new( LedgerInfo::new( BlockInfo::new( @@ -171,7 +174,7 @@ pub fn calculate_genesis( GENESIS_ROUND, genesis_block_id(), output - .ledger_update_output + .expect_ledger_update_output() .transaction_accumulator .root_hash(), genesis_version, @@ -182,9 +185,12 @@ pub fn calculate_genesis( ), AggregateSignature::empty(), /* signatures */ ); - chunk.ledger_info_opt = Some(ledger_info_with_sigs); + let executed_chunk = ExecutedChunk { + output, + ledger_info_opt: Some(ledger_info_with_sigs), + }; - let committer = GenesisCommitter::new(db.writer.clone(), chunk)?; + let committer = GenesisCommitter::new(db.writer.clone(), executed_chunk)?; info!( "Genesis calculated: ledger_info_with_sigs {:?}, waypoint {:?}", &committer.output.ledger_info_opt, committer.waypoint, diff --git a/execution/executor/src/fuzzing.rs b/execution/executor/src/fuzzing.rs index 88500a7db1d00..3b656889dc844 100644 --- a/execution/executor/src/fuzzing.rs +++ b/execution/executor/src/fuzzing.rs @@ -4,11 +4,11 @@ use crate::{ block_executor::{BlockExecutor, TransactionBlockExecutor}, - components::chunk_output::ChunkOutput, + workflow::do_get_execution_output::DoGetExecutionOutput, }; use anyhow::Result; use aptos_crypto::{hash::SPARSE_MERKLE_PLACEHOLDER_HASH, HashValue}; -use aptos_executor_types::BlockExecutorTrait; +use aptos_executor_types::{execution_output::ExecutionOutput, BlockExecutorTrait}; use aptos_storage_interface::{ cached_state_view::CachedStateView, chunk_to_commit::ChunkToCommit, DbReader, DbReaderWriter, DbWriter, @@ -72,8 +72,12 @@ impl TransactionBlockExecutor for FakeVM { transactions: ExecutableTransactions, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result { - ChunkOutput::by_transaction_execution::(transactions, state_view, onchain_config) + ) -> Result { + DoGetExecutionOutput::by_transaction_execution::( + transactions, + state_view, + onchain_config, + ) } } diff --git a/execution/executor/src/lib.rs b/execution/executor/src/lib.rs index 41a8c3859281f..e03c9443c573a 100644 --- a/execution/executor/src/lib.rs +++ b/execution/executor/src/lib.rs @@ -9,11 +9,10 @@ pub mod fuzzing; mod logging; pub mod metrics; #[cfg(test)] -mod mock_vm; -#[cfg(test)] mod tests; pub mod block_executor; pub mod chunk_executor; -pub mod components; pub mod db_bootstrapper; +pub mod types; +pub mod workflow; diff --git a/execution/executor/src/tests/chunk_executor_tests.rs b/execution/executor/src/tests/chunk_executor_tests.rs index 46de72e5d06fd..9dc2583c09f63 100644 --- a/execution/executor/src/tests/chunk_executor_tests.rs +++ b/execution/executor/src/tests/chunk_executor_tests.rs @@ -8,8 +8,10 @@ use crate::{ block_executor::BlockExecutor, chunk_executor::ChunkExecutor, db_bootstrapper::{generate_waypoint, maybe_bootstrap}, - mock_vm::{encode_mint_transaction, MockVM}, - tests::{self, create_blocks_and_chunks, create_transaction_chunks}, + tests::{ + self, create_blocks_and_chunks, create_transaction_chunks, + mock_vm::{encode_mint_transaction, MockVM}, + }, }; use aptos_crypto::HashValue; use aptos_db::AptosDB; diff --git a/execution/executor/src/mock_vm/mock_vm_test.rs b/execution/executor/src/tests/mock_vm/mock_vm_test.rs similarity index 100% rename from execution/executor/src/mock_vm/mock_vm_test.rs rename to execution/executor/src/tests/mock_vm/mock_vm_test.rs diff --git a/execution/executor/src/mock_vm/mod.rs b/execution/executor/src/tests/mock_vm/mod.rs similarity index 97% rename from execution/executor/src/mock_vm/mod.rs rename to execution/executor/src/tests/mock_vm/mod.rs index bba3c7891da84..552764e8f329e 100644 --- a/execution/executor/src/mock_vm/mod.rs +++ b/execution/executor/src/tests/mock_vm/mod.rs @@ -5,9 +5,13 @@ #[cfg(test)] mod mock_vm_test; -use crate::{block_executor::TransactionBlockExecutor, components::chunk_output::ChunkOutput}; +use crate::{ + block_executor::TransactionBlockExecutor, + workflow::do_get_execution_output::DoGetExecutionOutput, +}; use anyhow::Result; use aptos_crypto::{ed25519::Ed25519PrivateKey, PrivateKey, Uniform}; +use aptos_executor_types::execution_output::ExecutionOutput; use aptos_storage_interface::cached_state_view::CachedStateView; use aptos_types::{ account_address::AccountAddress, @@ -65,8 +69,12 @@ impl TransactionBlockExecutor for MockVM { transactions: ExecutableTransactions, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result { - ChunkOutput::by_transaction_execution::(transactions, state_view, onchain_config) + ) -> Result { + DoGetExecutionOutput::by_transaction_execution::( + transactions, + state_view, + onchain_config, + ) } } diff --git a/execution/executor/src/tests/mod.rs b/execution/executor/src/tests/mod.rs index e34cab4e0eccb..4a6118a488bc9 100644 --- a/execution/executor/src/tests/mod.rs +++ b/execution/executor/src/tests/mod.rs @@ -4,12 +4,8 @@ use crate::{ block_executor::BlockExecutor, - components::{chunk_output::ChunkOutput, executed_chunk::ExecutedChunk}, db_bootstrapper::{generate_waypoint, maybe_bootstrap}, - mock_vm::{ - encode_mint_transaction, encode_reconfiguration_transaction, encode_transfer_transaction, - MockVM, DISCARD_STATUS, KEEP_STATUS, - }, + workflow::{do_get_execution_output::DoGetExecutionOutput, ApplyExecutionOutput}, }; use aptos_crypto::{ed25519::Ed25519PrivateKey, HashValue, PrivateKey, SigningKey, Uniform}; use aptos_db::AptosDB; @@ -38,10 +34,16 @@ use aptos_types::{ write_set::{WriteOp, WriteSet, WriteSetMut}, }; use itertools::Itertools; +use mock_vm::{ + encode_mint_transaction, encode_reconfiguration_transaction, encode_transfer_transaction, + MockVM, DISCARD_STATUS, KEEP_STATUS, +}; use proptest::prelude::*; use std::{iter::once, sync::Arc}; mod chunk_executor_tests; +#[cfg(test)] +mod mock_vm; fn execute_and_commit_block( executor: &TestExecutor, @@ -486,18 +488,15 @@ fn apply_transaction_by_writeset( ) .unwrap(); - let chunk_output = ChunkOutput::by_transaction_output(txns, txn_outs, state_view).unwrap(); + let chunk_output = + DoGetExecutionOutput::by_transaction_output(txns, txn_outs, state_view).unwrap(); - let (executed, _, _) = chunk_output.apply_to_ledger(&ledger_view, None).unwrap(); - let ExecutedChunk { - output, - ledger_info_opt, - } = executed; + let output = ApplyExecutionOutput::run(chunk_output, &ledger_view, None).unwrap(); db.writer .save_transactions( output.expect_complete_result().as_chunk_to_commit(), - ledger_info_opt.as_ref(), + None, true, /* sync_commit */ ) .unwrap(); @@ -678,7 +677,7 @@ fn run_transactions_naive( for txn in transactions { let ledger_view: ExecutedTrees = db.reader.get_latest_executed_trees().unwrap(); - let out = ChunkOutput::by_transaction_execution::( + let out = DoGetExecutionOutput::by_transaction_execution::( vec![txn].into(), ledger_view .verified_state_view( @@ -690,15 +689,11 @@ fn run_transactions_naive( block_executor_onchain_config.clone(), ) .unwrap(); - let (executed, _, _) = out.apply_to_ledger(&ledger_view, None).unwrap(); - let ExecutedChunk { - output, - ledger_info_opt, - } = executed; + let output = ApplyExecutionOutput::run(out, &ledger_view, None).unwrap(); db.writer .save_transactions( output.expect_complete_result().as_chunk_to_commit(), - ledger_info_opt.as_ref(), + None, true, /* sync_commit */ ) .unwrap(); diff --git a/execution/executor/src/components/executed_chunk.rs b/execution/executor/src/types/executed_chunk.rs similarity index 82% rename from execution/executor/src/components/executed_chunk.rs rename to execution/executor/src/types/executed_chunk.rs index 50e8f0b185290..b72a77ce3337d 100644 --- a/execution/executor/src/components/executed_chunk.rs +++ b/execution/executor/src/types/executed_chunk.rs @@ -4,7 +4,7 @@ #![forbid(unsafe_code)] -use crate::components::partial_state_compute_result::PartialStateComputeResult; +use crate::types::partial_state_compute_result::PartialStateComputeResult; use aptos_types::ledger_info::LedgerInfoWithSignatures; #[derive(Debug)] diff --git a/execution/executor/src/components/in_memory_state_calculator_v2.rs b/execution/executor/src/types/in_memory_state_calculator_v2.rs similarity index 100% rename from execution/executor/src/components/in_memory_state_calculator_v2.rs rename to execution/executor/src/types/in_memory_state_calculator_v2.rs diff --git a/execution/executor/src/components/mod.rs b/execution/executor/src/types/mod.rs similarity index 60% rename from execution/executor/src/components/mod.rs rename to execution/executor/src/types/mod.rs index da78a4b5dff16..604b6b0cfb2f4 100644 --- a/execution/executor/src/components/mod.rs +++ b/execution/executor/src/types/mod.rs @@ -4,14 +4,7 @@ #![forbid(unsafe_code)] -pub mod apply_chunk_output; -pub mod block_tree; -pub mod chunk_commit_queue; -pub mod chunk_output; pub mod in_memory_state_calculator_v2; -pub mod chunk_result_verifier; -pub mod do_ledger_update; pub mod executed_chunk; pub mod partial_state_compute_result; -pub mod transaction_chunk; diff --git a/execution/executor/src/components/partial_state_compute_result.rs b/execution/executor/src/types/partial_state_compute_result.rs similarity index 100% rename from execution/executor/src/components/partial_state_compute_result.rs rename to execution/executor/src/types/partial_state_compute_result.rs diff --git a/execution/executor/src/components/chunk_output.rs b/execution/executor/src/workflow/do_get_execution_output.rs similarity index 86% rename from execution/executor/src/components/chunk_output.rs rename to execution/executor/src/workflow/do_get_execution_output.rs index 2e321b3bc2110..4245cf1e9252f 100644 --- a/execution/executor/src/components/chunk_output.rs +++ b/execution/executor/src/workflow/do_get_execution_output.rs @@ -1,66 +1,40 @@ -// Copyright © Aptos Foundation -// Parts of the project are originally copyright © Meta Platforms, Inc. +// Copyright (c) Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -#![forbid(unsafe_code)] - -use crate::{ - components::{apply_chunk_output::ApplyChunkOutput, executed_chunk::ExecutedChunk}, - metrics, -}; +use crate::metrics; use anyhow::Result; -use aptos_crypto::HashValue; use aptos_executor_service::{ local_executor_helper::SHARDED_BLOCK_EXECUTOR, remote_executor_client::{get_remote_addresses, REMOTE_SHARDED_BLOCK_EXECUTOR}, }; -use aptos_executor_types::state_checkpoint_output::StateCheckpointOutput; -use aptos_logger::{sample, sample::SampleRate, warn}; -use aptos_storage_interface::{ - cached_state_view::{CachedStateView, StateCache}, - state_delta::StateDelta, - ExecutedTrees, -}; +use aptos_executor_types::execution_output::ExecutionOutput; +use aptos_logger::prelude::*; +use aptos_storage_interface::cached_state_view::CachedStateView; use aptos_types::{ - account_config::CORE_CODE_ADDRESS, block_executor::{ config::BlockExecutorConfigFromOnchain, partitioner::{ExecutableTransactions, PartitionedTransactions}, }, contract_event::ContractEvent, - epoch_state::EpochState, transaction::{ authenticator::AccountAuthenticator, - block_epilogue::BlockEndInfo, signature_verified_transaction::{SignatureVerifiedTransaction, TransactionProvider}, BlockOutput, ExecutionStatus, Transaction, TransactionOutput, TransactionOutputProvider, TransactionStatus, }, }; use aptos_vm::{AptosVM, VMExecutor}; -use fail::fail_point; -use move_core_types::vm_status::StatusCode; -use std::{ops::Deref, sync::Arc, time::Duration}; +use move_core_types::{language_storage::CORE_CODE_ADDRESS, vm_status::StatusCode}; +use std::{sync::Arc, time::Duration}; -pub struct ChunkOutput { - /// Input transactions. - pub transactions: Vec, - /// Raw VM output. - pub transaction_outputs: Vec, - /// Carries the frozen base state view, so all in-mem nodes involved won't drop before the - /// execution result is processed; as well as all the accounts touched during execution, together - /// with their proofs. - pub state_cache: StateCache, - /// Optional StateCheckpoint payload - pub block_end_info: Option, -} +pub struct DoGetExecutionOutput; -impl ChunkOutput { +impl DoGetExecutionOutput { pub fn by_transaction_execution( transactions: ExecutableTransactions, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result { + ) -> Result { match transactions { ExecutableTransactions::Unsharded(txns) => { Self::by_transaction_execution_unsharded::(txns, state_view, onchain_config) @@ -75,11 +49,11 @@ impl ChunkOutput { transactions: Vec, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result { + ) -> Result { let block_output = Self::execute_block::(&transactions, &state_view, onchain_config)?; let (transaction_outputs, block_end_info) = block_output.into_inner(); - Ok(Self { + Ok(ExecutionOutput { transactions: transactions.into_iter().map(|t| t.into_inner()).collect(), transaction_outputs, state_cache: state_view.into_state_cache(), @@ -91,7 +65,7 @@ impl ChunkOutput { transactions: PartitionedTransactions, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> Result { + ) -> Result { let state_view_arc = Arc::new(state_view); let transaction_outputs = Self::execute_block_sharded::( transactions.clone(), @@ -104,7 +78,7 @@ impl ChunkOutput { // Unwrapping here is safe because the execution has finished and it is guaranteed that // the state view is not used anymore. let state_view = Arc::try_unwrap(state_view_arc).unwrap(); - Ok(Self { + Ok(ExecutionOutput { transactions: PartitionedTransactions::flatten(transactions) .into_iter() .map(|t| t.into_txn().into_inner()) @@ -119,7 +93,7 @@ impl ChunkOutput { transactions: Vec, transaction_outputs: Vec, state_view: CachedStateView, - ) -> Result { + ) -> Result { update_counters_for_processed_chunk(&transactions, &transaction_outputs, "output"); // collect all accounts touched and dedup @@ -131,7 +105,7 @@ impl ChunkOutput { // prime the state cache by fetching all touched accounts state_view.prime_cache_by_write_set(write_set)?; - Ok(Self { + Ok(ExecutionOutput { transactions, transaction_outputs, state_cache: state_view.into_state_cache(), @@ -139,39 +113,6 @@ impl ChunkOutput { }) } - pub fn apply_to_ledger( - self, - base_view: &ExecutedTrees, - known_state_checkpoint_hashes: Option>>, - ) -> Result<(ExecutedChunk, Vec, Vec)> { - fail_point!("executor::apply_to_ledger", |_| { - Err(anyhow::anyhow!("Injected error in apply_to_ledger.")) - }); - ApplyChunkOutput::apply_chunk(self, base_view, known_state_checkpoint_hashes) - } - - pub fn into_state_checkpoint_output( - self, - parent_state: &StateDelta, - block_id: HashValue, - ) -> Result<(Arc, Option, StateCheckpointOutput)> { - fail_point!("executor::into_state_checkpoint_output", |_| { - Err(anyhow::anyhow!( - "Injected error in into_state_checkpoint_output." - )) - }); - - // TODO(msmouse): If this code path is only used by block_executor, consider move it to the - // caller side. - ApplyChunkOutput::calculate_state_checkpoint( - self, - parent_state, - Some(block_id), - None, - /*is_block=*/ true, - ) - } - fn execute_block_sharded( partitioned_txns: PartitionedTransactions, state_view: Arc, @@ -179,14 +120,14 @@ impl ChunkOutput { ) -> Result> { if !get_remote_addresses().is_empty() { Ok(V::execute_block_sharded( - REMOTE_SHARDED_BLOCK_EXECUTOR.lock().deref(), + &REMOTE_SHARDED_BLOCK_EXECUTOR.lock(), partitioned_txns, state_view, onchain_config, )?) } else { Ok(V::execute_block_sharded( - SHARDED_BLOCK_EXECUTOR.lock().deref(), + &SHARDED_BLOCK_EXECUTOR.lock(), partitioned_txns, state_view, onchain_config, diff --git a/execution/executor/src/components/do_ledger_update.rs b/execution/executor/src/workflow/do_ledger_update.rs similarity index 98% rename from execution/executor/src/components/do_ledger_update.rs rename to execution/executor/src/workflow/do_ledger_update.rs index 2fe989faea05b..0a9e6988a5503 100644 --- a/execution/executor/src/components/do_ledger_update.rs +++ b/execution/executor/src/workflow/do_ledger_update.rs @@ -1,7 +1,9 @@ // Copyright (c) Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -use crate::{components::chunk_output::update_counters_for_processed_chunk, metrics::OTHER_TIMERS}; +use crate::{ + metrics::OTHER_TIMERS, workflow::do_get_execution_output::update_counters_for_processed_chunk, +}; use anyhow::Result; use aptos_crypto::{hash::CryptoHash, HashValue}; use aptos_executor_types::{ diff --git a/execution/executor/src/components/apply_chunk_output.rs b/execution/executor/src/workflow/do_state_checkpoint.rs similarity index 79% rename from execution/executor/src/components/apply_chunk_output.rs rename to execution/executor/src/workflow/do_state_checkpoint.rs index 42f091b5564fd..b0f8550e5c05b 100644 --- a/execution/executor/src/components/apply_chunk_output.rs +++ b/execution/executor/src/workflow/do_state_checkpoint.rs @@ -1,49 +1,41 @@ -// Copyright © Aptos Foundation -// Parts of the project are originally copyright © Meta Platforms, Inc. +// Copyright (c) Aptos Foundation // SPDX-License-Identifier: Apache-2.0 -#![forbid(unsafe_code)] - use crate::{ - components::{ - chunk_output::ChunkOutput, do_ledger_update::DoLedgerUpdate, executed_chunk::ExecutedChunk, - in_memory_state_calculator_v2::InMemoryStateCalculatorV2, - partial_state_compute_result::PartialStateComputeResult, - }, metrics::{EXECUTOR_ERRORS, OTHER_TIMERS}, + types::in_memory_state_calculator_v2::InMemoryStateCalculatorV2, }; -use anyhow::Result; use aptos_crypto::HashValue; use aptos_executor_types::{ + execution_output::ExecutionOutput, parsed_transaction_output::TransactionsWithParsedOutput, state_checkpoint_output::{StateCheckpointOutput, TransactionsByStatus}, ParsedTransactionOutput, }; use aptos_logger::error; use aptos_metrics_core::TimerHelper; -use aptos_storage_interface::{state_delta::StateDelta, ExecutedTrees}; +use aptos_storage_interface::state_delta::StateDelta; use aptos_types::{ epoch_state::EpochState, transaction::{ - block_epilogue::{BlockEndInfo, BlockEpiloguePayload}, - ExecutionStatus, Transaction, TransactionAuxiliaryData, TransactionOutput, - TransactionStatus, + BlockEndInfo, BlockEpiloguePayload, ExecutionStatus, Transaction, TransactionAuxiliaryData, + TransactionOutput, TransactionStatus, }, write_set::WriteSet, }; use std::{iter::repeat, sync::Arc}; -pub struct ApplyChunkOutput; +pub struct DoStateCheckpoint; -impl ApplyChunkOutput { - pub fn calculate_state_checkpoint( - chunk_output: ChunkOutput, +impl DoStateCheckpoint { + pub fn run( + chunk_output: ExecutionOutput, parent_state: &StateDelta, append_state_checkpoint_to_block: Option, known_state_checkpoints: Option>>, is_block: bool, - ) -> Result<(Arc, Option, StateCheckpointOutput)> { - let ChunkOutput { + ) -> anyhow::Result<(Arc, Option, StateCheckpointOutput)> { + let ExecutionOutput { state_cache, transactions, transaction_outputs, @@ -108,44 +100,12 @@ impl ApplyChunkOutput { )) } - pub fn apply_chunk( - chunk_output: ChunkOutput, - base_view: &ExecutedTrees, - known_state_checkpoint_hashes: Option>>, - ) -> Result<(ExecutedChunk, Vec, Vec)> { - let (result_state, next_epoch_state, state_checkpoint_output) = - Self::calculate_state_checkpoint( - chunk_output, - base_view.state(), - None, // append_state_checkpoint_to_block - known_state_checkpoint_hashes, - /*is_block=*/ false, - )?; - let (ledger_update_output, to_discard, to_retry) = - DoLedgerUpdate::run(state_checkpoint_output, base_view.txn_accumulator().clone())?; - let output = PartialStateComputeResult::new( - base_view.state().clone(), - result_state, - next_epoch_state, - ); - output.set_ledger_update_output(ledger_update_output); - - Ok(( - ExecutedChunk { - output, - ledger_info_opt: None, - }, - to_discard, - to_retry, - )) - } - fn sort_transactions_with_state_checkpoint( mut transactions: Vec, transaction_outputs: Vec, append_state_checkpoint_to_block: Option, block_end_info: Option, - ) -> Result<( + ) -> anyhow::Result<( bool, Vec, TransactionsWithParsedOutput, diff --git a/execution/executor/src/workflow/mod.rs b/execution/executor/src/workflow/mod.rs new file mode 100644 index 0000000000000..a9eb9ec5f7d66 --- /dev/null +++ b/execution/executor/src/workflow/mod.rs @@ -0,0 +1,45 @@ +// Copyright © Aptos Foundation +// Parts of the project are originally copyright © Meta Platforms, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#![forbid(unsafe_code)] + +use crate::types::partial_state_compute_result::PartialStateComputeResult; +use anyhow::Result; +use aptos_crypto::HashValue; +use aptos_executor_types::execution_output::ExecutionOutput; +use aptos_storage_interface::ExecutedTrees; +use do_ledger_update::DoLedgerUpdate; +use do_state_checkpoint::DoStateCheckpoint; + +pub mod do_get_execution_output; +pub mod do_ledger_update; +pub mod do_state_checkpoint; + +pub struct ApplyExecutionOutput; + +impl ApplyExecutionOutput { + pub fn run( + chunk_output: ExecutionOutput, + base_view: &ExecutedTrees, + known_state_checkpoint_hashes: Option>>, + ) -> Result { + let (result_state, next_epoch_state, state_checkpoint_output) = DoStateCheckpoint::run( + chunk_output, + base_view.state(), + None, // append_state_checkpoint_to_block + known_state_checkpoint_hashes, + /*is_block=*/ false, + )?; + let (ledger_update_output, _to_discard, _to_retry) = + DoLedgerUpdate::run(state_checkpoint_output, base_view.txn_accumulator().clone())?; + let output = PartialStateComputeResult::new( + base_view.state().clone(), + result_state, + next_epoch_state, + ); + output.set_ledger_update_output(ledger_update_output); + + Ok(output) + } +} diff --git a/experimental/execution/ptx-executor/Cargo.toml b/experimental/execution/ptx-executor/Cargo.toml index 0e791a70d6d19..5e4b035116772 100644 --- a/experimental/execution/ptx-executor/Cargo.toml +++ b/experimental/execution/ptx-executor/Cargo.toml @@ -15,6 +15,7 @@ rust-version = { workspace = true } [dependencies] anyhow = { workspace = true } aptos-executor = { workspace = true } +aptos-executor-types = { workspace = true } aptos-experimental-runtimes = { workspace = true } aptos-infallible = { workspace = true } aptos-logger = { workspace = true } diff --git a/experimental/execution/ptx-executor/src/lib.rs b/experimental/execution/ptx-executor/src/lib.rs index e41474632c007..c1f48b7d5634a 100644 --- a/experimental/execution/ptx-executor/src/lib.rs +++ b/experimental/execution/ptx-executor/src/lib.rs @@ -22,8 +22,10 @@ use crate::{ scheduler::PtxScheduler, sorter::PtxSorter, state_reader::PtxStateReader, }; use aptos_executor::{ - block_executor::TransactionBlockExecutor, components::chunk_output::ChunkOutput, + block_executor::TransactionBlockExecutor, + workflow::do_get_execution_output::DoGetExecutionOutput, }; +use aptos_executor_types::execution_output::ExecutionOutput; use aptos_experimental_runtimes::thread_manager::THREAD_MANAGER; use aptos_infallible::Mutex; use aptos_metrics_core::TimerHelper; @@ -119,8 +121,8 @@ impl TransactionBlockExecutor for PtxBlockExecutor { transactions: ExecutableTransactions, state_view: CachedStateView, onchain_config: BlockExecutorConfigFromOnchain, - ) -> anyhow::Result { - ChunkOutput::by_transaction_execution::( + ) -> anyhow::Result { + DoGetExecutionOutput::by_transaction_execution::( transactions, state_view, onchain_config, diff --git a/storage/aptosdb/src/state_store/mod.rs b/storage/aptosdb/src/state_store/mod.rs index a62226d3e6328..4288fcc8bebb6 100644 --- a/storage/aptosdb/src/state_store/mod.rs +++ b/storage/aptosdb/src/state_store/mod.rs @@ -40,7 +40,7 @@ use aptos_db_indexer_schemas::{ metadata::{MetadataKey, MetadataValue, StateSnapshotProgress}, schema::indexer_metadata::InternalIndexerMetadataSchema, }; -use aptos_executor::components::in_memory_state_calculator_v2::InMemoryStateCalculatorV2; +use aptos_executor::types::in_memory_state_calculator_v2::InMemoryStateCalculatorV2; use aptos_experimental_runtimes::thread_manager::THREAD_MANAGER; use aptos_infallible::Mutex; use aptos_jellyfish_merkle::iterator::JellyfishMerkleIterator;