Check mrenclave and scheduled height before produce blocks(fix #1295)

Author: DogLi

tee-worker/Cargo.lock

@@ -56,7 +56,7 @@ impl ScheduledEnclaveUpdate {
 			sidechain_block_number,
 			mr_enclave,
 		};
-		let old_enclaves = GLOBAL_SIDECHAIN_SCHEDULED_ENCLABES.get()?;
+		let old_enclaves = GLOBAL_SIDECHAIN_SCHEDULED_ENCLABES.get().unwrap_or_default();
 		// unwrap is safe here, because GLOBAL_SIDECHAIN_SCHEDULED_ENCLABES is initialized in `init_enclave()`
 		let mut scheduled_enclaves = Arc::<ScheduledEnclaves>::try_unwrap(old_enclaves).unwrap();
 		scheduled_enclaves.add_scheduled_enclave(scheduled_enclave)?;

tee-worker/core/parentchain/indirect-calls-executor/src/executor/litentry/get_scheduled_enclave.rs

@@ -126,6 +126,7 @@ itp-top-pool-author = { path = "../core-primitives/top-pool-author", default-fea
 itp-types = { path = "../core-primitives/types", default-features = false }
 itp-utils = { path = "../core-primitives/utils", default-features = false, features = ["sgx"] }
 its-block-verification = { path = "../sidechain/block-verification", default-features = false }
+its-consensus-common = { path = "../sidechain/consensus/common", default-features = false, features = ["sgx"] }
 its-primitives = { path = "../sidechain/primitives", default-features = false }
 its-sidechain = { path = "../sidechain/sidechain-crate", default-features = false, features = ["sgx"] }
 

tee-worker/enclave-runtime/Cargo.toml

@@ -60,6 +60,7 @@ use itc_tls_websocket_server::{
 use itp_attestation_handler::IntelAttestationHandler;
 use itp_component_container::{ComponentGetter, ComponentInitializer};
 use itp_enclave_scheduled::{ScheduledEnclaveHandle, ScheduledEnclaves};
+use itp_ocall_api::EnclaveAttestationOCallApi;
 use itp_primitives_cache::GLOBAL_PRIMITIVES_CACHE;
 use itp_settings::files::STATE_SNAPSHOTS_CACHE_SIZE;
 use itp_sgx_crypto::{aes, ed25519, rsa3072, AesSeal, Ed25519Seal, Rsa3072Seal};
@@ -72,6 +73,7 @@ use itp_stf_state_handler::{
 use itp_top_pool::pool::Options as PoolOptions;
 use itp_top_pool_author::author::AuthorTopFilter;
 use itp_types::ShardIdentifier;
+use its_consensus_common::block_production_suspension::set_global_block_suspender;
 use its_sidechain::block_composer::BlockComposer;
 use log::*;
 use sp_core::crypto::Pair;
@@ -130,6 +132,9 @@ pub(crate) fn init_enclave(mu_ra_url: String, untrusted_worker_url: String) -> E
 	let ocall_api = Arc::new(OcallApi);
 	GLOBAL_OCALL_API_COMPONENT.initialize(ocall_api.clone());
 
+	let mr_enclave = ocall_api.get_mrenclave_of_self()?.m;
+	set_global_block_suspender(false, mr_enclave);
+
 	// For debug purposes, list shards. no problem to panic if fails.
 	let shards = state_handler.list_shards().unwrap();
 	debug!("found the following {} shards on disk:", shards.len());

tee-worker/enclave-runtime/src/initialization/mod.rs

@@ -20,8 +20,8 @@ use crate::error::{Error, ServiceResult};
 use codec::Encode;
 use itp_enclave_api::{enclave_base::EnclaveBase, Enclave};
 use itp_settings::files::{
-	LAST_SLOT_BIN, LIGHT_CLIENT_DB, SHARDS_PATH, SHIELDING_KEY_FILE, SIDECHAIN_STORAGE_PATH,
-	SIGNING_KEY_FILE,
+	LAST_SLOT_BIN, LIGHT_CLIENT_DB, SCHEDULED_ENCLAVE_FILE, SHARDS_PATH, SHIELDING_KEY_FILE,
+	SIDECHAIN_STORAGE_PATH, SIGNING_KEY_FILE,
 };
 use itp_types::ShardIdentifier;
 use log::*;
@@ -49,6 +49,7 @@ pub(crate) fn initialize_shard_and_keys(
 	println!("[+] Generate key files");
 	generate_signing_key_file(enclave);
 	generate_shielding_key_file(enclave);
+	generate_scheduled_enclave_file();
 
 	Ok(())
 }
@@ -96,6 +97,14 @@ pub(crate) fn generate_signing_key_file(enclave: &Enclave) {
 	}
 }
 
+pub(crate) fn generate_scheduled_enclave_file() {
+	info!("*** Get scheduled enclaves from the TEE\n");
+	let path = Path::new(SCHEDULED_ENCLAVE_FILE);
+	if !path.exists() {
+		let _file = File::create(SCHEDULED_ENCLAVE_FILE).unwrap();
+	}
+}
+
 pub(crate) fn generate_shielding_key_file(enclave: &Enclave) {
 	info!("*** Get the public key from the TEE\n");
 	let pubkey = enclave.get_rsa_shielding_pubkey().unwrap();
@@ -117,6 +126,7 @@ fn purge_files(root_directory: &Path) -> ServiceResult<()> {
 
 	remove_file_if_it_exists(root_directory, LAST_SLOT_BIN)?;
 	remove_file_if_it_exists(root_directory, LIGHT_CLIENT_DB)?;
+	remove_file_if_it_exists(root_directory, SCHEDULED_ENCLAVE_FILE)?;
 	remove_file_if_it_exists(root_directory, light_client_backup_file().as_str())?;
 
 	Ok(())

tee-worker/service/src/setup.rs

@@ -20,7 +20,10 @@ sp-runtime = { default-features = false, git = "https://github.com/paritytech/su
 # local deps
 ita-stf = { path = "../../../app-libs/stf", default-features = false }
 itc-parentchain-block-import-dispatcher = { path = "../../../core/parentchain/block-import-dispatcher", default-features = false }
+itc-parentchain-indirect-calls-executor = { path = "../../../core/parentchain/indirect-calls-executor", default-features = false }
+itp-component-container = { path = "../../../core-primitives/component-container", default-features = false, optional = true }
 itp-enclave-metrics = { path = "../../../core-primitives/enclave-metrics", default-features = false }
+itp-enclave-scheduled = { path = "../../../core-primitives/sgx/enclave-scheduled", default-features = false, optional = true }
 itp-ocall-api = { path = "../../../core-primitives/ocall-api", default-features = false }
 itp-settings = { path = "../../../core-primitives/settings" }
 itp-sgx-crypto = { path = "../../../core-primitives/sgx/crypto", default-features = false }
@@ -61,6 +64,9 @@ sgx = [
     "itp-stf-executor/sgx",
     "itp-stf-state-handler/sgx",
     "itp-time-utils/sgx",
+    "itc-parentchain-indirect-calls-executor/sgx",
+    "itp-component-container/sgx",
+    "itp-enclave-scheduled/sgx",
     "itp-utils/sgx",
     "its-block-composer/sgx",
     "its-consensus-common/sgx",
@@ -80,6 +86,9 @@ std = [
     #local
     "ita-stf/std",
     "itc-parentchain-block-import-dispatcher/std",
+    "itc-parentchain-indirect-calls-executor/std",
+    "itp-enclave-scheduled/std",
+    "itp-component-container/std",
     "itp-enclave-metrics/std",
     "itp-ocall-api/std",
     "itp-sgx-crypto/std",

tee-worker/sidechain/consensus/aura/Cargo.toml

@@ -23,15 +23,23 @@ pub use its_consensus_common::BlockImport;
 use crate::{AuraVerifier, EnclaveOnChainOCallApi, SidechainBlockTrait};
 use ita_stf::hash::TrustedOperationOrHash;
 use itc_parentchain_block_import_dispatcher::triggered_dispatcher::TriggerParentchainBlockImport;
+use itc_parentchain_indirect_calls_executor::executor::litentry::get_scheduled_enclave::GLOBAL_SIDECHAIN_SCHEDULED_ENCLABES;
+use itp_component_container::component_container::ComponentGetter;
 use itp_enclave_metrics::EnclaveMetric;
+use itp_enclave_scheduled::ScheduledEnclaveHandle;
 use itp_ocall_api::{EnclaveMetricsOCallApi, EnclaveSidechainOCallApi};
 use itp_settings::sidechain::SLOT_DURATION;
 use itp_sgx_crypto::{key_repository::AccessKey, StateCrypto};
 use itp_sgx_externalities::SgxExternalities;
 use itp_stf_state_handler::handle_state::HandleState;
 use itp_top_pool_author::traits::{AuthorApi, OnBlockImported};
-use itp_types::H256;
-use its_consensus_common::Error as ConsensusError;
+use itp_types::{ShardIdentifier, H256};
+use its_consensus_common::{
+	block_production_suspension::{
+		BlockProductionSuspender, SuspendBlockProductionTrait, GLOBAL_BLOCK_SUSPENDER,
+	},
+	Error as ConsensusError,
+};
 use its_primitives::traits::{
 	BlockData, Header as HeaderTrait, ShardIdentifierFor, SignedBlock as SignedBlockTrait,
 };
@@ -193,6 +201,7 @@ impl<
 	type SidechainState = SgxExternalities;
 	type StateCrypto = <StateKeyRepository as AccessKey>::KeyType;
 	type Context = OCallApi;
+	type BlockSuspender = BlockProductionSuspender;
 
 	fn verifier(
 		&self,
@@ -236,9 +245,40 @@ impl<
 	where
 		F: FnOnce(&Self::SidechainState) -> Result<SignedSidechainBlock, ConsensusError>,
 	{
-		self.state_handler
+		let sidechain_block = self
+			.state_handler
 			.execute_on_current(shard, |state, _| verifying_function(state))
-			.map_err(|e| ConsensusError::Other(format!("{:?}", e).into()))?
+			.map_err(|e| ConsensusError::Other(format!("{:?}", e).into()))??;
+		let block_number = sidechain_block.block().header().block_number();
+		// get schedule block_number and mr_enclave
+		let scheduled_enclaves = GLOBAL_SIDECHAIN_SCHEDULED_ENCLABES
+			.get()
+			.map_err(|_| ConsensusError::GetScheduledEnclavesFailed)?;
+		if let Some(scheduled_enclave) = scheduled_enclaves.get_next_scheduled_enclave(block_number)
+		{
+			let block_suspender = self.block_suspender();
+			let scheduled_mr_enclave = scheduled_enclave.mr_enclave;
+			let scheduled_block_number = scheduled_enclave.sidechain_block_number;
+			let current_mr_enclave = block_suspender
+				.current_mr_enclave()
+				.map_err(|_| ConsensusError::GetMrEnclaveFailed)?;
+			if current_mr_enclave != scheduled_mr_enclave {
+				if scheduled_block_number <= block_number {
+					block_suspender
+						.suspend_for_production()
+						.map_err(|_| ConsensusError::SetBlockSuspenderFailed)?;
+					warn!("need to update, reason: enclave is outdated");
+				}
+				if scheduled_block_number == block_number {
+					let old_id = ShardIdentifier::from_slice(&current_mr_enclave[..]);
+					let new_id = ShardIdentifier::from_slice(&scheduled_mr_enclave[..]);
+					self.state_handler
+						.migrate_shard(old_id, new_id)
+						.map_err(|_| ConsensusError::MigrationFailed)?;
+				}
+			}
+		}
+		Ok(sidechain_block)
 	}
 
 	fn state_key(&self) -> Result<Self::StateCrypto, ConsensusError> {
@@ -251,6 +291,10 @@ impl<
 		&self.ocall_api
 	}
 
+	fn block_suspender(&self) -> Self::BlockSuspender {
+		GLOBAL_BLOCK_SUSPENDER.clone()
+	}
+
 	fn import_parentchain_block(
 		&self,
 		sidechain_block: &SignedSidechainBlock::Block,

tee-worker/sidechain/consensus/aura/src/block_importer.rs

@@ -34,7 +34,10 @@ use itc_parentchain_block_import_dispatcher::triggered_dispatcher::TriggerParent
 use itp_ocall_api::EnclaveOnChainOCallApi;
 use itp_time_utils::duration_now;
 use its_block_verification::slot::slot_author;
-use its_consensus_common::{Environment, Error as ConsensusError, Proposer};
+use its_consensus_common::{
+	block_production_suspension::{BlockProductionSuspender, GLOBAL_BLOCK_SUSPENDER},
+	Environment, Error as ConsensusError, Proposer,
+};
 use its_consensus_slots::{SimpleSlotWorker, Slot, SlotInfo};
 use its_primitives::{
 	traits::{Block as SidechainBlockTrait, Header as HeaderTrait, SignedBlock},
@@ -137,6 +140,7 @@ where
 	type Claim = AuthorityPair::Public;
 	type EpochData = Vec<AuthorityId<AuthorityPair>>;
 	type Output = SignedSidechainBlock;
+	type BlockSuspender = BlockProductionSuspender;
 
 	fn logging_target(&self) -> &'static str {
 		"aura"
@@ -187,6 +191,10 @@ where
 		self.environment.init(header, shard)
 	}
 
+	fn block_suspender(&mut self) -> Self::BlockSuspender {
+		GLOBAL_BLOCK_SUSPENDER.clone()
+	}
+
 	fn proposing_remaining_duration(&self, slot_info: &SlotInfo<ParentchainBlock>) -> Duration {
 		proposing_remaining_duration(slot_info, duration_now())
 	}

tee-worker/sidechain/consensus/aura/src/lib.rs

@@ -19,10 +19,13 @@ use crate::{block_importer::BlockImporter, test::fixtures::validateer, ShardIden
 use codec::Encode;
 use core::assert_matches::assert_matches;
 use itc_parentchain_block_import_dispatcher::trigger_parentchain_block_import_mock::TriggerParentchainBlockImportMock;
+use itc_parentchain_indirect_calls_executor::executor::litentry::get_scheduled_enclave::GLOBAL_SIDECHAIN_SCHEDULED_ENCLABES;
 use itc_parentchain_test::{
 	parentchain_block_builder::ParentchainBlockBuilder,
 	parentchain_header_builder::ParentchainHeaderBuilder,
 };
+use itp_component_container::ComponentInitializer;
+use itp_enclave_scheduled::ScheduledEnclaves;
 use itp_sgx_crypto::{aes::Aes, mocks::KeyRepositoryMock, StateCrypto};
 use itp_sgx_externalities::SgxExternalitiesDiffType;
 use itp_stf_state_handler::handle_state::HandleState;
@@ -146,15 +149,16 @@ fn default_authority_signed_block(
 
 #[test]
 fn simple_block_import_works() {
+	let scheduled_enclaves = ScheduledEnclaves::default();
+	GLOBAL_SIDECHAIN_SCHEDULED_ENCLABES.initialize(Arc::new(scheduled_enclaves));
 	let parentchain_header = ParentchainHeaderBuilder::default().build();
 	let (block_importer, state_handler, _) =
 		test_fixtures_with_default_import_trigger(&parentchain_header);
 	let signed_sidechain_block =
 		default_authority_signed_block(&parentchain_header, state_handler.as_ref());
 
-	block_importer
-		.import_block(signed_sidechain_block, &parentchain_header)
-		.unwrap();
+	let result = block_importer.import_block(signed_sidechain_block, &parentchain_header);
+	assert!(result.is_ok());
 }
 
 #[test]

tee-worker/sidechain/consensus/aura/src/test/block_importer_tests.rs

@@ -6,12 +6,15 @@ version = "0.9.0"
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false }
+lazy_static = { version = "1.1.0", features = ["spin_no_std"] }
 log = { version = "0.4", default-features = false }
 thiserror = { version = "1.0.26", optional = true }
 
 # local deps
+itc-parentchain-indirect-calls-executor = { path = "../../../core/parentchain/indirect-calls-executor", default-features = false }
 itc-parentchain-light-client = { path = "../../../core/parentchain/light-client", default-features = false }
 itp-block-import-queue = { path = "../../../core-primitives/block-import-queue", default-features = false }
+itp-component-container = { path = "../../../core-primitives/component-container", default-features = false, optional = true }
 itp-extrinsics-factory = { path = "../../../core-primitives/extrinsics-factory", default-features = false }
 itp-node-api-metadata = { path = "../../../core-primitives/node-api/metadata", default-features = false }
 itp-node-api-metadata-provider = { path = "../../../core-primitives/node-api/metadata-provider", default-features = false }
@@ -50,10 +53,12 @@ sgx = [
     "itc-parentchain-light-client/sgx",
     "itp-block-import-queue/sgx",
     "itp-extrinsics-factory/sgx",
+    "itp-component-container/sgx",
     "itp-node-api-metadata-provider/sgx",
     "itp-sgx-crypto/sgx",
     "itp-sgx-externalities/sgx",
     "its-state/sgx",
+    "itc-parentchain-indirect-calls-executor/sgx",
     # scs
     "its-block-verification/sgx",
 ]
@@ -63,10 +68,12 @@ std = [
     "thiserror",
     # local
     "itc-parentchain-light-client/std",
+    "itp-component-container/std",
     "itp-block-import-queue/std",
     "itp-extrinsics-factory/std",
     "itp-node-api-metadata/std",
     "itp-node-api-metadata-provider/std",
+    "itc-parentchain-indirect-calls-executor/std",
     "itp-ocall-api/std",
     "itp-sgx-crypto/std",
     "itp-sgx-externalities/std",