PoC: Broadcast blocks

Cargo.toml

@@ -1,13 +1,17 @@
 [workspace]
+resolver = "2"
 members = [
     "client",
     "primitives/api-client-extensions",
-    "primitives/settings",
+    "primitives/enclave-api",
+    "primitives/enclave-api/ffi",
     "primitives/node",
+    "primitives/settings",
     "primitives/worker",
     "stf",
     "worker",
-    "worker/worker-api",
+    "worker/rpc/client",
+    "worker/rpc/server",
     #	 "enclave",
 ]
 

client/Cargo.toml

@@ -80,7 +80,7 @@ path = "../primitives/api-client-extensions"
 path = "../stf"
 
 [dependencies.substratee-worker-api]
-path = "../worker/worker-api"
+path = "../worker/rpc/client"
 
 [dependencies.sp-keyring]
 git = "https://github.com/paritytech/substrate.git"

client/src/main.rs

@@ -64,7 +64,7 @@ use substrate_api_client::{
 
 use substrate_client_keystore::{LocalKeystore, KeystoreExt};
 use substratee_stf::{ShardIdentifier, TrustedCallSigned, TrustedOperation};
-use substratee_worker_api::direct_client::DirectApi as DirectWorkerApi;
+use substratee_worker_api::direct_client::{DirectClient as DirectWorkerApi, DirectApi};
 use substratee_api_client_extensions::SubstrateeRegistryApi;
 use substratee_worker_primitives::{DirectRequestStatus, RpcRequest, RpcResponse, RpcReturnValue};
 
@@ -311,7 +311,7 @@ fn main() {
                         println!("   AccountId: {}", enclave.pubkey.to_ss58check());
                         println!("   MRENCLAVE: {}", enclave.mr_enclave.to_base58());
                         println!("   RA timestamp: {}", timestamp);
-                        println!("   URL: {}", String::from_utf8(enclave.url).unwrap());
+                        println!("   URL: {}", enclave.url);
                     }
                     Ok(())
                 }),

enclave/Cargo.lock

@@ -973,6 +973,12 @@ dependencies = [
  "sgx_tstd",
 ]
 
+[[package]]
+name = "itoa"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
+
 [[package]]
 name = "jsonrpc-core"
 version = "16.1.0"
@@ -982,7 +988,7 @@ dependencies = [
  "log",
  "serde 1.0.118",
  "serde_derive",
- "serde_json",
+ "serde_json 1.0.60",
  "sp-std",
 ]
 
@@ -1867,12 +1873,23 @@ name = "serde_json"
 version = "1.0.60"
 source = "git+https://github.com/mesalock-linux/serde-json-sgx#380893814ad2a057758d825bab798aa117f7362a"
 dependencies = [
- "itoa",
+ "itoa 0.4.5",
  "ryu",
  "serde 1.0.118",
  "sgx_tstd",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
+dependencies = [
+ "itoa 0.4.7",
+ "ryu",
+ "serde 1.0.126",
+]
+
 [[package]]
 name = "sgx-externalities"
 version = "0.3.0"
@@ -2639,7 +2656,7 @@ dependencies = [
  "rustls",
  "serde 1.0.118",
  "serde_derive",
- "serde_json",
+ "serde_json 1.0.60",
  "sgx-externalities",
  "sgx_rand",
  "sgx_serialize",
@@ -2677,6 +2694,7 @@ dependencies = [
  "chrono 0.4.19",
  "parity-scale-codec",
  "primitive-types",
+ "serde_json 1.0.64",
  "sgx_tstd",
  "sp-core",
  "sp-runtime",

enclave/Enclave.edl

@@ -67,6 +67,13 @@ enclave {
 			[out, size=unchecked_extrinsic_size] uint8_t* unchecked_extrinsic, uint32_t unchecked_extrinsic_size
 		);
 
+		public sgx_status_t mock_register_enclave_xt(
+            [in, size=genesis_hash_size] uint8_t* genesis_hash, uint32_t genesis_hash_size,
+            [in] uint32_t* nonce,
+            [in, size=w_url_size] uint8_t* w_url, uint32_t w_url_size,
+            [out, size=unchecked_extrinsic_size] uint8_t* unchecked_extrinsic, uint32_t unchecked_extrinsic_size
+		);
+
 		public sgx_status_t dump_ra_to_disk();
 
 		public sgx_status_t run_key_provisioning_server(int fd, sgx_quote_sign_type_t quote_type);

enclave/src/lib.rs

@@ -37,7 +37,7 @@ use sgx_types::{sgx_epid_group_id_t, sgx_status_t, sgx_target_info_t, SgxResult}
 use substrate_api_client::compose_extrinsic_offline;
 use substratee_node_primitives::{CallWorkerFn, ShieldFundsFn};
 use substratee_worker_primitives::block::{
-    Block as SidechainBlock, SignedBlock as SignedSidechainBlock, StatePayload,
+    Block as SidechainBlock, SignedBlock as SignedSidechainBlock,
 };
 use substratee_worker_primitives::BlockHash;
 
@@ -56,7 +56,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use std::untrusted::time::SystemTimeEx;
 use utils::write_slice_and_whitespace_pad;
 
-use crate::utils::UnwrapOrSgxErrorUnexpected;
+use crate::utils::{UnwrapOrSgxErrorUnexpected, hash_from_slice};
 use chain_relay::{
     storage_proof::{StorageProof, StorageProofChecker},
     Block, Header, LightValidation,
@@ -69,7 +69,7 @@ use sgx_externalities::SgxExternalitiesTypeTrait;
 use substratee_stf::sgx::{shards_key_hash, storage_hashes_to_update_per_shard, OpaqueCall};
 use substratee_stf::State as StfState;
 use substratee_stf::{
-    AccountId, Getter, ShardIdentifier, Stf, TrustedCall, TrustedCallSigned, TrustedGetterSigned,
+    AccountId, Getter, ShardIdentifier, Stf, TrustedCall, TrustedCallSigned, TrustedGetterSigned, StatePayload
 };
 
 use rpc::author::{hash::TrustedOperationOrHash, Author, AuthorApi};
@@ -92,8 +92,9 @@ pub mod tests;
 pub mod tls_ra;
 pub mod top_pool;
 
-use substratee_settings::node::{BLOCK_CONFIRMED, CALL_CONFIRMED, RUNTIME_SPEC_VERSION, RUNTIME_TRANSACTION_VERSION, SUBSTRATEE_REGISTRY_MODULE, CALL_WORKER, SHIELD_FUNDS};
+use substratee_settings::node::{BLOCK_CONFIRMED, CALL_CONFIRMED, RUNTIME_SPEC_VERSION, RUNTIME_TRANSACTION_VERSION, SUBSTRATEE_REGISTRY_MODULE, CALL_WORKER, SHIELD_FUNDS, REGISTER_ENCLAVE};
 use substratee_settings::enclave::{CALL_TIMEOUT, GETTER_TIMEOUT};
+use codec::alloc::string::String;
 
 pub const CERTEXPIRYDAYS: i64 = 90i64;
 
@@ -190,6 +191,44 @@ pub unsafe extern "C" fn get_ecc_signing_pubkey(pubkey: *mut u8, pubkey_size: u3
     sgx_status_t::SGX_SUCCESS
 }
 
+
+#[no_mangle]
+pub unsafe extern "C" fn mock_register_enclave_xt(
+    genesis_hash: *const u8,
+    genesis_hash_size: u32,
+    nonce: *const u32,
+    w_url: *const u8,
+    w_url_size: u32,
+    unchecked_extrinsic: *mut u8,
+    unchecked_extrinsic_size: u32,
+) -> sgx_status_t {
+    let genesis_hash_slice = slice::from_raw_parts(genesis_hash, genesis_hash_size as usize);
+    let genesis_hash = hash_from_slice(genesis_hash_slice);
+
+    let mut url_slice = slice::from_raw_parts(w_url, w_url_size as usize);
+    let url: String = Decode::decode(&mut url_slice).unwrap();
+    let extrinsic_slice =
+        slice::from_raw_parts_mut(unchecked_extrinsic, unchecked_extrinsic_size as usize);
+
+    let signer = ed25519::unseal_pair().unwrap();
+
+    let call = [SUBSTRATEE_REGISTRY_MODULE, REGISTER_ENCLAVE];
+
+    let xt = compose_extrinsic_offline!(
+        signer,
+        (call, Vec::<u8>::new(), url.clone()),
+        *nonce,
+        Era::Immortal,
+        genesis_hash,
+        genesis_hash,
+        RUNTIME_SPEC_VERSION,
+        RUNTIME_TRANSACTION_VERSION
+    ).encode();
+
+    write_slice_and_whitespace_pad(extrinsic_slice, xt);
+    sgx_status_t::SGX_SUCCESS
+}
+
 fn create_extrinsics(
     validator: LightValidation,
     calls_buffer: Vec<OpaqueCall>,

enclave/src/rpc/worker_api_direct.rs

@@ -57,7 +57,7 @@ use chain_relay::Block;
 
 use substratee_node_primitives::Request;
 use substratee_worker_primitives::RpcReturnValue;
-use substratee_worker_primitives::{DirectRequestStatus, TrustedOperationStatus};
+use substratee_worker_primitives::{DirectRequestStatus, TrustedOperationStatus, block::SignedBlock};
 
 use crate::rsa3072;
 use crate::utils::write_slice_and_whitespace_pad;
@@ -357,6 +357,24 @@ fn init_io_handler() -> IoHandler {
         Ok(Value::String(format!("hello, {}", parsed)))
     });
 
+    let sidechain_import_import_name: &str = "sidechain_importBlock";
+    rpc_methods_vec.push(sidechain_import_import_name);
+    io.add_sync_method(sidechain_import_import_name, |sidechain_blocks: Params| {
+        // Todo: actually do something with the block, i.e., apply the state diff.
+        // However, this requires importing the stf here. So, before doing that and increase
+        // the size of init_io_handler even more, we should think about how we can do that more
+        // modularized.
+        debug!("sidechain_importBlock rpc. Params: {:?}", sidechain_blocks);
+        let block_vec: Vec<u8> = sidechain_blocks.parse()?;
+        let blocks: Vec<SignedBlock> = Decode::decode(&mut block_vec.as_slice())
+            .map_err(|_| jsonrpc_core::error::Error::invalid_params_with_details(
+                "Could not decode Vec<SignedBlock>",
+                block_vec
+            ))?;
+        info!("sidechain_importBlock. Blocks: {:?}", blocks);
+        Ok(Value::String("ok".to_owned()))
+    });
+
     // returns all rpcs methods
     let rpc_methods_string: String = convert_vec_to_string(rpc_methods_vec);
     io.add_sync_method("rpc_methods", move |_: Params| {
@@ -449,3 +467,42 @@ pub fn send_state<H: Encode>(hash: H, value_opt: Option<Vec<u8>>) -> Result<(),
 
     Ok(())
 }
+
+pub mod tests {
+    use super::init_io_handler;
+    use super::alloc::string::ToString;
+    use std::string::String;
+
+    fn rpc_response<T: ToString>(result: T) -> String {
+        format!(r#"{{"jsonrpc":"2.0","result":{},"id":1}}"#, result.to_string())
+    }
+
+    pub fn sidechain_import_block_is_ok() {
+        let io = init_io_handler();
+        let enclave_req = r#"{"jsonrpc":"2.0","method":"sidechain_importBlock","params":[4,0,0,0,0,0,0,0,0,228,0,145,188,97,251,138,131,108,29,6,107,10,152,67,29,148,190,114,167,223,169,197,163,93,228,76,169,171,80,15,209,101,11,211,96,0,0,0,0,83,52,167,255,37,229,185,231,38,66,122,3,55,139,5,190,125,85,94,177,190,99,22,149,92,97,154,30,142,89,24,144,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,136,220,52,23,213,5,142,196,180,80,62,12,18,234,26,10,137,190,32,15,233,137,34,66,61,67,52,1,79,166,176,238,0,0,0,175,124,84,84,32,238,162,224,130,203,26,66,7,121,44,59,196,200,100,31,173,226,165,106,187,135,223,149,30,46,191,95,116,203,205,102,100,85,82,74,158,197,166,218,181,130,119,127,162,134,227,129,118,85,123,76,21,113,90,1,160,77,110,15],"id":1}"#;
+
+        let response_string = io.handle_request_sync(enclave_req).unwrap();
+
+        assert_eq!(response_string, rpc_response("\"ok\""));
+    }
+
+    pub fn sidechain_import_block_returns_invalid_param_err() {
+        let io = init_io_handler();
+        let enclave_req = r#"{"jsonrpc":"2.0","method":"sidechain_importBlock","params":["SophisticatedInvalidParam"],"id":1}"#;
+
+        let response_string = io.handle_request_sync(enclave_req).unwrap();
+
+        let err_msg = r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid params: invalid type: string \"SophisticatedInvalidParam\", expected u8."},"id":1}"#;
+        assert_eq!(response_string, err_msg);
+    }
+
+    pub fn sidechain_import_block_returns_decode_err() {
+        let io = init_io_handler();
+        let enclave_req = r#"{"jsonrpc":"2.0","method":"sidechain_importBlock","params":[2],"id":1}"#;
+
+        let response_string = io.handle_request_sync(enclave_req).unwrap();
+
+        let err_msg = r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid parameters: Could not decode Vec<SignedBlock>","data":"[2]"},"id":1}"#;
+        assert_eq!(response_string, err_msg);
+    }
+}

enclave/src/tests.rs

@@ -29,7 +29,6 @@ use sgx_tunittest::*;
 use sgx_types::{sgx_status_t, size_t};
 
 use substrate_api_client::utils::storage_key;
-use substratee_worker_primitives::block::StatePayload;
 
 use codec::{Decode, Encode};
 use sp_core::{crypto::Pair, hashing::blake2_256, H256};
@@ -49,9 +48,9 @@ use chain_relay::{Block, Header};
 use sp_runtime::traits::Header as HeaderT;
 
 use sgx_externalities::SgxExternalitiesTypeTrait;
-use substratee_stf::sgx::AccountInfo;
+use substratee_stf::AccountInfo;
 use substratee_stf::StateTypeDiff as StfStateTypeDiff;
-use substratee_stf::{ShardIdentifier, Stf, TrustedCall};
+use substratee_stf::{ShardIdentifier, Stf, TrustedCall, StatePayload};
 use substratee_stf::{TrustedGetter, TrustedOperation};
 
 use substratee_settings::enclave::{GETTER_TIMEOUT};
@@ -115,6 +114,12 @@ pub extern "C" fn test_main_entrance() -> size_t {
         test_executing_call_updates_account_nonce,
         test_invalid_nonce_call_is_not_executed,
         test_non_root_shielding_call_is_not_executed,
+        substratee_stf::sgx::tests::apply_state_diff_works,
+        substratee_stf::sgx::tests::apply_state_diff_returns_storage_hash_mismatch_err,
+        substratee_stf::sgx::tests::apply_state_diff_returns_invalid_storage_diff_err,
+        rpc::worker_api_direct::tests::sidechain_import_block_is_ok,
+        rpc::worker_api_direct::tests::sidechain_import_block_returns_invalid_param_err,
+        rpc::worker_api_direct::tests::sidechain_import_block_returns_decode_err,
         // these unit tests (?) need an ipfs node running..
         //ipfs::test_creates_ipfs_content_struct_works,
         //ipfs::test_verification_ok_for_correct_content,

local-setup/launch.py

@@ -4,6 +4,7 @@
 """
 import signal
 from subprocess import Popen, STDOUT
+from time import sleep
 from typing import Union, IO
 
 from py.worker import Worker
@@ -32,17 +33,19 @@ def setup_worker(work_dir: str, std_err: Union[None, int, IO]):
 def main(processes):
     print('Starting substraTee-node-process in background')
     processes.append(
-        Popen([node_bin, '--tmp', '--dev', '-lruntime=debug'], stdout=node_log, stderr=STDOUT, bufsize=1)
+        Popen([node_bin, '--tmp', '--dev', '-lruntime=info'], stdout=node_log, stderr=STDOUT, bufsize=1)
     )
 
     w1 = setup_worker(w1_working_dir, worker1_log)
     w2 = setup_worker(w2_working_dir, worker2_log)
 
     print('Starting worker 1 in background')
-    processes.append(w1.run_in_background(log_file=worker1_log, flags=['-P', '2001'], subcommand_flags=['--skip-ra']))
-    print('Starting worker 2 in background')
-    processes.append(w2.run_in_background(log_file=worker2_log, subcommand_flags=['--skip-ra']))
+    processes.append(w1.run_in_background(log_file=worker1_log, flags=['-P', '2000'], subcommand_flags=['--skip-ra']))
 
+    # sleep to prevent nonce clash when bootstrapping the enclave's account
+    sleep(6)
+    print('Starting worker 2 in background')
+    processes.append(w2.run_in_background(log_file=worker2_log, flags=['-P', '3000'], subcommand_flags=['--skip-ra']))
     # keep script alive until terminated
     signal.pause()
 

local-setup/py/worker.py

@@ -1,3 +1,4 @@
+import os
 import pathlib
 import shutil
 import subprocess
@@ -108,16 +109,18 @@ def purge_shard(self, shard=None):
         if not shard:
             shard = self.mrenclave()
 
+        print(f'Purging shard: {shard}')
+
         if not self.shard_exists(shard):
             print('The shard to be purged does not exist.')
         else:
             shutil.rmtree(self._shard_path(shard))
 
-    @staticmethod
-    def purge_chain_relay_db():
-        db = pathlib.Path('./chain_relay_db.bin')
-        if db.exists():
-            db.unlink()
+    def purge_chain_relay_db(self):
+        print(f'purging chain_relay_db')
+        for db in pathlib.Path(self.cwd).glob('chain_relay_db.bin*'):
+                print(f'remove: {db}')
+                db.unlink()
 
     def mrenclave(self):
         """ Returns the mrenclave and caches it. """
@@ -140,8 +143,13 @@ def run_in_background(self, log_file: TextIO, flags: [str] = None, subcommand_fl
 
         :return: process handle for the spawned background process.
         """
+
+        # Todo: make this configurable
+        env = dict(os.environ, RUST_LOG='debug,ws=warn,sp_io=warn,substrate_api_client=info')
+
         return Popen(
             self._assemble_cmd(flags=flags, subcommand_flags=subcommand_flags),
+            env=env,
             stdout=log_file,
             stderr=STDOUT,
             bufsize=1,