Refactor the host <-> runtime interface machinery

.github/workflows/tests-misc.yml

@@ -64,9 +64,9 @@ jobs:
       - name: script
         run: |
           cd substrate/frame/examples/offchain-worker/
-          forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+          RUSTFLAGS="--cfg substrate_runtime" forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
           cd ../basic
-          forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
+          RUSTFLAGS="--cfg substrate_runtime" forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
 
   test-frame-ui:
     timeout-minutes: 60

polkadot/runtime/metrics/src/with_runtime_metrics.rs

@@ -142,5 +142,5 @@ impl Histogram {
 
 /// Returns current time in ns
 pub fn get_current_time() -> u128 {
-	frame_benchmarking::benchmarking::current_time()
+	frame_benchmarking::current_time()
 }

prdoc/pr_7375.prdoc

@@ -0,0 +1,87 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# yaml-language-server: $schema=https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: Refactor the host <-> runtime interface machinery (the `#[runtime_interface]` macro) and the way host functions are defined
+
+doc:
+  - audience: Node Dev
+    description: |
+      This PR refactors the way the host functions are defined.
+
+      Previously the way a given type was marshalled through the host <-> runtime boundary was
+      hardcoded for every type by the virtue of it implementing the relevant conversion traits.
+
+      This had two major consequences:
+        * It was not obvious how a given type is going to be passed just by looking at its type alone,
+          masking potentially expensive marshalling strategies. (For example, returning `Option<u32>`
+          was done through the SCALE codec and involved extra memory allocations!)
+        * It was not possible to use multiple marshalling strategies for a single type, making some
+          of the future improvements we'd like to do (e.g. move the runtime memory allocator into the runtime)
+          very hard to do.
+
+      So this PR disentangles this mess and makes the marshalling strategies explicit. This makes it
+      much more clear how a given type in a given host function is marshalled, and also makes it possible
+      to use different marshalling strategies for the same type in different host functions.
+
+      Before this PR you'd define a host function like this:
+
+      ```rust
+          #[runtime_interface]
+          trait MyInterface {
+              fn say_hello_world(name: &str) {
+                  println!("Hello {name}!");
+              }
+          }
+      ```
+
+      and after this PR you'll define it like this:
+
+      ```rust
+          #[runtime_interface]
+          trait MyInterface {
+              fn say_hello_world(name: PassFatPointerAndRead<&str>) {
+                  println!("Hello {name}!", name);
+              }
+          }
+      ```
+
+      In this case the strategy for passing the `&str` is now explicitly specified (`PassFatPointerAndRead`).
+      Note that the *actual* API generated by this macro and the way arguments are accessed is completely unchanged!
+      The `#[runtime_interface]` machinery automatically "strips" away the marshalling strategy wrappers,
+      so neither the body of the `say_hello_world` function here nor its callers need to be changed.
+
+      Furthermore, to explicitly distinguish between the runtime compilation mode and native mode,
+      `#[cfg(substrate_runtime)]` is now used instead of `#[cfg(not(feature = "std"))]`. That allows for fine-tuning
+      the compilation behavior without relying solely on the `std` feature.
+
+crates:
+  - name: sp-runtime-interface
+    bump: major
+    note: Rework of the `#[runtime_interface]` macro and associated types/trait.
+  - name: sp-runtime-interface-proc-macro
+    bump: major
+    note: Rework of the `#[runtime_interface]` macro.
+  - name: sp-wasm-interface
+    bump: major
+    note: The `Pointer` type now implements `Copy` and `Clone` unconditionally. The `Result` now always returns a `String`.
+  - name: sp-core
+    bump: major
+    note: Some types don't implement the traits related to the old `#[runtime_interface]` anymore. A few extra conversion impls.
+  - name: sp-io
+    bump: major
+    note: Requires the new `#[runtime_interface]` macro and associated machinery. Some types don't implement the traits related to the old `#[runtime_interface]` anymore.
+  - name: sp-statement-store
+    bump: major
+    note: Requires the new `#[runtime_interface]` macro and associated machinery. Some types don't implement the traits related to the old `#[runtime_interface]` anymore.
+  - name: sp-crypto-ec-utils
+    bump: minor
+    note: Requires the new `#[runtime_interface]` macro and associated machinery.
+  - name: frame-benchmarking
+    bump: major
+    note: Requires the new `#[runtime_interface]` macro and associated machinery. `Benchmarking::current_time` host call was changed.
+  - name: frame-support-procedural
+    bump: minor
+    note: Needs new `frame-benchmarking` due to the change to `current_time`.
+  - name: polkadot-runtime-metrics
+    bump: minor
+    note: Needs new `frame-benchmarking` due to the change to `current_time`.

substrate/client/executor/src/executor.rs

@@ -758,11 +758,11 @@ impl<D: NativeExecutionDispatch> sp_core::traits::ReadRuntimeVersion for NativeE
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use sp_runtime_interface::runtime_interface;
+	use sp_runtime_interface::{pass_by::PassFatPointerAndRead, runtime_interface};
 
 	#[runtime_interface]
 	trait MyInterface {
-		fn say_hello_world(data: &str) {
+		fn say_hello_world(data: PassFatPointerAndRead<&str>) {
 			println!("Hello world from: {}", data);
 		}
 	}

substrate/frame/benchmarking/src/utils.rs

@@ -24,6 +24,9 @@ use scale_info::TypeInfo;
 use serde::{Deserialize, Serialize};
 use sp_io::hashing::blake2_256;
 use sp_runtime::{traits::TrailingZeroInput, DispatchError};
+use sp_runtime_interface::pass_by::{
+	AllocateAndReturnByCodec, PassFatPointerAndDecode, PassFatPointerAndRead, PassPointerAndWrite,
+};
 use sp_storage::TrackedStorageKey;
 
 /// An alphabet of possible parameters to use for benchmarking.
@@ -245,18 +248,31 @@ sp_api::decl_runtime_apis! {
 	}
 }
 
+/// Get the number of nanoseconds passed since the UNIX epoch
+///
+/// WARNING! This is a non-deterministic call. Do not use this within
+/// consensus critical logic.
+pub fn current_time() -> u128 {
+	let mut out = [0; 16];
+	self::benchmarking::current_time(&mut out);
+	u128::from_le_bytes(out)
+}
+
 /// Interface that provides functions for benchmarking the runtime.
 #[sp_runtime_interface::runtime_interface]
 pub trait Benchmarking {
-	/// Get the number of nanoseconds passed since the UNIX epoch
+	/// Get the number of nanoseconds passed since the UNIX epoch, as u128 le-bytes.
+	///
+	/// You may want to use the standalone function [`current_time`].
 	///
 	/// WARNING! This is a non-deterministic call. Do not use this within
 	/// consensus critical logic.
-	fn current_time() -> u128 {
-		std::time::SystemTime::now()
+	fn current_time(out: PassPointerAndWrite<&mut [u8; 16], 16>) {
+		*out = std::time::SystemTime::now()
 			.duration_since(std::time::SystemTime::UNIX_EPOCH)
 			.expect("Unix time doesn't go backwards; qed")
 			.as_nanos()
+			.to_le_bytes();
 	}
 
 	/// Reset the trie database to the genesis state.
@@ -270,7 +286,7 @@ pub trait Benchmarking {
 	}
 
 	/// Get the read/write count.
-	fn read_write_count(&self) -> (u32, u32, u32, u32) {
+	fn read_write_count(&self) -> AllocateAndReturnByCodec<(u32, u32, u32, u32)> {
 		self.read_write_count()
 	}
 
@@ -280,17 +296,17 @@ pub trait Benchmarking {
 	}
 
 	/// Get the DB whitelist.
-	fn get_whitelist(&self) -> Vec<TrackedStorageKey> {
+	fn get_whitelist(&self) -> AllocateAndReturnByCodec<Vec<TrackedStorageKey>> {
 		self.get_whitelist()
 	}
 
 	/// Set the DB whitelist.
-	fn set_whitelist(&mut self, new: Vec<TrackedStorageKey>) {
+	fn set_whitelist(&mut self, new: PassFatPointerAndDecode<Vec<TrackedStorageKey>>) {
 		self.set_whitelist(new)
 	}
 
 	// Add a new item to the DB whitelist.
-	fn add_to_whitelist(&mut self, add: TrackedStorageKey) {
+	fn add_to_whitelist(&mut self, add: PassFatPointerAndDecode<TrackedStorageKey>) {
 		let mut whitelist = self.get_whitelist();
 		match whitelist.iter_mut().find(|x| x.key == add.key) {
 			// If we already have this key in the whitelist, update to be the most constrained
@@ -309,18 +325,20 @@ pub trait Benchmarking {
 	}
 
 	// Remove an item from the DB whitelist.
-	fn remove_from_whitelist(&mut self, remove: Vec<u8>) {
+	fn remove_from_whitelist(&mut self, remove: PassFatPointerAndRead<Vec<u8>>) {
 		let mut whitelist = self.get_whitelist();
 		whitelist.retain(|x| x.key != remove);
 		self.set_whitelist(whitelist);
 	}
 
-	fn get_read_and_written_keys(&self) -> Vec<(Vec<u8>, u32, u32, bool)> {
+	fn get_read_and_written_keys(
+		&self,
+	) -> AllocateAndReturnByCodec<Vec<(Vec<u8>, u32, u32, bool)>> {
 		self.get_read_and_written_keys()
 	}
 
 	/// Get current estimated proof size.
-	fn proof_size(&self) -> Option<u32> {
+	fn proof_size(&self) -> AllocateAndReturnByCodec<Option<u32>> {
 		self.proof_size()
 	}
 }
@@ -400,11 +418,11 @@ impl<'a> Recording for BenchmarkRecording<'a> {
 	fn start(&mut self) {
 		(self.on_before_start.take().expect("start called more than once"))();
 		self.start_pov = crate::benchmarking::proof_size();
-		self.start_extrinsic = Some(crate::benchmarking::current_time());
+		self.start_extrinsic = Some(current_time());
 	}
 
 	fn stop(&mut self) {
-		self.finish_extrinsic = Some(crate::benchmarking::current_time());
+		self.finish_extrinsic = Some(current_time());
 		self.end_pov = crate::benchmarking::proof_size();
 	}
 }

substrate/frame/benchmarking/src/v1.rs

@@ -1132,9 +1132,9 @@ macro_rules! impl_benchmark {
 					);
 
 					// Time the storage root recalculation.
-					let start_storage_root = $crate::benchmarking::current_time();
+					let start_storage_root = $crate::current_time();
 					$crate::__private::storage_root($crate::__private::StateVersion::V1);
-					let finish_storage_root = $crate::benchmarking::current_time();
+					let finish_storage_root = $crate::current_time();
 					let elapsed_storage_root = finish_storage_root - start_storage_root;
 
 					let skip_meta = [ $( stringify!($name_skip_meta).as_ref() ),* ];

substrate/frame/src/lib.rs

@@ -292,9 +292,12 @@ pub mod benchmarking {
 	}
 
 	pub mod prelude {
-		pub use super::shared::*;
 		pub use crate::prelude::*;
-		pub use frame_benchmarking::v2::*;
+		pub use frame_benchmarking::{
+			add_benchmark, benchmarking::add_to_whitelist, v1::account, v2::*, whitelist,
+			whitelisted_caller,
+		};
+		pub use frame_system::{Pallet as System, RawOrigin};
 	}
 }
 

substrate/frame/support/procedural/src/benchmark.rs

@@ -820,9 +820,9 @@ pub fn benchmarks(
 						);
 
 						// Time the storage root recalculation.
-						let start_storage_root = #krate::benchmarking::current_time();
+						let start_storage_root = #krate::current_time();
 						#krate::__private::storage_root(#krate::__private::StateVersion::V1);
-						let finish_storage_root = #krate::benchmarking::current_time();
+						let finish_storage_root = #krate::current_time();
 						let elapsed_storage_root = finish_storage_root - start_storage_root;
 
 						let skip_meta = [ #(#skip_meta_benchmark_names_str),* ];

substrate/primitives/application-crypto/check-features-variants.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env -S bash -eux
 
 export RUSTFLAGS="-Cdebug-assertions=y -Dwarnings"
+cargo check --release
+
+export RUSTFLAGS="$RUSTFLAGS --cfg substrate_runtime"
 T=wasm32-unknown-unknown
-cargo check --release 
 cargo check --release --target=$T --no-default-features
 cargo check --release --target=$T --no-default-features  --features="full_crypto"
 cargo check --release --target=$T --no-default-features  --features="serde"

substrate/primitives/core/Cargo.toml

@@ -71,6 +71,11 @@ w3f-bls = { optional = true, workspace = true }
 # bandersnatch crypto
 ark-vrf = { optional = true, workspace = true, features = ["bandersnatch", "ring"] }
 
+[target.'cfg(not(substrate_runtime))'.dependencies]
+sp-externalities = { workspace = true, default-features = false }
+futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
+dyn-clone = "1.0.17"
+
 [dev-dependencies]
 criterion = { workspace = true, default-features = true }
 regex = { workspace = true }
@@ -86,10 +91,9 @@ std = [
 	"bounded-collections/std",
 	"bs58/std",
 	"codec/std",
-	"dyn-clonable",
 	"ed25519-zebra/std",
 	"full_crypto",
-	"futures",
+	"futures/std",
 	"futures/thread-pool",
 	"hash-db/std",
 	"hash256-std-hasher/std",

substrate/primitives/core/check-features-variants.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env -S bash -eux
 
-export RUSTFLAGS="-Cdebug-assertions=y -Dwarnings"
+export RUSTFLAGS="-Cdebug-assertions=y -Dwarnings --cfg substrate_runtime"
 T=wasm32-unknown-unknown
 
 cargo check --target=$T --release --no-default-features  --features="bls-experimental"

substrate/primitives/core/src/crypto.rs

@@ -19,7 +19,7 @@
 
 use crate::{ed25519, sr25519, U256};
 use alloc::{format, str, vec::Vec};
-#[cfg(all(not(feature = "std"), feature = "serde"))]
+#[cfg(feature = "serde")]
 use alloc::{string::String, vec};
 use bip39::{Language, Mnemonic};
 use codec::{Decode, DecodeWithMemTracking, Encode, MaxEncodedLen};
@@ -32,7 +32,6 @@ use itertools::Itertools;
 use rand::{rngs::OsRng, RngCore};
 use scale_info::TypeInfo;
 pub use secrecy::{ExposeSecret, SecretString};
-use sp_runtime_interface::pass_by::PassByInner;
 pub use ss58_registry::{from_known_address_format, Ss58AddressFormat, Ss58AddressFormatRegistry};
 /// Trait to zeroize a memory buffer.
 pub use zeroize::Zeroize;
@@ -1052,11 +1051,11 @@ pub trait CryptoType {
 	Hash,
 	Encode,
 	Decode,
-	PassByInner,
 	crate::RuntimeDebug,
 	TypeInfo,
 )]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[repr(transparent)]
 pub struct KeyTypeId(pub [u8; 4]);
 
 impl From<u32> for KeyTypeId {
@@ -1071,6 +1070,18 @@ impl From<KeyTypeId> for u32 {
 	}
 }
 
+impl From<[u8; 4]> for KeyTypeId {
+	fn from(value: [u8; 4]) -> Self {
+		Self(value)
+	}
+}
+
+impl AsRef<[u8]> for KeyTypeId {
+	fn as_ref(&self) -> &[u8] {
+		&self.0
+	}
+}
+
 impl<'a> TryFrom<&'a str> for KeyTypeId {
 	type Error = ();
 
@@ -1208,6 +1219,7 @@ impl_from_entropy_base!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, U256);
 mod tests {
 	use super::*;
 	use crate::DeriveJunction;
+	use alloc::{string::String, vec};
 
 	struct TestCryptoTag;