Lean BEEFY to Full BEEFY - don't skip (older) mandatory blocks and import justifications

client/beefy/Cargo.toml

@@ -9,6 +9,7 @@ description = "BEEFY Client gadget for substrate"
 homepage = "https://substrate.io"
 
 [dependencies]
+async-trait = "0.1.50"
 codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] }
 fnv = "1.0.6"
 futures = "0.3"
@@ -22,6 +23,7 @@ beefy-primitives = { version = "4.0.0-dev", path = "../../primitives/beefy" }
 prometheus = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" }
 sc-chain-spec = { version = "4.0.0-dev", path = "../../client/chain-spec" }
 sc-client-api = { version = "4.0.0-dev", path = "../api" }
+sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
 sc-finality-grandpa = { version = "0.10.0-dev", path = "../../client/finality-grandpa" }
 sc-keystore = { version = "4.0.0-dev", path = "../keystore" }
 sc-network = { version = "0.10.0-dev", path = "../network" }
@@ -42,7 +44,7 @@ serde = "1.0.136"
 strum = { version = "0.24.1", features = ["derive"] }
 tempfile = "3.1.0"
 tokio = "1.17.0"
-sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
+sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" }
 sc-network-test = { version = "0.8.0", path = "../network/test" }
 sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" }
 sp-keyring = { version = "6.0.0", path = "../../primitives/keyring" }

client/beefy/rpc/src/lib.rs

@@ -164,8 +164,9 @@ where
 mod tests {
 	use super::*;
 
-	use beefy_gadget::notification::{
-		BeefyBestBlockStream, BeefySignedCommitment, BeefySignedCommitmentSender,
+	use beefy_gadget::{
+		justification::BeefySignedCommitment,
+		notification::{BeefyBestBlockStream, BeefySignedCommitmentSender},
 	};
 	use beefy_primitives::{known_payload_ids, Payload};
 	use codec::{Decode, Encode};

client/beefy/rpc/src/notification.rs

@@ -29,7 +29,7 @@ pub struct EncodedSignedCommitment(sp_core::Bytes);
 
 impl EncodedSignedCommitment {
 	pub fn new<Block>(
-		signed_commitment: beefy_gadget::notification::BeefySignedCommitment<Block>,
+		signed_commitment: beefy_gadget::justification::BeefySignedCommitment<Block>,
 	) -> Self
 	where
 		Block: BlockT,

client/beefy/src/error.rs

@@ -24,8 +24,12 @@ use std::fmt::Debug;
 
 #[derive(Debug, thiserror::Error, PartialEq)]
 pub enum Error {
+	#[error("Backend: {0}")]
+	Backend(String),
 	#[error("Keystore error: {0}")]
 	Keystore(String),
 	#[error("Signature error: {0}")]
 	Signature(String),
+	#[error("Session uninitialized")]
+	UninitSession,
 }

client/beefy/src/import.rs

@@ -0,0 +1,205 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use beefy_primitives::{crypto::Signature, BeefyApi, VersionedFinalityProof, BEEFY_ENGINE_ID};
+use codec::Encode;
+use log::error;
+use std::{collections::HashMap, sync::Arc};
+
+use sp_api::{ProvideRuntimeApi, TransactionFor};
+use sp_blockchain::{well_known_cache_keys, HeaderBackend};
+use sp_consensus::Error as ConsensusError;
+use sp_runtime::{
+	generic::BlockId,
+	traits::{Block as BlockT, Header as HeaderT, NumberFor},
+	EncodedJustification,
+};
+
+use sc_client_api::backend::Backend;
+use sc_consensus::{BlockCheckParams, BlockImport, BlockImportParams, ImportResult};
+
+use crate::{
+	justification::decode_and_verify_commitment, notification::BeefySignedCommitmentSender,
+};
+
+/// A block-import handler for BEEFY.
+///
+/// This scans each imported block for BEEFY justifications and verifies them.
+/// Wraps a `inner: BlockImport` and ultimately defers to it.
+///
+/// When using BEEFY, the block import worker should be using this block import object.
+pub struct BeefyBlockImport<Block: BlockT, Backend, RuntimeApi, I> {
+	backend: Arc<Backend>,
+	runtime: Arc<RuntimeApi>,
+	inner: I,
+	justification_sender: BeefySignedCommitmentSender<Block>,
+}
+
+impl<Block: BlockT, BE, Runtime, I: Clone> Clone for BeefyBlockImport<Block, BE, Runtime, I> {
+	fn clone(&self) -> Self {
+		BeefyBlockImport {
+			backend: self.backend.clone(),
+			runtime: self.runtime.clone(),
+			inner: self.inner.clone(),
+			justification_sender: self.justification_sender.clone(),
+		}
+	}
+}
+
+impl<Block: BlockT, BE, Runtime, I> BeefyBlockImport<Block, BE, Runtime, I> {
+	/// Create a new BeefyBlockImport.
+	pub fn new(
+		backend: Arc<BE>,
+		runtime: Arc<Runtime>,
+		inner: I,
+		justification_sender: BeefySignedCommitmentSender<Block>,
+	) -> BeefyBlockImport<Block, BE, Runtime, I> {
+		BeefyBlockImport { backend, runtime, inner, justification_sender }
+	}
+}
+
+impl<Block, BE, Runtime, I> BeefyBlockImport<Block, BE, Runtime, I>
+where
+	Block: BlockT,
+	BE: Backend<Block>,
+	Runtime: ProvideRuntimeApi<Block>,
+	Runtime::Api: BeefyApi<Block> + Send + Sync,
+{
+	fn decode_and_verify(
+		&self,
+		encoded: &EncodedJustification,
+		number: NumberFor<Block>,
+		hash: <Block as BlockT>::Hash,
+	) -> Result<VersionedFinalityProof<NumberFor<Block>, Signature>, ConsensusError> {
+		let block_id = BlockId::hash(hash);
+		let validator_set = self
+			.runtime
+			.runtime_api()
+			.validator_set(&block_id)
+			.map_err(|e| ConsensusError::ClientImport(e.to_string()))?
+			.ok_or_else(|| ConsensusError::ClientImport("Unknown validator set".to_string()))?;
+
+		decode_and_verify_commitment::<Block>(&encoded[..], number, &validator_set)
+	}
+
+	/// Import BEEFY justification: Send it to worker for processing and also append it to backend.
+	///
+	/// This function assumes:
+	/// - `justification` is verified and valid,
+	/// - the block referred by `justification` has been imported _and_ finalized.
+	fn import_beefy_justification_unchecked(
+		&self,
+		number: NumberFor<Block>,
+		justification: VersionedFinalityProof<NumberFor<Block>, Signature>,
+	) {
+		// Append the justification to the block in the backend.
+		if let Err(e) = self.backend.append_justification(
+			BlockId::Number(number),
+			(BEEFY_ENGINE_ID, justification.encode()),
+		) {
+			error!(target: "beefy", "🥩 Error {:?} on appending justification: {:?}", e, justification);
+		}
+		// Send the justification to the BEEFY voter for processing.
+		match justification {
+			// TODO #11838: Should not unpack, these channels should also use
+			// `VersionedFinalityProof`.
+			VersionedFinalityProof::V1(signed_commitment) => self
+				.justification_sender
+				.notify(|| Ok::<_, ()>(signed_commitment))
+				.expect("forwards closure result; the closure always returns Ok; qed."),
+		};
+	}
+}
+
+#[async_trait::async_trait]
+impl<Block, BE, Runtime, I> BlockImport<Block> for BeefyBlockImport<Block, BE, Runtime, I>
+where
+	Block: BlockT,
+	BE: Backend<Block>,
+	I: BlockImport<
+			Block,
+			Error = ConsensusError,
+			Transaction = sp_api::TransactionFor<Runtime, Block>,
+		> + Send
+		+ Sync,
+	Runtime: ProvideRuntimeApi<Block> + Send + Sync,
+	Runtime::Api: BeefyApi<Block>,
+{
+	type Error = ConsensusError;
+	type Transaction = TransactionFor<Runtime, Block>;
+
+	async fn import_block(
+		&mut self,
+		mut block: BlockImportParams<Block, Self::Transaction>,
+		new_cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error> {
+		let hash = block.post_hash();
+		let number = *block.header.number();
+
+		let beefy_proof = block
+			.justifications
+			.as_mut()
+			.and_then(|just| {
+				let decoded = just
+					.get(BEEFY_ENGINE_ID)
+					.map(|encoded| self.decode_and_verify(encoded, number, hash));
+				// Remove BEEFY justification from the list before giving to `inner`;
+				// we will append it to backend ourselves at the end if all goes well.
+				just.remove(BEEFY_ENGINE_ID);
+				decoded
+			})
+			.transpose()
+			.unwrap_or(None);
+
+		// Run inner block import.
+		let inner_import_result = self.inner.import_block(block, new_cache).await?;
+
+		match (beefy_proof, &inner_import_result) {
+			(Some(proof), ImportResult::Imported(_)) => {
+				let status = self.backend.blockchain().info();
+				if number <= status.finalized_number &&
+					Some(hash) ==
+						self.backend
+							.blockchain()
+							.hash(number)
+							.map_err(|e| ConsensusError::ClientImport(e.to_string()))?
+				{
+					// The proof is valid and the block is imported and final, we can import.
+					self.import_beefy_justification_unchecked(number, proof);
+				} else {
+					error!(
+						target: "beefy",
+						"🥩 Cannot import justification: {:?} for, not yet final, block number {:?}",
+						proof,
+						number,
+					);
+				}
+			},
+			_ => (),
+		}
+
+		Ok(inner_import_result)
+	}
+
+	async fn check_block(
+		&mut self,
+		block: BlockCheckParams<Block>,
+	) -> Result<ImportResult, Self::Error> {
+		self.inner.check_block(block).await
+	}
+}