Lean BEEFY to Full BEEFY - don't skip (older) mandatory blocks and import justifications

client/beefy/Cargo.toml

@@ -9,6 +9,7 @@ description = "BEEFY Client gadget for substrate"
 homepage = "https://substrate.io"
 
 [dependencies]
+async-trait = "0.1.50"
 codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] }
 fnv = "1.0.6"
 futures = "0.3"
@@ -22,6 +23,7 @@ beefy-primitives = { version = "4.0.0-dev", path = "../../primitives/beefy" }
 prometheus = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" }
 sc-chain-spec = { version = "4.0.0-dev", path = "../../client/chain-spec" }
 sc-client-api = { version = "4.0.0-dev", path = "../api" }
+sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
 sc-finality-grandpa = { version = "0.10.0-dev", path = "../../client/finality-grandpa" }
 sc-keystore = { version = "4.0.0-dev", path = "../keystore" }
 sc-network = { version = "0.10.0-dev", path = "../network" }
@@ -42,7 +44,6 @@ serde = "1.0.136"
 strum = { version = "0.23", features = ["derive"] }
 tempfile = "3.1.0"
 tokio = "1.17.0"
-sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
 sc-network-test = { version = "0.8.0", path = "../network/test" }
 sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" }
 sp-keyring = { version = "6.0.0", path = "../../primitives/keyring" }

client/beefy/rpc/src/lib.rs

@@ -164,8 +164,9 @@ where
 mod tests {
 	use super::*;
 
-	use beefy_gadget::notification::{
-		BeefyBestBlockStream, BeefySignedCommitment, BeefySignedCommitmentSender,
+	use beefy_gadget::{
+		justification::BeefySignedCommitment,
+		notification::{BeefyBestBlockStream, BeefySignedCommitmentSender},
 	};
 	use beefy_primitives::{known_payload_ids, Payload};
 	use codec::{Decode, Encode};

client/beefy/rpc/src/notification.rs

@@ -29,7 +29,7 @@ pub struct EncodedSignedCommitment(sp_core::Bytes);
 
 impl EncodedSignedCommitment {
 	pub fn new<Block>(
-		signed_commitment: beefy_gadget::notification::BeefySignedCommitment<Block>,
+		signed_commitment: beefy_gadget::justification::BeefySignedCommitment<Block>,
 	) -> Self
 	where
 		Block: BlockT,

client/beefy/src/error.rs

@@ -24,8 +24,12 @@ use std::fmt::Debug;
 
 #[derive(Debug, thiserror::Error, PartialEq)]
 pub enum Error {
+	#[error("Backend: {0}")]
+	Backend(String),
 	#[error("Keystore error: {0}")]
 	Keystore(String),
 	#[error("Signature error: {0}")]
 	Signature(String),
+	#[error("Session uninitialized")]
+	UninitSession,
 }

client/beefy/src/import.rs

@@ -0,0 +1,141 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, marker::PhantomData, sync::Arc};
+
+use sp_api::{ProvideRuntimeApi, TransactionFor};
+use sp_blockchain::well_known_cache_keys;
+use sp_consensus::Error as ConsensusError;
+use sp_runtime::{
+	generic::BlockId,
+	traits::{Block as BlockT, Header as HeaderT},
+};
+
+use sc_client_api::backend::Backend;
+use sc_consensus::{BlockCheckParams, BlockImport, BlockImportParams, ImportResult};
+
+use crate::{
+	justification::decode_and_verify_commitment, notification::BeefySignedCommitmentSender,
+	Client as BeefyClient,
+};
+use beefy_primitives::{BeefyApi, VersionedFinalityProof, BEEFY_ENGINE_ID};
+
+/// A block-import handler for BEEFY.
+///
+/// This scans each imported block for BEEFY justifications and verifies them.
+/// Wraps a `inner: BlockImport` and ultimately defers to it.
+///
+/// When using BEEFY, the block import worker should be using this block import object.
+pub struct BeefyBlockImport<Backend, Block: BlockT, Client, I> {
+	client: Arc<Client>,
+	inner: I,
+	justification_sender: BeefySignedCommitmentSender<Block>,
+	_phantom: PhantomData<Backend>,
+}
+
+impl<BE, Block: BlockT, Client, I: Clone> Clone for BeefyBlockImport<BE, Block, Client, I> {
+	fn clone(&self) -> Self {
+		BeefyBlockImport {
+			client: self.client.clone(),
+			inner: self.inner.clone(),
+			justification_sender: self.justification_sender.clone(),
+			_phantom: PhantomData,
+		}
+	}
+}
+
+impl<BE, Block: BlockT, Client, I> BeefyBlockImport<BE, Block, Client, I> {
+	/// Create a new BeefyBlockImport.
+	pub fn new(
+		client: Arc<Client>,
+		inner: I,
+		justification_sender: BeefySignedCommitmentSender<Block>,
+	) -> BeefyBlockImport<BE, Block, Client, I> {
+		BeefyBlockImport { inner, client, justification_sender, _phantom: PhantomData }
+	}
+}
+
+#[async_trait::async_trait]
+impl<BE, Block: BlockT, Client, I> BlockImport<Block> for BeefyBlockImport<BE, Block, Client, I>
+where
+	BE: Backend<Block>,
+	I: BlockImport<
+			Block,
+			Error = ConsensusError,
+			Transaction = sp_api::TransactionFor<Client, Block>,
+		> + Send
+		+ Sync,
+	Client: BeefyClient<Block, BE> + ProvideRuntimeApi<Block>,
+	Client::Api: BeefyApi<Block>,
+{
+	type Error = ConsensusError;
+	type Transaction = TransactionFor<Client, Block>;
+
+	async fn import_block(
+		&mut self,
+		block: BlockImportParams<Block, Self::Transaction>,
+		new_cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error> {
+		let hash = block.post_hash();
+		let number = *block.header.number();
+		let beefy_justification = block
+			.justifications
+			.as_ref()
+			.and_then(|just| just.get(BEEFY_ENGINE_ID).cloned());
+
+		// Run inner block import.
+		let import_result = self.inner.import_block(block, new_cache).await?;
+
+		if let Some(justification) = beefy_justification {
+			if let ImportResult::Imported(_) = &import_result {
+				let block_id = BlockId::hash(hash);
+				let validator_set = self
+					.client
+					.runtime_api()
+					.validator_set(&block_id)
+					.map_err(|e| ConsensusError::ClientImport(e.to_string()))?
+					// Deploying `BeefyBlockImport` on chains with dummy BeefyApi will
+					// effectively reject all blocks with BEEFY justifications.
+					.ok_or_else(|| {
+						ConsensusError::ClientImport("Unknown validator set".to_string())
+					})?;
+
+				match decode_and_verify_commitment::<Block>(
+					&justification[..],
+					number,
+					&validator_set,
+				)? {
+					// TODO: these channels should also use `VersionedFinalityProof`.
+					VersionedFinalityProof::V1(signed_commitment) => self
+						.justification_sender
+						.notify(|| Ok::<_, ()>(signed_commitment))
+						.expect("forwards closure result; the closure always returns Ok; qed."),
+				};
+			}
+		}
+
+		Ok(import_result)
+	}
+
+	async fn check_block(
+		&mut self,
+		block: BlockCheckParams<Block>,
+	) -> Result<ImportResult, Self::Error> {
+		self.inner.check_block(block).await
+	}
+}

client/beefy/src/justification.rs

@@ -0,0 +1,177 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::keystore::BeefyKeystore;
+use beefy_primitives::{
+	crypto::{AuthorityId, Signature},
+	ValidatorSet, VersionedFinalityProof,
+};
+use codec::{Decode, Encode};
+use sp_consensus::Error as ConsensusError;
+use sp_runtime::traits::{Block as BlockT, NumberFor};
+
+/// A commitment with matching BEEFY authorities' signatures.
+pub type BeefySignedCommitment<Block> =
+	beefy_primitives::SignedCommitment<NumberFor<Block>, beefy_primitives::crypto::Signature>;
+
+/// Decode and verify a Beefy SignedCommitment.
+pub(crate) fn decode_and_verify_commitment<Block: BlockT>(
+	encoded: &[u8],
+	target_number: NumberFor<Block>,
+	validator_set: &ValidatorSet<AuthorityId>,
+) -> Result<VersionedFinalityProof<NumberFor<Block>, Signature>, ConsensusError> {
+	let proof = <VersionedFinalityProof<NumberFor<Block>, Signature>>::decode(&mut &*encoded)
+		.map_err(|_| ConsensusError::InvalidJustification)?;
+	verify_with_validator_set::<Block>(target_number, validator_set, &proof).map(|_| proof)
+}
+
+/// Verify the Beefy finality proof against the validator set at the block it was generated.
+fn verify_with_validator_set<Block: BlockT>(
+	target_number: NumberFor<Block>,
+	validator_set: &ValidatorSet<AuthorityId>,
+	proof: &VersionedFinalityProof<NumberFor<Block>, Signature>,
+) -> Result<(), ConsensusError> {
+	match proof {
+		VersionedFinalityProof::V1(signed_commitment) => {
+			if validator_set.len() != signed_commitment.signatures.len() ||
+				signed_commitment.commitment.validator_set_id != validator_set.id() ||
+				signed_commitment.commitment.block_number != target_number
+			{
+				return Err(ConsensusError::InvalidJustification)
+			}
+
+			// Arrangement of signatures in the commitment should be in the same order
+			// as validators for that set.
+			let message = signed_commitment.commitment.encode();
+			let valid_signatures = validator_set
+				.validators()
+				.into_iter()
+				.zip(signed_commitment.signatures.iter())
+				.filter(|(id, signature)| {
+					signature
+						.as_ref()
+						.map(|sig| BeefyKeystore::verify(id, sig, &message[..]))
+						.unwrap_or(false)
+				})
+				.count();
+			if valid_signatures >= crate::round::threshold(validator_set.len()) {
+				Ok(())
+			} else {
+				Err(ConsensusError::InvalidJustification)
+			}
+		},
+	}
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use beefy_primitives::{known_payload_ids, Commitment, Payload, SignedCommitment};
+	use substrate_test_runtime_client::runtime::Block;
+
+	use super::*;
+	use crate::{keystore::tests::Keyring, tests::make_beefy_ids};
+
+	pub(crate) fn new_signed_commitment(
+		block_num: NumberFor<Block>,
+		validator_set: &ValidatorSet<AuthorityId>,
+		keys: &[Keyring],
+	) -> BeefySignedCommitment<Block> {
+		let commitment = Commitment {
+			payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+			block_number: block_num,
+			validator_set_id: validator_set.id(),
+		};
+		let message = commitment.encode();
+		let signatures = keys.iter().map(|key| Some(key.sign(&message))).collect();
+		SignedCommitment { commitment, signatures }
+	}
+
+	#[test]
+	fn should_verify_with_validator_set() {
+		let keys = &[Keyring::Alice, Keyring::Bob, Keyring::Charlie];
+		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
+
+		// build valid justification
+		let block_num = 42;
+		let proof = new_signed_commitment(block_num, &validator_set, keys);
+
+		let good_proof = proof.clone().into();
+		// should verify successfully
+		verify_with_validator_set::<Block>(block_num, &validator_set, &good_proof).unwrap();
+
+		// wrong block number -> should fail verification
+		let good_proof = proof.clone().into();
+		match verify_with_validator_set::<Block>(block_num + 1, &validator_set, &good_proof) {
+			Err(ConsensusError::InvalidJustification) => (),
+			_ => assert!(false, "Expected Err(ConsensusError::InvalidJustification)"),
+		};
+
+		// wrong validator set id -> should fail verification
+		let good_proof = proof.clone().into();
+		let other = ValidatorSet::new(make_beefy_ids(keys), 1).unwrap();
+		match verify_with_validator_set::<Block>(block_num, &other, &good_proof) {
+			Err(ConsensusError::InvalidJustification) => (),
+			_ => assert!(false, "Expected Err(ConsensusError::InvalidJustification)"),
+		};
+
+		// wrong signatures length -> should fail verification
+		let mut bad_proof = proof.clone();
+		// change length of signatures
+		bad_proof.signatures.pop().flatten().unwrap();
+		match verify_with_validator_set::<Block>(block_num + 1, &validator_set, &bad_proof.into()) {
+			Err(ConsensusError::InvalidJustification) => (),
+			_ => assert!(false, "Expected Err(ConsensusError::InvalidJustification)"),
+		};
+
+		// not enough signatures -> should fail verification
+		let mut bad_proof = proof.clone();
+		// remove a signature (but same length)
+		*bad_proof.signatures.first_mut().unwrap() = None;
+		match verify_with_validator_set::<Block>(block_num + 1, &validator_set, &bad_proof.into()) {
+			Err(ConsensusError::InvalidJustification) => (),
+			_ => assert!(false, "Expected Err(ConsensusError::InvalidJustification)"),
+		};
+
+		// not enough _correct_ signatures -> should fail verification
+		let mut bad_proof = proof.clone();
+		// change a signature to a different key
+		*bad_proof.signatures.first_mut().unwrap() =
+			Some(Keyring::Dave.sign(&proof.commitment.encode()));
+		match verify_with_validator_set::<Block>(block_num + 1, &validator_set, &bad_proof.into()) {
+			Err(ConsensusError::InvalidJustification) => (),
+			_ => assert!(false, "Expected Err(ConsensusError::InvalidJustification)"),
+		};
+	}
+
+	#[test]
+	fn should_decode_and_verify_commitment() {
+		let keys = &[Keyring::Alice, Keyring::Bob];
+		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
+		let block_num = 1;
+
+		// build valid justification
+		let proof = new_signed_commitment(block_num, &validator_set, keys);
+		let versioned_proof: VersionedFinalityProof<NumberFor<Block>, Signature> = proof.into();
+		let encoded = versioned_proof.encode();
+
+		// should successfully decode and verify
+		let verified =
+			decode_and_verify_commitment::<Block>(&encoded, block_num, &validator_set).unwrap();
+		assert_eq!(verified, versioned_proof);
+	}
+}