Improve chain connector storage (#9198)

* ♻️ Add ability to save inclusion proofs at every new block
- Add config to take keepInclusionProofsForHeights and inclusionProofKeys
- Add DB update to get/set/delete inclusionProofs
- Add handler in new block event to save inclusion proofs
- Add endpoint for chain to get inclusion proofs by height

* ♻️ Move inclusion proof collection to consensus

* ♻️ Save inclusionProofs on both block receive and generate

* ♻️  Improve chain connector plugin
- Improve storage to retrieve by key/val and delete efficiently instead of managing full array
- Reduce number of RPC calls during new block event
- Improve cleanup

* ♻️ Update plugin to manage last sent CCU and CCM

* ♻️ Update interop examples

* 🌱 Add license

* ♻️ Manage validatorsHash by height

* ♻️ Improve CCU storage management

* ♻️ Add tests and improve code base

* ✅ Add tests for block event handler

* ✅ Add ccmHandler unit tests

* ♻️ Unable CCU sending when receiving chain is syncing

* ♻️  Handle error at new block subscribe

Author: ishantiw

Diff for: examples/interop/pos-mainchain-fast/config/custom_config_node_one.json

@@ -1,7 +1,12 @@
 {
 	"system": {
-		"dataPath": "~/.lisk/pos-mainchain-node-one",
-		"logLevel": "info"
+		"dataPath": "~/.lisk/mainchain-node-one",
+		"logLevel": "debug",
+		"keepEventsForHeights": -1,
+		"keepInclusionProofsForHeights": -1,
+		"inclusionProofKeys": [
+			"83ed0d250000160811fdaf692ba77eabfbfc3a6bb3c4cf6a87beafd28cfe90b5dc64cb20ab46"
+		]
 	},
 	"rpc": {
 		"modes": ["ipc"],

Diff for: examples/interop/pos-mainchain-fast/config/custom_config_node_two.json

@@ -1,7 +1,12 @@
 {
 	"system": {
-		"dataPath": "~/.lisk/pos-mainchain-node-two",
-		"logLevel": "info"
+		"dataPath": "~/.lisk/mainchain-node-two",
+		"logLevel": "debug",
+		"keepEventsForHeights": -1,
+		"keepInclusionProofsForHeights": -1,
+		"inclusionProofKeys": [
+			"83ed0d250000ac894ab085f50b81fe000e99ccb27e5543de69f63d4f9105daab15dce90f81b3"
+		]
 	},
 	"rpc": {
 		"modes": ["ipc"],

Diff for: examples/interop/pos-mainchain-fast/config/default/config.json

@@ -1,13 +1,4 @@
 {
-	"system": {
-		"dataPath": "~/.lisk/pos-mainchain-fast",
-		"keepEventsForHeights": -1,
-		"keepInclusionProofsForHeights": -1,
-		"inclusionProofKeys": [
-			"83ed0d250000160811fdaf692ba77eabfbfc3a6bb3c4cf6a87beafd28cfe90b5dc64cb20ab46"
-		],
-		"logLevel": "info"
-	},
 	"rpc": {
 		"modes": ["ipc"],
 		"port": 7881,

Diff for: examples/interop/pos-sidechain-example-one/config/default/config.json

@@ -2,7 +2,11 @@
 	"system": {
 		"dataPath": "~/.lisk/pos-sidechain-example-one",
 		"keepEventsForHeights": 300,
-		"logLevel": "info"
+		"logLevel": "info",
+		"keepInclusionProofsForHeights": -1,
+		"inclusionProofKeys": [
+			"83ed0d250000fb5e512425fc9449316ec95969ebe71e2d576dbab833d61e2a5b9330fd70ee02"
+		]
 	},
 	"rpc": {
 		"modes": ["ipc"],

Diff for: examples/interop/pos-sidechain-example-two/config/default/config.json

@@ -2,7 +2,11 @@
 	"system": {
 		"dataPath": "~/.lisk/pos-sidechain-example-two",
 		"keepEventsForHeights": 300,
-		"logLevel": "info"
+		"logLevel": "info",
+		"keepInclusionProofsForHeights": -1,
+		"inclusionProofKeys": [
+			"83ed0d250000fb5e512425fc9449316ec95969ebe71e2d576dbab833d61e2a5b9330fd70ee02"
+		]
 	},
 	"rpc": {
 		"modes": ["ipc"],

Diff for: framework-plugins/lisk-framework-chain-connector-plugin/src/active_validators_update.ts

@@ -12,53 +12,31 @@
  * Removal or modification of this copyright notice is prohibited.
  */
 /* eslint-disable no-bitwise */
-import {
-	ActiveValidator,
-	Certificate,
-	LastCertificate,
-	utils,
-	ActiveValidatorsUpdate,
-} from 'lisk-sdk';
-import { ValidatorsData } from './types';
+import { ActiveValidator, utils, ActiveValidatorsUpdate } from 'lisk-sdk';
+import { ValidatorsDataWithHeight } from './types';
 
 /**
  * @see https://github.com/LiskHQ/lips/blob/main/proposals/lip-0053.md#computing-the-validators-update
  */
+
 export const calculateActiveValidatorsUpdate = (
-	certificate: Certificate,
-	validatorsHashPreimage: ValidatorsData[],
-	lastCertificate: LastCertificate,
+	validatorsDataAtLastCertificate: ValidatorsDataWithHeight,
+	validatorsDataAtNewCertificate: ValidatorsDataWithHeight,
 ): { activeValidatorsUpdate: ActiveValidatorsUpdate; certificateThreshold: bigint } => {
 	let certificateThreshold;
-	const validatorDataAtCertificate = validatorsHashPreimage.find(validatorsData =>
-		validatorsData.validatorsHash.equals(certificate.validatorsHash),
-	);
-
-	if (!validatorDataAtCertificate) {
-		throw new Error('No validators data found for the certificate height.');
-	}
-
-	const validatorDataAtLastCertificate = validatorsHashPreimage.find(validatorsData =>
-		validatorsData.validatorsHash.equals(lastCertificate.validatorsHash),
-	);
-
-	if (!validatorDataAtLastCertificate) {
-		throw new Error('No validators data found for the given last certificate height.');
-	}
-
 	// If the certificate threshold is not changed from last certificate then we assign zero
 	if (
-		validatorDataAtCertificate.certificateThreshold ===
-		validatorDataAtLastCertificate.certificateThreshold
+		validatorsDataAtNewCertificate.certificateThreshold ===
+		validatorsDataAtLastCertificate.certificateThreshold
 	) {
-		certificateThreshold = validatorDataAtLastCertificate.certificateThreshold;
+		certificateThreshold = validatorsDataAtLastCertificate.certificateThreshold;
 	} else {
-		certificateThreshold = validatorDataAtCertificate.certificateThreshold;
+		certificateThreshold = validatorsDataAtNewCertificate.certificateThreshold;
 	}
 
 	const activeValidatorsUpdate = getActiveValidatorsUpdate(
-		validatorDataAtLastCertificate.validators,
-		validatorDataAtCertificate.validators,
+		validatorsDataAtLastCertificate.validators,
+		validatorsDataAtNewCertificate.validators,
 	);
 
 	return { activeValidatorsUpdate, certificateThreshold };