Update MMR Runtime API with functionality to generate MMR proof for a…

… series of leaf indices (#10635)

* updated mmr rpc api with functions for batch generation of proof

* update code comments

* fix build errors

* added tests to mmr-rpc

* add tests to pallet-mmr

* update comments

* minor comment fix

* remove unused variables

* fix rust doc errors

* refactor mmr runtime api

* fix tests

* minor fix

* minor fix

* fix node-runtime

* revert to initial api

* impl from proof fot batchproof

* minor fix

* minor fix

* use explicit functions to convert btw batch proof and single proof

* minor fix

* add new variant to mmr error

* fmt

* update conversion to single leaf proof

* fix style nit

Co-authored-by: Adrian Catangiu <[email protected]>

bin/node/runtime/src/lib.rs

@@ -1832,8 +1832,12 @@ impl_runtime_apis! {
 		fn generate_proof(leaf_index: pallet_mmr::primitives::LeafIndex)
 			-> Result<(mmr::EncodableOpaqueLeaf, mmr::Proof<mmr::Hash>), mmr::Error>
 		{
-			Mmr::generate_proof(leaf_index)
-				.map(|(leaf, proof)| (mmr::EncodableOpaqueLeaf::from_leaf(&leaf), proof))
+			Mmr::generate_batch_proof(vec![leaf_index]).and_then(|(leaves, proof)|
+				Ok((
+					mmr::EncodableOpaqueLeaf::from_leaf(&leaves[0]),
+					mmr::BatchProof::into_single_leaf_proof(proof)?
+				))
+			)
 		}
 
 		fn verify_proof(leaf: mmr::EncodableOpaqueLeaf, proof: mmr::Proof<mmr::Hash>)
@@ -1843,7 +1847,7 @@ impl_runtime_apis! {
 				.into_opaque_leaf()
 				.try_decode()
 				.ok_or(mmr::Error::Verify)?;
-			Mmr::verify_leaf(leaf, proof)
+			Mmr::verify_leaves(vec![leaf], mmr::Proof::into_batch_proof(proof))
 		}
 
 		fn verify_proof_stateless(
@@ -1852,12 +1856,38 @@ impl_runtime_apis! {
 			proof: mmr::Proof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let node = mmr::DataOrHash::Data(leaf.into_opaque_leaf());
-			pallet_mmr::verify_leaf_proof::<mmr::Hashing, _>(root, node, proof)
+			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, vec![node], mmr::Proof::into_batch_proof(proof))
 		}
 
 		fn mmr_root() -> Result<mmr::Hash, mmr::Error> {
 			Ok(Mmr::mmr_root())
 		}
+
+		fn generate_batch_proof(leaf_indices: Vec<pallet_mmr::primitives::LeafIndex>)
+			-> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<mmr::Hash>), mmr::Error>
+		{
+			Mmr::generate_batch_proof(leaf_indices)
+				.map(|(leaves, proof)| (leaves.into_iter().map(|leaf| mmr::EncodableOpaqueLeaf::from_leaf(&leaf)).collect(), proof))
+		}
+
+		fn verify_batch_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::BatchProof<mmr::Hash>)
+			-> Result<(), mmr::Error>
+		{
+			let leaves = leaves.into_iter().map(|leaf|
+				leaf.into_opaque_leaf()
+				.try_decode()
+				.ok_or(mmr::Error::Verify)).collect::<Result<Vec<mmr::Leaf>, mmr::Error>>()?;
+			Mmr::verify_leaves(leaves, proof)
+		}
+
+		fn verify_batch_proof_stateless(
+			root: mmr::Hash,
+			leaves: Vec<mmr::EncodableOpaqueLeaf>,
+			proof: mmr::BatchProof<mmr::Hash>
+		) -> Result<(), mmr::Error> {
+			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
+			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
+		}
 	}
 
 	impl sp_session::SessionKeys<Block> for Runtime {

client/beefy/src/tests.rs

@@ -39,7 +39,9 @@ use beefy_primitives::{
 	crypto::AuthorityId, BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, BEEFY_ENGINE_ID,
 	KEY_TYPE as BeefyKeyType,
 };
-use sp_mmr_primitives::{EncodableOpaqueLeaf, Error as MmrError, LeafIndex, MmrApi, Proof};
+use sp_mmr_primitives::{
+	BatchProof, EncodableOpaqueLeaf, Error as MmrError, LeafIndex, MmrApi, Proof,
+};
 
 use sp_api::{ApiRef, ProvideRuntimeApi};
 use sp_consensus::BlockOrigin;
@@ -259,6 +261,22 @@ macro_rules! create_test_api {
 					fn mmr_root() -> Result<MmrRootHash, MmrError> {
 						Ok($mmr_root)
 					}
+
+					fn generate_batch_proof(_leaf_indices: Vec<LeafIndex>) -> Result<(Vec<EncodableOpaqueLeaf>, BatchProof<MmrRootHash>), MmrError> {
+						unimplemented!()
+					}
+
+					fn verify_batch_proof(_leaves: Vec<EncodableOpaqueLeaf>, _proof: BatchProof<MmrRootHash>) -> Result<(), MmrError> {
+						unimplemented!()
+					}
+
+					fn verify_batch_proof_stateless(
+						_root: MmrRootHash,
+						_leaves: Vec<EncodableOpaqueLeaf>,
+						_proof: BatchProof<MmrRootHash>
+					) -> Result<(), MmrError> {
+						unimplemented!()
+					}
 				}
 			}
 		}

frame/merkle-mountain-range/rpc/Cargo.toml

@@ -22,7 +22,7 @@ serde = { version = "1.0.136", features = ["derive"] }
 sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" }
 sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" }
 sp-core = { version = "6.0.0", path = "../../../primitives/core" }
-sp-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/merkle-mountain-range" }
+sp-mmr-primitives = { version = "4.0.0-dev", path = "../../../primitives/merkle-mountain-range" }
 sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
 
 [dev-dependencies]

frame/merkle-mountain-range/rpc/src/lib.rs

@@ -29,7 +29,7 @@ use serde::{Deserialize, Serialize};
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_core::Bytes;
-use sp_mmr_primitives::{Error as MmrError, LeafIndex, Proof};
+use sp_mmr_primitives::{BatchProof, Error as MmrError, LeafIndex, Proof};
 use sp_runtime::{generic::BlockId, traits::Block as BlockT};
 
 pub use sp_mmr_primitives::MmrApi as MmrRuntimeApi;
@@ -57,6 +57,34 @@ impl<BlockHash> LeafProof<BlockHash> {
 	}
 }
 
+/// Retrieved MMR leaves and their proof.
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub struct LeafBatchProof<BlockHash> {
+	/// Block hash the proof was generated for.
+	pub block_hash: BlockHash,
+	/// SCALE-encoded vector of `LeafData`.
+	pub leaves: Bytes,
+	/// SCALE-encoded proof data. See [sp_mmr_primitives::BatchProof].
+	pub proof: Bytes,
+}
+
+impl<BlockHash> LeafBatchProof<BlockHash> {
+	/// Create new `LeafBatchProof` from a given vector of `Leaf` and a
+	/// [sp_mmr_primitives::BatchProof].
+	pub fn new<Leaf, MmrHash>(
+		block_hash: BlockHash,
+		leaves: Vec<Leaf>,
+		proof: BatchProof<MmrHash>,
+	) -> Self
+	where
+		Leaf: Encode,
+		MmrHash: Encode,
+	{
+		Self { block_hash, leaves: Bytes(leaves.encode()), proof: Bytes(proof.encode()) }
+	}
+}
+
 /// MMR RPC methods.
 #[rpc]
 pub trait MmrApi<BlockHash> {
@@ -74,6 +102,23 @@ pub trait MmrApi<BlockHash> {
 		leaf_index: LeafIndex,
 		at: Option<BlockHash>,
 	) -> Result<LeafProof<BlockHash>>;
+
+	/// Generate MMR proof for the given leaf indices.
+	///
+	/// This method calls into a runtime with MMR pallet included and attempts to generate
+	/// MMR proof for a set of leaves at the given `leaf_indices`.
+	/// Optionally, a block hash at which the runtime should be queried can be specified.
+	///
+	/// Returns the leaves and a proof for these leaves (compact encoding, i.e. hash of
+	/// the leaves). Both parameters are SCALE-encoded.
+	/// The order of entries in the `leaves` field of the returned struct
+	/// is the same as the order of the entries in `leaf_indices` supplied
+	#[rpc(name = "mmr_generateBatchProof")]
+	fn generate_batch_proof(
+		&self,
+		leaf_indices: Vec<LeafIndex>,
+		at: Option<BlockHash>,
+	) -> Result<LeafBatchProof<BlockHash>>;
 }
 
 /// An implementation of MMR specific RPC methods.
@@ -117,6 +162,28 @@ where
 
 		Ok(LeafProof::new(block_hash, leaf, proof))
 	}
+
+	fn generate_batch_proof(
+		&self,
+		leaf_indices: Vec<LeafIndex>,
+		at: Option<<Block as BlockT>::Hash>,
+	) -> Result<LeafBatchProof<<Block as BlockT>::Hash>> {
+		let api = self.client.runtime_api();
+		let block_hash = at.unwrap_or_else(||
+			// If the block hash is not supplied assume the best block.
+			self.client.info().best_hash);
+
+		let (leaves, proof) = api
+			.generate_batch_proof_with_context(
+				&BlockId::hash(block_hash),
+				sp_core::ExecutionContext::OffchainCall(None),
+				leaf_indices,
+			)
+			.map_err(runtime_error_into_rpc_error)?
+			.map_err(mmr_error_into_rpc_error)?;
+
+		Ok(LeafBatchProof::new(block_hash, leaves, proof))
+	}
 }
 
 const RUNTIME_ERROR: i64 = 8000;
@@ -179,6 +246,28 @@ mod tests {
 		);
 	}
 
+	#[test]
+	fn should_serialize_leaf_batch_proof() {
+		// given
+		let leaf = vec![1_u8, 2, 3, 4];
+		let proof = BatchProof {
+			leaf_indices: vec![1],
+			leaf_count: 9,
+			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+		};
+
+		let leaf_proof = LeafBatchProof::new(H256::repeat_byte(0), vec![leaf], proof);
+
+		// when
+		let actual = serde_json::to_string(&leaf_proof).unwrap();
+
+		// then
+		assert_eq!(
+			actual,
+			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x041001020304","proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"#
+		);
+	}
+
 	#[test]
 	fn should_deserialize_leaf_proof() {
 		// given
@@ -205,4 +294,31 @@ mod tests {
 		// then
 		assert_eq!(actual, expected);
 	}
+
+	#[test]
+	fn should_deserialize_leaf_batch_proof() {
+		// given
+		let expected = LeafBatchProof {
+			block_hash: H256::repeat_byte(0),
+			leaves: Bytes(vec![vec![1_u8, 2, 3, 4]].encode()),
+			proof: Bytes(
+				BatchProof {
+					leaf_indices: vec![1],
+					leaf_count: 9,
+					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+				}
+				.encode(),
+			),
+		};
+
+		// when
+		let actual: LeafBatchProof<H256> = serde_json::from_str(r#"{
+			"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+			"leaves":"0x041001020304",
+			"proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"
+		}"#).unwrap();
+
+		// then
+		assert_eq!(actual, expected);
+	}
 }

frame/merkle-mountain-range/src/lib.rs

@@ -71,6 +71,7 @@ mod tests;
 
 pub use pallet::*;
 pub use sp_mmr_primitives::{self as primitives, Error, LeafDataProvider, LeafIndex, NodeIndex};
+use sp_std::prelude::*;
 
 /// The most common use case for MMRs is to store historical block hashes,
 /// so that any point in time in the future we can receive a proof about some past
@@ -228,22 +229,23 @@ type LeafOf<T, I> = <<T as Config<I>>::LeafData as primitives::LeafDataProvider>
 /// Hashing used for the pallet.
 pub(crate) type HashingOf<T, I> = <T as Config<I>>::Hashing;
 
-/// Stateless MMR proof verification.
+/// Stateless MMR proof verification for batch of leaves.
 ///
-/// This function can be used to verify received MMR proof (`proof`)
-/// for given leaf data (`leaf`) against a known MMR root hash (`root`).
-///
-/// The verification does not require any storage access.
-pub fn verify_leaf_proof<H, L>(
+/// This function can be used to verify received MMR [primitives::BatchProof] (`proof`)
+/// for given leaves set (`leaves`) against a known MMR root hash (`root`).
+/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
+/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the
+/// [primitives::BatchProof].
+pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
-	leaf: mmr::Node<H, L>,
-	proof: primitives::Proof<H::Output>,
+	leaves: Vec<mmr::Node<H, L>>,
+	proof: primitives::BatchProof<H::Output>,
 ) -> Result<(), primitives::Error>
 where
 	H: traits::Hash,
 	L: primitives::FullLeaf,
 {
-	let is_valid = mmr::verify_leaf_proof::<H, L>(root, leaf, proof)?;
+	let is_valid = mmr::verify_leaves_proof::<H, L>(root, leaves, proof)?;
 	if is_valid {
 		Ok(())
 	} else {
@@ -255,29 +257,36 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	fn offchain_key(pos: NodeIndex) -> sp_std::prelude::Vec<u8> {
 		(T::INDEXING_PREFIX, pos).encode()
 	}
-
-	/// Generate a MMR proof for the given `leaf_index`.
+	/// Generate a MMR proof for the given `leaf_indices`.
 	///
 	/// Note this method can only be used from an off-chain context
 	/// (Offchain Worker or Runtime API call), since it requires
 	/// all the leaves to be present.
 	/// It may return an error or panic if used incorrectly.
-	pub fn generate_proof(
-		leaf_index: LeafIndex,
-	) -> Result<(LeafOf<T, I>, primitives::Proof<<T as Config<I>>::Hash>), primitives::Error> {
+	pub fn generate_batch_proof(
+		leaf_indices: Vec<NodeIndex>,
+	) -> Result<
+		(Vec<LeafOf<T, I>>, primitives::BatchProof<<T as Config<I>>::Hash>),
+		primitives::Error,
+	> {
 		let mmr: ModuleMmr<mmr::storage::OffchainStorage, T, I> = mmr::Mmr::new(Self::mmr_leaves());
-		mmr.generate_proof(leaf_index)
+		mmr.generate_batch_proof(leaf_indices)
+	}
+
+	/// Return the on-chain MMR root hash.
+	pub fn mmr_root() -> <T as Config<I>>::Hash {
+		Self::mmr_root_hash()
 	}
 
-	/// Verify MMR proof for given `leaf`.
+	/// Verify MMR proof for given `leaves`.
 	///
 	/// This method is safe to use within the runtime code.
 	/// It will return `Ok(())` if the proof is valid
 	/// and an `Err(..)` if MMR is inconsistent (some leaves are missing)
 	/// or the proof is invalid.
-	pub fn verify_leaf(
-		leaf: LeafOf<T, I>,
-		proof: primitives::Proof<<T as Config<I>>::Hash>,
+	pub fn verify_leaves(
+		leaves: Vec<LeafOf<T, I>>,
+		proof: primitives::BatchProof<<T as Config<I>>::Hash>,
 	) -> Result<(), primitives::Error> {
 		if proof.leaf_count > Self::mmr_leaves() ||
 			proof.leaf_count == 0 ||
@@ -288,16 +297,11 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 		}
 
 		let mmr: ModuleMmr<mmr::storage::OffchainStorage, T, I> = mmr::Mmr::new(proof.leaf_count);
-		let is_valid = mmr.verify_leaf_proof(leaf, proof)?;
+		let is_valid = mmr.verify_leaves_proof(leaves, proof)?;
 		if is_valid {
 			Ok(())
 		} else {
 			Err(primitives::Error::Verify.log_debug("The proof is incorrect."))
 		}
 	}
-
-	/// Return the on-chain MMR root hash.
-	pub fn mmr_root() -> <T as Config<I>>::Hash {
-		Self::mmr_root_hash()
-	}
 }