From dd112e0b723e469a00e0a11bf56315e6b51a0711 Mon Sep 17 00:00:00 2001 From: tgmichel Date: Wed, 25 Aug 2021 17:10:14 +0200 Subject: [PATCH] Fix bloom matching (#457) * Fix bloom matching * Tests * Edit changelog * Handle empty `[]` requests + test --- client/rpc-core/CHANGELOG.md | 2 +- client/rpc-core/src/types/filter.rs | 410 +++++++++++++++++++++++++--- client/rpc/src/eth.rs | 7 +- 3 files changed, 378 insertions(+), 41 deletions(-) diff --git a/client/rpc-core/CHANGELOG.md b/client/rpc-core/CHANGELOG.md index f66539ef4b..db554ec8c6 100644 --- a/client/rpc-core/CHANGELOG.md +++ b/client/rpc-core/CHANGELOG.md @@ -1,4 +1,4 @@ # Changelog for `fc-rpc-core` ## Unreleased -- Add `FilteredParams::in_bloom()` function to check the possible existance of Filter addresses or topics in a block. +- Add `FilteredParams::address_in_bloom()` and `FilteredParams::topics_in_bloom()` functions to check the possible existance of Filter addresses or topics in a block. diff --git a/client/rpc-core/src/types/filter.rs b/client/rpc-core/src/types/filter.rs index 41772014b4..13c2c9ee9b 100644 --- a/client/rpc-core/src/types/filter.rs +++ b/client/rpc-core/src/types/filter.rs @@ -71,6 +71,62 @@ pub type FlatTopic = VariadicValue>; pub type BloomFilter<'a> = Vec>; +impl From<&VariadicValue> for Vec> { + fn from(address: &VariadicValue) -> Self { + let mut blooms = BloomFilter::new(); + match address { + VariadicValue::Single(address) => { + let bloom: Bloom = BloomInput::Raw(address.as_ref()).into(); + blooms.push(Some(bloom)) + } + VariadicValue::Multiple(addresses) => { + if addresses.len() == 0 { + blooms.push(None); + } else { + for address in addresses.into_iter() { + let bloom: Bloom = BloomInput::Raw(address.as_ref()).into(); + blooms.push(Some(bloom)); + } + } + } + _ => blooms.push(None), + } + blooms + } +} + +impl From<&VariadicValue>> for Vec> { + fn from(topics: &VariadicValue>) -> Self { + let mut blooms = BloomFilter::new(); + match topics { + VariadicValue::Single(topic) => { + if let Some(topic) = topic { + let bloom: Bloom = BloomInput::Raw(topic.as_ref()).into(); + blooms.push(Some(bloom)); + } else { + blooms.push(None); + } + } + VariadicValue::Multiple(topics) => { + if topics.len() == 0 { + blooms.push(None); + } else { + for topic in topics.into_iter() { + if let Some(topic) = topic { + let bloom: Bloom = BloomInput::Raw(topic.as_ref()).into(); + blooms.push(Some(bloom)); + } else { + blooms.push(None); + } + } + } + } + _ => blooms.push(None), + } + blooms + } +} + /// Filter #[derive(Debug, PartialEq, Clone, Deserialize, Eq, Hash)] #[serde(deny_unknown_fields)] @@ -122,55 +178,62 @@ impl FilteredParams { Self::default() } - pub fn bloom_filter<'a>( - address: &'a Option, - topics: &'a Option>, - ) -> BloomFilter<'a> { - let mut blooms = BloomFilter::new(); - // Address + /// Build an address-based BloomFilter. + pub fn adresses_bloom_filter<'a>(address: &'a Option) -> BloomFilter<'a> { if let Some(address) = address { - match address { - VariadicValue::Single(address) => { - let bloom: Bloom = BloomInput::Raw(address.as_ref()).into(); - blooms.push(Some(bloom)) - } - VariadicValue::Multiple(addresses) => { - for address in addresses.into_iter() { - let bloom: Bloom = BloomInput::Raw(address.as_ref()).into(); - blooms.push(Some(bloom)) - } - } - _ => blooms.push(None), - } + return address.into(); } - // Topics + Vec::new() + } + + /// Build a topic-based BloomFilter. + pub fn topics_bloom_filter<'a>(topics: &'a Option>) -> Vec> { + let mut output: Vec = Vec::new(); if let Some(topics) = topics { for flat in topics { - match flat { - VariadicValue::Single(topic) => { - if let Some(topic) = topic { - let bloom: Bloom = BloomInput::Raw(topic.as_ref()).into(); - blooms.push(Some(bloom)); - } else { - blooms.push(None); - } - } - _ => blooms.push(None), + output.push(flat.into()); + } + } + output + } + + /// Evaluates if a Bloom contains a provided sequence of topics. + pub fn topics_in_bloom(bloom: Bloom, topic_bloom_filters: &Vec) -> bool { + if topic_bloom_filters.len() == 0 { + // No filter provided, match. + return true; + } + // A logical OR evaluation over `topic_bloom_filters`. + for subset in topic_bloom_filters.into_iter() { + let mut matches = false; + for el in subset { + matches = match el { + Some(input) => bloom.contains_bloom(input), + // Wildcards are true. + None => true, + }; + // Each subset must be evaluated sequentially to true or break. + if !matches { + break; } } + // If any subset is fully evaluated to true, there is no further evaluation. + if matches { + return true; + } } - blooms + false } - /// Checks the possible existance of an address or topic(s) in a Bloom. - /// Wildcards (VariadicValue::Null) are matched as positive. - pub fn in_bloom(bloom: Bloom, bloom_input: &BloomFilter) -> bool { - if bloom_input.len() == 0 { + /// Evaluates if a Bloom contains the provided address(es). + pub fn address_in_bloom(bloom: Bloom, address_bloom_filter: &BloomFilter) -> bool { + if address_bloom_filter.len() == 0 { + // No filter provided, match. return true; } else { - for inner in bloom_input { - // Wildcard (None) or matching topic. - if match inner { + // Wildcards are true. + for el in address_bloom_filter { + if match el { Some(input) => bloom.contains_bloom(input), None => true, } { @@ -414,3 +477,274 @@ pub struct FilterPoolItem { /// On-memory stored filters created through the `eth_newFilter` RPC. pub type FilterPool = Arc>>; + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + fn block_bloom() -> Bloom { + let test_address = H160::from_str("1000000000000000000000000000000000000000").unwrap(); + let topic1 = + H256::from_str("1000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic2 = + H256::from_str("2000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + + let mut block_bloom = Bloom::default(); + block_bloom.accrue(BloomInput::Raw(&test_address[..])); + block_bloom.accrue(BloomInput::Raw(&topic1[..])); + block_bloom.accrue(BloomInput::Raw(&topic2[..])); + block_bloom + } + + #[test] + fn bloom_filter_should_match_by_address() { + let test_address = H160::from_str("1000000000000000000000000000000000000000").unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: Some(VariadicValue::Single(test_address)), + topics: None, + }; + let address_bloom = FilteredParams::adresses_bloom_filter(&filter.address); + assert!(FilteredParams::address_in_bloom( + block_bloom(), + &address_bloom + )); + } + + #[test] + fn bloom_filter_should_not_match_by_address() { + let test_address = H160::from_str("2000000000000000000000000000000000000000").unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: Some(VariadicValue::Single(test_address)), + topics: None, + }; + let address_bloom = FilteredParams::adresses_bloom_filter(&filter.address); + assert!(!FilteredParams::address_in_bloom( + block_bloom(), + &address_bloom + )); + } + #[test] + fn bloom_filter_should_match_by_topic() { + let topic1 = + H256::from_str("1000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic2 = + H256::from_str("2000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic3 = + H256::from_str("3000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: None, + topics: Some(VariadicValue::Multiple(vec![ + Some(VariadicValue::Single(Some(topic1))), + Some(VariadicValue::Multiple(vec![Some(topic2), Some(topic3)])), + ])), + }; + let topics_input = if let Some(_) = &filter.topics { + let filtered_params = FilteredParams::new(Some(filter.clone())); + Some(filtered_params.flat_topics) + } else { + None + }; + let topics_bloom = FilteredParams::topics_bloom_filter(&topics_input); + assert!(FilteredParams::topics_in_bloom( + block_bloom(), + &topics_bloom + )); + } + #[test] + fn bloom_filter_should_not_match_by_topic() { + let topic1 = + H256::from_str("1000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic2 = + H256::from_str("4000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic3 = + H256::from_str("5000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: None, + topics: Some(VariadicValue::Multiple(vec![ + Some(VariadicValue::Single(Some(topic1))), + Some(VariadicValue::Multiple(vec![Some(topic2), Some(topic3)])), + ])), + }; + let topics_input = if let Some(_) = &filter.topics { + let filtered_params = FilteredParams::new(Some(filter.clone())); + Some(filtered_params.flat_topics) + } else { + None + }; + let topics_bloom = FilteredParams::topics_bloom_filter(&topics_input); + assert!(!FilteredParams::topics_in_bloom( + block_bloom(), + &topics_bloom + )); + } + #[test] + fn bloom_filter_should_match_by_empty_topic() { + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: None, + topics: Some(VariadicValue::Multiple(vec![])), + }; + let topics_input = if let Some(_) = &filter.topics { + let filtered_params = FilteredParams::new(Some(filter.clone())); + Some(filtered_params.flat_topics) + } else { + None + }; + let topics_bloom = FilteredParams::topics_bloom_filter(&topics_input); + assert!(FilteredParams::topics_in_bloom( + block_bloom(), + &topics_bloom + )); + } + #[test] + fn bloom_filter_should_match_combined() { + let test_address = H160::from_str("1000000000000000000000000000000000000000").unwrap(); + let topic1 = + H256::from_str("1000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic2 = + H256::from_str("2000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic3 = + H256::from_str("3000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: Some(VariadicValue::Single(test_address)), + topics: Some(VariadicValue::Multiple(vec![ + Some(VariadicValue::Single(Some(topic1))), + Some(VariadicValue::Multiple(vec![Some(topic2), Some(topic3)])), + ])), + }; + let topics_input = if let Some(_) = &filter.topics { + let filtered_params = FilteredParams::new(Some(filter.clone())); + Some(filtered_params.flat_topics) + } else { + None + }; + let address_bloom = FilteredParams::adresses_bloom_filter(&filter.address); + let topics_bloom = FilteredParams::topics_bloom_filter(&topics_input); + let matches = FilteredParams::address_in_bloom(block_bloom(), &address_bloom) + && FilteredParams::topics_in_bloom(block_bloom(), &topics_bloom); + assert!(matches); + } + #[test] + fn bloom_filter_should_not_match_combined() { + let test_address = H160::from_str("2000000000000000000000000000000000000000").unwrap(); + let topic1 = + H256::from_str("1000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic2 = + H256::from_str("2000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic3 = + H256::from_str("3000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: Some(VariadicValue::Single(test_address)), + topics: Some(VariadicValue::Multiple(vec![ + Some(VariadicValue::Single(Some(topic1))), + Some(VariadicValue::Multiple(vec![Some(topic2), Some(topic3)])), + ])), + }; + let topics_input = if let Some(_) = &filter.topics { + let filtered_params = FilteredParams::new(Some(filter.clone())); + Some(filtered_params.flat_topics) + } else { + None + }; + let address_bloom = FilteredParams::adresses_bloom_filter(&filter.address); + let topics_bloom = FilteredParams::topics_bloom_filter(&topics_input); + let matches = FilteredParams::address_in_bloom(block_bloom(), &address_bloom) + && FilteredParams::topics_in_bloom(block_bloom(), &topics_bloom); + assert!(!matches); + } + #[test] + fn bloom_filter_should_match_wildcards_by_topic() { + let topic2 = + H256::from_str("2000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic3 = + H256::from_str("3000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: None, + topics: Some(VariadicValue::Multiple(vec![ + None, + Some(VariadicValue::Multiple(vec![Some(topic2), Some(topic3)])), + ])), + }; + let topics_input = if let Some(_) = &filter.topics { + let filtered_params = FilteredParams::new(Some(filter.clone())); + Some(filtered_params.flat_topics) + } else { + None + }; + let topics_bloom = FilteredParams::topics_bloom_filter(&topics_input); + assert!(FilteredParams::topics_in_bloom( + block_bloom(), + &topics_bloom + )); + } + #[test] + fn bloom_filter_should_not_match_wildcards_by_topic() { + let topic2 = + H256::from_str("4000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let topic3 = + H256::from_str("5000000000000000000000000000000000000000000000000000000000000000") + .unwrap(); + let filter = Filter { + from_block: None, + to_block: None, + block_hash: None, + address: None, + topics: Some(VariadicValue::Multiple(vec![ + None, + Some(VariadicValue::Multiple(vec![Some(topic2), Some(topic3)])), + ])), + }; + let topics_input = if let Some(_) = &filter.topics { + let filtered_params = FilteredParams::new(Some(filter.clone())); + Some(filtered_params.flat_topics) + } else { + None + }; + let topics_bloom = FilteredParams::topics_bloom_filter(&topics_input); + assert!(!FilteredParams::topics_in_bloom( + block_bloom(), + &topics_bloom + )); + } +} diff --git a/client/rpc/src/eth.rs b/client/rpc/src/eth.rs index e8847117a1..39ad576172 100644 --- a/client/rpc/src/eth.rs +++ b/client/rpc/src/eth.rs @@ -268,7 +268,8 @@ where } else { None }; - let bloom_filter = FilteredParams::bloom_filter(&filter.address, &topics_input); + let address_bloom_filter = FilteredParams::adresses_bloom_filter(&filter.address); + let topics_bloom_filter = FilteredParams::topics_bloom_filter(&topics_input); // Get schema cache. A single AuxStore read before the block range iteration. // This prevents having to do an extra DB read per block range iteration to getthe actual schema. @@ -320,7 +321,9 @@ where let block = handler.current_block(&id); if let Some(block) = block { - if FilteredParams::in_bloom(block.header.logs_bloom, &bloom_filter) { + if FilteredParams::address_in_bloom(block.header.logs_bloom, &address_bloom_filter) + && FilteredParams::topics_in_bloom(block.header.logs_bloom, &topics_bloom_filter) + { let statuses = handler.current_transaction_statuses(&id); if let Some(statuses) = statuses { filter_block_logs(ret, filter, block, statuses);