Skip to content

Commit

Permalink
fix: handle the arweave-rs exception (#598)
Browse files Browse the repository at this point in the history 
* fix: handle the arweave-rs exception

* fix: revert the client_v2.test.ts

* fix: fix the empty log bugs
  • Loading branch information
@imotai
imotai authored Jul 31, 2023
1 parent 518b627 commit bb44ae3
Show file tree
Hide file tree
Showing 7 changed files with 217 additions and 170 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ fastcrypto = {git = "https://github.com/MystenLabs/fastcrypto", rev = "306465d4f
ethers = {git="https://github.com/imotai/ethers-rs", rev="d526191b7972e8cf4412fee8b71cbf42e0ce7995"}
tonic = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"}
tonic-web = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"}
arweave-rs = {git="https://github.com/imotai/arweave-rs", rev="c46183c4cef05004685e53a869606d18abcbcdf2"}
arweave-rs = {git="https://github.com/imotai/arweave-rs", rev="ce7bcaadc614afb0e991953da6861ff173dce6ea"}
ejdb2rs = {git="https://github.com/imotai/ejdb2rs", rev="dfe9a00c66986c072b8375ce68086b9c98ded4fe"}
serde_json= "1.0"
2 changes: 1 addition & 1 deletion sdk/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export {
getDatabase,
getCollection,
addIndex,
deleteEventDatabase
deleteEventDatabase,
} from './store/database_v2'

export { Index, IndexType } from './proto/db3_database_v2'
Expand Down
158 changes: 87 additions & 71 deletions src/event/src/event_processor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ use ethers::{
use std::collections::HashSet;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use tokio::time::{sleep, Duration};
use tracing::{info, warn};

#[derive(Debug)]
Expand All @@ -43,7 +44,6 @@ pub struct EventProcessorConfig {

pub struct EventProcessor {
config: EventProcessorConfig,
provider: Arc<Provider<Ws>>,
running: Arc<AtomicBool>,
db_store: DBStoreV2,
block_number: Arc<AtomicU64>,
Expand All @@ -56,13 +56,8 @@ unsafe impl Send for EventProcessor {}
impl EventProcessor {
pub async fn new(config: EventProcessorConfig, db_store: DBStoreV2) -> Result<Self> {
info!("new event processor with config {:?}", config);
let provider = Provider::<Ws>::connect(&config.evm_node_url)
.await
.map_err(|e| DB3Error::StoreEventError(format!("{e}")))?;
let provider_arc = Arc::new(provider);
Ok(Self {
config,
provider: provider_arc,
running: Arc::new(AtomicBool::new(false)),
db_store,
block_number: Arc::new(AtomicU64::new(0)),
Expand Down Expand Up @@ -91,8 +86,6 @@ impl EventProcessor {
}

pub async fn start(&self) -> Result<()> {
let abi: Abi = serde_json::from_str(self.config.abi.as_str())
.map_err(|e| DB3Error::StoreEventError(format!("{e}")))?;
self.running
.store(true, std::sync::atomic::Ordering::Relaxed);
let address = self
Expand All @@ -102,88 +95,111 @@ impl EventProcessor {
.map_err(|e| DB3Error::StoreEventError(format!("{e}")))?;
let db_addr = DB3Address::from_hex(self.config.db_addr.as_str())
.map_err(|e| DB3Error::StoreEventError(format!("{e}")))?;
let filter = match self.config.start_block == 0 {
true => Filter::new().address(address),
false => {
info!(
"start process contract from block {} with address {}",
self.config.start_block, self.config.contract_addr
);
Filter::new()
.from_block(self.config.start_block)
.address(address)
}
};
let mut stream = self
.provider
.subscribe_logs(&filter)
.await
let abi: Abi = serde_json::from_str(self.config.abi.as_str())
.map_err(|e| DB3Error::StoreEventError(format!("{e}")))?;
info!(
"event processor for contract {}",
self.config.contract_addr.as_str()
);
while let Some(log) = stream.next().await {
let local_evm_node_url = self.config.evm_node_url.to_string();
loop {
if !self.running.load(Ordering::Relaxed) {
info!(
"stop event processor for contract {}",
self.config.contract_addr.as_str()
);
break;
}
if let Some(number) = log.block_number {
if number.as_u64() % 10 == 0 {
let provider =
Provider::<Ws>::connect_with_reconnects(local_evm_node_url.as_str(), 100)
.await
.map_err(|e| DB3Error::StoreEventError(format!("{e}")))?;
let provider_arc = Arc::new(provider);
let filter = match self.config.start_block == 0 {
true => Filter::new().address(address),
false => {
info!(
"contract {} sync status block {} event number {}",
self.config.contract_addr.as_str(),
self.block_number.load(Ordering::Relaxed),
self.event_number.load(Ordering::Relaxed)
"start process contract from block {} with address {}",
self.config.start_block, self.config.contract_addr
);
Filter::new()
.from_block(self.config.start_block)
.address(address)
}
self.block_number.store(number.as_u64(), Ordering::Relaxed)
}
for e in abi.events() {
// verify
let event_signature = log
.topics
.get(0)
.ok_or(DB3Error::StoreEventError(format!("")))?;
};

if event_signature != &e.signature() {
continue;
}
if !self.config.target_events.contains(e.name.as_str()) {
continue;
}
let raw_log = RawLog {
topics: log.topics.clone(),
data: log.data.to_vec(),
};
if let Ok(log_entry) = e.parse_log(raw_log) {
let json_value = Self::log_to_doc(&log_entry);
match serde_json::to_string(&json_value) {
Ok(value) => {
let values = vec![value.to_string()];
if let Err(e) = self.db_store.add_docs(
&db_addr,
&DB3Address::ZERO,
e.name.as_str(),
&values,
None,
) {
warn!("fail to write json doc {} for {e}", value.as_str());
} else {
self.event_number.fetch_add(1, Ordering::Relaxed);
if let Ok(mut stream) = provider_arc.clone().subscribe_logs(&filter).await {
loop {
if let Some(log) = stream.next().await {
if !self.running.load(Ordering::Relaxed) {
info!(
"stop event processor for contract {}",
self.config.contract_addr.as_str()
);
break;
}
if let Some(number) = log.block_number {
if number.as_u64() % 10 == 0 {
info!(
"contract {} sync status block {} event number {}",
self.config.contract_addr.as_str(),
self.block_number.load(Ordering::Relaxed),
self.event_number.load(Ordering::Relaxed)
);
}
self.block_number.store(number.as_u64(), Ordering::Relaxed)
}
Err(e) => {
warn!("fail to convert to json for {e} ");
for e in abi.events() {
// verify
let event_signature = log
.topics
.get(0)
.ok_or(DB3Error::StoreEventError(format!("")))?;
if event_signature != &e.signature() {
continue;
}
if !self.config.target_events.contains(e.name.as_str()) {
continue;
}
let raw_log = RawLog {
topics: log.topics.clone(),
data: log.data.to_vec(),
};
if let Ok(log_entry) = e.parse_log(raw_log) {
let json_value = Self::log_to_doc(&log_entry);
match serde_json::to_string(&json_value) {
Ok(value) => {
let values = vec![value.to_string()];
if let Err(e) = self.db_store.add_docs(
&db_addr,
&DB3Address::ZERO,
e.name.as_str(),
&values,
None,
) {
warn!(
"fail to write json doc {} for {e}",
value.as_str()
);
} else {
self.event_number.fetch_add(1, Ordering::Relaxed);
}
}
Err(e) => {
warn!("fail to convert to json for {e} ");
}
}
break;
}
}
} else {
warn!("empty log from stream, sleep 5 seconds and reconnect to it");
sleep(Duration::from_millis(5 * 1000)).await;
break;
}
break;
}
} else {
warn!("fail to subscribe the log, sleep 5 seconds and reconnect to it");
sleep(Duration::from_millis(5 * 1000)).await;
}
}

Ok(())
}

Expand Down
135 changes: 69 additions & 66 deletions src/event/src/meta_store_event_processor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,10 +347,6 @@ impl MetaStoreEventProcessor {
evm_node_url
);
self.last_running.store(Some(running.clone()));
let provider = Provider::<Ws>::connect_with_reconnects(evm_node_url, 100)
.await
.map_err(|e| DB3Error::StoreEventError(format!("{e}")))?;
let provider = Arc::new(provider);
let address = contract_addr
.parse::<Address>()
.map_err(|_| DB3Error::InvalidAddress)?;
Expand Down Expand Up @@ -391,81 +387,88 @@ impl MetaStoreEventProcessor {
}
};
let local_contract_addr = contract_addr.to_string();
let local_evm_node_url = evm_node_url.to_string();
task::spawn(async move {
loop {
if !running.load(Ordering::Relaxed) {
break;
}
match provider.subscribe_logs(&filter).await {
Ok(mut stream) => loop {
if !running.load(Ordering::Relaxed) {
info!(
"stop event processor for contract {}",
local_contract_addr.as_str()
);
break;
}
match stream.next().await {
Some(log) => {
if let Ok(provider) =
Provider::<Ws>::connect_with_reconnects(local_evm_node_url.as_str(), 100).await
{
let provider = Arc::new(provider);
match provider.subscribe_logs(&filter).await {
Ok(mut stream) => loop {
if !running.load(Ordering::Relaxed) {
info!(
"block number {:?} transacion {:?} sender address {:?} ",
log.block_number, log.transaction_hash, log.address
"stop event processor for contract {}",
local_contract_addr.as_str()
);
if let (Ok(Some(transaction)), Ok(event_signature)) = (
provider
.get_transaction(log.transaction_hash.unwrap())
.await
.map_err(|e| {
DB3Error::StoreEventError(format!(
"fail to get transaction {e}"
))
}),
log.topics.get(0).ok_or(DB3Error::StoreEventError(format!(
"fail to get topics"
))),
) {
if event_signature == &(CreateDatabaseFilter::signature()) {
if let Ok(_) = Self::handle_create_doc_database(
&log,
&transaction,
network,
&wallet,
&local_state_store,
&local_storage,
&local_db_store,
&mutation_type,
)
.await
{}
} else if event_signature
== &(CreateCollectionFilter::signature())
{
if let Ok(_) = Self::handle_create_collection(
&log,
&transaction,
network,
&wallet,
&local_state_store,
&local_storage,
&local_db_store,
&mutation_type,
)
.await
{}
break;
}
match stream.next().await {
Some(log) => {
info!(
"block number {:?} transacion {:?} sender address {:?} ",
log.block_number, log.transaction_hash, log.address
);
if let (Ok(Some(transaction)), Ok(event_signature)) = (
provider
.get_transaction(log.transaction_hash.unwrap())
.await
.map_err(|e| {
DB3Error::StoreEventError(format!(
"fail to get transaction {e}"
))
}),
log.topics.get(0).ok_or(DB3Error::StoreEventError(
format!("fail to get topics"),
)),
) {
if event_signature == &(CreateDatabaseFilter::signature()) {
if let Ok(_) = Self::handle_create_doc_database(
&log,
&transaction,
network,
&wallet,
&local_state_store,
&local_storage,
&local_db_store,
&mutation_type,
)
.await
{}
} else if event_signature
== &(CreateCollectionFilter::signature())
{
if let Ok(_) = Self::handle_create_collection(
&log,
&transaction,
network,
&wallet,
&local_state_store,
&local_storage,
&local_db_store,
&mutation_type,
)
.await
{}
}
}
}
None => {
warn!("empty log from stream, sleep 5 seconds and reconnect to it");
sleep(Duration::from_millis(5 * 1000)).await;
break;
}
}
None => {
warn!("empty log from stream");
sleep(Duration::from_millis(5 * 1000)).await;
}
},
Err(e) => {
warn!("fail get stream for error {e}");
}
},
Err(e) => {
warn!("fail get stream for error {e}");
}
sleep(Duration::from_millis(5 * 1000)).await;
}
sleep(Duration::from_millis(5 * 1000)).await;
}
warn!("the meta contract event listener exits");
});
Expand Down
Loading

0 comments on commit bb44ae3

Please sign in to comment.