improve reconnect logic
This commit is contained in:
parent
86b4f39a75
commit
7b223efa4d
@ -370,7 +370,7 @@ impl Web3Rpcs {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
warn!("unable to process block from rpc {}: {:#?}", rpc_name, err);
|
warn!("error while processing block from rpc {}: {:#?}", rpc_name, err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
|
@ -11,6 +11,7 @@ use ethers::prelude::{Bytes, Middleware, ProviderError, TxHash, H256, U64};
|
|||||||
use ethers::types::{Address, Transaction, U256};
|
use ethers::types::{Address, Transaction, U256};
|
||||||
use futures::future::try_join_all;
|
use futures::future::try_join_all;
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
|
use futures::stream::FuturesUnordered;
|
||||||
use log::{debug, error, info, trace, warn, Level};
|
use log::{debug, error, info, trace, warn, Level};
|
||||||
use migration::sea_orm::DatabaseConnection;
|
use migration::sea_orm::DatabaseConnection;
|
||||||
use ordered_float::OrderedFloat;
|
use ordered_float::OrderedFloat;
|
||||||
@ -701,8 +702,12 @@ impl Web3Rpc {
|
|||||||
RequestRevertHandler::ErrorLevel
|
RequestRevertHandler::ErrorLevel
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// this does loop. just only when reconnect is enabled
|
||||||
|
#[allow(clippy::never_loop)]
|
||||||
loop {
|
loop {
|
||||||
let mut futures = vec![];
|
debug!("subscription loop started");
|
||||||
|
|
||||||
|
let mut futures = FuturesUnordered::new();
|
||||||
|
|
||||||
let http_interval_receiver = http_interval_sender.as_ref().map(|x| x.subscribe());
|
let http_interval_receiver = http_interval_sender.as_ref().map(|x| x.subscribe());
|
||||||
|
|
||||||
@ -735,6 +740,7 @@ impl Web3Rpc {
|
|||||||
|
|
||||||
// health check loop
|
// health check loop
|
||||||
loop {
|
loop {
|
||||||
|
// TODO: do we need this to be abortable?
|
||||||
if rpc.should_disconnect() {
|
if rpc.should_disconnect() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -839,6 +845,7 @@ impl Web3Rpc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(block_sender) = &block_sender {
|
if let Some(block_sender) = &block_sender {
|
||||||
|
// TODO: do we need this to be abortable?
|
||||||
let f = self.clone().subscribe_new_heads(
|
let f = self.clone().subscribe_new_heads(
|
||||||
authorization.clone(),
|
authorization.clone(),
|
||||||
http_interval_receiver,
|
http_interval_receiver,
|
||||||
@ -850,6 +857,7 @@ impl Web3Rpc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(tx_id_sender) = &tx_id_sender {
|
if let Some(tx_id_sender) = &tx_id_sender {
|
||||||
|
// TODO: do we need this to be abortable?
|
||||||
let f = self
|
let f = self
|
||||||
.clone()
|
.clone()
|
||||||
.subscribe_pending_transactions(authorization.clone(), tx_id_sender.clone());
|
.subscribe_pending_transactions(authorization.clone(), tx_id_sender.clone());
|
||||||
@ -857,32 +865,48 @@ impl Web3Rpc {
|
|||||||
futures.push(flatten_handle(tokio::spawn(f)));
|
futures.push(flatten_handle(tokio::spawn(f)));
|
||||||
}
|
}
|
||||||
|
|
||||||
match try_join_all(futures).await {
|
while let Some(x) = futures.next().await {
|
||||||
Ok(_) => {
|
match x {
|
||||||
// futures all exited without error. break instead of restarting subscriptions
|
Ok(_) => {
|
||||||
break;
|
// future exited without error
|
||||||
}
|
// TODO: think about this more. we never set it to false. this can't be right
|
||||||
Err(err) => {
|
info!("future on {} exited successfully", self)
|
||||||
if self.reconnect.load(atomic::Ordering::Acquire) {
|
}
|
||||||
warn!("{} connection ended. err={:?}", self, err);
|
Err(err) => {
|
||||||
|
if self.reconnect.load(atomic::Ordering::Acquire) {
|
||||||
|
warn!("{} connection ended. reconnecting. err={:?}", self, err);
|
||||||
|
|
||||||
self.clone()
|
let disconnect_sender = self.disconnect_watch.as_ref().unwrap();
|
||||||
.retrying_connect(
|
|
||||||
|
// TODO: i'm not sure if this is necessary, but telling everything to disconnect seems like a better idea than relying on timeouts and dropped futures.
|
||||||
|
disconnect_sender.send_replace(true);
|
||||||
|
disconnect_sender.send_replace(false);
|
||||||
|
|
||||||
|
// we call retrying_connect here with initial_delay=true. above, initial_delay=false
|
||||||
|
self.retrying_connect(
|
||||||
block_sender.as_ref(),
|
block_sender.as_ref(),
|
||||||
chain_id,
|
chain_id,
|
||||||
authorization.db_conn.as_ref(),
|
authorization.db_conn.as_ref(),
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
} else if *disconnect_receiver.borrow() {
|
|
||||||
info!("{} is disconnecting", self);
|
continue;
|
||||||
break;
|
}
|
||||||
} else {
|
|
||||||
error!("{} subscription exited. err={:?}", self, err);
|
// reconnect is not enabled.
|
||||||
return Err(err);
|
if *disconnect_receiver.borrow() {
|
||||||
|
info!("{} is disconnecting", self);
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
error!("{} subscription exited. err={:?}", self, err);
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("all subscriptions on {} completed", self);
|
info!("all subscriptions on {} completed", self);
|
||||||
@ -1070,7 +1094,11 @@ impl Web3Rpc {
|
|||||||
self.send_head_block_result(Ok(None), &block_sender, block_map)
|
self.send_head_block_result(Ok(None), &block_sender, block_map)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(())
|
if self.should_disconnect() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("new_heads subscription exited. reconnect needed"))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Turn on the firehose of pending transactions
|
/// Turn on the firehose of pending transactions
|
||||||
@ -1124,7 +1152,11 @@ impl Web3Rpc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
if self.should_disconnect() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("pending_transactions subscription exited. reconnect needed"))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// be careful with this; it might wait forever!
|
/// be careful with this; it might wait forever!
|
||||||
|
Loading…
Reference in New Issue
Block a user