web3-proxy/web3_proxy/src/rpcs/many.rs

1660 lines
64 KiB
Rust
Raw Normal View History

///! Load balanced communication with a group of web3 rpc providers
use super::blockchain::{BlocksByHashCache, Web3ProxyBlock};
use super::consensus::ConsensusWeb3Rpcs;
use super::one::Web3Rpc;
use super::request::{OpenRequestHandle, OpenRequestResult, RequestRevertHandler};
use crate::app::{flatten_handle, AnyhowJoinHandle, Web3ProxyApp};
use crate::config::{BlockAndRpc, TxHashAndRpc, Web3RpcConfig};
use crate::frontend::authorization::{Authorization, RequestMetadata};
2023-01-17 09:54:40 +03:00
use crate::frontend::rpc_proxy_ws::ProxyMode;
use crate::jsonrpc::{JsonRpcForwardedResponse, JsonRpcRequest};
2022-08-24 03:59:05 +03:00
use crate::rpcs::transactions::TxStatus;
2023-02-27 10:52:37 +03:00
use anyhow::Context;
2022-05-28 07:26:24 +03:00
use counter::Counter;
2022-05-05 22:07:09 +03:00
use derive_more::From;
use ethers::prelude::{ProviderError, TxHash, H256, U64};
2023-02-09 22:56:07 +03:00
use futures::future::try_join_all;
2022-05-05 22:07:09 +03:00
use futures::stream::FuturesUnordered;
use futures::StreamExt;
2023-01-24 12:58:31 +03:00
use hashbrown::{HashMap, HashSet};
2023-02-15 04:41:40 +03:00
use itertools::Itertools;
2022-12-17 07:05:01 +03:00
use log::{debug, error, info, trace, warn, Level};
2022-11-14 21:24:52 +03:00
use migration::sea_orm::DatabaseConnection;
2022-09-22 02:50:55 +03:00
use moka::future::{Cache, ConcurrentCacheExt};
2023-02-15 22:42:25 +03:00
use ordered_float::OrderedFloat;
2023-02-27 07:00:13 +03:00
use parking_lot::RwLock;
2022-05-21 01:16:15 +03:00
use serde::ser::{SerializeStruct, Serializer};
use serde::Serialize;
use serde_json::json;
2022-05-05 22:07:09 +03:00
use serde_json::value::RawValue;
2023-02-15 04:41:40 +03:00
use std::cmp::min_by_key;
2022-12-08 09:54:38 +03:00
use std::collections::BTreeMap;
2023-02-16 11:26:58 +03:00
use std::sync::atomic::{self, Ordering};
2022-05-05 22:07:09 +03:00
use std::sync::Arc;
2023-01-20 05:30:30 +03:00
use std::{cmp, fmt};
2022-11-12 09:11:58 +03:00
use thread_fast_rng::rand::seq::SliceRandom;
2023-02-27 07:00:13 +03:00
use tokio::sync::{broadcast, watch};
2022-05-17 19:23:27 +03:00
use tokio::task;
2022-09-05 08:53:58 +03:00
use tokio::time::{interval, sleep, sleep_until, Duration, Instant, MissedTickBehavior};
2022-05-05 22:07:09 +03:00
/// A collection of web3 connections. Sends requests either the current best server or all servers.
#[derive(From)]
pub struct Web3Rpcs {
/// if watch_consensus_head_sender is some, Web3Rpc inside self will send blocks here when they get them
pub(crate) block_sender: flume::Sender<(Option<Web3ProxyBlock>, Arc<Web3Rpc>)>,
2022-08-27 02:44:25 +03:00
/// any requests will be forwarded to one (or more) of these connections
2023-02-27 07:00:13 +03:00
/// TODO: i tried to make this an AsyncRwLock, but then we have trouble serializing it
/// TODO: maybe an ArcSwap would be better. writes are rare
pub(crate) by_name: RwLock<HashMap<String, Arc<Web3Rpc>>>,
pub(crate) http_interval_sender: Option<Arc<broadcast::Sender<()>>>,
2023-01-23 09:02:08 +03:00
/// all providers with the same consensus head block. won't update if there is no `self.watch_consensus_head_sender`
/// TODO: document that this is a watch sender and not a broadcast! if things get busy, blocks might get missed
/// TODO: why is watch_consensus_head_sender in an Option, but this one isn't?
/// Geth's subscriptions have the same potential for skipping blocks.
pub(super) watch_consensus_rpcs_sender: watch::Sender<Option<Arc<ConsensusWeb3Rpcs>>>,
2023-01-23 09:02:08 +03:00
/// this head receiver makes it easy to wait until there is a new block
pub(super) watch_consensus_head_sender: Option<watch::Sender<Option<Web3ProxyBlock>>>,
pub(super) pending_transaction_cache:
Cache<TxHash, TxStatus, hashbrown::hash_map::DefaultHashBuilder>,
pub(super) pending_tx_id_receiver: flume::Receiver<TxHashAndRpc>,
pub(super) pending_tx_id_sender: flume::Sender<TxHashAndRpc>,
/// TODO: this map is going to grow forever unless we do some sort of pruning. maybe store pruned in redis?
/// all blocks, including orphans
pub(super) blocks_by_hash: BlocksByHashCache,
2022-09-01 08:58:55 +03:00
/// blocks on the heaviest chain
pub(super) blocks_by_number: Cache<U64, H256, hashbrown::hash_map::DefaultHashBuilder>,
/// the number of rpcs required to agree on consensus for the head block (thundering herd protection)
pub(super) min_head_rpcs: usize,
/// the soft limit required to agree on consensus for the head block. (thundering herd protection)
2022-08-27 06:11:58 +03:00
pub(super) min_sum_soft_limit: u32,
/// how far behind the highest known block height we can be before we stop serving requests
pub(super) max_block_lag: Option<U64>,
/// how old our consensus head block we can be before we stop serving requests
pub(super) max_block_age: Option<u64>,
2022-05-05 22:07:09 +03:00
}
impl Web3Rpcs {
/// Spawn durable connections to multiple Web3 providers.
2022-08-26 20:26:17 +03:00
#[allow(clippy::too_many_arguments)]
pub async fn spawn(
chain_id: u64,
db_conn: Option<DatabaseConnection>,
http_client: Option<reqwest::Client>,
max_block_age: Option<u64>,
max_block_lag: Option<U64>,
min_head_rpcs: usize,
min_sum_soft_limit: u32,
pending_transaction_cache: Cache<TxHash, TxStatus, hashbrown::hash_map::DefaultHashBuilder>,
pending_tx_sender: Option<broadcast::Sender<TxStatus>>,
2023-02-15 04:41:40 +03:00
watch_consensus_head_sender: Option<watch::Sender<Option<Web3ProxyBlock>>>,
2022-06-14 08:43:28 +03:00
) -> anyhow::Result<(Arc<Self>, AnyhowJoinHandle<()>)> {
let (pending_tx_id_sender, pending_tx_id_receiver) = flume::unbounded();
let (block_sender, block_receiver) = flume::unbounded::<BlockAndRpc>();
2022-05-05 22:07:09 +03:00
2023-02-25 20:48:40 +03:00
// TODO: query the rpc to get the actual expected block time, or get from config? maybe have this be part of a health check?
2023-01-03 22:37:42 +03:00
let expected_block_time_ms = match chain_id {
// ethereum
1 => 12_000,
// polygon
137 => 2_000,
// fantom
250 => 1_000,
// arbitrum
2023-01-03 22:38:06 +03:00
42161 => 500,
2023-01-03 22:37:42 +03:00
// anything else
_ => {
warn!("unexpected chain_id. polling every {} seconds", 10);
10_000
}
};
2022-06-29 22:15:05 +03:00
let http_interval_sender = if http_client.is_some() {
2023-02-07 02:20:36 +03:00
let (sender, _) = broadcast::channel(1);
2022-06-29 22:15:05 +03:00
// TODO: what interval? follow a websocket also? maybe by watching synced connections with a timeout. will need debounce
2023-02-07 00:48:50 +03:00
let mut interval = interval(Duration::from_millis(expected_block_time_ms / 2));
2022-06-29 22:15:05 +03:00
interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
let sender = Arc::new(sender);
let f = {
let sender = sender.clone();
async move {
loop {
interval.tick().await;
2023-02-07 00:48:50 +03:00
// trace!("http interval ready");
2022-07-16 08:21:08 +03:00
if sender.send(()).is_err() {
2023-02-07 02:20:36 +03:00
// errors are okay. they mean that all receivers have been dropped, or the rpcs just haven't started yet
trace!("no http receivers");
2023-02-07 01:13:15 +03:00
};
2022-06-29 22:15:05 +03:00
}
}
};
// TODO: do something with this handle?
tokio::spawn(f);
Some(sender)
} else {
None
};
// these blocks don't have full transactions, but they do have rather variable amounts of transaction hashes
// TODO: how can we do the weigher better? need to know actual allocated size
// TODO: limits from config
let blocks_by_hash: BlocksByHashCache = Cache::builder()
.max_capacity(1024 * 1024 * 1024)
.weigher(|_k, v: &Web3ProxyBlock| {
1 + v.block.transactions.len().try_into().unwrap_or(u32::MAX)
})
.time_to_idle(Duration::from_secs(600))
2022-11-11 21:40:52 +03:00
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default());
2022-09-17 05:30:06 +03:00
// all block numbers are the same size, so no need for weigher
// TODO: limits from config
let blocks_by_number = Cache::builder()
.time_to_idle(Duration::from_secs(600))
.max_capacity(10_000)
2022-11-11 21:40:52 +03:00
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default());
2022-09-05 08:53:58 +03:00
let (watch_consensus_rpcs_sender, _) = watch::channel(Default::default());
2023-01-23 09:02:08 +03:00
// by_name starts empty. self.apply_server_configs will add to it
let by_name = Default::default();
2023-01-23 09:02:08 +03:00
2022-05-16 01:02:14 +03:00
let connections = Arc::new(Self {
block_sender,
by_name,
http_interval_sender,
watch_consensus_rpcs_sender,
watch_consensus_head_sender,
pending_transaction_cache,
pending_tx_id_sender,
pending_tx_id_receiver,
blocks_by_hash,
blocks_by_number,
2022-08-27 06:11:58 +03:00
min_sum_soft_limit,
min_head_rpcs,
max_block_age,
max_block_lag,
2022-05-16 01:02:14 +03:00
});
let authorization = Arc::new(Authorization::internal(db_conn)?);
let handle = {
let connections = connections.clone();
tokio::spawn(async move {
connections
.subscribe(authorization, block_receiver, pending_tx_sender)
.await
})
};
2022-06-14 08:43:28 +03:00
Ok((connections, handle))
2022-05-18 19:35:06 +03:00
}
/// update the rpcs in this group
pub async fn apply_server_configs(
&self,
app: &Web3ProxyApp,
rpc_configs: HashMap<String, Web3RpcConfig>,
) -> anyhow::Result<()> {
// safety checks
if rpc_configs.len() < app.config.min_synced_rpcs {
return Err(anyhow::anyhow!(
"Only {}/{} rpcs! Add more rpcs or reduce min_synced_rpcs.",
rpc_configs.len(),
app.config.min_synced_rpcs
));
}
// safety check on sum soft limit
// TODO: will need to think about this more once sum_soft_limit is dynamic
let sum_soft_limit = rpc_configs.values().fold(0, |acc, x| acc + x.soft_limit);
// TODO: < is a bit dangerous, we should require a buffer
if sum_soft_limit < self.min_sum_soft_limit {
return Err(anyhow::anyhow!(
"Only {}/{} soft limit! Add more rpcs, increase soft limits, or reduce min_sum_soft_limit.",
sum_soft_limit,
self.min_sum_soft_limit
));
}
// turn configs into connections (in parallel)
// TODO: move this into a helper function. then we can use it when configs change (will need a remove function too)
let mut spawn_handles: FuturesUnordered<_> = rpc_configs
.into_iter()
.filter_map(|(server_name, server_config)| {
if server_config.disabled {
info!("{} is disabled", server_name);
return None;
}
let db_conn = app.db_conn();
let http_client = app.http_client.clone();
let vredis_pool = app.vredis_pool.clone();
let block_sender = if self.watch_consensus_head_sender.is_some() {
Some(self.block_sender.clone())
} else {
None
};
let pending_tx_id_sender = Some(self.pending_tx_id_sender.clone());
let blocks_by_hash = self.blocks_by_hash.clone();
let http_interval_sender = self.http_interval_sender.clone();
let chain_id = app.config.chain_id;
debug!("spawning {}", server_name);
let handle = tokio::spawn(async move {
server_config
.spawn(
server_name,
db_conn,
vredis_pool,
chain_id,
http_client,
http_interval_sender,
blocks_by_hash,
block_sender,
pending_tx_id_sender,
true,
)
.await
});
Some(handle)
})
.collect();
while let Some(x) = spawn_handles.next().await {
match x {
Ok(Ok((connection, _handle))) => {
// web3 connection worked
2023-02-27 07:00:13 +03:00
let old_rpc = self
.by_name
.write()
.insert(connection.name.clone(), connection);
if let Some(old_rpc) = old_rpc {
2023-02-27 10:52:37 +03:00
old_rpc.disconnect().await.context("disconnect old rpc")?;
}
// TODO: what should we do with the new handle? make sure error logs aren't dropped
}
Ok(Err(err)) => {
// if we got an error here, the app can continue on
// TODO: include context about which connection failed
// TODO: will this retry automatically? i don't think so
error!("Unable to create connection. err={:?}", err);
}
Err(err) => {
// something actually bad happened. exit with an error
return Err(err.into());
}
}
}
Ok(())
}
2023-02-27 07:00:13 +03:00
pub fn get(&self, conn_name: &str) -> Option<Arc<Web3Rpc>> {
self.by_name.read().get(conn_name).cloned()
2022-08-26 20:26:17 +03:00
}
2023-02-27 07:00:13 +03:00
pub fn min_head_rpcs(&self) -> usize {
self.min_head_rpcs
}
2022-07-22 22:30:39 +03:00
/// subscribe to blocks and transactions from all the backend rpcs.
/// blocks are processed by all the `Web3Rpc`s and then sent to the `block_receiver`
/// transaction ids from all the `Web3Rpc`s are deduplicated and forwarded to `pending_tx_sender`
async fn subscribe(
self: Arc<Self>,
authorization: Arc<Authorization>,
2022-08-26 20:26:17 +03:00
block_receiver: flume::Receiver<BlockAndRpc>,
2022-08-24 03:59:05 +03:00
pending_tx_sender: Option<broadcast::Sender<TxStatus>>,
) -> anyhow::Result<()> {
2022-06-16 05:53:37 +03:00
let mut futures = vec![];
2022-05-18 19:35:06 +03:00
2022-06-14 08:43:28 +03:00
// setup the transaction funnel
// it skips any duplicates (unless they are being orphaned)
// fetches new transactions from the notifying rpc
// forwards new transacitons to pending_tx_receipt_sender
if let Some(pending_tx_sender) = pending_tx_sender.clone() {
2022-06-14 09:42:52 +03:00
let clone = self.clone();
let authorization = authorization.clone();
let pending_tx_id_receiver = self.pending_tx_id_receiver.clone();
let handle = task::spawn(async move {
2022-08-26 20:26:17 +03:00
// TODO: set up this future the same as the block funnel
2022-06-16 05:53:37 +03:00
while let Ok((pending_tx_id, rpc)) = pending_tx_id_receiver.recv_async().await {
2022-08-26 20:26:17 +03:00
let f = clone.clone().process_incoming_tx_id(
authorization.clone(),
2022-06-16 05:53:37 +03:00
rpc,
pending_tx_id,
pending_tx_sender.clone(),
);
tokio::spawn(f);
}
Ok(())
});
2022-06-16 05:53:37 +03:00
futures.push(flatten_handle(handle));
}
2022-06-14 08:43:28 +03:00
// setup the block funnel
if self.watch_consensus_head_sender.is_some() {
let connections = Arc::clone(&self);
let pending_tx_sender = pending_tx_sender.clone();
2022-05-18 19:35:06 +03:00
let handle = task::Builder::default()
2022-08-26 20:26:17 +03:00
.name("process_incoming_blocks")
2022-05-17 19:23:27 +03:00
.spawn(async move {
connections
.process_incoming_blocks(&authorization, block_receiver, pending_tx_sender)
2022-05-17 19:23:27 +03:00
.await
2022-09-02 23:16:20 +03:00
})?;
2022-05-18 19:35:06 +03:00
2022-06-16 05:53:37 +03:00
futures.push(flatten_handle(handle));
2022-05-16 01:02:14 +03:00
}
2022-06-14 08:43:28 +03:00
if futures.is_empty() {
// no transaction or block subscriptions.
2022-08-11 00:29:50 +03:00
let handle = task::Builder::default().name("noop").spawn(async move {
loop {
sleep(Duration::from_secs(600)).await;
2022-08-26 20:26:17 +03:00
// TODO: "every interval, check that the provider is still connected"
2022-08-11 00:29:50 +03:00
}
2022-09-02 23:16:20 +03:00
})?;
2022-08-11 00:29:50 +03:00
futures.push(flatten_handle(handle));
2022-06-14 08:43:28 +03:00
}
2022-06-16 05:53:37 +03:00
if let Err(e) = try_join_all(futures).await {
2022-06-16 20:51:49 +03:00
error!("subscriptions over: {:?}", self);
2022-06-16 05:53:37 +03:00
return Err(e);
}
2022-05-18 19:35:06 +03:00
2022-06-14 08:43:28 +03:00
info!("subscriptions over: {:?}", self);
Ok(())
2022-05-05 22:07:09 +03:00
}
2022-05-28 07:26:24 +03:00
/// Send the same request to all the handles. Returning the most common success or most common error.
/// TODO: option to return the fastest response and handles for all the others instead?
2022-05-12 02:50:52 +03:00
pub async fn try_send_parallel_requests(
2022-05-28 07:26:24 +03:00
&self,
2022-08-24 03:14:49 +03:00
active_request_handles: Vec<OpenRequestHandle>,
2022-12-24 04:32:58 +03:00
id: Box<RawValue>,
2022-05-28 07:26:24 +03:00
method: &str,
params: Option<&serde_json::Value>,
error_level: Level,
2022-09-23 01:14:24 +03:00
// TODO: remove this box once i figure out how to do the options
2022-12-24 04:32:58 +03:00
) -> anyhow::Result<JsonRpcForwardedResponse> {
2022-08-26 20:26:17 +03:00
// TODO: if only 1 active_request_handles, do self.try_send_request?
2022-05-05 22:07:09 +03:00
2022-05-28 07:26:24 +03:00
let responses = active_request_handles
.into_iter()
.map(|active_request_handle| async move {
let result: Result<Box<RawValue>, _> = active_request_handle
.request(method, &json!(&params), error_level.into(), None)
.await;
2022-05-28 07:26:24 +03:00
result
})
.collect::<FuturesUnordered<_>>()
.collect::<Vec<Result<Box<RawValue>, ProviderError>>>()
.await;
2022-05-28 21:45:45 +03:00
// TODO: Strings are not great keys, but we can't use RawValue or ProviderError as keys because they don't implement Hash or Eq
2022-12-24 04:32:58 +03:00
let mut count_map: HashMap<String, _> = HashMap::new();
2022-05-28 07:26:24 +03:00
let mut counts: Counter<String> = Counter::new();
2022-12-24 04:32:58 +03:00
let mut any_ok_with_json_result = false;
let mut any_ok_but_maybe_json_error = false;
for partial_response in responses {
if partial_response.is_ok() {
any_ok_with_json_result = true;
}
let response =
JsonRpcForwardedResponse::try_from_response_result(partial_response, id.clone());
// TODO: better key?
2022-05-28 07:26:24 +03:00
let s = format!("{:?}", response);
if count_map.get(&s).is_none() {
if response.is_ok() {
2022-12-24 04:32:58 +03:00
any_ok_but_maybe_json_error = true;
2022-05-28 07:26:24 +03:00
}
2022-05-17 19:23:27 +03:00
2022-05-28 07:26:24 +03:00
count_map.insert(s.clone(), response);
}
2022-05-05 22:07:09 +03:00
2022-05-28 07:26:24 +03:00
counts.update([s].into_iter());
2022-05-05 22:07:09 +03:00
}
2022-05-28 07:26:24 +03:00
for (most_common, _) in counts.most_common_ordered() {
2022-12-24 04:32:58 +03:00
let most_common = count_map
.remove(&most_common)
.expect("most_common key must exist");
match most_common {
Ok(x) => {
if any_ok_with_json_result && x.error.is_some() {
// this one may be an "Ok", but the json has an error inside it
continue;
}
// return the most common success
return Ok(x);
}
Err(err) => {
if any_ok_but_maybe_json_error {
// the most common is an error, but there is an Ok in here somewhere. loop to find it
continue;
}
return Err(err);
}
2022-05-05 22:07:09 +03:00
}
}
2022-05-28 07:26:24 +03:00
// TODO: what should we do if we get here? i don't think we will
unimplemented!("this shouldn't be possible")
2022-05-05 22:07:09 +03:00
}
2023-02-16 08:05:41 +03:00
pub async fn best_available_rpc(
2022-07-02 04:20:28 +03:00
&self,
2023-01-19 14:05:39 +03:00
authorization: &Arc<Authorization>,
request_metadata: Option<&Arc<RequestMetadata>>,
skip: &[Arc<Web3Rpc>],
2023-02-11 07:45:57 +03:00
// TODO: if we are checking for the consensus head, i don' think we need min_block_needed/max_block_needed
2023-01-19 14:05:39 +03:00
min_block_needed: Option<&U64>,
2023-02-11 07:45:57 +03:00
max_block_needed: Option<&U64>,
2023-01-19 14:05:39 +03:00
) -> anyhow::Result<OpenRequestResult> {
2023-02-15 04:41:40 +03:00
let usable_rpcs_by_tier_and_head_number: BTreeMap<(u64, Option<U64>), Vec<Arc<Web3Rpc>>> = {
let synced_connections = self.watch_consensus_rpcs_sender.borrow().clone();
2022-11-07 00:05:03 +03:00
if synced_connections.is_none() {
return Ok(OpenRequestResult::NotReady);
}
let synced_connections =
synced_connections.expect("synced_connections can't be None here");
let head_block_num = synced_connections.head_block.number();
let head_block_age = synced_connections.head_block.age();
2023-01-03 19:33:49 +03:00
2023-02-15 04:41:40 +03:00
let needed_blocks_comparison = match (min_block_needed, max_block_needed) {
(None, None) => {
// no required block given. treat this like the requested the consensus head block
cmp::Ordering::Equal
}
(None, Some(max_block_needed)) => max_block_needed.cmp(head_block_num),
(Some(min_block_needed), None) => min_block_needed.cmp(head_block_num),
(Some(min_block_needed), Some(max_block_needed)) => {
match min_block_needed.cmp(max_block_needed) {
cmp::Ordering::Equal => min_block_needed.cmp(head_block_num),
cmp::Ordering::Greater => {
return Err(anyhow::anyhow!(
"Invalid blocks bounds requested. min ({}) > max ({})",
min_block_needed,
max_block_needed
))
}
2023-02-15 23:33:43 +03:00
cmp::Ordering::Less => min_block_needed.cmp(head_block_num),
2023-02-15 04:41:40 +03:00
}
}
};
2023-01-04 09:37:51 +03:00
2023-02-16 08:05:41 +03:00
trace!("needed_blocks_comparison: {:?}", needed_blocks_comparison);
2023-02-15 04:41:40 +03:00
// collect "usable_rpcs_by_head_num_and_weight"
// TODO: MAKE SURE None SORTS LAST?
2023-01-20 05:30:30 +03:00
let mut m = BTreeMap::new();
2023-01-04 09:37:51 +03:00
2023-02-15 04:41:40 +03:00
match needed_blocks_comparison {
2023-01-20 05:30:30 +03:00
cmp::Ordering::Less => {
2023-02-15 04:41:40 +03:00
// need an old block. check all the rpcs. ignore rpcs that are still syncing
2023-02-16 08:05:41 +03:00
trace!("old block needed");
2023-02-15 04:41:40 +03:00
let min_block_age =
self.max_block_age.map(|x| head_block_age.saturating_sub(x));
let min_sync_num = self.max_block_lag.map(|x| head_block_num.saturating_sub(x));
// TODO: cache this somehow?
// TODO: maybe have a helper on synced_connections? that way sum_soft_limits/min_synced_rpcs will be DRY
2023-01-20 05:30:30 +03:00
for x in self
2023-02-15 04:41:40 +03:00
.by_name
2023-02-27 07:00:13 +03:00
.read()
2023-01-20 05:30:30 +03:00
.values()
2023-02-11 07:45:57 +03:00
.filter(|x| {
2023-02-15 04:41:40 +03:00
// TODO: move a bunch of this onto a rpc.is_synced function
#[allow(clippy::if_same_then_else)]
2023-02-15 04:41:40 +03:00
if skip.contains(x) {
// we've already tried this server or have some other reason to skip it
false
} else if max_block_needed
.map(|max_block_needed| !x.has_block_data(max_block_needed))
.unwrap_or(false)
{
2023-02-15 04:41:40 +03:00
// server does not have the max block
false
2023-02-11 07:45:57 +03:00
} else {
!min_block_needed
.map(|min_block_needed| !x.has_block_data(min_block_needed))
.unwrap_or(false)
2023-02-11 07:45:57 +03:00
}
})
2023-01-20 05:30:30 +03:00
.cloned()
{
let x_head_block = x.head_block.read().clone();
2023-02-15 04:41:40 +03:00
if let Some(x_head) = x_head_block {
// TODO: should nodes that are ahead of the consensus block have priority? seems better to spread the load
let x_head_num = x_head.number().min(head_block_num);
2023-01-20 05:30:30 +03:00
2023-02-15 04:41:40 +03:00
// TODO: do we really need to check head_num and age?
if let Some(min_sync_num) = min_sync_num.as_ref() {
if x_head_num < min_sync_num {
2023-02-16 08:05:41 +03:00
trace!("rpc is still syncing");
2023-02-15 04:41:40 +03:00
continue;
}
}
if let Some(min_block_age) = min_block_age {
2023-02-16 08:05:41 +03:00
if x_head.age() > min_block_age {
2023-02-15 04:41:40 +03:00
// rpc is still syncing
2023-02-16 08:05:41 +03:00
trace!("block is too old");
2023-02-15 04:41:40 +03:00
continue;
}
2023-01-20 05:30:30 +03:00
}
2023-02-15 04:41:40 +03:00
let key = (x.tier, Some(*x_head_num));
m.entry(key).or_insert_with(Vec::new).push(x);
2023-01-20 05:30:30 +03:00
}
}
2023-02-15 04:41:40 +03:00
// TODO: check min_synced_rpcs and min_sum_soft_limits? or maybe better to just try to serve the request?
2023-01-20 05:30:30 +03:00
}
cmp::Ordering::Equal => {
2023-02-16 08:05:41 +03:00
// using the consensus head block. filter the synced rpcs
// the key doesn't matter if we are checking synced connections
// they are all at the same block and it is already sized to what we need
let key = (0, None);
for x in synced_connections.rpcs.iter() {
if skip.contains(x) {
trace!("skipping: {}", x);
continue;
}
trace!("not skipped!");
2023-01-20 05:30:30 +03:00
m.entry(key).or_insert_with(Vec::new).push(x.clone());
2023-01-20 05:30:30 +03:00
}
}
cmp::Ordering::Greater => {
2023-02-15 04:41:40 +03:00
// TODO? if the blocks is close, maybe we could wait for change on a watch_consensus_connections_receiver().subscribe()
return Ok(OpenRequestResult::NotReady);
2023-01-20 05:30:30 +03:00
}
2023-01-04 23:12:44 +03:00
}
2022-11-25 10:41:53 +03:00
2023-01-04 23:12:44 +03:00
m
};
2022-07-02 04:20:28 +03:00
2023-02-16 08:05:41 +03:00
trace!(
"usable_rpcs_by_tier_and_head_number: {:#?}",
usable_rpcs_by_tier_and_head_number
);
2022-12-08 09:54:38 +03:00
let mut earliest_retry_at = None;
2022-11-12 09:11:58 +03:00
2023-02-15 04:41:40 +03:00
for mut usable_rpcs in usable_rpcs_by_tier_and_head_number.into_values() {
// sort the tier randomly
if usable_rpcs.len() == 1 {
// TODO: include an rpc from the next tier?
} else {
// we can't get the rng outside of this loop because it is not Send
// this function should be pretty fast anyway, so it shouldn't matter too much
let mut rng = thread_fast_rng::thread_fast_rng();
usable_rpcs.shuffle(&mut rng);
2022-12-08 09:54:38 +03:00
};
2023-02-15 04:41:40 +03:00
// now that the rpcs are shuffled, try to get an active request handle for one of them
// pick the first two and try the one with the lower rpc.latency.ewma
// TODO: chunks or tuple windows?
for (rpc_a, rpc_b) in usable_rpcs.into_iter().circular_tuple_windows() {
2023-02-16 08:05:41 +03:00
trace!("{} vs {}", rpc_a, rpc_b);
// TODO: cached key to save a read lock
// TODO: ties to the server with the smallest block_data_limit
2023-02-15 22:42:25 +03:00
let best_rpc = min_by_key(rpc_a, rpc_b, |x| {
2023-02-16 11:26:58 +03:00
OrderedFloat(x.head_latency.read().value())
2023-02-15 22:42:25 +03:00
});
2023-02-16 08:05:41 +03:00
trace!("winner: {}", best_rpc);
2023-02-15 04:41:40 +03:00
// just because it has lower latency doesn't mean we are sure to get a connection
match best_rpc.try_request_handle(authorization, None).await {
2022-12-08 09:54:38 +03:00
Ok(OpenRequestResult::Handle(handle)) => {
// trace!("opened handle: {}", best_rpc);
2022-12-08 09:54:38 +03:00
return Ok(OpenRequestResult::Handle(handle));
}
Ok(OpenRequestResult::RetryAt(retry_at)) => {
earliest_retry_at = earliest_retry_at.min(Some(retry_at));
}
2023-02-15 04:41:40 +03:00
Ok(OpenRequestResult::NotReady) => {
2022-12-20 00:53:38 +03:00
// TODO: log a warning? emit a stat?
2023-02-16 08:05:41 +03:00
trace!("best_rpc not ready");
2022-12-08 09:54:38 +03:00
}
Err(err) => {
warn!("No request handle for {}. err={:?}", best_rpc, err)
}
2022-05-06 23:44:12 +03:00
}
2022-05-05 22:07:09 +03:00
}
}
2022-12-08 09:54:38 +03:00
if let Some(request_metadata) = request_metadata {
request_metadata.no_servers.fetch_add(1, Ordering::Release);
}
2022-08-07 09:48:57 +03:00
match earliest_retry_at {
2022-09-28 20:01:11 +03:00
None => {
2022-11-25 10:41:53 +03:00
// none of the servers gave us a time to retry at
2022-10-12 00:31:34 +03:00
2023-01-19 14:05:39 +03:00
// TODO: bring this back? need to think about how to do this with `allow_backups`
2022-11-25 10:41:53 +03:00
// we could return an error here, but maybe waiting a second will fix the problem
// TODO: configurable max wait? the whole max request time, or just some portion?
2022-12-08 09:54:38 +03:00
// let handle = sorted_rpcs
// .get(0)
// .expect("at least 1 is available")
// .wait_for_request_handle(authorization, Duration::from_secs(3), false)
// .await?;
// Ok(OpenRequestResult::Handle(handle))
2022-12-20 00:53:38 +03:00
// TODO: should we log here?
2023-02-15 04:41:40 +03:00
Ok(OpenRequestResult::NotReady)
2022-09-28 20:01:11 +03:00
}
Some(earliest_retry_at) => {
warn!("no servers on {:?}! {:?}", self, earliest_retry_at);
2022-10-12 00:31:34 +03:00
2022-09-28 20:01:11 +03:00
Ok(OpenRequestResult::RetryAt(earliest_retry_at))
}
2022-08-07 09:48:57 +03:00
}
2022-05-05 22:07:09 +03:00
}
/// get all rpc servers that are not rate limited
2023-01-24 12:58:31 +03:00
/// this prefers synced servers, but it will return servers even if they aren't fully in sync.
/// This is useful for broadcasting signed transactions.
2022-08-07 09:48:57 +03:00
// TODO: better type on this that can return an anyhow::Result
2023-01-24 12:58:31 +03:00
pub async fn all_connections(
2022-07-09 06:34:39 +03:00
&self,
authorization: &Arc<Authorization>,
2023-02-11 07:45:57 +03:00
min_block_needed: Option<&U64>,
max_block_needed: Option<&U64>,
2023-01-12 01:51:01 +03:00
max_count: Option<usize>,
2023-01-19 14:05:39 +03:00
allow_backups: bool,
2022-08-24 03:14:49 +03:00
) -> Result<Vec<OpenRequestHandle>, Option<Instant>> {
2022-08-07 09:48:57 +03:00
let mut earliest_retry_at = None;
2022-05-05 22:07:09 +03:00
2023-01-24 12:58:31 +03:00
let mut max_count = if let Some(max_count) = max_count {
max_count
2023-01-12 01:51:01 +03:00
} else {
2023-02-27 07:00:13 +03:00
self.by_name.read().len()
2023-01-12 01:51:01 +03:00
};
trace!("max_count: {}", max_count);
let mut selected_rpcs = Vec::with_capacity(max_count);
2023-01-24 12:58:31 +03:00
let mut tried = HashSet::new();
let mut synced_rpcs = {
let synced_rpcs = self.watch_consensus_rpcs_sender.borrow();
if let Some(synced_rpcs) = synced_rpcs.as_ref() {
synced_rpcs.rpcs.clone()
} else {
vec![]
}
};
// synced connections are all on the same block. sort them by tier with higher soft limits first
synced_rpcs.sort_by_cached_key(rpc_sync_status_sort_key);
trace!("synced_rpcs: {:#?}", synced_rpcs);
// if there aren't enough synced connections, include more connections
// TODO: only do this sorting if the synced_rpcs isn't enough
let mut all_rpcs: Vec<_> = self.by_name.read().values().cloned().collect();
all_rpcs.sort_by_cached_key(rpc_sync_status_sort_key);
trace!("all_rpcs: {:#?}", all_rpcs);
for rpc in itertools::chain(synced_rpcs, all_rpcs) {
2023-01-12 01:51:01 +03:00
if max_count == 0 {
break;
}
if tried.contains(&rpc.name) {
2023-01-24 12:58:31 +03:00
continue;
}
trace!("trying {}", rpc);
2023-01-24 12:58:31 +03:00
tried.insert(rpc.name.clone());
if !allow_backups && rpc.backup {
trace!("{} is a backup. skipping", rpc);
2023-01-19 14:05:39 +03:00
continue;
}
2023-02-11 07:45:57 +03:00
if let Some(block_needed) = min_block_needed {
if !rpc.has_block_data(block_needed) {
trace!("{} is missing min_block_needed. skipping", rpc);
2023-02-11 07:45:57 +03:00
continue;
}
}
if let Some(block_needed) = max_block_needed {
if !rpc.has_block_data(block_needed) {
trace!("{} is missing max_block_needed. skipping", rpc);
2022-07-19 04:31:12 +03:00
continue;
}
2022-07-09 07:25:59 +03:00
}
2022-05-05 22:07:09 +03:00
// check rate limits and increment our connection counter
match rpc.try_request_handle(authorization, None).await {
2022-08-24 03:14:49 +03:00
Ok(OpenRequestResult::RetryAt(retry_at)) => {
// this rpc is not available. skip it
trace!("{} is rate limited. skipping", rpc);
2022-08-07 09:48:57 +03:00
earliest_retry_at = earliest_retry_at.min(Some(retry_at));
}
2023-01-12 01:51:01 +03:00
Ok(OpenRequestResult::Handle(handle)) => {
trace!("{} is available", rpc);
2023-01-12 01:51:01 +03:00
max_count -= 1;
selected_rpcs.push(handle)
}
2023-02-15 04:41:40 +03:00
Ok(OpenRequestResult::NotReady) => {
warn!("no request handle for {}", rpc)
2022-08-07 09:48:57 +03:00
}
Err(err) => {
warn!("error getting request handle for {}. err={:?}", rpc, err)
}
2022-05-05 22:07:09 +03:00
}
}
if !selected_rpcs.is_empty() {
return Ok(selected_rpcs);
}
2022-05-22 02:34:05 +03:00
// return the earliest retry_after (if no rpcs are synced, this will be None)
2022-08-07 09:48:57 +03:00
Err(earliest_retry_at)
2022-05-05 22:07:09 +03:00
}
2022-05-28 07:26:24 +03:00
2022-05-29 04:23:58 +03:00
/// be sure there is a timeout on this or it might loop forever
2023-01-23 09:02:08 +03:00
/// TODO: think more about wait_for_sync
2023-01-17 09:54:40 +03:00
pub async fn try_send_best_consensus_head_connection(
2022-05-29 04:23:58 +03:00
&self,
authorization: &Arc<Authorization>,
2022-05-29 04:23:58 +03:00
request: JsonRpcRequest,
request_metadata: Option<&Arc<RequestMetadata>>,
2022-07-22 22:30:39 +03:00
min_block_needed: Option<&U64>,
2023-02-11 07:45:57 +03:00
max_block_needed: Option<&U64>,
2022-05-29 04:23:58 +03:00
) -> anyhow::Result<JsonRpcForwardedResponse> {
2022-07-02 04:20:28 +03:00
let mut skip_rpcs = vec![];
let mut method_not_available_response = None;
2022-07-02 04:20:28 +03:00
let mut watch_consensus_connections = self.watch_consensus_rpcs_sender.subscribe();
2023-01-23 09:02:08 +03:00
// TODO: maximum retries? right now its the total number of servers
2022-05-29 04:23:58 +03:00
loop {
let num_skipped = skip_rpcs.len();
2023-02-27 07:00:13 +03:00
if num_skipped >= self.by_name.read().len() {
2023-01-31 20:47:19 +03:00
break;
2022-07-02 04:20:28 +03:00
}
2022-07-19 04:31:12 +03:00
match self
2023-02-16 08:05:41 +03:00
.best_available_rpc(
authorization,
2022-10-12 00:31:34 +03:00
request_metadata,
&skip_rpcs,
min_block_needed,
2023-02-11 07:45:57 +03:00
max_block_needed,
2022-10-12 00:31:34 +03:00
)
2022-09-10 03:58:33 +03:00
.await?
2022-07-19 04:31:12 +03:00
{
2022-09-10 03:58:33 +03:00
OpenRequestResult::Handle(active_request_handle) => {
2022-07-02 04:20:28 +03:00
// save the rpc in case we get an error and want to retry on another server
// TODO: look at backend_requests instead
2022-07-02 04:20:28 +03:00
skip_rpcs.push(active_request_handle.clone_connection());
if let Some(request_metadata) = request_metadata {
let rpc = active_request_handle.clone_connection();
request_metadata
.response_from_backup_rpc
.store(rpc.backup, Ordering::Release);
request_metadata.backend_requests.lock().push(rpc);
}
// TODO: get the log percent from the user data
2022-05-29 04:23:58 +03:00
let response_result = active_request_handle
.request(
&request.method,
&json!(request.params),
2023-01-17 09:54:40 +03:00
RequestRevertHandler::Save,
None,
)
2022-05-29 04:23:58 +03:00
.await;
2022-07-02 04:20:28 +03:00
match JsonRpcForwardedResponse::try_from_response_result(
2022-06-04 00:45:44 +03:00
response_result,
request.id.clone(),
) {
Ok(response) => {
if let Some(error) = &response.error.as_ref() {
2022-12-20 02:59:01 +03:00
// trace!(?response, "rpc error");
2022-07-02 04:20:28 +03:00
if let Some(request_metadata) = request_metadata {
request_metadata
.error_response
.store(true, Ordering::Release);
}
2022-07-09 07:25:59 +03:00
// some errors should be retried on other nodes
let error_msg = error.message.as_str();
// different providers do different codes. check all of them
// TODO: there's probably more strings to add here
let rate_limit_substrings = ["limit", "exceeded"];
for rate_limit_substr in rate_limit_substrings {
if error_msg.contains(rate_limit_substr) {
2023-01-20 04:57:36 +03:00
warn!("rate limited by {}", skip_rpcs.last().unwrap());
continue;
}
}
match error.code {
-32000 => {
// TODO: regex?
let retry_prefixes = [
"header not found",
"header for hash not found",
"missing trie node",
"node not started",
"RPC timeout",
];
for retry_prefix in retry_prefixes {
if error_msg.starts_with(retry_prefix) {
continue;
}
}
}
-32601 => {
let error_msg = error.message.as_str();
// sometimes a provider does not support all rpc methods
// we check other connections rather than returning the error
// but sometimes the method is something that is actually unsupported,
// so we save the response here to return it later
// some providers look like this
if error_msg.starts_with("the method")
&& error_msg.ends_with("is not available")
{
method_not_available_response = Some(response);
continue;
}
2023-02-25 10:31:10 +03:00
// others look like this (this is the example in the official spec)
if error_msg == "Method not found" {
method_not_available_response = Some(response);
2022-07-09 07:25:59 +03:00
continue;
}
}
_ => {}
2022-07-02 04:20:28 +03:00
}
2022-06-04 00:45:44 +03:00
} else {
2022-12-20 02:59:01 +03:00
// trace!(?response, "rpc success");
2022-06-04 00:45:44 +03:00
}
2022-05-29 04:23:58 +03:00
2022-06-04 00:45:44 +03:00
return Ok(response);
}
2022-11-12 11:24:32 +03:00
Err(err) => {
2022-09-10 03:58:33 +03:00
let rpc = skip_rpcs
.last()
.expect("there must have been a provider if we got an error");
2022-12-17 07:05:01 +03:00
// TODO: emit a stat. if a server is getting skipped a lot, something is not right
debug!(
2022-11-12 11:24:32 +03:00
"Backend server error on {}! Retrying on another. err={:?}",
rpc, err
);
2022-05-29 04:23:58 +03:00
2022-06-04 00:45:44 +03:00
continue;
}
}
2022-05-29 04:23:58 +03:00
}
2022-09-10 03:58:33 +03:00
OpenRequestResult::RetryAt(retry_at) => {
2022-08-07 09:48:57 +03:00
// TODO: move this to a helper function
// sleep (TODO: with a lock?) until our rate limits should be available
// TODO: if a server catches up sync while we are waiting, we could stop waiting
warn!(
"All rate limits exceeded. waiting for change in synced servers or {:?}",
retry_at
);
2022-05-29 04:23:58 +03:00
2022-10-12 00:31:34 +03:00
// TODO: have a separate column for rate limited?
if let Some(request_metadata) = request_metadata {
request_metadata.no_servers.fetch_add(1, Ordering::Release);
}
tokio::select! {
_ = sleep_until(retry_at) => {
skip_rpcs.pop();
}
_ = watch_consensus_connections.changed() => {
watch_consensus_connections.borrow_and_update();
2023-01-23 09:02:08 +03:00
}
}
2022-05-29 04:23:58 +03:00
}
2023-02-15 04:41:40 +03:00
OpenRequestResult::NotReady => {
2022-10-12 00:31:34 +03:00
if let Some(request_metadata) = request_metadata {
request_metadata.no_servers.fetch_add(1, Ordering::Release);
}
2023-02-16 08:05:41 +03:00
break;
2022-05-29 04:23:58 +03:00
}
}
}
2022-07-02 04:20:28 +03:00
2023-02-16 08:05:41 +03:00
// TODO: do we need this here, or do we do it somewhere else? like, the code could change and a try operator in here would skip this increment
2022-10-12 00:31:34 +03:00
if let Some(request_metadata) = request_metadata {
request_metadata
.error_response
.store(true, Ordering::Release);
}
2023-02-16 08:05:41 +03:00
if let Some(r) = method_not_available_response {
// TODO: emit a stat for unsupported methods? it would be best to block them at the proxy instead of at the backend
return Ok(r);
}
2023-02-27 07:00:13 +03:00
let num_conns = self.by_name.read().len();
let num_skipped = skip_rpcs.len();
2022-11-28 09:52:16 +03:00
if num_skipped == 0 {
2023-02-16 08:05:41 +03:00
error!("No servers synced ({} known). None skipped", num_conns);
2022-12-24 04:32:58 +03:00
// TODO: what error code?
2023-02-16 08:05:41 +03:00
Ok(JsonRpcForwardedResponse::from_str(
"No servers synced",
Some(-32000),
Some(request.id),
2023-02-16 08:05:41 +03:00
))
} else {
// TODO: warn? debug? trace?
warn!(
"Requested data was not available on {}/{} servers",
num_skipped, num_conns
);
// TODO: what error code?
// cloudflare gives {"jsonrpc":"2.0","error":{"code":-32043,"message":"Requested data cannot be older than 128 blocks."},"id":1}
2023-02-16 08:05:41 +03:00
Ok(JsonRpcForwardedResponse::from_str(
"Requested data is not available",
Some(-32043),
Some(request.id),
2023-02-16 08:05:41 +03:00
))
}
2022-05-29 04:23:58 +03:00
}
2022-07-02 04:20:28 +03:00
/// be sure there is a timeout on this or it might loop forever
#[allow(clippy::too_many_arguments)]
2023-01-17 09:54:40 +03:00
pub async fn try_send_all_synced_connections(
2022-05-28 07:26:24 +03:00
&self,
authorization: &Arc<Authorization>,
2022-12-24 04:32:58 +03:00
request: &JsonRpcRequest,
request_metadata: Option<Arc<RequestMetadata>>,
2023-02-11 07:45:57 +03:00
min_block_needed: Option<&U64>,
max_block_needed: Option<&U64>,
error_level: Level,
2023-01-12 01:51:01 +03:00
max_count: Option<usize>,
always_include_backups: bool,
2022-05-28 07:26:24 +03:00
) -> anyhow::Result<JsonRpcForwardedResponse> {
loop {
match self
.all_connections(
authorization,
2023-02-11 07:45:57 +03:00
min_block_needed,
max_block_needed,
max_count,
always_include_backups,
)
.await
{
2022-05-28 07:26:24 +03:00
Ok(active_request_handles) => {
// TODO: benchmark this compared to waiting on unbounded futures
// TODO: do something with this handle?
// TODO: this is not working right. simplify
2022-10-12 00:31:34 +03:00
if let Some(request_metadata) = request_metadata {
let mut backup_used = false;
request_metadata.backend_requests.lock().extend(
active_request_handles.iter().map(|x| {
let rpc = x.clone_connection();
if rpc.backup {
// TODO: its possible we serve from a synced connection though. think about this more
backup_used = true;
}
x.clone_connection()
}),
);
2022-10-12 00:31:34 +03:00
request_metadata
.response_from_backup_rpc
.store(true, Ordering::Release);
2022-10-12 00:31:34 +03:00
}
2022-12-24 04:32:58 +03:00
return self
2022-05-28 07:26:24 +03:00
.try_send_parallel_requests(
active_request_handles,
2022-12-24 04:32:58 +03:00
request.id.clone(),
2022-05-28 07:26:24 +03:00
request.method.as_ref(),
request.params.as_ref(),
error_level,
2022-05-28 07:26:24 +03:00
)
2022-12-24 04:32:58 +03:00
.await;
2022-05-28 07:26:24 +03:00
}
Err(None) => {
2022-11-12 11:24:32 +03:00
warn!("No servers in sync on {:?}! Retrying", self);
2022-05-29 04:23:58 +03:00
2022-10-12 00:31:34 +03:00
if let Some(request_metadata) = &request_metadata {
request_metadata.no_servers.fetch_add(1, Ordering::Release);
}
2022-05-28 07:26:24 +03:00
// TODO: i don't think this will ever happen
2022-06-03 00:47:43 +03:00
// TODO: return a 502? if it does?
// return Err(anyhow::anyhow!("no available rpcs!"));
// TODO: sleep how long?
// TODO: subscribe to something in ConsensusConnections instead
2022-06-03 00:47:43 +03:00
sleep(Duration::from_millis(200)).await;
continue;
2022-05-28 07:26:24 +03:00
}
2022-08-07 09:48:57 +03:00
Err(Some(retry_at)) => {
2022-05-28 07:26:24 +03:00
// TODO: move this to a helper function
// sleep (TODO: with a lock?) until our rate limits should be available
// TODO: if a server catches up sync while we are waiting, we could stop waiting
2022-06-03 00:47:43 +03:00
warn!("All rate limits exceeded. Sleeping");
2022-10-12 00:31:34 +03:00
if let Some(request_metadata) = &request_metadata {
request_metadata.no_servers.fetch_add(1, Ordering::Release);
}
2022-08-07 09:48:57 +03:00
sleep_until(retry_at).await;
2022-05-28 07:26:24 +03:00
2022-06-03 00:47:43 +03:00
continue;
2022-05-28 07:26:24 +03:00
}
}
}
}
2023-01-17 09:54:40 +03:00
pub async fn try_proxy_connection(
&self,
proxy_mode: ProxyMode,
authorization: &Arc<Authorization>,
request: JsonRpcRequest,
request_metadata: Option<&Arc<RequestMetadata>>,
min_block_needed: Option<&U64>,
2023-02-11 07:45:57 +03:00
max_block_needed: Option<&U64>,
2023-01-17 09:54:40 +03:00
) -> anyhow::Result<JsonRpcForwardedResponse> {
match proxy_mode {
ProxyMode::Best => {
self.try_send_best_consensus_head_connection(
authorization,
request,
request_metadata,
min_block_needed,
2023-02-11 07:45:57 +03:00
max_block_needed,
2023-01-17 09:54:40 +03:00
)
.await
}
ProxyMode::Fastest(x) => todo!("Fastest"),
ProxyMode::Versus => todo!("Versus"),
}
}
2022-05-05 22:07:09 +03:00
}
2022-07-16 07:13:02 +03:00
impl fmt::Debug for Web3Rpcs {
2022-08-24 02:13:56 +03:00
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// TODO: the default formatter takes forever to write. this is too quiet though
f.debug_struct("Web3Rpcs")
2023-02-15 04:41:40 +03:00
.field("rpcs", &self.by_name)
2022-08-24 02:13:56 +03:00
.finish_non_exhaustive()
}
}
impl Serialize for Web3Rpcs {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct("Web3Rpcs", 6)?;
2022-09-22 02:50:55 +03:00
2023-02-27 07:00:13 +03:00
{
let by_name = self.by_name.read();
let rpcs: Vec<&Web3Rpc> = by_name.values().map(|x| x.as_ref()).collect();
// TODO: coordinate with frontend team to rename "conns" to "rpcs"
state.serialize_field("conns", &rpcs)?;
}
2022-09-22 02:50:55 +03:00
2023-01-23 09:02:08 +03:00
{
let consensus_rpcs = self.watch_consensus_rpcs_sender.borrow();
// TODO: rename synced_connections to consensus_rpcs
if let Some(consensus_rpcs) = consensus_rpcs.as_ref() {
state.serialize_field("synced_connections", consensus_rpcs)?;
} else {
state.serialize_field("synced_connections", &None::<()>)?;
}
2023-01-23 09:02:08 +03:00
}
2022-09-22 02:50:55 +03:00
self.blocks_by_hash.sync();
self.blocks_by_number.sync();
state.serialize_field("block_hashes_count", &self.blocks_by_hash.entry_count())?;
state.serialize_field("block_hashes_size", &self.blocks_by_hash.weighted_size())?;
state.serialize_field("block_numbers_count", &self.blocks_by_number.entry_count())?;
state.serialize_field("block_numbers_size", &self.blocks_by_number.weighted_size())?;
state.end()
}
}
/// sort by block number (descending) and tier (ascending)
2023-02-11 07:24:20 +03:00
/// TODO: should this be moved into a `impl Web3Rpc`?
2023-02-15 22:42:25 +03:00
/// TODO: i think we still have sorts scattered around the code that should use this
2023-02-11 07:24:20 +03:00
/// TODO: take AsRef or something like that? We don't need an Arc here
2023-02-15 22:42:25 +03:00
fn rpc_sync_status_sort_key(x: &Arc<Web3Rpc>) -> (U64, u64, OrderedFloat<f64>) {
let reversed_head_block = U64::MAX
2023-02-11 07:24:20 +03:00
- x.head_block
.read()
.as_ref()
2023-02-15 22:42:25 +03:00
.map(|x| *x.number())
.unwrap_or_default();
2023-02-11 07:24:20 +03:00
let tier = x.tier;
2023-02-16 11:26:58 +03:00
// TODO: use request instead of head latency
let head_ewma = x.head_latency.read().value();
2023-02-11 07:24:20 +03:00
2023-02-16 11:26:58 +03:00
let active_requests = x.active_requests.load(atomic::Ordering::Relaxed) as f64;
// TODO: i'm not sure head * active is exactly right. but we'll see
// TODO: i don't think this actually counts as peak. investigate with atomics.rs and peak_ewma.rs
let peak_ewma = OrderedFloat(head_ewma * active_requests);
(reversed_head_block, tier, peak_ewma)
}
mod tests {
2022-11-30 00:34:42 +03:00
// TODO: why is this allow needed? does tokio::test get in the way somehow?
2022-12-03 08:31:03 +03:00
#![allow(unused_imports)]
use super::*;
use crate::rpcs::consensus::ConsensusFinder;
use crate::rpcs::{blockchain::Web3ProxyBlock, provider::Web3Provider};
2022-12-01 01:11:14 +03:00
use ethers::types::{Block, U256};
2022-11-25 10:41:53 +03:00
use log::{trace, LevelFilter};
use parking_lot::RwLock;
2022-12-03 08:31:03 +03:00
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock as AsyncRwLock;
#[tokio::test]
async fn test_sort_connections_by_sync_status() {
let block_0 = Block {
number: Some(0.into()),
hash: Some(H256::random()),
..Default::default()
};
let block_1 = Block {
number: Some(1.into()),
hash: Some(H256::random()),
parent_hash: block_0.hash.unwrap(),
..Default::default()
};
let block_2 = Block {
number: Some(2.into()),
hash: Some(H256::random()),
parent_hash: block_1.hash.unwrap(),
..Default::default()
};
let blocks: Vec<_> = [block_0, block_1, block_2]
.into_iter()
2023-02-15 04:41:40 +03:00
.map(|x| Web3ProxyBlock::try_new(Arc::new(x)).unwrap())
.collect();
2023-02-11 07:24:20 +03:00
let mut rpcs: Vec<_> = [
Web3Rpc {
name: "a".to_string(),
tier: 0,
head_block: RwLock::new(None),
..Default::default()
},
Web3Rpc {
name: "b".to_string(),
tier: 0,
head_block: RwLock::new(blocks.get(1).cloned()),
..Default::default()
},
Web3Rpc {
name: "c".to_string(),
tier: 0,
head_block: RwLock::new(blocks.get(2).cloned()),
..Default::default()
},
Web3Rpc {
name: "d".to_string(),
tier: 1,
head_block: RwLock::new(None),
..Default::default()
},
Web3Rpc {
name: "e".to_string(),
tier: 1,
head_block: RwLock::new(blocks.get(1).cloned()),
..Default::default()
},
Web3Rpc {
name: "f".to_string(),
tier: 1,
head_block: RwLock::new(blocks.get(2).cloned()),
..Default::default()
},
]
.into_iter()
.map(Arc::new)
.collect();
2023-02-11 07:45:57 +03:00
rpcs.sort_by_cached_key(rpc_sync_status_sort_key);
let names_in_sort_order: Vec<_> = rpcs.iter().map(|x| x.name.as_str()).collect();
assert_eq!(names_in_sort_order, ["c", "f", "b", "e", "a", "d"]);
}
#[tokio::test]
2022-11-25 10:41:53 +03:00
async fn test_server_selection_by_height() {
// TODO: do this better. can test_env_logger and tokio test be stacked?
let _ = env_logger::builder()
.filter_level(LevelFilter::Error)
.filter_module("web3_proxy", LevelFilter::Trace)
.is_test(true)
.try_init();
2022-12-01 01:11:14 +03:00
let now: U256 = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs()
.into();
let lagged_block = Block {
hash: Some(H256::random()),
number: Some(0.into()),
2022-12-01 01:11:14 +03:00
timestamp: now - 1,
..Default::default()
};
let head_block = Block {
hash: Some(H256::random()),
number: Some(1.into()),
parent_hash: lagged_block.hash.unwrap(),
2022-12-01 01:11:14 +03:00
timestamp: now,
..Default::default()
};
let lagged_block = Arc::new(lagged_block);
let head_block = Arc::new(head_block);
2023-02-15 04:41:40 +03:00
let mut lagged_block: Web3ProxyBlock = lagged_block.try_into().unwrap();
let mut head_block: Web3ProxyBlock = head_block.try_into().unwrap();
2022-12-03 08:31:03 +03:00
let block_data_limit = u64::MAX;
let head_rpc = Web3Rpc {
name: "synced".to_string(),
soft_limit: 1_000,
automatic_block_limit: false,
backup: false,
block_data_limit: block_data_limit.into(),
2023-01-04 09:37:51 +03:00
tier: 0,
2022-12-03 08:31:03 +03:00
head_block: RwLock::new(Some(head_block.clone())),
provider: AsyncRwLock::new(Some(Arc::new(Web3Provider::Mock))),
..Default::default()
};
let lagged_rpc = Web3Rpc {
name: "lagged".to_string(),
soft_limit: 1_000,
2022-12-06 00:13:36 +03:00
automatic_block_limit: false,
backup: false,
block_data_limit: block_data_limit.into(),
2023-01-04 09:37:51 +03:00
tier: 0,
2022-12-03 08:31:03 +03:00
head_block: RwLock::new(Some(lagged_block.clone())),
provider: AsyncRwLock::new(Some(Arc::new(Web3Provider::Mock))),
..Default::default()
};
assert!(head_rpc.has_block_data(lagged_block.number()));
assert!(head_rpc.has_block_data(head_block.number()));
assert!(lagged_rpc.has_block_data(lagged_block.number()));
assert!(!lagged_rpc.has_block_data(head_block.number()));
let head_rpc = Arc::new(head_rpc);
let lagged_rpc = Arc::new(lagged_rpc);
2023-02-15 04:41:40 +03:00
let rpcs_by_name = HashMap::from([
(head_rpc.name.clone(), head_rpc.clone()),
(lagged_rpc.name.clone(), lagged_rpc.clone()),
]);
let (block_sender, _block_receiver) = flume::unbounded();
let (pending_tx_id_sender, pending_tx_id_receiver) = flume::unbounded();
let (watch_consensus_rpcs_sender, _watch_consensus_rpcs_receiver) =
watch::channel(Default::default());
let (watch_consensus_head_sender, _watch_consensus_head_receiver) =
watch::channel(Default::default());
2023-01-23 09:02:08 +03:00
// TODO: make a Web3Rpcs::new
2023-02-15 04:41:40 +03:00
let rpcs = Web3Rpcs {
block_sender,
2023-02-27 07:00:13 +03:00
by_name: RwLock::new(rpcs_by_name),
http_interval_sender: None,
watch_consensus_head_sender: Some(watch_consensus_head_sender),
watch_consensus_rpcs_sender,
pending_transaction_cache: Cache::builder()
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default()),
pending_tx_id_receiver,
pending_tx_id_sender,
blocks_by_hash: Cache::builder()
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default()),
blocks_by_number: Cache::builder()
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default()),
// TODO: test max_block_age?
max_block_age: None,
// TODO: test max_block_lag?
max_block_lag: None,
min_head_rpcs: 1,
min_sum_soft_limit: 1,
};
let authorization = Arc::new(Authorization::internal(None).unwrap());
2023-02-27 07:00:13 +03:00
let mut consensus_finder = ConsensusFinder::new(None, None);
// process None so that
2023-02-15 04:41:40 +03:00
rpcs.process_block_from_rpc(
&authorization,
&mut consensus_finder,
None,
lagged_rpc.clone(),
&None,
)
.await
.expect("its lagged, but it should still be seen as consensus if its the first to report");
rpcs.process_block_from_rpc(
&authorization,
&mut consensus_finder,
None,
head_rpc.clone(),
&None,
)
.await
.unwrap();
// no head block because the rpcs haven't communicated through their channels
2023-02-15 04:41:40 +03:00
assert!(rpcs.head_block_hash().is_none());
// all_backend_connections gives all non-backup servers regardless of sync status
assert_eq!(
2023-02-15 04:41:40 +03:00
rpcs.all_connections(&authorization, None, None, None, false)
.await
.unwrap()
.len(),
2
);
// best_synced_backend_connection requires servers to be synced with the head block
2023-02-15 04:41:40 +03:00
let x = rpcs
2023-02-16 08:05:41 +03:00
.best_available_rpc(&authorization, None, &[], None, None)
2022-11-28 09:52:16 +03:00
.await
.unwrap();
dbg!(&x);
2023-02-15 04:41:40 +03:00
assert!(matches!(x, OpenRequestResult::NotReady));
2023-02-15 04:41:40 +03:00
// add lagged blocks to the rpcs. both servers should be allowed
lagged_block = rpcs.try_cache_block(lagged_block, true).await.unwrap();
2023-02-15 04:41:40 +03:00
rpcs.process_block_from_rpc(
&authorization,
&mut consensus_finder,
Some(lagged_block.clone()),
lagged_rpc,
&None,
)
.await
.unwrap();
rpcs.process_block_from_rpc(
&authorization,
&mut consensus_finder,
Some(lagged_block.clone()),
head_rpc.clone(),
&None,
)
.await
.unwrap();
2023-02-15 04:41:40 +03:00
assert_eq!(rpcs.num_synced_rpcs(), 2);
2023-02-15 04:41:40 +03:00
// add head block to the rpcs. lagged_rpc should not be available
head_block = rpcs.try_cache_block(head_block, true).await.unwrap();
2023-02-15 04:41:40 +03:00
rpcs.process_block_from_rpc(
&authorization,
&mut consensus_finder,
Some(head_block.clone()),
head_rpc,
&None,
)
.await
.unwrap();
2023-02-15 04:41:40 +03:00
assert_eq!(rpcs.num_synced_rpcs(), 1);
2022-11-25 10:41:53 +03:00
assert!(matches!(
2023-02-16 08:05:41 +03:00
rpcs.best_available_rpc(&authorization, None, &[], None, None)
2022-11-25 10:41:53 +03:00
.await,
Ok(OpenRequestResult::Handle(_))
));
assert!(matches!(
2023-02-16 08:05:41 +03:00
rpcs.best_available_rpc(&authorization, None, &[], Some(&0.into()), None)
2022-11-25 10:41:53 +03:00
.await,
Ok(OpenRequestResult::Handle(_))
));
assert!(matches!(
2023-02-16 08:05:41 +03:00
rpcs.best_available_rpc(&authorization, None, &[], Some(&1.into()), None)
2022-11-25 10:41:53 +03:00
.await,
Ok(OpenRequestResult::Handle(_))
));
// future block should not get a handle
2023-02-16 08:05:41 +03:00
let future_rpc = rpcs
.best_available_rpc(&authorization, None, &[], Some(&2.into()), None)
.await;
assert!(matches!(future_rpc, Ok(OpenRequestResult::NotReady)));
2022-11-25 10:41:53 +03:00
}
#[tokio::test]
async fn test_server_selection_by_archive() {
// TODO: do this better. can test_env_logger and tokio test be stacked?
let _ = env_logger::builder()
.filter_level(LevelFilter::Error)
.filter_module("web3_proxy", LevelFilter::Trace)
.is_test(true)
.try_init();
2022-12-01 01:11:14 +03:00
let now: U256 = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs()
.into();
2022-12-03 08:31:03 +03:00
let head_block = Block {
2022-11-25 10:41:53 +03:00
hash: Some(H256::random()),
number: Some(1_000_000.into()),
parent_hash: H256::random(),
2022-12-01 01:11:14 +03:00
timestamp: now,
2022-11-25 10:41:53 +03:00
..Default::default()
};
2023-02-15 04:41:40 +03:00
let head_block: Web3ProxyBlock = Arc::new(head_block).try_into().unwrap();
2022-11-25 10:41:53 +03:00
let pruned_rpc = Web3Rpc {
2022-11-25 10:41:53 +03:00
name: "pruned".to_string(),
soft_limit: 3_000,
2022-12-06 00:13:36 +03:00
automatic_block_limit: false,
backup: false,
2022-11-25 10:41:53 +03:00
block_data_limit: 64.into(),
2023-01-04 09:37:51 +03:00
tier: 1,
2022-12-03 08:31:03 +03:00
head_block: RwLock::new(Some(head_block.clone())),
2023-02-16 08:05:41 +03:00
provider: AsyncRwLock::new(Some(Arc::new(Web3Provider::Mock))),
..Default::default()
2022-11-25 10:41:53 +03:00
};
let archive_rpc = Web3Rpc {
2022-11-25 10:41:53 +03:00
name: "archive".to_string(),
soft_limit: 1_000,
2022-12-06 00:13:36 +03:00
automatic_block_limit: false,
backup: false,
2022-11-25 10:41:53 +03:00
block_data_limit: u64::MAX.into(),
2023-01-04 09:37:51 +03:00
tier: 2,
2022-12-03 08:31:03 +03:00
head_block: RwLock::new(Some(head_block.clone())),
2023-02-16 08:05:41 +03:00
provider: AsyncRwLock::new(Some(Arc::new(Web3Provider::Mock))),
..Default::default()
2022-11-25 10:41:53 +03:00
};
assert!(pruned_rpc.has_block_data(head_block.number()));
assert!(archive_rpc.has_block_data(head_block.number()));
2022-11-25 10:41:53 +03:00
assert!(!pruned_rpc.has_block_data(&1.into()));
assert!(archive_rpc.has_block_data(&1.into()));
let pruned_rpc = Arc::new(pruned_rpc);
let archive_rpc = Arc::new(archive_rpc);
2023-02-15 04:41:40 +03:00
let rpcs_by_name = HashMap::from([
2022-11-25 10:41:53 +03:00
(pruned_rpc.name.clone(), pruned_rpc.clone()),
(archive_rpc.name.clone(), archive_rpc.clone()),
]);
let (block_sender, _) = flume::unbounded();
let (pending_tx_id_sender, pending_tx_id_receiver) = flume::unbounded();
let (watch_consensus_rpcs_sender, _) = watch::channel(Default::default());
let (watch_consensus_head_sender, _watch_consensus_head_receiver) =
watch::channel(Default::default());
2023-01-23 09:02:08 +03:00
// TODO: make a Web3Rpcs::new
2023-02-15 04:41:40 +03:00
let rpcs = Web3Rpcs {
block_sender,
2023-02-27 07:00:13 +03:00
by_name: RwLock::new(rpcs_by_name),
http_interval_sender: None,
watch_consensus_head_sender: Some(watch_consensus_head_sender),
watch_consensus_rpcs_sender,
pending_transaction_cache: Cache::builder()
2022-11-25 10:41:53 +03:00
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default()),
pending_tx_id_receiver,
pending_tx_id_sender,
blocks_by_hash: Cache::builder()
2022-11-25 10:41:53 +03:00
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default()),
blocks_by_number: Cache::builder()
2022-11-25 10:41:53 +03:00
.build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::default()),
min_head_rpcs: 1,
2023-02-15 04:41:40 +03:00
min_sum_soft_limit: 4_000,
max_block_age: None,
max_block_lag: None,
2022-11-25 10:41:53 +03:00
};
let authorization = Arc::new(Authorization::internal(None).unwrap());
2023-02-27 07:00:13 +03:00
let mut connection_heads = ConsensusFinder::new(None, None);
2022-11-25 10:41:53 +03:00
2023-02-15 04:41:40 +03:00
// min sum soft limit will require tier 2
rpcs.process_block_from_rpc(
&authorization,
&mut connection_heads,
Some(head_block.clone()),
pruned_rpc.clone(),
&None,
)
.await
.unwrap_err();
rpcs.process_block_from_rpc(
&authorization,
&mut connection_heads,
Some(head_block.clone()),
archive_rpc.clone(),
&None,
)
.await
.unwrap();
2022-11-25 10:41:53 +03:00
2023-02-15 04:41:40 +03:00
assert_eq!(rpcs.num_synced_rpcs(), 2);
2022-11-25 10:41:53 +03:00
// best_synced_backend_connection requires servers to be synced with the head block
2023-02-11 07:45:57 +03:00
// TODO: test with and without passing the head_block.number?
2023-02-16 08:05:41 +03:00
let best_available_server = rpcs
.best_available_rpc(&authorization, None, &[], Some(head_block.number()), None)
2022-11-25 10:41:53 +03:00
.await;
2023-02-16 08:05:41 +03:00
debug!("best_available_server: {:#?}", best_available_server);
2023-02-15 04:41:40 +03:00
2022-11-25 10:41:53 +03:00
assert!(matches!(
2023-02-16 08:05:41 +03:00
best_available_server.unwrap(),
2022-11-25 10:41:53 +03:00
OpenRequestResult::Handle(_)
));
2023-02-16 08:05:41 +03:00
let best_available_server_from_none = rpcs
.best_available_rpc(&authorization, None, &[], None, None)
.await;
// assert_eq!(best_available_server, best_available_server_from_none);
2023-02-15 04:41:40 +03:00
let best_archive_server = rpcs
2023-02-16 08:05:41 +03:00
.best_available_rpc(&authorization, None, &[], Some(&1.into()), None)
2022-11-25 10:41:53 +03:00
.await;
match best_archive_server {
Ok(OpenRequestResult::Handle(x)) => {
assert_eq!(x.clone_connection().name, "archive".to_string())
}
x => {
error!("unexpected result: {:?}", x);
}
}
}
}