web3-proxy/web3_proxy/src/rpcs/blockchain.rs

563 lines
20 KiB
Rust
Raw Normal View History

//! Keep track of the blockchain as seen by a Web3Rpcs.
use super::consensus::ConsensusFinder;
use super::many::Web3Rpcs;
use super::one::Web3Rpc;
2022-08-24 03:59:05 +03:00
use super::transactions::TxStatus;
use crate::config::BlockAndRpc;
use crate::errors::{Web3ProxyError, Web3ProxyErrorContext, Web3ProxyResult};
use crate::frontend::authorization::Authorization;
2022-08-26 20:26:17 +03:00
use derive_more::From;
2022-08-27 06:11:58 +03:00
use ethers::prelude::{Block, TxHash, H256, U64};
use log::{debug, trace, warn};
2023-05-18 23:34:22 +03:00
use quick_cache_ttl::CacheWithTTL;
use serde::ser::SerializeStruct;
2022-09-01 08:58:55 +03:00
use serde::Serialize;
use serde_json::json;
use std::hash::Hash;
2022-09-01 08:58:55 +03:00
use std::{cmp::Ordering, fmt::Display, sync::Arc};
use tokio::sync::broadcast;
2022-09-05 08:53:58 +03:00
// TODO: type for Hydrated Blocks with their full transactions?
2022-08-30 23:01:42 +03:00
pub type ArcBlock = Arc<Block<TxHash>>;
2023-05-18 23:34:22 +03:00
pub type BlocksByHashCache = Arc<CacheWithTTL<H256, Web3ProxyBlock>>;
pub type BlocksByNumberCache = Arc<CacheWithTTL<U64, H256>>;
2022-08-30 23:01:42 +03:00
/// A block and its age.
#[derive(Clone, Debug, Default, From)]
pub struct Web3ProxyBlock {
2022-12-03 08:31:03 +03:00
pub block: ArcBlock,
/// number of seconds this block was behind the current time when received
/// this is only set if the block is from a subscription
pub received_age: Option<u64>,
2022-08-26 20:26:17 +03:00
}
impl Serialize for Web3ProxyBlock {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
// TODO: i'm not sure about this name
let mut state = serializer.serialize_struct("saved_block", 2)?;
state.serialize_field("age", &self.age())?;
let block = json!({
2023-04-21 05:55:18 +03:00
"hash": self.block.hash,
"parent_hash": self.block.parent_hash,
"number": self.block.number,
"timestamp": self.block.timestamp,
});
state.serialize_field("block", &block)?;
state.end()
}
}
impl PartialEq for Web3ProxyBlock {
2022-12-17 07:05:01 +03:00
fn eq(&self, other: &Self) -> bool {
match (self.block.hash, other.block.hash) {
(None, None) => true,
(Some(_), None) => false,
(None, Some(_)) => false,
(Some(s), Some(o)) => s == o,
}
}
}
impl Eq for Web3ProxyBlock {}
impl Hash for Web3ProxyBlock {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.block.hash.hash(state);
}
}
impl Web3ProxyBlock {
/// A new block has arrived over a subscription
2023-02-15 04:41:40 +03:00
pub fn try_new(block: ArcBlock) -> Option<Self> {
if block.number.is_none() || block.hash.is_none() {
return None;
}
let mut x = Self {
block,
received_age: None,
};
2022-12-06 00:13:36 +03:00
// no need to recalulate lag every time
// if the head block gets too old, a health check restarts this connection
// TODO: emit a stat for received_age
x.received_age = Some(x.age());
2022-12-06 00:13:36 +03:00
2023-02-15 04:41:40 +03:00
Some(x)
2022-12-06 00:13:36 +03:00
}
pub fn age(&self) -> u64 {
let now = chrono::Utc::now().timestamp();
2022-12-03 08:31:03 +03:00
let block_timestamp = self.block.timestamp.as_u32() as i64;
2022-12-03 08:31:03 +03:00
2023-01-03 19:33:49 +03:00
if block_timestamp < now {
2022-12-03 08:31:03 +03:00
// this server is still syncing from too far away to serve requests
// u64 is safe because we checked equality above
(now - block_timestamp) as u64
2022-12-03 08:31:03 +03:00
} else {
0
2022-12-06 00:13:36 +03:00
}
2022-12-03 08:31:03 +03:00
}
#[inline(always)]
pub fn parent_hash(&self) -> &H256 {
&self.block.parent_hash
}
#[inline(always)]
pub fn hash(&self) -> &H256 {
self.block
.hash
.as_ref()
.expect("saved blocks must have a hash")
2022-12-03 08:31:03 +03:00
}
#[inline(always)]
pub fn number(&self) -> &U64 {
self.block
.number
.as_ref()
.expect("saved blocks must have a number")
2022-12-03 08:31:03 +03:00
}
}
2023-02-15 04:41:40 +03:00
impl TryFrom<ArcBlock> for Web3ProxyBlock {
2023-03-20 23:45:21 +03:00
type Error = Web3ProxyError;
2023-02-15 04:41:40 +03:00
fn try_from(x: ArcBlock) -> Result<Self, Self::Error> {
if x.number.is_none() || x.hash.is_none() {
2023-03-20 23:45:21 +03:00
return Err(Web3ProxyError::NoBlockNumberOrHash);
2023-02-15 04:41:40 +03:00
}
let b = Web3ProxyBlock {
block: x,
received_age: None,
2023-02-15 04:41:40 +03:00
};
Ok(b)
2022-12-03 08:31:03 +03:00
}
}
impl Display for Web3ProxyBlock {
2022-09-01 08:58:55 +03:00
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{} ({}, {}s old)",
self.number(),
self.hash(),
self.age()
)
2022-09-01 08:58:55 +03:00
}
}
impl Web3Rpcs {
2022-12-03 08:31:03 +03:00
/// add a block to our mappings and track the heaviest chain
pub async fn try_cache_block(
&self,
block: Web3ProxyBlock,
heaviest_chain: bool,
) -> Web3ProxyResult<Web3ProxyBlock> {
2022-09-03 00:35:03 +03:00
// TODO: i think we can rearrange this function to make it faster on the hot path
2023-05-18 23:34:22 +03:00
if block.hash().is_zero() {
debug!("Skipping block without hash!");
return Ok(block);
}
let block_num = block.number();
2022-10-27 00:39:26 +03:00
// this block is very likely already in block_hashes
// TODO: use their get_with
let block_hash = *block.hash();
2022-09-30 07:18:18 +03:00
// TODO: think more about heaviest_chain. would be better to do the check inside this function
2022-09-05 09:13:36 +03:00
if heaviest_chain {
// this is the only place that writes to block_numbers
// multiple inserts should be okay though
2022-12-03 08:31:03 +03:00
// TODO: info that there was a fork?
self.blocks_by_number
.get_or_insert_async(block_num, async move { block_hash })
.await;
}
let block = self
.blocks_by_hash
.get_or_insert_async(&block_hash, async move { block })
.await;
2022-09-05 09:13:36 +03:00
Ok(block)
}
2022-08-27 05:13:36 +03:00
/// Get a block from caches with fallback.
/// Will query a specific node or the best available.
2022-08-26 20:26:17 +03:00
pub async fn block(
&self,
authorization: &Arc<Authorization>,
2022-08-26 20:26:17 +03:00
hash: &H256,
rpc: Option<&Arc<Web3Rpc>>,
) -> Web3ProxyResult<Web3ProxyBlock> {
// first, try to get the hash from our cache
2022-09-03 00:35:03 +03:00
// the cache is set last, so if its here, its everywhere
// TODO: use try_get_with
if let Some(block) = self.blocks_by_hash.get(hash) {
2022-09-05 08:53:58 +03:00
return Ok(block);
}
// block not in cache. we need to ask an rpc for it
2022-09-23 00:51:52 +03:00
let get_block_params = (*hash, false);
let block: Option<ArcBlock> = if let Some(rpc) = rpc {
// TODO: request_with_metadata would probably be better
rpc.authorized_request::<_, Option<ArcBlock>>(
"eth_getBlockByHash",
&get_block_params,
authorization,
None,
)
.await?
} else {
// TODO: request_with_metadata would probably be better
self.internal_request::<_, Option<ArcBlock>>("eth_getBlockByHash", &get_block_params)
2022-12-06 03:18:31 +03:00
.await?
2022-08-27 05:13:36 +03:00
};
match block {
Some(block) => {
let block = self.try_cache_block(block.try_into()?, false).await?;
Ok(block)
}
// TODO: better error. some blocks are known, just not this one
None => Err(Web3ProxyError::NoBlocksKnown),
}
}
/// Convenience method to get the cannonical block at a given block height.
pub async fn block_hash(
&self,
authorization: &Arc<Authorization>,
num: &U64,
) -> Web3ProxyResult<(H256, u64)> {
2023-02-06 04:58:03 +03:00
let (block, block_depth) = self.cannonical_block(authorization, num).await?;
let hash = *block.hash();
2023-02-06 04:58:03 +03:00
Ok((hash, block_depth))
}
/// Get the heaviest chain's block from cache or backend rpc
2023-01-23 09:02:08 +03:00
/// Caution! If a future block is requested, this might wait forever. Be sure to have a timeout outside of this!
/// TODO: take a RequestMetadata
pub async fn cannonical_block(
&self,
authorization: &Arc<Authorization>,
num: &U64,
) -> Web3ProxyResult<(Web3ProxyBlock, u64)> {
2022-08-28 02:49:41 +03:00
// we only have blocks by hash now
2022-09-05 08:53:58 +03:00
// maybe save them during save_block in a blocks_by_number Cache<U64, Vec<ArcBlock>>
2022-08-28 02:49:41 +03:00
// if theres multiple, use petgraph to find the one on the main chain (and remove the others if they have enough confirmations)
2022-08-26 20:26:17 +03:00
2023-01-23 09:02:08 +03:00
let mut consensus_head_receiver = self
.watch_consensus_head_sender
2023-01-23 09:02:08 +03:00
.as_ref()
2023-03-20 23:45:21 +03:00
.web3_context("need new head subscriptions to fetch cannonical_block")?
.subscribe();
2023-01-23 09:02:08 +03:00
2022-09-03 00:35:03 +03:00
// be sure the requested block num exists
// TODO: is this okay? what if we aren't synced?!
2023-02-15 04:41:40 +03:00
let mut head_block_num = *consensus_head_receiver
.borrow_and_update()
.as_ref()
2023-03-20 23:45:21 +03:00
.web3_context("no consensus head block")?
2023-02-15 04:41:40 +03:00
.number();
2022-11-03 02:14:16 +03:00
2023-01-23 09:02:08 +03:00
loop {
if num <= &head_block_num {
break;
2023-01-23 09:02:08 +03:00
}
2022-11-03 02:14:16 +03:00
trace!("waiting for future block {} > {}", num, head_block_num);
2023-01-23 09:02:08 +03:00
consensus_head_receiver.changed().await?;
2023-02-15 04:41:40 +03:00
if let Some(head) = consensus_head_receiver.borrow_and_update().as_ref() {
head_block_num = *head.number();
}
}
let block_depth = (head_block_num - num).as_u64();
2023-01-23 09:02:08 +03:00
2022-09-03 00:35:03 +03:00
// try to get the hash from our cache
// deref to not keep the lock open
if let Some(block_hash) = self.blocks_by_number.get(num) {
2022-09-03 00:35:03 +03:00
// TODO: sometimes this needs to fetch the block. why? i thought block_numbers would only be set if the block hash was set
// TODO: pass authorization through here?
let block = self.block(authorization, &block_hash, None).await?;
2022-11-03 02:14:16 +03:00
2023-02-06 04:58:03 +03:00
return Ok((block, block_depth));
2022-09-03 00:35:03 +03:00
}
// block number not in cache. we need to ask an rpc for it
// TODO: this error is too broad
let response = self
.internal_request::<_, Option<ArcBlock>>("eth_getBlockByNumber", &(*num, false))
.await?
.ok_or(Web3ProxyError::NoBlocksKnown)?;
let block = Web3ProxyBlock::try_from(response)?;
// the block was fetched using eth_getBlockByNumber, so it should have all fields and be on the heaviest chain
let block = self.try_cache_block(block, true).await?;
2023-02-06 04:58:03 +03:00
Ok((block, block_depth))
}
2022-08-26 20:26:17 +03:00
pub(super) async fn process_incoming_blocks(
2022-08-24 03:59:05 +03:00
&self,
authorization: &Arc<Authorization>,
block_receiver: flume::Receiver<BlockAndRpc>,
2022-09-05 08:53:58 +03:00
// TODO: document that this is a watch sender and not a broadcast! if things get busy, blocks might get missed
// Geth's subscriptions have the same potential for skipping blocks.
2022-08-24 03:59:05 +03:00
pending_tx_sender: Option<broadcast::Sender<TxStatus>>,
2023-05-24 00:40:34 +03:00
) -> Web3ProxyResult<()> {
2023-02-27 07:00:13 +03:00
let mut connection_heads = ConsensusFinder::new(self.max_block_age, self.max_block_lag);
2022-08-24 03:59:05 +03:00
2023-02-07 02:20:36 +03:00
loop {
match block_receiver.recv_async().await {
2023-02-07 02:20:36 +03:00
Ok((new_block, rpc)) => {
let rpc_name = rpc.name.clone();
if let Err(err) = self
.process_block_from_rpc(
authorization,
&mut connection_heads,
new_block,
rpc,
&pending_tx_sender,
)
.await
{
2023-03-30 15:42:56 +03:00
warn!(
"error while processing block from rpc {}: {:#?}",
rpc_name, err
);
2023-02-07 02:20:36 +03:00
}
}
Err(err) => {
warn!("block_receiver exited! {:#?}", err);
return Err(err.into());
}
}
2022-08-24 03:59:05 +03:00
}
}
2022-08-26 20:26:17 +03:00
/// `connection_heads` is a mapping of rpc_names to head block hashes.
2022-11-21 01:52:08 +03:00
/// self.blockchain_map is a mapping of hashes to the complete ArcBlock.
2022-08-27 02:44:25 +03:00
/// TODO: return something?
pub(crate) async fn process_block_from_rpc(
&self,
authorization: &Arc<Authorization>,
consensus_finder: &mut ConsensusFinder,
2023-02-27 07:00:13 +03:00
new_block: Option<Web3ProxyBlock>,
rpc: Arc<Web3Rpc>,
_pending_tx_sender: &Option<broadcast::Sender<TxStatus>>,
2023-03-20 23:45:21 +03:00
) -> Web3ProxyResult<()> {
// TODO: how should we handle an error here?
if !consensus_finder
2023-02-27 07:00:13 +03:00
.update_rpc(new_block.clone(), rpc.clone(), self)
.await
2023-03-20 23:45:21 +03:00
.web3_context("failed to update rpc")?
{
// nothing changed. no need to scan for a new consensus head
return Ok(());
}
2022-12-01 01:11:14 +03:00
2023-05-18 23:34:22 +03:00
let new_consensus_rpcs = match consensus_finder
.find_consensus_connections(authorization, self)
.await
2023-03-30 15:42:56 +03:00
{
Err(err) => {
return Err(err).web3_context("error while finding consensus head block!");
2023-03-30 15:42:56 +03:00
}
Ok(None) => {
return Err(Web3ProxyError::NoConsensusHeadBlock);
2023-03-30 15:42:56 +03:00
}
Ok(Some(x)) => x,
};
2023-01-23 09:02:08 +03:00
2023-05-18 23:34:22 +03:00
trace!("new_synced_connections: {:#?}", new_consensus_rpcs);
2023-05-12 03:04:33 +03:00
let watch_consensus_head_sender = self.watch_consensus_head_sender.as_ref().unwrap();
2023-05-18 23:34:22 +03:00
let consensus_tier = new_consensus_rpcs.tier;
// TODO: think more about this unwrap
let total_tiers = consensus_finder.worst_tier().unwrap_or(10);
2023-05-18 23:34:22 +03:00
let backups_needed = new_consensus_rpcs.backups_needed;
let consensus_head_block = new_consensus_rpcs.head_block.clone();
let num_consensus_rpcs = new_consensus_rpcs.num_consensus_rpcs();
let num_active_rpcs = consensus_finder.len();
let total_rpcs = self.by_name.load().len();
2023-01-23 09:02:08 +03:00
let old_consensus_head_connections = self
.watch_consensus_rpcs_sender
2023-05-18 23:34:22 +03:00
.send_replace(Some(Arc::new(new_consensus_rpcs)));
let backups_voted_str = if backups_needed { "B " } else { "" };
2023-01-20 05:14:47 +03:00
match old_consensus_head_connections.as_ref() {
None => {
debug!(
"first {}/{} {}{}/{}/{} block={}, rpc={}",
consensus_tier,
total_tiers,
backups_voted_str,
num_consensus_rpcs,
num_active_rpcs,
total_rpcs,
consensus_head_block,
rpc,
);
if backups_needed {
// TODO: what else should be in this error?
warn!("Backup RPCs are in use!");
2022-12-03 08:31:03 +03:00
}
// this should already be cached
let consensus_head_block = self.try_cache_block(consensus_head_block, true).await?;
watch_consensus_head_sender
.send(Some(consensus_head_block))
2023-03-20 23:45:21 +03:00
.or(Err(Web3ProxyError::WatchSendError))
.web3_context(
"watch_consensus_head_sender failed sending first consensus_head_block",
)?;
}
Some(old_consensus_connections) => {
let old_head_block = &old_consensus_connections.head_block;
// TODO: do this log item better
let rpc_head_str = new_block
.map(|x| x.to_string())
.unwrap_or_else(|| "None".to_string());
match consensus_head_block.number().cmp(old_head_block.number()) {
Ordering::Equal => {
// multiple blocks with the same fork!
if consensus_head_block.hash() == old_head_block.hash() {
// no change in hash. no need to use watch_consensus_head_sender
// TODO: trace level if rpc is backup
debug!(
"con {}/{} {}{}/{}/{} con={} rpc={}@{}",
2023-02-15 23:33:43 +03:00
consensus_tier,
total_tiers,
backups_voted_str,
num_consensus_rpcs,
num_active_rpcs,
total_rpcs,
consensus_head_block,
rpc,
rpc_head_str,
)
} else {
// hash changed
debug!(
"unc {}/{} {}{}/{}/{} con_head={} old={} rpc={}@{}",
2023-02-15 23:33:43 +03:00
consensus_tier,
total_tiers,
backups_voted_str,
num_consensus_rpcs,
num_active_rpcs,
total_rpcs,
consensus_head_block,
old_head_block,
rpc,
rpc_head_str,
);
let consensus_head_block = self
.try_cache_block(consensus_head_block, true)
.await
2023-03-20 23:45:21 +03:00
.web3_context("save consensus_head_block as heaviest chain")?;
watch_consensus_head_sender
.send(Some(consensus_head_block))
2023-03-20 23:45:21 +03:00
.or(Err(Web3ProxyError::WatchSendError))
.web3_context("watch_consensus_head_sender failed sending uncled consensus_head_block")?;
}
}
Ordering::Less => {
// this is unlikely but possible
// TODO: better log
warn!(
2023-03-23 00:23:14 +03:00
"chain rolled back {}/{} {}{}/{}/{} con={} old={} rpc={}@{}",
consensus_tier,
total_tiers,
backups_voted_str,
num_consensus_rpcs,
num_active_rpcs,
total_rpcs,
consensus_head_block,
old_head_block,
rpc,
rpc_head_str,
);
if backups_needed {
// TODO: what else should be in this error?
warn!("Backup RPCs are in use!");
}
// TODO: tell save_block to remove any higher block numbers from the cache. not needed because we have other checks on requested blocks being > head, but still seems like a good idea
let consensus_head_block = self
.try_cache_block(consensus_head_block, true)
.await
2023-03-20 23:45:21 +03:00
.web3_context(
"save_block sending consensus_head_block as heaviest chain",
)?;
watch_consensus_head_sender
.send(Some(consensus_head_block))
2023-03-20 23:45:21 +03:00
.or(Err(Web3ProxyError::WatchSendError))
.web3_context("watch_consensus_head_sender failed sending rollback consensus_head_block")?;
}
Ordering::Greater => {
debug!(
"new {}/{} {}{}/{}/{} con={} rpc={}@{}",
consensus_tier,
total_tiers,
backups_voted_str,
num_consensus_rpcs,
num_active_rpcs,
total_rpcs,
consensus_head_block,
rpc,
rpc_head_str,
);
if backups_needed {
// TODO: what else should be in this error?
warn!("Backup RPCs are in use!");
}
let consensus_head_block =
self.try_cache_block(consensus_head_block, true).await?;
2023-03-20 23:45:21 +03:00
watch_consensus_head_sender.send(Some(consensus_head_block))
.or(Err(Web3ProxyError::WatchSendError))
.web3_context("watch_consensus_head_sender failed sending new consensus_head_block")?;
}
}
}
}
Ok(())
}
}