tornadocontrib/web3-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

loud log on rollback

Browse Source
This commit is contained in:
Bryan Stitt 2023-06-09 13:30:00 -07:00
parent bd87fcb13c
commit 8bf0ee473d
3 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
web3_proxy/src/rpcs/blockchain.rs
View File

@ -403,9 +403,11 @@ impl Web3Rpcs {
let num_active_rpcs = consensus_finder.len();
let total_rpcs = self.by_name.load().len();
let new_consensus_rpcs = Arc::new(new_consensus_rpcs);
let old_consensus_head_connections = self
.watch_consensus_rpcs_sender
.send_replace(Some(Arc::new(new_consensus_rpcs)));
.send_replace(Some(new_consensus_rpcs.clone()));
let backups_voted_str = if backups_needed { "B " } else { "" };
@ -494,9 +496,9 @@ impl Web3Rpcs {
}
Ordering::Less => {
// this is unlikely but possible
// TODO: better log
// TODO: better log that includes all the votes
warn!(
"chain rolled back {}/{} {}{}/{}/{} con={} old={} rpc={}@{}",
"chain rolled back {}/{} {}{}/{}/{} con={} old={} rpc={}@{}. {:#?} -> {:#?}",
consensus_tier,
total_tiers,
backups_voted_str,
@ -507,6 +509,8 @@ impl Web3Rpcs {
old_head_block,
rpc,
rpc_head_str,
old_consensus_connections,
new_consensus_rpcs,
);
if backups_needed {

9
web3_proxy/src/rpcs/consensus.rs
View File

@ -470,6 +470,9 @@ impl ConsensusFinder {
// create the histogram
let mut hist = Histogram::<u32>::new_with_bounds(1, max_latency, 3).unwrap();
// TODO: resize shouldn't be necessary, but i've seen it error
hist.auto(true);
for weighted_latency_ms in weighted_latencies.values() {
hist.record(*weighted_latency_ms)?;
}
@ -550,7 +553,7 @@ impl ConsensusFinder {
let num_known = self.rpc_heads.len();
if num_known < web3_rpcs.min_head_rpcs {
if num_known < web3_rpcs.min_synced_rpcs {
// this keeps us from serving requests when the proxy first starts
trace!("not enough servers known");
return Ok(None);
@ -671,7 +674,7 @@ impl ConsensusFinder {
continue;
}
// TODO: different mins for backup vs primary
if rpc_names.len() < web3_rpcs.min_head_rpcs {
if rpc_names.len() < web3_rpcs.min_synced_rpcs {
continue;
}
@ -683,7 +686,7 @@ impl ConsensusFinder {
.filter_map(|x| web3_rpcs.get(x))
.collect();
if consensus_rpcs.len() < web3_rpcs.min_head_rpcs {
if consensus_rpcs.len() < web3_rpcs.min_synced_rpcs {
continue;
}
// consensus found!

20
web3_proxy/src/rpcs/many.rs
View File

@ -61,7 +61,7 @@ pub struct Web3Rpcs {
/// blocks on the heaviest chain
pub(super) blocks_by_number: BlocksByNumberCache,
/// the number of rpcs required to agree on consensus for the head block (thundering herd protection)
pub(super) min_head_rpcs: usize,
pub(super) min_synced_rpcs: usize,
/// the soft limit required to agree on consensus for the head block. (thundering herd protection)
pub(super) min_sum_soft_limit: u32,
/// how far behind the highest known block height we can be before we stop serving requests
@ -121,7 +121,7 @@ impl Web3Rpcs {
by_name,
max_block_age,
max_block_lag,
min_head_rpcs,
min_synced_rpcs: min_head_rpcs,
min_sum_soft_limit,
name,
pending_transaction_cache,
@ -254,10 +254,10 @@ impl Web3Rpcs {
let num_rpcs = self.by_name.load().len();
if num_rpcs < self.min_head_rpcs {
if num_rpcs < self.min_synced_rpcs {
return Err(Web3ProxyError::NotEnoughRpcs {
num_known: num_rpcs,
min_head_rpcs: self.min_head_rpcs,
min_head_rpcs: self.min_synced_rpcs,
});
}
@ -277,7 +277,7 @@ impl Web3Rpcs {
}
pub fn min_head_rpcs(&self) -> usize {
self.min_head_rpcs
self.min_synced_rpcs
}
/// subscribe to blocks and transactions from all the backend rpcs.
@ -570,7 +570,7 @@ impl Web3Rpcs {
.cloned(),
);
if potential_rpcs.len() >= self.min_head_rpcs {
if potential_rpcs.len() >= self.min_synced_rpcs {
// we have enough potential rpcs. try to load balance
potential_rpcs.sort_by_cached_key(|x| {
x.shuffle_for_load_balancing_on(max_block_needed.copied())
@ -617,7 +617,7 @@ impl Web3Rpcs {
potential_rpcs.extend(more_rpcs);
if potential_rpcs.len() >= self.min_head_rpcs {
if potential_rpcs.len() >= self.min_synced_rpcs {
// we have enough potential rpcs. try to load balance
potential_rpcs.sort_by_cached_key(|x| {
x.shuffle_for_load_balancing_on(max_block_needed.copied())
@ -1501,7 +1501,7 @@ mod tests {
max_block_age: None,
// TODO: test max_block_lag?
max_block_lag: None,
min_head_rpcs: 1,
min_synced_rpcs: 1,
min_sum_soft_limit: 1,
};
@ -1777,7 +1777,7 @@ mod tests {
blocks_by_number: CacheBuilder::new(100)
.time_to_live(Duration::from_secs(120))
.build(),
min_head_rpcs: 1,
min_synced_rpcs: 1,
min_sum_soft_limit: 4_000,
max_block_age: None,
max_block_lag: None,
@ -1957,7 +1957,7 @@ mod tests {
pending_tx_id_sender,
blocks_by_hash: Cache::new(10_000),
blocks_by_number: Cache::new(10_000),
min_head_rpcs: 1,
min_synced_rpcs: 1,
min_sum_soft_limit: 1_000,
max_block_age: None,
max_block_lag: None,