change weight to tier
This commit is contained in:
parent
5a567ebeea
commit
c0fc999e02
2
TODO.md
2
TODO.md
|
@ -293,6 +293,7 @@ These are not yet ordered. There might be duplicates. We might not actually need
|
||||||
- [x] serde collect unknown fields in config instead of crash
|
- [x] serde collect unknown fields in config instead of crash
|
||||||
- [x] upgrade user tier by address
|
- [x] upgrade user tier by address
|
||||||
- [x] all_backend_connections skips syncing servers
|
- [x] all_backend_connections skips syncing servers
|
||||||
|
- [x] change weight back to tier
|
||||||
- [-] fix multiple origin and referer checks
|
- [-] fix multiple origin and referer checks
|
||||||
- [-] let users choose a % to log (or maybe x/second). someone like curve logging all reverts will be a BIG database very quickly
|
- [-] let users choose a % to log (or maybe x/second). someone like curve logging all reverts will be a BIG database very quickly
|
||||||
- this must be opt-in or spawned since it will slow things down and will make their calls less private
|
- this must be opt-in or spawned since it will slow things down and will make their calls less private
|
||||||
|
@ -541,7 +542,6 @@ in another repo: event subscriber
|
||||||
- if someone subscribes to all pending transactions, how should that count against rate limits
|
- if someone subscribes to all pending transactions, how should that count against rate limits
|
||||||
- when those rate limits are hit, what should happen?
|
- when those rate limits are hit, what should happen?
|
||||||
- missing pending transactions might be okay, but not missing confirmed blocks
|
- missing pending transactions might be okay, but not missing confirmed blocks
|
||||||
- [ ] double check weight sorting code
|
|
||||||
- [ ] sea-orm brings in async-std, but we are using tokio. benchmark switching
|
- [ ] sea-orm brings in async-std, but we are using tokio. benchmark switching
|
||||||
- [ ] this query always times out, but erigon can serve it quickly: `curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"debug_traceBlockByNumber","params":["latest"],"id":1}' 127.0.0.1:8544' 127.0.0.1:8544`
|
- [ ] this query always times out, but erigon can serve it quickly: `curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"debug_traceBlockByNumber","params":["latest"],"id":1}' 127.0.0.1:8544' 127.0.0.1:8544`
|
||||||
{"jsonrpc":"2.0","id":null,"error":{"code":-32099,"message":"deadline has elapsed"}}
|
{"jsonrpc":"2.0","id":null,"error":{"code":-32099,"message":"deadline has elapsed"}}
|
||||||
|
|
|
@ -315,7 +315,7 @@ mod tests {
|
||||||
block_data_limit: None,
|
block_data_limit: None,
|
||||||
soft_limit: 100,
|
soft_limit: 100,
|
||||||
hard_limit: None,
|
hard_limit: None,
|
||||||
weight: 1,
|
tier: 0,
|
||||||
subscribe_txs: Some(false),
|
subscribe_txs: Some(false),
|
||||||
extra: Default::default(),
|
extra: Default::default(),
|
||||||
},
|
},
|
||||||
|
@ -329,7 +329,7 @@ mod tests {
|
||||||
block_data_limit: None,
|
block_data_limit: None,
|
||||||
soft_limit: 100,
|
soft_limit: 100,
|
||||||
hard_limit: None,
|
hard_limit: None,
|
||||||
weight: 1,
|
tier: 0,
|
||||||
subscribe_txs: Some(false),
|
subscribe_txs: Some(false),
|
||||||
extra: Default::default(),
|
extra: Default::default(),
|
||||||
},
|
},
|
||||||
|
|
|
@ -198,9 +198,9 @@ pub struct Web3ConnectionConfig {
|
||||||
pub soft_limit: u32,
|
pub soft_limit: u32,
|
||||||
/// the requests per second at which the server throws errors (rate limit or otherwise)
|
/// the requests per second at which the server throws errors (rate limit or otherwise)
|
||||||
pub hard_limit: Option<u64>,
|
pub hard_limit: Option<u64>,
|
||||||
/// All else equal, a server with a lower weight receives more requests. Ranges 0-100
|
/// All else equal, a server with a lower tier receives all requests
|
||||||
#[serde(default = "default_weight")]
|
#[serde(default = "default_tier")]
|
||||||
pub weight: u32,
|
pub tier: u64,
|
||||||
/// Subscribe to the firehose of pending transactions
|
/// Subscribe to the firehose of pending transactions
|
||||||
/// Don't do this with free rpcs
|
/// Don't do this with free rpcs
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
@ -210,7 +210,7 @@ pub struct Web3ConnectionConfig {
|
||||||
pub extra: HashMap<String, serde_json::Value>,
|
pub extra: HashMap<String, serde_json::Value>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_weight() -> u32 {
|
fn default_tier() -> u64 {
|
||||||
0
|
0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -270,7 +270,7 @@ impl Web3ConnectionConfig {
|
||||||
block_sender,
|
block_sender,
|
||||||
tx_id_sender,
|
tx_id_sender,
|
||||||
true,
|
true,
|
||||||
self.weight,
|
self.tier,
|
||||||
open_request_handle_metrics,
|
open_request_handle_metrics,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|
|
@ -85,8 +85,8 @@ pub struct Web3Connection {
|
||||||
pub(super) automatic_block_limit: bool,
|
pub(super) automatic_block_limit: bool,
|
||||||
/// TODO: have an enum for this so that "no limit" prints pretty?
|
/// TODO: have an enum for this so that "no limit" prints pretty?
|
||||||
pub(super) block_data_limit: AtomicU64,
|
pub(super) block_data_limit: AtomicU64,
|
||||||
/// Lower weight are higher priority when sending requests. 0 to 99.
|
/// Lower tiers are higher priority when sending requests
|
||||||
pub(super) weight: f64,
|
pub(super) tier: u64,
|
||||||
/// TODO: should this be an AsyncRwLock?
|
/// TODO: should this be an AsyncRwLock?
|
||||||
pub(super) head_block: RwLock<Option<SavedBlock>>,
|
pub(super) head_block: RwLock<Option<SavedBlock>>,
|
||||||
pub(super) open_request_handle_metrics: Arc<OpenRequestHandleMetrics>,
|
pub(super) open_request_handle_metrics: Arc<OpenRequestHandleMetrics>,
|
||||||
|
@ -94,7 +94,7 @@ pub struct Web3Connection {
|
||||||
|
|
||||||
impl Web3Connection {
|
impl Web3Connection {
|
||||||
/// Connect to a web3 rpc
|
/// Connect to a web3 rpc
|
||||||
// TODO: have this take a builder (which will have channels attached)
|
// TODO: have this take a builder (which will have channels attached). or maybe just take the config and give the config public fields
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub async fn spawn(
|
pub async fn spawn(
|
||||||
name: String,
|
name: String,
|
||||||
|
@ -114,7 +114,7 @@ impl Web3Connection {
|
||||||
block_sender: Option<flume::Sender<BlockAndRpc>>,
|
block_sender: Option<flume::Sender<BlockAndRpc>>,
|
||||||
tx_id_sender: Option<flume::Sender<(TxHash, Arc<Self>)>>,
|
tx_id_sender: Option<flume::Sender<(TxHash, Arc<Self>)>>,
|
||||||
reconnect: bool,
|
reconnect: bool,
|
||||||
weight: u32,
|
tier: u64,
|
||||||
open_request_handle_metrics: Arc<OpenRequestHandleMetrics>,
|
open_request_handle_metrics: Arc<OpenRequestHandleMetrics>,
|
||||||
) -> anyhow::Result<(Arc<Web3Connection>, AnyhowJoinHandle<()>)> {
|
) -> anyhow::Result<(Arc<Web3Connection>, AnyhowJoinHandle<()>)> {
|
||||||
let hard_limit = hard_limit.map(|(hard_rate_limit, redis_pool)| {
|
let hard_limit = hard_limit.map(|(hard_rate_limit, redis_pool)| {
|
||||||
|
@ -128,9 +128,6 @@ impl Web3Connection {
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
// turn weight 0 into 100% and weight 100 into 0%
|
|
||||||
let weight = (100 - weight) as f64 / 100.0;
|
|
||||||
|
|
||||||
// TODO: should we do this even if block_sender is None? then we would know limits on private relays
|
// TODO: should we do this even if block_sender is None? then we would know limits on private relays
|
||||||
let block_data_limit: AtomicU64 = block_data_limit.unwrap_or_default().into();
|
let block_data_limit: AtomicU64 = block_data_limit.unwrap_or_default().into();
|
||||||
let automatic_block_limit =
|
let automatic_block_limit =
|
||||||
|
@ -151,7 +148,7 @@ impl Web3Connection {
|
||||||
automatic_block_limit,
|
automatic_block_limit,
|
||||||
block_data_limit,
|
block_data_limit,
|
||||||
head_block: RwLock::new(Default::default()),
|
head_block: RwLock::new(Default::default()),
|
||||||
weight,
|
tier,
|
||||||
open_request_handle_metrics,
|
open_request_handle_metrics,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1116,7 +1113,7 @@ impl Serialize for Web3Connection {
|
||||||
S: Serializer,
|
S: Serializer,
|
||||||
{
|
{
|
||||||
// 3 is the number of fields in the struct.
|
// 3 is the number of fields in the struct.
|
||||||
let mut state = serializer.serialize_struct("Web3Connection", 8)?;
|
let mut state = serializer.serialize_struct("Web3Connection", 9)?;
|
||||||
|
|
||||||
// the url is excluded because it likely includes private information. just show the name that we use in keys
|
// the url is excluded because it likely includes private information. just show the name that we use in keys
|
||||||
state.serialize_field("name", &self.name)?;
|
state.serialize_field("name", &self.name)?;
|
||||||
|
@ -1132,7 +1129,8 @@ impl Serialize for Web3Connection {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
state.serialize_field("weight", &self.weight)?;
|
state.serialize_field("tier", &self.tier)?;
|
||||||
|
state.serialize_field("weight", &1.0)?;
|
||||||
|
|
||||||
state.serialize_field("soft_limit", &self.soft_limit)?;
|
state.serialize_field("soft_limit", &self.soft_limit)?;
|
||||||
|
|
||||||
|
@ -1222,7 +1220,7 @@ mod tests {
|
||||||
soft_limit: 1_000,
|
soft_limit: 1_000,
|
||||||
automatic_block_limit: false,
|
automatic_block_limit: false,
|
||||||
block_data_limit: block_data_limit.into(),
|
block_data_limit: block_data_limit.into(),
|
||||||
weight: 100.0,
|
tier: 0,
|
||||||
head_block: RwLock::new(Some(head_block.clone())),
|
head_block: RwLock::new(Some(head_block.clone())),
|
||||||
open_request_handle_metrics: Arc::new(metrics),
|
open_request_handle_metrics: Arc::new(metrics),
|
||||||
};
|
};
|
||||||
|
@ -1269,7 +1267,7 @@ mod tests {
|
||||||
soft_limit: 1_000,
|
soft_limit: 1_000,
|
||||||
automatic_block_limit: false,
|
automatic_block_limit: false,
|
||||||
block_data_limit: block_data_limit.into(),
|
block_data_limit: block_data_limit.into(),
|
||||||
weight: 100.0,
|
tier: 0,
|
||||||
head_block: RwLock::new(Some(head_block.clone())),
|
head_block: RwLock::new(Some(head_block.clone())),
|
||||||
open_request_handle_metrics: Arc::new(metrics),
|
open_request_handle_metrics: Arc::new(metrics),
|
||||||
};
|
};
|
||||||
|
@ -1320,7 +1318,7 @@ mod tests {
|
||||||
soft_limit: 1_000,
|
soft_limit: 1_000,
|
||||||
automatic_block_limit: false,
|
automatic_block_limit: false,
|
||||||
block_data_limit: block_data_limit.into(),
|
block_data_limit: block_data_limit.into(),
|
||||||
weight: 100.0,
|
tier: 0,
|
||||||
head_block: RwLock::new(Some(head_block.clone())),
|
head_block: RwLock::new(Some(head_block.clone())),
|
||||||
open_request_handle_metrics: Arc::new(metrics),
|
open_request_handle_metrics: Arc::new(metrics),
|
||||||
};
|
};
|
||||||
|
|
|
@ -411,10 +411,9 @@ impl Web3Connections {
|
||||||
skip: &[Arc<Web3Connection>],
|
skip: &[Arc<Web3Connection>],
|
||||||
min_block_needed: Option<&U64>,
|
min_block_needed: Option<&U64>,
|
||||||
) -> anyhow::Result<OpenRequestResult> {
|
) -> anyhow::Result<OpenRequestResult> {
|
||||||
let usable_rpcs_by_head_num: BTreeMap<U64, Vec<Arc<Web3Connection>>> =
|
let usable_rpcs_by_head_num_and_weight: BTreeMap<(U64, u64), Vec<Arc<Web3Connection>>> =
|
||||||
if let Some(min_block_needed) = min_block_needed {
|
if let Some(min_block_needed) = min_block_needed {
|
||||||
// need a potentially old block. check all the rpcs
|
// need a potentially old block. check all the rpcs
|
||||||
// TODO: we are going to be checking "has_block_data" a lot now
|
|
||||||
let mut m = BTreeMap::new();
|
let mut m = BTreeMap::new();
|
||||||
|
|
||||||
for x in self
|
for x in self
|
||||||
|
@ -429,7 +428,9 @@ impl Web3Connections {
|
||||||
match x_head_block {
|
match x_head_block {
|
||||||
None => continue,
|
None => continue,
|
||||||
Some(x_head) => {
|
Some(x_head) => {
|
||||||
m.entry(x_head.number()).or_insert_with(Vec::new).push(x);
|
let key = (x_head.number(), u64::MAX - x.tier);
|
||||||
|
|
||||||
|
m.entry(key).or_insert_with(Vec::new).push(x);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -437,7 +438,6 @@ impl Web3Connections {
|
||||||
m
|
m
|
||||||
} else {
|
} else {
|
||||||
// need latest. filter the synced rpcs
|
// need latest. filter the synced rpcs
|
||||||
// TODO: double check has_block_data?
|
|
||||||
let synced_connections = self.synced_connections.load();
|
let synced_connections = self.synced_connections.load();
|
||||||
|
|
||||||
let head_block = match synced_connections.head_block.as_ref() {
|
let head_block = match synced_connections.head_block.as_ref() {
|
||||||
|
@ -445,26 +445,32 @@ impl Web3Connections {
|
||||||
Some(x) => x,
|
Some(x) => x,
|
||||||
};
|
};
|
||||||
|
|
||||||
// TODO: different allowed_lag depending on the chain
|
// TODO: self.allowed_lag instead of taking as an arg
|
||||||
if head_block.syncing(allowed_lag) {
|
if head_block.syncing(allowed_lag) {
|
||||||
return Ok(OpenRequestResult::NotReady);
|
return Ok(OpenRequestResult::NotReady);
|
||||||
}
|
}
|
||||||
|
|
||||||
let head_num = head_block.number();
|
let head_num = head_block.number();
|
||||||
|
|
||||||
let c: Vec<_> = synced_connections
|
let mut m = BTreeMap::new();
|
||||||
|
|
||||||
|
for x in synced_connections
|
||||||
.conns
|
.conns
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|x| !skip.contains(x))
|
.filter(|x| !skip.contains(x))
|
||||||
.cloned()
|
{
|
||||||
.collect();
|
let key = (head_num, u64::MAX - x.tier);
|
||||||
|
|
||||||
BTreeMap::from([(head_num, c)])
|
m.entry(key).or_insert_with(Vec::new).push(x.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
m
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut earliest_retry_at = None;
|
let mut earliest_retry_at = None;
|
||||||
|
|
||||||
for usable_rpcs in usable_rpcs_by_head_num.into_values().rev() {
|
for usable_rpcs in usable_rpcs_by_head_num_and_weight.into_values().rev() {
|
||||||
|
// under heavy load, it is possible for even our best server to be negative
|
||||||
let mut minimum = f64::MAX;
|
let mut minimum = f64::MAX;
|
||||||
|
|
||||||
// we sort on a combination of values. cache them here so that we don't do this math multiple times.
|
// we sort on a combination of values. cache them here so that we don't do this math multiple times.
|
||||||
|
@ -475,24 +481,22 @@ impl Web3Connections {
|
||||||
// TODO: get active requests out of redis (that's definitely too slow)
|
// TODO: get active requests out of redis (that's definitely too slow)
|
||||||
// TODO: do something with hard limit instead? (but that is hitting redis too much)
|
// TODO: do something with hard limit instead? (but that is hitting redis too much)
|
||||||
let active_requests = rpc.active_requests() as f64;
|
let active_requests = rpc.active_requests() as f64;
|
||||||
let soft_limit = rpc.soft_limit as f64 * rpc.weight;
|
let soft_limit = rpc.soft_limit as f64;
|
||||||
|
|
||||||
// TODO: maybe store weight as the percentile
|
|
||||||
let available_requests = soft_limit - active_requests;
|
let available_requests = soft_limit - active_requests;
|
||||||
|
|
||||||
trace!("available requests on {}: {}", rpc, available_requests);
|
trace!("available requests on {}: {}", rpc, available_requests);
|
||||||
|
|
||||||
// under heavy load, it is possible for even our best server to be negative
|
|
||||||
minimum = available_requests.min(minimum);
|
minimum = available_requests.min(minimum);
|
||||||
|
|
||||||
// TODO: clone needed?
|
|
||||||
(rpc, available_requests)
|
(rpc, available_requests)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
trace!("minimum available requests: {}", minimum);
|
trace!("minimum available requests: {}", minimum);
|
||||||
|
|
||||||
// weights can't have negative numbers. shift up if any are negative
|
// choose_multiple_weighted can't have negative numbers. shift up if any are negative
|
||||||
|
// TODO: is this a correct way to shift?
|
||||||
if minimum < 0.0 {
|
if minimum < 0.0 {
|
||||||
available_request_map = available_request_map
|
available_request_map = available_request_map
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
@ -943,7 +947,7 @@ mod tests {
|
||||||
soft_limit: 1_000,
|
soft_limit: 1_000,
|
||||||
automatic_block_limit: true,
|
automatic_block_limit: true,
|
||||||
block_data_limit: block_data_limit.into(),
|
block_data_limit: block_data_limit.into(),
|
||||||
weight: 100.0,
|
tier: 0,
|
||||||
head_block: RwLock::new(Some(head_block.clone())),
|
head_block: RwLock::new(Some(head_block.clone())),
|
||||||
open_request_handle_metrics: Arc::new(Default::default()),
|
open_request_handle_metrics: Arc::new(Default::default()),
|
||||||
};
|
};
|
||||||
|
@ -962,7 +966,7 @@ mod tests {
|
||||||
soft_limit: 1_000,
|
soft_limit: 1_000,
|
||||||
automatic_block_limit: false,
|
automatic_block_limit: false,
|
||||||
block_data_limit: block_data_limit.into(),
|
block_data_limit: block_data_limit.into(),
|
||||||
weight: 100.0,
|
tier: 0,
|
||||||
head_block: RwLock::new(Some(lagged_block.clone())),
|
head_block: RwLock::new(Some(lagged_block.clone())),
|
||||||
open_request_handle_metrics: Arc::new(Default::default()),
|
open_request_handle_metrics: Arc::new(Default::default()),
|
||||||
};
|
};
|
||||||
|
@ -1165,7 +1169,7 @@ mod tests {
|
||||||
soft_limit: 3_000,
|
soft_limit: 3_000,
|
||||||
automatic_block_limit: false,
|
automatic_block_limit: false,
|
||||||
block_data_limit: 64.into(),
|
block_data_limit: 64.into(),
|
||||||
weight: 1.0,
|
tier: 1,
|
||||||
head_block: RwLock::new(Some(head_block.clone())),
|
head_block: RwLock::new(Some(head_block.clone())),
|
||||||
open_request_handle_metrics: Arc::new(Default::default()),
|
open_request_handle_metrics: Arc::new(Default::default()),
|
||||||
};
|
};
|
||||||
|
@ -1184,8 +1188,7 @@ mod tests {
|
||||||
soft_limit: 1_000,
|
soft_limit: 1_000,
|
||||||
automatic_block_limit: false,
|
automatic_block_limit: false,
|
||||||
block_data_limit: u64::MAX.into(),
|
block_data_limit: u64::MAX.into(),
|
||||||
// TODO: does weight = 0 work?
|
tier: 2,
|
||||||
weight: 0.01,
|
|
||||||
head_block: RwLock::new(Some(head_block.clone())),
|
head_block: RwLock::new(Some(head_block.clone())),
|
||||||
open_request_handle_metrics: Arc::new(Default::default()),
|
open_request_handle_metrics: Arc::new(Default::default()),
|
||||||
};
|
};
|
||||||
|
|
Loading…
Reference in New Issue