From a3f2eff51255acfaac11615bc5a193572ca5a5ad Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Thu, 2 Feb 2023 13:44:57 -0800 Subject: [PATCH] simple request counts --- Cargo.lock | 59 +++++++- web3_proxy/Cargo.toml | 1 + web3_proxy/src/bin/web3_proxy_cli/main.rs | 3 + .../bin/web3_proxy_cli/popularity_contest.rs | 135 ++++++++++++++++++ 4 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 web3_proxy/src/bin/web3_proxy_cli/popularity_contest.rs diff --git a/Cargo.lock b/Cargo.lock index e1e9641c..8ca5d318 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -649,6 +649,18 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.12.0" @@ -960,7 +972,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45" dependencies = [ - "encode_unicode", + "encode_unicode 0.3.6", "lazy_static", "libc", "regex", @@ -975,7 +987,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ - "encode_unicode", + "encode_unicode 0.3.6", "lazy_static", "libc", "windows-sys", @@ -1153,6 +1165,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -1452,6 +1486,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -3590,6 +3630,20 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prettytable" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46480520d1b77c9a3482d39939fcf96831537a250ec62d4fd8fbdf8e0302e781" +dependencies = [ + "csv", + "encode_unicode 1.0.0", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "primitive-types" version = "0.12.1" @@ -5881,6 +5935,7 @@ dependencies = [ "num-traits", "pagerduty-rs", "parking_lot 0.12.1", + "prettytable", "proctitle", "redis-rate-limiter", "regex", diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml index 54c341a7..a4c384dc 100644 --- a/web3_proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -55,6 +55,7 @@ num = "0.4.0" num-traits = "0.2.15" pagerduty-rs = { version = "0.1.6", default-features = false, features = ["async", "rustls", "sync"] } parking_lot = { version = "0.12.1", features = ["arc_lock"] } +prettytable = "*" proctitle = "0.1.1" regex = "1.7.1" reqwest = { version = "0.11.14", default-features = false, features = ["json", "tokio-rustls"] } diff --git a/web3_proxy/src/bin/web3_proxy_cli/main.rs b/web3_proxy/src/bin/web3_proxy_cli/main.rs index 0b8dbe3a..fba010b2 100644 --- a/web3_proxy/src/bin/web3_proxy_cli/main.rs +++ b/web3_proxy/src/bin/web3_proxy_cli/main.rs @@ -10,6 +10,7 @@ mod daemon; mod drop_migration_lock; mod list_user_tier; mod pagerduty; +mod popularity_contest; mod rpc_accounting; mod sentryd; mod transfer_key; @@ -78,6 +79,7 @@ enum SubCommand { CreateUser(create_user::CreateUserSubCommand), DropMigrationLock(drop_migration_lock::DropMigrationLockSubCommand), Pagerduty(pagerduty::PagerdutySubCommand), + PopularityContest(popularity_contest::PopularityContestSubCommand), Proxyd(daemon::ProxydSubCommand), RpcAccounting(rpc_accounting::RpcAccountingSubCommand), Sentryd(sentryd::SentrydSubCommand), @@ -361,6 +363,7 @@ fn main() -> anyhow::Result<()> { x.main(pagerduty_async, top_config).await } + SubCommand::PopularityContest(x) => x.main().await, SubCommand::Sentryd(x) => { if cli_config.sentry_url.is_none() { warn!("sentry_url is not set! Logs will only show in this console"); diff --git a/web3_proxy/src/bin/web3_proxy_cli/popularity_contest.rs b/web3_proxy/src/bin/web3_proxy_cli/popularity_contest.rs new file mode 100644 index 00000000..b8b0e565 --- /dev/null +++ b/web3_proxy/src/bin/web3_proxy_cli/popularity_contest.rs @@ -0,0 +1,135 @@ +use std::collections::BTreeMap; + +// show what nodes are used most often +use argh::FromArgs; +use log::info; +use prettytable::{row, Table}; + +#[derive(FromArgs, PartialEq, Debug)] +/// Second subcommand. +#[argh(subcommand, name = "popularity_contest")] +pub struct PopularityContestSubCommand { + #[argh(positional)] + /// the web3-proxy url + /// TODO: query multiple and add them together + rpc: String, +} + +#[derive(Debug)] +struct BackendRpcData<'a> { + name: &'a str, + tier: u64, + backup: bool, + block_data_limit: u64, + requests: u64, +} + +impl PopularityContestSubCommand { + pub async fn main(self) -> anyhow::Result<()> { + let x: serde_json::Value = reqwest::get(format!("{}/status", self.rpc)) + .await? + .json() + .await?; + + let conns = x + .as_object() + .unwrap() + .get("balanced_rpcs") + .unwrap() + .as_object() + .unwrap() + .get("conns") + .unwrap() + .as_array() + .unwrap(); + + let mut by_tier = BTreeMap::>::new(); + let mut tier_requests = BTreeMap::::new(); + let mut total_requests = 0; + + for conn in conns { + let conn = conn.as_object().unwrap(); + + let name = conn + .get("display_name") + .unwrap_or_else(|| conn.get("name").unwrap()) + .as_str() + .unwrap(); + + if name.ends_with("http") { + continue; + } + + let tier = conn.get("tier").unwrap().as_u64().unwrap(); + + let backup = conn.get("backup").unwrap().as_bool().unwrap(); + + let block_data_limit = conn + .get("block_data_limit") + .unwrap() + .as_u64() + .unwrap_or(u64::MAX); + + let requests = conn.get("total_requests").unwrap().as_u64().unwrap(); + + let rpc_data = BackendRpcData { + name, + tier, + backup, + block_data_limit, + requests, + }; + + total_requests += rpc_data.requests; + + *tier_requests.entry(tier).or_default() += rpc_data.requests; + + by_tier.entry(tier).or_default().push(rpc_data); + } + + info!("tier_requests: {:#?}", tier_requests); + info!("by_tier: {:#?}", by_tier); + + let mut table = Table::new(); + + table.add_row(row![ + "name", + "tier", + "rpc_requests", + "tier_request_pct", + "total_pct" + ]); + + let total_requests = total_requests as f32; + + for (tier, rpcs) in by_tier.iter() { + let t = (*tier_requests.get(tier).unwrap()) as f32; + + for rpc in rpcs.iter() { + let tier_request_pct = if t == 0.0 { + 0.0 + } else { + (rpc.requests as f32) / t * 100.0 + }; + + let total_request_pct = if total_requests == 0.0 { + 0.0 + } else { + (rpc.requests as f32) / total_requests * 100.0 + }; + + table.add_row(row![ + rpc.name, + tier, + rpc.requests, + tier_request_pct, + total_request_pct + ]); + } + } + + table.printstd(); + + Ok(()) + } +}