web3-proxy/web3_proxy/src/bin/web3_proxy.rs

407 lines
14 KiB
Rust
Raw Normal View History

2022-08-06 08:49:52 +03:00
//! Web3_proxy is a fast caching and load balancing proxy for web3 (Ethereum or similar) JsonRPC servers.
//!
//! Signed transactions (eth_sendRawTransaction) are sent in parallel to the configured private RPCs (eden, ethermine, flashbots, etc.).
//!
//! All other requests are sent to an RPC server on the latest block (alchemy, moralis, rivet, your own node, or one of many other providers).
//! If multiple servers are in sync, the fastest server is prioritized. Since the fastest server is most likely to serve requests, slow servers are unlikely to ever get any requests.
2022-05-29 17:50:08 +03:00
//#![warn(missing_docs)]
2022-07-25 21:21:58 +03:00
#![forbid(unsafe_code)]
2022-11-21 01:38:01 +03:00
use anyhow::Context;
2022-10-21 01:51:56 +03:00
use futures::StreamExt;
2022-12-03 08:31:03 +03:00
use log::{debug, error, info, warn};
use num::Zero;
2022-05-05 22:07:09 +03:00
use std::fs;
2022-11-21 01:38:01 +03:00
use std::path::Path;
2022-05-12 21:49:57 +03:00
use std::sync::atomic::{self, AtomicUsize};
use tokio::runtime;
2022-10-21 01:51:56 +03:00
use tokio::sync::broadcast;
use web3_proxy::app::{flatten_handle, flatten_handles, Web3ProxyApp};
2022-08-12 22:07:14 +03:00
use web3_proxy::config::{CliConfig, TopConfig};
2022-09-20 09:56:24 +03:00
use web3_proxy::{frontend, metrics_frontend};
2022-06-16 05:53:37 +03:00
2023-01-15 23:12:52 +03:00
#[cfg(feature = "deadlock")]
use parking_lot::deadlock;
#[cfg(feature = "deadlock")]
use std::thread;
#[cfg(feature = "deadlock")]
use tokio::time::Duration;
2022-07-23 02:26:04 +03:00
fn run(
2022-10-21 01:51:56 +03:00
shutdown_sender: broadcast::Sender<()>,
2022-07-23 02:26:04 +03:00
cli_config: CliConfig,
2022-08-12 22:07:14 +03:00
top_config: TopConfig,
2022-07-23 02:26:04 +03:00
) -> anyhow::Result<()> {
2022-11-12 11:24:32 +03:00
debug!("{:?}", cli_config);
debug!("{:?}", top_config);
2022-05-05 22:07:09 +03:00
2022-11-04 22:52:46 +03:00
let mut shutdown_receiver = shutdown_sender.subscribe();
2023-01-15 23:12:52 +03:00
#[cfg(feature = "deadlock")]
{
// spawn a thread for deadlock detection
thread::spawn(move || loop {
thread::sleep(Duration::from_secs(10));
let deadlocks = deadlock::check_deadlock();
if deadlocks.is_empty() {
continue;
}
2022-05-16 08:16:32 +03:00
2023-01-15 23:12:52 +03:00
println!("{} deadlocks detected", deadlocks.len());
for (i, threads) in deadlocks.iter().enumerate() {
println!("Deadlock #{}", i);
for t in threads {
println!("Thread Id {:#?}", t.thread_id());
println!("{:#?}", t.backtrace());
}
2022-05-16 08:16:32 +03:00
}
2023-01-15 23:12:52 +03:00
});
}
2022-05-16 08:16:32 +03:00
2022-07-14 02:24:47 +03:00
// set up tokio's async runtime
let mut rt_builder = runtime::Builder::new_multi_thread();
2022-08-12 22:07:14 +03:00
let chain_id = top_config.app.chain_id;
2022-07-14 02:24:47 +03:00
rt_builder.enable_all().thread_name_fn(move || {
static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0);
// TODO: what ordering? i think we want seqcst so that these all happen in order, but that might be stricter than we really need
let worker_id = ATOMIC_ID.fetch_add(1, atomic::Ordering::SeqCst);
// TODO: i think these max at 15 characters
format!("web3-{}-{}", chain_id, worker_id)
});
if cli_config.workers > 0 {
rt_builder.worker_threads(cli_config.workers);
}
2022-07-08 21:27:06 +03:00
// start tokio's async runtime
let rt = rt_builder.build()?;
2022-07-09 02:02:32 +03:00
2022-09-14 09:18:13 +03:00
let num_workers = rt.metrics().num_workers();
2022-11-23 02:34:31 +03:00
info!("num_workers: {}", num_workers);
2022-07-09 02:02:32 +03:00
2022-05-12 21:49:57 +03:00
rt.block_on(async {
let app_frontend_port = cli_config.port;
let app_prometheus_port = cli_config.prometheus_port;
2022-12-03 08:31:03 +03:00
// start the main app
2022-10-31 23:05:58 +03:00
let mut spawned_app =
2022-10-21 01:51:56 +03:00
Web3ProxyApp::spawn(top_config, num_workers, shutdown_sender.subscribe()).await?;
2022-10-31 23:05:58 +03:00
let frontend_handle =
tokio::spawn(frontend::serve(app_frontend_port, spawned_app.app.clone()));
2023-01-15 23:54:08 +03:00
// TODO: should we put this in a dedicated thread?
let prometheus_handle = tokio::spawn(metrics_frontend::serve(
spawned_app.app.clone(),
app_prometheus_port,
));
2022-06-14 08:43:28 +03:00
2022-06-16 05:53:37 +03:00
// if everything is working, these should both run forever
tokio::select! {
2022-10-31 23:05:58 +03:00
x = flatten_handles(spawned_app.app_handles) => {
2022-07-08 21:27:06 +03:00
match x {
Ok(_) => info!("app_handle exited"),
Err(e) => {
return Err(e);
}
}
2022-06-14 08:43:28 +03:00
}
2022-06-16 05:53:37 +03:00
x = flatten_handle(frontend_handle) => {
2022-07-08 21:27:06 +03:00
match x {
Ok(_) => info!("frontend exited"),
Err(e) => {
return Err(e);
}
}
2022-06-14 08:43:28 +03:00
}
x = flatten_handle(prometheus_handle) => {
match x {
Ok(_) => info!("prometheus exited"),
Err(e) => {
return Err(e);
}
}
}
2022-10-21 01:51:56 +03:00
x = tokio::signal::ctrl_c() => {
match x {
Ok(_) => info!("quiting from ctrl-c"),
Err(e) => {
return Err(e.into());
}
}
2022-07-23 02:26:04 +03:00
}
2022-11-04 22:52:46 +03:00
x = shutdown_receiver.recv() => {
match x {
Ok(_) => info!("quiting from shutdown receiver"),
Err(e) => {
return Err(e.into());
}
}
}
2022-06-16 05:53:37 +03:00
};
2022-05-05 22:07:09 +03:00
2022-10-21 01:51:56 +03:00
// one of the handles stopped. send a value so the others know to shut down
if let Err(err) = shutdown_sender.send(()) {
2022-11-12 11:24:32 +03:00
warn!("shutdown sender err={:?}", err);
};
2022-10-21 01:51:56 +03:00
2022-12-03 08:31:03 +03:00
// wait for things like saving stats to the database to complete
info!("waiting on important background tasks");
let mut background_errors = 0;
2022-10-31 23:05:58 +03:00
while let Some(x) = spawned_app.background_handles.next().await {
2022-10-21 01:51:56 +03:00
match x {
2022-12-03 08:31:03 +03:00
Err(e) => {
error!("{:?}", e);
background_errors += 1;
}
Ok(Err(e)) => {
error!("{:?}", e);
background_errors += 1;
}
2022-10-21 01:51:56 +03:00
Ok(Ok(_)) => continue,
}
}
2022-10-10 07:15:07 +03:00
2022-12-03 08:31:03 +03:00
if background_errors.is_zero() {
info!("finished");
2023-01-15 23:54:08 +03:00
Ok(())
2022-12-03 08:31:03 +03:00
} else {
// TODO: collect instead?
2023-01-15 23:54:08 +03:00
Err(anyhow::anyhow!("finished with errors!"))
2022-12-03 08:31:03 +03:00
}
2022-05-18 19:35:06 +03:00
})
2022-05-05 22:07:09 +03:00
}
2022-07-23 02:26:04 +03:00
fn main() -> anyhow::Result<()> {
// if RUST_LOG isn't set, configure a default
2022-11-12 12:26:05 +03:00
let rust_log = match std::env::var("RUST_LOG") {
Ok(x) => x,
Err(_) => "info,ethers=debug,redis_rate_limit=debug,web3_proxy=debug".to_string(),
};
2022-07-23 02:26:04 +03:00
// this probably won't matter for us in docker, but better safe than sorry
fdlimit::raise_fd_limit();
// initial configuration from flags
let cli_config: CliConfig = argh::from_env();
2022-11-21 01:38:01 +03:00
// convert to absolute path so error logging is most helpful
let config_path = Path::new(&cli_config.config)
.canonicalize()
.context(format!(
"checking full path of {} and {}",
".", // TODO: get cwd somehow
cli_config.config
))?;
2022-07-23 02:26:04 +03:00
// advanced configuration is on disk
2022-11-21 01:38:01 +03:00
let top_config: String = fs::read_to_string(config_path.clone())
.context(format!("reading config at {}", config_path.display()))?;
let top_config: TopConfig = toml::from_str(&top_config)
.context(format!("parsing config at {}", config_path.display()))?;
2022-07-23 02:26:04 +03:00
// TODO: this doesn't seem to do anything
2022-08-12 22:07:14 +03:00
proctitle::set_title(format!("web3_proxy-{}", top_config.app.chain_id));
2022-07-23 02:26:04 +03:00
2022-11-12 12:26:05 +03:00
let logger = env_logger::builder().parse_filters(&rust_log).build();
2022-11-12 11:24:32 +03:00
2022-11-12 12:26:05 +03:00
let max_level = logger.filter();
2022-11-12 11:24:32 +03:00
2022-10-25 00:07:29 +03:00
// connect to sentry for error reporting
// if no sentry, only log to stdout
let _sentry_guard = if let Some(sentry_url) = top_config.app.sentry_url.clone() {
2022-11-12 12:26:05 +03:00
let logger = sentry::integrations::log::SentryLogger::with_dest(logger);
log::set_boxed_logger(Box::new(logger)).unwrap();
2022-10-25 00:07:29 +03:00
let guard = sentry::init((
sentry_url,
sentry::ClientOptions {
release: sentry::release_name!(),
// TODO: Set this a to lower value (from config) in production
traces_sample_rate: 1.0,
..Default::default()
},
));
Some(guard)
} else {
2022-11-12 11:24:32 +03:00
log::set_boxed_logger(Box::new(logger)).unwrap();
2022-10-25 00:07:29 +03:00
None
};
2022-11-12 12:26:05 +03:00
log::set_max_level(max_level);
2022-10-25 00:07:29 +03:00
// we used to do this earlier, but now we attach sentry
debug!("CLI config @ {:#?}", cli_config.config);
2022-08-12 22:07:14 +03:00
// tokio has code for catching ctrl+c so we use that
// this shutdown sender is currently only used in tests, but we might make a /shutdown endpoint or something
2022-11-21 01:52:08 +03:00
// we do not need this receiver. new receivers are made by `shutdown_sender.subscribe()`
2022-11-04 22:52:46 +03:00
let (shutdown_sender, _) = broadcast::channel(1);
2022-07-23 02:26:04 +03:00
2022-10-21 01:51:56 +03:00
run(shutdown_sender, cli_config, top_config)
2022-07-23 02:26:04 +03:00
}
#[cfg(test)]
mod tests {
use ethers::{
2022-11-21 01:52:08 +03:00
prelude::{Http, Provider, U256},
2022-07-23 02:26:04 +03:00
utils::Anvil,
};
use hashbrown::HashMap;
use std::env;
2023-01-15 23:12:52 +03:00
use std::thread;
2022-07-23 02:26:04 +03:00
2022-11-21 01:52:08 +03:00
use web3_proxy::{
config::{AppConfig, Web3ConnectionConfig},
rpcs::blockchain::ArcBlock,
};
2022-07-23 02:26:04 +03:00
use super::*;
#[tokio::test]
async fn it_works() {
// TODO: move basic setup into a test fixture
let path = env::var("PATH").unwrap();
println!("path: {}", path);
2022-07-23 03:36:07 +03:00
// TODO: how should we handle logs in this?
// TODO: option for super verbose logs
2022-07-23 02:26:04 +03:00
std::env::set_var("RUST_LOG", "info,web3_proxy=debug");
2022-11-12 11:24:32 +03:00
let _ = env_logger::builder().is_test(true).try_init();
2022-07-23 02:26:04 +03:00
let anvil = Anvil::new().spawn();
println!("Anvil running at `{}`", anvil.endpoint());
2022-07-23 03:36:07 +03:00
let anvil_provider = Provider::<Http>::try_from(anvil.endpoint()).unwrap();
2022-07-23 02:26:04 +03:00
// mine a block because my code doesn't like being on block 0
2022-11-21 01:55:42 +03:00
// TODO: make block 0 okay? is it okay now?
2022-07-23 03:36:07 +03:00
let _: U256 = anvil_provider
.request("evm_mine", None::<()>)
.await
.unwrap();
2022-07-23 02:26:04 +03:00
// make a test CliConfig
let cli_config = CliConfig {
port: 0,
prometheus_port: 0,
2022-07-23 03:19:13 +03:00
workers: 4,
2022-07-23 02:26:04 +03:00
config: "./does/not/exist/test.toml".to_string(),
cookie_key_filename: "./does/not/exist/development_cookie_key".to_string(),
2022-07-23 02:26:04 +03:00
};
2022-11-21 01:55:42 +03:00
// make a test TopConfig
2022-11-21 05:42:01 +03:00
// TODO: load TopConfig from a file? CliConfig could have `cli_config.load_top_config`. would need to inject our endpoint ports
2022-12-28 19:36:22 +03:00
let top_config = TopConfig {
2022-08-12 22:07:14 +03:00
app: AppConfig {
2022-07-23 02:26:04 +03:00
chain_id: 31337,
2022-11-01 22:12:57 +03:00
default_user_max_requests_per_period: Some(6_000_000),
2022-08-27 06:11:58 +03:00
min_sum_soft_limit: 1,
2022-08-27 03:33:45 +03:00
min_synced_rpcs: 1,
2022-11-01 22:12:57 +03:00
public_requests_per_period: Some(1_000_000),
2022-07-23 02:26:04 +03:00
response_cache_max_bytes: 10_usize.pow(7),
2022-10-18 00:47:58 +03:00
redirect_public_url: Some("example.com/".to_string()),
2022-11-30 00:29:17 +03:00
redirect_rpc_key_url: Some("example.com/{{rpc_key_id}}".to_string()),
2022-09-02 23:16:20 +03:00
..Default::default()
2022-07-23 02:26:04 +03:00
},
balanced_rpcs: HashMap::from([
(
"anvil".to_string(),
Web3ConnectionConfig {
disabled: false,
display_name: None,
url: anvil.endpoint(),
block_data_limit: None,
soft_limit: 100,
hard_limit: None,
2023-01-04 09:37:51 +03:00
tier: 0,
subscribe_txs: Some(false),
2022-12-28 19:36:22 +03:00
extra: Default::default(),
},
2022-07-23 02:26:04 +03:00
),
(
"anvil_ws".to_string(),
Web3ConnectionConfig {
disabled: false,
display_name: None,
url: anvil.ws_endpoint(),
block_data_limit: None,
soft_limit: 100,
hard_limit: None,
2023-01-04 09:37:51 +03:00
tier: 0,
subscribe_txs: Some(false),
2022-12-28 19:36:22 +03:00
extra: Default::default(),
},
2022-07-23 02:26:04 +03:00
),
]),
private_rpcs: None,
2022-12-28 19:49:21 +03:00
extra: Default::default(),
2022-07-23 02:26:04 +03:00
};
2022-11-04 22:52:46 +03:00
let (shutdown_sender, _) = broadcast::channel(1);
2022-07-23 02:26:04 +03:00
// spawn another thread for running the app
2022-07-23 03:36:07 +03:00
// TODO: allow launching into the local tokio runtime instead of creating a new one?
2022-10-21 01:51:56 +03:00
let handle = {
let shutdown_sender = shutdown_sender.clone();
2022-12-28 19:36:22 +03:00
thread::spawn(move || run(shutdown_sender, cli_config, top_config))
2022-10-21 01:51:56 +03:00
};
2022-07-23 02:26:04 +03:00
// TODO: do something to the node. query latest block, mine another block, query again
2022-07-23 03:19:13 +03:00
let proxy_provider = Provider::<Http>::try_from(anvil.endpoint()).unwrap();
2022-07-23 02:26:04 +03:00
2022-11-06 23:52:11 +03:00
let anvil_result = anvil_provider
2022-11-21 01:52:08 +03:00
.request::<_, Option<ArcBlock>>("eth_getBlockByNumber", ("latest", true))
2022-07-23 03:19:13 +03:00
.await
2022-11-06 23:52:11 +03:00
.unwrap()
2022-07-23 03:19:13 +03:00
.unwrap();
2022-11-06 23:52:11 +03:00
let proxy_result = proxy_provider
2022-11-21 01:52:08 +03:00
.request::<_, Option<ArcBlock>>("eth_getBlockByNumber", ("latest", true))
2022-07-23 03:19:13 +03:00
.await
2022-11-06 23:52:11 +03:00
.unwrap()
2022-07-23 03:19:13 +03:00
.unwrap();
assert_eq!(anvil_result, proxy_result);
let first_block_num = anvil_result.number.unwrap();
2022-07-23 03:36:07 +03:00
let _: U256 = anvil_provider
.request("evm_mine", None::<()>)
.await
.unwrap();
2022-07-23 03:19:13 +03:00
2022-11-06 23:52:11 +03:00
let anvil_result = anvil_provider
2022-11-21 01:52:08 +03:00
.request::<_, Option<ArcBlock>>("eth_getBlockByNumber", ("latest", true))
2022-07-23 03:19:13 +03:00
.await
2022-11-06 23:52:11 +03:00
.unwrap()
2022-07-23 03:19:13 +03:00
.unwrap();
2022-11-06 23:52:11 +03:00
let proxy_result = proxy_provider
2022-11-21 01:52:08 +03:00
.request::<_, Option<ArcBlock>>("eth_getBlockByNumber", ("latest", true))
2022-07-23 03:19:13 +03:00
.await
2022-11-06 23:52:11 +03:00
.unwrap()
2022-07-23 03:19:13 +03:00
.unwrap();
assert_eq!(anvil_result, proxy_result);
let second_block_num = anvil_result.number.unwrap();
2022-11-21 01:55:42 +03:00
assert_eq!(first_block_num, second_block_num - 1);
2022-07-23 03:19:13 +03:00
// tell the test app to shut down
2022-07-23 02:26:04 +03:00
shutdown_sender.send(()).unwrap();
println!("waiting for shutdown...");
2022-07-23 03:40:15 +03:00
// TODO: panic if a timeout is reached
2022-07-23 02:26:04 +03:00
handle.join().unwrap().unwrap();
}
}