use crate::app::AnyhowJoinHandle; use crate::rpcs::blockchain::{BlockHashesCache, Web3ProxyBlock}; use crate::rpcs::one::Web3Rpc; use argh::FromArgs; use ethers::prelude::TxHash; use ethers::types::{U256, U64}; use hashbrown::HashMap; use log::warn; use migration::sea_orm::DatabaseConnection; use serde::Deserialize; use std::sync::Arc; use tokio::sync::broadcast; pub type BlockAndRpc = (Option, Arc); pub type TxHashAndRpc = (TxHash, Arc); #[derive(Debug, FromArgs)] /// Web3_proxy is a fast caching and load balancing proxy for web3 (Ethereum or similar) JsonRPC servers. pub struct CliConfig { /// path to a toml of rpc servers #[argh(option, default = "\"./config/development.toml\".to_string()")] pub config: String, /// what port the proxy should listen on #[argh(option, default = "8544")] pub port: u16, /// what port the proxy should expose prometheus stats on #[argh(option, default = "8543")] pub prometheus_port: u16, /// number of worker threads. Defaults to the number of logical processors #[argh(option, default = "0")] pub workers: usize, /// path to a binary file used to encrypt cookies. Should be at least 64 bytes. #[argh(option, default = "\"./data/development_cookie_key\".to_string()")] pub cookie_key_filename: String, } #[derive(Clone, Debug, Deserialize)] pub struct TopConfig { pub app: AppConfig, pub balanced_rpcs: HashMap, // TODO: instead of an option, give it a default pub private_rpcs: Option>, /// unknown config options get put here #[serde(flatten, default = "HashMap::default")] pub extra: HashMap, } /// shared configuration between Web3Rpcs // TODO: no String, only &str #[derive(Clone, Debug, Default, Deserialize)] pub struct AppConfig { /// Request limit for allowed origins for anonymous users. /// These requests get rate limited by IP. #[serde(default = "default_allowed_origin_requests_per_period")] pub allowed_origin_requests_per_period: HashMap, /// erigon defaults to pruning beyond 90,000 blocks #[serde(default = "default_archive_depth")] pub archive_depth: u64, /// EVM chain id. 1 for ETH /// TODO: better type for chain_id? max of `u64::MAX / 2 - 36` pub chain_id: u64, /// Database is used for user data. /// Currently supports mysql or compatible backend. pub db_url: Option, /// minimum size of the connection pool for the database. /// If none, the number of workers are used. pub db_min_connections: Option, /// maximum size of the connection pool for the database. /// If none, the minimum * 2 is used. pub db_max_connections: Option, /// Read-only replica of db_url. pub db_replica_url: Option, /// minimum size of the connection pool for the database replica. /// If none, db_min_connections is used. pub db_replica_min_connections: Option, /// maximum size of the connection pool for the database replica. /// If none, db_max_connections is used. pub db_replica_max_connections: Option, /// Default request limit for registered users. /// 0 = block all requests /// None = allow all requests pub default_user_max_requests_per_period: Option, /// minimum amount to increase eth_estimateGas results pub gas_increase_min: Option, /// percentage to increase eth_estimateGas results. 100 == 100% pub gas_increase_percent: Option, /// Restrict user registration. /// None = no code needed pub invite_code: Option, pub login_domain: Option, /// do not serve any requests if the best known block is older than this many seconds. pub max_block_age: Option, /// do not serve any requests if the best known block is behind the best known block by more than this many blocks. pub max_block_lag: Option, /// Rate limit for bearer token authenticated entrypoints. /// This is separate from the rpc limits. #[serde(default = "default_bearer_token_max_concurrent_requests")] pub bearer_token_max_concurrent_requests: u64, /// Rate limit for the login entrypoint. /// This is separate from the rpc limits. #[serde(default = "default_login_rate_limit_per_period")] pub login_rate_limit_per_period: u64, /// The soft limit prevents thundering herds as new blocks are seen. #[serde(default = "default_min_sum_soft_limit")] pub min_sum_soft_limit: u32, /// Another knob for preventing thundering herds as new blocks are seen. #[serde(default = "default_min_synced_rpcs")] pub min_synced_rpcs: usize, /// Concurrent request limit for anonymous users. /// Some(0) = block all requests /// None = allow all requests pub public_max_concurrent_requests: Option, /// Request limit for anonymous users. /// Some(0) = block all requests /// None = allow all requests pub public_requests_per_period: Option, /// Salt for hashing recent ips pub public_recent_ips_salt: Option, /// RPC responses are cached locally #[serde(default = "default_response_cache_max_bytes")] pub response_cache_max_bytes: u64, /// the stats page url for an anonymous user. pub redirect_public_url: Option, /// the stats page url for a logged in user. if set, must contain "{rpc_key_id}" pub redirect_rpc_key_url: Option, /// Optionally send errors to pub sentry_url: Option, /// Track rate limits in a redis (or compatible backend) /// It is okay if this data is lost. pub volatile_redis_url: Option, /// maximum size of the connection pool for the cache /// If none, the minimum * 2 is used pub volatile_redis_max_connections: Option, /// unknown config options get put here #[serde(flatten, default = "HashMap::default")] pub extra: HashMap, } fn default_archive_depth() -> u64 { 90_000 } fn default_allowed_origin_requests_per_period() -> HashMap { HashMap::new() } /// This might cause a thundering herd! fn default_min_sum_soft_limit() -> u32 { 1 } /// Only require 1 server. This might cause a thundering herd! fn default_min_synced_rpcs() -> usize { 1 } /// Having a low amount of concurrent requests for bearer tokens keeps us from hammering the database. fn default_bearer_token_max_concurrent_requests() -> u64 { 2 } /// Having a low amount of requests per period (usually minute) for login is safest. fn default_login_rate_limit_per_period() -> u64 { 10 } fn default_response_cache_max_bytes() -> u64 { // TODO: default to some percentage of the system? // 100 megabytes 10u64.pow(8) } /// Configuration for a backend web3 RPC server #[derive(Clone, Debug, Default, Deserialize)] pub struct Web3RpcConfig { /// simple way to disable a connection without deleting the row #[serde(default)] pub disabled: bool, /// a name used in /status and other user facing messages pub display_name: Option, /// (deprecated) rpc url pub url: Option, /// while not absolutely required, a ws:// or wss:// connection will be able to subscribe to head blocks pub ws_url: Option, /// while not absolutely required, a http:// or https:// connection will allow erigon to stream JSON pub http_url: Option, /// block data limit. If None, will be queried pub block_data_limit: Option, /// the requests per second at which the server starts slowing down pub soft_limit: u32, /// the requests per second at which the server throws errors (rate limit or otherwise) pub hard_limit: Option, /// only use this rpc if everything else is lagging too far. this allows us to ignore fast but very low limit rpcs #[serde(default)] pub backup: bool, /// All else equal, a server with a lower tier receives all requests #[serde(default = "default_tier")] pub tier: u64, /// Subscribe to the firehose of pending transactions /// Don't do this with free rpcs #[serde(default)] pub subscribe_txs: bool, /// unknown config options get put here #[serde(flatten, default = "HashMap::default")] pub extra: HashMap, } fn default_tier() -> u64 { 0 } impl Web3RpcConfig { /// Create a Web3Rpc from config /// TODO: move this into Web3Rpc? (just need to make things pub(crate)) #[allow(clippy::too_many_arguments)] pub async fn spawn( self, name: String, db_conn: Option, redis_pool: Option, chain_id: u64, http_client: Option, http_interval_sender: Option>>, block_map: BlockHashesCache, block_sender: Option>, tx_id_sender: Option>, reconnect: bool, ) -> anyhow::Result<(Arc, AnyhowJoinHandle<()>)> { if !self.extra.is_empty() { warn!("unknown Web3RpcConfig fields!: {:?}", self.extra.keys()); } Web3Rpc::spawn( self, name, chain_id, db_conn, http_client, http_interval_sender, redis_pool, block_map, block_sender, tx_id_sender, reconnect, ) .await } }