From e0c57a22ae4c45fc43d81d67cc43b09bcc463239 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Thu, 6 Jul 2023 03:51:39 -0700 Subject: [PATCH] derivative and serde didn't work together like we'd hoped --- Cargo.lock | 12 ++++++++ web3_proxy/Cargo.toml | 3 +- web3_proxy/src/compute_units.rs | 2 ++ web3_proxy/src/config.rs | 51 ++++++++++++++++++--------------- web3_proxy/src/rpcs/request.rs | 4 +-- web3_proxy/tests/common/app.rs | 31 +++++++++++--------- 6 files changed, 63 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 200bc0a8..b7862c09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5462,6 +5462,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-inline-default" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa824cde50b5f01ff28a955114d8152a07cd62d81f53459dad0f2610136be844" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_derive" version = "1.0.166" @@ -7130,6 +7141,7 @@ dependencies = [ "sentry", "sentry-tracing", "serde", + "serde-inline-default", "serde_json", "serde_prometheus", "siwe", diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml index 4c7b7bfc..e40ca55c 100644 --- a/web3_proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -48,7 +48,6 @@ check-if-email-exists = "0.9.0" chrono = { version = "0.4.26" } console-subscriber = { version = "0.1.10", features = ["env-filter", "parking_lot"], optional = true } counter = "0.5.7" -derivative = "2.2.0" derive_more = { version = "0.99.17", features = ["nightly"] } ethbloom = { version = "0.13.0" } ethers = { version = "2.0.7", default-features = false, features = ["rustls", "ws"] } @@ -105,6 +104,8 @@ uuid = { version = "1.4.0", default-features = false, features = ["fast-rng", "v # TODO: why doesn't this work in dev-dependencies test-log = { version = "0.2.12", default-features = false, features = ["trace"] } +serde-inline-default = "0.1.1" +derivative = "2.2.0" [dev-dependencies] env_logger = "0.10" diff --git a/web3_proxy/src/compute_units.rs b/web3_proxy/src/compute_units.rs index a82a8ee4..0bd7b053 100644 --- a/web3_proxy/src/compute_units.rs +++ b/web3_proxy/src/compute_units.rs @@ -109,6 +109,8 @@ impl ComputeUnit { (_, "web3_clientVersion") => 15, (_, "web3_sha3") => 15, (_, method) => { + // TODO: emit a stat + // TODO: do some filtering on these to tarpit peers that are trying to do silly things like sql/influx inject warn!("unknown method {}", method); return Self::unimplemented(); } diff --git a/web3_proxy/src/config.rs b/web3_proxy/src/config.rs index d7bc1be4..5232852d 100644 --- a/web3_proxy/src/config.rs +++ b/web3_proxy/src/config.rs @@ -2,13 +2,13 @@ use crate::app::Web3ProxyJoinHandle; use crate::rpcs::blockchain::{BlocksByHashCache, Web3ProxyBlock}; use crate::rpcs::one::Web3Rpc; use argh::FromArgs; -use derivative::Derivative; use ethers::prelude::{Address, TxHash}; use ethers::types::{U256, U64}; use hashbrown::HashMap; use migration::sea_orm::DatabaseConnection; use sentry::types::Dsn; use serde::Deserialize; +use serde_inline_default::serde_inline_default; use std::sync::Arc; use std::time::Duration; use tracing::warn; @@ -53,17 +53,16 @@ pub struct TopConfig { /// shared configuration between Web3Rpcs // TODO: no String, only &str -#[derive(Clone, Debug, Derivative, Deserialize, PartialEq, Eq)] -#[derivative(Default)] +#[serde_inline_default] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] pub struct AppConfig { /// Request limit for allowed origins for anonymous users. /// These requests get rate limited by IP. - #[serde(default)] + #[serde(default = "Default::default")] pub allowed_origin_requests_per_period: HashMap, /// erigon defaults to pruning beyond 90,000 blocks - #[serde(default)] - #[derivative(Default(value = "90_000"))] + #[serde_inline_default(90_000u64)] pub archive_depth: u64, /// EVM chain id. 1 for ETH @@ -115,8 +114,7 @@ pub struct AppConfig { /// Used by /debug/:rpc_key urls for logging requests and responses. No other endpoints log request/response data. pub kafka_urls: Option, - #[serde(default)] - #[derivative(Default(value = r#""ssl".to_string()"#))] + #[serde_inline_default("ssl".to_string())] pub kafka_protocol: String, /// domain in sign-in-with-ethereum messages @@ -127,18 +125,15 @@ pub struct AppConfig { /// Rate limit for the login entrypoint. /// This is separate from the rpc limits. - #[serde(default)] - #[derivative(Default(value = "10"))] + #[serde_inline_default(10u64)] pub login_rate_limit_per_period: u64, /// The soft limit prevents thundering herds as new blocks are seen. - #[serde(default)] - #[derivative(Default(value = "1"))] + #[serde_inline_default(1u32)] pub min_sum_soft_limit: u32, /// Another knob for preventing thundering herds as new blocks are seen. - #[serde(default)] - #[derivative(Default(value = "1"))] + #[serde_inline_default(1usize)] pub min_synced_rpcs: usize, /// Concurrent request limit for anonymous users. @@ -155,8 +150,7 @@ pub struct AppConfig { pub public_recent_ips_salt: Option, /// RPC responses are cached locally - #[serde(default)] - #[derivative(Default(value = "10u64.pow(8)"))] + #[serde_inline_default(10u64.pow(8))] pub response_cache_max_bytes: u64, /// the stats page url for an anonymous user. @@ -224,11 +218,11 @@ pub fn average_block_interval(chain_id: u64) -> Duration { } /// Configuration for a backend web3 RPC server -#[derive(Clone, Debug, Derivative, Deserialize, PartialEq, Eq)] -#[derivative(Default)] +#[serde_inline_default] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] pub struct Web3RpcConfig { /// simple way to disable a connection without deleting the row - #[serde(default)] + #[serde(default = "Default::default")] pub disabled: bool, /// a name used in /status and other user facing messages pub display_name: Option, @@ -239,17 +233,16 @@ pub struct Web3RpcConfig { /// block data limit. If None, will be queried pub block_data_limit: Option, /// the requests per second at which the server starts slowing down - #[serde(default)] - #[derivative(Default(value = "1"))] + #[serde_inline_default(1u32)] pub soft_limit: u32, /// the requests per second at which the server throws errors (rate limit or otherwise) pub hard_limit: Option, /// only use this rpc if everything else is lagging too far. this allows us to ignore fast but very low limit rpcs - #[serde(default)] + #[serde(default = "Default::default")] pub backup: bool, /// Subscribe to the firehose of pending transactions /// Don't do this with free rpcs - #[serde(default)] + #[serde(default = "Default::default")] pub subscribe_txs: bool, /// unknown config options get put here #[serde(flatten, default = "HashMap::default")] @@ -291,3 +284,15 @@ impl Web3RpcConfig { .await } } + +#[cfg(test)] +mod tests { + use super::Web3RpcConfig; + + #[test] + fn expected_defaults() { + let a: Web3RpcConfig = serde_json::from_str("{}").unwrap(); + + assert_eq!(a.soft_limit, 1); + } +} diff --git a/web3_proxy/src/rpcs/request.rs b/web3_proxy/src/rpcs/request.rs index c45c8b9f..c38a138f 100644 --- a/web3_proxy/src/rpcs/request.rs +++ b/web3_proxy/src/rpcs/request.rs @@ -275,9 +275,9 @@ impl OpenRequestHandle { } else if msg.contains("limit") || msg.contains("request") { // TODO: too verbose if self.rpc.backup { - trace!("rate limit from {}", self.rpc); + trace!(%msg, "rate limit from {}", self.rpc); } else { - warn!("rate limit from {}", self.rpc); + warn!(%msg, "rate limit from {}", self.rpc); } ResponseTypes::RateLimit } else { diff --git a/web3_proxy/tests/common/app.rs b/web3_proxy/tests/common/app.rs index 06192ed6..e04188f9 100644 --- a/web3_proxy/tests/common/app.rs +++ b/web3_proxy/tests/common/app.rs @@ -10,6 +10,7 @@ use ethers::{ use hashbrown::HashMap; use migration::sea_orm::DatabaseConnection; use parking_lot::Mutex; +use serde_json::json; use std::{ env, process::Command as SyncCommand, @@ -242,21 +243,23 @@ impl TestApp { // make a test TopConfig // TODO: test influx // TODO: test redis + let app_config: AppConfig = serde_json::from_value(json!({ + "chain_id": 31337, + "db_url": db_url, + "default_user_max_requests_per_period": Some(6_000_000), + "deposit_factory_contract": Address::from_str( + "4e3BC2054788De923A04936C6ADdB99A05B0Ea36", + ) + .ok(), + "min_sum_soft_limit": 1, + "min_synced_rpcs": 1, + "public_requests_per_period": Some(1_000_000), + "response_cache_max_bytes": 10_u64.pow(7), + })) + .unwrap(); + let top_config = TopConfig { - app: AppConfig { - chain_id: 31337, - db_url, - default_user_max_requests_per_period: Some(6_000_000), - deposit_factory_contract: Address::from_str( - "4e3BC2054788De923A04936C6ADdB99A05B0Ea36", - ) - .ok(), - min_sum_soft_limit: 1, - min_synced_rpcs: 1, - public_requests_per_period: Some(1_000_000), - response_cache_max_bytes: 10_u64.pow(7), - ..Default::default() - }, + app: app_config, balanced_rpcs: HashMap::from([( "anvil".to_string(), Web3RpcConfig {