derivative and serde didn't work together like we'd hoped

This commit is contained in:
Bryan Stitt 2023-07-06 03:51:39 -07:00
parent 0e3fdfa0ea
commit e0c57a22ae
6 changed files with 63 additions and 40 deletions

12
Cargo.lock generated

@ -5462,6 +5462,17 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-inline-default"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa824cde50b5f01ff28a955114d8152a07cd62d81f53459dad0f2610136be844"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "serde_derive"
version = "1.0.166"
@ -7130,6 +7141,7 @@ dependencies = [
"sentry",
"sentry-tracing",
"serde",
"serde-inline-default",
"serde_json",
"serde_prometheus",
"siwe",

@ -48,7 +48,6 @@ check-if-email-exists = "0.9.0"
chrono = { version = "0.4.26" }
console-subscriber = { version = "0.1.10", features = ["env-filter", "parking_lot"], optional = true }
counter = "0.5.7"
derivative = "2.2.0"
derive_more = { version = "0.99.17", features = ["nightly"] }
ethbloom = { version = "0.13.0" }
ethers = { version = "2.0.7", default-features = false, features = ["rustls", "ws"] }
@ -105,6 +104,8 @@ uuid = { version = "1.4.0", default-features = false, features = ["fast-rng", "v
# TODO: why doesn't this work in dev-dependencies
test-log = { version = "0.2.12", default-features = false, features = ["trace"] }
serde-inline-default = "0.1.1"
derivative = "2.2.0"
[dev-dependencies]
env_logger = "0.10"

@ -109,6 +109,8 @@ impl ComputeUnit {
(_, "web3_clientVersion") => 15,
(_, "web3_sha3") => 15,
(_, method) => {
// TODO: emit a stat
// TODO: do some filtering on these to tarpit peers that are trying to do silly things like sql/influx inject
warn!("unknown method {}", method);
return Self::unimplemented();
}

@ -2,13 +2,13 @@ use crate::app::Web3ProxyJoinHandle;
use crate::rpcs::blockchain::{BlocksByHashCache, Web3ProxyBlock};
use crate::rpcs::one::Web3Rpc;
use argh::FromArgs;
use derivative::Derivative;
use ethers::prelude::{Address, TxHash};
use ethers::types::{U256, U64};
use hashbrown::HashMap;
use migration::sea_orm::DatabaseConnection;
use sentry::types::Dsn;
use serde::Deserialize;
use serde_inline_default::serde_inline_default;
use std::sync::Arc;
use std::time::Duration;
use tracing::warn;
@ -53,17 +53,16 @@ pub struct TopConfig {
/// shared configuration between Web3Rpcs
// TODO: no String, only &str
#[derive(Clone, Debug, Derivative, Deserialize, PartialEq, Eq)]
#[derivative(Default)]
#[serde_inline_default]
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
pub struct AppConfig {
/// Request limit for allowed origins for anonymous users.
/// These requests get rate limited by IP.
#[serde(default)]
#[serde(default = "Default::default")]
pub allowed_origin_requests_per_period: HashMap<String, u64>,
/// erigon defaults to pruning beyond 90,000 blocks
#[serde(default)]
#[derivative(Default(value = "90_000"))]
#[serde_inline_default(90_000u64)]
pub archive_depth: u64,
/// EVM chain id. 1 for ETH
@ -115,8 +114,7 @@ pub struct AppConfig {
/// Used by /debug/:rpc_key urls for logging requests and responses. No other endpoints log request/response data.
pub kafka_urls: Option<String>,
#[serde(default)]
#[derivative(Default(value = r#""ssl".to_string()"#))]
#[serde_inline_default("ssl".to_string())]
pub kafka_protocol: String,
/// domain in sign-in-with-ethereum messages
@ -127,18 +125,15 @@ pub struct AppConfig {
/// Rate limit for the login entrypoint.
/// This is separate from the rpc limits.
#[serde(default)]
#[derivative(Default(value = "10"))]
#[serde_inline_default(10u64)]
pub login_rate_limit_per_period: u64,
/// The soft limit prevents thundering herds as new blocks are seen.
#[serde(default)]
#[derivative(Default(value = "1"))]
#[serde_inline_default(1u32)]
pub min_sum_soft_limit: u32,
/// Another knob for preventing thundering herds as new blocks are seen.
#[serde(default)]
#[derivative(Default(value = "1"))]
#[serde_inline_default(1usize)]
pub min_synced_rpcs: usize,
/// Concurrent request limit for anonymous users.
@ -155,8 +150,7 @@ pub struct AppConfig {
pub public_recent_ips_salt: Option<String>,
/// RPC responses are cached locally
#[serde(default)]
#[derivative(Default(value = "10u64.pow(8)"))]
#[serde_inline_default(10u64.pow(8))]
pub response_cache_max_bytes: u64,
/// the stats page url for an anonymous user.
@ -224,11 +218,11 @@ pub fn average_block_interval(chain_id: u64) -> Duration {
}
/// Configuration for a backend web3 RPC server
#[derive(Clone, Debug, Derivative, Deserialize, PartialEq, Eq)]
#[derivative(Default)]
#[serde_inline_default]
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
pub struct Web3RpcConfig {
/// simple way to disable a connection without deleting the row
#[serde(default)]
#[serde(default = "Default::default")]
pub disabled: bool,
/// a name used in /status and other user facing messages
pub display_name: Option<String>,
@ -239,17 +233,16 @@ pub struct Web3RpcConfig {
/// block data limit. If None, will be queried
pub block_data_limit: Option<u64>,
/// the requests per second at which the server starts slowing down
#[serde(default)]
#[derivative(Default(value = "1"))]
#[serde_inline_default(1u32)]
pub soft_limit: u32,
/// the requests per second at which the server throws errors (rate limit or otherwise)
pub hard_limit: Option<u64>,
/// only use this rpc if everything else is lagging too far. this allows us to ignore fast but very low limit rpcs
#[serde(default)]
#[serde(default = "Default::default")]
pub backup: bool,
/// Subscribe to the firehose of pending transactions
/// Don't do this with free rpcs
#[serde(default)]
#[serde(default = "Default::default")]
pub subscribe_txs: bool,
/// unknown config options get put here
#[serde(flatten, default = "HashMap::default")]
@ -291,3 +284,15 @@ impl Web3RpcConfig {
.await
}
}
#[cfg(test)]
mod tests {
use super::Web3RpcConfig;
#[test]
fn expected_defaults() {
let a: Web3RpcConfig = serde_json::from_str("{}").unwrap();
assert_eq!(a.soft_limit, 1);
}
}

@ -275,9 +275,9 @@ impl OpenRequestHandle {
} else if msg.contains("limit") || msg.contains("request") {
// TODO: too verbose
if self.rpc.backup {
trace!("rate limit from {}", self.rpc);
trace!(%msg, "rate limit from {}", self.rpc);
} else {
warn!("rate limit from {}", self.rpc);
warn!(%msg, "rate limit from {}", self.rpc);
}
ResponseTypes::RateLimit
} else {

@ -10,6 +10,7 @@ use ethers::{
use hashbrown::HashMap;
use migration::sea_orm::DatabaseConnection;
use parking_lot::Mutex;
use serde_json::json;
use std::{
env,
process::Command as SyncCommand,
@ -242,21 +243,23 @@ impl TestApp {
// make a test TopConfig
// TODO: test influx
// TODO: test redis
let app_config: AppConfig = serde_json::from_value(json!({
"chain_id": 31337,
"db_url": db_url,
"default_user_max_requests_per_period": Some(6_000_000),
"deposit_factory_contract": Address::from_str(
"4e3BC2054788De923A04936C6ADdB99A05B0Ea36",
)
.ok(),
"min_sum_soft_limit": 1,
"min_synced_rpcs": 1,
"public_requests_per_period": Some(1_000_000),
"response_cache_max_bytes": 10_u64.pow(7),
}))
.unwrap();
let top_config = TopConfig {
app: AppConfig {
chain_id: 31337,
db_url,
default_user_max_requests_per_period: Some(6_000_000),
deposit_factory_contract: Address::from_str(
"4e3BC2054788De923A04936C6ADdB99A05B0Ea36",
)
.ok(),
min_sum_soft_limit: 1,
min_synced_rpcs: 1,
public_requests_per_period: Some(1_000_000),
response_cache_max_bytes: 10_u64.pow(7),
..Default::default()
},
app: app_config,
balanced_rpcs: HashMap::from([(
"anvil".to_string(),
Web3RpcConfig {