fix archive node detection when nodes used to be archive but than changed flags
This commit is contained in:
Bryan Stitt
parent
43d550190b
commit
4dfb7a5451
175
Cargo.lock
generated
175
Cargo.lock
generated
@ -272,7 +272,7 @@ version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"hermit-abi 0.1.19",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
@ -1356,17 +1356,38 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.9.3"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
|
||||
checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"is-terminal",
|
||||
"log",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
|
||||
dependencies = [
|
||||
"errno-dragonfly",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno-dragonfly"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "error-chain"
|
||||
version = "0.12.4"
|
||||
@ -1737,7 +1758,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"windows-sys",
|
||||
"windows-sys 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2170,6 +2191,15 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
@ -2419,6 +2449,16 @@ dependencies = [
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "io-lifetimes"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7d367024b3f3414d8e01f437f704f41a9f64ab36f9067fa73e526ad4c763c87"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.5.1"
|
||||
@ -2435,6 +2475,18 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aae5bc6e2eb41c9def29a3e0f1306382807764b9b53112030eff57435667352d"
|
||||
dependencies = [
|
||||
"hermit-abi 0.2.6",
|
||||
"io-lifetimes",
|
||||
"rustix",
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
@ -2556,9 +2608,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.126"
|
||||
version = "0.2.137"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
|
||||
checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
@ -2566,6 +2618,12 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.8"
|
||||
@ -2698,7 +2756,7 @@ dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
"windows-sys",
|
||||
"windows-sys 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2885,7 +2943,7 @@ version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"hermit-abi 0.1.19",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@ -3074,7 +3132,7 @@ dependencies = [
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"thread-id",
|
||||
"windows-sys",
|
||||
"windows-sys 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3770,6 +3828,20 @@ dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.36.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b1fbb4dfc4eb1d390c02df47760bb19a84bb80b301ecc947ab5406394d8223e"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"io-lifetimes",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.20.6"
|
||||
@ -3884,9 +3956,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sea-orm"
|
||||
version = "0.10.3"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8744afc95ca462de12c2cea5a56d7e406f3be2b2683d3b05066e1afdba898bc5"
|
||||
checksum = "3120bc435b8640963ffda698f877610e07e077157e216eb99408d819c344034d"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
@ -3912,9 +3984,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sea-orm-cli"
|
||||
version = "0.10.3"
|
||||
version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "531871fd7d043270d6c26071d48bbdc2e04058e570adf7a8a75d9063c96771e9"
|
||||
checksum = "d037d297869f0960e56f49b6e8c427660f9cc9c3f867132f4d1014676525dbe7"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"clap",
|
||||
@ -3928,9 +4000,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sea-orm-macros"
|
||||
version = "0.10.3"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ca4d01381fdcabc3818b6d39c5f1f0c885900af90da638e4001406907462784"
|
||||
checksum = "c54bacfeb842813c16821e21f9456c358861a448294075184ea1d6307e386d08"
|
||||
dependencies = [
|
||||
"bae",
|
||||
"heck 0.3.3",
|
||||
@ -3941,9 +4013,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sea-orm-migration"
|
||||
version = "0.10.3"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e73a6b3823f2d02b09e16060a2d547c781557f756755b80f413679bdb2a149e"
|
||||
checksum = "8b0c22c789077e49c8ff6b1560e6b7a1e5f2a8f91aeebfb9b9e874fa1bc1aa29"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"clap",
|
||||
@ -5549,43 +5621,100 @@ version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
|
||||
dependencies = [
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_msvc",
|
||||
"windows_aarch64_msvc 0.36.1",
|
||||
"windows_i686_gnu 0.36.1",
|
||||
"windows_i686_msvc 0.36.1",
|
||||
"windows_x86_64_gnu 0.36.1",
|
||||
"windows_x86_64_msvc 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc 0.42.0",
|
||||
"windows_i686_gnu 0.42.0",
|
||||
"windows_i686_msvc 0.42.0",
|
||||
"windows_x86_64_gnu 0.42.0",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.10.1"
|
||||
|
||||
@ -10,6 +10,6 @@ path = "src/mod.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
sea-orm = "0.10.3"
|
||||
sea-orm = "0.10.4"
|
||||
serde = "1.0.147"
|
||||
uuid = "1.2.2"
|
||||
|
||||
@ -12,7 +12,7 @@ path = "src/lib.rs"
|
||||
tokio = { version = "1.22.0", features = ["full", "tracing"] }
|
||||
|
||||
[dependencies.sea-orm-migration]
|
||||
version = "0.10.3"
|
||||
version = "0.10.4"
|
||||
features = [
|
||||
# Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI.
|
||||
# View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime.
|
||||
|
||||
@ -31,7 +31,7 @@ counter = "0.5.7"
|
||||
derive_more = "0.99.17"
|
||||
dotenv = "0.15.0"
|
||||
ethers = { version = "1.0.1", default-features = false, features = ["rustls", "ws"] }
|
||||
env_logger = "0.9.3"
|
||||
env_logger = "0.10.0"
|
||||
fdlimit = "0.2.1"
|
||||
flume = "0.10.14"
|
||||
futures = { version = "0.3.25", features = ["thread-pool"] }
|
||||
|
||||
@ -294,27 +294,29 @@ mod tests {
|
||||
balanced_rpcs: HashMap::from([
|
||||
(
|
||||
"anvil".to_string(),
|
||||
Web3ConnectionConfig::new(
|
||||
false,
|
||||
None,
|
||||
anvil.endpoint(),
|
||||
100,
|
||||
None,
|
||||
1,
|
||||
Some(false),
|
||||
),
|
||||
Web3ConnectionConfig {
|
||||
disabled: false,
|
||||
display_name: None,
|
||||
url: anvil.endpoint(),
|
||||
block_data_limit: None,
|
||||
soft_limit: 100,
|
||||
hard_limit: None,
|
||||
weight: 1,
|
||||
subscribe_txs: Some(false),
|
||||
},
|
||||
),
|
||||
(
|
||||
"anvil_ws".to_string(),
|
||||
Web3ConnectionConfig::new(
|
||||
false,
|
||||
None,
|
||||
anvil.ws_endpoint(),
|
||||
100,
|
||||
None,
|
||||
0,
|
||||
Some(true),
|
||||
),
|
||||
Web3ConnectionConfig {
|
||||
disabled: false,
|
||||
display_name: None,
|
||||
url: anvil.ws_endpoint(),
|
||||
block_data_limit: None,
|
||||
soft_limit: 100,
|
||||
hard_limit: None,
|
||||
weight: 1,
|
||||
subscribe_txs: Some(false),
|
||||
},
|
||||
),
|
||||
]),
|
||||
private_rpcs: None,
|
||||
|
||||
@ -3,7 +3,6 @@ use crate::rpcs::connection::Web3Connection;
|
||||
use crate::rpcs::request::OpenRequestHandleMetrics;
|
||||
use crate::{app::AnyhowJoinHandle, rpcs::blockchain::ArcBlock};
|
||||
use argh::FromArgs;
|
||||
use derive_more::Constructor;
|
||||
use ethers::prelude::TxHash;
|
||||
use hashbrown::HashMap;
|
||||
use migration::sea_orm::DatabaseConnection;
|
||||
@ -176,7 +175,7 @@ fn default_response_cache_max_bytes() -> usize {
|
||||
}
|
||||
|
||||
/// Configuration for a backend web3 RPC server
|
||||
#[derive(Debug, Deserialize, Constructor)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Web3ConnectionConfig {
|
||||
/// simple way to disable a connection without deleting the row
|
||||
@ -186,6 +185,8 @@ pub struct Web3ConnectionConfig {
|
||||
pub display_name: Option<String>,
|
||||
/// websocket (or http if no websocket)
|
||||
pub url: String,
|
||||
/// block data limit. If None, will be queried
|
||||
pub block_data_limit: Option<u64>,
|
||||
/// the requests per second at which the server starts slowing down
|
||||
pub soft_limit: u32,
|
||||
/// the requests per second at which the server throws errors (rate limit or otherwise)
|
||||
@ -247,6 +248,7 @@ impl Web3ConnectionConfig {
|
||||
http_interval_sender,
|
||||
hard_limit,
|
||||
self.soft_limit,
|
||||
self.block_data_limit,
|
||||
block_map,
|
||||
block_sender,
|
||||
tx_id_sender,
|
||||
|
||||
@ -45,6 +45,12 @@ pub enum RateLimitResult {
|
||||
UnknownKey,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum AuthorizatioType {
|
||||
Internal,
|
||||
Frontend,
|
||||
}
|
||||
|
||||
/// TODO: include the authorization checks in this?
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Authorization {
|
||||
@ -54,6 +60,7 @@ pub struct Authorization {
|
||||
pub origin: Option<Origin>,
|
||||
pub referer: Option<Referer>,
|
||||
pub user_agent: Option<UserAgent>,
|
||||
pub authorization_type: AuthorizatioType,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@ -163,7 +170,7 @@ impl From<RpcSecretKey> for Uuid {
|
||||
}
|
||||
|
||||
impl Authorization {
|
||||
pub fn local(db_conn: Option<DatabaseConnection>) -> anyhow::Result<Self> {
|
||||
pub fn internal(db_conn: Option<DatabaseConnection>) -> anyhow::Result<Self> {
|
||||
let authorization_checks = AuthorizationChecks {
|
||||
// any error logs on a local (internal) query are likely problems. log them all
|
||||
log_revert_chance: 1.0,
|
||||
@ -174,10 +181,18 @@ impl Authorization {
|
||||
let ip: IpAddr = "127.0.0.1".parse().expect("localhost should always parse");
|
||||
let user_agent = UserAgent::from_str(APP_USER_AGENT).ok();
|
||||
|
||||
Self::try_new(authorization_checks, db_conn, ip, None, None, user_agent)
|
||||
Self::try_new(
|
||||
authorization_checks,
|
||||
db_conn,
|
||||
ip,
|
||||
None,
|
||||
None,
|
||||
user_agent,
|
||||
AuthorizatioType::Internal,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn public(
|
||||
pub fn external(
|
||||
allowed_origin_requests_per_period: &HashMap<String, u64>,
|
||||
db_conn: Option<DatabaseConnection>,
|
||||
ip: IpAddr,
|
||||
@ -208,6 +223,7 @@ impl Authorization {
|
||||
origin,
|
||||
referer,
|
||||
user_agent,
|
||||
AuthorizatioType::Frontend,
|
||||
)
|
||||
}
|
||||
|
||||
@ -218,6 +234,7 @@ impl Authorization {
|
||||
origin: Option<Origin>,
|
||||
referer: Option<Referer>,
|
||||
user_agent: Option<UserAgent>,
|
||||
authorization_type: AuthorizatioType,
|
||||
) -> anyhow::Result<Self> {
|
||||
// check ip
|
||||
match &authorization_checks.allowed_ips {
|
||||
@ -272,6 +289,7 @@ impl Authorization {
|
||||
origin,
|
||||
referer,
|
||||
user_agent,
|
||||
authorization_type,
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -444,7 +462,7 @@ impl Web3ProxyApp {
|
||||
// TODO: dry this up with rate_limit_by_rpc_key?
|
||||
|
||||
// we don't care about user agent or origin or referer
|
||||
let authorization = Authorization::public(
|
||||
let authorization = Authorization::external(
|
||||
&self.config.allowed_origin_requests_per_period,
|
||||
self.db_conn(),
|
||||
ip,
|
||||
@ -498,7 +516,7 @@ impl Web3ProxyApp {
|
||||
) -> anyhow::Result<RateLimitResult> {
|
||||
// ip rate limits don't check referer or user agent
|
||||
// the do check
|
||||
let authorization = Authorization::public(
|
||||
let authorization = Authorization::external(
|
||||
allowed_origin_requests_per_period,
|
||||
self.db_conn.clone(),
|
||||
ip,
|
||||
@ -687,6 +705,7 @@ impl Web3ProxyApp {
|
||||
origin,
|
||||
referer,
|
||||
user_agent,
|
||||
AuthorizatioType::Frontend,
|
||||
)?;
|
||||
|
||||
let user_max_requests_per_period = match authorization.checks.max_requests_per_period {
|
||||
|
||||
@ -9,7 +9,7 @@ use anyhow::Context;
|
||||
use ethers::prelude::{Bytes, Middleware, ProviderError, TxHash, H256, U64};
|
||||
use futures::future::try_join_all;
|
||||
use futures::StreamExt;
|
||||
use log::{debug, error, info, warn, Level};
|
||||
use log::{debug, error, info, trace, warn, Level};
|
||||
use migration::sea_orm::DatabaseConnection;
|
||||
use parking_lot::RwLock;
|
||||
use redis_rate_limiter::{RedisPool, RedisRateLimitResult, RedisRateLimiter};
|
||||
@ -37,10 +37,10 @@ pub struct Web3Connection {
|
||||
pub(super) http_client: Option<reqwest::Client>,
|
||||
/// keep track of currently open requests. We sort on this
|
||||
pub(super) active_requests: AtomicU32,
|
||||
/// keep track of total requests
|
||||
/// TODO: is this type okay?
|
||||
/// TODO: replace this with something in metered?
|
||||
pub(super) total_requests: AtomicU64,
|
||||
/// keep track of total requests from the frontend
|
||||
pub(super) frontend_requests: AtomicU64,
|
||||
/// keep track of total requests from web3-proxy itself
|
||||
pub(super) internal_requests: AtomicU64,
|
||||
/// provider is in a RwLock so that we can replace it if re-connecting
|
||||
/// it is an async lock because we hold it open across awaits
|
||||
pub(super) provider: AsyncRwLock<Option<Arc<Web3Provider>>>,
|
||||
@ -75,6 +75,7 @@ impl Web3Connection {
|
||||
hard_limit: Option<(u64, RedisPool)>,
|
||||
// TODO: think more about this type
|
||||
soft_limit: u32,
|
||||
block_data_limit: Option<u64>,
|
||||
block_map: BlockHashesCache,
|
||||
block_sender: Option<flume::Sender<BlockAndRpc>>,
|
||||
tx_id_sender: Option<flume::Sender<(TxHash, Arc<Self>)>>,
|
||||
@ -96,17 +97,20 @@ impl Web3Connection {
|
||||
// turn weight 0 into 100% and weight 100 into 0%
|
||||
let weight = (100 - weight) as f64 / 100.0;
|
||||
|
||||
let block_data_limit = block_data_limit.unwrap_or_default().into();
|
||||
|
||||
let new_connection = Self {
|
||||
name,
|
||||
display_name,
|
||||
http_client,
|
||||
url: url_str,
|
||||
active_requests: 0.into(),
|
||||
total_requests: 0.into(),
|
||||
frontend_requests: 0.into(),
|
||||
internal_requests: 0.into(),
|
||||
provider: AsyncRwLock::new(None),
|
||||
hard_limit,
|
||||
soft_limit,
|
||||
block_data_limit: Default::default(),
|
||||
block_data_limit,
|
||||
head_block_id: RwLock::new(Default::default()),
|
||||
weight,
|
||||
open_request_handle_metrics,
|
||||
@ -119,7 +123,7 @@ impl Web3Connection {
|
||||
.retrying_reconnect(block_sender.as_ref(), false)
|
||||
.await?;
|
||||
|
||||
let authorization = Arc::new(Authorization::local(db_conn)?);
|
||||
let authorization = Arc::new(Authorization::internal(db_conn)?);
|
||||
|
||||
// check the server's chain_id here
|
||||
// TODO: move this outside the `new` function and into a `start` function or something. that way we can do retries from there
|
||||
@ -153,7 +157,12 @@ impl Web3Connection {
|
||||
}
|
||||
}
|
||||
|
||||
let will_subscribe_to_blocks = block_sender.is_some();
|
||||
// TODO: should we do this even if block_sender is None? then we would know limits on private relays
|
||||
let check_block_limit_needed = (new_connection
|
||||
.block_data_limit
|
||||
.load(atomic::Ordering::Acquire)
|
||||
== 0)
|
||||
&& block_sender.is_some();
|
||||
|
||||
// subscribe to new blocks and new transactions
|
||||
// TODO: make transaction subscription optional (just pass None for tx_id_sender)
|
||||
@ -179,7 +188,7 @@ impl Web3Connection {
|
||||
// TODO: would be great if rpcs exposed this
|
||||
// TODO: move this to a helper function so we can recheck on errors or as the chain grows
|
||||
// TODO: move this to a helper function that checks
|
||||
if will_subscribe_to_blocks {
|
||||
if check_block_limit_needed {
|
||||
// TODO: make sure the server isn't still syncing
|
||||
|
||||
// TODO: don't sleep. wait for new heads subscription instead
|
||||
@ -194,32 +203,30 @@ impl Web3Connection {
|
||||
Ok((new_connection, handle))
|
||||
}
|
||||
|
||||
/// TODO: should check_block_data_limit take authorization?
|
||||
async fn check_block_data_limit(
|
||||
self: &Arc<Self>,
|
||||
authorization: &Arc<Authorization>,
|
||||
) -> anyhow::Result<Option<u64>> {
|
||||
let mut limit = None;
|
||||
|
||||
for block_data_limit in [u64::MAX, 90_000, 128, 64, 32] {
|
||||
// TODO: binary search between 90k and max?
|
||||
// TODO: start at 0 or 1
|
||||
for block_data_limit in [0, 32, 64, 128, 256, 512, 1024, 90_000, u64::MAX] {
|
||||
let mut head_block_id = self.head_block_id.read().clone();
|
||||
|
||||
// TODO: subscribe to a channel instead of polling. subscribe to http_interval_sender?
|
||||
while head_block_id.is_none() {
|
||||
warn!("no head block yet. retrying rpc {}", self);
|
||||
|
||||
// TODO: sleep for the block time, or maybe subscribe to a channel instead of this simple pull
|
||||
sleep(Duration::from_secs(13)).await;
|
||||
|
||||
head_block_id = self.head_block_id.read().clone();
|
||||
}
|
||||
let head_block_num = head_block_id.expect("is_none was checked above").num;
|
||||
|
||||
debug_assert_ne!(head_block_num, U64::zero());
|
||||
|
||||
// TODO: subtract 1 from block_data_limit for safety?
|
||||
let maybe_archive_block = head_block_num
|
||||
.saturating_sub((block_data_limit).into())
|
||||
.max(U64::one());
|
||||
let maybe_archive_block = head_block_num.saturating_sub((block_data_limit).into());
|
||||
|
||||
// TODO: wait for the handle BEFORE we check the current block number. it might be delayed too!
|
||||
// TODO: what should the request be?
|
||||
@ -233,17 +240,22 @@ impl Web3Connection {
|
||||
maybe_archive_block,
|
||||
)),
|
||||
// error here are expected, so keep the level low
|
||||
Level::Debug.into(),
|
||||
Level::Trace.into(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// // trace!(?archive_result, rpc=%self);
|
||||
|
||||
if archive_result.is_ok() {
|
||||
limit = Some(block_data_limit);
|
||||
trace!(
|
||||
"archive_result on {} for {}: {:?}",
|
||||
block_data_limit,
|
||||
self.name,
|
||||
archive_result
|
||||
);
|
||||
|
||||
if archive_result.is_err() {
|
||||
break;
|
||||
}
|
||||
|
||||
limit = Some(block_data_limit);
|
||||
}
|
||||
|
||||
if let Some(limit) = limit {
|
||||
@ -251,6 +263,8 @@ impl Web3Connection {
|
||||
.store(limit, atomic::Ordering::Release);
|
||||
}
|
||||
|
||||
debug!("block data limit on {}: {:?}", self.name, limit);
|
||||
|
||||
Ok(limit)
|
||||
}
|
||||
|
||||
@ -946,7 +960,7 @@ impl Serialize for Web3Connection {
|
||||
|
||||
state.serialize_field(
|
||||
"total_requests",
|
||||
&self.total_requests.load(atomic::Ordering::Relaxed),
|
||||
&self.frontend_requests.load(atomic::Ordering::Relaxed),
|
||||
)?;
|
||||
|
||||
let head_block_id = &*self.head_block_id.read();
|
||||
@ -999,7 +1013,8 @@ mod tests {
|
||||
url: "ws://example.com".to_string(),
|
||||
http_client: None,
|
||||
active_requests: 0.into(),
|
||||
total_requests: 0.into(),
|
||||
frontend_requests: 0.into(),
|
||||
internal_requests: 0.into(),
|
||||
provider: AsyncRwLock::new(None),
|
||||
hard_limit: None,
|
||||
soft_limit: 1_000,
|
||||
@ -1027,13 +1042,15 @@ mod tests {
|
||||
|
||||
let metrics = OpenRequestHandleMetrics::default();
|
||||
|
||||
// TODO: this is getting long. have a `impl Default`
|
||||
let x = Web3Connection {
|
||||
name: "name".to_string(),
|
||||
display_name: None,
|
||||
url: "ws://example.com".to_string(),
|
||||
http_client: None,
|
||||
active_requests: 0.into(),
|
||||
total_requests: 0.into(),
|
||||
frontend_requests: 0.into(),
|
||||
internal_requests: 0.into(),
|
||||
provider: AsyncRwLock::new(None),
|
||||
hard_limit: None,
|
||||
soft_limit: 1_000,
|
||||
|
||||
@ -215,7 +215,7 @@ impl Web3Connections {
|
||||
min_head_rpcs,
|
||||
});
|
||||
|
||||
let authorization = Arc::new(Authorization::local(db_conn.clone())?);
|
||||
let authorization = Arc::new(Authorization::internal(db_conn.clone())?);
|
||||
|
||||
let handle = {
|
||||
let connections = connections.clone();
|
||||
@ -857,7 +857,8 @@ mod tests {
|
||||
url: "ws://example.com/synced".to_string(),
|
||||
http_client: None,
|
||||
active_requests: 0.into(),
|
||||
total_requests: 0.into(),
|
||||
frontend_requests: 0.into(),
|
||||
internal_requests: 0.into(),
|
||||
provider: AsyncRwLock::new(Some(Arc::new(Web3Provider::Mock))),
|
||||
hard_limit: None,
|
||||
soft_limit: 1_000,
|
||||
@ -873,7 +874,8 @@ mod tests {
|
||||
url: "ws://example.com/lagged".to_string(),
|
||||
http_client: None,
|
||||
active_requests: 0.into(),
|
||||
total_requests: 0.into(),
|
||||
frontend_requests: 0.into(),
|
||||
internal_requests: 0.into(),
|
||||
provider: AsyncRwLock::new(Some(Arc::new(Web3Provider::Mock))),
|
||||
hard_limit: None,
|
||||
soft_limit: 1_000,
|
||||
@ -914,7 +916,7 @@ mod tests {
|
||||
min_sum_soft_limit: 1,
|
||||
};
|
||||
|
||||
let authorization = Arc::new(Authorization::local(None).unwrap());
|
||||
let authorization = Arc::new(Authorization::internal(None).unwrap());
|
||||
|
||||
let (head_block_sender, _head_block_receiver) =
|
||||
watch::channel::<ArcBlock>(Default::default());
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use super::connection::Web3Connection;
|
||||
use super::provider::Web3Provider;
|
||||
use crate::frontend::authorization::Authorization;
|
||||
use crate::frontend::authorization::{AuthorizatioType, Authorization};
|
||||
use crate::metered::{JsonRpcErrorCount, ProviderErrorCount};
|
||||
use anyhow::Context;
|
||||
use chrono::Utc;
|
||||
@ -42,14 +42,16 @@ pub struct OpenRequestHandle {
|
||||
|
||||
/// Depending on the context, RPC errors can require different handling.
|
||||
pub enum RequestErrorHandler {
|
||||
/// Potentially save the revert. Users can tune how often this happens
|
||||
SaveReverts,
|
||||
/// Log at the trace level. Use when errors are expected.
|
||||
TraceLevel,
|
||||
/// Log at the debug level. Use when errors are expected.
|
||||
DebugLevel,
|
||||
/// Log at the error level. Use when errors are bad.
|
||||
ErrorLevel,
|
||||
/// Log at the warn level. Use when errors do not cause problems.
|
||||
WarnLevel,
|
||||
/// Potentially save the revert. Users can tune how often this happens
|
||||
SaveReverts,
|
||||
}
|
||||
|
||||
// TODO: second param could be skipped since we don't need it here
|
||||
@ -65,6 +67,7 @@ struct EthCallFirstParams {
|
||||
impl From<Level> for RequestErrorHandler {
|
||||
fn from(level: Level) -> Self {
|
||||
match level {
|
||||
Level::Trace => RequestErrorHandler::TraceLevel,
|
||||
Level::Debug => RequestErrorHandler::DebugLevel,
|
||||
Level::Error => RequestErrorHandler::ErrorLevel,
|
||||
Level::Warn => RequestErrorHandler::WarnLevel,
|
||||
@ -136,7 +139,16 @@ impl OpenRequestHandle {
|
||||
|
||||
// TODO: handle overflows?
|
||||
// TODO: what ordering?
|
||||
conn.total_requests.fetch_add(1, atomic::Ordering::Relaxed);
|
||||
match authorization.as_ref().authorization_type {
|
||||
AuthorizatioType::Frontend => {
|
||||
conn.frontend_requests
|
||||
.fetch_add(1, atomic::Ordering::Relaxed);
|
||||
}
|
||||
AuthorizatioType::Internal => {
|
||||
conn.internal_requests
|
||||
.fetch_add(1, atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
let metrics = conn.open_request_handle_metrics.clone();
|
||||
let used = false.into();
|
||||
@ -273,6 +285,7 @@ impl OpenRequestHandle {
|
||||
false
|
||||
};
|
||||
|
||||
// TODO: think more about the method and param logs. those can be sensitive information
|
||||
match error_handler {
|
||||
RequestErrorHandler::DebugLevel => {
|
||||
// TODO: think about this revert check more. sometimes we might want reverts logged so this needs a flag
|
||||
@ -283,19 +296,38 @@ impl OpenRequestHandle {
|
||||
);
|
||||
}
|
||||
}
|
||||
RequestErrorHandler::ErrorLevel => {
|
||||
error!(
|
||||
RequestErrorHandler::TraceLevel => {
|
||||
trace!(
|
||||
"bad response from {}! method={} params={:?} err={:?}",
|
||||
self.conn, method, params, err
|
||||
self.conn,
|
||||
method,
|
||||
params,
|
||||
err
|
||||
);
|
||||
}
|
||||
RequestErrorHandler::ErrorLevel => {
|
||||
// TODO: include params if not running in release mode
|
||||
error!(
|
||||
"bad response from {}! method={} err={:?}",
|
||||
self.conn, method, err
|
||||
);
|
||||
}
|
||||
RequestErrorHandler::WarnLevel => {
|
||||
// TODO: include params if not running in release mode
|
||||
warn!(
|
||||
"bad response from {}! method={} params={:?} err={:?}",
|
||||
self.conn, method, params, err
|
||||
"bad response from {}! method={} err={:?}",
|
||||
self.conn, method, err
|
||||
);
|
||||
}
|
||||
RequestErrorHandler::SaveReverts => {
|
||||
trace!(
|
||||
"bad response from {}! method={} params={:?} err={:?}",
|
||||
self.conn,
|
||||
method,
|
||||
params,
|
||||
err
|
||||
);
|
||||
|
||||
// TODO: do not unwrap! (doesn't matter much since we check method as a string above)
|
||||
let method: Method = Method::try_from_value(&method.to_string()).unwrap();
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user