From 1aa6b4cdb489eed4331f7f02e038d407315a4a75 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Mon, 25 Jul 2022 22:36:02 +0000 Subject: [PATCH] better logs --- README.md | 5 +++-- TODO.md | 16 ++++++++-------- web3-proxy/Cargo.toml | 1 + web3-proxy/src/connections.rs | 27 ++++++++++++++++++++++----- web3-proxy/src/main.rs | 2 +- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f860c64c..6e056eff 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ All other requests are sent to an RPC server on the latest block (alchemy, moral Each server has different limits to configure. The `soft_limit` is the number of parallel active requests where a server starts to slow down. The `hard_limit` is where a server starts giving rate limits or other errors. ``` -$ cargo run --release -p web3-proxy -- --help +$ cargo run --release -- --help ``` ``` Compiling web3-proxy v0.1.0 (/home/bryan/src/web3-proxy/web3-proxy) @@ -31,7 +31,7 @@ Options: Start the server with the defaults (listen on `http://localhost:8544` and use `./config/example.toml` which proxies to a bunch of public nodes: ``` -cargo run --release -p web3-proxy -- --config ./config/example.toml +cargo run --release -- --config ./config/example.toml ``` ## Common commands @@ -75,6 +75,7 @@ Run the proxy under gdb for advanced debugging: cargo build --release && RUST_LOG=web3_proxy=debug rust-gdb --args target/debug/web3-proxy --listen-port 7503 --rpc-config-path ./config/production-eth.toml +TODO: also enable debug symbols in the release build by modifying the root Cargo.toml ## Load Testing diff --git a/TODO.md b/TODO.md index 71a04f8a..673e88fa 100644 --- a/TODO.md +++ b/TODO.md @@ -58,13 +58,6 @@ - we can improve this by only publishing the synced connections once a threshold of total available soft and hard limits is passed. how can we do this without hammering redis? at least its only once per block per server - [x] instead of tracking `pending_synced_connections`, have a mapping of where all connections are individually. then each change, re-check for consensus. - [x] synced connections swap threshold set to 1 so that it always serves something -- [ ] if we request an old block, more servers can handle it than we currently use. - - [ ] instead of the one list of just heads, store our intermediate mappings (rpcs_by_hash, rpcs_by_num, blocks_by_hash) in SyncedConnections. this shouldn't be too much slower than what we have now - - [ ] remove the if/else where we optionally route to archive and refactor to require a BlockNumber enum - - [ ] then check syncedconnections for the blockNum. if num given, use the cannonical chain to figure out the winning hash - - [ ] this means if someone requests a recent but not ancient block, they can use all our servers, even the slower ones - -- [ ] nice output when cargo doc is run - [ ] basic request method stats ## V1 @@ -79,7 +72,14 @@ - [cancelled] eth_getBlockByNumber and similar calls served from the block map - will need all Block **and** Block in caches or fetched efficiently - so maybe we don't want this. we can just use the general request cache for these. they will only require 1 request and it means requests won't get in the way as much on writes as new blocks arrive. +- [ ] cli tool for managing users and resetting api keys - [ ] incoming rate limiting by api key +- [ ] nice output when cargo doc is run +- [ ] if we request an old block, more servers can handle it than we currently use. + - [ ] instead of the one list of just heads, store our intermediate mappings (rpcs_by_hash, rpcs_by_num, blocks_by_hash) in SyncedConnections. this shouldn't be too much slower than what we have now + - [ ] remove the if/else where we optionally route to archive and refactor to require a BlockNumber enum + - [ ] then check syncedconnections for the blockNum. if num given, use the cannonical chain to figure out the winning hash + - [ ] this means if someone requests a recent but not ancient block, they can use all our servers, even the slower ones - [ ] refactor so configs can change while running - create the app without applying any config to it - have a blocking future watching the config file and calling app.apply_config() on first load and on change @@ -110,7 +110,7 @@ - [ ] 60 second timeout is too short. Maybe do that for free tier and larger timeout for paid. Problem is that some queries can take over 1000 seconds new endpoints for users: -- think about where to put this. a separate app might be better. this repo could just have a cli tool for managing users +- think about where to put this. a separate app might be better - [ ] GET /user/login/$address - returns a JSON string for the user to sign - [ ] POST /user/login/$address diff --git a/web3-proxy/Cargo.toml b/web3-proxy/Cargo.toml index 8b8ad4d3..f1809f5c 100644 --- a/web3-proxy/Cargo.toml +++ b/web3-proxy/Cargo.toml @@ -2,6 +2,7 @@ name = "web3-proxy" version = "0.1.0" edition = "2021" +default-run = "web3-proxy" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/web3-proxy/src/connections.rs b/web3-proxy/src/connections.rs index 35ad6561..95494734 100644 --- a/web3-proxy/src/connections.rs +++ b/web3-proxy/src/connections.rs @@ -600,6 +600,13 @@ impl Web3Connections { let mut connection_heads = IndexMap::>>::new(); while let Ok((new_block, rpc)) = block_receiver.recv_async().await { + if let Some(current_block) = connection_heads.get(rpc.url()) { + if current_block.hash == new_block.hash { + // duplicate block + continue; + } + } + let new_block_hash = if let Some(hash) = new_block.hash { hash } else { @@ -810,13 +817,17 @@ impl Web3Connections { if new_head_block { self.chain.add_block(new_block.clone(), true); - // TODO: include the fastest rpc here? info!( - "{}/{} rpcs at {} ({}). publishing new head!", + "{}/{} rpcs at {} ({}). head at {:?}", pending_synced_connections.conns.len(), self.conns.len(), pending_synced_connections.head_block_hash, pending_synced_connections.head_block_num, + pending_synced_connections + .conns + .iter() + .map(|x| format!("{}", x)) + .collect::>(), ); // TODO: what if the hashes don't match? if pending_synced_connections.head_block_hash == new_block_hash { @@ -833,12 +844,18 @@ impl Web3Connections { // TODO: mark any orphaned transactions as unconfirmed } + } else if num_best_rpcs == self.conns.len() { + debug!( + "all {} rpcs at {} ({})", + num_best_rpcs, + pending_synced_connections.head_block_hash, + pending_synced_connections.head_block_num, + ); } else { - // TODO: i'm seeing 4/4 print twice. maybe because of http providers? - // TODO: only do this log if there was a change trace!( + ?pending_synced_connections, "{}/{} rpcs at {} ({})", - pending_synced_connections.conns.len(), + num_best_rpcs, self.conns.len(), pending_synced_connections.head_block_hash, pending_synced_connections.head_block_num, diff --git a/web3-proxy/src/main.rs b/web3-proxy/src/main.rs index 10853be0..f07ad56b 100644 --- a/web3-proxy/src/main.rs +++ b/web3-proxy/src/main.rs @@ -156,7 +156,7 @@ mod tests { use hashbrown::HashMap; use std::env; - use web3_proxy::config::{RpcSharedConfig, Web3ConnectionConfig}; + use crate::config::{RpcSharedConfig, Web3ConnectionConfig}; use super::*;