From d98b6aeff3400599d8271d0d0f13289f60f4231b Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Tue, 20 Sep 2022 06:58:40 +0000 Subject: [PATCH] split error counts --- TODO.md | 7 +-- web3_proxy/src/metered/jsonrpc_error_count.rs | 60 +++++++++++++++++++ web3_proxy/src/metered/mod.rs | 5 ++ .../src/metered/provider_error_count.rs | 57 ++++++++++++++++++ web3_proxy/src/metrics_frontend.rs | 58 ++++++++++++++++++ 5 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 web3_proxy/src/metered/jsonrpc_error_count.rs create mode 100644 web3_proxy/src/metered/mod.rs create mode 100644 web3_proxy/src/metered/provider_error_count.rs create mode 100644 web3_proxy/src/metrics_frontend.rs diff --git a/TODO.md b/TODO.md index 87c7865c..77d0ba75 100644 --- a/TODO.md +++ b/TODO.md @@ -152,12 +152,11 @@ These are roughly in order of completition - [x] when there are a LOT of concurrent requests, we see errors. i thought that was a problem with redis cell, but it happens with my simpler rate limit. now i think the problem is actually with bb8 - https://docs.rs/redis/latest/redis/aio/struct.ConnectionManager.html or https://crates.io/crates/deadpool-redis? - WARN http_request: redis_rate_limit::errors: redis error err=Response was of incompatible type: "Response type not string compatible." (response was int(500237)) id=01GC6514JWN5PS1NCWJCGJTC94 method=POST -- [ ] web3_proxy_error_count{path = "backend_rpc/request"} is inflated by a bunch of reverts. do not log reverts as warn. +- [x] web3_proxy_error_count{path = "backend_rpc/request"} is inflated by a bunch of reverts. do not log reverts as warn. - erigon gives `method=eth_call reqid=986147 t=1.151551ms err="execution reverted"` - - [ ] maybe change request to return a JsonRpcResponse or an anyhow - - [ ] opt-in debug mode that inspects responses for reverts and saves the request to the database for the user +- [ ] opt-in debug mode that inspects responses for reverts and saves the request to the database for the user - this must be opt-in or spawned since it will slow things down and will make their calls less private -- [ ] add configurable size limits to all the Caches +- [-] add configurable size limits to all the Caches - [ ] Api keys need option to lock to IP, cors header, referer, etc - [ ] active requests per second per api key - [ ] distribution of methods per api key (eth_call, eth_getLogs, etc.) diff --git a/web3_proxy/src/metered/jsonrpc_error_count.rs b/web3_proxy/src/metered/jsonrpc_error_count.rs new file mode 100644 index 00000000..424e8b7c --- /dev/null +++ b/web3_proxy/src/metered/jsonrpc_error_count.rs @@ -0,0 +1,60 @@ +//! A module providing the `JsonRpcErrorCount` metric. + +use ethers::providers::ProviderError; +use metered::metric::{Advice, Enter, OnResult}; +use metered::{ + atomic::AtomicInt, + clear::Clear, + metric::{Counter, Metric}, +}; +use serde::Serialize; +use std::ops::Deref; + +/// A metric counting how many times an expression typed std `Result` as +/// returned an `Err` variant. +/// +/// This is a light-weight metric. +/// +/// By default, `ErrorCount` uses a lock-free `u64` `Counter`, which makes sense +/// in multithread scenarios. Non-threaded applications can gain performance by +/// using a `std::cell:Cell` instead. +#[derive(Clone, Default, Debug, Serialize)] +pub struct JsonRpcErrorCount>(pub C); + +impl Metric> for JsonRpcErrorCount {} + +impl Enter for JsonRpcErrorCount { + type E = (); + fn enter(&self) {} +} + +impl OnResult> for JsonRpcErrorCount { + /// Unlike the default ErrorCount, this one does not increment for internal jsonrpc errors + /// TODO: count errors like this on another helper + fn on_result(&self, _: (), r: &Result) -> Advice { + match r { + Ok(_) => {} + Err(ProviderError::JsonRpcClientError(_)) => { + self.0.incr(); + } + Err(_) => { + // TODO: count jsonrpc errors + } + } + Advice::Return + } +} + +impl Clear for JsonRpcErrorCount { + fn clear(&self) { + self.0.clear() + } +} + +impl Deref for JsonRpcErrorCount { + type Target = C; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/web3_proxy/src/metered/mod.rs b/web3_proxy/src/metered/mod.rs new file mode 100644 index 00000000..f8f61bbc --- /dev/null +++ b/web3_proxy/src/metered/mod.rs @@ -0,0 +1,5 @@ +mod jsonrpc_error_count; +mod provider_error_count; + +pub use self::jsonrpc_error_count::JsonRpcErrorCount; +pub use self::provider_error_count::ProviderErrorCount; diff --git a/web3_proxy/src/metered/provider_error_count.rs b/web3_proxy/src/metered/provider_error_count.rs new file mode 100644 index 00000000..5670e3ba --- /dev/null +++ b/web3_proxy/src/metered/provider_error_count.rs @@ -0,0 +1,57 @@ +//! A module providing the `JsonRpcErrorCount` metric. + +use ethers::providers::ProviderError; +use metered::metric::{Advice, Enter, OnResult}; +use metered::{ + atomic::AtomicInt, + clear::Clear, + metric::{Counter, Metric}, +}; +use serde::Serialize; +use std::ops::Deref; + +/// A metric counting how many times an expression typed std `Result` as +/// returned an `Err` variant. +/// +/// This is a light-weight metric. +/// +/// By default, `ErrorCount` uses a lock-free `u64` `Counter`, which makes sense +/// in multithread scenarios. Non-threaded applications can gain performance by +/// using a `std::cell:Cell` instead. +#[derive(Clone, Default, Debug, Serialize)] +pub struct ProviderErrorCount>(pub C); + +impl Metric> for ProviderErrorCount {} + +impl Enter for ProviderErrorCount { + type E = (); + fn enter(&self) {} +} + +impl OnResult> for ProviderErrorCount { + /// Unlike the default ErrorCount, this one does not increment for internal jsonrpc errors + fn on_result(&self, _: (), r: &Result) -> Advice { + match r { + Ok(_) => {} + Err(ProviderError::JsonRpcClientError(_)) => {} + Err(_) => { + self.0.incr(); + } + } + Advice::Return + } +} + +impl Clear for ProviderErrorCount { + fn clear(&self) { + self.0.clear() + } +} + +impl Deref for ProviderErrorCount { + type Target = C; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/web3_proxy/src/metrics_frontend.rs b/web3_proxy/src/metrics_frontend.rs new file mode 100644 index 00000000..fbf01786 --- /dev/null +++ b/web3_proxy/src/metrics_frontend.rs @@ -0,0 +1,58 @@ +use axum::headers::HeaderName; +use axum::http::HeaderValue; +use axum::response::{IntoResponse, Response}; +use axum::{routing::get, Extension, Router}; +use std::net::SocketAddr; +use std::sync::Arc; +use tracing::{info, instrument}; + +use crate::app::Web3ProxyApp; + +/// Run a prometheus metrics server on the given port. +#[instrument(skip_all)] +pub async fn serve(app: Arc, port: u16) -> anyhow::Result<()> { + // build our application with a route + // order most to least common + // TODO: 404 any unhandled routes? + let app = Router::new().route("/", get(root)).layer(Extension(app)); + + // run our app with hyper + // TODO: allow only listening on localhost? + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + info!("prometheus listening on port {}", port); + // TODO: into_make_service is enough if we always run behind a proxy. make into_make_service_with_connect_info optional? + + /* + It sequentially looks for an IP in: + - x-forwarded-for header (de-facto standard) + - x-real-ip header + - forwarded header (new standard) + - axum::extract::ConnectInfo (if not behind proxy) + + So we probably won't need into_make_service_with_connect_info, but it shouldn't hurt + */ + let service = app.into_make_service_with_connect_info::(); + // let service = app.into_make_service(); + + // `axum::Server` is a re-export of `hyper::Server` + axum::Server::bind(&addr) + // TODO: option to use with_connect_info. we want it in dev, but not when running behind a proxy, but not + .serve(service) + .await + .map_err(Into::into) +} + +#[instrument(skip_all)] +async fn root(Extension(app): Extension>) -> Response { + let serialized = app.prometheus_metrics(); + + let mut r = serialized.into_response(); + + // // TODO: is there an easier way to do this? + r.headers_mut().insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/openmetrics-text; version=1.0.0; charset=utf-8"), + ); + + r +}