split error counts
This commit is contained in:
parent
abe5001792
commit
d98b6aeff3
7
TODO.md
7
TODO.md
@ -152,12 +152,11 @@ These are roughly in order of completition
|
|||||||
- [x] when there are a LOT of concurrent requests, we see errors. i thought that was a problem with redis cell, but it happens with my simpler rate limit. now i think the problem is actually with bb8
|
- [x] when there are a LOT of concurrent requests, we see errors. i thought that was a problem with redis cell, but it happens with my simpler rate limit. now i think the problem is actually with bb8
|
||||||
- https://docs.rs/redis/latest/redis/aio/struct.ConnectionManager.html or https://crates.io/crates/deadpool-redis?
|
- https://docs.rs/redis/latest/redis/aio/struct.ConnectionManager.html or https://crates.io/crates/deadpool-redis?
|
||||||
- WARN http_request: redis_rate_limit::errors: redis error err=Response was of incompatible type: "Response type not string compatible." (response was int(500237)) id=01GC6514JWN5PS1NCWJCGJTC94 method=POST
|
- WARN http_request: redis_rate_limit::errors: redis error err=Response was of incompatible type: "Response type not string compatible." (response was int(500237)) id=01GC6514JWN5PS1NCWJCGJTC94 method=POST
|
||||||
- [ ] web3_proxy_error_count{path = "backend_rpc/request"} is inflated by a bunch of reverts. do not log reverts as warn.
|
- [x] web3_proxy_error_count{path = "backend_rpc/request"} is inflated by a bunch of reverts. do not log reverts as warn.
|
||||||
- erigon gives `method=eth_call reqid=986147 t=1.151551ms err="execution reverted"`
|
- erigon gives `method=eth_call reqid=986147 t=1.151551ms err="execution reverted"`
|
||||||
- [ ] maybe change request to return a JsonRpcResponse or an anyhow
|
- [ ] opt-in debug mode that inspects responses for reverts and saves the request to the database for the user
|
||||||
- [ ] opt-in debug mode that inspects responses for reverts and saves the request to the database for the user
|
|
||||||
- this must be opt-in or spawned since it will slow things down and will make their calls less private
|
- this must be opt-in or spawned since it will slow things down and will make their calls less private
|
||||||
- [ ] add configurable size limits to all the Caches
|
- [-] add configurable size limits to all the Caches
|
||||||
- [ ] Api keys need option to lock to IP, cors header, referer, etc
|
- [ ] Api keys need option to lock to IP, cors header, referer, etc
|
||||||
- [ ] active requests per second per api key
|
- [ ] active requests per second per api key
|
||||||
- [ ] distribution of methods per api key (eth_call, eth_getLogs, etc.)
|
- [ ] distribution of methods per api key (eth_call, eth_getLogs, etc.)
|
||||||
|
60
web3_proxy/src/metered/jsonrpc_error_count.rs
Normal file
60
web3_proxy/src/metered/jsonrpc_error_count.rs
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
//! A module providing the `JsonRpcErrorCount` metric.
|
||||||
|
|
||||||
|
use ethers::providers::ProviderError;
|
||||||
|
use metered::metric::{Advice, Enter, OnResult};
|
||||||
|
use metered::{
|
||||||
|
atomic::AtomicInt,
|
||||||
|
clear::Clear,
|
||||||
|
metric::{Counter, Metric},
|
||||||
|
};
|
||||||
|
use serde::Serialize;
|
||||||
|
use std::ops::Deref;
|
||||||
|
|
||||||
|
/// A metric counting how many times an expression typed std `Result` as
|
||||||
|
/// returned an `Err` variant.
|
||||||
|
///
|
||||||
|
/// This is a light-weight metric.
|
||||||
|
///
|
||||||
|
/// By default, `ErrorCount` uses a lock-free `u64` `Counter`, which makes sense
|
||||||
|
/// in multithread scenarios. Non-threaded applications can gain performance by
|
||||||
|
/// using a `std::cell:Cell<u64>` instead.
|
||||||
|
#[derive(Clone, Default, Debug, Serialize)]
|
||||||
|
pub struct JsonRpcErrorCount<C: Counter = AtomicInt<u64>>(pub C);
|
||||||
|
|
||||||
|
impl<C: Counter, T> Metric<Result<T, ProviderError>> for JsonRpcErrorCount<C> {}
|
||||||
|
|
||||||
|
impl<C: Counter> Enter for JsonRpcErrorCount<C> {
|
||||||
|
type E = ();
|
||||||
|
fn enter(&self) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C: Counter, T> OnResult<Result<T, ProviderError>> for JsonRpcErrorCount<C> {
|
||||||
|
/// Unlike the default ErrorCount, this one does not increment for internal jsonrpc errors
|
||||||
|
/// TODO: count errors like this on another helper
|
||||||
|
fn on_result(&self, _: (), r: &Result<T, ProviderError>) -> Advice {
|
||||||
|
match r {
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(ProviderError::JsonRpcClientError(_)) => {
|
||||||
|
self.0.incr();
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// TODO: count jsonrpc errors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Advice::Return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C: Counter> Clear for JsonRpcErrorCount<C> {
|
||||||
|
fn clear(&self) {
|
||||||
|
self.0.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C: Counter> Deref for JsonRpcErrorCount<C> {
|
||||||
|
type Target = C;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
}
|
5
web3_proxy/src/metered/mod.rs
Normal file
5
web3_proxy/src/metered/mod.rs
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
mod jsonrpc_error_count;
|
||||||
|
mod provider_error_count;
|
||||||
|
|
||||||
|
pub use self::jsonrpc_error_count::JsonRpcErrorCount;
|
||||||
|
pub use self::provider_error_count::ProviderErrorCount;
|
57
web3_proxy/src/metered/provider_error_count.rs
Normal file
57
web3_proxy/src/metered/provider_error_count.rs
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
//! A module providing the `JsonRpcErrorCount` metric.
|
||||||
|
|
||||||
|
use ethers::providers::ProviderError;
|
||||||
|
use metered::metric::{Advice, Enter, OnResult};
|
||||||
|
use metered::{
|
||||||
|
atomic::AtomicInt,
|
||||||
|
clear::Clear,
|
||||||
|
metric::{Counter, Metric},
|
||||||
|
};
|
||||||
|
use serde::Serialize;
|
||||||
|
use std::ops::Deref;
|
||||||
|
|
||||||
|
/// A metric counting how many times an expression typed std `Result` as
|
||||||
|
/// returned an `Err` variant.
|
||||||
|
///
|
||||||
|
/// This is a light-weight metric.
|
||||||
|
///
|
||||||
|
/// By default, `ErrorCount` uses a lock-free `u64` `Counter`, which makes sense
|
||||||
|
/// in multithread scenarios. Non-threaded applications can gain performance by
|
||||||
|
/// using a `std::cell:Cell<u64>` instead.
|
||||||
|
#[derive(Clone, Default, Debug, Serialize)]
|
||||||
|
pub struct ProviderErrorCount<C: Counter = AtomicInt<u64>>(pub C);
|
||||||
|
|
||||||
|
impl<C: Counter, T> Metric<Result<T, ProviderError>> for ProviderErrorCount<C> {}
|
||||||
|
|
||||||
|
impl<C: Counter> Enter for ProviderErrorCount<C> {
|
||||||
|
type E = ();
|
||||||
|
fn enter(&self) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C: Counter, T> OnResult<Result<T, ProviderError>> for ProviderErrorCount<C> {
|
||||||
|
/// Unlike the default ErrorCount, this one does not increment for internal jsonrpc errors
|
||||||
|
fn on_result(&self, _: (), r: &Result<T, ProviderError>) -> Advice {
|
||||||
|
match r {
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(ProviderError::JsonRpcClientError(_)) => {}
|
||||||
|
Err(_) => {
|
||||||
|
self.0.incr();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Advice::Return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C: Counter> Clear for ProviderErrorCount<C> {
|
||||||
|
fn clear(&self) {
|
||||||
|
self.0.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C: Counter> Deref for ProviderErrorCount<C> {
|
||||||
|
type Target = C;
|
||||||
|
|
||||||
|
fn deref(&self) -> &Self::Target {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
}
|
58
web3_proxy/src/metrics_frontend.rs
Normal file
58
web3_proxy/src/metrics_frontend.rs
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
use axum::headers::HeaderName;
|
||||||
|
use axum::http::HeaderValue;
|
||||||
|
use axum::response::{IntoResponse, Response};
|
||||||
|
use axum::{routing::get, Extension, Router};
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tracing::{info, instrument};
|
||||||
|
|
||||||
|
use crate::app::Web3ProxyApp;
|
||||||
|
|
||||||
|
/// Run a prometheus metrics server on the given port.
|
||||||
|
#[instrument(skip_all)]
|
||||||
|
pub async fn serve(app: Arc<Web3ProxyApp>, port: u16) -> anyhow::Result<()> {
|
||||||
|
// build our application with a route
|
||||||
|
// order most to least common
|
||||||
|
// TODO: 404 any unhandled routes?
|
||||||
|
let app = Router::new().route("/", get(root)).layer(Extension(app));
|
||||||
|
|
||||||
|
// run our app with hyper
|
||||||
|
// TODO: allow only listening on localhost?
|
||||||
|
let addr = SocketAddr::from(([0, 0, 0, 0], port));
|
||||||
|
info!("prometheus listening on port {}", port);
|
||||||
|
// TODO: into_make_service is enough if we always run behind a proxy. make into_make_service_with_connect_info optional?
|
||||||
|
|
||||||
|
/*
|
||||||
|
It sequentially looks for an IP in:
|
||||||
|
- x-forwarded-for header (de-facto standard)
|
||||||
|
- x-real-ip header
|
||||||
|
- forwarded header (new standard)
|
||||||
|
- axum::extract::ConnectInfo (if not behind proxy)
|
||||||
|
|
||||||
|
So we probably won't need into_make_service_with_connect_info, but it shouldn't hurt
|
||||||
|
*/
|
||||||
|
let service = app.into_make_service_with_connect_info::<SocketAddr>();
|
||||||
|
// let service = app.into_make_service();
|
||||||
|
|
||||||
|
// `axum::Server` is a re-export of `hyper::Server`
|
||||||
|
axum::Server::bind(&addr)
|
||||||
|
// TODO: option to use with_connect_info. we want it in dev, but not when running behind a proxy, but not
|
||||||
|
.serve(service)
|
||||||
|
.await
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument(skip_all)]
|
||||||
|
async fn root(Extension(app): Extension<Arc<Web3ProxyApp>>) -> Response {
|
||||||
|
let serialized = app.prometheus_metrics();
|
||||||
|
|
||||||
|
let mut r = serialized.into_response();
|
||||||
|
|
||||||
|
// // TODO: is there an easier way to do this?
|
||||||
|
r.headers_mut().insert(
|
||||||
|
HeaderName::from_static("content-type"),
|
||||||
|
HeaderValue::from_static("application/openmetrics-text; version=1.0.0; charset=utf-8"),
|
||||||
|
);
|
||||||
|
|
||||||
|
r
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user