split error counts

TODO.md

@@ -152,12 +152,11 @@ These are roughly in order of completition
   - [x] when there are a LOT of concurrent requests, we see errors. i thought that was a problem with redis cell, but it happens with my simpler rate limit. now i think the problem is actually with bb8
   - https://docs.rs/redis/latest/redis/aio/struct.ConnectionManager.html or https://crates.io/crates/deadpool-redis?
   - WARN http_request: redis_rate_limit::errors: redis error err=Response was of incompatible type: "Response type not string compatible." (response was int(500237)) id=01GC6514JWN5PS1NCWJCGJTC94 method=POST
-- [ ] web3_proxy_error_count{path = "backend_rpc/request"} is inflated by a bunch of reverts. do not log reverts as warn. 
+- [x] web3_proxy_error_count{path = "backend_rpc/request"} is inflated by a bunch of reverts. do not log reverts as warn. 
   - erigon gives `method=eth_call reqid=986147 t=1.151551ms err="execution reverted"`
-  - [ ] maybe change request to return a JsonRpcResponse or an anyhow
-  - [ ] opt-in debug mode that inspects responses for reverts and saves the request to the database for the user
+- [ ] opt-in debug mode that inspects responses for reverts and saves the request to the database for the user
   - this must be opt-in or spawned since it will slow things down and will make their calls less private
-- [ ] add configurable size limits to all the Caches
+- [-] add configurable size limits to all the Caches
 - [ ] Api keys need option to lock to IP, cors header, referer, etc
 - [ ] active requests per second per api key
 - [ ] distribution of methods per api key (eth_call, eth_getLogs, etc.)

web3_proxy/src/metered/jsonrpc_error_count.rs

@@ -0,0 +1,60 @@
+//! A module providing the `JsonRpcErrorCount` metric.
+
+use ethers::providers::ProviderError;
+use metered::metric::{Advice, Enter, OnResult};
+use metered::{
+    atomic::AtomicInt,
+    clear::Clear,
+    metric::{Counter, Metric},
+};
+use serde::Serialize;
+use std::ops::Deref;
+
+/// A metric counting how many times an expression typed std `Result` as
+/// returned an `Err` variant.
+///
+/// This is a light-weight metric.
+///
+/// By default, `ErrorCount` uses a lock-free `u64` `Counter`, which makes sense
+/// in multithread scenarios. Non-threaded applications can gain performance by
+/// using a `std::cell:Cell<u64>` instead.
+#[derive(Clone, Default, Debug, Serialize)]
+pub struct JsonRpcErrorCount<C: Counter = AtomicInt<u64>>(pub C);
+
+impl<C: Counter, T> Metric<Result<T, ProviderError>> for JsonRpcErrorCount<C> {}
+
+impl<C: Counter> Enter for JsonRpcErrorCount<C> {
+    type E = ();
+    fn enter(&self) {}
+}
+
+impl<C: Counter, T> OnResult<Result<T, ProviderError>> for JsonRpcErrorCount<C> {
+    /// Unlike the default ErrorCount, this one does not increment for internal jsonrpc errors
+    /// TODO: count errors like this on another helper
+    fn on_result(&self, _: (), r: &Result<T, ProviderError>) -> Advice {
+        match r {
+            Ok(_) => {}
+            Err(ProviderError::JsonRpcClientError(_)) => {
+                self.0.incr();
+            }
+            Err(_) => {
+                // TODO: count jsonrpc errors
+            }
+        }
+        Advice::Return
+    }
+}
+
+impl<C: Counter> Clear for JsonRpcErrorCount<C> {
+    fn clear(&self) {
+        self.0.clear()
+    }
+}
+
+impl<C: Counter> Deref for JsonRpcErrorCount<C> {
+    type Target = C;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}

web3_proxy/src/metered/mod.rs

@@ -0,0 +1,5 @@
+mod jsonrpc_error_count;
+mod provider_error_count;
+
+pub use self::jsonrpc_error_count::JsonRpcErrorCount;
+pub use self::provider_error_count::ProviderErrorCount;

web3_proxy/src/metered/provider_error_count.rs

@@ -0,0 +1,57 @@
+//! A module providing the `JsonRpcErrorCount` metric.
+
+use ethers::providers::ProviderError;
+use metered::metric::{Advice, Enter, OnResult};
+use metered::{
+    atomic::AtomicInt,
+    clear::Clear,
+    metric::{Counter, Metric},
+};
+use serde::Serialize;
+use std::ops::Deref;
+
+/// A metric counting how many times an expression typed std `Result` as
+/// returned an `Err` variant.
+///
+/// This is a light-weight metric.
+///
+/// By default, `ErrorCount` uses a lock-free `u64` `Counter`, which makes sense
+/// in multithread scenarios. Non-threaded applications can gain performance by
+/// using a `std::cell:Cell<u64>` instead.
+#[derive(Clone, Default, Debug, Serialize)]
+pub struct ProviderErrorCount<C: Counter = AtomicInt<u64>>(pub C);
+
+impl<C: Counter, T> Metric<Result<T, ProviderError>> for ProviderErrorCount<C> {}
+
+impl<C: Counter> Enter for ProviderErrorCount<C> {
+    type E = ();
+    fn enter(&self) {}
+}
+
+impl<C: Counter, T> OnResult<Result<T, ProviderError>> for ProviderErrorCount<C> {
+    /// Unlike the default ErrorCount, this one does not increment for internal jsonrpc errors
+    fn on_result(&self, _: (), r: &Result<T, ProviderError>) -> Advice {
+        match r {
+            Ok(_) => {}
+            Err(ProviderError::JsonRpcClientError(_)) => {}
+            Err(_) => {
+                self.0.incr();
+            }
+        }
+        Advice::Return
+    }
+}
+
+impl<C: Counter> Clear for ProviderErrorCount<C> {
+    fn clear(&self) {
+        self.0.clear()
+    }
+}
+
+impl<C: Counter> Deref for ProviderErrorCount<C> {
+    type Target = C;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}

web3_proxy/src/metrics_frontend.rs

@@ -0,0 +1,58 @@
+use axum::headers::HeaderName;
+use axum::http::HeaderValue;
+use axum::response::{IntoResponse, Response};
+use axum::{routing::get, Extension, Router};
+use std::net::SocketAddr;
+use std::sync::Arc;
+use tracing::{info, instrument};
+
+use crate::app::Web3ProxyApp;
+
+/// Run a prometheus metrics server on the given port.
+#[instrument(skip_all)]
+pub async fn serve(app: Arc<Web3ProxyApp>, port: u16) -> anyhow::Result<()> {
+    // build our application with a route
+    // order most to least common
+    // TODO: 404 any unhandled routes?
+    let app = Router::new().route("/", get(root)).layer(Extension(app));
+
+    // run our app with hyper
+    // TODO: allow only listening on localhost?
+    let addr = SocketAddr::from(([0, 0, 0, 0], port));
+    info!("prometheus listening on port {}", port);
+    // TODO: into_make_service is enough if we always run behind a proxy. make into_make_service_with_connect_info optional?
+
+    /*
+    It sequentially looks for an IP in:
+      - x-forwarded-for header (de-facto standard)
+      - x-real-ip header
+      - forwarded header (new standard)
+      - axum::extract::ConnectInfo (if not behind proxy)
+
+    So we probably won't need into_make_service_with_connect_info, but it shouldn't hurt
+    */
+    let service = app.into_make_service_with_connect_info::<SocketAddr>();
+    // let service = app.into_make_service();
+
+    // `axum::Server` is a re-export of `hyper::Server`
+    axum::Server::bind(&addr)
+        // TODO: option to use with_connect_info. we want it in dev, but not when running behind a proxy, but not
+        .serve(service)
+        .await
+        .map_err(Into::into)
+}
+
+#[instrument(skip_all)]
+async fn root(Extension(app): Extension<Arc<Web3ProxyApp>>) -> Response {
+    let serialized = app.prometheus_metrics();
+
+    let mut r = serialized.into_response();
+
+    // // TODO: is there an easier way to do this?
+    r.headers_mut().insert(
+        HeaderName::from_static("content-type"),
+        HeaderValue::from_static("application/openmetrics-text; version=1.0.0; charset=utf-8"),
+    );
+
+    r
+}