default to less detailed stats

Cargo.lock

@@ -1347,7 +1347,7 @@ dependencies = [
 
 [[package]]
 name = "entities"
-version = "0.10.0"
+version = "0.11.0"
 dependencies = [
  "sea-orm",
  "serde",
@@ -2720,7 +2720,7 @@ dependencies = [
 
 [[package]]
 name = "migration"
-version = "0.10.0"
+version = "0.11.0"
 dependencies = [
  "sea-orm-migration",
  "tokio",
@@ -5499,7 +5499,7 @@ dependencies = [
 
 [[package]]
 name = "web3_proxy"
-version = "0.10.0"
+version = "0.11.0"
 dependencies = [
  "anyhow",
  "arc-swap",

entities/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "entities"
-version = "0.10.0"
+version = "0.11.0"
 edition = "2021"
 
 [lib]

entities/src/rpc_accounting.rs

@@ -10,7 +10,7 @@ pub struct Model {
     pub id: u64,
     pub rpc_key_id: Option<u64>,
     pub chain_id: u64,
-    pub method: String,
+    pub method: Option<String>,
     pub archive_request: bool,
     pub error_response: bool,
     pub period_datetime: DateTimeUtc,

entities/src/rpc_key.rs

@@ -1,5 +1,6 @@
 //! SeaORM Entity. Generated by sea-orm-codegen 0.10.1
 
+use super::sea_orm_active_enums::LogLevel;
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
@@ -14,6 +15,8 @@ pub struct Model {
     pub description: Option<String>,
     pub private_txs: bool,
     pub active: bool,
+    pub log_level: LogLevel,
+    pub log_revert_chance: f64,
     #[sea_orm(column_type = "Text", nullable)]
     pub allowed_ips: Option<String>,
     #[sea_orm(column_type = "Text", nullable)]
@@ -22,7 +25,12 @@ pub struct Model {
     pub allowed_referers: Option<String>,
     #[sea_orm(column_type = "Text", nullable)]
     pub allowed_user_agents: Option<String>,
-    pub log_revert_chance: f64,
+}
+
+pub enum RpcKeyLogLevels {
+    None,
+    Basic,
+    Detailed,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

entities/src/sea_orm_active_enums.rs

@@ -3,6 +3,34 @@
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
+#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
+#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "log_level")]
+pub enum LogLevel {
+    #[sea_orm(string_value = "none")]
+    None,
+    #[sea_orm(string_value = "aggregate")]
+    Aggregate,
+    #[sea_orm(string_value = "detailed")]
+    Detailed,
+}
+
+impl Default for LogLevel {
+    fn default() -> Self {
+        Self::None
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
+#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
+pub enum Method {
+    #[sea_orm(string_value = "eth_call")]
+    EthCall,
+    #[sea_orm(string_value = "eth_estimateGas")]
+    EthEstimateGas,
+    #[sea_orm(string_value = "eth_sendRawTransaction")]
+    EthSendRawTransaction,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "role")]
 pub enum Role {
@@ -13,13 +41,3 @@ pub enum Role {
     #[sea_orm(string_value = "collaborator")]
     Collaborator,
 }
-#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
-pub enum Method {
-    #[sea_orm(string_value = "eth_call")]
-    EthCall,
-    #[sea_orm(string_value = "eth_estimateGas")]
-    EthEstimateGas,
-    #[sea_orm(string_value = "eth_sendRawTransaction")]
-    EthSendRawTransaction,
-}

migration/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "migration"
-version = "0.10.0"
+version = "0.11.0"
 edition = "2021"
 publish = false
 

migration/src/lib.rs

@@ -10,6 +10,7 @@ mod m20221027_002407_user_tiers;
 mod m20221031_211916_clean_up;
 mod m20221101_222349_archive_request;
 mod m20221108_200345_save_anon_stats;
+mod m20221211_124002_request_method_privacy;
 
 pub struct Migrator;
 
@@ -27,6 +28,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20221031_211916_clean_up::Migration),
             Box::new(m20221101_222349_archive_request::Migration),
             Box::new(m20221108_200345_save_anon_stats::Migration),
+            Box::new(m20221211_124002_request_method_privacy::Migration),
         ]
     }
 }

web3_proxy/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "web3_proxy"
-version = "0.10.0"
+version = "0.11.0"
 edition = "2021"
 default-run = "web3_proxy"
 

web3_proxy/src/app/mod.rs

@@ -17,6 +17,7 @@ use anyhow::Context;
 use axum::headers::{Origin, Referer, UserAgent};
 use deferred_rate_limiter::DeferredRateLimiter;
 use derive_more::From;
+use entities::sea_orm_active_enums::LogLevel;
 use ethers::core::utils::keccak256;
 use ethers::prelude::{Address, Block, Bytes, TxHash, H256, U64};
 use futures::future::join_all;
@@ -84,6 +85,7 @@ pub struct AuthorizationChecks {
     pub allowed_user_agents: Option<Vec<UserAgent>>,
     /// if None, allow any IP Address
     pub allowed_ips: Option<Vec<IpNet>>,
+    pub log_level: LogLevel,
     /// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database.
     /// TODO: f32 would be fine
     pub log_revert_chance: f64,
@@ -180,6 +182,8 @@ pub async fn drop_migration_lock(db_conn: &DatabaseConnection) -> Result<(), DbE
 
     db_conn.execute(drop_lock_statment).await?;
 
+    debug!("migration lock unlocked");
+
     Ok(())
 }
 

web3_proxy/src/app_stats.rs

@@ -3,6 +3,7 @@ use axum::headers::Origin;
 use chrono::{TimeZone, Utc};
 use derive_more::From;
 use entities::rpc_accounting;
+use entities::sea_orm_active_enums::LogLevel;
 use hashbrown::HashMap;
 use hdrhistogram::{Histogram, RecordError};
 use log::{error, info};
@@ -34,7 +35,7 @@ impl ProxyResponseStat {
     /// TODO: think more about this. probably rename it
     fn key(&self) -> ProxyResponseAggregateKey {
         // include either the rpc_key_id or the origin
-        let (rpc_key_id, origin) = match (
+        let (mut rpc_key_id, origin) = match (
             self.authorization.checks.rpc_key_id,
             &self.authorization.origin,
         ) {
@@ -52,10 +53,27 @@ impl ProxyResponseStat {
             }
         };
 
+        let method = match self.authorization.checks.log_level {
+            LogLevel::None => {
+                // No rpc_key logging. Only save fully anonymized metric
+                rpc_key_id = None;
+                // keep the method since the rpc key is not attached
+                Some(self.method.clone())
+            }
+            LogLevel::Aggregate => {
+                // Lose the method
+                None
+            }
+            LogLevel::Detailed => {
+                // include the method
+                Some(self.method.clone())
+            }
+        };
+
         ProxyResponseAggregateKey {
             archive_request: self.archive_request,
             error_response: self.error_response,
-            method: self.method.clone(),
+            method,
             origin,
             rpc_key_id,
         }
@@ -89,7 +107,7 @@ struct ProxyResponseAggregateKey {
     archive_request: bool,
     error_response: bool,
     rpc_key_id: Option<NonZeroU64>,
-    method: String,
+    method: Option<String>,
     /// TODO: should this be Origin or String?
     origin: Option<Origin>,
 }

web3_proxy/src/frontend/authorization.rs

@@ -46,7 +46,7 @@ pub enum RateLimitResult {
 }
 
 #[derive(Clone, Debug)]
-pub enum AuthorizatioType {
+pub enum AuthorizationType {
     Internal,
     Frontend,
 }
@@ -60,7 +60,7 @@ pub struct Authorization {
     pub origin: Option<Origin>,
     pub referer: Option<Referer>,
     pub user_agent: Option<UserAgent>,
-    pub authorization_type: AuthorizatioType,
+    pub authorization_type: AuthorizationType,
 }
 
 #[derive(Debug)]
@@ -188,7 +188,7 @@ impl Authorization {
             None,
             None,
             user_agent,
-            AuthorizatioType::Internal,
+            AuthorizationType::Internal,
         )
     }
 
@@ -223,7 +223,7 @@ impl Authorization {
             origin,
             referer,
             user_agent,
-            AuthorizatioType::Frontend,
+            AuthorizationType::Frontend,
         )
     }
 
@@ -234,7 +234,7 @@ impl Authorization {
         origin: Option<Origin>,
         referer: Option<Referer>,
         user_agent: Option<UserAgent>,
-        authorization_type: AuthorizatioType,
+        authorization_type: AuthorizationType,
     ) -> anyhow::Result<Self> {
         // check ip
         match &authorization_checks.allowed_ips {
@@ -664,6 +664,7 @@ impl Web3ProxyApp {
                             allowed_origins,
                             allowed_referers,
                             allowed_user_agents,
+                            log_level: rpc_key_model.log_level,
                             log_revert_chance: rpc_key_model.log_revert_chance,
                             max_concurrent_requests: user_tier_model.max_concurrent_requests,
                             max_requests_per_period: user_tier_model.max_requests_per_period,
@@ -705,7 +706,7 @@ impl Web3ProxyApp {
             origin,
             referer,
             user_agent,
-            AuthorizatioType::Frontend,
+            AuthorizationType::Frontend,
         )?;
 
         let user_max_requests_per_period = match authorization.checks.max_requests_per_period {

web3_proxy/src/frontend/users.rs

@@ -18,6 +18,7 @@ use axum::{
 };
 use axum_client_ip::ClientIp;
 use axum_macros::debug_handler;
+use entities::sea_orm_active_enums::LogLevel;
 use entities::{revert_log, rpc_key, user};
 use ethers::{prelude::Address, types::Bytes};
 use hashbrown::HashMap;
@@ -26,8 +27,8 @@ use ipnet::IpNet;
 use itertools::Itertools;
 use log::warn;
 use migration::sea_orm::{
-    self, ActiveModelTrait, ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder,
+    self, ActiveModelTrait, ColumnTrait, EntityTrait, IntoActiveModel, PaginatorTrait, QueryFilter,
-    TransactionTrait, TryIntoModel,
+    QueryOrder, TransactionTrait, TryIntoModel,
 };
 use redis_rate_limiter::redis::AsyncCommands;
 use serde::Deserialize;
@@ -502,23 +503,24 @@ pub async fn rpc_keys_delete(
 /// the JSON input to the `rpc_keys_management` handler.
 /// If `key_id` is set, it updates an existing key.
 /// If `key_id` is not set, it creates a new key.
+/// `log_request_method` cannot be change once the key is created
+/// `user_tier` cannot be changed here
 #[derive(Debug, Deserialize)]
 pub struct UserKeyManagement {
     key_id: Option<u64>,
-    description: Option<String>,
-    private_txs: Option<bool>,
     active: Option<bool>,
-    // TODO: enable log_revert_trace: Option<f64>,
     allowed_ips: Option<String>,
     allowed_origins: Option<String>,
     allowed_referers: Option<String>,
     allowed_user_agents: Option<String>,
-    // do not allow! `user_tier: Option<u64>,`
+    description: Option<String>,
+    log_level: Option<LogLevel>,
+    // TODO: enable log_revert_trace: Option<f64>,
+    private_txs: Option<bool>,
 }
 
 /// `POST /user/keys` or `PUT /user/keys` -- Use a bearer token to create or update an existing key.
 #[debug_handler]
-
 pub async fn rpc_keys_management(
     Extension(app): Extension<Arc<Web3ProxyApp>>,
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
@@ -532,15 +534,14 @@ pub async fn rpc_keys_management(
 
     let mut uk = if let Some(existing_key_id) = payload.key_id {
         // get the key and make sure it belongs to the user
-        let uk = rpc_key::Entity::find()
+        rpc_key::Entity::find()
             .filter(rpc_key::Column::UserId.eq(user.id))
             .filter(rpc_key::Column::Id.eq(existing_key_id))
             .one(&db_conn)
             .await
             .context("failed loading user's key")?
-            .context("key does not exist or is not controlled by this bearer token")?;
+            .context("key does not exist or is not controlled by this bearer token")?
-
+            .into_active_model()
-        uk.try_into().unwrap()
     } else {
         // make a new key
         // TODO: limit to 10 keys?
@@ -549,6 +550,7 @@ pub async fn rpc_keys_management(
         rpc_key::ActiveModel {
             user_id: sea_orm::Set(user.id),
             secret_key: sea_orm::Set(secret_key.into()),
+            log_level: sea_orm::Set(payload.log_level.unwrap_or(LogLevel::None)),
             ..Default::default()
         }
     };

web3_proxy/src/rpcs/connections.rs

@@ -10,7 +10,6 @@ use crate::config::{BlockAndRpc, TxHashAndRpc, Web3ConnectionConfig};
 use crate::frontend::authorization::{Authorization, RequestMetadata};
 use crate::jsonrpc::{JsonRpcForwardedResponse, JsonRpcRequest};
 use crate::rpcs::transactions::TxStatus;
-use anyhow::Context;
 use arc_swap::ArcSwap;
 use counter::Counter;
 use derive_more::From;

web3_proxy/src/rpcs/request.rs

@@ -1,6 +1,6 @@
 use super::connection::Web3Connection;
 use super::provider::Web3Provider;
-use crate::frontend::authorization::{AuthorizatioType, Authorization};
+use crate::frontend::authorization::{Authorization, AuthorizationType};
 use crate::metered::{JsonRpcErrorCount, ProviderErrorCount};
 use anyhow::Context;
 use chrono::Utc;
@@ -173,11 +173,11 @@ impl OpenRequestHandle {
         // TODO: handle overflows?
         // TODO: what ordering?
         match authorization.as_ref().authorization_type {
-            AuthorizatioType::Frontend => {
+            AuthorizationType::Frontend => {
                 conn.frontend_requests
                     .fetch_add(1, atomic::Ordering::Relaxed);
             }
-            AuthorizatioType::Internal => {
+            AuthorizationType::Internal => {
                 conn.internal_requests
                     .fetch_add(1, atomic::Ordering::Relaxed);
             }