From 4c2d3634c5ec680af721b72e2cbeb8d4ee4bdacc Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Sun, 11 Dec 2022 20:39:54 -0800 Subject: [PATCH] default to less detailed stats --- Cargo.lock | 6 ++-- entities/Cargo.toml | 2 +- entities/src/rpc_accounting.rs | 2 +- entities/src/rpc_key.rs | 10 ++++++- entities/src/sea_orm_active_enums.rs | 38 +++++++++++++++++------- migration/Cargo.toml | 2 +- migration/src/lib.rs | 2 ++ web3_proxy/Cargo.toml | 2 +- web3_proxy/src/app/mod.rs | 4 +++ web3_proxy/src/app_stats.rs | 24 +++++++++++++-- web3_proxy/src/frontend/authorization.rs | 13 ++++---- web3_proxy/src/frontend/users.rs | 24 ++++++++------- web3_proxy/src/rpcs/connections.rs | 1 - web3_proxy/src/rpcs/request.rs | 6 ++-- 14 files changed, 94 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53cb1efd..b46e0886 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1347,7 +1347,7 @@ dependencies = [ [[package]] name = "entities" -version = "0.10.0" +version = "0.11.0" dependencies = [ "sea-orm", "serde", @@ -2720,7 +2720,7 @@ dependencies = [ [[package]] name = "migration" -version = "0.10.0" +version = "0.11.0" dependencies = [ "sea-orm-migration", "tokio", @@ -5499,7 +5499,7 @@ dependencies = [ [[package]] name = "web3_proxy" -version = "0.10.0" +version = "0.11.0" dependencies = [ "anyhow", "arc-swap", diff --git a/entities/Cargo.toml b/entities/Cargo.toml index 821037bb..71846a0b 100644 --- a/entities/Cargo.toml +++ b/entities/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "entities" -version = "0.10.0" +version = "0.11.0" edition = "2021" [lib] diff --git a/entities/src/rpc_accounting.rs b/entities/src/rpc_accounting.rs index 15fe09b0..88e3fa5c 100644 --- a/entities/src/rpc_accounting.rs +++ b/entities/src/rpc_accounting.rs @@ -10,7 +10,7 @@ pub struct Model { pub id: u64, pub rpc_key_id: Option, pub chain_id: u64, - pub method: String, + pub method: Option, pub archive_request: bool, pub error_response: bool, pub period_datetime: DateTimeUtc, diff --git a/entities/src/rpc_key.rs b/entities/src/rpc_key.rs index 4e93745f..6038d242 100644 --- a/entities/src/rpc_key.rs +++ b/entities/src/rpc_key.rs @@ -1,5 +1,6 @@ //! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 +use super::sea_orm_active_enums::LogLevel; use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; @@ -14,6 +15,8 @@ pub struct Model { pub description: Option, pub private_txs: bool, pub active: bool, + pub log_level: LogLevel, + pub log_revert_chance: f64, #[sea_orm(column_type = "Text", nullable)] pub allowed_ips: Option, #[sea_orm(column_type = "Text", nullable)] @@ -22,7 +25,12 @@ pub struct Model { pub allowed_referers: Option, #[sea_orm(column_type = "Text", nullable)] pub allowed_user_agents: Option, - pub log_revert_chance: f64, +} + +pub enum RpcKeyLogLevels { + None, + Basic, + Detailed, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/entities/src/sea_orm_active_enums.rs b/entities/src/sea_orm_active_enums.rs index bd97d400..185f4901 100644 --- a/entities/src/sea_orm_active_enums.rs +++ b/entities/src/sea_orm_active_enums.rs @@ -3,6 +3,34 @@ use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; +#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "log_level")] +pub enum LogLevel { + #[sea_orm(string_value = "none")] + None, + #[sea_orm(string_value = "aggregate")] + Aggregate, + #[sea_orm(string_value = "detailed")] + Detailed, +} + +impl Default for LogLevel { + fn default() -> Self { + Self::None + } +} + +#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")] +pub enum Method { + #[sea_orm(string_value = "eth_call")] + EthCall, + #[sea_orm(string_value = "eth_estimateGas")] + EthEstimateGas, + #[sea_orm(string_value = "eth_sendRawTransaction")] + EthSendRawTransaction, +} + #[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "role")] pub enum Role { @@ -13,13 +41,3 @@ pub enum Role { #[sea_orm(string_value = "collaborator")] Collaborator, } -#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")] -pub enum Method { - #[sea_orm(string_value = "eth_call")] - EthCall, - #[sea_orm(string_value = "eth_estimateGas")] - EthEstimateGas, - #[sea_orm(string_value = "eth_sendRawTransaction")] - EthSendRawTransaction, -} diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 15f073c3..be9a7417 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "migration" -version = "0.10.0" +version = "0.11.0" edition = "2021" publish = false diff --git a/migration/src/lib.rs b/migration/src/lib.rs index e8ff40d9..381099b8 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -10,6 +10,7 @@ mod m20221027_002407_user_tiers; mod m20221031_211916_clean_up; mod m20221101_222349_archive_request; mod m20221108_200345_save_anon_stats; +mod m20221211_124002_request_method_privacy; pub struct Migrator; @@ -27,6 +28,7 @@ impl MigratorTrait for Migrator { Box::new(m20221031_211916_clean_up::Migration), Box::new(m20221101_222349_archive_request::Migration), Box::new(m20221108_200345_save_anon_stats::Migration), + Box::new(m20221211_124002_request_method_privacy::Migration), ] } } diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml index 7affefc9..a654e2ea 100644 --- a/web3_proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "web3_proxy" -version = "0.10.0" +version = "0.11.0" edition = "2021" default-run = "web3_proxy" diff --git a/web3_proxy/src/app/mod.rs b/web3_proxy/src/app/mod.rs index 30417d9c..01498076 100644 --- a/web3_proxy/src/app/mod.rs +++ b/web3_proxy/src/app/mod.rs @@ -17,6 +17,7 @@ use anyhow::Context; use axum::headers::{Origin, Referer, UserAgent}; use deferred_rate_limiter::DeferredRateLimiter; use derive_more::From; +use entities::sea_orm_active_enums::LogLevel; use ethers::core::utils::keccak256; use ethers::prelude::{Address, Block, Bytes, TxHash, H256, U64}; use futures::future::join_all; @@ -84,6 +85,7 @@ pub struct AuthorizationChecks { pub allowed_user_agents: Option>, /// if None, allow any IP Address pub allowed_ips: Option>, + pub log_level: LogLevel, /// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database. /// TODO: f32 would be fine pub log_revert_chance: f64, @@ -180,6 +182,8 @@ pub async fn drop_migration_lock(db_conn: &DatabaseConnection) -> Result<(), DbE db_conn.execute(drop_lock_statment).await?; + debug!("migration lock unlocked"); + Ok(()) } diff --git a/web3_proxy/src/app_stats.rs b/web3_proxy/src/app_stats.rs index 70551df1..09c9ae63 100644 --- a/web3_proxy/src/app_stats.rs +++ b/web3_proxy/src/app_stats.rs @@ -3,6 +3,7 @@ use axum::headers::Origin; use chrono::{TimeZone, Utc}; use derive_more::From; use entities::rpc_accounting; +use entities::sea_orm_active_enums::LogLevel; use hashbrown::HashMap; use hdrhistogram::{Histogram, RecordError}; use log::{error, info}; @@ -34,7 +35,7 @@ impl ProxyResponseStat { /// TODO: think more about this. probably rename it fn key(&self) -> ProxyResponseAggregateKey { // include either the rpc_key_id or the origin - let (rpc_key_id, origin) = match ( + let (mut rpc_key_id, origin) = match ( self.authorization.checks.rpc_key_id, &self.authorization.origin, ) { @@ -52,10 +53,27 @@ impl ProxyResponseStat { } }; + let method = match self.authorization.checks.log_level { + LogLevel::None => { + // No rpc_key logging. Only save fully anonymized metric + rpc_key_id = None; + // keep the method since the rpc key is not attached + Some(self.method.clone()) + } + LogLevel::Aggregate => { + // Lose the method + None + } + LogLevel::Detailed => { + // include the method + Some(self.method.clone()) + } + }; + ProxyResponseAggregateKey { archive_request: self.archive_request, error_response: self.error_response, - method: self.method.clone(), + method, origin, rpc_key_id, } @@ -89,7 +107,7 @@ struct ProxyResponseAggregateKey { archive_request: bool, error_response: bool, rpc_key_id: Option, - method: String, + method: Option, /// TODO: should this be Origin or String? origin: Option, } diff --git a/web3_proxy/src/frontend/authorization.rs b/web3_proxy/src/frontend/authorization.rs index 254346ad..38dc51f9 100644 --- a/web3_proxy/src/frontend/authorization.rs +++ b/web3_proxy/src/frontend/authorization.rs @@ -46,7 +46,7 @@ pub enum RateLimitResult { } #[derive(Clone, Debug)] -pub enum AuthorizatioType { +pub enum AuthorizationType { Internal, Frontend, } @@ -60,7 +60,7 @@ pub struct Authorization { pub origin: Option, pub referer: Option, pub user_agent: Option, - pub authorization_type: AuthorizatioType, + pub authorization_type: AuthorizationType, } #[derive(Debug)] @@ -188,7 +188,7 @@ impl Authorization { None, None, user_agent, - AuthorizatioType::Internal, + AuthorizationType::Internal, ) } @@ -223,7 +223,7 @@ impl Authorization { origin, referer, user_agent, - AuthorizatioType::Frontend, + AuthorizationType::Frontend, ) } @@ -234,7 +234,7 @@ impl Authorization { origin: Option, referer: Option, user_agent: Option, - authorization_type: AuthorizatioType, + authorization_type: AuthorizationType, ) -> anyhow::Result { // check ip match &authorization_checks.allowed_ips { @@ -664,6 +664,7 @@ impl Web3ProxyApp { allowed_origins, allowed_referers, allowed_user_agents, + log_level: rpc_key_model.log_level, log_revert_chance: rpc_key_model.log_revert_chance, max_concurrent_requests: user_tier_model.max_concurrent_requests, max_requests_per_period: user_tier_model.max_requests_per_period, @@ -705,7 +706,7 @@ impl Web3ProxyApp { origin, referer, user_agent, - AuthorizatioType::Frontend, + AuthorizationType::Frontend, )?; let user_max_requests_per_period = match authorization.checks.max_requests_per_period { diff --git a/web3_proxy/src/frontend/users.rs b/web3_proxy/src/frontend/users.rs index 6d780bed..2ee769fb 100644 --- a/web3_proxy/src/frontend/users.rs +++ b/web3_proxy/src/frontend/users.rs @@ -18,6 +18,7 @@ use axum::{ }; use axum_client_ip::ClientIp; use axum_macros::debug_handler; +use entities::sea_orm_active_enums::LogLevel; use entities::{revert_log, rpc_key, user}; use ethers::{prelude::Address, types::Bytes}; use hashbrown::HashMap; @@ -26,8 +27,8 @@ use ipnet::IpNet; use itertools::Itertools; use log::warn; use migration::sea_orm::{ - self, ActiveModelTrait, ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, - TransactionTrait, TryIntoModel, + self, ActiveModelTrait, ColumnTrait, EntityTrait, IntoActiveModel, PaginatorTrait, QueryFilter, + QueryOrder, TransactionTrait, TryIntoModel, }; use redis_rate_limiter::redis::AsyncCommands; use serde::Deserialize; @@ -502,23 +503,24 @@ pub async fn rpc_keys_delete( /// the JSON input to the `rpc_keys_management` handler. /// If `key_id` is set, it updates an existing key. /// If `key_id` is not set, it creates a new key. +/// `log_request_method` cannot be change once the key is created +/// `user_tier` cannot be changed here #[derive(Debug, Deserialize)] pub struct UserKeyManagement { key_id: Option, - description: Option, - private_txs: Option, active: Option, - // TODO: enable log_revert_trace: Option, allowed_ips: Option, allowed_origins: Option, allowed_referers: Option, allowed_user_agents: Option, - // do not allow! `user_tier: Option,` + description: Option, + log_level: Option, + // TODO: enable log_revert_trace: Option, + private_txs: Option, } /// `POST /user/keys` or `PUT /user/keys` -- Use a bearer token to create or update an existing key. #[debug_handler] - pub async fn rpc_keys_management( Extension(app): Extension>, TypedHeader(Authorization(bearer)): TypedHeader>, @@ -532,15 +534,14 @@ pub async fn rpc_keys_management( let mut uk = if let Some(existing_key_id) = payload.key_id { // get the key and make sure it belongs to the user - let uk = rpc_key::Entity::find() + rpc_key::Entity::find() .filter(rpc_key::Column::UserId.eq(user.id)) .filter(rpc_key::Column::Id.eq(existing_key_id)) .one(&db_conn) .await .context("failed loading user's key")? - .context("key does not exist or is not controlled by this bearer token")?; - - uk.try_into().unwrap() + .context("key does not exist or is not controlled by this bearer token")? + .into_active_model() } else { // make a new key // TODO: limit to 10 keys? @@ -549,6 +550,7 @@ pub async fn rpc_keys_management( rpc_key::ActiveModel { user_id: sea_orm::Set(user.id), secret_key: sea_orm::Set(secret_key.into()), + log_level: sea_orm::Set(payload.log_level.unwrap_or(LogLevel::None)), ..Default::default() } }; diff --git a/web3_proxy/src/rpcs/connections.rs b/web3_proxy/src/rpcs/connections.rs index 5a79e868..6497a7f5 100644 --- a/web3_proxy/src/rpcs/connections.rs +++ b/web3_proxy/src/rpcs/connections.rs @@ -10,7 +10,6 @@ use crate::config::{BlockAndRpc, TxHashAndRpc, Web3ConnectionConfig}; use crate::frontend::authorization::{Authorization, RequestMetadata}; use crate::jsonrpc::{JsonRpcForwardedResponse, JsonRpcRequest}; use crate::rpcs::transactions::TxStatus; -use anyhow::Context; use arc_swap::ArcSwap; use counter::Counter; use derive_more::From; diff --git a/web3_proxy/src/rpcs/request.rs b/web3_proxy/src/rpcs/request.rs index 6ef3c97a..08f4e647 100644 --- a/web3_proxy/src/rpcs/request.rs +++ b/web3_proxy/src/rpcs/request.rs @@ -1,6 +1,6 @@ use super::connection::Web3Connection; use super::provider::Web3Provider; -use crate::frontend::authorization::{AuthorizatioType, Authorization}; +use crate::frontend::authorization::{Authorization, AuthorizationType}; use crate::metered::{JsonRpcErrorCount, ProviderErrorCount}; use anyhow::Context; use chrono::Utc; @@ -173,11 +173,11 @@ impl OpenRequestHandle { // TODO: handle overflows? // TODO: what ordering? match authorization.as_ref().authorization_type { - AuthorizatioType::Frontend => { + AuthorizationType::Frontend => { conn.frontend_requests .fetch_add(1, atomic::Ordering::Relaxed); } - AuthorizatioType::Internal => { + AuthorizationType::Internal => { conn.internal_requests .fetch_add(1, atomic::Ordering::Relaxed); }