tornadocontrib/web3-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

default to less detailed stats

Browse Source
This commit is contained in:
Bryan Stitt 2022-12-11 20:39:54 -08:00
parent 86f6b16761
commit 4c2d3634c5
14 changed files with 94 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -1347,7 +1347,7 @@ dependencies = [
[[package]]
name = "entities"
version = "0.10.0"
version = "0.11.0"
dependencies = [
"sea-orm",
"serde",
@ -2720,7 +2720,7 @@ dependencies = [
[[package]]
name = "migration"
version = "0.10.0"
version = "0.11.0"
dependencies = [
"sea-orm-migration",
"tokio",
@ -5499,7 +5499,7 @@ dependencies = [
[[package]]
name = "web3_proxy"
version = "0.10.0"
version = "0.11.0"
dependencies = [
"anyhow",
"arc-swap",

2
entities/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "entities"
version = "0.10.0"
version = "0.11.0"
edition = "2021"
[lib]

2
entities/src/rpc_accounting.rs
View File

@ -10,7 +10,7 @@ pub struct Model {
pub id: u64,
pub rpc_key_id: Option<u64>,
pub chain_id: u64,
pub method: String,
pub method: Option<String>,
pub archive_request: bool,
pub error_response: bool,
pub period_datetime: DateTimeUtc,

10
entities/src/rpc_key.rs
View File

@ -1,5 +1,6 @@
//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1
use super::sea_orm_active_enums::LogLevel;
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
@ -14,6 +15,8 @@ pub struct Model {
pub description: Option<String>,
pub private_txs: bool,
pub active: bool,
pub log_level: LogLevel,
pub log_revert_chance: f64,
#[sea_orm(column_type = "Text", nullable)]
pub allowed_ips: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
@ -22,7 +25,12 @@ pub struct Model {
pub allowed_referers: Option<String>,
#[sea_orm(column_type = "Text", nullable)]
pub allowed_user_agents: Option<String>,
pub log_revert_chance: f64,
}
pub enum RpcKeyLogLevels {
None,
Basic,
Detailed,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

38
entities/src/sea_orm_active_enums.rs
View File

@ -3,6 +3,34 @@
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "log_level")]
pub enum LogLevel {
#[sea_orm(string_value = "none")]
None,
#[sea_orm(string_value = "aggregate")]
Aggregate,
#[sea_orm(string_value = "detailed")]
Detailed,
}
impl Default for LogLevel {
fn default() -> Self {
Self::None
}
}
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
pub enum Method {
#[sea_orm(string_value = "eth_call")]
EthCall,
#[sea_orm(string_value = "eth_estimateGas")]
EthEstimateGas,
#[sea_orm(string_value = "eth_sendRawTransaction")]
EthSendRawTransaction,
}
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "role")]
pub enum Role {
@ -13,13 +41,3 @@ pub enum Role {
#[sea_orm(string_value = "collaborator")]
Collaborator,
}
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
pub enum Method {
#[sea_orm(string_value = "eth_call")]
EthCall,
#[sea_orm(string_value = "eth_estimateGas")]
EthEstimateGas,
#[sea_orm(string_value = "eth_sendRawTransaction")]
EthSendRawTransaction,
}

2
migration/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "migration"
version = "0.10.0"
version = "0.11.0"
edition = "2021"
publish = false

2
migration/src/lib.rs
View File

@ -10,6 +10,7 @@ mod m20221027_002407_user_tiers;
mod m20221031_211916_clean_up;
mod m20221101_222349_archive_request;
mod m20221108_200345_save_anon_stats;
mod m20221211_124002_request_method_privacy;
pub struct Migrator;
@ -27,6 +28,7 @@ impl MigratorTrait for Migrator {
Box::new(m20221031_211916_clean_up::Migration),
Box::new(m20221101_222349_archive_request::Migration),
Box::new(m20221108_200345_save_anon_stats::Migration),
Box::new(m20221211_124002_request_method_privacy::Migration),
]
}
}

2
web3_proxy/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "web3_proxy"
version = "0.10.0"
version = "0.11.0"
edition = "2021"
default-run = "web3_proxy"

4
web3_proxy/src/app/mod.rs
View File

@ -17,6 +17,7 @@ use anyhow::Context;
use axum::headers::{Origin, Referer, UserAgent};
use deferred_rate_limiter::DeferredRateLimiter;
use derive_more::From;
use entities::sea_orm_active_enums::LogLevel;
use ethers::core::utils::keccak256;
use ethers::prelude::{Address, Block, Bytes, TxHash, H256, U64};
use futures::future::join_all;
@ -84,6 +85,7 @@ pub struct AuthorizationChecks {
pub allowed_user_agents: Option<Vec<UserAgent>>,
/// if None, allow any IP Address
pub allowed_ips: Option<Vec<IpNet>>,
pub log_level: LogLevel,
/// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database.
/// TODO: f32 would be fine
pub log_revert_chance: f64,
@ -180,6 +182,8 @@ pub async fn drop_migration_lock(db_conn: &DatabaseConnection) -> Result<(), DbE
db_conn.execute(drop_lock_statment).await?;
debug!("migration lock unlocked");
Ok(())
}

24
web3_proxy/src/app_stats.rs
View File

@ -3,6 +3,7 @@ use axum::headers::Origin;
use chrono::{TimeZone, Utc};
use derive_more::From;
use entities::rpc_accounting;
use entities::sea_orm_active_enums::LogLevel;
use hashbrown::HashMap;
use hdrhistogram::{Histogram, RecordError};
use log::{error, info};
@ -34,7 +35,7 @@ impl ProxyResponseStat {
/// TODO: think more about this. probably rename it
fn key(&self) -> ProxyResponseAggregateKey {
// include either the rpc_key_id or the origin
let (rpc_key_id, origin) = match (
let (mut rpc_key_id, origin) = match (
self.authorization.checks.rpc_key_id,
&self.authorization.origin,
) {
@ -52,10 +53,27 @@ impl ProxyResponseStat {
}
};
let method = match self.authorization.checks.log_level {
LogLevel::None => {
// No rpc_key logging. Only save fully anonymized metric
rpc_key_id = None;
// keep the method since the rpc key is not attached
Some(self.method.clone())
}
LogLevel::Aggregate => {
// Lose the method
None
}
LogLevel::Detailed => {
// include the method
Some(self.method.clone())
}
};
ProxyResponseAggregateKey {
archive_request: self.archive_request,
error_response: self.error_response,
method: self.method.clone(),
method,
origin,
rpc_key_id,
}
@ -89,7 +107,7 @@ struct ProxyResponseAggregateKey {
archive_request: bool,
error_response: bool,
rpc_key_id: Option<NonZeroU64>,
method: String,
method: Option<String>,
/// TODO: should this be Origin or String?
origin: Option<Origin>,
}

13
web3_proxy/src/frontend/authorization.rs
View File

@ -46,7 +46,7 @@ pub enum RateLimitResult {
}
#[derive(Clone, Debug)]
pub enum AuthorizatioType {
pub enum AuthorizationType {
Internal,
Frontend,
}
@ -60,7 +60,7 @@ pub struct Authorization {
pub origin: Option<Origin>,
pub referer: Option<Referer>,
pub user_agent: Option<UserAgent>,
pub authorization_type: AuthorizatioType,
pub authorization_type: AuthorizationType,
}
#[derive(Debug)]
@ -188,7 +188,7 @@ impl Authorization {
None,
None,
user_agent,
AuthorizatioType::Internal,
AuthorizationType::Internal,
)
}
@ -223,7 +223,7 @@ impl Authorization {
origin,
referer,
user_agent,
AuthorizatioType::Frontend,
AuthorizationType::Frontend,
)
}
@ -234,7 +234,7 @@ impl Authorization {
origin: Option<Origin>,
referer: Option<Referer>,
user_agent: Option<UserAgent>,
authorization_type: AuthorizatioType,
authorization_type: AuthorizationType,
) -> anyhow::Result<Self> {
// check ip
match &authorization_checks.allowed_ips {
@ -664,6 +664,7 @@ impl Web3ProxyApp {
allowed_origins,
allowed_referers,
allowed_user_agents,
log_level: rpc_key_model.log_level,
log_revert_chance: rpc_key_model.log_revert_chance,
max_concurrent_requests: user_tier_model.max_concurrent_requests,
max_requests_per_period: user_tier_model.max_requests_per_period,
@ -705,7 +706,7 @@ impl Web3ProxyApp {
origin,
referer,
user_agent,
AuthorizatioType::Frontend,
AuthorizationType::Frontend,
)?;
let user_max_requests_per_period = match authorization.checks.max_requests_per_period {

24
web3_proxy/src/frontend/users.rs
View File

@ -18,6 +18,7 @@ use axum::{
};
use axum_client_ip::ClientIp;
use axum_macros::debug_handler;
use entities::sea_orm_active_enums::LogLevel;
use entities::{revert_log, rpc_key, user};
use ethers::{prelude::Address, types::Bytes};
use hashbrown::HashMap;
@ -26,8 +27,8 @@ use ipnet::IpNet;
use itertools::Itertools;
use log::warn;
use migration::sea_orm::{
self, ActiveModelTrait, ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder,
TransactionTrait, TryIntoModel,
self, ActiveModelTrait, ColumnTrait, EntityTrait, IntoActiveModel, PaginatorTrait, QueryFilter,
QueryOrder, TransactionTrait, TryIntoModel,
};
use redis_rate_limiter::redis::AsyncCommands;
use serde::Deserialize;
@ -502,23 +503,24 @@ pub async fn rpc_keys_delete(
/// the JSON input to the `rpc_keys_management` handler.
/// If `key_id` is set, it updates an existing key.
/// If `key_id` is not set, it creates a new key.
/// `log_request_method` cannot be change once the key is created
/// `user_tier` cannot be changed here
#[derive(Debug, Deserialize)]
pub struct UserKeyManagement {
key_id: Option<u64>,
description: Option<String>,
private_txs: Option<bool>,
active: Option<bool>,
// TODO: enable log_revert_trace: Option<f64>,
allowed_ips: Option<String>,
allowed_origins: Option<String>,
allowed_referers: Option<String>,
allowed_user_agents: Option<String>,
// do not allow! `user_tier: Option<u64>,`
description: Option<String>,
log_level: Option<LogLevel>,
// TODO: enable log_revert_trace: Option<f64>,
private_txs: Option<bool>,
}
/// `POST /user/keys` or `PUT /user/keys` -- Use a bearer token to create or update an existing key.
#[debug_handler]
pub async fn rpc_keys_management(
Extension(app): Extension<Arc<Web3ProxyApp>>,
TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
@ -532,15 +534,14 @@ pub async fn rpc_keys_management(
let mut uk = if let Some(existing_key_id) = payload.key_id {
// get the key and make sure it belongs to the user
let uk = rpc_key::Entity::find()
rpc_key::Entity::find()
.filter(rpc_key::Column::UserId.eq(user.id))
.filter(rpc_key::Column::Id.eq(existing_key_id))
.one(&db_conn)
.await
.context("failed loading user's key")?
.context("key does not exist or is not controlled by this bearer token")?;
uk.try_into().unwrap()
.context("key does not exist or is not controlled by this bearer token")?
.into_active_model()
} else {
// make a new key
// TODO: limit to 10 keys?
@ -549,6 +550,7 @@ pub async fn rpc_keys_management(
rpc_key::ActiveModel {
user_id: sea_orm::Set(user.id),
secret_key: sea_orm::Set(secret_key.into()),
log_level: sea_orm::Set(payload.log_level.unwrap_or(LogLevel::None)),
..Default::default()
}
};

1
web3_proxy/src/rpcs/connections.rs
View File

@ -10,7 +10,6 @@ use crate::config::{BlockAndRpc, TxHashAndRpc, Web3ConnectionConfig};
use crate::frontend::authorization::{Authorization, RequestMetadata};
use crate::jsonrpc::{JsonRpcForwardedResponse, JsonRpcRequest};
use crate::rpcs::transactions::TxStatus;
use anyhow::Context;
use arc_swap::ArcSwap;
use counter::Counter;
use derive_more::From;

6
web3_proxy/src/rpcs/request.rs
View File

@ -1,6 +1,6 @@
use super::connection::Web3Connection;
use super::provider::Web3Provider;
use crate::frontend::authorization::{AuthorizatioType, Authorization};
use crate::frontend::authorization::{Authorization, AuthorizationType};
use crate::metered::{JsonRpcErrorCount, ProviderErrorCount};
use anyhow::Context;
use chrono::Utc;
@ -173,11 +173,11 @@ impl OpenRequestHandle {
// TODO: handle overflows?
// TODO: what ordering?
match authorization.as_ref().authorization_type {
AuthorizatioType::Frontend => {
AuthorizationType::Frontend => {
conn.frontend_requests
.fetch_add(1, atomic::Ordering::Relaxed);
}
AuthorizatioType::Internal => {
AuthorizationType::Internal => {
conn.internal_requests
.fetch_add(1, atomic::Ordering::Relaxed);
}