From 6038351cb8844705cc2d7772c261e8efb6303b8a Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Mon, 19 Jun 2023 13:00:57 -0700 Subject: [PATCH] change default tracking in prep for premium-only keys --- Cargo.lock | 4 +- entities/Cargo.toml | 2 +- entities/src/rpc_key.rs | 3 - entities/src/sea_orm_active_enums.rs | 19 ------ migration/Cargo.toml | 2 +- migration/src/lib.rs | 2 + .../src/m20230619_172237_default_tracking.rs | 63 +++++++++++++++++++ web3_proxy/src/frontend/authorization.rs | 28 --------- web3_proxy/src/frontend/users/rpc_keys.rs | 8 +-- web3_proxy/src/stats/mod.rs | 48 +++----------- web3_proxy/src/stats/stat_buffer.rs | 2 +- 11 files changed, 78 insertions(+), 103 deletions(-) create mode 100644 migration/src/m20230619_172237_default_tracking.rs diff --git a/Cargo.lock b/Cargo.lock index d551fc1a..c2f7f8f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "entities" -version = "0.31.0" +version = "0.32.0" dependencies = [ "ethers", "sea-orm", @@ -3330,7 +3330,7 @@ dependencies = [ [[package]] name = "migration" -version = "0.31.0" +version = "0.32.0" dependencies = [ "sea-orm-migration", "tokio", diff --git a/entities/Cargo.toml b/entities/Cargo.toml index afd8dad2..459f573a 100644 --- a/entities/Cargo.toml +++ b/entities/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "entities" -version = "0.31.0" +version = "0.32.0" edition = "2021" [lib] diff --git a/entities/src/rpc_key.rs b/entities/src/rpc_key.rs index 547aabdf..8d35bf48 100644 --- a/entities/src/rpc_key.rs +++ b/entities/src/rpc_key.rs @@ -1,6 +1,5 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7 -use super::sea_orm_active_enums::TrackingLevel; use crate::serialization; use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; @@ -26,8 +25,6 @@ pub struct Model { #[sea_orm(column_type = "Text", nullable)] pub allowed_user_agents: Option, pub log_revert_chance: f64, - // TODO: rename this with a migration - pub log_level: TrackingLevel, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/entities/src/sea_orm_active_enums.rs b/entities/src/sea_orm_active_enums.rs index a6e536ae..00fb46aa 100644 --- a/entities/src/sea_orm_active_enums.rs +++ b/entities/src/sea_orm_active_enums.rs @@ -3,25 +3,6 @@ use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; -// TODO: rename TrackingLevel to StatLevel? AccountingLevel? What? -#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "log_level")] -pub enum TrackingLevel { - /// TODO: rename to minimal - #[sea_orm(string_value = "none")] - None, - #[sea_orm(string_value = "aggregated")] - Aggregated, - #[sea_orm(string_value = "detailed")] - Detailed, -} - -impl Default for TrackingLevel { - fn default() -> Self { - Self::None - } -} - #[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")] pub enum Method { diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 7b07bccb..9dbae336 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "migration" -version = "0.31.0" +version = "0.32.0" edition = "2021" publish = false diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 86596da2..850aed73 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -31,6 +31,7 @@ mod m20230514_114803_admin_add_credits; mod m20230607_221917_total_deposits; mod m20230615_221201_handle_payment_uncles; mod m20230618_230611_longer_payload; +mod m20230619_172237_default_tracking; pub struct Migrator; @@ -69,6 +70,7 @@ impl MigratorTrait for Migrator { Box::new(m20230607_221917_total_deposits::Migration), Box::new(m20230615_221201_handle_payment_uncles::Migration), Box::new(m20230618_230611_longer_payload::Migration), + Box::new(m20230619_172237_default_tracking::Migration), ] } } diff --git a/migration/src/m20230619_172237_default_tracking.rs b/migration/src/m20230619_172237_default_tracking.rs new file mode 100644 index 00000000..aca4fb54 --- /dev/null +++ b/migration/src/m20230619_172237_default_tracking.rs @@ -0,0 +1,63 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // new keys get set to aggregated logging + // TODO: rename "none" to "minimal" + manager + .alter_table( + Table::alter() + .table(RpcKey::Table) + .modify_column( + ColumnDef::new(RpcKey::LogLevel) + .enumeration( + Alias::new("log_level"), + [ + Alias::new("none"), + Alias::new("aggregated"), + Alias::new("detailed"), + ], + ) + .not_null() + .default("detailed"), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // new keys get set to none logging + manager + .alter_table( + Table::alter() + .table(RpcKey::Table) + .modify_column( + ColumnDef::new(RpcKey::LogLevel) + .enumeration( + Alias::new("log_level"), + [ + Alias::new("none"), + Alias::new("aggregated"), + Alias::new("detailed"), + ], + ) + .not_null() + .default("none"), + ) + .to_owned(), + ) + .await + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum RpcKey { + Table, + LogLevel, +} diff --git a/web3_proxy/src/frontend/authorization.rs b/web3_proxy/src/frontend/authorization.rs index e0fa5023..dc2ae0f8 100644 --- a/web3_proxy/src/frontend/authorization.rs +++ b/web3_proxy/src/frontend/authorization.rs @@ -14,7 +14,6 @@ use chrono::Utc; use core::fmt; use deferred_rate_limiter::DeferredRateLimitResult; use derive_more::From; -use entities::sea_orm_active_enums::TrackingLevel; use entities::{balance, login, rpc_key, user, user_tier}; use ethers::types::{Bytes, U64}; use ethers::utils::keccak256; @@ -111,8 +110,6 @@ pub struct AuthorizationChecks { pub allowed_user_agents: Option>, /// if None, allow any IP Address pub allowed_ips: Option>, - /// how detailed any rpc account entries should be - pub tracking_level: TrackingLevel, /// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database. /// depending on the caller, errors might be expected. this keeps us from bloating our database /// u16::MAX == 100% @@ -523,28 +520,6 @@ impl RequestMetadata { self.backend_requests.lock().clone() } - pub fn tracking_level(&self) -> TrackingLevel { - if let Some(authorization) = self.authorization.as_ref() { - authorization.checks.tracking_level.clone() - } else { - TrackingLevel::None - } - } - - pub fn opt_in_method(&self) -> Option { - match self.tracking_level() { - TrackingLevel::None | TrackingLevel::Aggregated => None, - TrackingLevel::Detailed => self.method.clone(), - } - } - - pub fn take_opt_in_method(&mut self) -> Option { - match self.tracking_level() { - TrackingLevel::None | TrackingLevel::Aggregated => None, - TrackingLevel::Detailed => self.method.take(), - } - } - pub fn try_send_stat(mut self) -> Web3ProxyResult> { if let Some(stat_sender) = self.stat_sender.take() { trace!("sending stat! {:?}", self); @@ -693,7 +668,6 @@ impl Authorization { let authorization_checks = AuthorizationChecks { // any error logs on a local (internal) query are likely problems. log them all log_revert_chance: 100, - tracking_level: TrackingLevel::Detailed, // default for everything else should be fine. we don't have a user_id or ip to give ..Default::default() }; @@ -734,7 +708,6 @@ impl Authorization { let authorization_checks = AuthorizationChecks { max_requests_per_period, proxy_mode, - tracking_level: TrackingLevel::Detailed, ..Default::default() }; @@ -1383,7 +1356,6 @@ impl Web3ProxyApp { proxy_mode, rpc_secret_key: Some(rpc_secret_key), rpc_secret_key_id: rpc_key_id, - tracking_level: rpc_key_model.log_level, user_id: rpc_key_model.user_id, }) } diff --git a/web3_proxy/src/frontend/users/rpc_keys.rs b/web3_proxy/src/frontend/users/rpc_keys.rs index c07f9bc5..ece913d8 100644 --- a/web3_proxy/src/frontend/users/rpc_keys.rs +++ b/web3_proxy/src/frontend/users/rpc_keys.rs @@ -10,7 +10,7 @@ use axum::{ }; use axum_macros::debug_handler; use entities; -use entities::sea_orm_active_enums::{Role, TrackingLevel}; +use entities::sea_orm_active_enums::Role; use entities::{rpc_key, secondary_user}; use hashbrown::HashMap; use http::HeaderValue; @@ -100,7 +100,6 @@ pub struct UserKeyManagement { allowed_referers: Option, allowed_user_agents: Option, description: Option, - log_level: Option, // TODO: enable log_revert_trace: Option, private_txs: Option, } @@ -169,14 +168,9 @@ pub async fn rpc_keys_management( // TODO: limit to 10 keys? let secret_key = RpcSecretKey::new(); - let log_level = payload - .log_level - .web3_context("log level must be 'none', 'detailed', or 'aggregated'")?; - Ok(rpc_key::ActiveModel { user_id: sea_orm::Set(user.id), secret_key: sea_orm::Set(secret_key.into()), - log_level: sea_orm::Set(log_level), ..Default::default() }) } diff --git a/web3_proxy/src/stats/mod.rs b/web3_proxy/src/stats/mod.rs index 757ca8ec..86c8149f 100644 --- a/web3_proxy/src/stats/mod.rs +++ b/web3_proxy/src/stats/mod.rs @@ -14,7 +14,6 @@ use anyhow::{anyhow, Context}; use axum::headers::Origin; use chrono::{DateTime, Months, TimeZone, Utc}; use derive_more::From; -use entities::sea_orm_active_enums::TrackingLevel; use entities::{balance, referee, referrer, rpc_accounting_v2, rpc_key}; use influxdb2::models::DataPoint; use log::trace; @@ -85,7 +84,7 @@ fn round_timestamp(timestamp: i64, period_seconds: i64) -> i64 { impl RpcQueryStats { /// rpc keys can opt into multiple levels of tracking. - /// we always need enough to handle billing, so even the "none" level still has some minimal tracking. + /// we always need enough to handle billing, so the "none" level was changed to "minimal" tracking. /// This "accounting_key" is used in the relational database. /// anonymous users are also saved in the relational database so that the host can do their own cost accounting. fn accounting_key(&self, period_seconds: i64) -> RpcQueryKey { @@ -93,29 +92,10 @@ impl RpcQueryStats { let rpc_secret_key_id = self.authorization.checks.rpc_secret_key_id; - let (method, origin) = match self.authorization.checks.tracking_level { - TrackingLevel::None => { - // this RPC key requested no tracking. this is the default - // do not store the method or the origin - (None, None) - } - TrackingLevel::Aggregated => { - // this RPC key requested tracking aggregated across all methods and origins - // TODO: think about this more. do we want the origin or not? grouping free cost per site might be useful. i'd rather not collect things if we don't have a planned purpose though - let method = None; - let origin = None; + let method = self.method.clone(); - (method, origin) - } - TrackingLevel::Detailed => { - // detailed tracking keeps track of the method and origin - // depending on the request, the origin might still be None - let method = self.method.clone(); - let origin = self.authorization.origin.clone(); - - (method, origin) - } - }; + // we used to optionally store origin, but wallets don't set it, so its almost always None + let origin = None; // Depending on method, add some arithmetic around calculating credits_used // I think balance should not go here, this looks more like a key thingy @@ -151,26 +131,12 @@ impl RpcQueryStats { } } - /// rpc keys can opt into more detailed tracking - fn opt_in_timeseries_key(&self) -> Option { + /// stats for a single key + fn owned_timeseries_key(&self) -> Option { // we don't store origin in the timeseries db. its only optionaly used for accounting let origin = None; - // depending on tracking level, we either skip opt-in stats, track without method, or track with method - let method = match self.authorization.checks.tracking_level { - TrackingLevel::None => { - // this RPC key requested no tracking. this is the default. - return None; - } - TrackingLevel::Aggregated => { - // this RPC key requested tracking aggregated across all methods - None - } - TrackingLevel::Detailed => { - // detailed tracking keeps track of the method - self.method.clone() - } - }; + let method = self.method.clone(); let key = RpcQueryKey { response_timestamp: self.response_timestamp, diff --git a/web3_proxy/src/stats/stat_buffer.rs b/web3_proxy/src/stats/stat_buffer.rs index 40300b82..69e65adc 100644 --- a/web3_proxy/src/stats/stat_buffer.rs +++ b/web3_proxy/src/stats/stat_buffer.rs @@ -122,7 +122,7 @@ impl StatBuffer { self.global_timeseries_buffer.entry(global_timeseries_key).or_default().add(stat.clone()); - if let Some(opt_in_timeseries_key) = stat.opt_in_timeseries_key() { + if let Some(opt_in_timeseries_key) = stat.owned_timeseries_key() { self.opt_in_timeseries_buffer.entry(opt_in_timeseries_key).or_default().add(stat.clone()); } }