change default tracking in prep for premium-only keys

Cargo.lock

@@ -1676,7 +1676,7 @@ dependencies = [
 
 [[package]]
 name = "entities"
-version = "0.31.0"
+version = "0.32.0"
 dependencies = [
  "ethers",
  "sea-orm",
@@ -3330,7 +3330,7 @@ dependencies = [
 
 [[package]]
 name = "migration"
-version = "0.31.0"
+version = "0.32.0"
 dependencies = [
  "sea-orm-migration",
  "tokio",

entities/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "entities"
-version = "0.31.0"
+version = "0.32.0"
 edition = "2021"
 
 [lib]

entities/src/rpc_key.rs

@@ -1,6 +1,5 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7
 
-use super::sea_orm_active_enums::TrackingLevel;
 use crate::serialization;
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
@@ -26,8 +25,6 @@ pub struct Model {
     #[sea_orm(column_type = "Text", nullable)]
     pub allowed_user_agents: Option<String>,
     pub log_revert_chance: f64,
-    // TODO: rename this with a migration
-    pub log_level: TrackingLevel,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

entities/src/sea_orm_active_enums.rs

@@ -3,25 +3,6 @@
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
-// TODO: rename TrackingLevel to StatLevel? AccountingLevel? What?
-#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "log_level")]
-pub enum TrackingLevel {
-    /// TODO: rename to minimal
-    #[sea_orm(string_value = "none")]
-    None,
-    #[sea_orm(string_value = "aggregated")]
-    Aggregated,
-    #[sea_orm(string_value = "detailed")]
-    Detailed,
-}
-
-impl Default for TrackingLevel {
-    fn default() -> Self {
-        Self::None
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
 pub enum Method {

migration/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "migration"
-version = "0.31.0"
+version = "0.32.0"
 edition = "2021"
 publish = false
 

migration/src/lib.rs

@@ -31,6 +31,7 @@ mod m20230514_114803_admin_add_credits;
 mod m20230607_221917_total_deposits;
 mod m20230615_221201_handle_payment_uncles;
 mod m20230618_230611_longer_payload;
+mod m20230619_172237_default_tracking;
 
 pub struct Migrator;
 
@@ -69,6 +70,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20230607_221917_total_deposits::Migration),
             Box::new(m20230615_221201_handle_payment_uncles::Migration),
             Box::new(m20230618_230611_longer_payload::Migration),
+            Box::new(m20230619_172237_default_tracking::Migration),
         ]
     }
 }

migration/src/m20230619_172237_default_tracking.rs

@@ -0,0 +1,63 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // new keys get set to aggregated logging
+        // TODO: rename "none" to "minimal"
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(RpcKey::Table)
+                    .modify_column(
+                        ColumnDef::new(RpcKey::LogLevel)
+                            .enumeration(
+                                Alias::new("log_level"),
+                                [
+                                    Alias::new("none"),
+                                    Alias::new("aggregated"),
+                                    Alias::new("detailed"),
+                                ],
+                            )
+                            .not_null()
+                            .default("detailed"),
+                    )
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // new keys get set to none logging
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(RpcKey::Table)
+                    .modify_column(
+                        ColumnDef::new(RpcKey::LogLevel)
+                            .enumeration(
+                                Alias::new("log_level"),
+                                [
+                                    Alias::new("none"),
+                                    Alias::new("aggregated"),
+                                    Alias::new("detailed"),
+                                ],
+                            )
+                            .not_null()
+                            .default("none"),
+                    )
+                    .to_owned(),
+            )
+            .await
+    }
+}
+
+/// Learn more at https://docs.rs/sea-query#iden
+#[derive(Iden)]
+enum RpcKey {
+    Table,
+    LogLevel,
+}

web3_proxy/src/frontend/authorization.rs

@@ -14,7 +14,6 @@ use chrono::Utc;
 use core::fmt;
 use deferred_rate_limiter::DeferredRateLimitResult;
 use derive_more::From;
-use entities::sea_orm_active_enums::TrackingLevel;
 use entities::{balance, login, rpc_key, user, user_tier};
 use ethers::types::{Bytes, U64};
 use ethers::utils::keccak256;
@@ -111,8 +110,6 @@ pub struct AuthorizationChecks {
     pub allowed_user_agents: Option<Vec<UserAgent>>,
     /// if None, allow any IP Address
     pub allowed_ips: Option<Vec<IpNet>>,
-    /// how detailed any rpc account entries should be
-    pub tracking_level: TrackingLevel,
     /// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database.
     /// depending on the caller, errors might be expected. this keeps us from bloating our database
     /// u16::MAX == 100%
@@ -523,28 +520,6 @@ impl RequestMetadata {
         self.backend_requests.lock().clone()
     }
 
-    pub fn tracking_level(&self) -> TrackingLevel {
-        if let Some(authorization) = self.authorization.as_ref() {
-            authorization.checks.tracking_level.clone()
-        } else {
-            TrackingLevel::None
-        }
-    }
-
-    pub fn opt_in_method(&self) -> Option<String> {
-        match self.tracking_level() {
-            TrackingLevel::None | TrackingLevel::Aggregated => None,
-            TrackingLevel::Detailed => self.method.clone(),
-        }
-    }
-
-    pub fn take_opt_in_method(&mut self) -> Option<String> {
-        match self.tracking_level() {
-            TrackingLevel::None | TrackingLevel::Aggregated => None,
-            TrackingLevel::Detailed => self.method.take(),
-        }
-    }
-
     pub fn try_send_stat(mut self) -> Web3ProxyResult<Option<Self>> {
         if let Some(stat_sender) = self.stat_sender.take() {
             trace!("sending stat! {:?}", self);
@@ -693,7 +668,6 @@ impl Authorization {
         let authorization_checks = AuthorizationChecks {
             // any error logs on a local (internal) query are likely problems. log them all
             log_revert_chance: 100,
-            tracking_level: TrackingLevel::Detailed,
             // default for everything else should be fine. we don't have a user_id or ip to give
             ..Default::default()
         };
@@ -734,7 +708,6 @@ impl Authorization {
         let authorization_checks = AuthorizationChecks {
             max_requests_per_period,
             proxy_mode,
-            tracking_level: TrackingLevel::Detailed,
             ..Default::default()
         };
 
@@ -1383,7 +1356,6 @@ impl Web3ProxyApp {
                             proxy_mode,
                             rpc_secret_key: Some(rpc_secret_key),
                             rpc_secret_key_id: rpc_key_id,
-                            tracking_level: rpc_key_model.log_level,
                             user_id: rpc_key_model.user_id,
                         })
                     }

web3_proxy/src/frontend/users/rpc_keys.rs

@@ -10,7 +10,7 @@ use axum::{
 };
 use axum_macros::debug_handler;
 use entities;
-use entities::sea_orm_active_enums::{Role, TrackingLevel};
+use entities::sea_orm_active_enums::Role;
 use entities::{rpc_key, secondary_user};
 use hashbrown::HashMap;
 use http::HeaderValue;
@@ -100,7 +100,6 @@ pub struct UserKeyManagement {
     allowed_referers: Option<String>,
     allowed_user_agents: Option<String>,
     description: Option<String>,
-    log_level: Option<TrackingLevel>,
     // TODO: enable log_revert_trace: Option<f64>,
     private_txs: Option<bool>,
 }
@@ -169,14 +168,9 @@ pub async fn rpc_keys_management(
             // TODO: limit to 10 keys?
             let secret_key = RpcSecretKey::new();
 
-            let log_level = payload
-                .log_level
-                .web3_context("log level must be 'none', 'detailed', or 'aggregated'")?;
-
             Ok(rpc_key::ActiveModel {
                 user_id: sea_orm::Set(user.id),
                 secret_key: sea_orm::Set(secret_key.into()),
-                log_level: sea_orm::Set(log_level),
                 ..Default::default()
             })
         }

web3_proxy/src/stats/mod.rs

@@ -14,7 +14,6 @@ use anyhow::{anyhow, Context};
 use axum::headers::Origin;
 use chrono::{DateTime, Months, TimeZone, Utc};
 use derive_more::From;
-use entities::sea_orm_active_enums::TrackingLevel;
 use entities::{balance, referee, referrer, rpc_accounting_v2, rpc_key};
 use influxdb2::models::DataPoint;
 use log::trace;
@@ -85,7 +84,7 @@ fn round_timestamp(timestamp: i64, period_seconds: i64) -> i64 {
 
 impl RpcQueryStats {
     /// rpc keys can opt into multiple levels of tracking.
-    /// we always need enough to handle billing, so even the "none" level still has some minimal tracking.
+    /// we always need enough to handle billing, so the "none" level was changed to "minimal" tracking.
     /// This "accounting_key" is used in the relational database.
     /// anonymous users are also saved in the relational database so that the host can do their own cost accounting.
     fn accounting_key(&self, period_seconds: i64) -> RpcQueryKey {
@@ -93,29 +92,10 @@ impl RpcQueryStats {
 
         let rpc_secret_key_id = self.authorization.checks.rpc_secret_key_id;
 
-        let (method, origin) = match self.authorization.checks.tracking_level {
+        let method = self.method.clone();
-            TrackingLevel::None => {
-                // this RPC key requested no tracking. this is the default
-                // do not store the method or the origin
-                (None, None)
-            }
-            TrackingLevel::Aggregated => {
-                // this RPC key requested tracking aggregated across all methods and origins
-                // TODO: think about this more. do we want the origin or not? grouping free cost per site might be useful. i'd rather not collect things if we don't have a planned purpose though
-                let method = None;
-                let origin = None;
 
-                (method, origin)
+        // we used to optionally store origin, but wallets don't set it, so its almost always None
-            }
+        let origin = None;
-            TrackingLevel::Detailed => {
-                // detailed tracking keeps track of the method and origin
-                // depending on the request, the origin might still be None
-                let method = self.method.clone();
-                let origin = self.authorization.origin.clone();
-
-                (method, origin)
-            }
-        };
 
         // Depending on method, add some arithmetic around calculating credits_used
         // I think balance should not go here, this looks more like a key thingy
@@ -151,26 +131,12 @@ impl RpcQueryStats {
         }
     }
 
-    /// rpc keys can opt into more detailed tracking
+    /// stats for a single key
-    fn opt_in_timeseries_key(&self) -> Option<RpcQueryKey> {
+    fn owned_timeseries_key(&self) -> Option<RpcQueryKey> {
         // we don't store origin in the timeseries db. its only optionaly used for accounting
         let origin = None;
 
-        // depending on tracking level, we either skip opt-in stats, track without method, or track with method
+        let method = self.method.clone();
-        let method = match self.authorization.checks.tracking_level {
-            TrackingLevel::None => {
-                // this RPC key requested no tracking. this is the default.
-                return None;
-            }
-            TrackingLevel::Aggregated => {
-                // this RPC key requested tracking aggregated across all methods
-                None
-            }
-            TrackingLevel::Detailed => {
-                // detailed tracking keeps track of the method
-                self.method.clone()
-            }
-        };
 
         let key = RpcQueryKey {
             response_timestamp: self.response_timestamp,

web3_proxy/src/stats/stat_buffer.rs

@@ -122,7 +122,7 @@ impl StatBuffer {
 
                                 self.global_timeseries_buffer.entry(global_timeseries_key).or_default().add(stat.clone());
 
-                                if let Some(opt_in_timeseries_key) = stat.opt_in_timeseries_key() {
+                                if let Some(opt_in_timeseries_key) = stat.owned_timeseries_key() {
                                     self.opt_in_timeseries_buffer.entry(opt_in_timeseries_key).or_default().add(stat.clone());
                                 }
                             }