tornadocontrib/ web3-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change default tracking in prep for premium-only keys

This commit is contained in:
Bryan Stitt 2023-06-19 13:00:57 -07:00
parent b35cd58a76
commit 6038351cb8
11 changed files with 78 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@ -1676,7 +1676,7 @@ dependencies = [
[[package]]
name = "entities"
version = "0.31.0"
version = "0.32.0"
dependencies = [
"ethers",
"sea-orm",
@ -3330,7 +3330,7 @@ dependencies = [
[[package]]
name = "migration"
version = "0.31.0"
version = "0.32.0"
dependencies = [
"sea-orm-migration",
"tokio",

2
entities/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "entities"
version = "0.31.0"
version = "0.32.0"
edition = "2021"
[lib]

3
entities/src/rpc_key.rs
View File

@ -1,6 +1,5 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7
use super::sea_orm_active_enums::TrackingLevel;
use crate::serialization;
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
@ -26,8 +25,6 @@ pub struct Model {
#[sea_orm(column_type = "Text", nullable)]
pub allowed_user_agents: Option<String>,
pub log_revert_chance: f64,
// TODO: rename this with a migration
pub log_level: TrackingLevel,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

19
entities/src/sea_orm_active_enums.rs
View File

@ -3,25 +3,6 @@
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
// TODO: rename TrackingLevel to StatLevel? AccountingLevel? What?
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "log_level")]
pub enum TrackingLevel {
/// TODO: rename to minimal
#[sea_orm(string_value = "none")]
None,
#[sea_orm(string_value = "aggregated")]
Aggregated,
#[sea_orm(string_value = "detailed")]
Detailed,
}
impl Default for TrackingLevel {
fn default() -> Self {
Self::None
}
}
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
pub enum Method {

2
migration/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "migration"
version = "0.31.0"
version = "0.32.0"
edition = "2021"
publish = false

2
migration/src/lib.rs
View File

@ -31,6 +31,7 @@ mod m20230514_114803_admin_add_credits;
mod m20230607_221917_total_deposits;
mod m20230615_221201_handle_payment_uncles;
mod m20230618_230611_longer_payload;
mod m20230619_172237_default_tracking;
pub struct Migrator;
@ -69,6 +70,7 @@ impl MigratorTrait for Migrator {
Box::new(m20230607_221917_total_deposits::Migration),
Box::new(m20230615_221201_handle_payment_uncles::Migration),
Box::new(m20230618_230611_longer_payload::Migration),
Box::new(m20230619_172237_default_tracking::Migration),
]
}
}

63
migration/src/m20230619_172237_default_tracking.rs Normal file
View File

@ -0,0 +1,63 @@
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// new keys get set to aggregated logging
// TODO: rename "none" to "minimal"
manager
.alter_table(
Table::alter()
.table(RpcKey::Table)
.modify_column(
ColumnDef::new(RpcKey::LogLevel)
.enumeration(
Alias::new("log_level"),
[
Alias::new("none"),
Alias::new("aggregated"),
Alias::new("detailed"),
],
)
.not_null()
.default("detailed"),
)
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// new keys get set to none logging
manager
.alter_table(
Table::alter()
.table(RpcKey::Table)
.modify_column(
ColumnDef::new(RpcKey::LogLevel)
.enumeration(
Alias::new("log_level"),
[
Alias::new("none"),
Alias::new("aggregated"),
Alias::new("detailed"),
],
)
.not_null()
.default("none"),
)
.to_owned(),
)
.await
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum RpcKey {
Table,
LogLevel,
}

28
web3_proxy/src/frontend/authorization.rs
View File

@ -14,7 +14,6 @@ use chrono::Utc;
use core::fmt;
use deferred_rate_limiter::DeferredRateLimitResult;
use derive_more::From;
use entities::sea_orm_active_enums::TrackingLevel;
use entities::{balance, login, rpc_key, user, user_tier};
use ethers::types::{Bytes, U64};
use ethers::utils::keccak256;
@ -111,8 +110,6 @@ pub struct AuthorizationChecks {
pub allowed_user_agents: Option<Vec<UserAgent>>,
/// if None, allow any IP Address
pub allowed_ips: Option<Vec<IpNet>>,
/// how detailed any rpc account entries should be
pub tracking_level: TrackingLevel,
/// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database.
/// depending on the caller, errors might be expected. this keeps us from bloating our database
/// u16::MAX == 100%
@ -523,28 +520,6 @@ impl RequestMetadata {
self.backend_requests.lock().clone()
}
pub fn tracking_level(&self) -> TrackingLevel {
if let Some(authorization) = self.authorization.as_ref() {
authorization.checks.tracking_level.clone()
} else {
TrackingLevel::None
}
}
pub fn opt_in_method(&self) -> Option<String> {
match self.tracking_level() {
TrackingLevel::None | TrackingLevel::Aggregated => None,
TrackingLevel::Detailed => self.method.clone(),
}
}
pub fn take_opt_in_method(&mut self) -> Option<String> {
match self.tracking_level() {
TrackingLevel::None | TrackingLevel::Aggregated => None,
TrackingLevel::Detailed => self.method.take(),
}
}
pub fn try_send_stat(mut self) -> Web3ProxyResult<Option<Self>> {
if let Some(stat_sender) = self.stat_sender.take() {
trace!("sending stat! {:?}", self);
@ -693,7 +668,6 @@ impl Authorization {
let authorization_checks = AuthorizationChecks {
// any error logs on a local (internal) query are likely problems. log them all
log_revert_chance: 100,
tracking_level: TrackingLevel::Detailed,
// default for everything else should be fine. we don't have a user_id or ip to give
..Default::default()
};
@ -734,7 +708,6 @@ impl Authorization {
let authorization_checks = AuthorizationChecks {
max_requests_per_period,
proxy_mode,
tracking_level: TrackingLevel::Detailed,
..Default::default()
};
@ -1383,7 +1356,6 @@ impl Web3ProxyApp {
proxy_mode,
rpc_secret_key: Some(rpc_secret_key),
rpc_secret_key_id: rpc_key_id,
tracking_level: rpc_key_model.log_level,
user_id: rpc_key_model.user_id,
})
}

8
web3_proxy/src/frontend/users/rpc_keys.rs
View File

@ -10,7 +10,7 @@ use axum::{
};
use axum_macros::debug_handler;
use entities;
use entities::sea_orm_active_enums::{Role, TrackingLevel};
use entities::sea_orm_active_enums::Role;
use entities::{rpc_key, secondary_user};
use hashbrown::HashMap;
use http::HeaderValue;
@ -100,7 +100,6 @@ pub struct UserKeyManagement {
allowed_referers: Option<String>,
allowed_user_agents: Option<String>,
description: Option<String>,
log_level: Option<TrackingLevel>,
// TODO: enable log_revert_trace: Option<f64>,
private_txs: Option<bool>,
}
@ -169,14 +168,9 @@ pub async fn rpc_keys_management(
// TODO: limit to 10 keys?
let secret_key = RpcSecretKey::new();
let log_level = payload
.log_level
.web3_context("log level must be 'none', 'detailed', or 'aggregated'")?;
Ok(rpc_key::ActiveModel {
user_id: sea_orm::Set(user.id),
secret_key: sea_orm::Set(secret_key.into()),
log_level: sea_orm::Set(log_level),
..Default::default()
})
}

48
web3_proxy/src/stats/mod.rs
View File

@ -14,7 +14,6 @@ use anyhow::{anyhow, Context};
use axum::headers::Origin;
use chrono::{DateTime, Months, TimeZone, Utc};
use derive_more::From;
use entities::sea_orm_active_enums::TrackingLevel;
use entities::{balance, referee, referrer, rpc_accounting_v2, rpc_key};
use influxdb2::models::DataPoint;
use log::trace;
@ -85,7 +84,7 @@ fn round_timestamp(timestamp: i64, period_seconds: i64) -> i64 {
impl RpcQueryStats {
/// rpc keys can opt into multiple levels of tracking.
/// we always need enough to handle billing, so even the "none" level still has some minimal tracking.
/// we always need enough to handle billing, so the "none" level was changed to "minimal" tracking.
/// This "accounting_key" is used in the relational database.
/// anonymous users are also saved in the relational database so that the host can do their own cost accounting.
fn accounting_key(&self, period_seconds: i64) -> RpcQueryKey {
@ -93,29 +92,10 @@ impl RpcQueryStats {
let rpc_secret_key_id = self.authorization.checks.rpc_secret_key_id;
let (method, origin) = match self.authorization.checks.tracking_level {
TrackingLevel::None => {
// this RPC key requested no tracking. this is the default
// do not store the method or the origin
(None, None)
}
TrackingLevel::Aggregated => {
// this RPC key requested tracking aggregated across all methods and origins
// TODO: think about this more. do we want the origin or not? grouping free cost per site might be useful. i'd rather not collect things if we don't have a planned purpose though
let method = None;
let origin = None;
let method = self.method.clone();
(method, origin)
}
TrackingLevel::Detailed => {
// detailed tracking keeps track of the method and origin
// depending on the request, the origin might still be None
let method = self.method.clone();
let origin = self.authorization.origin.clone();
(method, origin)
}
};
// we used to optionally store origin, but wallets don't set it, so its almost always None
let origin = None;
// Depending on method, add some arithmetic around calculating credits_used
// I think balance should not go here, this looks more like a key thingy
@ -151,26 +131,12 @@ impl RpcQueryStats {
}
}
/// rpc keys can opt into more detailed tracking
fn opt_in_timeseries_key(&self) -> Option<RpcQueryKey> {
/// stats for a single key
fn owned_timeseries_key(&self) -> Option<RpcQueryKey> {
// we don't store origin in the timeseries db. its only optionaly used for accounting
let origin = None;
// depending on tracking level, we either skip opt-in stats, track without method, or track with method
let method = match self.authorization.checks.tracking_level {
TrackingLevel::None => {
// this RPC key requested no tracking. this is the default.
return None;
}
TrackingLevel::Aggregated => {
// this RPC key requested tracking aggregated across all methods
None
}
TrackingLevel::Detailed => {
// detailed tracking keeps track of the method
self.method.clone()
}
};
let method = self.method.clone();
let key = RpcQueryKey {
response_timestamp: self.response_timestamp,

2
web3_proxy/src/stats/stat_buffer.rs
View File

@ -122,7 +122,7 @@ impl StatBuffer {
self.global_timeseries_buffer.entry(global_timeseries_key).or_default().add(stat.clone());
if let Some(opt_in_timeseries_key) = stat.opt_in_timeseries_key() {
if let Some(opt_in_timeseries_key) = stat.owned_timeseries_key() {
self.opt_in_timeseries_buffer.entry(opt_in_timeseries_key).or_default().add(stat.clone());
}
}