change default tracking in prep for premium-only keys
This commit is contained in:
parent
b35cd58a76
commit
6038351cb8
4
Cargo.lock
generated
4
Cargo.lock
generated
@ -1676,7 +1676,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "entities"
|
name = "entities"
|
||||||
version = "0.31.0"
|
version = "0.32.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ethers",
|
"ethers",
|
||||||
"sea-orm",
|
"sea-orm",
|
||||||
@ -3330,7 +3330,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "migration"
|
name = "migration"
|
||||||
version = "0.31.0"
|
version = "0.32.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"sea-orm-migration",
|
"sea-orm-migration",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "entities"
|
name = "entities"
|
||||||
version = "0.31.0"
|
version = "0.32.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7
|
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7
|
||||||
|
|
||||||
use super::sea_orm_active_enums::TrackingLevel;
|
|
||||||
use crate::serialization;
|
use crate::serialization;
|
||||||
use sea_orm::entity::prelude::*;
|
use sea_orm::entity::prelude::*;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@ -26,8 +25,6 @@ pub struct Model {
|
|||||||
#[sea_orm(column_type = "Text", nullable)]
|
#[sea_orm(column_type = "Text", nullable)]
|
||||||
pub allowed_user_agents: Option<String>,
|
pub allowed_user_agents: Option<String>,
|
||||||
pub log_revert_chance: f64,
|
pub log_revert_chance: f64,
|
||||||
// TODO: rename this with a migration
|
|
||||||
pub log_level: TrackingLevel,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||||
|
@ -3,25 +3,6 @@
|
|||||||
use sea_orm::entity::prelude::*;
|
use sea_orm::entity::prelude::*;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
// TODO: rename TrackingLevel to StatLevel? AccountingLevel? What?
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
|
|
||||||
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "log_level")]
|
|
||||||
pub enum TrackingLevel {
|
|
||||||
/// TODO: rename to minimal
|
|
||||||
#[sea_orm(string_value = "none")]
|
|
||||||
None,
|
|
||||||
#[sea_orm(string_value = "aggregated")]
|
|
||||||
Aggregated,
|
|
||||||
#[sea_orm(string_value = "detailed")]
|
|
||||||
Detailed,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for TrackingLevel {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
|
||||||
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
|
#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")]
|
||||||
pub enum Method {
|
pub enum Method {
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "migration"
|
name = "migration"
|
||||||
version = "0.31.0"
|
version = "0.32.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
publish = false
|
publish = false
|
||||||
|
|
||||||
|
@ -31,6 +31,7 @@ mod m20230514_114803_admin_add_credits;
|
|||||||
mod m20230607_221917_total_deposits;
|
mod m20230607_221917_total_deposits;
|
||||||
mod m20230615_221201_handle_payment_uncles;
|
mod m20230615_221201_handle_payment_uncles;
|
||||||
mod m20230618_230611_longer_payload;
|
mod m20230618_230611_longer_payload;
|
||||||
|
mod m20230619_172237_default_tracking;
|
||||||
|
|
||||||
pub struct Migrator;
|
pub struct Migrator;
|
||||||
|
|
||||||
@ -69,6 +70,7 @@ impl MigratorTrait for Migrator {
|
|||||||
Box::new(m20230607_221917_total_deposits::Migration),
|
Box::new(m20230607_221917_total_deposits::Migration),
|
||||||
Box::new(m20230615_221201_handle_payment_uncles::Migration),
|
Box::new(m20230615_221201_handle_payment_uncles::Migration),
|
||||||
Box::new(m20230618_230611_longer_payload::Migration),
|
Box::new(m20230618_230611_longer_payload::Migration),
|
||||||
|
Box::new(m20230619_172237_default_tracking::Migration),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
63
migration/src/m20230619_172237_default_tracking.rs
Normal file
63
migration/src/m20230619_172237_default_tracking.rs
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
use sea_orm_migration::prelude::*;
|
||||||
|
|
||||||
|
#[derive(DeriveMigrationName)]
|
||||||
|
pub struct Migration;
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl MigrationTrait for Migration {
|
||||||
|
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||||
|
// new keys get set to aggregated logging
|
||||||
|
// TODO: rename "none" to "minimal"
|
||||||
|
manager
|
||||||
|
.alter_table(
|
||||||
|
Table::alter()
|
||||||
|
.table(RpcKey::Table)
|
||||||
|
.modify_column(
|
||||||
|
ColumnDef::new(RpcKey::LogLevel)
|
||||||
|
.enumeration(
|
||||||
|
Alias::new("log_level"),
|
||||||
|
[
|
||||||
|
Alias::new("none"),
|
||||||
|
Alias::new("aggregated"),
|
||||||
|
Alias::new("detailed"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.not_null()
|
||||||
|
.default("detailed"),
|
||||||
|
)
|
||||||
|
.to_owned(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||||
|
// new keys get set to none logging
|
||||||
|
manager
|
||||||
|
.alter_table(
|
||||||
|
Table::alter()
|
||||||
|
.table(RpcKey::Table)
|
||||||
|
.modify_column(
|
||||||
|
ColumnDef::new(RpcKey::LogLevel)
|
||||||
|
.enumeration(
|
||||||
|
Alias::new("log_level"),
|
||||||
|
[
|
||||||
|
Alias::new("none"),
|
||||||
|
Alias::new("aggregated"),
|
||||||
|
Alias::new("detailed"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.not_null()
|
||||||
|
.default("none"),
|
||||||
|
)
|
||||||
|
.to_owned(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Learn more at https://docs.rs/sea-query#iden
|
||||||
|
#[derive(Iden)]
|
||||||
|
enum RpcKey {
|
||||||
|
Table,
|
||||||
|
LogLevel,
|
||||||
|
}
|
@ -14,7 +14,6 @@ use chrono::Utc;
|
|||||||
use core::fmt;
|
use core::fmt;
|
||||||
use deferred_rate_limiter::DeferredRateLimitResult;
|
use deferred_rate_limiter::DeferredRateLimitResult;
|
||||||
use derive_more::From;
|
use derive_more::From;
|
||||||
use entities::sea_orm_active_enums::TrackingLevel;
|
|
||||||
use entities::{balance, login, rpc_key, user, user_tier};
|
use entities::{balance, login, rpc_key, user, user_tier};
|
||||||
use ethers::types::{Bytes, U64};
|
use ethers::types::{Bytes, U64};
|
||||||
use ethers::utils::keccak256;
|
use ethers::utils::keccak256;
|
||||||
@ -111,8 +110,6 @@ pub struct AuthorizationChecks {
|
|||||||
pub allowed_user_agents: Option<Vec<UserAgent>>,
|
pub allowed_user_agents: Option<Vec<UserAgent>>,
|
||||||
/// if None, allow any IP Address
|
/// if None, allow any IP Address
|
||||||
pub allowed_ips: Option<Vec<IpNet>>,
|
pub allowed_ips: Option<Vec<IpNet>>,
|
||||||
/// how detailed any rpc account entries should be
|
|
||||||
pub tracking_level: TrackingLevel,
|
|
||||||
/// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database.
|
/// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database.
|
||||||
/// depending on the caller, errors might be expected. this keeps us from bloating our database
|
/// depending on the caller, errors might be expected. this keeps us from bloating our database
|
||||||
/// u16::MAX == 100%
|
/// u16::MAX == 100%
|
||||||
@ -523,28 +520,6 @@ impl RequestMetadata {
|
|||||||
self.backend_requests.lock().clone()
|
self.backend_requests.lock().clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn tracking_level(&self) -> TrackingLevel {
|
|
||||||
if let Some(authorization) = self.authorization.as_ref() {
|
|
||||||
authorization.checks.tracking_level.clone()
|
|
||||||
} else {
|
|
||||||
TrackingLevel::None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn opt_in_method(&self) -> Option<String> {
|
|
||||||
match self.tracking_level() {
|
|
||||||
TrackingLevel::None | TrackingLevel::Aggregated => None,
|
|
||||||
TrackingLevel::Detailed => self.method.clone(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn take_opt_in_method(&mut self) -> Option<String> {
|
|
||||||
match self.tracking_level() {
|
|
||||||
TrackingLevel::None | TrackingLevel::Aggregated => None,
|
|
||||||
TrackingLevel::Detailed => self.method.take(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn try_send_stat(mut self) -> Web3ProxyResult<Option<Self>> {
|
pub fn try_send_stat(mut self) -> Web3ProxyResult<Option<Self>> {
|
||||||
if let Some(stat_sender) = self.stat_sender.take() {
|
if let Some(stat_sender) = self.stat_sender.take() {
|
||||||
trace!("sending stat! {:?}", self);
|
trace!("sending stat! {:?}", self);
|
||||||
@ -693,7 +668,6 @@ impl Authorization {
|
|||||||
let authorization_checks = AuthorizationChecks {
|
let authorization_checks = AuthorizationChecks {
|
||||||
// any error logs on a local (internal) query are likely problems. log them all
|
// any error logs on a local (internal) query are likely problems. log them all
|
||||||
log_revert_chance: 100,
|
log_revert_chance: 100,
|
||||||
tracking_level: TrackingLevel::Detailed,
|
|
||||||
// default for everything else should be fine. we don't have a user_id or ip to give
|
// default for everything else should be fine. we don't have a user_id or ip to give
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
@ -734,7 +708,6 @@ impl Authorization {
|
|||||||
let authorization_checks = AuthorizationChecks {
|
let authorization_checks = AuthorizationChecks {
|
||||||
max_requests_per_period,
|
max_requests_per_period,
|
||||||
proxy_mode,
|
proxy_mode,
|
||||||
tracking_level: TrackingLevel::Detailed,
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -1383,7 +1356,6 @@ impl Web3ProxyApp {
|
|||||||
proxy_mode,
|
proxy_mode,
|
||||||
rpc_secret_key: Some(rpc_secret_key),
|
rpc_secret_key: Some(rpc_secret_key),
|
||||||
rpc_secret_key_id: rpc_key_id,
|
rpc_secret_key_id: rpc_key_id,
|
||||||
tracking_level: rpc_key_model.log_level,
|
|
||||||
user_id: rpc_key_model.user_id,
|
user_id: rpc_key_model.user_id,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -10,7 +10,7 @@ use axum::{
|
|||||||
};
|
};
|
||||||
use axum_macros::debug_handler;
|
use axum_macros::debug_handler;
|
||||||
use entities;
|
use entities;
|
||||||
use entities::sea_orm_active_enums::{Role, TrackingLevel};
|
use entities::sea_orm_active_enums::Role;
|
||||||
use entities::{rpc_key, secondary_user};
|
use entities::{rpc_key, secondary_user};
|
||||||
use hashbrown::HashMap;
|
use hashbrown::HashMap;
|
||||||
use http::HeaderValue;
|
use http::HeaderValue;
|
||||||
@ -100,7 +100,6 @@ pub struct UserKeyManagement {
|
|||||||
allowed_referers: Option<String>,
|
allowed_referers: Option<String>,
|
||||||
allowed_user_agents: Option<String>,
|
allowed_user_agents: Option<String>,
|
||||||
description: Option<String>,
|
description: Option<String>,
|
||||||
log_level: Option<TrackingLevel>,
|
|
||||||
// TODO: enable log_revert_trace: Option<f64>,
|
// TODO: enable log_revert_trace: Option<f64>,
|
||||||
private_txs: Option<bool>,
|
private_txs: Option<bool>,
|
||||||
}
|
}
|
||||||
@ -169,14 +168,9 @@ pub async fn rpc_keys_management(
|
|||||||
// TODO: limit to 10 keys?
|
// TODO: limit to 10 keys?
|
||||||
let secret_key = RpcSecretKey::new();
|
let secret_key = RpcSecretKey::new();
|
||||||
|
|
||||||
let log_level = payload
|
|
||||||
.log_level
|
|
||||||
.web3_context("log level must be 'none', 'detailed', or 'aggregated'")?;
|
|
||||||
|
|
||||||
Ok(rpc_key::ActiveModel {
|
Ok(rpc_key::ActiveModel {
|
||||||
user_id: sea_orm::Set(user.id),
|
user_id: sea_orm::Set(user.id),
|
||||||
secret_key: sea_orm::Set(secret_key.into()),
|
secret_key: sea_orm::Set(secret_key.into()),
|
||||||
log_level: sea_orm::Set(log_level),
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -14,7 +14,6 @@ use anyhow::{anyhow, Context};
|
|||||||
use axum::headers::Origin;
|
use axum::headers::Origin;
|
||||||
use chrono::{DateTime, Months, TimeZone, Utc};
|
use chrono::{DateTime, Months, TimeZone, Utc};
|
||||||
use derive_more::From;
|
use derive_more::From;
|
||||||
use entities::sea_orm_active_enums::TrackingLevel;
|
|
||||||
use entities::{balance, referee, referrer, rpc_accounting_v2, rpc_key};
|
use entities::{balance, referee, referrer, rpc_accounting_v2, rpc_key};
|
||||||
use influxdb2::models::DataPoint;
|
use influxdb2::models::DataPoint;
|
||||||
use log::trace;
|
use log::trace;
|
||||||
@ -85,7 +84,7 @@ fn round_timestamp(timestamp: i64, period_seconds: i64) -> i64 {
|
|||||||
|
|
||||||
impl RpcQueryStats {
|
impl RpcQueryStats {
|
||||||
/// rpc keys can opt into multiple levels of tracking.
|
/// rpc keys can opt into multiple levels of tracking.
|
||||||
/// we always need enough to handle billing, so even the "none" level still has some minimal tracking.
|
/// we always need enough to handle billing, so the "none" level was changed to "minimal" tracking.
|
||||||
/// This "accounting_key" is used in the relational database.
|
/// This "accounting_key" is used in the relational database.
|
||||||
/// anonymous users are also saved in the relational database so that the host can do their own cost accounting.
|
/// anonymous users are also saved in the relational database so that the host can do their own cost accounting.
|
||||||
fn accounting_key(&self, period_seconds: i64) -> RpcQueryKey {
|
fn accounting_key(&self, period_seconds: i64) -> RpcQueryKey {
|
||||||
@ -93,29 +92,10 @@ impl RpcQueryStats {
|
|||||||
|
|
||||||
let rpc_secret_key_id = self.authorization.checks.rpc_secret_key_id;
|
let rpc_secret_key_id = self.authorization.checks.rpc_secret_key_id;
|
||||||
|
|
||||||
let (method, origin) = match self.authorization.checks.tracking_level {
|
|
||||||
TrackingLevel::None => {
|
|
||||||
// this RPC key requested no tracking. this is the default
|
|
||||||
// do not store the method or the origin
|
|
||||||
(None, None)
|
|
||||||
}
|
|
||||||
TrackingLevel::Aggregated => {
|
|
||||||
// this RPC key requested tracking aggregated across all methods and origins
|
|
||||||
// TODO: think about this more. do we want the origin or not? grouping free cost per site might be useful. i'd rather not collect things if we don't have a planned purpose though
|
|
||||||
let method = None;
|
|
||||||
let origin = None;
|
|
||||||
|
|
||||||
(method, origin)
|
|
||||||
}
|
|
||||||
TrackingLevel::Detailed => {
|
|
||||||
// detailed tracking keeps track of the method and origin
|
|
||||||
// depending on the request, the origin might still be None
|
|
||||||
let method = self.method.clone();
|
let method = self.method.clone();
|
||||||
let origin = self.authorization.origin.clone();
|
|
||||||
|
|
||||||
(method, origin)
|
// we used to optionally store origin, but wallets don't set it, so its almost always None
|
||||||
}
|
let origin = None;
|
||||||
};
|
|
||||||
|
|
||||||
// Depending on method, add some arithmetic around calculating credits_used
|
// Depending on method, add some arithmetic around calculating credits_used
|
||||||
// I think balance should not go here, this looks more like a key thingy
|
// I think balance should not go here, this looks more like a key thingy
|
||||||
@ -151,26 +131,12 @@ impl RpcQueryStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// rpc keys can opt into more detailed tracking
|
/// stats for a single key
|
||||||
fn opt_in_timeseries_key(&self) -> Option<RpcQueryKey> {
|
fn owned_timeseries_key(&self) -> Option<RpcQueryKey> {
|
||||||
// we don't store origin in the timeseries db. its only optionaly used for accounting
|
// we don't store origin in the timeseries db. its only optionaly used for accounting
|
||||||
let origin = None;
|
let origin = None;
|
||||||
|
|
||||||
// depending on tracking level, we either skip opt-in stats, track without method, or track with method
|
let method = self.method.clone();
|
||||||
let method = match self.authorization.checks.tracking_level {
|
|
||||||
TrackingLevel::None => {
|
|
||||||
// this RPC key requested no tracking. this is the default.
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
TrackingLevel::Aggregated => {
|
|
||||||
// this RPC key requested tracking aggregated across all methods
|
|
||||||
None
|
|
||||||
}
|
|
||||||
TrackingLevel::Detailed => {
|
|
||||||
// detailed tracking keeps track of the method
|
|
||||||
self.method.clone()
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let key = RpcQueryKey {
|
let key = RpcQueryKey {
|
||||||
response_timestamp: self.response_timestamp,
|
response_timestamp: self.response_timestamp,
|
||||||
|
@ -122,7 +122,7 @@ impl StatBuffer {
|
|||||||
|
|
||||||
self.global_timeseries_buffer.entry(global_timeseries_key).or_default().add(stat.clone());
|
self.global_timeseries_buffer.entry(global_timeseries_key).or_default().add(stat.clone());
|
||||||
|
|
||||||
if let Some(opt_in_timeseries_key) = stat.opt_in_timeseries_key() {
|
if let Some(opt_in_timeseries_key) = stat.owned_timeseries_key() {
|
||||||
self.opt_in_timeseries_buffer.entry(opt_in_timeseries_key).or_default().add(stat.clone());
|
self.opt_in_timeseries_buffer.entry(opt_in_timeseries_key).or_default().add(stat.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user