From 8b35bf5e633463639af0b11a4b12e0b91178be63 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Tue, 1 Nov 2022 18:54:39 +0000 Subject: [PATCH] clean up migration --- Cargo.lock | 4 +- README.md | 7 ++ entities/Cargo.toml | 2 +- entities/src/mod.rs | 7 +- entities/src/prelude.rs | 7 +- .../src/{revert_logs.rs => revert_log.rs} | 14 +-- entities/src/rpc_accounting.rs | 14 +-- entities/src/{rpc_keys.rs => rpc_key.rs} | 21 ++-- entities/src/sea_orm_active_enums.rs | 3 +- entities/src/secondary_user.rs | 4 +- entities/src/user.rs | 33 ++++-- entities/src/user_tier.rs | 28 +++++ migration/Cargo.toml | 2 +- migration/src/lib.rs | 2 + migration/src/m20221031_211916_clean_up.rs | 103 ++++++++++++++++++ web3_proxy/src/app.rs | 17 +-- web3_proxy/src/bin/web3_proxy.rs | 2 +- .../src/bin/web3_proxy_cli/check_config.rs | 2 +- .../src/bin/web3_proxy_cli/create_user.rs | 20 ++-- web3_proxy/src/config.rs | 2 +- web3_proxy/src/frontend/authorization.rs | 76 +++++++------ web3_proxy/src/frontend/status.rs | 6 +- web3_proxy/src/frontend/users.rs | 50 ++++----- web3_proxy/src/rpcs/blockchain.rs | 2 +- web3_proxy/src/rpcs/request.rs | 15 +-- web3_proxy/src/user_queries.rs | 25 ++--- 26 files changed, 308 insertions(+), 160 deletions(-) rename entities/src/{revert_logs.rs => revert_log.rs} (72%) rename entities/src/{rpc_keys.rs => rpc_key.rs} (71%) create mode 100644 entities/src/user_tier.rs create mode 100644 migration/src/m20221031_211916_clean_up.rs diff --git a/Cargo.lock b/Cargo.lock index 8397e313..1ca24beb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,7 +1348,7 @@ dependencies = [ [[package]] name = "entities" -version = "0.6.0" +version = "0.8.0" dependencies = [ "sea-orm", "serde", @@ -2657,7 +2657,7 @@ dependencies = [ [[package]] name = "migration" -version = "0.7.0" +version = "0.8.0" dependencies = [ "sea-orm-migration", "tokio", diff --git a/README.md b/README.md index 99c53e5d..a70b6c2e 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,13 @@ $ websocat ws://127.0.0.1:8544 You can copy `config/example.toml` to `config/production-$CHAINNAME.toml` and then run `docker-compose up --build -d` start proxies for many chains. + +Run migrations (useful during development. in production, the migrations run on application start) +``` +cd migration +cargo run up +``` + ## Database entities This command only needs to be run during development. Production should use the already generated entities. diff --git a/entities/Cargo.toml b/entities/Cargo.toml index 3e8ce886..e65d8ee8 100644 --- a/entities/Cargo.toml +++ b/entities/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "entities" -version = "0.6.0" +version = "0.8.0" edition = "2021" [lib] diff --git a/entities/src/mod.rs b/entities/src/mod.rs index fbcc2c1f..8bcaf5a0 100644 --- a/entities/src/mod.rs +++ b/entities/src/mod.rs @@ -1,10 +1,11 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 pub mod prelude; -pub mod revert_logs; +pub mod revert_log; pub mod rpc_accounting; +pub mod rpc_key; pub mod sea_orm_active_enums; pub mod secondary_user; pub mod user; -pub mod rpc_keys; +pub mod user_tier; diff --git a/entities/src/prelude.rs b/entities/src/prelude.rs index ae61d8d4..0f31fa60 100644 --- a/entities/src/prelude.rs +++ b/entities/src/prelude.rs @@ -1,7 +1,8 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 -pub use super::revert_logs::Entity as RevertLogs; +pub use super::revert_log::Entity as RevertLog; pub use super::rpc_accounting::Entity as RpcAccounting; -pub use super::rpc_keys::Entity as UserKeys; +pub use super::rpc_key::Entity as RpcKey; pub use super::secondary_user::Entity as SecondaryUser; pub use super::user::Entity as User; +pub use super::user_tier::Entity as UserTier; diff --git a/entities/src/revert_logs.rs b/entities/src/revert_log.rs similarity index 72% rename from entities/src/revert_logs.rs rename to entities/src/revert_log.rs index 3e323650..a2ac9735 100644 --- a/entities/src/revert_logs.rs +++ b/entities/src/revert_log.rs @@ -1,11 +1,11 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 use super::sea_orm_active_enums::Method; use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] -#[sea_orm(table_name = "revert_logs")] +#[sea_orm(table_name = "revert_log")] pub struct Model { #[sea_orm(primary_key)] pub id: u64, @@ -21,18 +21,18 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { #[sea_orm( - belongs_to = "super::rpc_keys::Entity", + belongs_to = "super::rpc_key::Entity", from = "Column::RpcKeyId", - to = "super::rpc_keys::Column::Id", + to = "super::rpc_key::Column::Id", on_update = "NoAction", on_delete = "NoAction" )] - UserKeys, + RpcKey, } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::UserKeys.def() + Relation::RpcKey.def() } } diff --git a/entities/src/rpc_accounting.rs b/entities/src/rpc_accounting.rs index 5f57adc4..c1eab9c5 100644 --- a/entities/src/rpc_accounting.rs +++ b/entities/src/rpc_accounting.rs @@ -1,4 +1,4 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; @@ -13,14 +13,12 @@ pub struct Model { pub method: String, pub error_response: bool, pub period_datetime: DateTimeUtc, - // TODO: migration to make these u32 pub frontend_requests: u64, pub backend_requests: u64, pub backend_retries: u64, pub no_servers: u64, pub cache_misses: u64, pub cache_hits: u64, - // TODO: end migration to make these u32 pub sum_request_bytes: u64, pub min_request_bytes: u64, pub mean_request_bytes: f64, @@ -47,18 +45,18 @@ pub struct Model { #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { #[sea_orm( - belongs_to = "super::rpc_keys::Entity", + belongs_to = "super::rpc_key::Entity", from = "Column::RpcKeyId", - to = "super::rpc_keys::Column::Id", + to = "super::rpc_key::Column::Id", on_update = "NoAction", on_delete = "NoAction" )] - RpcKeys, + RpcKey, } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::RpcKeys.def() + Relation::RpcKey.def() } } diff --git a/entities/src/rpc_keys.rs b/entities/src/rpc_key.rs similarity index 71% rename from entities/src/rpc_keys.rs rename to entities/src/rpc_key.rs index dc4bb5d1..4e93745f 100644 --- a/entities/src/rpc_keys.rs +++ b/entities/src/rpc_key.rs @@ -1,22 +1,19 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] -#[sea_orm(table_name = "rpc_keys")] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "rpc_key")] pub struct Model { #[sea_orm(primary_key)] pub id: u64, pub user_id: u64, #[sea_orm(unique)] - pub rpc_key: Uuid, + pub secret_key: Uuid, pub description: Option, pub private_txs: bool, pub active: bool, - pub requests_per_minute: Option, - #[sea_orm(column_type = "Decimal(Some((5, 4)))")] - pub log_revert_chance: Decimal, #[sea_orm(column_type = "Text", nullable)] pub allowed_ips: Option, #[sea_orm(column_type = "Text", nullable)] @@ -25,7 +22,7 @@ pub struct Model { pub allowed_referers: Option, #[sea_orm(column_type = "Text", nullable)] pub allowed_user_agents: Option, - pub max_concurrent_requests: Option, + pub log_revert_chance: f64, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] @@ -38,8 +35,8 @@ pub enum Relation { on_delete = "NoAction" )] User, - #[sea_orm(has_many = "super::revert_logs::Entity")] - RevertLogs, + #[sea_orm(has_many = "super::revert_log::Entity")] + RevertLog, #[sea_orm(has_many = "super::rpc_accounting::Entity")] RpcAccounting, } @@ -50,9 +47,9 @@ impl Related for Entity { } } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { - Relation::RevertLogs.def() + Relation::RevertLog.def() } } diff --git a/entities/src/sea_orm_active_enums.rs b/entities/src/sea_orm_active_enums.rs index 879a218d..bd97d400 100644 --- a/entities/src/sea_orm_active_enums.rs +++ b/entities/src/sea_orm_active_enums.rs @@ -1,7 +1,6 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 use sea_orm::entity::prelude::*; -use sea_orm::EnumIter; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] diff --git a/entities/src/secondary_user.rs b/entities/src/secondary_user.rs index df409517..275f3223 100644 --- a/entities/src/secondary_user.rs +++ b/entities/src/secondary_user.rs @@ -1,4 +1,4 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 use super::sea_orm_active_enums::Role; use sea_orm::entity::prelude::*; @@ -10,9 +10,7 @@ pub struct Model { #[sea_orm(primary_key)] pub id: u64, pub user_id: u64, - pub address: Vec, pub description: Option, - pub email: Option, pub role: Role, } diff --git a/entities/src/user.rs b/entities/src/user.rs index a57dd584..fd121e59 100644 --- a/entities/src/user.rs +++ b/entities/src/user.rs @@ -1,4 +1,4 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.10.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; @@ -12,14 +12,35 @@ pub struct Model { pub address: Vec, pub description: Option, pub email: Option, + pub user_tier_id: u64, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { + #[sea_orm( + belongs_to = "super::user_tier::Entity", + from = "Column::UserTierId", + to = "super::user_tier::Column::Id", + on_update = "NoAction", + on_delete = "NoAction" + )] + UserTier, + #[sea_orm(has_many = "super::rpc_key::Entity")] + RpcKey, #[sea_orm(has_many = "super::secondary_user::Entity")] SecondaryUser, - #[sea_orm(has_many = "super::rpc_keys::Entity")] - RpcKeys, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::UserTier.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::RpcKey.def() + } } impl Related for Entity { @@ -28,10 +49,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::RpcKeys.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/entities/src/user_tier.rs b/entities/src/user_tier.rs new file mode 100644 index 00000000..885471dd --- /dev/null +++ b/entities/src/user_tier.rs @@ -0,0 +1,28 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.10.1 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "user_tier")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: u64, + pub title: String, + pub max_requests_per_minute: Option, + pub max_concurrent_requests: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::user::Entity")] + User, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/migration/Cargo.toml b/migration/Cargo.toml index 8422bb72..36f70ee1 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "migration" -version = "0.7.0" +version = "0.8.0" edition = "2021" publish = false diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 6e4397f6..03b5a127 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -7,6 +7,7 @@ mod m20221007_213828_accounting; mod m20221025_210326_add_chain_id_to_reverts; mod m20221026_230819_rename_user_keys; mod m20221027_002407_user_tiers; +mod m20221031_211916_clean_up; pub struct Migrator; @@ -21,6 +22,7 @@ impl MigratorTrait for Migrator { Box::new(m20221025_210326_add_chain_id_to_reverts::Migration), Box::new(m20221026_230819_rename_user_keys::Migration), Box::new(m20221027_002407_user_tiers::Migration), + Box::new(m20221031_211916_clean_up::Migration), ] } } diff --git a/migration/src/m20221031_211916_clean_up.rs b/migration/src/m20221031_211916_clean_up.rs new file mode 100644 index 00000000..65bce367 --- /dev/null +++ b/migration/src/m20221031_211916_clean_up.rs @@ -0,0 +1,103 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // rename tables from plural to singluar + manager + .rename_table( + Table::rename() + .table(Alias::new("revert_logs"), Alias::new("revert_log")) + .to_owned(), + ) + .await?; + + manager + .rename_table( + Table::rename() + .table(Alias::new("rpc_keys"), Alias::new("rpc_key")) + .to_owned(), + ) + .await?; + + // on rpc_key table, rename rpc_key to secret_key + manager + .alter_table( + Table::alter() + .table(Alias::new("rpc_key")) + .rename_column(Alias::new("rpc_key"), Alias::new("secret_key")) + .to_owned(), + ) + .await?; + + // on revert_log table, rename user_key_id to rpc_key_id + manager + .alter_table( + Table::alter() + .table(Alias::new("revert_log")) + .rename_column(Alias::new("user_key_id"), Alias::new("rpc_key_id")) + .to_owned(), + ) + .await?; + + // on rpc_accounting table, rename user_key_id to rpc_key_id + manager + .alter_table( + Table::alter() + .table(Alias::new("rpc_accounting")) + .rename_column(Alias::new("user_key_id"), Alias::new("rpc_key_id")) + .to_owned(), + ) + .await?; + + // on secondary_users table, remove "email" and "address" column + manager + .alter_table( + Table::alter() + .table(Alias::new("secondary_user")) + .drop_column(Alias::new("email")) + .drop_column(Alias::new("address")) + .to_owned(), + ) + .await?; + + // on user_tier table, rename requests_per_minute to max_requests_per_minute + manager + .alter_table( + Table::alter() + .table(Alias::new("user_tier")) + .rename_column( + Alias::new("requests_per_minute"), + Alias::new("max_requests_per_minute"), + ) + .to_owned(), + ) + .await?; + + manager + .alter_table( + Table::alter() + .table(Alias::new("rpc_key")) + .drop_column(Alias::new("log_revert_chance")) + .add_column( + ColumnDef::new(Alias::new("log_revert_chance")) + .double() + .not_null() + .default(0.0), + ) + .to_owned(), + ) + .await?; + + // rename column rpc_key to rpc_secret_key + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + todo!(); + } +} diff --git a/web3_proxy/src/app.rs b/web3_proxy/src/app.rs index a3522bf3..a4ab0beb 100644 --- a/web3_proxy/src/app.rs +++ b/web3_proxy/src/app.rs @@ -29,7 +29,6 @@ use metered::{metered, ErrorCount, HitCount, ResponseTime, Throughput}; use migration::{Migrator, MigratorTrait}; use moka::future::Cache; use redis_rate_limiter::{DeadpoolRuntime, RedisConfig, RedisPool, RedisRateLimiter}; -use sea_orm::prelude::Decimal; use sea_orm::DatabaseConnection; use serde::Serialize; use serde_json::json; @@ -68,9 +67,9 @@ pub struct UserKeyData { pub user_id: u64, /// database id of the rpc key pub rpc_key_id: u64, - /// if None, allow unlimited queries + /// if None, allow unlimited queries. inherited from the user_tier pub max_requests_per_period: Option, - // if None, allow unlimited concurrent requests + // if None, allow unlimited concurrent requests. inherited from the user_tier pub max_concurrent_requests: Option, /// if None, allow any Origin pub allowed_origins: Option>, @@ -81,7 +80,8 @@ pub struct UserKeyData { /// if None, allow any IP Address pub allowed_ips: Option>, /// Chance to save reverting eth_call, eth_estimateGas, and eth_sendRawTransaction to the database. - pub log_revert_chance: Decimal, + /// TODO: f32 would be fine + pub log_revert_chance: f64, } /// The application @@ -108,7 +108,8 @@ pub struct Web3ProxyApp { pub frontend_key_rate_limiter: Option>, pub login_rate_limiter: Option, pub vredis_pool: Option, - pub rpc_key_cache: Cache, + // TODO: this key should be our RpcSecretKey class, not Ulid + pub rpc_secret_key_cache: Cache, pub rpc_key_semaphores: Cache, hashbrown::hash_map::DefaultHashBuilder>, pub ip_semaphores: Cache, hashbrown::hash_map::DefaultHashBuilder>, pub bearer_token_semaphores: @@ -456,9 +457,9 @@ impl Web3ProxyApp { // if there is no database of users, there will be no keys and so this will be empty // TODO: max_capacity from config // TODO: ttl from config - let rpc_key_cache = Cache::builder() + let rpc_secret_key_cache = Cache::builder() .max_capacity(10_000) - .time_to_live(Duration::from_secs(60)) + .time_to_live(Duration::from_secs(600)) .build_with_hasher(hashbrown::hash_map::DefaultHashBuilder::new()); // create semaphores for concurrent connection limits @@ -488,7 +489,7 @@ impl Web3ProxyApp { vredis_pool, app_metrics, open_request_handle_metrics, - rpc_key_cache, + rpc_secret_key_cache, bearer_token_semaphores, ip_semaphores, rpc_key_semaphores, diff --git a/web3_proxy/src/bin/web3_proxy.rs b/web3_proxy/src/bin/web3_proxy.rs index 7de292d3..471a5d03 100644 --- a/web3_proxy/src/bin/web3_proxy.rs +++ b/web3_proxy/src/bin/web3_proxy.rs @@ -267,7 +267,7 @@ mod tests { let app_config = TopConfig { app: AppConfig { chain_id: 31337, - default_user_requests_per_minute: Some(6_000_000), + default_user_max_requests_per_minute: Some(6_000_000), min_sum_soft_limit: 1, min_synced_rpcs: 1, public_requests_per_minute: Some(1_000_000), diff --git a/web3_proxy/src/bin/web3_proxy_cli/check_config.rs b/web3_proxy/src/bin/web3_proxy_cli/check_config.rs index 286650ca..89358b19 100644 --- a/web3_proxy/src/bin/web3_proxy_cli/check_config.rs +++ b/web3_proxy/src/bin/web3_proxy_cli/check_config.rs @@ -35,7 +35,7 @@ impl CheckConfigSubCommand { Some(_) => {} } - match top_config.app.default_user_requests_per_minute { + match top_config.app.default_user_max_requests_per_minute { None => { info!("app.default_user_requests_per_minute is None. Fully open to registered requests!") } diff --git a/web3_proxy/src/bin/web3_proxy_cli/create_user.rs b/web3_proxy/src/bin/web3_proxy_cli/create_user.rs index ede684dd..c7969a2c 100644 --- a/web3_proxy/src/bin/web3_proxy_cli/create_user.rs +++ b/web3_proxy/src/bin/web3_proxy_cli/create_user.rs @@ -1,12 +1,12 @@ use anyhow::Context; use argh::FromArgs; -use entities::{rpc_keys, user}; +use entities::{rpc_key, user}; use ethers::prelude::Address; use sea_orm::{ActiveModelTrait, TransactionTrait}; use tracing::info; use ulid::Ulid; use uuid::Uuid; -use web3_proxy::frontend::authorization::RpcApiKey; +use web3_proxy::frontend::authorization::RpcSecretKey; #[derive(FromArgs, PartialEq, Debug, Eq)] /// Create a new user and api key @@ -25,12 +25,7 @@ pub struct CreateUserSubCommand { /// the user's first api ULID or UUID key. /// If none given, one will be created. #[argh(option)] - rpc_key: RpcApiKey, - - /// the key's maximum requests per minute. - /// Default to "None" which the code sees as "unlimited" requests. - #[argh(option)] - rpm: Option, + rpc_secret_key: RpcSecretKey, /// an optional short description of the key's purpose. #[argh(option)] @@ -74,10 +69,9 @@ impl CreateUserSubCommand { ); // create a key for the new user - let uk = rpc_keys::ActiveModel { + let uk = rpc_key::ActiveModel { user_id: u.id, - rpc_key: sea_orm::Set(self.rpc_key.into()), - requests_per_minute: sea_orm::Set(self.rpm), + secret_key: sea_orm::Set(self.rpc_secret_key.into()), description: sea_orm::Set(self.description), ..Default::default() }; @@ -87,8 +81,8 @@ impl CreateUserSubCommand { txn.commit().await?; - info!("user key as ULID: {}", Ulid::from(self.rpc_key)); - info!("user key as UUID: {}", Uuid::from(self.rpc_key)); + info!("user key as ULID: {}", Ulid::from(self.rpc_secret_key)); + info!("user key as UUID: {}", Uuid::from(self.rpc_secret_key)); Ok(()) } diff --git a/web3_proxy/src/config.rs b/web3_proxy/src/config.rs index 66a9fc8d..7ab12fe6 100644 --- a/web3_proxy/src/config.rs +++ b/web3_proxy/src/config.rs @@ -74,7 +74,7 @@ pub struct AppConfig { /// Default request limit for registered users. /// 0 = block all requests /// None = allow all requests - pub default_user_requests_per_minute: Option, + pub default_user_max_requests_per_minute: Option, /// Restrict user registration. /// None = no code needed diff --git a/web3_proxy/src/frontend/authorization.rs b/web3_proxy/src/frontend/authorization.rs index 1b13b665..d7fda9fb 100644 --- a/web3_proxy/src/frontend/authorization.rs +++ b/web3_proxy/src/frontend/authorization.rs @@ -10,12 +10,12 @@ use axum::headers::{Header, Origin, Referer, UserAgent}; use axum::TypedHeader; use chrono::Utc; use deferred_rate_limiter::DeferredRateLimitResult; -use entities::{rpc_keys, user}; +use entities::{rpc_key, user, user_tier}; use http::HeaderValue; use ipnet::IpNet; use redis_rate_limiter::redis::AsyncCommands; use redis_rate_limiter::RedisRateLimitResult; -use sea_orm::{prelude::Decimal, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; +use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; use std::fmt::Display; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::{net::IpAddr, str::FromStr, sync::Arc}; @@ -28,7 +28,7 @@ use uuid::Uuid; /// This lets us use UUID and ULID while we transition to only ULIDs /// TODO: include the key's description. #[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] -pub enum RpcApiKey { +pub enum RpcSecretKey { Ulid(Ulid), Uuid(Uuid), } @@ -55,7 +55,7 @@ pub struct AuthorizedKey { pub user_id: u64, pub rpc_key_id: u64, // TODO: just use an f32? even an f16 is probably fine - pub log_revert_chance: Decimal, + pub log_revert_chance: f64, } #[derive(Debug)] @@ -107,19 +107,19 @@ impl RequestMetadata { } } -impl RpcApiKey { +impl RpcSecretKey { pub fn new() -> Self { Ulid::new().into() } } -impl Default for RpcApiKey { +impl Default for RpcSecretKey { fn default() -> Self { Self::new() } } -impl Display for RpcApiKey { +impl Display for RpcSecretKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // TODO: do this without dereferencing let ulid: Ulid = (*self).into(); @@ -128,7 +128,7 @@ impl Display for RpcApiKey { } } -impl FromStr for RpcApiKey { +impl FromStr for RpcSecretKey { type Err = anyhow::Error; fn from_str(s: &str) -> Result { @@ -143,32 +143,32 @@ impl FromStr for RpcApiKey { } } -impl From for RpcApiKey { +impl From for RpcSecretKey { fn from(x: Ulid) -> Self { - RpcApiKey::Ulid(x) + RpcSecretKey::Ulid(x) } } -impl From for RpcApiKey { +impl From for RpcSecretKey { fn from(x: Uuid) -> Self { - RpcApiKey::Uuid(x) + RpcSecretKey::Uuid(x) } } -impl From for Ulid { - fn from(x: RpcApiKey) -> Self { +impl From for Ulid { + fn from(x: RpcSecretKey) -> Self { match x { - RpcApiKey::Ulid(x) => x, - RpcApiKey::Uuid(x) => Ulid::from(x.as_u128()), + RpcSecretKey::Ulid(x) => x, + RpcSecretKey::Uuid(x) => Ulid::from(x.as_u128()), } } } -impl From for Uuid { - fn from(x: RpcApiKey) -> Self { +impl From for Uuid { + fn from(x: RpcSecretKey) -> Self { match x { - RpcApiKey::Ulid(x) => Uuid::from_u128(x.0), - RpcApiKey::Uuid(x) => x, + RpcSecretKey::Ulid(x) => Uuid::from_u128(x.0), + RpcSecretKey::Uuid(x) => x, } } } @@ -297,7 +297,7 @@ pub async fn ip_is_authorized( pub async fn key_is_authorized( app: &Web3ProxyApp, - rpc_key: RpcApiKey, + rpc_key: RpcSecretKey, ip: IpAddr, origin: Option, referer: Option, @@ -502,27 +502,32 @@ impl Web3ProxyApp { // check the local cache for user data, or query the database #[instrument(level = "trace")] - pub(crate) async fn user_data(&self, rpc_key: RpcApiKey) -> anyhow::Result { + pub(crate) async fn user_data( + &self, + rpc_secret_key: RpcSecretKey, + ) -> anyhow::Result { let user_data: Result<_, Arc> = self - .rpc_key_cache - .try_get_with(rpc_key.into(), async move { - trace!(?rpc_key, "user_cache miss"); + .rpc_secret_key_cache + .try_get_with(rpc_secret_key.into(), async move { + trace!(?rpc_secret_key, "user cache miss"); let db_conn = self.db_conn().context("Getting database connection")?; - let rpc_key: Uuid = rpc_key.into(); + let rpc_secret_key: Uuid = rpc_secret_key.into(); // TODO: join the user table to this to return the User? we don't always need it - // TODO: also attach secondary users - match rpc_keys::Entity::find() - .filter(rpc_keys::Column::RpcKey.eq(rpc_key)) - .filter(rpc_keys::Column::Active.eq(true)) + // TODO: join on secondary users + // TODO: join on user tier + match rpc_key::Entity::find() + .filter(rpc_key::Column::SecretKey.eq(rpc_secret_key)) + .filter(rpc_key::Column::Active.eq(true)) .one(&db_conn) .await? { Some(rpc_key_model) => { // TODO: move these splits into helper functions // TODO: can we have sea orm handle this for us? + // let user_tier_model = rpc_key_model. let allowed_ips: Option> = if let Some(allowed_ips) = rpc_key_model.allowed_ips { @@ -575,16 +580,18 @@ impl Web3ProxyApp { None }; + // let user_tier_model = user_tier + Ok(UserKeyData { user_id: rpc_key_model.user_id, rpc_key_id: rpc_key_model.id, - max_requests_per_period: rpc_key_model.requests_per_minute, - max_concurrent_requests: rpc_key_model.max_concurrent_requests, allowed_ips, allowed_origins, allowed_referers, allowed_user_agents, log_revert_chance: rpc_key_model.log_revert_chance, + max_concurrent_requests: None, // todo! user_tier_model.max_concurrent_requests, + max_requests_per_period: None, // todo! user_tier_model.max_requests_per_period, }) } None => Ok(UserKeyData::default()), @@ -597,7 +604,10 @@ impl Web3ProxyApp { } #[instrument(level = "trace")] - pub async fn rate_limit_by_key(&self, rpc_key: RpcApiKey) -> anyhow::Result { + pub async fn rate_limit_by_key( + &self, + rpc_key: RpcSecretKey, + ) -> anyhow::Result { let user_data = self.user_data(rpc_key).await?; if user_data.rpc_key_id == 0 { diff --git a/web3_proxy/src/frontend/status.rs b/web3_proxy/src/frontend/status.rs index 1d9d28f4..f1d4d046 100644 --- a/web3_proxy/src/frontend/status.rs +++ b/web3_proxy/src/frontend/status.rs @@ -39,14 +39,14 @@ pub async fn prometheus(Extension(app): Extension>) -> impl In #[instrument(level = "trace")] pub async fn status(Extension(app): Extension>) -> impl IntoResponse { app.pending_transactions.sync(); - app.rpc_key_cache.sync(); + app.rpc_secret_key_cache.sync(); // TODO: what else should we include? uptime, cache hit rates, cpu load let body = json!({ "pending_transactions_count": app.pending_transactions.entry_count(), "pending_transactions_size": app.pending_transactions.weighted_size(), - "user_cache_count": app.rpc_key_cache.entry_count(), - "user_cache_size": app.rpc_key_cache.weighted_size(), + "user_cache_count": app.rpc_secret_key_cache.entry_count(), + "user_cache_size": app.rpc_secret_key_cache.weighted_size(), "balanced_rpcs": app.balanced_rpcs, "private_rpcs": app.private_rpcs, }); diff --git a/web3_proxy/src/frontend/users.rs b/web3_proxy/src/frontend/users.rs index c2a2fd30..f01d2768 100644 --- a/web3_proxy/src/frontend/users.rs +++ b/web3_proxy/src/frontend/users.rs @@ -1,6 +1,6 @@ //! Handle registration, logins, and managing account data. -use super::authorization::{login_is_authorized, RpcApiKey}; +use super::authorization::{login_is_authorized, RpcSecretKey}; use super::errors::FrontendResult; use crate::app::Web3ProxyApp; use crate::user_queries::{ @@ -18,7 +18,7 @@ use axum::{ }; use axum_client_ip::ClientIp; use axum_macros::debug_handler; -use entities::{revert_logs, rpc_keys, user}; +use entities::{revert_log, rpc_key, user}; use ethers::{prelude::Address, types::Bytes}; use hashbrown::HashMap; use http::{HeaderValue, StatusCode}; @@ -250,6 +250,8 @@ pub async fn user_login_post( // the only thing we need from them is an address // everything else is optional + // TODO: different invite codes should allow different levels + // TODO: maybe decrement a count on the invite code? let u = user::ActiveModel { address: sea_orm::Set(our_msg.address.into()), ..Default::default() @@ -259,14 +261,13 @@ pub async fn user_login_post( // create the user's first api key // TODO: rename to UserApiKey? RpcApiKey? - let rpc_key = RpcApiKey::new(); + let rpc_secret_key = RpcSecretKey::new(); // TODO: variable requests per minute depending on the invite code - let uk = rpc_keys::ActiveModel { + let uk = rpc_key::ActiveModel { user_id: sea_orm::Set(u.id), - rpc_key: sea_orm::Set(rpc_key.into()), + secret_key: sea_orm::Set(rpc_secret_key.into()), description: sea_orm::Set(Some("first".to_string())), - requests_per_minute: sea_orm::Set(app.config.default_user_requests_per_minute), ..Default::default() }; @@ -284,8 +285,8 @@ pub async fn user_login_post( } Some(u) => { // the user is already registered - let uks = rpc_keys::Entity::find() - .filter(rpc_keys::Column::UserId.eq(u.id)) + let uks = rpc_key::Entity::find() + .filter(rpc_key::Column::UserId.eq(u.id)) .all(&db_conn) .await .context("failed loading user's key")?; @@ -460,8 +461,8 @@ pub async fn rpc_keys_get( let db_conn = app.db_conn().context("getting db to fetch user's keys")?; - let uks = rpc_keys::Entity::find() - .filter(rpc_keys::Column::UserId.eq(user.id)) + let uks = rpc_key::Entity::find() + .filter(rpc_key::Column::UserId.eq(user.id)) .all(&db_conn) .await .context("failed loading user's key")?; @@ -525,9 +526,9 @@ pub async fn rpc_keys_management( let mut uk = if let Some(existing_key_id) = payload.key_id { // get the key and make sure it belongs to the user - let uk = rpc_keys::Entity::find() - .filter(rpc_keys::Column::UserId.eq(user.id)) - .filter(rpc_keys::Column::Id.eq(existing_key_id)) + let uk = rpc_key::Entity::find() + .filter(rpc_key::Column::UserId.eq(user.id)) + .filter(rpc_key::Column::Id.eq(existing_key_id)) .one(&db_conn) .await .context("failed loading user's key")? @@ -537,12 +538,11 @@ pub async fn rpc_keys_management( } else { // make a new key // TODO: limit to 10 keys? - let rpc_key = RpcApiKey::new(); + let secret_key = RpcSecretKey::new(); - rpc_keys::ActiveModel { + rpc_key::ActiveModel { user_id: sea_orm::Set(user.id), - rpc_key: sea_orm::Set(rpc_key.into()), - requests_per_minute: sea_orm::Set(app.config.default_user_requests_per_minute), + secret_key: sea_orm::Set(secret_key.into()), ..Default::default() } }; @@ -671,7 +671,7 @@ pub async fn rpc_keys_management( uk }; - let uk: rpc_keys::Model = uk.try_into()?; + let uk: rpc_key::Model = uk.try_into()?; Ok(Json(uk).into_response()) } @@ -702,8 +702,8 @@ pub async fn user_revert_logs_get( let db_conn = app.db_conn().context("getting db for user's revert logs")?; - let uks = rpc_keys::Entity::find() - .filter(rpc_keys::Column::UserId.eq(user.id)) + let uks = rpc_key::Entity::find() + .filter(rpc_key::Column::UserId.eq(user.id)) .all(&db_conn) .await .context("failed loading user's key")?; @@ -712,17 +712,17 @@ pub async fn user_revert_logs_get( let uks: Vec<_> = uks.into_iter().map(|x| x.id).collect(); // get paginated logs - let q = revert_logs::Entity::find() - .filter(revert_logs::Column::Timestamp.gte(query_start)) - .filter(revert_logs::Column::RpcKeyId.is_in(uks)) - .order_by_asc(revert_logs::Column::Timestamp); + let q = revert_log::Entity::find() + .filter(revert_log::Column::Timestamp.gte(query_start)) + .filter(revert_log::Column::RpcKeyId.is_in(uks)) + .order_by_asc(revert_log::Column::Timestamp); let q = if chain_id == 0 { // don't do anything q } else { // filter on chain id - q.filter(revert_logs::Column::ChainId.eq(chain_id)) + q.filter(revert_log::Column::ChainId.eq(chain_id)) }; let revert_logs = q.paginate(&db_conn, page_size).fetch_page(page).await?; diff --git a/web3_proxy/src/rpcs/blockchain.rs b/web3_proxy/src/rpcs/blockchain.rs index 7aeb2481..a858a382 100644 --- a/web3_proxy/src/rpcs/blockchain.rs +++ b/web3_proxy/src/rpcs/blockchain.rs @@ -402,7 +402,7 @@ impl Web3Connections { // if we get here, something is wrong. clear synced connections let empty_synced_connections = SyncedConnections::default(); - let old_synced_connections = self + let _ = self .synced_connections .swap(Arc::new(empty_synced_connections)); diff --git a/web3_proxy/src/rpcs/request.rs b/web3_proxy/src/rpcs/request.rs index 2218e9c4..5675e39e 100644 --- a/web3_proxy/src/rpcs/request.rs +++ b/web3_proxy/src/rpcs/request.rs @@ -4,7 +4,7 @@ use crate::frontend::authorization::AuthorizedRequest; use crate::metered::{JsonRpcErrorCount, ProviderErrorCount}; use anyhow::Context; use chrono::Utc; -use entities::revert_logs; +use entities::revert_log; use entities::sea_orm_active_enums::Method; use ethers::providers::{HttpClientError, ProviderError, WsClientError}; use ethers::types::{Address, Bytes}; @@ -12,9 +12,7 @@ use metered::metered; use metered::HitCount; use metered::ResponseTime; use metered::Throughput; -use num_traits::cast::FromPrimitive; use rand::Rng; -use sea_orm::prelude::Decimal; use sea_orm::ActiveEnum; use sea_orm::ActiveModelTrait; use serde_json::json; @@ -94,7 +92,7 @@ impl AuthorizedRequest { .expect("address should always convert to a Vec"); let call_data = params.data.map(|x| format!("{}", x)); - let rl = revert_logs::ActiveModel { + let rl = revert_log::ActiveModel { rpc_key_id: sea_orm::Set(authorized_request.rpc_key_id), method: sea_orm::Set(method), to: sea_orm::Set(to), @@ -222,16 +220,13 @@ impl OpenRequestHandle { } else { let log_revert_chance = y.log_revert_chance; - if log_revert_chance.is_zero() { + if log_revert_chance == 0.0 { trace!(%method, "no chance. skipping save on revert"); RequestErrorHandler::DebugLevel - } else if log_revert_chance == Decimal::ONE { + } else if log_revert_chance == 1.0 { trace!(%method, "gaurenteed chance. SAVING on revert"); error_handler - } else if Decimal::from_f32(rand::thread_rng().gen_range(0.0f32..=1.0)) - .expect("f32 should always convert to a Decimal") - > log_revert_chance - { + } else if rand::thread_rng().gen_range(0.0f64..=1.0) > log_revert_chance { trace!(%method, "missed chance. skipping save on revert"); RequestErrorHandler::DebugLevel } else { diff --git a/web3_proxy/src/user_queries.rs b/web3_proxy/src/user_queries.rs index 4f7807aa..8e6ab886 100644 --- a/web3_proxy/src/user_queries.rs +++ b/web3_proxy/src/user_queries.rs @@ -4,7 +4,7 @@ use axum::{ TypedHeader, }; use chrono::NaiveDateTime; -use entities::{rpc_accounting, rpc_keys}; +use entities::{rpc_accounting, rpc_key}; use hashbrown::HashMap; use migration::Expr; use num::Zero; @@ -272,11 +272,11 @@ pub async fn get_aggregate_rpc_stats_from_params( // TODO: are these joins correct? // TODO: what about keys where they are the secondary users? let q = q - .join(JoinType::InnerJoin, rpc_accounting::Relation::RpcKeys.def()) - .column(rpc_keys::Column::UserId) - .group_by(rpc_keys::Column::UserId); + .join(JoinType::InnerJoin, rpc_accounting::Relation::RpcKey.def()) + .column(rpc_key::Column::UserId) + .group_by(rpc_key::Column::UserId); - let condition = condition.add(rpc_keys::Column::UserId.eq(user_id)); + let condition = condition.add(rpc_key::Column::UserId.eq(user_id)); (condition, q) }; @@ -394,21 +394,20 @@ pub async fn get_detailed_stats( // TODO: move authentication here? // TODO: what about keys where this user is a secondary user? let q = q - .join(JoinType::InnerJoin, rpc_accounting::Relation::RpcKeys.def()) - .column(rpc_keys::Column::UserId) - .group_by(rpc_keys::Column::UserId); + .join(JoinType::InnerJoin, rpc_accounting::Relation::RpcKey.def()) + .column(rpc_key::Column::UserId) + .group_by(rpc_key::Column::UserId); - let condition = condition.add(rpc_keys::Column::UserId.eq(user_id)); + let condition = condition.add(rpc_key::Column::UserId.eq(user_id)); let q = if rpc_key_id == 0 { - q.column(rpc_keys::Column::UserId) - .group_by(rpc_keys::Column::UserId) + q.column(rpc_key::Column::UserId) + .group_by(rpc_key::Column::UserId) } else { response.insert("rpc_key_id", serde_json::to_value(rpc_key_id)?); // no need to group_by user_id when we are grouping by key_id - q.column(rpc_keys::Column::Id) - .group_by(rpc_keys::Column::Id) + q.column(rpc_key::Column::Id).group_by(rpc_key::Column::Id) }; (condition, q)