From b90f80f46b9a4c568ad9ac9b105767068cf903f3 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Sat, 6 Aug 2022 00:07:12 +0000 Subject: [PATCH] back to ids i still think uuids are a better idea, but sea orm has some kinks to work out --- entities/src/block_list.rs | 7 +- entities/src/secondary_user.rs | 11 +- entities/src/user.rs | 7 +- entities/src/user_keys.rs | 10 +- .../src/m20220101_000001_create_table.rs | 62 ++++++----- web3_proxy/Cargo.toml | 1 + web3_proxy/src/app.rs | 45 +++++--- web3_proxy/src/bin/web3_proxy_cli.rs | 100 ------------------ .../src/bin/web3_proxy_cli/create_user.rs | 54 ++++++++++ web3_proxy/src/bin/web3_proxy_cli/main.rs | 62 +++++++++++ web3_proxy/src/bin/web3_proxy_cli/two.rs | 16 +++ web3_proxy/src/frontend/mod.rs | 2 +- web3_proxy/src/frontend/users.rs | 2 +- web3_proxy/src/jsonrpc.rs | 5 +- 14 files changed, 217 insertions(+), 167 deletions(-) delete mode 100644 web3_proxy/src/bin/web3_proxy_cli.rs create mode 100644 web3_proxy/src/bin/web3_proxy_cli/create_user.rs create mode 100644 web3_proxy/src/bin/web3_proxy_cli/main.rs create mode 100644 web3_proxy/src/bin/web3_proxy_cli/two.rs diff --git a/entities/src/block_list.rs b/entities/src/block_list.rs index 600015be..32fa5cd1 100644 --- a/entities/src/block_list.rs +++ b/entities/src/block_list.rs @@ -2,15 +2,14 @@ use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; -use sea_orm::prelude::Uuid; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] #[sea_orm(table_name = "block_list")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub uuid: Uuid, + #[sea_orm(primary_key)] + pub id: i64, #[sea_orm(unique)] - pub address: String, + pub address: Vec, pub description: Option, } diff --git a/entities/src/secondary_user.rs b/entities/src/secondary_user.rs index 32ce425a..cb4f3f8f 100644 --- a/entities/src/secondary_user.rs +++ b/entities/src/secondary_user.rs @@ -2,16 +2,15 @@ use super::sea_orm_active_enums::Role; use sea_orm::entity::prelude::*; -use sea_orm::prelude::Uuid; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] #[sea_orm(table_name = "secondary_user")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub uuid: Uuid, - pub user_id: Uuid, - pub address: String, + #[sea_orm(primary_key)] + pub id: i64, + pub user_id: i64, + pub address: Vec, pub description: Option, pub email: Option, pub role: Role, @@ -22,7 +21,7 @@ pub enum Relation { #[sea_orm( belongs_to = "super::user::Entity", from = "Column::UserId", - to = "super::user::Column::Uuid", + to = "super::user::Column::Id", on_update = "NoAction", on_delete = "NoAction" )] diff --git a/entities/src/user.rs b/entities/src/user.rs index 0f34e7b7..fa6cc96c 100644 --- a/entities/src/user.rs +++ b/entities/src/user.rs @@ -2,15 +2,14 @@ use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; -use sea_orm::prelude::Uuid; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] #[sea_orm(table_name = "user")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub uuid: Uuid, + #[sea_orm(primary_key)] + pub id: i64, #[sea_orm(unique)] - pub address: String, + pub address: Vec, pub description: Option, pub email: Option, } diff --git a/entities/src/user_keys.rs b/entities/src/user_keys.rs index 48919bbf..a1aa9465 100644 --- a/entities/src/user_keys.rs +++ b/entities/src/user_keys.rs @@ -7,9 +7,9 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] #[sea_orm(table_name = "user_keys")] pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub uuid: Uuid, - pub user_uuid: Uuid, + #[sea_orm(primary_key)] + pub id: i64, + pub user_id: i64, #[sea_orm(unique)] pub api_key: Uuid, pub description: Option, @@ -21,8 +21,8 @@ pub struct Model { pub enum Relation { #[sea_orm( belongs_to = "super::user::Entity", - from = "Column::UserUuid", - to = "super::user::Column::Uuid", + from = "Column::UserId", + to = "super::user::Column::Id", on_update = "NoAction", on_delete = "NoAction" )] diff --git a/migration/src/m20220101_000001_create_table.rs b/migration/src/m20220101_000001_create_table.rs index 1a5b7e51..ea628d41 100644 --- a/migration/src/m20220101_000001_create_table.rs +++ b/migration/src/m20220101_000001_create_table.rs @@ -12,15 +12,15 @@ impl MigrationTrait for Migration { Table::create() .table(User::Table) .col( - ColumnDef::new(User::Uuid) - .uuid() + ColumnDef::new(User::Id) + .big_integer() .not_null() - .extra("DEFAULT (UUID_TO_BIN(UUID()))".to_string()) - .primary_key(), + .primary_key() + .auto_increment(), ) .col( ColumnDef::new(User::Address) - .string_len(42) + .binary_len(20) .not_null() .unique_key(), ) @@ -36,16 +36,20 @@ impl MigrationTrait for Migration { Table::create() .table(SecondaryUser::Table) .col( - ColumnDef::new(SecondaryUser::Uuid) - .uuid() + ColumnDef::new(SecondaryUser::Id) + .big_integer() .not_null() - .extra("DEFAULT (UUID_TO_BIN(UUID()))".to_string()) - .primary_key(), + .primary_key() + .auto_increment(), + ) + .col( + ColumnDef::new(SecondaryUser::UserId) + .big_integer() + .not_null(), ) - .col(ColumnDef::new(SecondaryUser::UserId).uuid().not_null()) .col( ColumnDef::new(SecondaryUser::Address) - .string_len(42) + .binary_len(20) .not_null(), ) .col(ColumnDef::new(SecondaryUser::Description).string()) @@ -59,7 +63,7 @@ impl MigrationTrait for Migration { .foreign_key( sea_query::ForeignKey::create() .from(SecondaryUser::Table, SecondaryUser::UserId) - .to(User::Table, User::Uuid), + .to(User::Table, User::Id), ) .to_owned(), ) @@ -71,15 +75,15 @@ impl MigrationTrait for Migration { Table::create() .table(BlockList::Table) .col( - ColumnDef::new(BlockList::Uuid) - .uuid() + ColumnDef::new(BlockList::Id) + .big_integer() .not_null() - .extra("DEFAULT (UUID_TO_BIN(UUID()))".to_string()) - .primary_key(), + .primary_key() + .auto_increment(), ) .col( ColumnDef::new(BlockList::Address) - .string() + .binary_len(20) .not_null() .unique_key(), ) @@ -94,13 +98,13 @@ impl MigrationTrait for Migration { Table::create() .table(UserKeys::Table) .col( - ColumnDef::new(UserKeys::Uuid) - .uuid() + ColumnDef::new(UserKeys::Id) + .big_integer() .not_null() - .extra("DEFAULT (UUID_TO_BIN(UUID()))".to_string()) - .primary_key(), + .primary_key() + .auto_increment(), ) - .col(ColumnDef::new(UserKeys::UserUuid).uuid().not_null()) + .col(ColumnDef::new(UserKeys::UserId).big_integer().not_null()) .col( ColumnDef::new(UserKeys::ApiKey) .uuid() @@ -123,8 +127,8 @@ impl MigrationTrait for Migration { .index(sea_query::Index::create().col(UserKeys::Active)) .foreign_key( sea_query::ForeignKey::create() - .from(UserKeys::Table, UserKeys::UserUuid) - .to(User::Table, User::Uuid), + .from(UserKeys::Table, UserKeys::UserId) + .to(User::Table, User::Id), ) .to_owned(), ) @@ -156,7 +160,7 @@ impl MigrationTrait for Migration { #[derive(Iden)] enum User { Table, - Uuid, + Id, Address, Description, Email, @@ -172,7 +176,7 @@ enum User { #[derive(Iden)] enum SecondaryUser { Table, - Uuid, + Id, UserId, Address, Description, @@ -184,7 +188,7 @@ enum SecondaryUser { #[derive(Iden)] enum BlockList { Table, - Uuid, + Id, Address, Description, } @@ -202,8 +206,8 @@ enum BlockList { #[derive(Iden)] enum UserKeys { Table, - Uuid, - UserUuid, + Id, + UserId, ApiKey, Description, PrivateTxs, diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml index b97e1468..f198e477 100644 --- a/web3_proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -48,6 +48,7 @@ sea-orm = { version = "0.9.1", features = ["macros"] } serde = { version = "1.0.142", features = [] } serde_json = { version = "1.0.83", default-features = false, features = ["alloc", "raw_value"] } tokio = { version = "1.20.1", features = ["full", "tracing"] } +# TODO: make sure this uuid version matches what is in sea orm. PR on sea orm to put builder into prelude uuid = "1.1.2" toml = "0.5.9" tracing = "0.1.36" diff --git a/web3_proxy/src/app.rs b/web3_proxy/src/app.rs index 87e1c428..2e545bbe 100644 --- a/web3_proxy/src/app.rs +++ b/web3_proxy/src/app.rs @@ -14,6 +14,7 @@ use migration::{Migrator, MigratorTrait}; use parking_lot::RwLock; use redis_cell_client::bb8::ErrorSink; use redis_cell_client::{bb8, RedisCellClient, RedisConnectionManager}; +use sea_orm::DatabaseConnection; use serde_json::json; use std::fmt; use std::mem::size_of_val; @@ -236,6 +237,31 @@ fn block_needed( } } +pub async fn get_migrated_db( + db_url: String, + min_connections: u32, +) -> anyhow::Result { + let mut db_opt = sea_orm::ConnectOptions::new(db_url); + + // TODO: load all these options from the config file + // TODO: sqlx logging only in debug. way too verbose for production + db_opt + .max_connections(100) + .min_connections(min_connections) + .connect_timeout(Duration::from_secs(8)) + .idle_timeout(Duration::from_secs(8)) + .max_lifetime(Duration::from_secs(60)) + .sqlx_logging(false); + // .sqlx_logging_level(log::LevelFilter::Info); + + let db = sea_orm::Database::connect(db_opt).await?; + + // TODO: if error, roll back? + Migrator::up(&db, None).await?; + + Ok(db) +} + // TODO: think more about TxState. d #[derive(Clone)] pub enum TxState { @@ -296,24 +322,11 @@ impl Web3ProxyApp { )> { // first, we connect to mysql and make sure the latest migrations have run let db_conn = if let Some(db_url) = app_config.shared.db_url { - let mut db_opt = sea_orm::ConnectOptions::new(db_url); + let min_connections = num_workers.try_into()?; - // TODO: load all these options from the config file - db_opt - .max_connections(100) - .min_connections(num_workers.try_into()?) - .connect_timeout(Duration::from_secs(8)) - .idle_timeout(Duration::from_secs(8)) - .max_lifetime(Duration::from_secs(60)) - .sqlx_logging(true); - // .sqlx_logging_level(log::LevelFilter::Info); + let db = get_migrated_db(db_url, min_connections).await?; - let db_conn = sea_orm::Database::connect(db_opt).await?; - - // TODO: if error, roll back - Migrator::up(&db_conn, None).await?; - - Some(db_conn) + Some(db) } else { info!("no database"); None diff --git a/web3_proxy/src/bin/web3_proxy_cli.rs b/web3_proxy/src/bin/web3_proxy_cli.rs deleted file mode 100644 index 1c22c81b..00000000 --- a/web3_proxy/src/bin/web3_proxy_cli.rs +++ /dev/null @@ -1,100 +0,0 @@ -use argh::FromArgs; -use entities::{user, user_keys}; -use fstrings::{format_args_f, println_f}; -use sea_orm::ActiveModelTrait; -use web3_proxy::users::new_api_key; - -#[derive(Debug, FromArgs)] -/// Command line interface for admins to interact with web3-proxy -pub struct TopConfig { - /// what host the client should connect to - #[argh( - option, - default = "\"mysql://root:dev_web3_proxy@127.0.0.1:3306/dev_web3_proxy\".to_string()" - )] - pub db_url: String, - - /// this one cli can do multiple things - #[argh(subcommand)] - sub_command: SubCommand, -} - -#[derive(FromArgs, PartialEq, Debug)] -#[argh(subcommand)] -enum SubCommand { - CreateUser(CreateUserSubCommand), - Two(SubCommandTwo), - // TODO: sub command to downgrade migrations? - // TODO: sub command to add new api keys to an existing user? -} - -#[derive(FromArgs, PartialEq, Debug)] -/// First subcommand. -#[argh(subcommand, name = "create_user")] -struct CreateUserSubCommand { - #[argh(option)] - /// the user's ethereum address - address: String, - - #[argh(option)] - /// the user's optional email - email: Option, -} - -impl CreateUserSubCommand { - async fn main(self, db: &sea_orm::DatabaseConnection) -> anyhow::Result<()> { - let u = user::ActiveModel { - address: sea_orm::Set(self.address), - email: sea_orm::Set(self.email), - ..Default::default() - }; - - // TODO: proper error message - let u = u.insert(db).await?; - - println_f!("user: {u:?}"); - - // create a key for the new user - let uk = user_keys::ActiveModel { - user_uuid: sea_orm::Set(u.uuid), - api_key: sea_orm::Set(new_api_key()), - ..Default::default() - }; - - println_f!("user key: {uk:?}"); - - Ok(()) - } -} - -#[derive(FromArgs, PartialEq, Debug)] -/// Second subcommand. -#[argh(subcommand, name = "two")] -struct SubCommandTwo { - #[argh(switch)] - /// whether to fooey - fooey: bool, -} - -impl SubCommandTwo { - async fn main(self) -> anyhow::Result<()> { - todo!() - } -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let cli_config: TopConfig = argh::from_env(); - - println!("hello, {}", cli_config.db_url); - - match cli_config.sub_command { - SubCommand::CreateUser(x) => { - // TODO: more advanced settings - let db_conn = sea_orm::Database::connect(cli_config.db_url).await?; - - x.main(&db_conn).await - } - SubCommand::Two(x) => x.main().await, - } -} diff --git a/web3_proxy/src/bin/web3_proxy_cli/create_user.rs b/web3_proxy/src/bin/web3_proxy_cli/create_user.rs new file mode 100644 index 00000000..848cc687 --- /dev/null +++ b/web3_proxy/src/bin/web3_proxy_cli/create_user.rs @@ -0,0 +1,54 @@ +use anyhow::Context; +use argh::FromArgs; +use entities::{user, user_keys}; +use ethers::types::Address; +use fstrings::{format_args_f, println_f}; +use sea_orm::ActiveModelTrait; +use web3_proxy::users::new_api_key; + +#[derive(FromArgs, PartialEq, Debug)] +/// First subcommand. +#[argh(subcommand, name = "create_user")] +pub struct CreateUserSubCommand { + #[argh(option)] + /// the user's ethereum address + address: String, + + #[argh(option)] + /// the user's optional email + email: Option, +} + +impl CreateUserSubCommand { + pub async fn main(self, db: &sea_orm::DatabaseConnection) -> anyhow::Result<()> { + let address = self + .address + .parse::
() + .context("Failed parsing new user address")? + .to_fixed_bytes() + .into(); + + let u = user::ActiveModel { + address: sea_orm::Set(address), + email: sea_orm::Set(self.email), + ..Default::default() + }; + + let u = u.insert(db).await.context("Failed saving new user")?; + + println_f!("user: {u:?}"); + + // create a key for the new user + let uk = user_keys::ActiveModel { + user_id: sea_orm::Set(u.id), + api_key: sea_orm::Set(new_api_key()), + ..Default::default() + }; + + let uk = uk.insert(db).await.context("Failed saving new user key")?; + + println_f!("user key: {uk:?}"); + + Ok(()) + } +} diff --git a/web3_proxy/src/bin/web3_proxy_cli/main.rs b/web3_proxy/src/bin/web3_proxy_cli/main.rs new file mode 100644 index 00000000..1cc0e263 --- /dev/null +++ b/web3_proxy/src/bin/web3_proxy_cli/main.rs @@ -0,0 +1,62 @@ +mod create_user; +mod two; + +use argh::FromArgs; +use tracing::info; +use web3_proxy::app::get_migrated_db; + +#[derive(Debug, FromArgs)] +/// Command line interface for admins to interact with web3-proxy +pub struct TopConfig { + /// what host the client should connect to + #[argh( + option, + default = "\"mysql://root:dev_web3_proxy@127.0.0.1:3306/dev_web3_proxy\".to_string()" + )] + pub db_url: String, + + /// this one cli can do multiple things + #[argh(subcommand)] + sub_command: SubCommand, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand)] +enum SubCommand { + CreateUser(create_user::CreateUserSubCommand), + Two(two::SubCommandTwo), + // TODO: sub command to downgrade migrations? + // TODO: sub command to add new api keys to an existing user? +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // if RUST_LOG isn't set, configure a default + // TODO: is there a better way to do this? + if std::env::var("RUST_LOG").is_err() { + // std::env::set_var("RUST_LOG", "info,web3_proxy=debug,web3_proxy_cli=debug"); + std::env::set_var("RUST_LOG", "info,web3_proxy=debug,web3_proxy_cli=debug"); + } + + // install global collector configured based on RUST_LOG env var. + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .compact() + .init(); + + // this probably won't matter for us in docker, but better safe than sorry + fdlimit::raise_fd_limit(); + + let cli_config: TopConfig = argh::from_env(); + + info!("hello, {}", cli_config.db_url); + + match cli_config.sub_command { + SubCommand::CreateUser(x) => { + let db = get_migrated_db(cli_config.db_url, 1).await?; + + x.main(&db).await + } + SubCommand::Two(x) => x.main().await, + } +} diff --git a/web3_proxy/src/bin/web3_proxy_cli/two.rs b/web3_proxy/src/bin/web3_proxy_cli/two.rs new file mode 100644 index 00000000..ed0ee02d --- /dev/null +++ b/web3_proxy/src/bin/web3_proxy_cli/two.rs @@ -0,0 +1,16 @@ +use argh::FromArgs; + +#[derive(FromArgs, PartialEq, Debug)] +/// Second subcommand. +#[argh(subcommand, name = "two")] +pub struct SubCommandTwo { + #[argh(switch)] + /// whether to fooey + fooey: bool, +} + +impl SubCommandTwo { + pub async fn main(self) -> anyhow::Result<()> { + todo!() + } +} diff --git a/web3_proxy/src/frontend/mod.rs b/web3_proxy/src/frontend/mod.rs index 8b6cda3f..b72d43e6 100644 --- a/web3_proxy/src/frontend/mod.rs +++ b/web3_proxy/src/frontend/mod.rs @@ -40,7 +40,7 @@ pub async fn rate_limit_by_key( // TODO: probably want a cache on this match user_keys::Entity::find() .select_only() - .column(user_keys::Column::UserUuid) + .column(user_keys::Column::UserId) .filter(user_keys::Column::ApiKey.eq(user_key)) .filter(user_keys::Column::Active.eq(true)) .one(db) diff --git a/web3_proxy/src/frontend/users.rs b/web3_proxy/src/frontend/users.rs index 540e335b..95b4af2e 100644 --- a/web3_proxy/src/frontend/users.rs +++ b/web3_proxy/src/frontend/users.rs @@ -44,7 +44,7 @@ pub async fn create_user( } let user = user::ActiveModel { - address: sea_orm::Set(payload.address.to_string()), + address: sea_orm::Set(payload.address.to_fixed_bytes().into()), email: sea_orm::Set(payload.email), ..Default::default() }; diff --git a/web3_proxy/src/jsonrpc.rs b/web3_proxy/src/jsonrpc.rs index dc81963e..59c18ddf 100644 --- a/web3_proxy/src/jsonrpc.rs +++ b/web3_proxy/src/jsonrpc.rs @@ -4,6 +4,7 @@ use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}; use serde::Serialize; use serde_json::value::RawValue; use std::fmt; +use tracing::warn; #[derive(Clone, serde::Deserialize)] pub struct JsonRpcRequest { @@ -166,6 +167,8 @@ impl JsonRpcForwardedResponse { pub fn from_anyhow_error(err: anyhow::Error, id: Box) -> Self { let err = format!("{:?}", err); + warn!("forwarding error. {:?}", err); + JsonRpcForwardedResponse { jsonrpc: "2.0".to_string(), id, @@ -173,7 +176,7 @@ impl JsonRpcForwardedResponse { error: Some(JsonRpcErrorData { // TODO: set this jsonrpc error code to match the http status code code: -32099, - message: err, + message: "internal server error".to_string(), data: None, }), }