From e295307afc0977197c5bff71c6dc4619f1fb11a6 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Fri, 5 Aug 2022 19:22:23 +0000 Subject: [PATCH] fix directory structure --- Cargo.lock | 26 ++++- Cargo.toml | 2 +- .../src/m20220101_000001_create_table.rs | 2 +- web3-proxy/src/bin/users.rs | 7 -- {web3-proxy => web3_proxy}/Cargo.toml | 6 +- .../examples/subscribe_blocks.rs | 0 .../examples/watch_blocks.rs | 0 {web3-proxy => web3_proxy}/src/app.rs | 0 {web3-proxy => web3_proxy}/src/bb8_helpers.rs | 0 .../src/bin/web3_proxy.rs | 17 +-- web3_proxy/src/bin/web3_proxy_cli.rs | 106 ++++++++++++++++++ {web3-proxy => web3_proxy}/src/config.rs | 0 {web3-proxy => web3_proxy}/src/connection.rs | 0 {web3-proxy => web3_proxy}/src/connections.rs | 0 {web3-proxy => web3_proxy}/src/firewall.rs | 0 .../src/frontend/errors.rs | 0 .../src/frontend/http.rs | 1 + .../src/frontend/http_proxy.rs | 2 +- .../src/frontend/mod.rs | 56 ++++++--- .../src/frontend/users.rs | 0 .../src/frontend/ws_proxy.rs | 2 +- {web3-proxy => web3_proxy}/src/jsonrpc.rs | 0 web3_proxy/src/lib.rs | 9 ++ web3_proxy/src/users.rs | 11 ++ 24 files changed, 207 insertions(+), 40 deletions(-) delete mode 100644 web3-proxy/src/bin/users.rs rename {web3-proxy => web3_proxy}/Cargo.toml (95%) rename {web3-proxy => web3_proxy}/examples/subscribe_blocks.rs (100%) rename {web3-proxy => web3_proxy}/examples/watch_blocks.rs (100%) rename {web3-proxy => web3_proxy}/src/app.rs (100%) rename {web3-proxy => web3_proxy}/src/bb8_helpers.rs (100%) rename web3-proxy/src/main.rs => web3_proxy/src/bin/web3_proxy.rs (96%) create mode 100644 web3_proxy/src/bin/web3_proxy_cli.rs rename {web3-proxy => web3_proxy}/src/config.rs (100%) rename {web3-proxy => web3_proxy}/src/connection.rs (100%) rename {web3-proxy => web3_proxy}/src/connections.rs (100%) rename {web3-proxy => web3_proxy}/src/firewall.rs (100%) rename {web3-proxy => web3_proxy}/src/frontend/errors.rs (100%) rename {web3-proxy => web3_proxy}/src/frontend/http.rs (95%) rename {web3-proxy => web3_proxy}/src/frontend/http_proxy.rs (97%) rename {web3-proxy => web3_proxy}/src/frontend/mod.rs (60%) rename {web3-proxy => web3_proxy}/src/frontend/users.rs (100%) rename {web3-proxy => web3_proxy}/src/frontend/ws_proxy.rs (99%) rename {web3-proxy => web3_proxy}/src/jsonrpc.rs (100%) create mode 100644 web3_proxy/src/lib.rs create mode 100644 web3_proxy/src/users.rs diff --git a/Cargo.lock b/Cargo.lock index 614cfa77..b196f89a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1875,6 +1875,28 @@ dependencies = [ "libc", ] +[[package]] +name = "fstrings" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7845a0f15da505ac36baad0486612dab57f8b8d34e19c5470a265bbcdd572ae6" +dependencies = [ + "fstrings-proc-macro", + "proc-macro-hack", +] + +[[package]] +name = "fstrings-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b58c0e7581dc33478a32299182cbe5ae3b8c028be26728a47fb0a113c92d9d" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -5176,7 +5198,6 @@ dependencies = [ "anyhow", "arc-swap", "argh", - "async-std", "axum", "axum-client-ip", "counter", @@ -5187,6 +5208,7 @@ dependencies = [ "ethers", "fdlimit", "flume", + "fstrings", "futures", "hashbrown", "indexmap", @@ -5197,6 +5219,7 @@ dependencies = [ "parking_lot 0.12.1", "petgraph", "proctitle", + "rand", "redis-cell-client", "regex", "reqwest", @@ -5212,6 +5235,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "uuid 1.1.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 102a84c0..63f29dcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = [ "migration", "linkedhashmap", "redis-cell-client", - "web3-proxy", + "web3_proxy", ] # TODO: enable lto (and maybe other things proven with benchmarks) once rapid development is done diff --git a/migration/src/m20220101_000001_create_table.rs b/migration/src/m20220101_000001_create_table.rs index 5b934fc8..16d61c76 100644 --- a/migration/src/m20220101_000001_create_table.rs +++ b/migration/src/m20220101_000001_create_table.rs @@ -107,7 +107,7 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(UserKeys::UserUuid).uuid().not_null()) .col( ColumnDef::new(UserKeys::ApiKey) - .string_len(32) + .uuid() .not_null() .unique_key(), ) diff --git a/web3-proxy/src/bin/users.rs b/web3-proxy/src/bin/users.rs deleted file mode 100644 index 2ebc7713..00000000 --- a/web3-proxy/src/bin/users.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Manage users. -//! -//! While most user management will (and should) happen through the web api, - -fn main() { - println!("hello, world"); -} diff --git a/web3-proxy/Cargo.toml b/web3_proxy/Cargo.toml similarity index 95% rename from web3-proxy/Cargo.toml rename to web3_proxy/Cargo.toml index 3411f338..b97e1468 100644 --- a/web3-proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -2,7 +2,7 @@ name = "web3-proxy" version = "0.1.0" edition = "2021" -default-run = "web3-proxy" +default-run = "web3_proxy" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -29,6 +29,7 @@ ethers = { version = "0.17.0", features = ["rustls", "ws"] } fdlimit = "0.2.1" flume = "0.10.14" futures = { version = "0.3.21", features = ["thread-pool"] } +fstrings = "0.2.3" hashbrown = { version = "0.12.3", features = ["serde"] } indexmap = "1.9.1" linkedhashmap = { path = "../linkedhashmap", features = ["inline-more"] } @@ -37,6 +38,7 @@ num = "0.4.0" parking_lot = { version = "0.12.1", features = ["arc_lock"] } petgraph = "0.6.2" proctitle = "0.1.1" +rand = "0.8.5" # TODO: regex has several "perf" features that we might want to use regex = "1.6.0" reqwest = { version = "0.11.11", default-features = false, features = ["json", "tokio-rustls"] } @@ -46,7 +48,7 @@ sea-orm = { version = "0.9.1", features = ["macros"] } serde = { version = "1.0.142", features = [] } serde_json = { version = "1.0.83", default-features = false, features = ["alloc", "raw_value"] } tokio = { version = "1.20.1", features = ["full", "tracing"] } -async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } +uuid = "1.1.2" toml = "0.5.9" tracing = "0.1.36" # TODO: tracing-subscriber has serde and serde_json features that we might want to use diff --git a/web3-proxy/examples/subscribe_blocks.rs b/web3_proxy/examples/subscribe_blocks.rs similarity index 100% rename from web3-proxy/examples/subscribe_blocks.rs rename to web3_proxy/examples/subscribe_blocks.rs diff --git a/web3-proxy/examples/watch_blocks.rs b/web3_proxy/examples/watch_blocks.rs similarity index 100% rename from web3-proxy/examples/watch_blocks.rs rename to web3_proxy/examples/watch_blocks.rs diff --git a/web3-proxy/src/app.rs b/web3_proxy/src/app.rs similarity index 100% rename from web3-proxy/src/app.rs rename to web3_proxy/src/app.rs diff --git a/web3-proxy/src/bb8_helpers.rs b/web3_proxy/src/bb8_helpers.rs similarity index 100% rename from web3-proxy/src/bb8_helpers.rs rename to web3_proxy/src/bb8_helpers.rs diff --git a/web3-proxy/src/main.rs b/web3_proxy/src/bin/web3_proxy.rs similarity index 96% rename from web3-proxy/src/main.rs rename to web3_proxy/src/bin/web3_proxy.rs index c312d168..ae6222cf 100644 --- a/web3-proxy/src/main.rs +++ b/web3_proxy/src/bin/web3_proxy.rs @@ -8,15 +8,6 @@ //#![warn(missing_docs)] #![forbid(unsafe_code)] -pub mod app; -pub mod bb8_helpers; -pub mod config; -pub mod connection; -pub mod connections; -pub mod firewall; -pub mod frontend; -pub mod jsonrpc; - use parking_lot::deadlock; use std::fs; use std::sync::atomic::{self, AtomicUsize}; @@ -25,9 +16,9 @@ use std::time::Duration; use tokio::runtime; use tracing::{debug, info}; use tracing_subscriber::EnvFilter; - -use crate::app::{flatten_handle, Web3ProxyApp}; -use crate::config::{AppConfig, CliConfig}; +use web3_proxy::app::{flatten_handle, Web3ProxyApp}; +use web3_proxy::config::{AppConfig, CliConfig}; +use web3_proxy::frontend; fn run( shutdown_receiver: flume::Receiver<()>, @@ -156,7 +147,7 @@ mod tests { use hashbrown::HashMap; use std::env; - use crate::config::{RpcSharedConfig, Web3ConnectionConfig}; + use web3_proxy::config::{RpcSharedConfig, Web3ConnectionConfig}; use super::*; diff --git a/web3_proxy/src/bin/web3_proxy_cli.rs b/web3_proxy/src/bin/web3_proxy_cli.rs new file mode 100644 index 00000000..7cfb619f --- /dev/null +++ b/web3_proxy/src/bin/web3_proxy_cli.rs @@ -0,0 +1,106 @@ +use argh::FromArgs; +use entities::{user, user_keys}; +use ethers::prelude::Bytes; +use fstrings::{format_args_f, println_f}; +use rand::prelude::*; +use sea_orm::{prelude::Uuid, ActiveModelTrait}; +use web3_proxy::users::new_api_key; + +#[derive(Debug, FromArgs)] +/// Command line interface for admins to interact with web3-proxy +pub struct TopConfig { + /// what host the client should connect to + #[argh( + option, + default = "\"mysql://root:dev_web3_proxy@127.0.0.1:3306/dev_web3_proxy\".to_string()" + )] + pub db_url: String, + + /// this one cli can do multiple things + #[argh(subcommand)] + sub_command: SubCommand, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand)] +enum SubCommand { + CreateUser(CreateUserSubCommand), + Two(SubCommandTwo), + // TODO: sub command to downgrade migrations? + // TODO: sub command to add new api keys to an existing user? +} + +#[derive(FromArgs, PartialEq, Debug)] +/// First subcommand. +#[argh(subcommand, name = "create_user")] +struct CreateUserSubCommand { + #[argh(option)] + /// the user's ethereum address + address: String, + + #[argh(option)] + /// the user's optional email + email: Option, +} + +impl CreateUserSubCommand { + async fn main(self, db: &sea_orm::DatabaseConnection) -> anyhow::Result<()> { + let u = user::ActiveModel { + address: sea_orm::Set(self.address), + email: sea_orm::Set(self.email), + ..Default::default() + }; + + // TODO: proper error message + let u = u.insert(db).await?; + + println_f!("user: {u:?}"); + + // TODO: use chacha20? + let api_key = new_api_key(); + + // TODO: create a key, too + // TODO: why are active and private_txs ints instead of bools? + let uk = user_keys::ActiveModel { + user_uuid: sea_orm::Set(u.uuid), + // api_key: api_key, + ..Default::default() + }; + + println_f!("user key: {uk:?}"); + + Ok(()) + } +} + +#[derive(FromArgs, PartialEq, Debug)] +/// Second subcommand. +#[argh(subcommand, name = "two")] +struct SubCommandTwo { + #[argh(switch)] + /// whether to fooey + fooey: bool, +} + +impl SubCommandTwo { + async fn main(self) -> anyhow::Result<()> { + todo!() + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let cli_config: TopConfig = argh::from_env(); + + println!("hello, {}", cli_config.db_url); + + match cli_config.sub_command { + SubCommand::CreateUser(x) => { + // TODO: more advanced settings + let db_conn = sea_orm::Database::connect(cli_config.db_url).await?; + + x.main(&db_conn).await + } + SubCommand::Two(x) => x.main().await, + } +} diff --git a/web3-proxy/src/config.rs b/web3_proxy/src/config.rs similarity index 100% rename from web3-proxy/src/config.rs rename to web3_proxy/src/config.rs diff --git a/web3-proxy/src/connection.rs b/web3_proxy/src/connection.rs similarity index 100% rename from web3-proxy/src/connection.rs rename to web3_proxy/src/connection.rs diff --git a/web3-proxy/src/connections.rs b/web3_proxy/src/connections.rs similarity index 100% rename from web3-proxy/src/connections.rs rename to web3_proxy/src/connections.rs diff --git a/web3-proxy/src/firewall.rs b/web3_proxy/src/firewall.rs similarity index 100% rename from web3-proxy/src/firewall.rs rename to web3_proxy/src/firewall.rs diff --git a/web3-proxy/src/frontend/errors.rs b/web3_proxy/src/frontend/errors.rs similarity index 100% rename from web3-proxy/src/frontend/errors.rs rename to web3_proxy/src/frontend/errors.rs diff --git a/web3-proxy/src/frontend/http.rs b/web3_proxy/src/frontend/http.rs similarity index 95% rename from web3-proxy/src/frontend/http.rs rename to web3_proxy/src/frontend/http.rs index db6c3bc6..de5fc935 100644 --- a/web3-proxy/src/frontend/http.rs +++ b/web3_proxy/src/frontend/http.rs @@ -14,6 +14,7 @@ pub async fn health(Extension(app): Extension>) -> impl IntoRe } /// Very basic status page +/// TODO: replace this with proper stats and monitoring pub async fn status(Extension(app): Extension>) -> impl IntoResponse { // TODO: what else should we include? uptime? prometheus? let balanced_rpcs = app.balanced_rpcs(); diff --git a/web3-proxy/src/frontend/http_proxy.rs b/web3_proxy/src/frontend/http_proxy.rs similarity index 97% rename from web3-proxy/src/frontend/http_proxy.rs rename to web3_proxy/src/frontend/http_proxy.rs index b47a9c9d..ec40130c 100644 --- a/web3-proxy/src/frontend/http_proxy.rs +++ b/web3_proxy/src/frontend/http_proxy.rs @@ -6,7 +6,7 @@ use super::errors::handle_anyhow_error; use super::{rate_limit_by_ip, rate_limit_by_key}; use crate::{app::Web3ProxyApp, jsonrpc::JsonRpcRequestEnum}; -pub async fn proxy_web3_rpc( +pub async fn public_proxy_web3_rpc( Json(payload): Json, Extension(app): Extension>, ClientIp(ip): ClientIp, diff --git a/web3-proxy/src/frontend/mod.rs b/web3_proxy/src/frontend/mod.rs similarity index 60% rename from web3-proxy/src/frontend/mod.rs rename to web3_proxy/src/frontend/mod.rs index 24c4165c..8b6cda3f 100644 --- a/web3-proxy/src/frontend/mod.rs +++ b/web3_proxy/src/frontend/mod.rs @@ -11,7 +11,9 @@ use axum::{ routing::{get, post}, Extension, Router, }; +use entities::user_keys; use reqwest::StatusCode; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QuerySelect}; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use tracing::debug; @@ -26,13 +28,48 @@ pub async fn rate_limit_by_ip(app: &Web3ProxyApp, ip: &IpAddr) -> Result<(), imp rate_limit_by_key(app, &rate_limiter_key).await } +/// if Ok(()), rate limits are acceptable +/// if Err(response), rate limits exceeded pub async fn rate_limit_by_key( app: &Web3ProxyApp, user_key: &str, ) -> Result<(), impl IntoResponse> { let db = app.db_conn(); - // TODO: query the db to make sure this key is active + // query the db to make sure this key is active + // TODO: probably want a cache on this + match user_keys::Entity::find() + .select_only() + .column(user_keys::Column::UserUuid) + .filter(user_keys::Column::ApiKey.eq(user_key)) + .filter(user_keys::Column::Active.eq(true)) + .one(db) + .await + { + Ok(Some(_)) => { + // user key is valid + } + Ok(None) => { + // invalid user key + // TODO: rate limit by ip here, too? maybe tarpit? + return Err(handle_anyhow_error( + Some(StatusCode::FORBIDDEN), + None, + anyhow::anyhow!("unknown api key"), + ) + .await + .into_response()); + } + Err(e) => { + return Err(handle_anyhow_error( + Some(StatusCode::INTERNAL_SERVER_ERROR), + None, + e.into(), + ) + .await + .into_response()); + } + } if let Some(rate_limiter) = app.rate_limiter() { if rate_limiter.throttle_key(user_key).await.is_err() { @@ -55,23 +92,15 @@ pub async fn rate_limit_by_key( } pub async fn run(port: u16, proxy_app: Arc) -> anyhow::Result<()> { - // TODO: check auth (from authp?) here // build our application with a route // order most to least common let app = Router::new() - // `POST /` goes to `proxy_web3_rpc` - .route("/", post(http_proxy::proxy_web3_rpc)) - // `websocket /` goes to `proxy_web3_ws` - .route("/", get(ws_proxy::websocket_handler)) - // `POST /rpc/:key` goes to `proxy_web3_rpc` - .route("/rpc/:key", post(http_proxy::user_proxy_web3_rpc)) - // `websocket /` goes to `proxy_web3_ws` - .route("/rpc/:key", get(ws_proxy::user_websocket_handler)) - // `GET /health` goes to `health` + .route("/", post(http_proxy::public_proxy_web3_rpc)) + .route("/", get(ws_proxy::public_websocket_handler)) + .route("/u/:key", post(http_proxy::user_proxy_web3_rpc)) + .route("/u/:key", get(ws_proxy::user_websocket_handler)) .route("/health", get(http::health)) - // `GET /status` goes to `status` .route("/status", get(http::status)) - // `POST /users` goes to `create_user` .route("/users", post(users::create_user)) .layer(Extension(proxy_app)); @@ -80,6 +109,7 @@ pub async fn run(port: u16, proxy_app: Arc) -> anyhow::Result<()> // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` + // TODO: allow only listening on localhost? let addr = SocketAddr::from(([0, 0, 0, 0], port)); debug!("listening on port {}", port); // TODO: into_make_service is enough if we always run behind a proxy. make into_make_service_with_connect_info optional? diff --git a/web3-proxy/src/frontend/users.rs b/web3_proxy/src/frontend/users.rs similarity index 100% rename from web3-proxy/src/frontend/users.rs rename to web3_proxy/src/frontend/users.rs diff --git a/web3-proxy/src/frontend/ws_proxy.rs b/web3_proxy/src/frontend/ws_proxy.rs similarity index 99% rename from web3-proxy/src/frontend/ws_proxy.rs rename to web3_proxy/src/frontend/ws_proxy.rs index 59871531..9894b4c1 100644 --- a/web3-proxy/src/frontend/ws_proxy.rs +++ b/web3_proxy/src/frontend/ws_proxy.rs @@ -22,7 +22,7 @@ use crate::{ use super::{rate_limit_by_ip, rate_limit_by_key}; -pub async fn websocket_handler( +pub async fn public_websocket_handler( Extension(app): Extension>, ClientIp(ip): ClientIp, ws: WebSocketUpgrade, diff --git a/web3-proxy/src/jsonrpc.rs b/web3_proxy/src/jsonrpc.rs similarity index 100% rename from web3-proxy/src/jsonrpc.rs rename to web3_proxy/src/jsonrpc.rs diff --git a/web3_proxy/src/lib.rs b/web3_proxy/src/lib.rs new file mode 100644 index 00000000..22318914 --- /dev/null +++ b/web3_proxy/src/lib.rs @@ -0,0 +1,9 @@ +pub mod app; +pub mod bb8_helpers; +pub mod config; +pub mod connection; +pub mod connections; +pub mod firewall; +pub mod frontend; +pub mod jsonrpc; +pub mod users; diff --git a/web3_proxy/src/users.rs b/web3_proxy/src/users.rs new file mode 100644 index 00000000..c54ea1c3 --- /dev/null +++ b/web3_proxy/src/users.rs @@ -0,0 +1,11 @@ +use rand::prelude::*; +use uuid::{Builder, Uuid}; + +pub fn new_api_key() -> Uuid { + // TODO: chacha20? + let mut rng = thread_rng(); + + let random_bytes = rng.gen(); + + Builder::from_random_bytes(random_bytes).into_uuid() +}