From 0ef728df9349823c3efb8718acdffef0c0638247 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Fri, 20 May 2022 05:27:18 +0000 Subject: [PATCH] axum and compact logs --- Cargo.lock | 222 +----------------------- Cargo.toml | 1 - web3-proxy-frontend/Cargo.toml | 17 -- web3-proxy-frontend/src/main.rs | 68 -------- web3-proxy/Cargo.toml | 8 +- web3-proxy/examples/subscribe_blocks.rs | 6 +- web3-proxy/examples/watch_blocks.rs | 6 +- web3-proxy/src/app.rs | 4 +- web3-proxy/src/main.rs | 59 +------ 9 files changed, 27 insertions(+), 364 deletions(-) delete mode 100644 web3-proxy-frontend/Cargo.toml delete mode 100644 web3-proxy-frontend/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 2e15490c..16c630f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,27 +124,6 @@ dependencies = [ "term", ] -[[package]] -name = "async-stream" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" -dependencies = [ - "async-stream-impl", - "futures-core", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-trait" version = "0.1.53" @@ -221,6 +200,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-tungstenite 0.17.1", "tower", "tower-http", "tower-layer", @@ -689,43 +669,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "console-api" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24cb05777feccbb2642d4f2df44d0505601a2cd88ca517d8c913f263a5a8dc8b" -dependencies = [ - "prost", - "prost-types", - "tonic", - "tracing-core", -] - -[[package]] -name = "console-subscriber" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f21a16ee925aa9d2bad2e296beffd6c5b1bfaad50af509d305b8e7f23af20fb" -dependencies = [ - "console-api", - "crossbeam-channel", - "crossbeam-utils", - "futures", - "hdrhistogram", - "humantime", - "parking_lot 0.11.2", - "prost-types", - "serde", - "serde_json", - "thread_local", - "tokio", - "tokio-stream", - "tonic", - "tracing", - "tracing-core", - "tracing-subscriber", -] - [[package]] name = "const-oid" version = "0.7.1" @@ -753,15 +696,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", -] - [[package]] name = "criterion" version = "0.3.5" @@ -1449,18 +1383,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" -[[package]] -name = "flate2" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" -dependencies = [ - "cfg-if", - "crc32fast", - "libc", - "miniz_oxide", -] - [[package]] name = "flume" version = "0.10.12" @@ -1752,19 +1674,6 @@ dependencies = [ "hashbrown 0.12.1", ] -[[package]] -name = "hdrhistogram" -version = "7.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31672b7011be2c4f7456c4ddbcb40e7e9a4a9fad8efe49a6ebaf5f307d0109c0" -dependencies = [ - "base64 0.13.0", - "byteorder", - "flate2", - "nom", - "num-traits", -] - [[package]] name = "headers" version = "0.3.7" @@ -1888,12 +1797,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - [[package]] name = "hyper" version = "0.14.18" @@ -1931,18 +1834,6 @@ dependencies = [ "tokio-rustls", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "idna" version = "0.2.3" @@ -2238,12 +2129,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.5.1" @@ -2315,16 +2200,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" -[[package]] -name = "nom" -version = "7.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nonzero_ext" version = "0.3.0" @@ -2776,39 +2651,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "prost" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc03e116981ff7d8da8e5c220e374587b98d294af7ba7dd7fda761158f00086f" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" -dependencies = [ - "bytes", - "prost", -] - [[package]] name = "quanta" version = "0.9.3" @@ -3711,16 +3553,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "1.7.0" @@ -3820,38 +3652,6 @@ dependencies = [ "serde", ] -[[package]] -name = "tonic" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.0", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "tokio", - "tokio-stream", - "tokio-util 0.7.1", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - [[package]] name = "tower" version = "0.4.12" @@ -3860,11 +3660,8 @@ checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" dependencies = [ "futures-core", "futures-util", - "indexmap", "pin-project", "pin-project-lite", - "rand", - "slab", "tokio", "tokio-util 0.7.1", "tower-layer", @@ -4305,7 +4102,7 @@ dependencies = [ "anyhow", "arc-swap", "argh", - "console-subscriber", + "axum", "dashmap", "derive_more", "ethers", @@ -4324,26 +4121,13 @@ dependencies = [ "serde_json", "tokio", "toml", + "tower", "tracing", "tracing-subscriber", "url", "warp", ] -[[package]] -name = "web3-proxy-frontend" -version = "0.1.0" -dependencies = [ - "axum", - "console-subscriber", - "hashbrown 0.12.1", - "serde", - "serde_json", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "webpki" version = "0.22.0" diff --git a/Cargo.toml b/Cargo.toml index 6814b735..3a68c758 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ members = [ "linkedhashmap", "web3-proxy", - "web3-proxy-frontend", ] # TODO: enable lto (and maybe other things proven with benchmarks) once rapid development is done diff --git a/web3-proxy-frontend/Cargo.toml b/web3-proxy-frontend/Cargo.toml deleted file mode 100644 index 69dae5df..00000000 --- a/web3-proxy-frontend/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "web3-proxy-frontend" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -axum = "0.5.6" -console-subscriber = { version = "0.1.5", features = ["parking_lot"] } -hashbrown = "0.12.1" -serde = { version = "1.0.137", features = [] } -serde_json = { version = "1.0.81", default-features = false, features = ["alloc", "raw_value"] } -tokio = { version = "1.18.2", features = ["full", "tracing"] } -tracing = "0.1.34" -# TODO: tracing-subscriber has serde and serde_json features that we might want to use -tracing-subscriber = { version = "0.3.11", features = ["parking_lot"] } diff --git a/web3-proxy-frontend/src/main.rs b/web3-proxy-frontend/src/main.rs deleted file mode 100644 index c99c8a0a..00000000 --- a/web3-proxy-frontend/src/main.rs +++ /dev/null @@ -1,68 +0,0 @@ -/// this should move into web3-proxy once the basics are working -use axum::{ - http::StatusCode, - response::IntoResponse, - routing::{get, post}, - Json, Router, -}; -use serde::{Deserialize, Serialize}; -use std::net::SocketAddr; - -#[tokio::main] -async fn main() { - // initialize tracing. this is too slow for production, but helpful while developing - console_subscriber::init(); - - // the real app will load the port and other settings out of command flags or a config file - let port = 8544; - - // build our application with a route - let app = Router::new() - // `GET /` goes to `root` - .route("/", get(root)) - // `POST /users` goes to `create_user` - .route("/users", post(create_user)); - - // run our app with hyper - // `axum::Server` is a re-export of `hyper::Server` - let addr = SocketAddr::from(([0, 0, 0, 0], port)); - tracing::debug!("listening on {}", addr); - axum::Server::bind(&addr) - .serve(app.into_make_service()) - .await - .unwrap(); -} - -// basic handler that responds with a static string -async fn root() -> &'static str { - "Hello, World!" -} - -async fn create_user( - // this argument tells axum to parse the request body - // as JSON into a `CreateUser` type - Json(payload): Json, -) -> impl IntoResponse { - // insert your application logic here - let user = User { - id: 1337, - username: payload.username, - }; - - // this will be converted into a JSON response - // with a status code of `201 Created` - (StatusCode::CREATED, Json(user)) -} - -// the input to our `create_user` handler -#[derive(Deserialize)] -struct CreateUser { - username: String, -} - -// the output to our `create_user` handler -#[derive(Serialize)] -struct User { - id: u64, - username: String, -} diff --git a/web3-proxy/Cargo.toml b/web3-proxy/Cargo.toml index ae78f0d2..204466b5 100644 --- a/web3-proxy/Cargo.toml +++ b/web3-proxy/Cargo.toml @@ -9,8 +9,7 @@ edition = "2021" anyhow = "1.0.57" arc-swap = "1.5.0" argh = "0.1.7" -# axum = "*" # TODO: use this instead of warp? -console-subscriber = { version = "0.1.5", features = ["parking_lot"] } +axum = { version = "0.5.6", features = ["serde_json", "tokio-tungstenite"] } dashmap = "5.3.3" derive_more = "0.99.17" ethers = { git = "https://github.com/SatoshiAndKin/ethers-rs", features = ["rustls", "ws"] } @@ -34,7 +33,8 @@ tokio = { version = "1.18.2", features = ["full", "tracing"] } toml = "0.5.9" tracing = "0.1.34" # TODO: tracing-subscriber has serde and serde_json features that we might want to use -tracing-subscriber = { version = "0.3.11", features = ["parking_lot"] } +tracing-subscriber = { version = "0.3.11", features = ["env-filter", "parking_lot"] } url = "2.2.2" -# TODO: replace warp with axum? +# TODO: replace warp with axum warp = "0.3.2" +tower = "*" \ No newline at end of file diff --git a/web3-proxy/examples/subscribe_blocks.rs b/web3-proxy/examples/subscribe_blocks.rs index 01498c26..9511accd 100644 --- a/web3-proxy/examples/subscribe_blocks.rs +++ b/web3-proxy/examples/subscribe_blocks.rs @@ -4,7 +4,11 @@ use std::time::Duration; #[tokio::main] async fn main() -> anyhow::Result<()> { - console_subscriber::init(); + // install global collector configured based on RUST_LOG env var. + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .compact() + .init(); fdlimit::raise_fd_limit(); diff --git a/web3-proxy/examples/watch_blocks.rs b/web3-proxy/examples/watch_blocks.rs index 7488b020..1c4c8994 100644 --- a/web3-proxy/examples/watch_blocks.rs +++ b/web3-proxy/examples/watch_blocks.rs @@ -4,7 +4,11 @@ use std::{str::FromStr, time::Duration}; #[tokio::main] async fn main() -> anyhow::Result<()> { - console_subscriber::init(); + // install global collector configured based on RUST_LOG env var. + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .compact() + .init(); fdlimit::raise_fd_limit(); diff --git a/web3-proxy/src/app.rs b/web3-proxy/src/app.rs index dcb12945..20627bc5 100644 --- a/web3-proxy/src/app.rs +++ b/web3-proxy/src/app.rs @@ -110,7 +110,7 @@ impl Web3ProxyApp { pub async fn proxy_web3_rpc( self: Arc, request: JsonRpcRequestEnum, - ) -> anyhow::Result { + ) -> anyhow::Result { // TODO: i don't always see this in the logs. why? debug!("Received request: {:?}", request); @@ -126,7 +126,7 @@ impl Web3ProxyApp { // TODO: i don't always see this in the logs. why? debug!("Forwarding response: {:?}", response); - Ok(warp::reply::json(&response)) + Ok(response) } // #[instrument(skip_all)] diff --git a/web3-proxy/src/main.rs b/web3-proxy/src/main.rs index 63cf7419..467bc8ac 100644 --- a/web3-proxy/src/main.rs +++ b/web3-proxy/src/main.rs @@ -2,6 +2,7 @@ mod app; mod config; mod connection; mod connections; +mod frontend; mod jsonrpc; use jsonrpc::{JsonRpcErrorData, JsonRpcForwardedResponse}; @@ -13,9 +14,8 @@ use std::sync::Arc; use std::thread; use std::time::Duration; use tokio::runtime; -use tracing::{info, warn}; -use warp::Filter; -use warp::Reply; +use tracing::{debug, error, info, trace, warn}; +use tracing_subscriber::EnvFilter; use crate::app::Web3ProxyApp; use crate::config::{CliConfig, RpcConfig}; @@ -25,8 +25,10 @@ fn main() -> anyhow::Result<()> { // TODO: what should the default log level be? // install global collector configured based on RUST_LOG env var. - tracing_subscriber::fmt().compact().init(); - // console_subscriber::init(); + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .compact() + .init(); fdlimit::raise_fd_limit(); @@ -77,55 +79,10 @@ fn main() -> anyhow::Result<()> { // spawn the root task rt.block_on(async { - let listen_port = cli_config.port; - let app = rpc_config.try_build().await?; let app: Arc = Arc::new(app); - let proxy_rpc_filter = warp::any() - .and(warp::post()) - .and(warp::body::json()) - .then(move |json_body| app.clone().proxy_web3_rpc(json_body)); - - // TODO: filter for displaying connections and their block heights - - // TODO: warp trace is super verbose. how do we make this more readable? - // let routes = proxy_rpc_filter.with(warp::trace::request()); - let routes = proxy_rpc_filter.map(handle_anyhow_errors); - - warp::serve(routes).run(([0, 0, 0, 0], listen_port)).await; - - Ok(()) + frontend::run(cli_config.port, app).await }) } - -/// convert result into a jsonrpc error. use this at the end of your warp filter -fn handle_anyhow_errors( - res: anyhow::Result, -) -> warp::http::Response { - match res { - Ok(r) => r.into_response(), - Err(e) => { - warn!("Responding with error: {:?}", e); - - let e = JsonRpcForwardedResponse { - jsonrpc: "2.0".to_string(), - // TODO: what id can we use? how do we make sure the incoming id gets attached to this? - id: RawValue::from_string("0".to_string()).unwrap(), - result: None, - error: Some(JsonRpcErrorData { - code: -32099, - message: format!("{:?}", e), - data: None, - }), - }; - - warp::reply::with_status( - serde_json::to_string(&e).unwrap(), - warp::http::StatusCode::INTERNAL_SERVER_ERROR, - ) - } - .into_response(), - } -}