axum and compact logs

This commit is contained in:
Bryan Stitt 2022-05-20 05:27:18 +00:00
parent 7f8e444356
commit 0ef728df93
9 changed files with 27 additions and 364 deletions

222
Cargo.lock generated

@ -124,27 +124,6 @@ dependencies = [
"term",
]
[[package]]
name = "async-stream"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"
dependencies = [
"async-stream-impl",
"futures-core",
]
[[package]]
name = "async-stream-impl"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "async-trait"
version = "0.1.53"
@ -221,6 +200,7 @@ dependencies = [
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.17.1",
"tower",
"tower-http",
"tower-layer",
@ -689,43 +669,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "console-api"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24cb05777feccbb2642d4f2df44d0505601a2cd88ca517d8c913f263a5a8dc8b"
dependencies = [
"prost",
"prost-types",
"tonic",
"tracing-core",
]
[[package]]
name = "console-subscriber"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f21a16ee925aa9d2bad2e296beffd6c5b1bfaad50af509d305b8e7f23af20fb"
dependencies = [
"console-api",
"crossbeam-channel",
"crossbeam-utils",
"futures",
"hdrhistogram",
"humantime",
"parking_lot 0.11.2",
"prost-types",
"serde",
"serde_json",
"thread_local",
"tokio",
"tokio-stream",
"tonic",
"tracing",
"tracing-core",
"tracing-subscriber",
]
[[package]]
name = "const-oid"
version = "0.7.1"
@ -753,15 +696,6 @@ dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.3.5"
@ -1449,18 +1383,6 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"
[[package]]
name = "flate2"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af"
dependencies = [
"cfg-if",
"crc32fast",
"libc",
"miniz_oxide",
]
[[package]]
name = "flume"
version = "0.10.12"
@ -1752,19 +1674,6 @@ dependencies = [
"hashbrown 0.12.1",
]
[[package]]
name = "hdrhistogram"
version = "7.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31672b7011be2c4f7456c4ddbcb40e7e9a4a9fad8efe49a6ebaf5f307d0109c0"
dependencies = [
"base64 0.13.0",
"byteorder",
"flate2",
"nom",
"num-traits",
]
[[package]]
name = "headers"
version = "0.3.7"
@ -1888,12 +1797,6 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.18"
@ -1931,18 +1834,6 @@ dependencies = [
"tokio-rustls",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
dependencies = [
"hyper",
"pin-project-lite",
"tokio",
"tokio-io-timeout",
]
[[package]]
name = "idna"
version = "0.2.3"
@ -2238,12 +2129,6 @@ dependencies = [
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.5.1"
@ -2315,16 +2200,6 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
[[package]]
name = "nom"
version = "7.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "nonzero_ext"
version = "0.3.0"
@ -2776,39 +2651,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "prost"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc03e116981ff7d8da8e5c220e374587b98d294af7ba7dd7fda761158f00086f"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-derive"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "prost-types"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68"
dependencies = [
"bytes",
"prost",
]
[[package]]
name = "quanta"
version = "0.9.3"
@ -3711,16 +3553,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "tokio-io-timeout"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
dependencies = [
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-macros"
version = "1.7.0"
@ -3820,38 +3652,6 @@ dependencies = [
"serde",
]
[[package]]
name = "tonic"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb"
dependencies = [
"async-stream",
"async-trait",
"axum",
"base64 0.13.0",
"bytes",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"prost-derive",
"tokio",
"tokio-stream",
"tokio-util 0.7.1",
"tower",
"tower-layer",
"tower-service",
"tracing",
"tracing-futures",
]
[[package]]
name = "tower"
version = "0.4.12"
@ -3860,11 +3660,8 @@ checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e"
dependencies = [
"futures-core",
"futures-util",
"indexmap",
"pin-project",
"pin-project-lite",
"rand",
"slab",
"tokio",
"tokio-util 0.7.1",
"tower-layer",
@ -4305,7 +4102,7 @@ dependencies = [
"anyhow",
"arc-swap",
"argh",
"console-subscriber",
"axum",
"dashmap",
"derive_more",
"ethers",
@ -4324,26 +4121,13 @@ dependencies = [
"serde_json",
"tokio",
"toml",
"tower",
"tracing",
"tracing-subscriber",
"url",
"warp",
]
[[package]]
name = "web3-proxy-frontend"
version = "0.1.0"
dependencies = [
"axum",
"console-subscriber",
"hashbrown 0.12.1",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "webpki"
version = "0.22.0"

@ -2,7 +2,6 @@
members = [
"linkedhashmap",
"web3-proxy",
"web3-proxy-frontend",
]
# TODO: enable lto (and maybe other things proven with benchmarks) once rapid development is done

@ -1,17 +0,0 @@
[package]
name = "web3-proxy-frontend"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
axum = "0.5.6"
console-subscriber = { version = "0.1.5", features = ["parking_lot"] }
hashbrown = "0.12.1"
serde = { version = "1.0.137", features = [] }
serde_json = { version = "1.0.81", default-features = false, features = ["alloc", "raw_value"] }
tokio = { version = "1.18.2", features = ["full", "tracing"] }
tracing = "0.1.34"
# TODO: tracing-subscriber has serde and serde_json features that we might want to use
tracing-subscriber = { version = "0.3.11", features = ["parking_lot"] }

@ -1,68 +0,0 @@
/// this should move into web3-proxy once the basics are working
use axum::{
http::StatusCode,
response::IntoResponse,
routing::{get, post},
Json, Router,
};
use serde::{Deserialize, Serialize};
use std::net::SocketAddr;
#[tokio::main]
async fn main() {
// initialize tracing. this is too slow for production, but helpful while developing
console_subscriber::init();
// the real app will load the port and other settings out of command flags or a config file
let port = 8544;
// build our application with a route
let app = Router::new()
// `GET /` goes to `root`
.route("/", get(root))
// `POST /users` goes to `create_user`
.route("/users", post(create_user));
// run our app with hyper
// `axum::Server` is a re-export of `hyper::Server`
let addr = SocketAddr::from(([0, 0, 0, 0], port));
tracing::debug!("listening on {}", addr);
axum::Server::bind(&addr)
.serve(app.into_make_service())
.await
.unwrap();
}
// basic handler that responds with a static string
async fn root() -> &'static str {
"Hello, World!"
}
async fn create_user(
// this argument tells axum to parse the request body
// as JSON into a `CreateUser` type
Json(payload): Json<CreateUser>,
) -> impl IntoResponse {
// insert your application logic here
let user = User {
id: 1337,
username: payload.username,
};
// this will be converted into a JSON response
// with a status code of `201 Created`
(StatusCode::CREATED, Json(user))
}
// the input to our `create_user` handler
#[derive(Deserialize)]
struct CreateUser {
username: String,
}
// the output to our `create_user` handler
#[derive(Serialize)]
struct User {
id: u64,
username: String,
}

@ -9,8 +9,7 @@ edition = "2021"
anyhow = "1.0.57"
arc-swap = "1.5.0"
argh = "0.1.7"
# axum = "*" # TODO: use this instead of warp?
console-subscriber = { version = "0.1.5", features = ["parking_lot"] }
axum = { version = "0.5.6", features = ["serde_json", "tokio-tungstenite"] }
dashmap = "5.3.3"
derive_more = "0.99.17"
ethers = { git = "https://github.com/SatoshiAndKin/ethers-rs", features = ["rustls", "ws"] }
@ -34,7 +33,8 @@ tokio = { version = "1.18.2", features = ["full", "tracing"] }
toml = "0.5.9"
tracing = "0.1.34"
# TODO: tracing-subscriber has serde and serde_json features that we might want to use
tracing-subscriber = { version = "0.3.11", features = ["parking_lot"] }
tracing-subscriber = { version = "0.3.11", features = ["env-filter", "parking_lot"] }
url = "2.2.2"
# TODO: replace warp with axum?
# TODO: replace warp with axum
warp = "0.3.2"
tower = "*"

@ -4,7 +4,11 @@ use std::time::Duration;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
console_subscriber::init();
// install global collector configured based on RUST_LOG env var.
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.compact()
.init();
fdlimit::raise_fd_limit();

@ -4,7 +4,11 @@ use std::{str::FromStr, time::Duration};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
console_subscriber::init();
// install global collector configured based on RUST_LOG env var.
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.compact()
.init();
fdlimit::raise_fd_limit();

@ -110,7 +110,7 @@ impl Web3ProxyApp {
pub async fn proxy_web3_rpc(
self: Arc<Web3ProxyApp>,
request: JsonRpcRequestEnum,
) -> anyhow::Result<impl warp::Reply> {
) -> anyhow::Result<JsonRpcForwardedResponseEnum> {
// TODO: i don't always see this in the logs. why?
debug!("Received request: {:?}", request);
@ -126,7 +126,7 @@ impl Web3ProxyApp {
// TODO: i don't always see this in the logs. why?
debug!("Forwarding response: {:?}", response);
Ok(warp::reply::json(&response))
Ok(response)
}
// #[instrument(skip_all)]

@ -2,6 +2,7 @@ mod app;
mod config;
mod connection;
mod connections;
mod frontend;
mod jsonrpc;
use jsonrpc::{JsonRpcErrorData, JsonRpcForwardedResponse};
@ -13,9 +14,8 @@ use std::sync::Arc;
use std::thread;
use std::time::Duration;
use tokio::runtime;
use tracing::{info, warn};
use warp::Filter;
use warp::Reply;
use tracing::{debug, error, info, trace, warn};
use tracing_subscriber::EnvFilter;
use crate::app::Web3ProxyApp;
use crate::config::{CliConfig, RpcConfig};
@ -25,8 +25,10 @@ fn main() -> anyhow::Result<()> {
// TODO: what should the default log level be?
// install global collector configured based on RUST_LOG env var.
tracing_subscriber::fmt().compact().init();
// console_subscriber::init();
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::from_default_env())
.compact()
.init();
fdlimit::raise_fd_limit();
@ -77,55 +79,10 @@ fn main() -> anyhow::Result<()> {
// spawn the root task
rt.block_on(async {
let listen_port = cli_config.port;
let app = rpc_config.try_build().await?;
let app: Arc<Web3ProxyApp> = Arc::new(app);
let proxy_rpc_filter = warp::any()
.and(warp::post())
.and(warp::body::json())
.then(move |json_body| app.clone().proxy_web3_rpc(json_body));
// TODO: filter for displaying connections and their block heights
// TODO: warp trace is super verbose. how do we make this more readable?
// let routes = proxy_rpc_filter.with(warp::trace::request());
let routes = proxy_rpc_filter.map(handle_anyhow_errors);
warp::serve(routes).run(([0, 0, 0, 0], listen_port)).await;
Ok(())
frontend::run(cli_config.port, app).await
})
}
/// convert result into a jsonrpc error. use this at the end of your warp filter
fn handle_anyhow_errors<T: warp::Reply>(
res: anyhow::Result<T>,
) -> warp::http::Response<warp::hyper::Body> {
match res {
Ok(r) => r.into_response(),
Err(e) => {
warn!("Responding with error: {:?}", e);
let e = JsonRpcForwardedResponse {
jsonrpc: "2.0".to_string(),
// TODO: what id can we use? how do we make sure the incoming id gets attached to this?
id: RawValue::from_string("0".to_string()).unwrap(),
result: None,
error: Some(JsonRpcErrorData {
code: -32099,
message: format!("{:?}", e),
data: None,
}),
};
warp::reply::with_status(
serde_json::to_string(&e).unwrap(),
warp::http::StatusCode::INTERNAL_SERVER_ERROR,
)
}
.into_response(),
}
}