From a0056ca8815930c9baaa0aa2bbbc4a25cb4a1f15 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Mon, 24 Oct 2022 21:07:29 +0000 Subject: [PATCH] optional tracing-sentry --- Cargo.lock | 233 ++++++++++++++++++++++++++----- Cargo.toml | 8 +- TODO.md | 1 + web3_proxy/Cargo.toml | 5 +- web3_proxy/src/bin/web3_proxy.rs | 47 ++++++- web3_proxy/src/config.rs | 22 ++- 6 files changed, 266 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1a02e30..aa9c1428 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "async-channel" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" dependencies = [ "concurrent-queue", "event-listener", @@ -192,9 +192,9 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5262ed948da60dd8956c6c5aca4d4163593dddb7b32d73267c93dab7b2e98940" +checksum = "0da5b41ee986eed3f524c380e6d64965aea573882a8907682ad100f7859305ca" dependencies = [ "async-channel", "async-executor", @@ -202,7 +202,6 @@ dependencies = [ "async-lock", "blocking", "futures-lite", - "num_cpus", "once_cell", "tokio", ] @@ -1215,6 +1214,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid 1.2.1", +] + [[package]] name = "deferred-rate-limiter" version = "0.2.0" @@ -1352,6 +1361,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dotenvy" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d8c417d7a8cb362e0c37e5d815f5eb7c37f79ff93707329d5a194e42e54ca0" + [[package]] name = "dtoa" version = "0.4.8" @@ -2296,6 +2311,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "http" version = "0.2.8" @@ -2671,6 +2697,12 @@ dependencies = [ "value-bag", ] +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchers" version = "0.1.0" @@ -3419,9 +3451,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd6e8b7189a73169290e89bd24c771071f1012d8fe6f738f5226531f0b03d89" +checksum = "73d946ec7d256b04dfadc4e6a3292324e6f417124750fc5c0950f981b703a0f1" dependencies = [ "bytes", "chrono", @@ -3429,7 +3461,7 @@ dependencies = [ "postgres-protocol", "serde", "serde_json", - "time 0.3.15", + "time 0.3.16", "uuid 1.2.1", ] @@ -4073,7 +4105,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "time 0.3.15", + "time 0.3.16", "tracing", "url", "uuid 1.2.1", @@ -4127,9 +4159,9 @@ dependencies = [ [[package]] name = "sea-query" -version = "0.26.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a1de9d0334895f7eb51bd1603c3c9f6737413f905a134001f1e198f43bfd70" +checksum = "15e36a9680854b4e5d2ea5230f94e7ef81562260905b770fe20c3df9b3e7a536" dependencies = [ "chrono", "postgres-types", @@ -4137,7 +4169,7 @@ dependencies = [ "sea-query-derive", "sea-query-driver", "serde_json", - "time 0.3.15", + "time 0.3.16", "uuid 1.2.1", ] @@ -4156,9 +4188,9 @@ dependencies = [ [[package]] name = "sea-query-driver" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbda46eb3484cae1efb7bc68bca50f553a5b42c076cf4cbfae05b27f707549d4" +checksum = "7d7f0cae2e7ebb2affc378c40bc343c8197181d601d6755c3e66f1bd18cac253" dependencies = [ "proc-macro2", "quote", @@ -4167,9 +4199,9 @@ dependencies = [ [[package]] name = "sea-schema" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d070aba647637b533bd669a8e430bdc8f7d5249c9b53402da34347563bbfca" +checksum = "f737a6819fcad9727207d7090ccd6a33c0b384f3794e8838f952b52d599e3a17" dependencies = [ "futures", "sea-query", @@ -4239,6 +4271,109 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "930c0acf610d3fdb5e2ab6213019aaa04e227ebe9547b0649ba599b16d788bd7" +[[package]] +name = "sentry" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73642819e7fa63eb264abc818a2f65ac8764afbe4870b5ee25bcecc491be0d4c" +dependencies = [ + "httpdate", + "reqwest", + "sentry-anyhow", + "sentry-backtrace", + "sentry-contexts", + "sentry-core", + "sentry-panic", + "tokio", +] + +[[package]] +name = "sentry-anyhow" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d3479158a7396b4dfd346a1944d523427a225ff3c39102e8e985bc21cdfea8" +dependencies = [ + "anyhow", + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-backtrace" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49bafa55eefc6dbc04c7dac91e8c8ab9e89e9414f3193c105cabd991bbc75134" +dependencies = [ + "backtrace", + "once_cell", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-contexts" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c63317c4051889e73f0b00ce4024cae3e6a225f2e18a27d2c1522eb9ce2743da" +dependencies = [ + "hostname", + "libc", + "rustc_version", + "sentry-core", + "uname", +] + +[[package]] +name = "sentry-core" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a4591a2d128af73b1b819ab95f143bc6a2fbe48cd23a4c45e1ee32177e66ae6" +dependencies = [ + "once_cell", + "rand 0.8.5", + "sentry-types", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-panic" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696c74c5882d5a0d5b4a31d0ff3989b04da49be7983b7f52a52c667da5b480bf" +dependencies = [ + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-tracing" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea50bcf843510179a7ba41c9fe4d83a8958e9f8adf707ec731ff999297536344" +dependencies = [ + "sentry-core", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sentry-types" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "823923ae5f54a729159d720aa12181673044ee5c79cbda3be09e56f885e5468f" +dependencies = [ + "debugid", + "getrandom", + "hex", + "serde", + "serde_json", + "thiserror", + "time 0.3.16", + "url", + "uuid 1.2.1", +] + [[package]] name = "serde" version = "1.0.147" @@ -4422,7 +4557,7 @@ dependencies = [ "rand 0.8.5", "sha3", "thiserror", - "time 0.3.15", + "time 0.3.16", ] [[package]] @@ -4536,9 +4671,9 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.1.8" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b7922be017ee70900be125523f38bdd644f4f06a1b16e8fa5a8ee8c34bffd4" +checksum = "f87e292b4291f154971a43c3774364e2cbcaec599d3f5bf6fa9d122885dbc38a" dependencies = [ "itertools", "nom", @@ -4547,9 +4682,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f82cbe94f41641d6c410ded25bbf5097c240cefdf8e3b06d04198d0a96af6a4" +checksum = "9249290c05928352f71c077cc44a464d880c63f26f7534728cca008e135c0428" dependencies = [ "sqlx-core", "sqlx-macros", @@ -4557,9 +4692,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b69bf218860335ddda60d6ce85ee39f6cf6e5630e300e19757d1de15886a093" +checksum = "dcbc16ddba161afc99e14d1713a453747a2b07fc097d2009f4c300ec99286105" dependencies = [ "ahash", "atoi", @@ -4570,6 +4705,7 @@ dependencies = [ "crc", "crossbeam-queue", "digest 0.10.3", + "dotenvy", "either", "event-listener", "futures-channel", @@ -4595,14 +4731,14 @@ dependencies = [ "rustls-pemfile", "serde", "serde_json", - "sha-1", + "sha1", "sha2 0.10.2", "smallvec", "sqlformat", "sqlx-rt", "stringprep", "thiserror", - "time 0.3.15", + "time 0.3.16", "tokio-stream", "url", "uuid 1.2.1", @@ -4611,11 +4747,11 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40c63177cf23d356b159b60acd27c54af7423f1736988502e36bae9a712118f" +checksum = "b850fa514dc11f2ee85be9d055c512aa866746adfacd1cb42d867d68e6a5b0d9" dependencies = [ - "dotenv", + "dotenvy", "either", "heck 0.4.0", "once_cell", @@ -4631,9 +4767,9 @@ dependencies = [ [[package]] name = "sqlx-rt" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874e93a365a598dc3dadb197565952cb143ae4aa716f7bcc933a8d836f6bf89f" +checksum = "24c5b2d25fa654cc5f841750b8e1cdedbe21189bf9a9382ee90bfa9dd3562396" dependencies = [ "once_cell", "tokio", @@ -4882,21 +5018,32 @@ dependencies = [ [[package]] name = "time" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c" +checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca" dependencies = [ "itoa 1.0.2", "libc", "num_threads", + "serde", + "time-core", "time-macros", ] [[package]] -name = "time-macros" -version = "0.2.4" +name = "time-core" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb801831d812c562ae7d2bfb531f26e66e4e1f6b17307ba4149c5064710e5b" +dependencies = [ + "time-core", +] [[package]] name = "tiny-keccak" @@ -5242,6 +5389,15 @@ dependencies = [ "serde", ] +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + [[package]] name = "unicase" version = "2.6.0" @@ -5311,6 +5467,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -5531,11 +5688,13 @@ dependencies = [ "reqwest", "rustc-hash", "sea-orm", + "sentry", + "sentry-tracing", "serde", "serde_json", "serde_prometheus", "siwe", - "time 0.3.15", + "time 0.3.16", "tokio", "tokio-stream", "toml", @@ -5709,7 +5868,7 @@ dependencies = [ "hmac", "pbkdf2 0.10.1", "sha1", - "time 0.3.15", + "time 0.3.16", "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index bde23c3c..945941fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,10 @@ members = [ "web3_proxy", ] +[profile.release] +# we leave debug = true on so that sentry can give us line numbers +debug = true # TODO: enable lto (and maybe other things proven with benchmarks) once rapid development is done -# TODO: we can't do panic = abort because the websockets disconnect by panicking sometimes -#[profile.release] -#debug = true #lto = true + +# TODO: we can't do panic = abort because the websockets disconnect by panicking sometimes diff --git a/TODO.md b/TODO.md index 4462d993..7c907315 100644 --- a/TODO.md +++ b/TODO.md @@ -192,6 +192,7 @@ These are roughly in order of completition - need an flume::watch on unflushed stats that we can subscribe to. wait for it to flip to true - [x] don't use unix timestamps for response_millis since leap seconds will confuse it - [x] config to allow origins even on the anonymous endpoints +- [ ] log errors to sentry - [-] ability to domain lock or ip lock said key - the code to check the database and use these entries already exists, but users don't have a way to set them - [-] new endpoints for users (not totally sure about the exact paths, but these features are all needed): diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml index 31d94fcb..0a5393c1 100644 --- a/web3_proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -55,11 +55,13 @@ handlebars = "4.3.5" rustc-hash = "1.1.0" siwe = "0.5.0" sea-orm = { version = "0.9.3", features = ["macros"] } +sentry = { version = "0.27.0", default-features = false, features = ["backtrace", "contexts", "panic", "anyhow", "reqwest", "rustls"] } +sentry-tracing = "0.27.0" serde = { version = "1.0.147", features = [] } serde_json = { version = "1.0.87", default-features = false, features = ["alloc", "raw_value"] } serde_prometheus = "0.1.6" # TODO: make sure this time version matches siwe. PR to put this in their prelude -time = "0.3.15" +time = "0.3.16" tokio = { version = "1.21.2", features = ["full", "tracing"] } # TODO: make sure this uuid version matches sea-orm. PR to put this in their prelude tokio-stream = { version = "0.1.11", features = ["sync"] } @@ -74,3 +76,4 @@ tracing-subscriber = { version = "0.3.16", features = ["env-filter", "parking_lo ulid = { version = "1.0.0", features = ["serde"] } url = "2.3.1" uuid = "1.2.1" + diff --git a/web3_proxy/src/bin/web3_proxy.rs b/web3_proxy/src/bin/web3_proxy.rs index b3d08048..bf51a672 100644 --- a/web3_proxy/src/bin/web3_proxy.rs +++ b/web3_proxy/src/bin/web3_proxy.rs @@ -17,6 +17,7 @@ use tokio::runtime; use tokio::sync::broadcast; use tokio::time::Duration; use tracing::{debug, info, warn}; +use tracing_subscriber::prelude::*; use tracing_subscriber::EnvFilter; use web3_proxy::app::{flatten_handle, flatten_handles, Web3ProxyApp}; use web3_proxy::config::{CliConfig, TopConfig}; @@ -147,12 +148,6 @@ fn main() -> anyhow::Result<()> { ); } - // install global collector configured based on RUST_LOG env var. - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .compact() - .init(); - // this probably won't matter for us in docker, but better safe than sorry fdlimit::raise_fd_limit(); @@ -160,13 +155,50 @@ fn main() -> anyhow::Result<()> { let cli_config: CliConfig = argh::from_env(); // advanced configuration is on disk - info!("Loading config @ {}", cli_config.config); let top_config: String = fs::read_to_string(cli_config.config.clone())?; let top_config: TopConfig = toml::from_str(&top_config)?; // TODO: this doesn't seem to do anything proctitle::set_title(format!("web3_proxy-{}", top_config.app.chain_id)); + // connect to sentry for error reporting + // if no sentry, only log to stdout + let _sentry_guard = if let Some(sentry_url) = top_config.app.sentry_url.clone() { + let guard = sentry::init(( + sentry_url, + sentry::ClientOptions { + release: sentry::release_name!(), + // TODO: Set this a to lower value (from config) in production + traces_sample_rate: 1.0, + ..Default::default() + }, + )); + + // TODO: how do we put the EnvFilter on this? + tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .compact() + .with_filter(EnvFilter::from_default_env()), + ) + .with(sentry_tracing::layer()) + .init(); + + Some(guard) + } else { + // install global collector configured based on RUST_LOG env var. + // TODO: attach sentry here + tracing_subscriber::fmt() + .compact() + .with_env_filter(EnvFilter::from_default_env()) + .init(); + + None + }; + + // we used to do this earlier, but now we attach sentry + debug!("CLI config @ {:#?}", cli_config.config); + // tokio has code for catching ctrl+c so we use that // this shutdown sender is currently only used in tests, but we might make a /shutdown endpoint or something let (shutdown_sender, _shutdown_receiver) = broadcast::channel(1); @@ -198,6 +230,7 @@ mod tests { // TODO: option for super verbose logs std::env::set_var("RUST_LOG", "info,web3_proxy=debug"); // install global collector configured based on RUST_LOG env var. + // TODO: sentry is needed here! tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .compact() diff --git a/web3_proxy/src/config.rs b/web3_proxy/src/config.rs index 43d76bfd..57106449 100644 --- a/web3_proxy/src/config.rs +++ b/web3_proxy/src/config.rs @@ -50,55 +50,73 @@ pub struct TopConfig { #[derive(Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] pub struct AppConfig { + /// Request limit for allowed origins for anonymous users. + pub allowed_origin_requests_per_minute: HashMap, + /// EVM chain id. 1 for ETH /// TODO: better type for chain_id? max of `u64::MAX / 2 - 36` https://github.com/ethereum/EIPs/issues/2294 pub chain_id: u64, + /// Database is used for user data. /// Currently supports mysql or compatible backend. pub db_url: Option, + /// minimum size of the connection pool for the database. /// If none, the number of workers are used. pub db_min_connections: Option, + /// maximum size of the connection pool for the database. /// If none, the minimum * 2 is used. pub db_max_connections: Option, + /// Default request limit for registered users. /// 0 = block all requests /// None = allow all requests pub default_user_requests_per_minute: Option, + /// Restrict user registration. /// None = no code needed pub invite_code: Option, + /// The soft limit prevents thundering herds as new blocks are seen. #[serde(default = "default_min_sum_soft_limit")] pub min_sum_soft_limit: u32, + /// Another knob for preventing thundering herds as new blocks are seen. #[serde(default = "default_min_synced_rpcs")] pub min_synced_rpcs: usize, + /// Request limit for anonymous users. /// Some(0) = block all requests /// None = allow all requests #[serde(default = "default_public_requests_per_minute")] pub public_requests_per_minute: Option, - /// Request limit for allowed origins for anonymous users. - pub allowed_origin_requests_per_minute: HashMap, + /// Rate limit for the login entrypoint. /// This is separate from the rpc limits. #[serde(default = "default_login_rate_limit_per_minute")] pub login_rate_limit_per_minute: u64, + /// Track rate limits in a redis (or compatible backend) /// It is okay if this data is lost. pub volatile_redis_url: Option, + /// maximum size of the connection pool for the cache /// If none, the minimum * 2 is used pub volatile_redis_max_connections: Option, + /// RPC responses are cached locally #[serde(default = "default_response_cache_max_bytes")] pub response_cache_max_bytes: usize, + /// the stats page url for an anonymous user. pub redirect_public_url: Option, + /// the stats page url for a logged in user. if set, must contain "{user_id}" pub redirect_user_url: Option, + + /// https://sentry.io + pub sentry_url: Option, } /// This might cause a thundering herd!