diff --git a/Cargo.lock b/Cargo.lock index 8f03e629..1cf401cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -733,6 +733,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "cipher" version = "0.3.0" @@ -2951,6 +2957,17 @@ dependencies = [ "syn", ] +[[package]] +name = "os_info" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4750134fb6a5d49afc80777394ad5d95b04bc12068c6abb92fae8f43817270f" +dependencies = [ + "log", + "serde", + "winapi", +] + [[package]] name = "os_str_bytes" version = "6.1.0" @@ -4175,25 +4192,28 @@ checksum = "930c0acf610d3fdb5e2ab6213019aaa04e227ebe9547b0649ba599b16d788bd7" [[package]] name = "sentry" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73642819e7fa63eb264abc818a2f65ac8764afbe4870b5ee25bcecc491be0d4c" +checksum = "a120fb5e8b7975736bf1fc57de380531e617a6a8f5a55d037bcea25a7f5e8371" dependencies = [ "httpdate", "reqwest", + "rustls", "sentry-anyhow", "sentry-backtrace", "sentry-contexts", "sentry-core", "sentry-panic", "tokio", + "ureq", + "webpki-roots", ] [[package]] name = "sentry-anyhow" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d3479158a7396b4dfd346a1944d523427a225ff3c39102e8e985bc21cdfea8" +checksum = "0a3485a5784c7f962c6dab90f3006bd980b3741812293b7da7a438ecb603a637" dependencies = [ "anyhow", "sentry-backtrace", @@ -4202,9 +4222,9 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49bafa55eefc6dbc04c7dac91e8c8ab9e89e9414f3193c105cabd991bbc75134" +checksum = "2ac56ff9aae25b024a5aad4f0242808dfde29161c82d183adce778338c6822ef" dependencies = [ "backtrace", "once_cell", @@ -4214,12 +4234,13 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c63317c4051889e73f0b00ce4024cae3e6a225f2e18a27d2c1522eb9ce2743da" +checksum = "188506b08b5e64004c71b7a5edb34959083e6e1288fada3b8d18d0bc7449ce1e" dependencies = [ "hostname", "libc", + "os_info", "rustc_version", "sentry-core", "uname", @@ -4227,9 +4248,9 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a4591a2d128af73b1b819ab95f143bc6a2fbe48cd23a4c45e1ee32177e66ae6" +checksum = "ff58433a7ad557b586a09c42c4298d5f3ddb0c777e1a79d950e510d7b93fce0e" dependencies = [ "once_cell", "rand 0.8.5", @@ -4240,9 +4261,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696c74c5882d5a0d5b4a31d0ff3989b04da49be7983b7f52a52c667da5b480bf" +checksum = "4145005d9b5c117132765c34e2cb33e9d24d16e73d7f3a357122b77fe3a3b815" dependencies = [ "sentry-backtrace", "sentry-core", @@ -4250,9 +4271,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea50bcf843510179a7ba41c9fe4d83a8958e9f8adf707ec731ff999297536344" +checksum = "fc8d5bae1e1c06d96a966efc425bf1479a90464de99757d40601ce449f91fbed" dependencies = [ "sentry-core", "tracing-core", @@ -4261,10 +4282,11 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823923ae5f54a729159d720aa12181673044ee5c79cbda3be09e56f885e5468f" +checksum = "fb30d75498a041005a774ec1b6b7d9589c5906d17ebaca338cb685dc92170f9b" dependencies = [ + "chrono", "debugid", "getrandom", "hex", @@ -5346,6 +5368,22 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" +dependencies = [ + "base64 0.13.0", + "chunked_transfer", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "url" version = "2.3.1" @@ -5599,9 +5637,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.3" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" +checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" dependencies = [ "webpki", ] diff --git a/TODO.md b/TODO.md index f7700d9c..b0c7a7c2 100644 --- a/TODO.md +++ b/TODO.md @@ -223,16 +223,19 @@ These are roughly in order of completition - [x] instruments are missing. maybe that is why sentry had broken traces - [x] description should default to an empty string instead of being nullable - [x] include if archive query or not in the stats -- [-] add configurable size limits to all the Caches - - [ ] instead of configuring each cache with MB sizes, have one value for total memory footprint and then percentages for each cache -- [ ] proper authentication on rpc_key_id +- [x] fix test not shutting down +- [x] proper authentication on rpc_key_id - we have bearer token auth for user_id, but rpc_key_id needs more code -- [ ] fix tests +- [-] add configurable size limits to all the Caches + - instead of configuring each cache with MB sizes, have one value for total memory footprint and then percentages for each cache + +- [ ] actually block unauthenticated requests instead of emitting warning of "allowing without auth during development!" ## V1 -These are not yet ordered. +These are not yet ordered. There might be duplicates. We might not actually need all of these. +- [ ] logging of "bad response!" is way too verbose - [ ] config parsing is strict right now. this makes it hard to deploy on git push since configs need to change along with it - [ ] i think our "best" server picking is incorrect somehow. - we upgraded erigon to a version with a broken websocket @@ -294,7 +297,6 @@ These are not yet ordered. - what is going on? when the server fist launches they are in sync - [ ] related BUG? WARN web3_proxy::rpcs::blockchain: Missing connection_head_block in block_hashes. Fetching now connection_head_hash=0x4b7a…14b5 conn_name=local_erigon_alpha_archive rpc=local_erigon_alpha_archive - i see this a lot more than expected. why is it happening so much? better logs needed - - [ ] after adding semaphores (or maybe something else), CPU load seems a lot higher. investigate - [ ] proper support for Finalized and Safe block queries - [ ] admin-only page for viewing user stat pages @@ -496,8 +498,8 @@ in another repo: event subscriber - [ ] have a log all option? instead of just reverts, log all request/responses? can be very useful for debugging but would flood our database. maybe better for them to do that on their client side - [ ] failsafe. if no blocks or transactions in some time, warn and reset the connection - [ ] WARN http_request:request: web3_proxy::block_number: could not get block from params err=unexpected params length id=01GF4HTRKM4JV6NX52XSF9AYMW method=POST authorized_request=User(Some(SqlxMySqlPoolConnection), AuthorizedKey { ip: 10.11.12.15, origin: None, user_key_id: 4, log_revert_chance: 0.0000 }) -- [ ] document "backend requests is cache_misses + backend_retries" - [ ] having tons of worker threads can actually make us slower if they keep waking to steal work from eachother. need benchmarks - [ ] change the wrk data to log requests and errors to a file - [ ] if redis is not set and login page is visited, users get a 502. should be 501 - [ ] allow passing the authorization header to the anonymous rpc endpoint +- [ ] sentry profiling diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml index 2a745258..86d5c09f 100644 --- a/web3_proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -55,8 +55,8 @@ handlebars = "4.3.5" rustc-hash = "1.1.0" siwe = "0.5.0" sea-orm = { version = "0.10.1", features = ["macros"] } -sentry = { version = "0.27.0", default-features = false, features = ["backtrace", "contexts", "panic", "anyhow", "reqwest", "rustls"] } -sentry-tracing = "0.27.0" +sentry = { version = "0.28.0", default-features = false, features = ["backtrace", "contexts", "panic", "anyhow", "reqwest", "rustls"] } +sentry-tracing = "0.28.0" serde = { version = "1.0.147", features = [] } serde_json = { version = "1.0.87", default-features = false, features = ["alloc", "raw_value"] } serde_prometheus = "0.1.6" diff --git a/web3_proxy/src/user_queries.rs b/web3_proxy/src/user_queries.rs index 6e0b287d..5208820a 100644 --- a/web3_proxy/src/user_queries.rs +++ b/web3_proxy/src/user_queries.rs @@ -285,6 +285,22 @@ pub async fn query_user_stats<'a>( (condition, q) }; + // get_user_id_from_params checks that the bearer is connected to this user_id + // TODO: match on user_id and rpc_key_id? + let user_id = get_user_id_from_params(redis_conn, bearer, params).await?; + let (condition, q) = if user_id == 0 { + // 0 means everyone. don't filter on user + (condition, q) + } else { + let q = q.left_join(rpc_key::Entity); + + let condition = condition.add(rpc_key::Column::UserId.eq(user_id)); + + response.insert("user_id", serde_json::Value::Number(user_id.into())); + + (condition, q) + }; + // filter on rpc_key_id // TODO: move getting the param and checking the bearer token into a helper function let (condition, q) = if let Some(rpc_key_id) = params.get("rpc_key_id") { @@ -299,34 +315,28 @@ pub async fn query_user_stats<'a>( if rpc_key_id == 0 { (condition, q) } else { - // TODO: make sure that the bearer token is allowed to view this rpc_key_id - let q = q.group_by(rpc_accounting::Column::RpcKeyId); + response.insert("rpc_key_id", serde_json::Value::Number(rpc_key_id.into())); let condition = condition.add(rpc_accounting::Column::RpcKeyId.eq(rpc_key_id)); - response.insert("rpc_key_id", serde_json::Value::Number(rpc_key_id.into())); + let q = q.group_by(rpc_accounting::Column::RpcKeyId); - (condition, q) + if user_id == 0 { + // no user id, we did not join above + let q = q.left_join(rpc_key::Entity); + + (condition, q) + } else { + // user_id added a join on rpc_key already. only filter on user_id + let condition = condition.add(rpc_key::Column::UserId.eq(user_id)); + + (condition, q) + } } } else { (condition, q) }; - // get_user_id_from_params checks that the bearer is connected to this user_id - let user_id = get_user_id_from_params(redis_conn, bearer, ¶ms).await?; - let (condition, q) = if user_id == 0 { - // 0 means everyone. don't filter on user - (condition, q) - } else { - let q = q.left_join(rpc_key::Entity); - - let condition = condition.add(rpc_key::Column::UserId.eq(user_id)); - - response.insert("user_id", serde_json::Value::Number(user_id.into())); - - (condition, q) - }; - // now that all the conditions are set up. add them to the query let q = q.filter(condition);