From 5e14333e61f39635ef9b85925b51290b9e45ad1a Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Sat, 21 May 2022 20:40:22 +0000 Subject: [PATCH] start adding redis-cell for rate limits --- .dockerignore | 1 + Cargo.lock | 265 +++++++++----------------------- Cargo.toml | 1 + TODO.md | 6 +- config/example.toml | 9 +- docker-compose.common.yml | 1 + docker-compose.prod.yml | 96 ++++++++++++ docker-compose.yml | 87 +---------- redis-cell-client/Cargo.toml | 9 ++ redis-cell-client/LICENSE | 21 +++ redis-cell-client/README.md | 2 + redis-cell-client/src/client.rs | 24 +++ redis-cell-client/src/lib.rs | 3 + redis-cell-server/Dockerfile | 17 ++ web3-proxy/Cargo.toml | 6 +- web3-proxy/src/config.rs | 2 +- web3-proxy/src/connection.rs | 3 +- web3-proxy/src/connections.rs | 1 + 18 files changed, 266 insertions(+), 288 deletions(-) create mode 100644 docker-compose.prod.yml create mode 100644 redis-cell-client/Cargo.toml create mode 100644 redis-cell-client/LICENSE create mode 100644 redis-cell-client/README.md create mode 100644 redis-cell-client/src/client.rs create mode 100644 redis-cell-client/src/lib.rs create mode 100644 redis-cell-server/Dockerfile diff --git a/.dockerignore b/.dockerignore index d1247e40..ec04274d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ config/*.toml .git Dockerfile +redis-cell-server/ target web3-proxy/flamegraph.svg web3-proxy/perf.data diff --git a/Cargo.lock b/Cargo.lock index 16c630f6..90607052 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -200,7 +200,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-tungstenite 0.17.1", + "tokio-tungstenite", "tower", "tower-http", "tower-layer", @@ -409,16 +409,6 @@ dependencies = [ "serde", ] -[[package]] -name = "buf_redux" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" -dependencies = [ - "memchr", - "safemem", -] - [[package]] name = "bumpalo" version = "3.9.1" @@ -641,6 +631,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "combine" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a604e93b79d1808327a6fca85a6f2d69de66461e7620f5a4cbf5fb4d1d7c948" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util 0.7.1", +] + [[package]] name = "console" version = "0.14.1" @@ -957,6 +961,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + [[package]] name = "dunce" version = "1.0.2" @@ -1262,7 +1272,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", - "tokio-tungstenite 0.17.1", + "tokio-tungstenite", "tracing", "tracing-futures", "url", @@ -1674,31 +1684,6 @@ dependencies = [ "hashbrown 0.12.1", ] -[[package]] -name = "headers" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cff78e5788be1e0ab65b04d306b2ed5092c815ec97ec70f4ebd5aee158aa55d" -dependencies = [ - "base64 0.13.0", - "bitflags", - "bytes", - "headers-core", - "http", - "httpdate", - "mime", - "sha-1 0.10.0", -] - -[[package]] -name = "headers-core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" -dependencies = [ - "http", -] - [[package]] name = "heck" version = "0.3.3" @@ -2119,16 +2104,6 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" -[[package]] -name = "mime_guess" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "miniz_oxide" version = "0.5.1" @@ -2161,24 +2136,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "multipart" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" -dependencies = [ - "buf_redux", - "httparse", - "log", - "mime", - "mime_guess", - "quick-error", - "rand", - "safemem", - "tempfile", - "twoway", -] - [[package]] name = "nanorand" version = "0.7.0" @@ -2667,12 +2624,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" version = "1.0.18" @@ -2757,6 +2708,34 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "redis" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80b5f38d7f5a020856a0e16e40a9cfabf88ae8f0e4c2dcd8a3114c1e470852" +dependencies = [ + "async-trait", + "bytes", + "combine", + "dtoa", + "futures-util", + "itoa 0.4.8", + "percent-encoding", + "pin-project-lite", + "sha1", + "tokio", + "tokio-util 0.6.9", + "url", +] + +[[package]] +name = "redis-ratelimit-client" +version = "0.2.0" +dependencies = [ + "anyhow", + "redis", +] + [[package]] name = "redox_syscall" version = "0.2.13" @@ -2779,9 +2758,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick", "memchr", @@ -2799,9 +2778,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "remove_dir_all" @@ -2968,12 +2947,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" -[[package]] -name = "safemem" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" - [[package]] name = "salsa20" version = "0.9.0" @@ -2992,12 +2965,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scoped-tls" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" - [[package]] name = "scopeguard" version = "1.1.0" @@ -3118,19 +3085,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha-1" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug 0.3.0", -] - [[package]] name = "sha-1" version = "0.10.0" @@ -3142,6 +3096,21 @@ dependencies = [ "digest 0.10.3", ] +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.8.2" @@ -3575,30 +3544,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "tokio-stream" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511de3f85caf1c98983545490c3d09685fa8eb634e57eec22bb4db271f46cbd8" -dependencies = [ - "futures-util", - "log", - "pin-project", - "tokio", - "tungstenite 0.14.0", -] - [[package]] name = "tokio-tungstenite" version = "0.17.1" @@ -3610,7 +3555,7 @@ dependencies = [ "rustls", "tokio", "tokio-rustls", - "tungstenite 0.17.2", + "tungstenite", "webpki", "webpki-roots", ] @@ -3780,25 +3725,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" -[[package]] -name = "tungstenite" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5" -dependencies = [ - "base64 0.13.0", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand", - "sha-1 0.9.8", - "thiserror", - "url", - "utf-8", -] - [[package]] name = "tungstenite" version = "0.17.2" @@ -3813,22 +3739,13 @@ dependencies = [ "log", "rand", "rustls", - "sha-1 0.10.0", + "sha-1", "thiserror", "url", "utf-8", "webpki", ] -[[package]] -name = "twoway" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" -dependencies = [ - "memchr", -] - [[package]] name = "typenum" version = "1.15.0" @@ -3847,15 +3764,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "unicase" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.8" @@ -3962,36 +3870,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "warp" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cef4e1e9114a4b7f1ac799f16ce71c14de5778500c5450ec6b7b920c55b587e" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "headers", - "http", - "hyper", - "log", - "mime", - "mime_guess", - "multipart", - "percent-encoding", - "pin-project", - "scoped-tls", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-stream", - "tokio-tungstenite 0.15.0", - "tokio-util 0.6.9", - "tower-service", - "tracing", -] - [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -4125,7 +4003,6 @@ dependencies = [ "tracing", "tracing-subscriber", "url", - "warp", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3a68c758..a3721e3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "linkedhashmap", + "redis-cell-client", "web3-proxy", ] diff --git a/TODO.md b/TODO.md index e1301094..4d576c3f 100644 --- a/TODO.md +++ b/TODO.md @@ -1,9 +1,11 @@ # Todo -- [ ] some production configs are occassionally stuck waiting at 100% cpu +- [ ] use redis and redis-cell for rate limits +- [x] some production configs are occassionally stuck waiting at 100% cpu - they stop processing new blocks. i'm guessing 2 blocks arrive at the same time, but i thought our locks would handle that - even after removing a bunch of the locks, the deadlock still happens. i can't reliably reproduce. i just let it run for awhile and it happens. - running gdb shows the thread at tokio tungstenite thread is spinning near 100% cpu and none of the rest of the program is proceeding + - fixed by https://github.com/gakonst/ethers-rs/pull/1287 - [ ] should we use ethers-rs' quorum provider for the private rpcs? i think it would work well, but won't work with our current reconnect logic - [ ] improve caching - [ ] if the eth_call (or similar) params include a block, we can cache for longer @@ -13,7 +15,7 @@ - [ ] if chain split detected, don't send transactions - [ ] if a rpc fails to connect at start, retry later instead of skipping it forever - [ ] endpoint for health checks. if no synced servers, give a 502 error - - [ ] move from warp to auxm? +- [x] move from warp to auxm? - [ ] proper logging with useful instrumentation - [ ] handle websocket disconnect and reconnect - [ ] warning if no blocks for too long. maybe reconnect automatically? diff --git a/config/example.toml b/config/example.toml index 2bcd3479..67a061a9 100644 --- a/config/example.toml +++ b/config/example.toml @@ -1,24 +1,25 @@ [shared] chain_id = 1 +rate_limit_redis = "redis:6379" [balanced_rpcs] [balanced_rpcs.erigon_archive] - url = "http://127.0.0.1:8549" + url = "http://10.11.12.16:8549" # TODO: double check soft_limit on erigon soft_limit = 100_000 [balanced_rpcs.erigon_archive_ws] - url = "ws://127.0.0.1:8549" + url = "ws://10.11.12.16:8549" # TODO: double check soft_limit on erigon soft_limit = 100_000 [balanced_rpcs.geth] - url = "http://127.0.0.1:8545" + url = "http://10.11.12.16:8545" soft_limit = 200_000 [balanced_rpcs.geth_ws] - url = "ws://127.0.0.1:8546" + url = "ws://10.11.12.16:8546" soft_limit = 200_000 [private_rpcs] diff --git a/docker-compose.common.yml b/docker-compose.common.yml index 4c26fa0e..5d7e9baa 100644 --- a/docker-compose.common.yml +++ b/docker-compose.common.yml @@ -1,5 +1,6 @@ services: base: + # TODO: build in dev but use docker hub in prod? build: . restart: unless-stopped command: --config /config.toml --workers 8 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..e04887a4 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,96 @@ +--- +version: "3.4" + +services: + redis: + build: ./redis-cell-server/ + + arbitrum: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-arbitrum.toml:/config.toml + ports: + - 7500:8544 + + avalanche-c: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-avalanche-c.toml:/config.toml + ports: + - 7501:8544 + + bsc: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-bsc.toml:/config.toml + ports: + - 7502:8544 + + eth: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-eth.toml:/config.toml + ports: + - 7503:8544 + + eth-archive: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-eth-archive.toml:/config.toml + ports: + - 7603:8544 + + fantom: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-fantom.toml:/config.toml + ports: + - 7504:8544 + + gnosis: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-gnosis.toml:/config.toml + ports: + - 7505:8544 + + goerli: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-goerli.toml:/config.toml + ports: + - 7506:8544 + + optimism: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-optimism.toml:/config.toml + ports: + - 7507:8544 + + polygon: + extends: + file: docker-compose.common.yml + service: base + volumes: + - ./config/production-polygon.toml:/config.toml + ports: + - 7508:8544 diff --git a/docker-compose.yml b/docker-compose.yml index 42e87626..2e5eb94f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,93 +1,16 @@ --- +# development config version: "3.4" services: - arbitrum: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-arbitrum.toml:/config.toml - ports: - - 7500:8544 - - avalanche-c: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-avalanche-c.toml:/config.toml - ports: - - 7501:8544 - - bsc: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-bsc.toml:/config.toml - ports: - - 7502:8544 + redis: + build: ./redis-cell-server/ eth: extends: file: docker-compose.common.yml service: base volumes: - - ./config/production-eth.toml:/config.toml + - ./config/example.toml:/config.toml ports: - - 7503:8544 - - eth-archive: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-eth-archive.toml:/config.toml - ports: - - 7603:8544 - - fantom: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-fantom.toml:/config.toml - ports: - - 7504:8544 - - gnosis: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-gnosis.toml:/config.toml - ports: - - 7505:8544 - - goerli: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-goerli.toml:/config.toml - ports: - - 7506:8544 - - optimism: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-optimism.toml:/config.toml - ports: - - 7507:8544 - - polygon: - extends: - file: docker-compose.common.yml - service: base - volumes: - - ./config/production-polygon.toml:/config.toml - ports: - - 7508:8544 + - 8544:8544 diff --git a/redis-cell-client/Cargo.toml b/redis-cell-client/Cargo.toml new file mode 100644 index 00000000..15680f34 --- /dev/null +++ b/redis-cell-client/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "redis-ratelimit-client" +version = "0.2.0" +authors = ["Bryan Stitt "] +edition = "2018" + +[dependencies] +anyhow = "1.0.57" +redis = { version = "0.21.5", features = ["aio", "tokio", "tokio-comp"] } diff --git a/redis-cell-client/LICENSE b/redis-cell-client/LICENSE new file mode 100644 index 00000000..14faaa3b --- /dev/null +++ b/redis-cell-client/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Tin Rabzelj + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/redis-cell-client/README.md b/redis-cell-client/README.md new file mode 100644 index 00000000..0635a2c9 --- /dev/null +++ b/redis-cell-client/README.md @@ -0,0 +1,2 @@ +Based on https://github.com/tinrab/rusty-redis-rate-limiting/blob/46d95040708df92d46cef22d319d0182e12a2c0c/src/rate_limiter.rs +Upgraded \ No newline at end of file diff --git a/redis-cell-client/src/client.rs b/redis-cell-client/src/client.rs new file mode 100644 index 00000000..84461c30 --- /dev/null +++ b/redis-cell-client/src/client.rs @@ -0,0 +1,24 @@ +use std::time; +use std::time::{Duration, SystemTime}; + +use redis::aio::MultiplexedConnection; +use redis::AsyncCommands; + +const KEY_PREFIX: &str = "rate-limit"; + +#[derive(Clone)] +pub struct RedisCellClient { + conn: MultiplexedConnection, +} + +impl RedisCellClient { + pub async fn open(redis_address: &str) -> anyhow::Result { + let client = redis::Client::open(redis_address)?; + + let conn = client.get_multiplexed_tokio_connection().await?; + + Ok(Self { conn }) + } + + // TODO: what else? +} diff --git a/redis-cell-client/src/lib.rs b/redis-cell-client/src/lib.rs new file mode 100644 index 00000000..bdcb980f --- /dev/null +++ b/redis-cell-client/src/lib.rs @@ -0,0 +1,3 @@ +mod client; + +pub use client::RedisCellClient; diff --git a/redis-cell-server/Dockerfile b/redis-cell-server/Dockerfile new file mode 100644 index 00000000..f7cc5999 --- /dev/null +++ b/redis-cell-server/Dockerfile @@ -0,0 +1,17 @@ +FROM rust:1-bullseye as builder + +WORKDIR /usr/src/redis-cell +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/src/web3-proxy/target \ + { \ + set -eux; \ + git clone -b v0.3.0 https://github.com/brandur/redis-cell .; \ + cargo build --release; \ + } + +FROM redis:bullseye + +COPY --from=builder /usr/src/redis-cell/target/release/libredis_cell.so /usr/lib/redis/modules/ + +CMD ["redis-server", "--loadmodule", "/usr/lib/redis/modules/libredis_cell.so"] diff --git a/web3-proxy/Cargo.toml b/web3-proxy/Cargo.toml index 204466b5..efd513ee 100644 --- a/web3-proxy/Cargo.toml +++ b/web3-proxy/Cargo.toml @@ -24,7 +24,7 @@ linkedhashmap = { path = "../linkedhashmap", features = ["inline-more"] } parking_lot = { version = "0.12.0", features = ["deadlock_detection"] } proctitle = "0.1.1" # TODO: regex has several "perf" features that we might want to use -regex = "1.5.5" +regex = "1.5.6" reqwest = { version = "0.11.10", default-features = false, features = ["json", "tokio-rustls"] } rustc-hash = "1.1.0" serde = { version = "1.0.137", features = [] } @@ -35,6 +35,4 @@ tracing = "0.1.34" # TODO: tracing-subscriber has serde and serde_json features that we might want to use tracing-subscriber = { version = "0.3.11", features = ["env-filter", "parking_lot"] } url = "2.2.2" -# TODO: replace warp with axum -warp = "0.3.2" -tower = "*" \ No newline at end of file +tower = "0.4.12" diff --git a/web3-proxy/src/config.rs b/web3-proxy/src/config.rs index 9ad28941..05248094 100644 --- a/web3-proxy/src/config.rs +++ b/web3-proxy/src/config.rs @@ -20,7 +20,7 @@ pub struct CliConfig { pub workers: usize, /// path to a toml of rpc servers - #[argh(option, default = "\"./config/example.toml\".to_string()")] + #[argh(option, default = "\"./config/development.toml\".to_string()")] pub config: String, } diff --git a/web3-proxy/src/connection.rs b/web3-proxy/src/connection.rs index 4f7b6b30..9304c0f9 100644 --- a/web3-proxy/src/connection.rs +++ b/web3-proxy/src/connection.rs @@ -73,8 +73,9 @@ pub struct Web3Connection { url: String, /// keep track of currently open requests. We sort on this active_requests: AtomicU32, - // TODO: put this in a RwLock so that we can replace it if re-connecting + /// provider is in a RwLock so that we can replace it if re-connecting provider: RwLock>, + /// this should store rate limits in redis/memcache/etc so that restarts and multiple proxies don't block eachother ratelimiter: Option, /// used for load balancing to the least loaded server soft_limit: u32, diff --git a/web3-proxy/src/connections.rs b/web3-proxy/src/connections.rs index 05c93dff..7671f9a6 100644 --- a/web3-proxy/src/connections.rs +++ b/web3-proxy/src/connections.rs @@ -255,6 +255,7 @@ impl Web3Connections { cmp::Ordering::Greater => { // the rpc's newest block is the new overall best block // TODO: if trace, do the full block hash? + // TODO: only accept this block if it is a child of the current head_block info!("new head: {}", new_block_hash); pending_synced_connections.inner.clear();