start adding redis-cell for rate limits
This commit is contained in:
Bryan Stitt
parent
307062d8d2
commit
5e14333e61
@ -1,6 +1,7 @@
|
||||
config/*.toml
|
||||
.git
|
||||
Dockerfile
|
||||
redis-cell-server/
|
||||
target
|
||||
web3-proxy/flamegraph.svg
|
||||
web3-proxy/perf.data
|
||||
|
||||
265
Cargo.lock
generated
265
Cargo.lock
generated
@ -200,7 +200,7 @@ dependencies = [
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.17.1",
|
||||
"tokio-tungstenite",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tower-layer",
|
||||
@ -409,16 +409,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "buf_redux"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"safemem",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.9.1"
|
||||
@ -641,6 +631,20 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a604e93b79d1808327a6fca85a6f2d69de66461e7620f5a4cbf5fb4d1d7c948"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tokio-util 0.7.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.14.1"
|
||||
@ -957,6 +961,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
|
||||
|
||||
[[package]]
|
||||
name = "dunce"
|
||||
version = "1.0.2"
|
||||
@ -1262,7 +1272,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.17.1",
|
||||
"tokio-tungstenite",
|
||||
"tracing",
|
||||
"tracing-futures",
|
||||
"url",
|
||||
@ -1674,31 +1684,6 @@ dependencies = [
|
||||
"hashbrown 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "headers"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4cff78e5788be1e0ab65b04d306b2ed5092c815ec97ec70f4ebd5aee158aa55d"
|
||||
dependencies = [
|
||||
"base64 0.13.0",
|
||||
"bitflags",
|
||||
"bytes",
|
||||
"headers-core",
|
||||
"http",
|
||||
"httpdate",
|
||||
"mime",
|
||||
"sha-1 0.10.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "headers-core"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
|
||||
dependencies = [
|
||||
"http",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.3.3"
|
||||
@ -2119,16 +2104,6 @@ version = "0.3.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
||||
|
||||
[[package]]
|
||||
name = "mime_guess"
|
||||
version = "2.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
|
||||
dependencies = [
|
||||
"mime",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.5.1"
|
||||
@ -2161,24 +2136,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multipart"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182"
|
||||
dependencies = [
|
||||
"buf_redux",
|
||||
"httparse",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"quick-error",
|
||||
"rand",
|
||||
"safemem",
|
||||
"tempfile",
|
||||
"twoway",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nanorand"
|
||||
version = "0.7.0"
|
||||
@ -2667,12 +2624,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.18"
|
||||
@ -2757,6 +2708,34 @@ dependencies = [
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "0.21.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a80b5f38d7f5a020856a0e16e40a9cfabf88ae8f0e4c2dcd8a3114c1e470852"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"combine",
|
||||
"dtoa",
|
||||
"futures-util",
|
||||
"itoa 0.4.8",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"sha1",
|
||||
"tokio",
|
||||
"tokio-util 0.6.9",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis-ratelimit-client"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"redis",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.13"
|
||||
@ -2779,9 +2758,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.5.5"
|
||||
version = "1.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
|
||||
checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@ -2799,9 +2778,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.25"
|
||||
version = "0.6.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
|
||||
checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
@ -2968,12 +2947,6 @@ version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
|
||||
|
||||
[[package]]
|
||||
name = "safemem"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
|
||||
|
||||
[[package]]
|
||||
name = "salsa20"
|
||||
version = "0.9.0"
|
||||
@ -2992,12 +2965,6 @@ dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scoped-tls"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
@ -3118,19 +3085,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha-1"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6"
|
||||
dependencies = [
|
||||
"block-buffer 0.9.0",
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest 0.9.0",
|
||||
"opaque-debug 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha-1"
|
||||
version = "0.10.0"
|
||||
@ -3142,6 +3096,21 @@ dependencies = [
|
||||
"digest 0.10.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770"
|
||||
dependencies = [
|
||||
"sha1_smol",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1_smol"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.8.2"
|
||||
@ -3575,30 +3544,6 @@ dependencies = [
|
||||
"webpki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "511de3f85caf1c98983545490c3d09685fa8eb634e57eec22bb4db271f46cbd8"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"pin-project",
|
||||
"tokio",
|
||||
"tungstenite 0.14.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.17.1"
|
||||
@ -3610,7 +3555,7 @@ dependencies = [
|
||||
"rustls",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tungstenite 0.17.2",
|
||||
"tungstenite",
|
||||
"webpki",
|
||||
"webpki-roots",
|
||||
]
|
||||
@ -3780,25 +3725,6 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5"
|
||||
dependencies = [
|
||||
"base64 0.13.0",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand",
|
||||
"sha-1 0.9.8",
|
||||
"thiserror",
|
||||
"url",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.17.2"
|
||||
@ -3813,22 +3739,13 @@ dependencies = [
|
||||
"log",
|
||||
"rand",
|
||||
"rustls",
|
||||
"sha-1 0.10.0",
|
||||
"sha-1",
|
||||
"thiserror",
|
||||
"url",
|
||||
"utf-8",
|
||||
"webpki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "twoway"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.15.0"
|
||||
@ -3847,15 +3764,6 @@ dependencies = [
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.8"
|
||||
@ -3962,36 +3870,6 @@ dependencies = [
|
||||
"try-lock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "warp"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cef4e1e9114a4b7f1ac799f16ce71c14de5778500c5450ec6b7b920c55b587e"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"headers",
|
||||
"http",
|
||||
"hyper",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"multipart",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
"scoped-tls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-tungstenite 0.15.0",
|
||||
"tokio-util 0.6.9",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.2+wasi-snapshot-preview1"
|
||||
@ -4125,7 +4003,6 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"warp",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
[workspace]
|
||||
members = [
|
||||
"linkedhashmap",
|
||||
"redis-cell-client",
|
||||
"web3-proxy",
|
||||
]
|
||||
|
||||
|
||||
6
TODO.md
6
TODO.md
@ -1,9 +1,11 @@
|
||||
# Todo
|
||||
|
||||
- [ ] some production configs are occassionally stuck waiting at 100% cpu
|
||||
- [ ] use redis and redis-cell for rate limits
|
||||
- [x] some production configs are occassionally stuck waiting at 100% cpu
|
||||
- they stop processing new blocks. i'm guessing 2 blocks arrive at the same time, but i thought our locks would handle that
|
||||
- even after removing a bunch of the locks, the deadlock still happens. i can't reliably reproduce. i just let it run for awhile and it happens.
|
||||
- running gdb shows the thread at tokio tungstenite thread is spinning near 100% cpu and none of the rest of the program is proceeding
|
||||
- fixed by https://github.com/gakonst/ethers-rs/pull/1287
|
||||
- [ ] should we use ethers-rs' quorum provider for the private rpcs? i think it would work well, but won't work with our current reconnect logic
|
||||
- [ ] improve caching
|
||||
- [ ] if the eth_call (or similar) params include a block, we can cache for longer
|
||||
@ -13,7 +15,7 @@
|
||||
- [ ] if chain split detected, don't send transactions
|
||||
- [ ] if a rpc fails to connect at start, retry later instead of skipping it forever
|
||||
- [ ] endpoint for health checks. if no synced servers, give a 502 error
|
||||
- [ ] move from warp to auxm?
|
||||
- [x] move from warp to auxm?
|
||||
- [ ] proper logging with useful instrumentation
|
||||
- [ ] handle websocket disconnect and reconnect
|
||||
- [ ] warning if no blocks for too long. maybe reconnect automatically?
|
||||
|
||||
@ -1,24 +1,25 @@
|
||||
[shared]
|
||||
chain_id = 1
|
||||
rate_limit_redis = "redis:6379"
|
||||
|
||||
[balanced_rpcs]
|
||||
|
||||
[balanced_rpcs.erigon_archive]
|
||||
url = "http://127.0.0.1:8549"
|
||||
url = "http://10.11.12.16:8549"
|
||||
# TODO: double check soft_limit on erigon
|
||||
soft_limit = 100_000
|
||||
|
||||
[balanced_rpcs.erigon_archive_ws]
|
||||
url = "ws://127.0.0.1:8549"
|
||||
url = "ws://10.11.12.16:8549"
|
||||
# TODO: double check soft_limit on erigon
|
||||
soft_limit = 100_000
|
||||
|
||||
[balanced_rpcs.geth]
|
||||
url = "http://127.0.0.1:8545"
|
||||
url = "http://10.11.12.16:8545"
|
||||
soft_limit = 200_000
|
||||
|
||||
[balanced_rpcs.geth_ws]
|
||||
url = "ws://127.0.0.1:8546"
|
||||
url = "ws://10.11.12.16:8546"
|
||||
soft_limit = 200_000
|
||||
|
||||
[private_rpcs]
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
services:
|
||||
base:
|
||||
# TODO: build in dev but use docker hub in prod?
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
command: --config /config.toml --workers 8
|
||||
|
||||
96
docker-compose.prod.yml
Normal file
96
docker-compose.prod.yml
Normal file
@ -0,0 +1,96 @@
|
||||
---
|
||||
version: "3.4"
|
||||
|
||||
services:
|
||||
redis:
|
||||
build: ./redis-cell-server/
|
||||
|
||||
arbitrum:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-arbitrum.toml:/config.toml
|
||||
ports:
|
||||
- 7500:8544
|
||||
|
||||
avalanche-c:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-avalanche-c.toml:/config.toml
|
||||
ports:
|
||||
- 7501:8544
|
||||
|
||||
bsc:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-bsc.toml:/config.toml
|
||||
ports:
|
||||
- 7502:8544
|
||||
|
||||
eth:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-eth.toml:/config.toml
|
||||
ports:
|
||||
- 7503:8544
|
||||
|
||||
eth-archive:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-eth-archive.toml:/config.toml
|
||||
ports:
|
||||
- 7603:8544
|
||||
|
||||
fantom:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-fantom.toml:/config.toml
|
||||
ports:
|
||||
- 7504:8544
|
||||
|
||||
gnosis:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-gnosis.toml:/config.toml
|
||||
ports:
|
||||
- 7505:8544
|
||||
|
||||
goerli:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-goerli.toml:/config.toml
|
||||
ports:
|
||||
- 7506:8544
|
||||
|
||||
optimism:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-optimism.toml:/config.toml
|
||||
ports:
|
||||
- 7507:8544
|
||||
|
||||
polygon:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-polygon.toml:/config.toml
|
||||
ports:
|
||||
- 7508:8544
|
||||
@ -1,93 +1,16 @@
|
||||
---
|
||||
# development config
|
||||
version: "3.4"
|
||||
|
||||
services:
|
||||
arbitrum:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-arbitrum.toml:/config.toml
|
||||
ports:
|
||||
- 7500:8544
|
||||
|
||||
avalanche-c:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-avalanche-c.toml:/config.toml
|
||||
ports:
|
||||
- 7501:8544
|
||||
|
||||
bsc:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-bsc.toml:/config.toml
|
||||
ports:
|
||||
- 7502:8544
|
||||
redis:
|
||||
build: ./redis-cell-server/
|
||||
|
||||
eth:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-eth.toml:/config.toml
|
||||
- ./config/example.toml:/config.toml
|
||||
ports:
|
||||
- 7503:8544
|
||||
|
||||
eth-archive:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-eth-archive.toml:/config.toml
|
||||
ports:
|
||||
- 7603:8544
|
||||
|
||||
fantom:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-fantom.toml:/config.toml
|
||||
ports:
|
||||
- 7504:8544
|
||||
|
||||
gnosis:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-gnosis.toml:/config.toml
|
||||
ports:
|
||||
- 7505:8544
|
||||
|
||||
goerli:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-goerli.toml:/config.toml
|
||||
ports:
|
||||
- 7506:8544
|
||||
|
||||
optimism:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-optimism.toml:/config.toml
|
||||
ports:
|
||||
- 7507:8544
|
||||
|
||||
polygon:
|
||||
extends:
|
||||
file: docker-compose.common.yml
|
||||
service: base
|
||||
volumes:
|
||||
- ./config/production-polygon.toml:/config.toml
|
||||
ports:
|
||||
- 7508:8544
|
||||
- 8544:8544
|
||||
|
||||
9
redis-cell-client/Cargo.toml
Normal file
9
redis-cell-client/Cargo.toml
Normal file
@ -0,0 +1,9 @@
|
||||
[package]
|
||||
name = "redis-ratelimit-client"
|
||||
version = "0.2.0"
|
||||
authors = ["Bryan Stitt <bryan@stitthappens.com>"]
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.57"
|
||||
redis = { version = "0.21.5", features = ["aio", "tokio", "tokio-comp"] }
|
||||
21
redis-cell-client/LICENSE
Normal file
21
redis-cell-client/LICENSE
Normal file
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2020 Tin Rabzelj
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
2
redis-cell-client/README.md
Normal file
2
redis-cell-client/README.md
Normal file
@ -0,0 +1,2 @@
|
||||
Based on https://github.com/tinrab/rusty-redis-rate-limiting/blob/46d95040708df92d46cef22d319d0182e12a2c0c/src/rate_limiter.rs
|
||||
Upgraded
|
||||
24
redis-cell-client/src/client.rs
Normal file
24
redis-cell-client/src/client.rs
Normal file
@ -0,0 +1,24 @@
|
||||
use std::time;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use redis::aio::MultiplexedConnection;
|
||||
use redis::AsyncCommands;
|
||||
|
||||
const KEY_PREFIX: &str = "rate-limit";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RedisCellClient {
|
||||
conn: MultiplexedConnection,
|
||||
}
|
||||
|
||||
impl RedisCellClient {
|
||||
pub async fn open(redis_address: &str) -> anyhow::Result<Self> {
|
||||
let client = redis::Client::open(redis_address)?;
|
||||
|
||||
let conn = client.get_multiplexed_tokio_connection().await?;
|
||||
|
||||
Ok(Self { conn })
|
||||
}
|
||||
|
||||
// TODO: what else?
|
||||
}
|
||||
3
redis-cell-client/src/lib.rs
Normal file
3
redis-cell-client/src/lib.rs
Normal file
@ -0,0 +1,3 @@
|
||||
mod client;
|
||||
|
||||
pub use client::RedisCellClient;
|
||||
17
redis-cell-server/Dockerfile
Normal file
17
redis-cell-server/Dockerfile
Normal file
@ -0,0 +1,17 @@
|
||||
FROM rust:1-bullseye as builder
|
||||
|
||||
WORKDIR /usr/src/redis-cell
|
||||
RUN \
|
||||
--mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/src/web3-proxy/target \
|
||||
{ \
|
||||
set -eux; \
|
||||
git clone -b v0.3.0 https://github.com/brandur/redis-cell .; \
|
||||
cargo build --release; \
|
||||
}
|
||||
|
||||
FROM redis:bullseye
|
||||
|
||||
COPY --from=builder /usr/src/redis-cell/target/release/libredis_cell.so /usr/lib/redis/modules/
|
||||
|
||||
CMD ["redis-server", "--loadmodule", "/usr/lib/redis/modules/libredis_cell.so"]
|
||||
@ -24,7 +24,7 @@ linkedhashmap = { path = "../linkedhashmap", features = ["inline-more"] }
|
||||
parking_lot = { version = "0.12.0", features = ["deadlock_detection"] }
|
||||
proctitle = "0.1.1"
|
||||
# TODO: regex has several "perf" features that we might want to use
|
||||
regex = "1.5.5"
|
||||
regex = "1.5.6"
|
||||
reqwest = { version = "0.11.10", default-features = false, features = ["json", "tokio-rustls"] }
|
||||
rustc-hash = "1.1.0"
|
||||
serde = { version = "1.0.137", features = [] }
|
||||
@ -35,6 +35,4 @@ tracing = "0.1.34"
|
||||
# TODO: tracing-subscriber has serde and serde_json features that we might want to use
|
||||
tracing-subscriber = { version = "0.3.11", features = ["env-filter", "parking_lot"] }
|
||||
url = "2.2.2"
|
||||
# TODO: replace warp with axum
|
||||
warp = "0.3.2"
|
||||
tower = "*"
|
||||
tower = "0.4.12"
|
||||
|
||||
@ -20,7 +20,7 @@ pub struct CliConfig {
|
||||
pub workers: usize,
|
||||
|
||||
/// path to a toml of rpc servers
|
||||
#[argh(option, default = "\"./config/example.toml\".to_string()")]
|
||||
#[argh(option, default = "\"./config/development.toml\".to_string()")]
|
||||
pub config: String,
|
||||
}
|
||||
|
||||
|
||||
@ -73,8 +73,9 @@ pub struct Web3Connection {
|
||||
url: String,
|
||||
/// keep track of currently open requests. We sort on this
|
||||
active_requests: AtomicU32,
|
||||
// TODO: put this in a RwLock so that we can replace it if re-connecting
|
||||
/// provider is in a RwLock so that we can replace it if re-connecting
|
||||
provider: RwLock<Arc<Web3Provider>>,
|
||||
/// this should store rate limits in redis/memcache/etc so that restarts and multiple proxies don't block eachother
|
||||
ratelimiter: Option<Web3RateLimiter>,
|
||||
/// used for load balancing to the least loaded server
|
||||
soft_limit: u32,
|
||||
|
||||
@ -255,6 +255,7 @@ impl Web3Connections {
|
||||
cmp::Ordering::Greater => {
|
||||
// the rpc's newest block is the new overall best block
|
||||
// TODO: if trace, do the full block hash?
|
||||
// TODO: only accept this block if it is a child of the current head_block
|
||||
info!("new head: {}", new_block_hash);
|
||||
|
||||
pending_synced_connections.inner.clear();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user