Merge branch 'main' into 19-admin-imitate

This commit is contained in:
David 2023-02-27 10:56:29 +01:00 committed by GitHub
commit 27a85c946f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 191 additions and 145 deletions

224
Cargo.lock generated

@ -165,7 +165,7 @@ dependencies = [
"slab",
"socket2",
"waker-fn",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -282,9 +282,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "axum"
version = "0.6.6"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e246206a63c9830e118d12c894f56a82033da1a2361f5544deeee3df85c99d9"
checksum = "2bd379e511536bad07447f899300aa526e9bae8e6f66dc5e5ca45d7587b7c1ec"
dependencies = [
"async-trait",
"axum-core",
@ -312,7 +312,7 @@ dependencies = [
"tokio",
"tokio-tungstenite 0.18.0",
"tower",
"tower-http",
"tower-http 0.3.5",
"tower-layer",
"tower-service",
]
@ -664,9 +664,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "1.3.0"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
dependencies = [
"serde",
]
@ -940,7 +940,7 @@ dependencies = [
"encode_unicode 0.3.6",
"lazy_static",
"libc",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -1418,9 +1418,9 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "encoding_rs"
version = "0.8.31"
version = "0.8.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
dependencies = [
"cfg-if",
]
@ -1862,7 +1862,7 @@ dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -2141,9 +2141,9 @@ dependencies = [
[[package]]
name = "gimli"
version = "0.27.1"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec"
checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
[[package]]
name = "glob"
@ -2375,9 +2375,9 @@ dependencies = [
[[package]]
name = "http"
version = "0.2.8"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
dependencies = [
"bytes",
"fnv",
@ -2421,9 +2421,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.23"
version = "0.14.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c"
checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"
dependencies = [
"bytes",
"futures-channel",
@ -2604,7 +2604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"
dependencies = [
"libc",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -2632,7 +2632,7 @@ dependencies = [
"hermit-abi 0.2.6",
"io-lifetimes",
"rustix",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -2667,9 +2667,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.60"
version = "0.3.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
dependencies = [
"wasm-bindgen",
]
@ -2879,14 +2879,14 @@ dependencies = [
[[package]]
name = "mio"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys",
"windows-sys 0.45.0",
]
[[package]]
@ -2969,7 +2969,7 @@ dependencies = [
"libc",
"mio",
"walkdir",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -3191,7 +3191,7 @@ checksum = "bd10bab2b6df910bbe6c4987d76aa4221235103d9a9c000cfabcee6a6abc8f7a"
dependencies = [
"reqwest",
"serde",
"time 0.3.17",
"time 0.3.20",
"url",
]
@ -3245,7 +3245,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core 0.9.6",
"parking_lot_core 0.9.7",
]
[[package]]
@ -3264,9 +3264,9 @@ dependencies = [
[[package]]
name = "parking_lot_core"
version = "0.9.6"
version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"
checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
dependencies = [
"backtrace",
"cfg-if",
@ -3275,7 +3275,7 @@ dependencies = [
"redox_syscall",
"smallvec",
"thread-id",
"windows-sys",
"windows-sys 0.45.0",
]
[[package]]
@ -3374,9 +3374,9 @@ dependencies = [
[[package]]
name = "petgraph"
version = "0.6.2"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
dependencies = [
"fixedbitset",
"indexmap",
@ -3523,7 +3523,7 @@ dependencies = [
"libc",
"log",
"wepoll-ffi",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -3617,9 +3617,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
version = "1.0.50"
version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"
checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
dependencies = [
"unicode-ident",
]
@ -4042,7 +4042,7 @@ dependencies = [
"io-lifetimes",
"libc",
"linux-raw-sys",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -4185,7 +4185,7 @@ dependencies = [
"serde_json",
"sqlx",
"thiserror",
"time 0.3.17",
"time 0.3.20",
"tracing",
"url",
"uuid 1.3.0",
@ -4248,7 +4248,7 @@ dependencies = [
"rust_decimal",
"sea-query-derive",
"serde_json",
"time 0.3.17",
"time 0.3.20",
"uuid 1.3.0",
]
@ -4264,7 +4264,7 @@ dependencies = [
"sea-query",
"serde_json",
"sqlx",
"time 0.3.17",
"time 0.3.20",
"uuid 1.3.0",
]
@ -4363,9 +4363,9 @@ checksum = "930c0acf610d3fdb5e2ab6213019aaa04e227ebe9547b0649ba599b16d788bd7"
[[package]]
name = "sentry"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6f8ce69326daef9d845c3fd17149bd3dbd7caf5dc65dbbad9f5441a40ee407f"
checksum = "b5ce6d3512e2617c209ec1e86b0ca2fea06454cd34653c91092bf0f3ec41f8e3"
dependencies = [
"httpdate",
"reqwest",
@ -4383,9 +4383,9 @@ dependencies = [
[[package]]
name = "sentry-anyhow"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a80510663e6b711de2eed521a95dc38435a0e5858397d1acec79185e4a44215b"
checksum = "2d29d7126d9f4912d817eb2ecef8c8bcb46098ddfc2a24afb7c01f6094d9894d"
dependencies = [
"anyhow",
"sentry-backtrace",
@ -4394,9 +4394,9 @@ dependencies = [
[[package]]
name = "sentry-backtrace"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed6c0254d4cce319800609aa0d41b486ee57326494802045ff27434fc9a2030"
checksum = "0e7fe408d4d1f8de188a9309916e02e129cbe51ca19e55badea5a64899399b1a"
dependencies = [
"backtrace",
"once_cell",
@ -4406,9 +4406,9 @@ dependencies = [
[[package]]
name = "sentry-contexts"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3277dc5d2812562026f2095c7841f3d61bbe6789159b7da54f41d540787f818"
checksum = "5695096a059a89973ec541062d331ff4c9aeef9c2951416c894f0fff76340e7d"
dependencies = [
"hostname",
"libc",
@ -4420,9 +4420,9 @@ dependencies = [
[[package]]
name = "sentry-core"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5acbd3da4255938cf0384b6b140e6c07ff65919c26e4d7a989d8d90ee88fa91"
checksum = "5b22828bfd118a7b660cf7a155002a494755c0424cebb7061e4743ecde9c7dbc"
dependencies = [
"once_cell",
"rand",
@ -4433,9 +4433,9 @@ dependencies = [
[[package]]
name = "sentry-log"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4b922394014861334c24388a55825e4c715afb8ec7c1db900175aa9951f8241"
checksum = "bfa3a3f4477e77541c26eb84d0e355729dfa35c74c682eb8678f146db5126013"
dependencies = [
"log",
"sentry-core",
@ -4443,9 +4443,9 @@ dependencies = [
[[package]]
name = "sentry-panic"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "beebc7aedbd3aa470cd19caad208a5efe6c48902595c0d111a193d8ce4f7bd15"
checksum = "1f4ced2a7a8c14899d58eec402d946f69d5ed26a3fc363a7e8b1e5cb88473a01"
dependencies = [
"sentry-backtrace",
"sentry-core",
@ -4453,9 +4453,9 @@ dependencies = [
[[package]]
name = "sentry-types"
version = "0.29.3"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10d8587b12c0b8211bb3066979ee57af6e8657e23cf439dc6c8581fd86de24e8"
checksum = "360ee3270f7a4a1eee6c667f7d38360b995431598a73b740dfe420da548d9cc9"
dependencies = [
"debugid",
"getrandom",
@ -4463,7 +4463,7 @@ dependencies = [
"serde",
"serde_json",
"thiserror",
"time 0.3.17",
"time 0.3.20",
"url",
"uuid 1.3.0",
]
@ -4640,9 +4640,9 @@ dependencies = [
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
dependencies = [
"libc",
]
@ -4676,7 +4676,7 @@ dependencies = [
"rand",
"sha3",
"thiserror",
"time 0.3.17",
"time 0.3.20",
]
[[package]]
@ -4696,9 +4696,9 @@ dependencies = [
[[package]]
name = "slab"
version = "0.4.7"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef"
checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
dependencies = [
"autocfg",
]
@ -4836,7 +4836,7 @@ dependencies = [
"sqlx-rt",
"stringprep",
"thiserror",
"time 0.3.17",
"time 0.3.20",
"tokio-stream",
"url",
"uuid 1.3.0",
@ -4986,9 +4986,9 @@ dependencies = [
[[package]]
name = "sync_wrapper"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "tagptr"
@ -5113,9 +5113,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.17"
version = "0.3.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
dependencies = [
"itoa 1.0.5",
"serde",
@ -5131,9 +5131,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
[[package]]
name = "time-macros"
version = "0.2.6"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2"
checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36"
dependencies = [
"time-core",
]
@ -5158,9 +5158,9 @@ dependencies = [
[[package]]
name = "tinyvec_macros"
version = "0.1.0"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
@ -5180,7 +5180,7 @@ dependencies = [
"socket2",
"tokio-macros",
"tracing",
"windows-sys",
"windows-sys 0.42.0",
]
[[package]]
@ -5207,9 +5207,9 @@ dependencies = [
[[package]]
name = "tokio-stream"
version = "0.1.11"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce"
checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
dependencies = [
"futures-core",
"pin-project-lite",
@ -5247,9 +5247,9 @@ dependencies = [
[[package]]
name = "tokio-util"
version = "0.7.4"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"
checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2"
dependencies = [
"bytes",
"futures-core",
@ -5354,6 +5354,24 @@ dependencies = [
"tower-service",
]
[[package]]
name = "tower-http"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"
dependencies = [
"bitflags",
"bytes",
"futures-core",
"futures-util",
"http",
"http-body",
"http-range-header",
"pin-project-lite",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-layer"
version = "0.3.2"
@ -5679,9 +5697,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.83"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@ -5689,9 +5707,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.83"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
dependencies = [
"bumpalo",
"log",
@ -5704,9 +5722,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.33"
version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
dependencies = [
"cfg-if",
"js-sys",
@ -5716,9 +5734,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.83"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -5726,9 +5744,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.83"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
dependencies = [
"proc-macro2",
"quote",
@ -5739,9 +5757,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.83"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
[[package]]
name = "wasm-timer"
@ -5760,9 +5778,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.60"
version = "0.3.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
dependencies = [
"js-sys",
"wasm-bindgen",
@ -5819,12 +5837,12 @@ dependencies = [
"serde_prometheus",
"siwe",
"thread-fast-rng",
"time 0.3.17",
"time 0.3.20",
"tokio",
"tokio-stream",
"toml 0.7.2",
"tower",
"tower-http",
"tower-http 0.4.0",
"ulid",
"url",
"uuid 1.3.0",
@ -5919,6 +5937,30 @@ dependencies = [
"windows_x86_64_msvc",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.1"
@ -6025,7 +6067,7 @@ dependencies = [
"hmac",
"pbkdf2",
"sha1",
"time 0.3.17",
"time 0.3.20",
"zstd",
]

31
TODO.md

@ -334,7 +334,10 @@ These are not yet ordered. There might be duplicates. We might not actually need
- [x] improve eth_sendRawTransaction server selection
- [x] don't cache methods that are usually very large
- [x] use http provider when available
- [ ] don't use new_head_provider anywhere except new head subscription
- [x] per-chain rpc rate limits
- [-] if we subscribe to a server that is syncing, it gives us null block_data_limit. when it catches up, we don't ever send queries to it. we need to recheck block_data_limit
- [x] add a "failover" tier that is only used if balanced_rpcs has "no servers synced"
- use this tier (and private tier) to check timestamp on latest block. if we are behind that by more than a few seconds, something is wrong
- [-] proxy mode for benchmarking all backends
- [-] proxy mode for sending to multiple backends
- [-] let users choose a % of reverts to log (or maybe x/second). someone like curve logging all reverts will be a BIG database very quickly
@ -344,10 +347,16 @@ These are not yet ordered. There might be duplicates. We might not actually need
- [-] add configurable size limits to all the Caches
- instead of configuring each cache with MB sizes, have one value for total memory footprint and then percentages for each cache
- https://github.com/moka-rs/moka/issues/201
- [ ] refactor so configs can change while running
- this will probably be a rather large change, but is necessary when we have autoscaling
- create the app without applying any config to it
- have a blocking future watching the config file and calling app.apply_config() on first load and on change
- work started on this in the "config_reloads" branch. because of how we pass channels around during spawn, this requires a larger refactor.
- [ ] have multiple providers on each backend rpc. one websocket for newHeads. and then http providers for handling requests
- erigon only streams the JSON over HTTP. that code isn't enabled for websockets. so this should save memory on the erigon servers
- i think this also means we don't need to worry about changing the id that the user gives us.
- have the healthcheck get the block over http. if it errors, or doesn't match what the websocket says, something is wrong (likely a deadlock in the websocket code)
- [ ] don't use new_head_provider anywhere except new head subscription
- [ ] maybe we shouldn't route eth_getLogs to syncing nodes. serving queries slows down sync significantly
- change the send_best function to only include servers that are at least close to fully synced
- [ ] have private transactions be enabled by a url setting rather than a setting on the key
@ -371,10 +380,6 @@ These are not yet ordered. There might be duplicates. We might not actually need
- it won't happen with users often because they should be sticky to one proxy, but unauthenticated users will definitely hit this
- one option: we need the insert to be an upsert, but how do we merge historgrams?
- [ ] don't use systemtime. use chrono
- [ ] soft limit needs more thought
- it should be the min of total_sum_soft_limit (from only non-lagged servers) and min_sum_soft_limit
- otherwise it won't track anything and will just give errors.
- but if web3 proxy has just started, we should give some time otherwise we will thundering herd the first server that responds
- [ ] connection pool for websockets. use tokio-tungstenite directly. no need for ethers providers since serde_json is enough for us
- this should also get us closer to being able to do our own streaming json parser where we can
- [ ] get `oldest_allowed` out of config. or calculate automatically based on block time.
@ -391,14 +396,8 @@ These are not yet ordered. There might be duplicates. We might not actually need
- maybe sum available_requests grouped by archive/non-archive. only limit to non-archive if they have enough?
- [ ] some places we call it "accounting" others a "stat". be consistent
- [ ] cli commands to search users by key
- [ ] cli flag to set prometheus port
- [x] cli flag to set prometheus port
- [ ] flamegraphs show 25% of the time to be in moka-housekeeper. tune that
- [ ] remove/change the "active_requests" counter? maybe only once we have dynamic soft limits?
- [ ] refactor so configs can change while running
- this will probably be a rather large change, but is necessary when we have autoscaling
- create the app without applying any config to it
- have a blocking future watching the config file and calling app.apply_config() on first load and on change
- work started on this in the "config_reloads" branch. because of how we pass channels around during spawn, this requires a larger refactor.
- [ ] config parsing is strict right now. this makes it hard to deploy on git push since configs need to change along with it
- [ ] when displaying the user's data, they just see an opaque id for their tier. We should join that data
- [ ] add indexes to speed up stat queries
@ -486,12 +485,12 @@ These are not yet ordered. There might be duplicates. We might not actually need
- [ ] don't "unwrap" anywhere. give proper errors
- [ ] handle log subscriptions
- probably as a paid feature
- [ ] if we subscribe to a server that is syncing, it gives us null block_data_limit. when it catches up, we don't ever send queries to it. we need to recheck block_data_limit
- [ ] add a "failover" tier that is only used if balanced_rpcs has "no servers synced"
- use this tier (and private tier) to check timestamp on latest block. if we are behind that by more than a few seconds, something is wrong
- [ ] relevant erigon changelogs: add pendingTransactionWithBody subscription method (#5675)
- [ ] change_user_tier_by_key should not show the rpc key id. that way our ansible playbook won't expose it
- [ ] make sure all our responses follow the spec: https://www.jsonrpc.org/specification#examples
- [ ] min_sum_soft_limit should be automatic based on the apps average rps plus a buffer.
- if min_sum_soft_limit > max_sum_soft_limit, just wait for all? emit a warning
- [ ] a script for load testing a server and calculating its hard and soft limits
## V2

@ -1,2 +1,2 @@
[toolchain]
channel = "1.65.0"
channel = "1.67.1"

@ -27,7 +27,7 @@ thread-fast-rng = { path = "../thread-fast-rng" }
anyhow = { version = "1.0.69", features = ["backtrace"] }
argh = "0.1.10"
axum = { version = "0.6.6", features = ["headers", "ws"] }
axum = { version = "0.6.8", features = ["headers", "ws"] }
axum-client-ip = "0.4.0"
axum-macros = "0.3.4"
chrono = "0.4.23"
@ -44,7 +44,7 @@ glob = "0.3.1"
handlebars = "4.3.6"
hashbrown = { version = "0.13.2", features = ["serde"] }
hdrhistogram = "7.5.2"
http = "0.2.8"
http = "0.2.9"
ipnet = "2.7.1"
itertools = "0.10.5"
log = "0.4.17"
@ -60,17 +60,17 @@ proctitle = "0.1.1"
regex = "1.7.1"
reqwest = { version = "0.11.14", default-features = false, features = ["json", "tokio-rustls"] }
rustc-hash = "1.1.0"
sentry = { version = "0.29.3", default-features = false, features = ["backtrace", "contexts", "panic", "anyhow", "reqwest", "rustls", "log", "sentry-log"] }
sentry = { version = "0.30.0", default-features = false, features = ["backtrace", "contexts", "panic", "anyhow", "reqwest", "rustls", "log", "sentry-log"] }
serde = { version = "1.0.152", features = [] }
serde_json = { version = "1.0.93", default-features = false, features = ["alloc", "raw_value"] }
serde_prometheus = "0.2.1"
siwe = "0.5.0"
time = "0.3.17"
time = "0.3.20"
tokio = { version = "1.25.0", features = ["full"] }
tokio-stream = { version = "0.1.11", features = ["sync"] }
tokio-stream = { version = "0.1.12", features = ["sync"] }
toml = "0.7.2"
tower = "0.4.13"
tower-http = { version = "0.3.5", features = ["cors", "sensitive-headers"] }
tower-http = { version = "0.4.0", features = ["cors", "sensitive-headers"] }
ulid = { version = "1.0.0", features = ["serde"] }
url = "2.3.1"
uuid = "1.3.0"

@ -645,31 +645,31 @@ impl Web3ProxyApp {
let mut login_rate_limiter = None;
if let Some(redis_pool) = vredis_pool.as_ref() {
let rpc_rrl = RedisRateLimiter::new(
"web3_proxy",
"frontend",
// TODO: think about this unwrapping
top_config
.app
.public_requests_per_period
.unwrap_or(u64::MAX),
60.0,
redis_pool.clone(),
);
if let Some(public_requests_per_period) = top_config.app.public_requests_per_period {
// chain id is included in the app name so that rpc rate limits are per-chain
let rpc_rrl = RedisRateLimiter::new(
&format!("web3_proxy:{}", top_config.app.chain_id),
"frontend",
public_requests_per_period,
60.0,
redis_pool.clone(),
);
// these two rate limiters can share the base limiter
// these are deferred rate limiters because we don't want redis network requests on the hot path
// TODO: take cache_size from config
frontend_ip_rate_limiter = Some(DeferredRateLimiter::<IpAddr>::new(
10_000,
"ip",
rpc_rrl.clone(),
None,
));
frontend_registered_user_rate_limiter = Some(DeferredRateLimiter::<u64>::new(
10_000, "key", rpc_rrl, None,
));
// these two rate limiters can share the base limiter
// these are deferred rate limiters because we don't want redis network requests on the hot path
// TODO: take cache_size from config
frontend_ip_rate_limiter = Some(DeferredRateLimiter::<IpAddr>::new(
10_000,
"ip",
rpc_rrl.clone(),
None,
));
frontend_registered_user_rate_limiter = Some(DeferredRateLimiter::<u64>::new(
10_000, "key", rpc_rrl, None,
));
}
// login rate limiter
login_rate_limiter = Some(RedisRateLimiter::new(
"web3_proxy",
"login",

@ -70,7 +70,7 @@ pub async fn clean_block_number(
// convert the json value to a BlockNumber
let (block_num, change) = if let Some(obj) = x.as_object_mut() {
// it might be a Map like `{"blockHash": String("0xa5626dc20d3a0a209b1de85521717a3e859698de8ce98bca1b16822b7501f74b")}`
if let Some(block_hash) = obj.remove("blockHash") {
if let Some(block_hash) = obj.get("blockHash").cloned() {
let block_hash: H256 =
serde_json::from_value(block_hash).context("decoding blockHash")?;
@ -79,7 +79,7 @@ pub async fn clean_block_number(
.await
.context("fetching block number from hash")?;
// TODO: set change to true? i think not we should probably use hashes for everything.
// TODO: we do not change the
(*block.number(), false)
} else {
return Err(anyhow::anyhow!("blockHash missing"));
@ -176,6 +176,7 @@ pub async fn block_needed(
"eth_getLogs" => {
// TODO: think about this more
// TODO: jsonrpc has a specific code for this
// TODO: this shouldn't be a 500. this should
let obj = params[0]
.as_object_mut()
.ok_or_else(|| anyhow::anyhow!("invalid format"))?;

@ -887,7 +887,7 @@ impl Web3Rpcs {
continue;
}
// others look like this
// others look like this (this is the example in the official spec)
if error_msg == "Method not found" {
method_not_available_response = Some(response);
continue;

@ -358,7 +358,11 @@ impl Web3Rpc {
.store(limit, atomic::Ordering::Release);
}
info!("block data limit on {}: {:?}", self, limit);
if limit == Some(u64::MAX) {
info!("block data limit on {}: archive", self);
} else {
info!("block data limit on {}: {:?}", self, limit);
}
Ok(limit)
}
@ -707,13 +711,13 @@ impl Web3Rpc {
// TODO: move this into a function and the chaining should be easier
let head_block = conn.head_block.read().clone();
if let Some((block_hash, txid)) = head_block.and_then(|x| {
if let Some((block_number, txid)) = head_block.and_then(|x| {
let block = x.block.clone();
let block_hash = block.hash?;
let block_number = block.number?;
let txid = block.transactions.last().cloned()?;
Some((block_hash, txid))
Some((block_number, txid))
}) {
let to = conn
.wait_for_query::<_, Option<Transaction>>(
@ -755,7 +759,7 @@ impl Web3Rpc {
Ok(to) => {
conn.wait_for_query::<_, Option<Bytes>>(
"eth_getCode",
&(to, block_hash),
&(to, block_number),
revert_handler,
authorization.clone(),
Some(client),