wait on background threads
This commit is contained in:
parent
6c9189ef76
commit
12ee26a2b3
8
Cargo.lock
generated
8
Cargo.lock
generated
@ -409,9 +409,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "axum"
|
name = "axum"
|
||||||
version = "0.5.16"
|
version = "0.5.17"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043"
|
checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"axum-core",
|
"axum-core",
|
||||||
@ -454,9 +454,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "axum-core"
|
name = "axum-core"
|
||||||
version = "0.2.8"
|
version = "0.2.9"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b"
|
checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
24
TODO.md
24
TODO.md
@ -185,6 +185,11 @@ These are roughly in order of completition
|
|||||||
- [x] if unknown config items, error
|
- [x] if unknown config items, error
|
||||||
- unknown configs are almost always a mistake. usually from me changing config parsing on my side and old fields not being updated to the new way
|
- unknown configs are almost always a mistake. usually from me changing config parsing on my side and old fields not being updated to the new way
|
||||||
- [x] also need to change how we disable rpcs since i was using an unknown field
|
- [x] also need to change how we disable rpcs since i was using an unknown field
|
||||||
|
- [x] [paginate responses](https://www.sea-ql.org/SeaORM/docs/basic-crud/select/#paginate-result)
|
||||||
|
- [x] graceful shutdown. stop taking new requests and don't stop until all outstanding queries are handled
|
||||||
|
- https://github.com/tokio-rs/mini-redis/blob/master/src/shutdown.rs
|
||||||
|
- we need this because we need to be sure all the queries are saved in the db. maybe put stuff in Drop
|
||||||
|
- need an flume::watch on unflushed stats that we can subscribe to. wait for it to flip to true
|
||||||
- [-] ability to domain lock or ip lock said key
|
- [-] ability to domain lock or ip lock said key
|
||||||
- the code to check the database and use these entries already exists, but users don't have a way to set them
|
- the code to check the database and use these entries already exists, but users don't have a way to set them
|
||||||
- [-] new endpoints for users (not totally sure about the exact paths, but these features are all needed):
|
- [-] new endpoints for users (not totally sure about the exact paths, but these features are all needed):
|
||||||
@ -201,21 +206,20 @@ These are roughly in order of completition
|
|||||||
- [ ] generate a new key from a web endpoint
|
- [ ] generate a new key from a web endpoint
|
||||||
- [ ] modifying key settings such as private relay, revert logging, ip/origin/etc checks
|
- [ ] modifying key settings such as private relay, revert logging, ip/origin/etc checks
|
||||||
- [ ] GET logged reverts on an endpoint that **requires authentication**.
|
- [ ] GET logged reverts on an endpoint that **requires authentication**.
|
||||||
- [x] [paginate responses](https://www.sea-ql.org/SeaORM/docs/basic-crud/select/#paginate-result)
|
|
||||||
- [ ] per-user stats should probably be locked behind authentication. the code is written but disabled for easy development
|
- [ ] per-user stats should probably be locked behind authentication. the code is written but disabled for easy development
|
||||||
- if we do this, we should also have an admin-only endpoint for seeing these for support requests
|
- if we do this, we should also have an admin-only endpoint for seeing these for support requests
|
||||||
- [ ] endpoint for creating/modifying api keys and their advanced security features
|
- [ ] endpoint for creating/modifying api keys and their advanced security features
|
||||||
- [ ] graceful shutdown. stop taking new requests and don't stop until all outstanding queries are handled
|
|
||||||
- https://github.com/tokio-rs/mini-redis/blob/master/src/shutdown.rs
|
|
||||||
- we need this because we need to be sure all the queries are saved in the db. maybe put stuff in Drop
|
|
||||||
- need an flume::watch on unflushed stats that we can subscribe to. wait for it to flip to true
|
|
||||||
- [ ] include if archive query or not in the stats
|
- [ ] include if archive query or not in the stats
|
||||||
- this is already partially done, but we need to double check it works. preferrably with tests
|
- this is already partially done, but we need to double check it works. preferrably with tests
|
||||||
|
- [-] add configurable size limits to all the Caches
|
||||||
|
- [ ] instead of configuring each cache with MB sizes, have one value for total memory footprint and then percentages for each cache
|
||||||
|
- [-] let users choose a % to log (or maybe x/second). someone like curve logging all reverts will be a BIG database very quickly
|
||||||
|
- this must be opt-in or spawned since it will slow things down and will make their calls less private
|
||||||
|
- [ ] we currently default to 0.0 and don't expose a way to edit it. we have a database row, but we don't use it
|
||||||
|
- [ ] don't use unix timestamps for response_millis since leap seconds will confuse it
|
||||||
- [ ] WARN http_request:request: web3_proxy::block_number: could not get block from params err=unexpected params length id=01GF4HTRKM4JV6NX52XSF9AYMW method=POST authorized_request=User(Some(SqlxMySqlPoolConnection), AuthorizedKey { ip: 10.11.12.15, origin: None, user_key_id: 4, log_revert_chance: 0.0000 })
|
- [ ] WARN http_request:request: web3_proxy::block_number: could not get block from params err=unexpected params length id=01GF4HTRKM4JV6NX52XSF9AYMW method=POST authorized_request=User(Some(SqlxMySqlPoolConnection), AuthorizedKey { ip: 10.11.12.15, origin: None, user_key_id: 4, log_revert_chance: 0.0000 })
|
||||||
- ERROR http_request:request:try_send_all_upstream_servers: web3_proxy::rpcs::request: bad response! err=JsonRpcClientError(JsonRpcError(JsonRpcError { code: -32000, message: "INTERNAL_ERROR: existing tx with same hash", data: None })) method=eth_sendRawTransaction rpc=local_erigon_alpha_archive id=01GF4HV03Y4ZNKQV8DW5NDQ5CG method=POST authorized_request=User(Some(SqlxMySqlPoolConnection), AuthorizedKey { ip: 10.11.12.15, origin: None, user_key_id: 4, log_revert_chance: 0.0000 }) self=Web3Connections { conns: {"local_erigon_alpha_archive_ws": Web3Connection { name: "local_erigon_alpha_archive_ws", blocks: "all", .. }, "local_geth_ws": Web3Connection { name: "local_geth_ws", blocks: 64, .. }, "local_erigon_alpha_archive": Web3Connection { name: "local_erigon_alpha_archive", blocks: "all", .. }}, .. } authorized_request=Some(User(Some(SqlxMySqlPoolConnection), AuthorizedKey { ip: 10.11.12.15, origin: None, user_key_id: 4, log_revert_chance: 0.0000 })) request=JsonRpcRequest { id: RawValue(39), method: "eth_sendRawTransaction", .. } request_metadata=Some(RequestMetadata { datetime: 2022-10-11T22:14:57.406829095Z, period_seconds: 60, request_bytes: 633, backend_requests: 0, no_servers: 0, error_response: false, response_bytes: 0, response_millis: 0 }) block_needed=None
|
- ERROR http_request:request:try_send_all_upstream_servers: web3_proxy::rpcs::request: bad response! err=JsonRpcClientError(JsonRpcError(JsonRpcError { code: -32000, message: "INTERNAL_ERROR: existing tx with same hash", data: None })) method=eth_sendRawTransaction rpc=local_erigon_alpha_archive id=01GF4HV03Y4ZNKQV8DW5NDQ5CG method=POST authorized_request=User(Some(SqlxMySqlPoolConnection), AuthorizedKey { ip: 10.11.12.15, origin: None, user_key_id: 4, log_revert_chance: 0.0000 }) self=Web3Connections { conns: {"local_erigon_alpha_archive_ws": Web3Connection { name: "local_erigon_alpha_archive_ws", blocks: "all", .. }, "local_geth_ws": Web3Connection { name: "local_geth_ws", blocks: 64, .. }, "local_erigon_alpha_archive": Web3Connection { name: "local_erigon_alpha_archive", blocks: "all", .. }}, .. } authorized_request=Some(User(Some(SqlxMySqlPoolConnection), AuthorizedKey { ip: 10.11.12.15, origin: None, user_key_id: 4, log_revert_chance: 0.0000 })) request=JsonRpcRequest { id: RawValue(39), method: "eth_sendRawTransaction", .. } request_metadata=Some(RequestMetadata { datetime: 2022-10-11T22:14:57.406829095Z, period_seconds: 60, request_bytes: 633, backend_requests: 0, no_servers: 0, error_response: false, response_bytes: 0, response_millis: 0 }) block_needed=None
|
||||||
- why is it failing to get the block from params when its set to None? That should be the simple case
|
- why is it failing to get the block from params when its set to None? That should be the simple case
|
||||||
- [-] add configurable size limits to all the Caches
|
|
||||||
- [ ] instead of configuring each cache with MB sizes, have one value for total memory footprint and then percentages for each cache
|
|
||||||
- [ ] if user-specific caches have evictions that aren't from timeouts, log a warning
|
- [ ] if user-specific caches have evictions that aren't from timeouts, log a warning
|
||||||
- [ ] make the "not synced" error more verbose
|
- [ ] make the "not synced" error more verbose
|
||||||
- I think there is a bug in our synced_rpcs filtering. likely in has_block_data
|
- I think there is a bug in our synced_rpcs filtering. likely in has_block_data
|
||||||
@ -225,9 +229,6 @@ These are roughly in order of completition
|
|||||||
- [ ] emit global stat on no servers synced
|
- [ ] emit global stat on no servers synced
|
||||||
- [ ] emit global stat on error (maybe just use sentry, but graphs are handy)
|
- [ ] emit global stat on error (maybe just use sentry, but graphs are handy)
|
||||||
- if we wait until the error handler to emit the stat, i don't think we have access to the authorized_request
|
- if we wait until the error handler to emit the stat, i don't think we have access to the authorized_request
|
||||||
- [-] let users choose a % to log (or maybe x/second). someone like curve logging all reverts will be a BIG database very quickly
|
|
||||||
- this must be opt-in or spawned since it will slow things down and will make their calls less private
|
|
||||||
- [ ] we currently default to 0.0 and don't expose a way to edit it. we have a database row, but we don't use it
|
|
||||||
- [ ] endpoint (and cli script) to rotate api key
|
- [ ] endpoint (and cli script) to rotate api key
|
||||||
- [ ] if no bearer token found in redis (likely because it expired), send 401 unauthorized
|
- [ ] if no bearer token found in redis (likely because it expired), send 401 unauthorized
|
||||||
- [ ] user create script should allow multiple keys per user
|
- [ ] user create script should allow multiple keys per user
|
||||||
@ -235,9 +236,8 @@ These are roughly in order of completition
|
|||||||
- [ ] WARN http_request: web3_proxy::frontend::errors: anyhow err=UserKey was not a ULID or UUID id=01GER4VBTS0FDHEBR96D1JRDZF method=POST
|
- [ ] WARN http_request: web3_proxy::frontend::errors: anyhow err=UserKey was not a ULID or UUID id=01GER4VBTS0FDHEBR96D1JRDZF method=POST
|
||||||
- if invalid user id given, we give a 500. should be a different error code instead
|
- if invalid user id given, we give a 500. should be a different error code instead
|
||||||
- [ ] BUG: i think if all backend servers stop, the server doesn't properly reconnect. It appears to stop listening on 8854, but not shut down.
|
- [ ] BUG: i think if all backend servers stop, the server doesn't properly reconnect. It appears to stop listening on 8854, but not shut down.
|
||||||
- [ ] BUG? WARN web3_proxy::rpcs::blockchain: Missing connection_head_block in block_hashes. Fetching now connection_head_hash=0x4b7a…14b5 conn_name=local_erigon_alpha_archive rpc=local_erigon_alpha_archive
|
- [ ] BUG? WARN web3_proxy::rpcs::blockchain: Missing connection_head_block in block_hashes. Fetching now connection_head_hash=0x4b7a…14b5 conn_name=local_erigon_alpha_archive rpc=local_erigon_alpha_archive
|
||||||
- i see this a lot more than expected. why is it happening so much? better logs needed
|
- i see this a lot more than expected. why is it happening so much? better logs needed
|
||||||
- [ ] don't use unix timestamps for response_millis since leap seconds will confuse itt
|
|
||||||
- [ ] from what i thought, /status should show hashes > numbers!
|
- [ ] from what i thought, /status should show hashes > numbers!
|
||||||
- but block numbers count is maxed out (10k)
|
- but block numbers count is maxed out (10k)
|
||||||
- and block hashes count is tiny (83)
|
- and block hashes count is tiny (83)
|
||||||
|
@ -22,7 +22,7 @@ redis-rate-limiter = { path = "../redis-rate-limiter" }
|
|||||||
anyhow = { version = "1.0.65", features = ["backtrace"] }
|
anyhow = { version = "1.0.65", features = ["backtrace"] }
|
||||||
arc-swap = "1.5.1"
|
arc-swap = "1.5.1"
|
||||||
argh = "0.1.9"
|
argh = "0.1.9"
|
||||||
axum = { version = "0.5.16", features = ["headers", "serde_json", "tokio-tungstenite", "ws"] }
|
axum = { version = "0.5.17", features = ["headers", "serde_json", "tokio-tungstenite", "ws"] }
|
||||||
axum-client-ip = "0.2.0"
|
axum-client-ip = "0.2.0"
|
||||||
axum-macros = "0.2.3"
|
axum-macros = "0.2.3"
|
||||||
# TODO: import chrono from sea-orm so we always have the same version
|
# TODO: import chrono from sea-orm so we always have the same version
|
||||||
|
@ -23,7 +23,6 @@ use futures::future::Abortable;
|
|||||||
use futures::future::{join_all, AbortHandle};
|
use futures::future::{join_all, AbortHandle};
|
||||||
use futures::stream::FuturesUnordered;
|
use futures::stream::FuturesUnordered;
|
||||||
use futures::stream::StreamExt;
|
use futures::stream::StreamExt;
|
||||||
use futures::Future;
|
|
||||||
use hashbrown::HashMap;
|
use hashbrown::HashMap;
|
||||||
use ipnet::IpNet;
|
use ipnet::IpNet;
|
||||||
use metered::{metered, ErrorCount, HitCount, ResponseTime, Throughput};
|
use metered::{metered, ErrorCount, HitCount, ResponseTime, Throughput};
|
||||||
@ -36,7 +35,6 @@ use serde::Serialize;
|
|||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
use std::pin::Pin;
|
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::sync::atomic::{self, AtomicUsize};
|
use std::sync::atomic::{self, AtomicUsize};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@ -173,9 +171,13 @@ impl Web3ProxyApp {
|
|||||||
pub async fn spawn(
|
pub async fn spawn(
|
||||||
top_config: TopConfig,
|
top_config: TopConfig,
|
||||||
num_workers: usize,
|
num_workers: usize,
|
||||||
|
shutdown_receiver: broadcast::Receiver<()>,
|
||||||
) -> anyhow::Result<(
|
) -> anyhow::Result<(
|
||||||
Arc<Web3ProxyApp>,
|
Arc<Web3ProxyApp>,
|
||||||
Pin<Box<dyn Future<Output = anyhow::Result<()>>>>,
|
// this handle is the main loops that we can cancel. select on this
|
||||||
|
FuturesUnordered<AnyhowJoinHandle<()>>,
|
||||||
|
// this handle is the state saving background loops that we must let finish. join_all on this
|
||||||
|
FuturesUnordered<AnyhowJoinHandle<()>>,
|
||||||
)> {
|
)> {
|
||||||
// safety checks on the config
|
// safety checks on the config
|
||||||
if let Some(redirect) = &top_config.app.redirect_user_url {
|
if let Some(redirect) = &top_config.app.redirect_user_url {
|
||||||
@ -213,8 +215,10 @@ impl Web3ProxyApp {
|
|||||||
let balanced_rpcs = top_config.balanced_rpcs;
|
let balanced_rpcs = top_config.balanced_rpcs;
|
||||||
let private_rpcs = top_config.private_rpcs.unwrap_or_default();
|
let private_rpcs = top_config.private_rpcs.unwrap_or_default();
|
||||||
|
|
||||||
// TODO: try_join_all instead?
|
// these are safe to cancel
|
||||||
let handles = FuturesUnordered::new();
|
let cancellable_handles = FuturesUnordered::new();
|
||||||
|
// we must wait for these to end on their own (and they need to subscribe to shutdown_sender)
|
||||||
|
let important_background_handles = FuturesUnordered::new();
|
||||||
|
|
||||||
// make a http shared client
|
// make a http shared client
|
||||||
// TODO: can we configure the connection pool? should we?
|
// TODO: can we configure the connection pool? should we?
|
||||||
@ -267,15 +271,15 @@ impl Web3ProxyApp {
|
|||||||
// we do this in a channel so we don't slow down our response to the users
|
// we do this in a channel so we don't slow down our response to the users
|
||||||
let stat_sender = if let Some(db_conn) = db_conn.clone() {
|
let stat_sender = if let Some(db_conn) = db_conn.clone() {
|
||||||
// TODO: sender and receiver here are a little confusing. because the thing that reads the receiver is what actually submits the stats
|
// TODO: sender and receiver here are a little confusing. because the thing that reads the receiver is what actually submits the stats
|
||||||
let (stat_sender, stat_handle, save_handle) = {
|
let (stat_sender, save_handle, stat_handle) = {
|
||||||
// TODO: period from config instead of always being 60 seconds
|
// TODO: period from config instead of always being 60 seconds
|
||||||
let emitter = StatEmitter::new(top_config.app.chain_id, db_conn, 60);
|
let emitter = StatEmitter::new(top_config.app.chain_id, db_conn, 60);
|
||||||
|
|
||||||
emitter.spawn().await?
|
emitter.spawn(shutdown_receiver).await?
|
||||||
};
|
};
|
||||||
|
|
||||||
handles.push(stat_handle);
|
cancellable_handles.push(stat_handle);
|
||||||
handles.push(save_handle);
|
important_background_handles.push(save_handle);
|
||||||
|
|
||||||
Some(stat_sender)
|
Some(stat_sender)
|
||||||
} else {
|
} else {
|
||||||
@ -332,7 +336,7 @@ impl Web3ProxyApp {
|
|||||||
.context("spawning balanced rpcs")?;
|
.context("spawning balanced rpcs")?;
|
||||||
|
|
||||||
// save the handle to catch any errors
|
// save the handle to catch any errors
|
||||||
handles.push(balanced_handle);
|
cancellable_handles.push(balanced_handle);
|
||||||
|
|
||||||
// connect to the private rpcs
|
// connect to the private rpcs
|
||||||
// only some chains have this, so this is optional
|
// only some chains have this, so this is optional
|
||||||
@ -363,7 +367,7 @@ impl Web3ProxyApp {
|
|||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
// save the handle to catch any errors
|
// save the handle to catch any errors
|
||||||
handles.push(private_handle);
|
cancellable_handles.push(private_handle);
|
||||||
|
|
||||||
Some(private_rpcs)
|
Some(private_rpcs)
|
||||||
}
|
}
|
||||||
@ -477,9 +481,7 @@ impl Web3ProxyApp {
|
|||||||
|
|
||||||
let app = Arc::new(app);
|
let app = Arc::new(app);
|
||||||
|
|
||||||
let handle = Box::pin(flatten_handles(handles));
|
Ok((app, cancellable_handles, important_background_handles))
|
||||||
|
|
||||||
Ok((app, handle))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn prometheus_metrics(&self) -> String {
|
pub fn prometheus_metrics(&self) -> String {
|
||||||
|
@ -10,7 +10,7 @@ use sea_orm::{ActiveModelTrait, DatabaseConnection};
|
|||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::{sync::atomic::AtomicU32, time::Duration};
|
use std::{sync::atomic::AtomicU32, time::Duration};
|
||||||
use tokio::sync::Mutex as AsyncMutex;
|
use tokio::sync::{broadcast, Mutex as AsyncMutex};
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
use tracing::{error, info, trace};
|
use tracing::{error, info, trace};
|
||||||
|
|
||||||
@ -173,6 +173,7 @@ impl StatEmitter {
|
|||||||
|
|
||||||
pub async fn spawn(
|
pub async fn spawn(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
|
shutdown_receiver: broadcast::Receiver<()>,
|
||||||
) -> anyhow::Result<(
|
) -> anyhow::Result<(
|
||||||
flume::Sender<Web3ProxyStat>,
|
flume::Sender<Web3ProxyStat>,
|
||||||
JoinHandle<anyhow::Result<()>>,
|
JoinHandle<anyhow::Result<()>>,
|
||||||
@ -181,8 +182,11 @@ impl StatEmitter {
|
|||||||
let (aggregate_tx, aggregate_rx) = flume::unbounded::<Web3ProxyStat>();
|
let (aggregate_tx, aggregate_rx) = flume::unbounded::<Web3ProxyStat>();
|
||||||
|
|
||||||
// TODO: join and flatten these handles
|
// TODO: join and flatten these handles
|
||||||
let aggregate_handle = tokio::spawn(self.clone().aggregate_stats_loop(aggregate_rx));
|
let aggregate_handle = tokio::spawn(
|
||||||
let save_handle = { tokio::spawn(self.save_stats_loop()) };
|
self.clone()
|
||||||
|
.aggregate_stats_loop(aggregate_rx, shutdown_receiver),
|
||||||
|
);
|
||||||
|
let save_handle = tokio::spawn(self.save_stats_loop());
|
||||||
|
|
||||||
Ok((aggregate_tx, aggregate_handle, save_handle))
|
Ok((aggregate_tx, aggregate_handle, save_handle))
|
||||||
}
|
}
|
||||||
@ -191,28 +195,42 @@ impl StatEmitter {
|
|||||||
async fn aggregate_stats_loop(
|
async fn aggregate_stats_loop(
|
||||||
self: Arc<Self>,
|
self: Arc<Self>,
|
||||||
aggregate_rx: flume::Receiver<Web3ProxyStat>,
|
aggregate_rx: flume::Receiver<Web3ProxyStat>,
|
||||||
|
mut shutdown_receiver: broadcast::Receiver<()>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
// TODO: select on shutdown handle so we can be sure to save every aggregate!
|
// TODO: select on shutdown handle so we can be sure to save every aggregate!
|
||||||
while let Ok(x) = aggregate_rx.recv_async().await {
|
tokio::select! {
|
||||||
trace!(?x, "aggregating stat");
|
x = aggregate_rx.recv_async() => {
|
||||||
|
match x {
|
||||||
|
Ok(x) => {
|
||||||
|
trace!(?x, "aggregating stat");
|
||||||
|
|
||||||
// TODO: increment global stats (in redis? in local cache for prometheus?)
|
// TODO: increment global stats (in redis? in local cache for prometheus?)
|
||||||
|
|
||||||
// TODO: batch stats? spawn this?
|
// TODO: batch stats?
|
||||||
// TODO: where can we wait on this handle?
|
// TODO: where can we wait on this handle?
|
||||||
let clone = self.clone();
|
let clone = self.clone();
|
||||||
tokio::spawn(async move { clone.aggregate_stat(x).await });
|
tokio::spawn(async move { clone.aggregate_stat(x).await });
|
||||||
|
},
|
||||||
// no need to save manually. they save on expire
|
Err(err) => {
|
||||||
|
error!(?err, "aggregate_rx");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
x = shutdown_receiver.recv() => {
|
||||||
|
match x {
|
||||||
|
Ok(_) => info!("aggregate stats loop shutting down"),
|
||||||
|
Err(err) => error!(?err, "shutdown receiver"),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// shutting down. force a save
|
// shutting down. force a save of any pending stats
|
||||||
// we do not use invalidate_all because that is done on a background thread
|
// we do not use invalidate_all because that is done on a background thread
|
||||||
for (key, _) in self.aggregated_proxy_responses.into_iter() {
|
for (key, _) in self.aggregated_proxy_responses.into_iter() {
|
||||||
self.aggregated_proxy_responses.invalidate(&key).await;
|
self.aggregated_proxy_responses.invalidate(&key).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("stat aggregator exited");
|
info!("aggregate stats loop finished");
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -8,20 +8,22 @@
|
|||||||
//#![warn(missing_docs)]
|
//#![warn(missing_docs)]
|
||||||
#![forbid(unsafe_code)]
|
#![forbid(unsafe_code)]
|
||||||
|
|
||||||
|
use futures::StreamExt;
|
||||||
use parking_lot::deadlock;
|
use parking_lot::deadlock;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::sync::atomic::{self, AtomicUsize};
|
use std::sync::atomic::{self, AtomicUsize};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use tokio::runtime;
|
use tokio::runtime;
|
||||||
|
use tokio::sync::broadcast;
|
||||||
use tokio::time::Duration;
|
use tokio::time::Duration;
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info};
|
||||||
use tracing_subscriber::EnvFilter;
|
use tracing_subscriber::EnvFilter;
|
||||||
use web3_proxy::app::{flatten_handle, Web3ProxyApp};
|
use web3_proxy::app::{flatten_handle, flatten_handles, Web3ProxyApp};
|
||||||
use web3_proxy::config::{CliConfig, TopConfig};
|
use web3_proxy::config::{CliConfig, TopConfig};
|
||||||
use web3_proxy::{frontend, metrics_frontend};
|
use web3_proxy::{frontend, metrics_frontend};
|
||||||
|
|
||||||
fn run(
|
fn run(
|
||||||
shutdown_receiver: flume::Receiver<()>,
|
shutdown_sender: broadcast::Sender<()>,
|
||||||
cli_config: CliConfig,
|
cli_config: CliConfig,
|
||||||
top_config: TopConfig,
|
top_config: TopConfig,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
@ -71,7 +73,8 @@ fn run(
|
|||||||
let app_frontend_port = cli_config.port;
|
let app_frontend_port = cli_config.port;
|
||||||
let app_prometheus_port = cli_config.prometheus_port;
|
let app_prometheus_port = cli_config.prometheus_port;
|
||||||
|
|
||||||
let (app, app_handle) = Web3ProxyApp::spawn(top_config, num_workers).await?;
|
let (app, app_handles, mut important_background_handles) =
|
||||||
|
Web3ProxyApp::spawn(top_config, num_workers, shutdown_sender.subscribe()).await?;
|
||||||
|
|
||||||
let frontend_handle = tokio::spawn(frontend::serve(app_frontend_port, app.clone()));
|
let frontend_handle = tokio::spawn(frontend::serve(app_frontend_port, app.clone()));
|
||||||
|
|
||||||
@ -80,7 +83,7 @@ fn run(
|
|||||||
// if everything is working, these should both run forever
|
// if everything is working, these should both run forever
|
||||||
// TODO: join these instead and use shutdown handler properly. probably use tokio's ctrl+c helper
|
// TODO: join these instead and use shutdown handler properly. probably use tokio's ctrl+c helper
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
x = app_handle => {
|
x = flatten_handles(app_handles) => {
|
||||||
match x {
|
match x {
|
||||||
Ok(_) => info!("app_handle exited"),
|
Ok(_) => info!("app_handle exited"),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@ -104,17 +107,27 @@ fn run(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = shutdown_receiver.recv_async() => {
|
x = tokio::signal::ctrl_c() => {
|
||||||
// TODO: think more about this. we need some way for tests to tell the app to stop
|
match x {
|
||||||
info!("received shutdown signal");
|
Ok(_) => info!("quiting from ctrl-c"),
|
||||||
|
Err(e) => {
|
||||||
// TODO: wait for outstanding requests to complete. graceful shutdown will make our users happier
|
return Err(e.into());
|
||||||
|
}
|
||||||
return Ok(())
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// TODO: wait on all the handles to stop
|
// one of the handles stopped. send a value so the others know to shut down
|
||||||
|
shutdown_sender.send(())?;
|
||||||
|
|
||||||
|
// wait on all the important background tasks (like saving stats to the database) to complete
|
||||||
|
while let Some(x) = important_background_handles.next().await {
|
||||||
|
match x {
|
||||||
|
Err(e) => return Err(e.into()),
|
||||||
|
Ok(Err(e)) => return Err(e),
|
||||||
|
Ok(Ok(_)) => continue,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
info!("finished");
|
info!("finished");
|
||||||
|
|
||||||
@ -154,9 +167,9 @@ fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
// tokio has code for catching ctrl+c so we use that
|
// tokio has code for catching ctrl+c so we use that
|
||||||
// this shutdown sender is currently only used in tests, but we might make a /shutdown endpoint or something
|
// this shutdown sender is currently only used in tests, but we might make a /shutdown endpoint or something
|
||||||
let (_shutdown_sender, shutdown_receiver) = flume::bounded(1);
|
let (shutdown_sender, _shutdown_receiver) = broadcast::channel(1);
|
||||||
|
|
||||||
run(shutdown_receiver, cli_config, top_config)
|
run(shutdown_sender, cli_config, top_config)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@ -237,11 +250,15 @@ mod tests {
|
|||||||
private_rpcs: None,
|
private_rpcs: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
let (shutdown_sender, shutdown_receiver) = flume::bounded(1);
|
let (shutdown_sender, _shutdown_receiver) = broadcast::channel(1);
|
||||||
|
|
||||||
// spawn another thread for running the app
|
// spawn another thread for running the app
|
||||||
// TODO: allow launching into the local tokio runtime instead of creating a new one?
|
// TODO: allow launching into the local tokio runtime instead of creating a new one?
|
||||||
let handle = thread::spawn(move || run(shutdown_receiver, cli_config, app_config));
|
let handle = {
|
||||||
|
let shutdown_sender = shutdown_sender.clone();
|
||||||
|
|
||||||
|
thread::spawn(move || run(shutdown_sender, cli_config, app_config))
|
||||||
|
};
|
||||||
|
|
||||||
// TODO: do something to the node. query latest block, mine another block, query again
|
// TODO: do something to the node. query latest block, mine another block, query again
|
||||||
let proxy_provider = Provider::<Http>::try_from(anvil.endpoint()).unwrap();
|
let proxy_provider = Provider::<Http>::try_from(anvil.endpoint()).unwrap();
|
||||||
|
@ -94,7 +94,7 @@ pub async fn serve(port: u16, proxy_app: Arc<Web3ProxyApp>) -> anyhow::Result<()
|
|||||||
// create a unique id for each request
|
// create a unique id for each request
|
||||||
.layer(RequestIdLayer)
|
.layer(RequestIdLayer)
|
||||||
// application state
|
// application state
|
||||||
.layer(Extension(proxy_app))
|
.layer(Extension(proxy_app.clone()))
|
||||||
// 404 for any unknown routes
|
// 404 for any unknown routes
|
||||||
.fallback(errors::handler_404.into_service());
|
.fallback(errors::handler_404.into_service());
|
||||||
|
|
||||||
@ -118,18 +118,6 @@ pub async fn serve(port: u16, proxy_app: Arc<Web3ProxyApp>) -> anyhow::Result<()
|
|||||||
axum::Server::bind(&addr)
|
axum::Server::bind(&addr)
|
||||||
// TODO: option to use with_connect_info. we want it in dev, but not when running behind a proxy, but not
|
// TODO: option to use with_connect_info. we want it in dev, but not when running behind a proxy, but not
|
||||||
.serve(service)
|
.serve(service)
|
||||||
.with_graceful_shutdown(signal_shutdown())
|
|
||||||
.await
|
.await
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Tokio signal handler that will wait for a user to press CTRL+C.
|
|
||||||
/// Used in our hyper `Server` method `with_graceful_shutdown`.
|
|
||||||
async fn signal_shutdown() {
|
|
||||||
// TODO: take a shutdown_receiver and select on ctrl_c and it
|
|
||||||
info!("ctrl-c to quit");
|
|
||||||
tokio::signal::ctrl_c()
|
|
||||||
.await
|
|
||||||
.expect("expect tokio signal ctrl-c");
|
|
||||||
info!("signal shutdown");
|
|
||||||
}
|
|
||||||
|
@ -423,8 +423,8 @@ pub async fn user_balance_get(
|
|||||||
/// TODO: this will change as we add better support for secondary users.
|
/// TODO: this will change as we add better support for secondary users.
|
||||||
#[debug_handler]
|
#[debug_handler]
|
||||||
pub async fn user_balance_post(
|
pub async fn user_balance_post(
|
||||||
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
|
||||||
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
||||||
|
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
||||||
) -> FrontendResult {
|
) -> FrontendResult {
|
||||||
todo!("user_balance_post");
|
todo!("user_balance_post");
|
||||||
}
|
}
|
||||||
@ -434,8 +434,8 @@ pub async fn user_balance_post(
|
|||||||
/// TODO: one key per request? maybe /user/keys/:api_key?
|
/// TODO: one key per request? maybe /user/keys/:api_key?
|
||||||
#[debug_handler]
|
#[debug_handler]
|
||||||
pub async fn user_keys_get(
|
pub async fn user_keys_get(
|
||||||
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
|
||||||
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
||||||
|
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
||||||
) -> FrontendResult {
|
) -> FrontendResult {
|
||||||
todo!("user_keys_get");
|
todo!("user_keys_get");
|
||||||
}
|
}
|
||||||
@ -446,8 +446,8 @@ pub async fn user_keys_get(
|
|||||||
/// TODO: one key per request? maybe /user/keys/:api_key?
|
/// TODO: one key per request? maybe /user/keys/:api_key?
|
||||||
#[debug_handler]
|
#[debug_handler]
|
||||||
pub async fn user_keys_post(
|
pub async fn user_keys_post(
|
||||||
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
|
||||||
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
||||||
|
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
||||||
) -> FrontendResult {
|
) -> FrontendResult {
|
||||||
todo!("user_keys_post");
|
todo!("user_keys_post");
|
||||||
}
|
}
|
||||||
@ -459,8 +459,8 @@ pub async fn user_keys_post(
|
|||||||
/// TODO: this will change as we add better support for secondary users.
|
/// TODO: this will change as we add better support for secondary users.
|
||||||
#[debug_handler]
|
#[debug_handler]
|
||||||
pub async fn user_profile_get(
|
pub async fn user_profile_get(
|
||||||
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
|
||||||
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
||||||
|
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
||||||
) -> FrontendResult {
|
) -> FrontendResult {
|
||||||
todo!("user_profile_get");
|
todo!("user_profile_get");
|
||||||
}
|
}
|
||||||
@ -468,8 +468,8 @@ pub async fn user_profile_get(
|
|||||||
/// `GET /user/revert_logs` -- Use a bearer token to get the user's revert logs.
|
/// `GET /user/revert_logs` -- Use a bearer token to get the user's revert logs.
|
||||||
#[debug_handler]
|
#[debug_handler]
|
||||||
pub async fn user_revert_logs_get(
|
pub async fn user_revert_logs_get(
|
||||||
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
|
||||||
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
||||||
|
TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
|
||||||
) -> FrontendResult {
|
) -> FrontendResult {
|
||||||
todo!("user_revert_logs_get");
|
todo!("user_revert_logs_get");
|
||||||
}
|
}
|
||||||
@ -485,10 +485,8 @@ pub async fn user_revert_logs_get(
|
|||||||
/// TODO: this will change as we add better support for secondary users.
|
/// TODO: this will change as we add better support for secondary users.
|
||||||
#[debug_handler]
|
#[debug_handler]
|
||||||
pub async fn user_stats_detailed_get(
|
pub async fn user_stats_detailed_get(
|
||||||
// TODO: turn this back on when done debugging. maybe add a path field for this
|
|
||||||
// TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
|
|
||||||
bearer: Option<TypedHeader<Authorization<Bearer>>>,
|
|
||||||
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
Extension(app): Extension<Arc<Web3ProxyApp>>,
|
||||||
|
bearer: Option<TypedHeader<Authorization<Bearer>>>,
|
||||||
Query(params): Query<HashMap<String, String>>,
|
Query(params): Query<HashMap<String, String>>,
|
||||||
) -> FrontendResult {
|
) -> FrontendResult {
|
||||||
let x = get_detailed_stats(&app, bearer, params).await?;
|
let x = get_detailed_stats(&app, bearer, params).await?;
|
||||||
|
Loading…
Reference in New Issue
Block a user