From 4ec10e7a98ccd6e599da2507663e898f73ea506c Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Tue, 26 Jul 2022 00:38:00 +0000 Subject: [PATCH] first pass at a schema --- Cargo.lock | 64 +++++++++++++++++++++++++++++++++++-- TODO.md | 2 ++ config/example.toml | 5 ++- docker-compose.prod.yml | 4 ++- docker-compose.yml | 40 ++++++++++++----------- web3-proxy/Cargo.toml | 4 ++- web3-proxy/src/app.rs | 8 +++++ web3-proxy/src/bin/users.rs | 4 +++ 8 files changed, 105 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 959aceb6..e94b30ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,9 +180,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9496f0c1d1afb7a2af4338bbe1d969cddfead41d87a9fb3aaa6d0bbc7af648" +checksum = "c943a505c17b494638a38a9af129067f760c9c06794b9f57d499266909be8e72" dependencies = [ "async-trait", "axum-core", @@ -977,6 +977,37 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diesel" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" +dependencies = [ + "byteorder", + "diesel_derives", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "diesel_migrations" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" +dependencies = [ + "migrations_internals", + "migrations_macros", +] + [[package]] name = "diff" version = "0.1.12" @@ -1033,6 +1064,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dtoa" version = "0.4.8" @@ -2274,6 +2311,27 @@ dependencies = [ "autocfg", ] +[[package]] +name = "migrations_internals" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +dependencies = [ + "diesel", +] + +[[package]] +name = "migrations_macros" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "mime" version = "0.3.16" @@ -4281,6 +4339,8 @@ dependencies = [ "counter", "dashmap", "derive_more", + "diesel_migrations", + "dotenv", "ethers", "fdlimit", "flume", diff --git a/TODO.md b/TODO.md index 673e88fa..789dddff 100644 --- a/TODO.md +++ b/TODO.md @@ -138,6 +138,7 @@ new endpoints for users: ## V2 +- [ ] jwt auth so people can easily switch from infura - [ ] handle log subscriptions - [ ] most things that are cached locally should probably be in shared redis caches - [ ] automated soft limit @@ -154,6 +155,7 @@ in another repo: event subscriber ## "Maybe some day" and other Miscellaneous Things +- [ ] search for all the "TODO" items in the code and move them here - [ ] don't "unwrap" anywhere. give proper errors - [ ] instead of giving a rate limit error code, delay the connection's response at the start. reject if incoming requests is super high? - [ ] add the backend server to the header? diff --git a/config/example.toml b/config/example.toml index e588a073..5659d261 100644 --- a/config/example.toml +++ b/config/example.toml @@ -1,10 +1,9 @@ [shared] chain_id = 1 -# in prod, do `redis_url = "redis://redis:6379/"` -#redis_url = "redis://dev-redis:6379/" +redis_url = "redis://dev-redis:6379/" public_rate_limit_per_minute = 60_000 # 1GB of cache -response_cache_max_bytes = 10 ^ 9 +response_cache_max_bytes = 10000000000 [balanced_rpcs] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 702af10f..b620b068 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -3,10 +3,12 @@ version: "3.4" services: - # TODO: configure persistence? + # TODO: configure persistence? its just caches, but cold caches can be slow redis: build: ./redis-cell-server/ + # TODO: database and adminer + arbitrum: extends: file: docker-compose.common.yml diff --git a/docker-compose.yml b/docker-compose.yml index 916ca9a0..5d5f67fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,26 @@ version: "3.4" services: dev-redis: build: ./redis-cell-server/ - # TODO: expose these ports? + ports: + - 127.0.0.1:6379:6379 + # TODO: expose these ports? + + dev-db: + image: mysql + # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password + # (this is just an example, not intended to be a production configuration) + command: --default-authentication-plugin=mysql_native_password + environment: + MYSQL_ROOT_PASSWORD: dev_web3_proxy + ports: + - 127.0.0.1:3306:3306 + + dev-adminer: + image: adminer + ports: + - 127.0.0.1:8306:8080 + environment: + ADMINER_DEFAULT_SERVER: dev-db dev-eth: extends: @@ -14,21 +33,4 @@ services: volumes: - ./config/example.toml:/config.toml ports: - - 8544:8544 - - dev-user-portal: - - dev-db: - image: mysql - # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password - # (this is just an example, not intended to be a production configuration) - command: --default-authentication-plugin=mysql_native_password - environment: - MYSQL_ROOT_PASSWORD: example - - dev-adminer: - image: adminer - ports: - - 8080:8080 - environment: - ADMINER_DEFAULT_SERVER: dev-db + - 127.0.0.1:8544:8544 diff --git a/web3-proxy/Cargo.toml b/web3-proxy/Cargo.toml index f1809f5c..ad0822bf 100644 --- a/web3-proxy/Cargo.toml +++ b/web3-proxy/Cargo.toml @@ -14,11 +14,13 @@ deadlock_detection = ["parking_lot/deadlock_detection"] anyhow = { version = "1.0.58", features = ["backtrace"] } arc-swap = "1.5.0" argh = "0.1.8" -axum = { version = "0.5.13", features = ["serde_json", "tokio-tungstenite", "ws"] } +axum = { version = "0.5.14", features = ["serde_json", "tokio-tungstenite", "ws"] } axum-client-ip = "0.2.0" counter = "0.5.6" dashmap = "5.3.4" derive_more = "0.99.17" +diesel_migrations = "1.4.0" +dotenv = "0.15.0" ethers = { version = "0.15.0", features = ["rustls", "ws"] } fdlimit = "0.2.1" flume = "0.10.14" diff --git a/web3-proxy/src/app.rs b/web3-proxy/src/app.rs index 1f227455..a208a594 100644 --- a/web3-proxy/src/app.rs +++ b/web3-proxy/src/app.rs @@ -1,6 +1,7 @@ use axum::extract::ws::Message; use dashmap::mapref::entry::Entry as DashMapEntry; use dashmap::DashMap; +use diesel_migrations::embed_migrations; use ethers::core::utils::keccak256; use ethers::prelude::{Address, Block, BlockNumber, Bytes, Transaction, TxHash, H256, U64}; use futures::future::Abortable; @@ -35,6 +36,8 @@ use crate::jsonrpc::JsonRpcForwardedResponseEnum; use crate::jsonrpc::JsonRpcRequest; use crate::jsonrpc::JsonRpcRequestEnum; +embed_migrations!("../migrations/"); + // TODO: make this customizable? static APP_USER_AGENT: &str = concat!( "satoshiandkin/", @@ -287,6 +290,11 @@ impl Web3ProxyApp { Arc, Pin>>>, )> { + // // first, we connect to mysql and make sure the latest migrations have run + // let db_pool = todo!(app_config.db_url).await; + // let connection = db_pool.get().await; + // embedded_migrations::run_with_output(&connection, &mut std::io::stdout()); + let balanced_rpcs = app_config.balanced_rpcs.into_values().collect(); let private_rpcs = if let Some(private_rpcs) = app_config.private_rpcs { diff --git a/web3-proxy/src/bin/users.rs b/web3-proxy/src/bin/users.rs index 2e6431cd..2ebc7713 100644 --- a/web3-proxy/src/bin/users.rs +++ b/web3-proxy/src/bin/users.rs @@ -1,3 +1,7 @@ +//! Manage users. +//! +//! While most user management will (and should) happen through the web api, + fn main() { println!("hello, world"); }