first pass at a schema

Cargo.lock

@@ -180,9 +180,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "axum"
-version = "0.5.13"
+version = "0.5.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b9496f0c1d1afb7a2af4338bbe1d969cddfead41d87a9fb3aaa6d0bbc7af648"
+checksum = "c943a505c17b494638a38a9af129067f760c9c06794b9f57d499266909be8e72"
 dependencies = [
  "async-trait",
  "axum-core",
@@ -977,6 +977,37 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "diesel"
+version = "1.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d"
+dependencies = [
+ "byteorder",
+ "diesel_derives",
+]
+
+[[package]]
+name = "diesel_derives"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "diesel_migrations"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c"
+dependencies = [
+ "migrations_internals",
+ "migrations_macros",
+]
+
 [[package]]
 name = "diff"
 version = "0.1.12"
@@ -1033,6 +1064,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "dotenv"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
+
 [[package]]
 name = "dtoa"
 version = "0.4.8"
@@ -2274,6 +2311,27 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "migrations_internals"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860"
+dependencies = [
+ "diesel",
+]
+
+[[package]]
+name = "migrations_macros"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c"
+dependencies = [
+ "migrations_internals",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -4281,6 +4339,8 @@ dependencies = [
  "counter",
  "dashmap",
  "derive_more",
+ "diesel_migrations",
+ "dotenv",
  "ethers",
  "fdlimit",
  "flume",

TODO.md

@@ -138,6 +138,7 @@ new endpoints for users:
 
 ## V2
 
+- [ ] jwt auth so people can easily switch from infura
 - [ ] handle log subscriptions
 - [ ] most things that are cached locally should probably be in shared redis caches
 - [ ] automated soft limit
@@ -154,6 +155,7 @@ in another repo: event subscriber
 
 ## "Maybe some day" and other Miscellaneous Things
 
+- [ ] search for all the "TODO" items in the code and move them here
 - [ ] don't "unwrap" anywhere. give proper errors
 - [ ] instead of giving a rate limit error code, delay the connection's response at the start. reject if incoming requests is super high?
 - [ ] add the backend server to the header?

config/example.toml

@@ -1,10 +1,9 @@
 [shared]
 chain_id = 1
-# in prod, do `redis_url = "redis://redis:6379/"`
+redis_url = "redis://dev-redis:6379/"
-#redis_url = "redis://dev-redis:6379/"
 public_rate_limit_per_minute = 60_000
 # 1GB of cache
-response_cache_max_bytes = 10 ^ 9
+response_cache_max_bytes = 10000000000
 
 [balanced_rpcs]
 

docker-compose.prod.yml

@@ -3,10 +3,12 @@
 version: "3.4"
 
 services:
-  # TODO: configure persistence?
+  # TODO: configure persistence? its just caches, but cold caches can be slow
   redis:
     build: ./redis-cell-server/
 
+  # TODO: database and adminer
+
   arbitrum:
     extends:
       file: docker-compose.common.yml

docker-compose.yml

@@ -5,8 +5,27 @@ version: "3.4"
 services:
   dev-redis:
     build: ./redis-cell-server/
+    ports:
+      - 127.0.0.1:6379:6379
       # TODO: expose these ports?
 
+  dev-db:
+    image: mysql
+    # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password
+    # (this is just an example, not intended to be a production configuration)
+    command: --default-authentication-plugin=mysql_native_password
+    environment:
+      MYSQL_ROOT_PASSWORD: dev_web3_proxy
+    ports:
+      - 127.0.0.1:3306:3306
+
+  dev-adminer:
+    image: adminer
+    ports:
+      - 127.0.0.1:8306:8080
+    environment:
+      ADMINER_DEFAULT_SERVER: dev-db
+
   dev-eth:
     extends:
       file: docker-compose.common.yml
@@ -14,21 +33,4 @@ services:
     volumes:
       - ./config/example.toml:/config.toml
     ports:
-      - 8544:8544
+      - 127.0.0.1:8544:8544
-
-  dev-user-portal:
-
-  dev-db:
-    image: mysql
-    # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password
-    # (this is just an example, not intended to be a production configuration)
-    command: --default-authentication-plugin=mysql_native_password
-    environment:
-      MYSQL_ROOT_PASSWORD: example
-
-  dev-adminer:
-    image: adminer
-    ports:
-      - 8080:8080
-    environment:
-      ADMINER_DEFAULT_SERVER: dev-db

web3-proxy/Cargo.toml

@@ -14,11 +14,13 @@ deadlock_detection = ["parking_lot/deadlock_detection"]
 anyhow = { version = "1.0.58", features = ["backtrace"] }
 arc-swap = "1.5.0"
 argh = "0.1.8"
-axum = { version = "0.5.13", features = ["serde_json", "tokio-tungstenite", "ws"] }
+axum = { version = "0.5.14", features = ["serde_json", "tokio-tungstenite", "ws"] }
 axum-client-ip = "0.2.0"
 counter = "0.5.6"
 dashmap = "5.3.4"
 derive_more = "0.99.17"
+diesel_migrations = "1.4.0"
+dotenv = "0.15.0"
 ethers = { version = "0.15.0", features = ["rustls", "ws"] }
 fdlimit = "0.2.1"
 flume = "0.10.14"

web3-proxy/src/app.rs

@@ -1,6 +1,7 @@
 use axum::extract::ws::Message;
 use dashmap::mapref::entry::Entry as DashMapEntry;
 use dashmap::DashMap;
+use diesel_migrations::embed_migrations;
 use ethers::core::utils::keccak256;
 use ethers::prelude::{Address, Block, BlockNumber, Bytes, Transaction, TxHash, H256, U64};
 use futures::future::Abortable;
@@ -35,6 +36,8 @@ use crate::jsonrpc::JsonRpcForwardedResponseEnum;
 use crate::jsonrpc::JsonRpcRequest;
 use crate::jsonrpc::JsonRpcRequestEnum;
 
+embed_migrations!("../migrations/");
+
 // TODO: make this customizable?
 static APP_USER_AGENT: &str = concat!(
     "satoshiandkin/",
@@ -287,6 +290,11 @@ impl Web3ProxyApp {
         Arc<Web3ProxyApp>,
         Pin<Box<dyn Future<Output = anyhow::Result<()>>>>,
     )> {
+        // // first, we connect to mysql and make sure the latest migrations have run
+        // let db_pool = todo!(app_config.db_url).await;
+        // let connection = db_pool.get().await;
+        // embedded_migrations::run_with_output(&connection, &mut std::io::stdout());
+
         let balanced_rpcs = app_config.balanced_rpcs.into_values().collect();
 
         let private_rpcs = if let Some(private_rpcs) = app_config.private_rpcs {

web3-proxy/src/bin/users.rs

@@ -1,3 +1,7 @@
+//! Manage users.
+//!
+//! While most user management will (and should) happen through the web api,
+
 fn main() {
     println!("hello, world");
 }