first pass at a schema

2022-07-26 00:38:00 +00:00 · 2022-07-26 00:38:00 +00:00 · 4ec10e7a98
commit 4ec10e7a98
parent b2ebe247e6
8 changed files with 105 additions and 26 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -180,9 +180,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

 [[package]]
 name = "axum"
-version = "0.5.13"
+version = "0.5.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b9496f0c1d1afb7a2af4338bbe1d969cddfead41d87a9fb3aaa6d0bbc7af648"
+checksum = "c943a505c17b494638a38a9af129067f760c9c06794b9f57d499266909be8e72"
 dependencies = [
 "async-trait",
 "axum-core",
@ -977,6 +977,37 @@ dependencies = [
 "zeroize",
 ]

+[[package]]
+name = "diesel"
+version = "1.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d"
+dependencies = [
+ "byteorder",
+ "diesel_derives",
+]
+
+[[package]]
+name = "diesel_derives"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "diesel_migrations"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c"
+dependencies = [
+ "migrations_internals",
+ "migrations_macros",
+]
+
 [[package]]
 name = "diff"
 version = "0.1.12"
@ -1033,6 +1064,12 @@ dependencies = [
 "winapi 0.3.9",
 ]

+[[package]]
+name = "dotenv"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
+
 [[package]]
 name = "dtoa"
 version = "0.4.8"
@ -2274,6 +2311,27 @@ dependencies = [
 "autocfg",
 ]

+[[package]]
+name = "migrations_internals"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860"
+dependencies = [
+ "diesel",
+]
+
+[[package]]
+name = "migrations_macros"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c"
+dependencies = [
+ "migrations_internals",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@ -4281,6 +4339,8 @@ dependencies = [
 "counter",
 "dashmap",
 "derive_more",
+ "diesel_migrations",
+ "dotenv",
 "ethers",
 "fdlimit",
 "flume",
--- a/TODO.md
+++ b/TODO.md
@ -138,6 +138,7 @@ new endpoints for users:

 ## V2

+- [ ] jwt auth so people can easily switch from infura
 - [ ] handle log subscriptions
 - [ ] most things that are cached locally should probably be in shared redis caches
 - [ ] automated soft limit
@ -154,6 +155,7 @@ in another repo: event subscriber

 ## "Maybe some day" and other Miscellaneous Things

+- [ ] search for all the "TODO" items in the code and move them here
 - [ ] don't "unwrap" anywhere. give proper errors
 - [ ] instead of giving a rate limit error code, delay the connection's response at the start. reject if incoming requests is super high?
 - [ ] add the backend server to the header?
--- a/config/example.toml
+++ b/config/example.toml
@ -1,10 +1,9 @@
 [shared]
 chain_id = 1
-# in prod, do `redis_url = "redis://redis:6379/"`
-#redis_url = "redis://dev-redis:6379/"
+redis_url = "redis://dev-redis:6379/"
 public_rate_limit_per_minute = 60_000
 # 1GB of cache
-response_cache_max_bytes = 10 ^ 9
+response_cache_max_bytes = 10000000000

 [balanced_rpcs]

--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@ -3,10 +3,12 @@
 version: "3.4"

 services:
-  # TODO: configure persistence?
+  # TODO: configure persistence? its just caches, but cold caches can be slow
  redis:
    build: ./redis-cell-server/

+  # TODO: database and adminer
+
  arbitrum:
    extends:
      file: docker-compose.common.yml
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -5,7 +5,26 @@ version: "3.4"
 services:
  dev-redis:
    build: ./redis-cell-server/
-  # TODO: expose these ports?
+    ports:
+      - 127.0.0.1:6379:6379
+      # TODO: expose these ports?
+
+  dev-db:
+    image: mysql
+    # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password
+    # (this is just an example, not intended to be a production configuration)
+    command: --default-authentication-plugin=mysql_native_password
+    environment:
+      MYSQL_ROOT_PASSWORD: dev_web3_proxy
+    ports:
+      - 127.0.0.1:3306:3306
+
+  dev-adminer:
+    image: adminer
+    ports:
+      - 127.0.0.1:8306:8080
+    environment:
+      ADMINER_DEFAULT_SERVER: dev-db

  dev-eth:
    extends:
@ -14,21 +33,4 @@ services:
    volumes:
      - ./config/example.toml:/config.toml
    ports:
-      - 8544:8544
-
-  dev-user-portal:
-
-  dev-db:
-    image: mysql
-    # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password
-    # (this is just an example, not intended to be a production configuration)
-    command: --default-authentication-plugin=mysql_native_password
-    environment:
-      MYSQL_ROOT_PASSWORD: example
-
-  dev-adminer:
-    image: adminer
-    ports:
-      - 8080:8080
-    environment:
-      ADMINER_DEFAULT_SERVER: dev-db
+      - 127.0.0.1:8544:8544
--- a/web3-proxy/Cargo.toml
+++ b/web3-proxy/Cargo.toml
@ -14,11 +14,13 @@ deadlock_detection = ["parking_lot/deadlock_detection"]
 anyhow = { version = "1.0.58", features = ["backtrace"] }
 arc-swap = "1.5.0"
 argh = "0.1.8"
-axum = { version = "0.5.13", features = ["serde_json", "tokio-tungstenite", "ws"] }
+axum = { version = "0.5.14", features = ["serde_json", "tokio-tungstenite", "ws"] }
 axum-client-ip = "0.2.0"
 counter = "0.5.6"
 dashmap = "5.3.4"
 derive_more = "0.99.17"
+diesel_migrations = "1.4.0"
+dotenv = "0.15.0"
 ethers = { version = "0.15.0", features = ["rustls", "ws"] }
 fdlimit = "0.2.1"
 flume = "0.10.14"
--- a/web3-proxy/src/app.rs
+++ b/web3-proxy/src/app.rs
@ -1,6 +1,7 @@
 use axum::extract::ws::Message;
 use dashmap::mapref::entry::Entry as DashMapEntry;
 use dashmap::DashMap;
+use diesel_migrations::embed_migrations;
 use ethers::core::utils::keccak256;
 use ethers::prelude::{Address, Block, BlockNumber, Bytes, Transaction, TxHash, H256, U64};
 use futures::future::Abortable;
@ -35,6 +36,8 @@ use crate::jsonrpc::JsonRpcForwardedResponseEnum;
 use crate::jsonrpc::JsonRpcRequest;
 use crate::jsonrpc::JsonRpcRequestEnum;

+embed_migrations!("../migrations/");
+
 // TODO: make this customizable?
 static APP_USER_AGENT: &str = concat!(
    "satoshiandkin/",
@ -287,6 +290,11 @@ impl Web3ProxyApp {
        Arc<Web3ProxyApp>,
        Pin<Box<dyn Future<Output = anyhow::Result<()>>>>,
    )> {
+        // // first, we connect to mysql and make sure the latest migrations have run
+        // let db_pool = todo!(app_config.db_url).await;
+        // let connection = db_pool.get().await;
+        // embedded_migrations::run_with_output(&connection, &mut std::io::stdout());
+
        let balanced_rpcs = app_config.balanced_rpcs.into_values().collect();

        let private_rpcs = if let Some(private_rpcs) = app_config.private_rpcs {
--- a/web3-proxy/src/bin/users.rs
+++ b/web3-proxy/src/bin/users.rs
@ -1,3 +1,7 @@
+//! Manage users.
+//!
+//! While most user management will (and should) happen through the web api,
+
 fn main() {
    println!("hello, world");
 }