From 4cb65b0fa4975919d0ea26837cf542087a06f375 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Tue, 26 Jul 2022 04:53:38 +0000 Subject: [PATCH] setup database and stub migrations --- .gitignore | 1 + Cargo.lock | 947 ++++++++++++++++-- Cargo.toml | 2 + README.md | 1 + TODO.md | 1 + config/example.toml | 1 + diesel.toml | 5 - docker-compose.yml | 8 +- entities/Cargo.toml | 8 + entities/src/main.rs | 3 + migration/Cargo.toml | 21 + migration/README.md | 41 + migration/first_draft.sql | 45 + migration/src/lib.rs | 12 + .../src/m20220101_000001_create_table.rs | 48 + migration/src/main.rs | 6 + web3-proxy/Cargo.toml | 9 +- web3-proxy/diesel.toml | 5 - web3-proxy/migrations/.gitkeep | 0 .../2022-07-25-234230_first/down.sql | 4 - .../migrations/2022-07-25-234230_first/up.sql | 43 - web3-proxy/src/app.rs | 50 +- web3-proxy/src/bb8_helpers.rs | 2 + web3-proxy/src/config.rs | 3 +- web3-proxy/src/connections.rs | 2 + web3-proxy/src/main.rs | 2 + 26 files changed, 1122 insertions(+), 148 deletions(-) delete mode 100644 diesel.toml create mode 100644 entities/Cargo.toml create mode 100644 entities/src/main.rs create mode 100644 migration/Cargo.toml create mode 100644 migration/README.md create mode 100644 migration/first_draft.sql create mode 100644 migration/src/lib.rs create mode 100644 migration/src/m20220101_000001_create_table.rs create mode 100644 migration/src/main.rs delete mode 100644 web3-proxy/diesel.toml delete mode 100644 web3-proxy/migrations/.gitkeep delete mode 100644 web3-proxy/migrations/2022-07-25-234230_first/down.sql delete mode 100644 web3-proxy/migrations/2022-07-25-234230_first/up.sql diff --git a/.gitignore b/.gitignore index f1bf4b01..40d0aca3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /config/*.toml +/data flamegraph.svg perf.data perf.data.old diff --git a/Cargo.lock b/Cargo.lock index e94b30ea..06d83912 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,6 +59,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "ansi_term" version = "0.12.1" @@ -127,6 +133,140 @@ dependencies = [ "term", ] +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5262ed948da60dd8956c6c5aca4d4163593dddb7b32d73267c93dab7b2e98940" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "num_cpus", + "once_cell", + "tokio", +] + +[[package]] +name = "async-io" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e18f61464ae81cde0a23e713ae8fd299580c54d697a35820cfd0625b8b0e07" +dependencies = [ + "concurrent-queue", + "futures-lite", + "libc", + "log", + "once_cell", + "parking", + "polling", + "slab", + "socket2", + "waker-fn", + "winapi 0.3.9", +] + +[[package]] +name = "async-lock" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-attributes", + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-task" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" + [[package]] name = "async-trait" version = "0.1.53" @@ -149,6 +289,21 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "atoi" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + [[package]] name = "atty" version = "0.2.14" @@ -251,6 +406,19 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bae" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b8de67cc41132507eeece2584804efcb15f85ba516e34c944b7667f480397a" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "base16ct" version = "0.1.1" @@ -421,6 +589,20 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "blocking" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + [[package]] name = "bs58" version = "0.4.0" @@ -493,6 +675,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + [[package]] name = "camino" version = "1.0.9" @@ -566,8 +754,12 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ + "libc", "num-integer", "num-traits", + "serde", + "time 0.1.43", + "winapi 0.3.9", ] [[package]] @@ -592,16 +784,16 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.18" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" +checksum = "44bbe24bbd31a185bc2c4f7c2abe80bea13a20d57ee4e55be70ac512bdc76417" dependencies = [ "atty", "bitflags", "clap_derive", "clap_lex", "indexmap", - "lazy_static", + "once_cell", "strsim", "termcolor", "textwrap 0.15.0", @@ -609,9 +801,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.18" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" +checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -622,9 +814,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.2.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" dependencies = [ "os_str_bytes", ] @@ -711,6 +903,15 @@ dependencies = [ "tokio-util 0.7.2", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "console" version = "0.14.1" @@ -739,6 +940,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "const-oid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" + [[package]] name = "const-oid" version = "0.9.0" @@ -781,6 +988,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53757d12b596c16c78b83458d732a5d1a17ab3f53f2f7412f6fb57cc8a140ab3" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0165d2900ae6778e36e80bbc4da3b5eefccee9ba939761f9c2882a5d9af3ff" + [[package]] name = "crc32fast" version = "1.3.2" @@ -861,6 +1083,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.8" @@ -877,6 +1109,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" +dependencies = [ + "generic-array 0.14.5", + "subtle", +] + [[package]] name = "crypto-bigint" version = "0.4.8" @@ -921,6 +1163,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctor" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "ctr" version = "0.8.0" @@ -942,13 +1194,24 @@ dependencies = [ "parking_lot_core 0.9.3", ] +[[package]] +name = "der" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" +dependencies = [ + "const-oid 0.7.1", + "crypto-bigint 0.3.2", + "pem-rfc7468", +] + [[package]] name = "der" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dd2ae565c0a381dde7fade45fce95984c568bdcb4700a4fdbe3175e0380b2f" dependencies = [ - "const-oid", + "const-oid 0.9.0", "zeroize", ] @@ -977,37 +1240,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "diesel" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" -dependencies = [ - "byteorder", - "diesel_derives", -] - -[[package]] -name = "diesel_derives" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "diesel_migrations" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" -dependencies = [ - "migrations_internals", - "migrations_macros", -] - [[package]] name = "diff" version = "0.1.12" @@ -1088,7 +1320,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bd46e0c364655e5baf2f5e99b603e7a09905da9966d7928d7470af393b28670" dependencies = [ - "der", + "der 0.6.0", "elliptic-curve", "rfc6979", "signature", @@ -1107,13 +1339,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c47abd0a791d2ac0c7aa1118715f85b83689e4522c4e3a244e159d4fc9848a8d" dependencies = [ "base16ct", - "crypto-bigint", - "der", + "crypto-bigint 0.4.8", + "der 0.6.0", "digest 0.10.3", "ff", "generic-array 0.14.5", "group", - "pkcs8", + "pkcs8 0.9.0", "rand_core", "sec1", "subtle", @@ -1144,6 +1376,10 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "entities" +version = "0.1.0" + [[package]] name = "eth-keystore" version = "0.4.1" @@ -1163,7 +1399,7 @@ dependencies = [ "sha2 0.10.2", "sha3", "thiserror", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -1440,6 +1676,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "event-listener" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" + [[package]] name = "eyre" version = "0.6.8" @@ -1456,6 +1698,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "1.7.0" @@ -1682,12 +1930,38 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "futures-intrusive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62007592ac46aa7c2b6416f7deb9a8a8f63a01e0f1d6e1787d5630170db2b63e" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.11.2", +] + [[package]] name = "futures-io" version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-locks" version = "0.7.0" @@ -1799,6 +2073,18 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "gloo-timers" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "group" version = "0.12.0" @@ -1877,6 +2163,9 @@ name = "heck" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -2180,6 +2469,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lalrpop" version = "0.19.8" @@ -2217,6 +2515,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "lazycell" @@ -2230,6 +2531,12 @@ version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +[[package]] +name = "libm" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2264,6 +2571,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", + "value-bag", ] [[package]] @@ -2312,24 +2620,11 @@ dependencies = [ ] [[package]] -name = "migrations_internals" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +name = "migration" +version = "0.1.0" dependencies = [ - "diesel", -] - -[[package]] -name = "migrations_macros" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" -dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", - "syn", + "async-std", + "sea-orm-migration", ] [[package]] @@ -2338,6 +2633,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.5.1" @@ -2428,6 +2729,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nonempty" version = "0.7.0" @@ -2477,6 +2788,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566d173b2f9406afbc5510a90925d5a2cd80cae4605631f1212303df265de011" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.2" @@ -2526,6 +2854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2592,6 +2921,30 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" +[[package]] +name = "ouroboros" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f31a3b678685b150cba82b702dcdc5e155893f63610cf388d30cd988d4ca2bf" +dependencies = [ + "aliasable", + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084fd65d5dd8b3772edccb5ffd1e4b7eba43897ecd0f9401e330e8c542959408" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "parity-scale-codec" version = "3.1.2" @@ -2618,6 +2971,12 @@ dependencies = [ "syn", ] +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.2" @@ -2691,6 +3050,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + [[package]] name = "path-slash" version = "0.2.0" @@ -2721,6 +3086,15 @@ dependencies = [ "sha2 0.10.2", ] +[[package]] +name = "pem-rfc7468" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01de5d978f34aa4b2296576379fcc416034702fd94117c56ffd8a1a767cefb30" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -2829,14 +3203,36 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78f66c04ccc83dd4486fd46c33896f4e17b24a7a3a6400dedc48ed0ddd72320" +dependencies = [ + "der 0.5.1", + "pkcs8 0.8.0", + "zeroize", +] + +[[package]] +name = "pkcs8" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" +dependencies = [ + "der 0.5.1", + "spki 0.5.4", + "zeroize", +] + [[package]] name = "pkcs8" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" dependencies = [ - "der", - "spki", + "der 0.6.0", + "spki 0.6.0", ] [[package]] @@ -2873,6 +3269,53 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polling" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685404d509889fade3e86fe3a5803bca2ec09b0c0778d5ada6ec8bf7a8de5259" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "log", + "wepoll-ffi", + "winapi 0.3.9", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +dependencies = [ + "base64 0.13.0", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2 0.10.2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd6e8b7189a73169290e89bd24c771071f1012d8fe6f738f5226531f0b03d89" +dependencies = [ + "bytes", + "chrono", + "fallible-iterator", + "postgres-protocol", + "serde", + "serde_json", + "time 0.3.11", + "uuid 1.1.2", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -3160,7 +3603,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88c86280f057430a52f4861551b092a01b419b8eacefc7c995eacb9dc132fe32" dependencies = [ - "crypto-bigint", + "crypto-bigint 0.4.8", "hmac", "zeroize", ] @@ -3210,6 +3653,26 @@ dependencies = [ "syn", ] +[[package]] +name = "rsa" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf22754c49613d2b3b119f0e5d46e34a2c628a937e3024b8762de4e7d8c710b" +dependencies = [ + "byteorder", + "digest 0.10.3", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8 0.8.0", + "rand_core", + "smallvec", + "subtle", + "zeroize", +] + [[package]] name = "rust_decimal" version = "1.25.0" @@ -3328,6 +3791,164 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sea-orm" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05961693fe32ef4dfc964e48dcd0363151e3fab0215e3f3e138e034710339ca" +dependencies = [ + "async-stream", + "async-trait", + "chrono", + "futures", + "futures-util", + "log", + "once_cell", + "ouroboros", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-strum", + "serde", + "serde_json", + "sqlx", + "time 0.3.11", + "tracing", + "url", + "uuid 1.1.2", +] + +[[package]] +name = "sea-orm-cli" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f527a5c69fc9b9cd8d243fa45f5bcf14e5e2a3e94b003267be1b075221c5b04b" +dependencies = [ + "async-std", + "chrono", + "clap 3.2.15", + "dotenv", + "regex", + "sea-schema", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "sea-orm-macros" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8c8c00c8a57ae61991b34822f32ded409afb66a971d4952d857c329efbc355" +dependencies = [ + "bae", + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sea-orm-migration" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697d983cc6fd0ec497934cbfd16b3872b7e95be369e8851bdc65770c3b5ec57d" +dependencies = [ + "async-trait", + "clap 3.2.15", + "dotenv", + "sea-orm", + "sea-orm-cli", + "sea-schema", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "sea-query" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2997a3e57614c0ff2212890e143deadb4a31f8057cbe7fc163daa3f297a33527" +dependencies = [ + "chrono", + "postgres-types", + "rust_decimal", + "sea-query-derive", + "sea-query-driver", + "serde_json", + "time 0.3.11", + "uuid 1.1.2", +] + +[[package]] +name = "sea-query-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cdc022b4f606353fe5dc85b09713a04e433323b70163e81513b141c6ae6eb5" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", + "thiserror", +] + +[[package]] +name = "sea-query-driver" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbda46eb3484cae1efb7bc68bca50f553a5b42c076cf4cbfae05b27f707549d4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sea-schema" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d070aba647637b533bd669a8e430bdc8f7d5249c9b53402da34347563bbfca" +dependencies = [ + "futures", + "sea-query", + "sea-schema-derive", +] + +[[package]] +name = "sea-schema-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56821b7076f5096b8f726e2791ad255a99c82498e08ec477a65a96c461ff1927" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sea-strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391d06a6007842cfe79ac6f7f53911b76dfd69fc9a6769f1cf6569d12ce20e1b" +dependencies = [ + "sea-strum_macros", +] + +[[package]] +name = "sea-strum_macros" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b4397b825df6ccf1e98bcdabef3bbcfc47ff5853983467850eeab878384f21" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "sec1" version = "0.3.0" @@ -3335,9 +3956,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" dependencies = [ "base16ct", - "der", + "der 0.6.0", "generic-array 0.14.5", - "pkcs8", + "pkcs8 0.9.0", "subtle", "zeroize", ] @@ -3575,6 +4196,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" +dependencies = [ + "base64ct", + "der 0.5.1", +] + [[package]] name = "spki" version = "0.6.0" @@ -3582,9 +4213,121 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" dependencies = [ "base64ct", - "der", + "der 0.6.0", ] +[[package]] +name = "sqlformat" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4b7922be017ee70900be125523f38bdd644f4f06a1b16e8fa5a8ee8c34bffd4" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f82cbe94f41641d6c410ded25bbf5097c240cefdf8e3b06d04198d0a96af6a4" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b69bf218860335ddda60d6ce85ee39f6cf6e5630e300e19757d1de15886a093" +dependencies = [ + "ahash", + "atoi", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "digest 0.10.3", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-util", + "generic-array 0.14.5", + "hashlink", + "hex", + "indexmap", + "itoa 1.0.2", + "libc", + "log", + "memchr", + "num-bigint", + "once_cell", + "paste", + "percent-encoding", + "rand", + "rsa", + "rust_decimal", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha-1", + "sha2 0.10.2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "time 0.3.11", + "tokio-stream", + "url", + "uuid 1.1.2", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40c63177cf23d356b159b60acd27c54af7423f1736988502e36bae9a712118f" +dependencies = [ + "dotenv", + "either", + "heck 0.4.0", + "once_cell", + "proc-macro2", + "quote", + "serde_json", + "sha2 0.10.2", + "sqlx-core", + "sqlx-rt", + "syn", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874e93a365a598dc3dadb197565952cb143ae4aa716f7bcc933a8d836f6bf89f" +dependencies = [ + "once_cell", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -3604,6 +4347,16 @@ dependencies = [ "precomputed-hash", ] +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strsim" version = "0.10.0" @@ -3646,7 +4399,7 @@ checksum = "a55e00b6f95abd889ce398bd7eab2a9c62cd27281cf1bfba70847340557434cf" dependencies = [ "anyhow", "cfg-if 1.0.0", - "clap 3.1.18", + "clap 3.2.15", "console 0.14.1", "dialoguer", "fs2", @@ -3791,6 +4544,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "time" version = "0.3.11" @@ -4157,6 +4920,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.7.1" @@ -4191,18 +4960,44 @@ dependencies = [ "serde", ] +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" +dependencies = [ + "getrandom", + "serde", +] + [[package]] name = "valuable" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-bag" +version = "1.0.0-alpha.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" +dependencies = [ + "ctor", + "version_check", +] + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "walkdir" version = "2.3.2" @@ -4334,13 +5129,14 @@ dependencies = [ "anyhow", "arc-swap", "argh", + "async-std", "axum", "axum-client-ip", "counter", "dashmap", "derive_more", - "diesel_migrations", "dotenv", + "entities", "ethers", "fdlimit", "flume", @@ -4348,6 +5144,7 @@ dependencies = [ "hashbrown", "indexmap", "linkedhashmap", + "migration", "notify", "num", "parking_lot 0.12.1", @@ -4357,6 +5154,7 @@ dependencies = [ "regex", "reqwest", "rustc-hash", + "sea-orm", "serde", "serde_json", "tokio", @@ -4387,6 +5185,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + [[package]] name = "winapi" version = "0.2.8" @@ -4541,7 +5348,7 @@ dependencies = [ "hmac", "pbkdf2 0.10.1", "sha1", - "time", + "time 0.3.11", "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index 7e1f30de..102a84c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ + "entities", + "migration", "linkedhashmap", "redis-cell-client", "web3-proxy", diff --git a/README.md b/README.md index 6e056eff..7c87e47d 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ All other requests are sent to an RPC server on the latest block (alchemy, moral Each server has different limits to configure. The `soft_limit` is the number of parallel active requests where a server starts to slow down. The `hard_limit` is where a server starts giving rate limits or other errors. ``` +$ cargo install sea-orm-cli $ cargo run --release -- --help ``` ``` diff --git a/TODO.md b/TODO.md index 789dddff..a5a7f4fb 100644 --- a/TODO.md +++ b/TODO.md @@ -138,6 +138,7 @@ new endpoints for users: ## V2 +- [ ] sea-orm brings in async-std, but we are using tokio. benchmark switching - [ ] jwt auth so people can easily switch from infura - [ ] handle log subscriptions - [ ] most things that are cached locally should probably be in shared redis caches diff --git a/config/example.toml b/config/example.toml index 5659d261..888c4d1d 100644 --- a/config/example.toml +++ b/config/example.toml @@ -1,5 +1,6 @@ [shared] chain_id = 1 +db_url = "mysql://root:dev_web3_proxy@dev-mysql:3306/dev_web3_proxy" redis_url = "redis://dev-redis:6379/" public_rate_limit_per_minute = 60_000 # 1GB of cache diff --git a/diesel.toml b/diesel.toml deleted file mode 100644 index 92267c82..00000000 --- a/diesel.toml +++ /dev/null @@ -1,5 +0,0 @@ -# For documentation on how to configure this file, -# see diesel.rs/guides/configuring-diesel-cli - -[print_schema] -file = "src/schema.rs" diff --git a/docker-compose.yml b/docker-compose.yml index 5d5f67fe..85087ae6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,18 +11,18 @@ services: dev-db: image: mysql - # NOTE: use of "mysql_native_password" is not recommended: https://dev.mysql.com/doc/refman/8.0/en/upgrading-from-previous-series.html#upgrade-caching-sha2-password - # (this is just an example, not intended to be a production configuration) - command: --default-authentication-plugin=mysql_native_password environment: MYSQL_ROOT_PASSWORD: dev_web3_proxy + MYSQL_DATABASE: dev_web3_proxy ports: - 127.0.0.1:3306:3306 + volumes: + - ./data/dev_mysql:/var/lib/mysql dev-adminer: image: adminer ports: - - 127.0.0.1:8306:8080 + - 8306:8080 environment: ADMINER_DEFAULT_SERVER: dev-db diff --git a/entities/Cargo.toml b/entities/Cargo.toml new file mode 100644 index 00000000..f9763dd4 --- /dev/null +++ b/entities/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "entities" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/entities/src/main.rs b/entities/src/main.rs new file mode 100644 index 00000000..e7a11a96 --- /dev/null +++ b/entities/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/migration/Cargo.toml b/migration/Cargo.toml new file mode 100644 index 00000000..fca5222e --- /dev/null +++ b/migration/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "migration" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +name = "migration" +path = "src/lib.rs" + +[dependencies] +async-std = { version = "^1", features = ["attributes", "tokio1"] } + +[dependencies.sea-orm-migration] +version = "^0.9.0" +features = [ + # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. + # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. + "runtime-tokio-rustls", # `ASYNC_RUNTIME` featrure + "sqlx-mysql", # `DATABASE_DRIVER` feature +] \ No newline at end of file diff --git a/migration/README.md b/migration/README.md new file mode 100644 index 00000000..b3ea53eb --- /dev/null +++ b/migration/README.md @@ -0,0 +1,41 @@ +# Running Migrator CLI + +- Generate a new migration file + ```sh + cargo run -- migrate generate MIGRATION_NAME + ``` +- Apply all pending migrations + ```sh + cargo run + ``` + ```sh + cargo run -- up + ``` +- Apply first 10 pending migrations + ```sh + cargo run -- up -n 10 + ``` +- Rollback last applied migrations + ```sh + cargo run -- down + ``` +- Rollback last 10 applied migrations + ```sh + cargo run -- down -n 10 + ``` +- Drop all tables from the database, then reapply all migrations + ```sh + cargo run -- fresh + ``` +- Rollback all applied migrations, then reapply all migrations + ```sh + cargo run -- refresh + ``` +- Rollback all applied migrations + ```sh + cargo run -- reset + ``` +- Check the status of all migrations + ```sh + cargo run -- status + ``` diff --git a/migration/first_draft.sql b/migration/first_draft.sql new file mode 100644 index 00000000..a7cdc7c0 --- /dev/null +++ b/migration/first_draft.sql @@ -0,0 +1,45 @@ +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + chain INT, + primary_address VARCHAR(42), + description VARCHAR(255), + email VARCHAR(320), +) + +-- TODO: foreign key +-- TODO: how should we store addresses? +-- TODO: creation time? +-- TODO: permissions. likely similar to infura +CREATE TABLE secondary_users ( + id SERIAL PRIMARY KEY, + users_id BIGINT, + secondary_address VARCHAR(42), + chain INT, + description VARCHAR, + email VARCHAR(320), +) + +-- TODO: creation time? +CREATE TABLE blocklist ( + id SERIAL PRIMARY KEY, + blocked_address VARCHAR, + chain INT, + reason TEXT, +) + +-- TODO: foreign key +-- TODO: index on api_key +-- TODO: what size for api_key +-- TODO: track active with a timestamp? +-- TODO: creation time? +-- TODO: requests_per_second INT, +-- TODO: requests_per_day INT, +-- TODO: more security features. likely similar to infura +CREATE TABLE user_keys ( + id SERIAL PRIMARY KEY, + users_id BIGINT, + api_key VARCHAR, + description VARCHAR, + private_txs BOOLEAN, + active BOOLEAN, +) diff --git a/migration/src/lib.rs b/migration/src/lib.rs new file mode 100644 index 00000000..2c605afb --- /dev/null +++ b/migration/src/lib.rs @@ -0,0 +1,12 @@ +pub use sea_orm_migration::prelude::*; + +mod m20220101_000001_create_table; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![Box::new(m20220101_000001_create_table::Migration)] + } +} diff --git a/migration/src/m20220101_000001_create_table.rs b/migration/src/m20220101_000001_create_table.rs new file mode 100644 index 00000000..b0582446 --- /dev/null +++ b/migration/src/m20220101_000001_create_table.rs @@ -0,0 +1,48 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + todo!(); + + manager + .create_table( + Table::create() + .table(Post::Table) + .if_not_exists() + .col( + ColumnDef::new(Post::Id) + .integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Post::Title).string().not_null()) + .col(ColumnDef::new(Post::Text).string().not_null()) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + todo!(); + + manager + .drop_table(Table::drop().table(Post::Table).to_owned()) + .await + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum Post { + Table, + Id, + Title, + Text, +} diff --git a/migration/src/main.rs b/migration/src/main.rs new file mode 100644 index 00000000..c6b6e48d --- /dev/null +++ b/migration/src/main.rs @@ -0,0 +1,6 @@ +use sea_orm_migration::prelude::*; + +#[async_std::main] +async fn main() { + cli::run_cli(migration::Migrator).await; +} diff --git a/web3-proxy/Cargo.toml b/web3-proxy/Cargo.toml index ad0822bf..78772807 100644 --- a/web3-proxy/Cargo.toml +++ b/web3-proxy/Cargo.toml @@ -9,8 +9,13 @@ default-run = "web3-proxy" [features] default = ["deadlock_detection"] deadlock_detection = ["parking_lot/deadlock_detection"] +verbose = ["sea-orm/debug-print"] [dependencies] +redis-cell-client = { path = "../redis-cell-client" } +entities = { path = "../entities" } +migration = { path = "../migration" } + anyhow = { version = "1.0.58", features = ["backtrace"] } arc-swap = "1.5.0" argh = "0.1.8" @@ -19,7 +24,6 @@ axum-client-ip = "0.2.0" counter = "0.5.6" dashmap = "5.3.4" derive_more = "0.99.17" -diesel_migrations = "1.4.0" dotenv = "0.15.0" ethers = { version = "0.15.0", features = ["rustls", "ws"] } fdlimit = "0.2.1" @@ -30,7 +34,6 @@ indexmap = "1.9.1" linkedhashmap = { path = "../linkedhashmap", features = ["inline-more"] } notify = "4.0.17" num = "0.4.0" -redis-cell-client = { path = "../redis-cell-client" } parking_lot = { version = "0.12.1", features = ["arc_lock"] } petgraph = "0.6.2" proctitle = "0.1.1" @@ -39,9 +42,11 @@ regex = "1.6.0" reqwest = { version = "0.11.11", default-features = false, features = ["json", "tokio-rustls"] } rustc-hash = "1.1.0" # siwe = "0.4.0" # blocked by https://github.com/spruceid/siwe-rs/issues/36 +sea-orm = { version = "0.9.1", features = [ "macros" ] } serde = { version = "1.0.140", features = [] } serde_json = { version = "1.0.82", default-features = false, features = ["alloc", "raw_value"] } tokio = { version = "1.20.1", features = ["full", "tracing"] } +async-std = { version = "^1", features = ["attributes", "tokio1"] } toml = "0.5.9" tracing = "0.1.35" # TODO: tracing-subscriber has serde and serde_json features that we might want to use diff --git a/web3-proxy/diesel.toml b/web3-proxy/diesel.toml deleted file mode 100644 index 92267c82..00000000 --- a/web3-proxy/diesel.toml +++ /dev/null @@ -1,5 +0,0 @@ -# For documentation on how to configure this file, -# see diesel.rs/guides/configuring-diesel-cli - -[print_schema] -file = "src/schema.rs" diff --git a/web3-proxy/migrations/.gitkeep b/web3-proxy/migrations/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/web3-proxy/migrations/2022-07-25-234230_first/down.sql b/web3-proxy/migrations/2022-07-25-234230_first/down.sql deleted file mode 100644 index 89469da5..00000000 --- a/web3-proxy/migrations/2022-07-25-234230_first/down.sql +++ /dev/null @@ -1,4 +0,0 @@ -DROP TABLE users; -DROP TABLE secondary_users; -DROP TABLE blocklist; -DROP TABLE user_keys; diff --git a/web3-proxy/migrations/2022-07-25-234230_first/up.sql b/web3-proxy/migrations/2022-07-25-234230_first/up.sql deleted file mode 100644 index 8289313b..00000000 --- a/web3-proxy/migrations/2022-07-25-234230_first/up.sql +++ /dev/null @@ -1,43 +0,0 @@ -CREATE TABLE users ( - id SERIAL PRIMARY KEY, - primary_address VARCHAR NOT NULL, - chain INT NOT NULL, - description VARCHAR, - email VARCHAR DEFAULT NULL, -) - -CREATE TABLE secondary_users ( - id SERIAL PRIMARY KEY, - -- TODO: foreign key - users_id BIGINT, - -- TODO: how should we store addresses? - secondary_address VARCHAR NOT NULL, - chain INT NOT NULL, - description VARCHAR, - -- TODO: creation time? - -- TODO: permissions. likely similar to infura -) - -CREATE TABLE blocklist ( - id SERIAL PRIMARY KEY, - -- TODO: creation time? - blocked_address VARCHAR NOT NULL, - chain INT NOT NULL, - reason TEXT, -) - -CREATE TABLE user_keys ( - id SERIAL PRIMARY KEY, - -- TODO: foreign key - users_id BIGINT, - -- TODO: index on api_key - api_key VARCHAR NOT NULL, - description VARCHAR, - private_txs BOOLEAN, - -- TODO: track active with a timestamp? - active BOOLEAN, - -- TODO: creation time? - -- TODO: requests_per_second INT, - -- TODO: requests_per_day INT, - -- TODO: more security features. likely similar to infura -) diff --git a/web3-proxy/src/app.rs b/web3-proxy/src/app.rs index a208a594..70b5d5b1 100644 --- a/web3-proxy/src/app.rs +++ b/web3-proxy/src/app.rs @@ -1,7 +1,7 @@ +use anyhow::Context; use axum::extract::ws::Message; use dashmap::mapref::entry::Entry as DashMapEntry; use dashmap::DashMap; -use diesel_migrations::embed_migrations; use ethers::core::utils::keccak256; use ethers::prelude::{Address, Block, BlockNumber, Bytes, Transaction, TxHash, H256, U64}; use futures::future::Abortable; @@ -36,8 +36,6 @@ use crate::jsonrpc::JsonRpcForwardedResponseEnum; use crate::jsonrpc::JsonRpcRequest; use crate::jsonrpc::JsonRpcRequestEnum; -embed_migrations!("../migrations/"); - // TODO: make this customizable? static APP_USER_AGENT: &str = concat!( "satoshiandkin/", @@ -264,6 +262,7 @@ pub struct Web3ProxyApp { pending_tx_sender: broadcast::Sender, pending_transactions: Arc>, public_rate_limiter: Option, + db_conn: Option, } impl fmt::Debug for Web3ProxyApp { @@ -295,6 +294,27 @@ impl Web3ProxyApp { // let connection = db_pool.get().await; // embedded_migrations::run_with_output(&connection, &mut std::io::stdout()); + let db_conn = if let Some(db_url) = app_config.shared.db_url { + let mut db_opt = sea_orm::ConnectOptions::new(db_url); + + // TODO: load all these options from the config file + db_opt + .max_connections(100) + .min_connections(num_workers.try_into()?) + .connect_timeout(Duration::from_secs(8)) + .idle_timeout(Duration::from_secs(8)) + .max_lifetime(Duration::from_secs(60)) + .sqlx_logging(true); + // .sqlx_logging_level(log::LevelFilter::Info); + + let db_conn = sea_orm::Database::connect(db_opt).await?; + + Some(db_conn) + } else { + info!("no database"); + None + }; + let balanced_rpcs = app_config.balanced_rpcs.into_values().collect(); let private_rpcs = if let Some(private_rpcs) = app_config.private_rpcs { @@ -318,14 +338,13 @@ impl Web3ProxyApp { ); let redis_client_pool = match app_config.shared.redis_url { - Some(redis_address) => { - info!("Connecting to redis on {}", redis_address); + Some(redis_url) => { + info!("Connecting to redis on {}", redis_url); - let manager = RedisConnectionManager::new(redis_address)?; + let manager = RedisConnectionManager::new(redis_url)?; let min_size = num_workers as u32; let max_size = min_size * 4; - // TODO: min_idle? // TODO: set max_size based on max expected concurrent connections? set based on num_workers? let builder = bb8::Pool::builder() @@ -338,14 +357,17 @@ impl Web3ProxyApp { Some(pool) } None => { - warn!("No redis address"); + warn!("no redis connection"); None } }; let (head_block_sender, head_block_receiver) = watch::channel(Arc::new(Block::default())); // TODO: will one receiver lagging be okay? how big should this be? - let (pending_tx_sender, pending_tx_receiver) = broadcast::channel(16); + let (pending_tx_sender, pending_tx_receiver) = broadcast::channel(256); + + // TODO: use this? it could listen for confirmed transactions and then clear pending_transactions, but the head_block_sender is doing that + drop(pending_tx_receiver); // TODO: this will grow unbounded!! add some expiration to this. and probably move to redis let pending_transactions = Arc::new(DashMap::new()); @@ -364,7 +386,8 @@ impl Web3ProxyApp { Some(pending_tx_sender.clone()), pending_transactions.clone(), ) - .await?; + .await + .context("balanced rpcs")?; handles.push(balanced_handle); @@ -384,16 +407,14 @@ impl Web3ProxyApp { Some(pending_tx_sender.clone()), pending_transactions.clone(), ) - .await?; + .await + .context("private_rpcs")?; handles.push(private_handle); private_rpcs }; - // TODO: use this? it could listen for confirmed transactions and then clear pending_transactions, but the head_block_sender is doing that - drop(pending_tx_receiver); - // TODO: how much should we allow? let public_max_burst = app_config.shared.public_rate_limit_per_minute / 3; @@ -421,6 +442,7 @@ impl Web3ProxyApp { pending_tx_sender, pending_transactions, public_rate_limiter, + db_conn, }; let app = Arc::new(app); diff --git a/web3-proxy/src/bb8_helpers.rs b/web3-proxy/src/bb8_helpers.rs index 1826359c..847364e7 100644 --- a/web3-proxy/src/bb8_helpers.rs +++ b/web3-proxy/src/bb8_helpers.rs @@ -1,3 +1,5 @@ +// TODO: move this into redis-cell-client + use redis_cell_client::bb8; use tracing::warn; diff --git a/web3-proxy/src/config.rs b/web3-proxy/src/config.rs index 95744778..dc24e050 100644 --- a/web3-proxy/src/config.rs +++ b/web3-proxy/src/config.rs @@ -39,6 +39,7 @@ pub struct AppConfig { pub struct RpcSharedConfig { // TODO: better type for chain_id? max of `u64::MAX / 2 - 36` https://github.com/ethereum/EIPs/issues/2294 pub chain_id: u64, + pub db_url: Option, pub redis_url: Option, #[serde(default = "default_public_rate_limit_per_minute")] pub public_rate_limit_per_minute: u32, @@ -79,7 +80,7 @@ impl Web3ConnectionConfig { (None, None) => None, (Some(hard_limit), Some(redis_client_pool)) => Some((hard_limit, redis_client_pool)), (None, Some(_)) => None, - (Some(hard_limit), None) => { + (Some(_hard_limit), None) => { return Err(anyhow::anyhow!( "no redis client pool! needed for hard limit" )) diff --git a/web3-proxy/src/connections.rs b/web3-proxy/src/connections.rs index 95494734..1fd19ebb 100644 --- a/web3-proxy/src/connections.rs +++ b/web3-proxy/src/connections.rs @@ -517,6 +517,8 @@ impl Web3Connections { } pub fn has_synced_rpcs(&self) -> bool { + // TODO: require a minimum number of synced rpcs + // TODO: move this whole function to SyncedConnections if self.synced_connections.load().conns.is_empty() { return false; } diff --git a/web3-proxy/src/main.rs b/web3-proxy/src/main.rs index f07ad56b..c312d168 100644 --- a/web3-proxy/src/main.rs +++ b/web3-proxy/src/main.rs @@ -174,6 +174,7 @@ mod tests { tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .compact() + .with_test_writer() .init(); let anvil = Anvil::new().spawn(); @@ -200,6 +201,7 @@ mod tests { let app_config = AppConfig { shared: RpcSharedConfig { chain_id: 31337, + db_url: None, redis_url: None, public_rate_limit_per_minute: 0, response_cache_max_bytes: 10_usize.pow(7),