From 4b193a7112bfcd5696b6d58c970a41ebe30175a5 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Wed, 3 Aug 2022 00:27:26 +0000 Subject: [PATCH] basic database --- Cargo.lock | 99 ++++++--- migration/Cargo.toml | 6 +- migration/first_draft.sql | 4 +- .../src/m20220101_000001_create_table.rs | 200 ++++++++++++++++-- web3-proxy/Cargo.toml | 8 +- web3-proxy/src/app.rs | 10 +- web3-proxy/src/connection.rs | 5 +- 7 files changed, 272 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06d83912..78c92ec3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -556,7 +556,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ - "block-padding", + "block-padding 0.1.5", "byte-tools", "byteorder", "generic-array 0.12.4", @@ -568,6 +568,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ + "block-padding 0.2.1", "generic-array 0.14.5", ] @@ -589,6 +590,12 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + [[package]] name = "blocking" version = "1.2.0" @@ -874,7 +881,7 @@ dependencies = [ "serde", "serde_derive", "sha2 0.10.2", - "sha3", + "sha3 0.10.1", "thiserror", ] @@ -1397,7 +1404,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.2", - "sha3", + "sha3 0.10.1", "thiserror", "uuid 0.8.2", ] @@ -1414,7 +1421,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sha3", + "sha3 0.10.1", "thiserror", "uint", ] @@ -1448,9 +1455,9 @@ dependencies = [ [[package]] name = "ethers" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198a76d7ff7be414df68692a8f5bb86ec2698aa58bab5092ab89e37334064442" +checksum = "16142eeb3155cfa5aec6be3f828a28513a28bd995534f945fa70e7d608f16c10" dependencies = [ "ethers-addressbook", "ethers-contract", @@ -1464,9 +1471,9 @@ dependencies = [ [[package]] name = "ethers-addressbook" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae7e8db5a961eacf62c5ca68a3ddbee066f91717b826e2da0ed5bfed5b92d795" +checksum = "e23f8992ecf45ea9dd2983696aabc566c108723585f07f5dc8c9efb24e52d3db" dependencies = [ "ethers-core", "once_cell", @@ -1476,9 +1483,9 @@ dependencies = [ [[package]] name = "ethers-contract" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42af664b7ec2154ea6d8d393fb65e14e9e857039d0d96905acef196689063ffe" +checksum = "2e0010fffc97c5abcf75a30fd75676b1ed917b2b82beb8270391333618e2847d" dependencies = [ "ethers-contract-abigen", "ethers-core", @@ -1494,9 +1501,9 @@ dependencies = [ [[package]] name = "ethers-contract-abigen" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ecf7d921039dfe65b1148e9b3687d2a877b0413c33bee3a432e5e2e5c65f86" +checksum = "bda76ce804d524f693a898dc5857d08f4db443f3da64d0c36237fa05c0ecef30" dependencies = [ "Inflector", "cfg-if 1.0.0", @@ -1517,9 +1524,9 @@ dependencies = [ [[package]] name = "ethers-core" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a95c6dbf3cf0cb6e595f9a250a8c4b3a14740bd374e93f9176e3bd5318620d" +checksum = "0ebdd63c828f58aa067f40f9adcbea5e114fb1f90144b3a1e2858e0c9b1ff4e8" dependencies = [ "arrayvec", "bytes", @@ -1549,9 +1556,9 @@ dependencies = [ [[package]] name = "ethers-etherscan" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04ec38aae5f36aa433a9c532ec458f85141beff52965a2ab09fc3f77a47e091b" +checksum = "b279a3d00bd219caa2f9a34451b4accbfa9a1eaafc26dcda9d572591528435f0" dependencies = [ "ethers-core", "getrandom", @@ -1566,9 +1573,9 @@ dependencies = [ [[package]] name = "ethers-middleware" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27dc8ef6f63a3fe5021e8ca7a063fff5cf1faf954cf4296998ac78dad0bd5a30" +checksum = "b1e7e8632d28175352b9454bbcb604643b6ca1de4d36dc99b3f86860d75c132b" dependencies = [ "async-trait", "ethers-contract", @@ -1591,9 +1598,9 @@ dependencies = [ [[package]] name = "ethers-providers" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66c690ae7f12a5a53eb91a7de40564248b8cdb6141c1e4f860d12ed23be1250" +checksum = "e46482e4d1e79b20c338fd9db9e166184eb387f0a4e7c05c5b5c0aa2e8c8900c" dependencies = [ "async-trait", "auto_impl", @@ -1628,9 +1635,9 @@ dependencies = [ [[package]] name = "ethers-signers" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56536bcb4979e06a026a5edfd6bf8167368e23401e4c8a8a56345adb5ad91bd2" +checksum = "73a72ecad124e8ccd18d6a43624208cab0199e59621b1f0fa6b776b2e0529107" dependencies = [ "async-trait", "coins-bip32", @@ -1646,9 +1653,9 @@ dependencies = [ [[package]] name = "ethers-solc" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86afd894e3078fbb57c24ac2ab8ddb0feeaff867da310c2f1092650a306ef4d4" +checksum = "ebe5db405d0e584aa8dae154ffebb90f2305cae588fd11d9f6b857ebe3a79294" dependencies = [ "cfg-if 1.0.0", "colored", @@ -2401,6 +2408,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +[[package]] +name = "iri-string" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0f7638c1e223529f1bfdc48c8b133b9e0b434094d1d28473161ee48b235f78" +dependencies = [ + "nom", +] + [[package]] name = "itertools" version = "0.10.3" @@ -2450,7 +2466,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "sha2 0.10.2", - "sha3", + "sha3 0.10.1", ] [[package]] @@ -3383,9 +3399,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc2916cde080c1876ff40292a396541241fe0072ef928cd76582e9ea5d60d2" +checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" dependencies = [ "unicode-ident", ] @@ -4099,6 +4115,18 @@ dependencies = [ "digest 0.10.3", ] +[[package]] +name = "sha3" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "keccak", + "opaque-debug 0.3.0", +] + [[package]] name = "sha3" version = "0.10.1" @@ -4143,6 +4171,22 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "siwe" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a35b706108fb328661325c3b6882f6b1d62da47d533da0b87fca36ac769877db" +dependencies = [ + "hex", + "http", + "iri-string", + "k256", + "rand", + "sha3 0.9.1", + "thiserror", + "time 0.3.11", +] + [[package]] name = "slab" version = "0.4.7" @@ -5157,6 +5201,7 @@ dependencies = [ "sea-orm", "serde", "serde_json", + "siwe", "tokio", "tokio-stream", "toml", diff --git a/migration/Cargo.toml b/migration/Cargo.toml index fca5222e..77c0cd23 100644 --- a/migration/Cargo.toml +++ b/migration/Cargo.toml @@ -9,13 +9,13 @@ name = "migration" path = "src/lib.rs" [dependencies] -async-std = { version = "^1", features = ["attributes", "tokio1"] } +async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } [dependencies.sea-orm-migration] -version = "^0.9.0" +version = "0.9.1" features = [ # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. "runtime-tokio-rustls", # `ASYNC_RUNTIME` featrure "sqlx-mysql", # `DATABASE_DRIVER` feature -] \ No newline at end of file +] diff --git a/migration/first_draft.sql b/migration/first_draft.sql index 97a1219b..f8cec822 100644 --- a/migration/first_draft.sql +++ b/migration/first_draft.sql @@ -1,6 +1,6 @@ CREATE TABLE users ( id SERIAL PRIMARY KEY, - chain INT, + primary_chain INT, primary_address VARCHAR(42), description VARCHAR(255), email VARCHAR(320), @@ -14,7 +14,7 @@ CREATE TABLE secondary_users ( id SERIAL PRIMARY KEY, users_id BIGINT, secondary_address VARCHAR(42), - chain INT, + secondary_chain INT, description VARCHAR, email VARCHAR(320), ) diff --git a/migration/src/m20220101_000001_create_table.rs b/migration/src/m20220101_000001_create_table.rs index b0582446..0095ebe9 100644 --- a/migration/src/m20220101_000001_create_table.rs +++ b/migration/src/m20220101_000001_create_table.rs @@ -6,43 +6,209 @@ pub struct Migration; #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Replace the sample below with your own migration scripts - todo!(); - manager .create_table( Table::create() - .table(Post::Table) + .table(User::Table) .if_not_exists() .col( - ColumnDef::new(Post::Id) - .integer() + ColumnDef::new(User::Id) + .big_integer() .not_null() .auto_increment() .primary_key(), ) - .col(ColumnDef::new(Post::Title).string().not_null()) - .col(ColumnDef::new(Post::Text).string().not_null()) + .col( + ColumnDef::new(User::Address) + .string_len(42) + .not_null() + .unique_key(), + ) + .col(ColumnDef::new(User::Description).string().not_null()) + .col(ColumnDef::new(User::Email).string().not_null()) .to_owned(), ) - .await + .await?; + + manager + .create_table( + Table::create() + .table(SecondaryUser::Table) + .if_not_exists() + .col( + ColumnDef::new(SecondaryUser::Id) + .big_integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col( + ColumnDef::new(SecondaryUser::UserId) + .big_integer() + .not_null(), + ) + .col(ColumnDef::new(SecondaryUser::Address).string().not_null()) + .col( + ColumnDef::new(SecondaryUser::Description) + .string() + .not_null(), + ) + .col(ColumnDef::new(SecondaryUser::Email).string().not_null()) + .col( + ColumnDef::new(SecondaryUser::Role) + .enumeration("role", ["owner", "admin", "collaborator"]) + .not_null(), + ) + .to_owned(), + ) + .await?; + + // TODO: make sure from and to aren't backwards + manager + .create_foreign_key( + sea_query::ForeignKey::create() + .from(SecondaryUser::Table, SecondaryUser::UserId) + .to(User::Table, User::Id) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(BlockList::Table) + .if_not_exists() + .col( + ColumnDef::new(BlockList::Id) + .big_integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(BlockList::Address).string().not_null()) + .col(ColumnDef::new(BlockList::Chain).integer().not_null()) + .col(ColumnDef::new(BlockList::Description).string().not_null()) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(UserKeys::Table) + .if_not_exists() + .col( + ColumnDef::new(UserKeys::Id) + .big_integer() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(UserKeys::UserId).big_integer().not_null()) + .col(ColumnDef::new(UserKeys::ApiKey).string_len(32).not_null()) + .col(ColumnDef::new(UserKeys::Description).string().not_null()) + .col( + ColumnDef::new(UserKeys::PrivateTxs) + .boolean() + .default(true) + .not_null(), + ) + .col( + ColumnDef::new(UserKeys::Active) + .boolean() + .default(true) + .not_null(), + ) + .to_owned(), + ) + .await?; + + // TODO: make sure from and to aren't backwards + manager + .create_foreign_key( + sea_query::ForeignKey::create() + .from(UserKeys::Table, UserKeys::UserId) + .to(User::Table, User::Id) + .to_owned(), + ) + .await?; + + Ok(()) } async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Replace the sample below with your own migration scripts - todo!(); - manager - .drop_table(Table::drop().table(Post::Table).to_owned()) - .await + .drop_table(Table::drop().table(User::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(SecondaryUser::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(BlockList::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(UserKeys::Table).to_owned()) + .await?; + + Ok(()) } } /// Learn more at https://docs.rs/sea-query#iden #[derive(Iden)] -enum Post { +enum User { Table, Id, - Title, - Text, + Address, + Description, + Email, +} + +/* +-- TODO: foreign keys +-- TODO: how should we store addresses? +-- TODO: creation time? +-- TODO: permissions. likely similar to infura +// TODO: creation time? +*/ +#[derive(Iden)] +enum SecondaryUser { + Table, + Id, + UserId, + Address, + Description, + Email, + Role, +} + +// TODO: creation time? +#[derive(Iden)] +enum BlockList { + Table, + Id, + Address, + Chain, + Description, +} + +/* +-- TODO: foreign keys +-- TODO: index on api_key +-- TODO: what size for api_key +-- TODO: track active with a timestamp? +-- TODO: creation time? +-- TODO: requests_per_second INT, +-- TODO: requests_per_day INT, +-- TODO: more security features. likely similar to infura +*/ +#[derive(Iden)] +enum UserKeys { + Table, + Id, + UserId, + ApiKey, + Description, + PrivateTxs, + Active, } diff --git a/web3-proxy/Cargo.toml b/web3-proxy/Cargo.toml index b77f1752..beac6253 100644 --- a/web3-proxy/Cargo.toml +++ b/web3-proxy/Cargo.toml @@ -19,13 +19,13 @@ migration = { path = "../migration" } anyhow = { version = "1.0.58", features = ["backtrace"] } arc-swap = "1.5.0" argh = "0.1.8" -axum = { version = "0.5.14", features = ["serde_json", "tokio-tungstenite", "ws"] } +axum = { version = "0.5.13", features = ["serde_json", "tokio-tungstenite", "ws"] } axum-client-ip = "0.2.0" counter = "0.5.6" dashmap = "5.3.4" derive_more = "0.99.17" dotenv = "0.15.0" -ethers = { version = "0.15.0", features = ["rustls", "ws"] } +ethers = { version = "0.17.0", features = ["rustls", "ws"] } fdlimit = "0.2.1" flume = "0.10.14" futures = { version = "0.3.21", features = ["thread-pool"] } @@ -41,12 +41,12 @@ proctitle = "0.1.1" regex = "1.6.0" reqwest = { version = "0.11.11", default-features = false, features = ["json", "tokio-rustls"] } rustc-hash = "1.1.0" -# siwe = "0.4.0" # blocked by https://github.com/spruceid/siwe-rs/issues/36 +siwe = "0.4.1" sea-orm = { version = "0.9.1", features = ["macros"] } serde = { version = "1.0.140", features = [] } serde_json = { version = "1.0.82", default-features = false, features = ["alloc", "raw_value"] } tokio = { version = "1.20.1", features = ["full", "tracing"] } -async-std = { version = "^1", features = ["attributes", "tokio1"] } +async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } toml = "0.5.9" tracing = "0.1.35" # TODO: tracing-subscriber has serde and serde_json features that we might want to use diff --git a/web3-proxy/src/app.rs b/web3-proxy/src/app.rs index 70b5d5b1..7b09cff3 100644 --- a/web3-proxy/src/app.rs +++ b/web3-proxy/src/app.rs @@ -10,6 +10,7 @@ use futures::stream::FuturesUnordered; use futures::stream::StreamExt; use futures::Future; use linkedhashmap::LinkedHashMap; +use migration::{Migrator, MigratorTrait}; use parking_lot::RwLock; use redis_cell_client::bb8::ErrorSink; use redis_cell_client::{bb8, RedisCellClient, RedisConnectionManager}; @@ -289,11 +290,7 @@ impl Web3ProxyApp { Arc, Pin>>>, )> { - // // first, we connect to mysql and make sure the latest migrations have run - // let db_pool = todo!(app_config.db_url).await; - // let connection = db_pool.get().await; - // embedded_migrations::run_with_output(&connection, &mut std::io::stdout()); - + // first, we connect to mysql and make sure the latest migrations have run let db_conn = if let Some(db_url) = app_config.shared.db_url { let mut db_opt = sea_orm::ConnectOptions::new(db_url); @@ -309,6 +306,9 @@ impl Web3ProxyApp { let db_conn = sea_orm::Database::connect(db_opt).await?; + // TODO: if error, roll back + Migrator::up(&db_conn, None).await?; + Some(db_conn) } else { info!("no database"); diff --git a/web3-proxy/src/connection.rs b/web3-proxy/src/connection.rs index 5fb4cea6..9d5142f0 100644 --- a/web3-proxy/src/connection.rs +++ b/web3-proxy/src/connection.rs @@ -216,18 +216,19 @@ impl Web3Connection { // TODO: don't sleep. wait for new heads subscription instead // TODO: i think instead of atomics, we could maybe use a watch channel - sleep(Duration::from_millis(100)).await; + sleep(Duration::from_millis(200)).await; // we could take "archive" as a parameter, but we would want a safety check on it regardless // check common archive thresholds // TODO: would be great if rpcs exposed this // TODO: move this to a helper function so we can recheck on errors or as the chain grows + // TODO: move this to a helper function that checks for block_data_limit in [u64::MAX, 90_000, 128, 64, 32] { let mut head_block_num = new_connection.head_block.read().1; // TODO: wait until head block is set outside the loop? if we disconnect while starting we could actually get 0 though while head_block_num == U64::zero() { - info!(?new_connection, "no head block"); + warn!(?new_connection, "no head block"); // TODO: subscribe to a channel instead of polling? subscribe to http_interval_sender? sleep(Duration::from_secs(1)).await;