basic database

This commit is contained in:
Bryan Stitt 2022-08-03 00:27:26 +00:00
parent 6067bb1661
commit 4b193a7112
7 changed files with 272 additions and 60 deletions

99
Cargo.lock generated

@ -556,7 +556,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
dependencies = [
"block-padding",
"block-padding 0.1.5",
"byte-tools",
"byteorder",
"generic-array 0.12.4",
@ -568,6 +568,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
dependencies = [
"block-padding 0.2.1",
"generic-array 0.14.5",
]
@ -589,6 +590,12 @@ dependencies = [
"byte-tools",
]
[[package]]
name = "block-padding"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
[[package]]
name = "blocking"
version = "1.2.0"
@ -874,7 +881,7 @@ dependencies = [
"serde",
"serde_derive",
"sha2 0.10.2",
"sha3",
"sha3 0.10.1",
"thiserror",
]
@ -1397,7 +1404,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.2",
"sha3",
"sha3 0.10.1",
"thiserror",
"uuid 0.8.2",
]
@ -1414,7 +1421,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
"sha3",
"sha3 0.10.1",
"thiserror",
"uint",
]
@ -1448,9 +1455,9 @@ dependencies = [
[[package]]
name = "ethers"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "198a76d7ff7be414df68692a8f5bb86ec2698aa58bab5092ab89e37334064442"
checksum = "16142eeb3155cfa5aec6be3f828a28513a28bd995534f945fa70e7d608f16c10"
dependencies = [
"ethers-addressbook",
"ethers-contract",
@ -1464,9 +1471,9 @@ dependencies = [
[[package]]
name = "ethers-addressbook"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae7e8db5a961eacf62c5ca68a3ddbee066f91717b826e2da0ed5bfed5b92d795"
checksum = "e23f8992ecf45ea9dd2983696aabc566c108723585f07f5dc8c9efb24e52d3db"
dependencies = [
"ethers-core",
"once_cell",
@ -1476,9 +1483,9 @@ dependencies = [
[[package]]
name = "ethers-contract"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42af664b7ec2154ea6d8d393fb65e14e9e857039d0d96905acef196689063ffe"
checksum = "2e0010fffc97c5abcf75a30fd75676b1ed917b2b82beb8270391333618e2847d"
dependencies = [
"ethers-contract-abigen",
"ethers-core",
@ -1494,9 +1501,9 @@ dependencies = [
[[package]]
name = "ethers-contract-abigen"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57ecf7d921039dfe65b1148e9b3687d2a877b0413c33bee3a432e5e2e5c65f86"
checksum = "bda76ce804d524f693a898dc5857d08f4db443f3da64d0c36237fa05c0ecef30"
dependencies = [
"Inflector",
"cfg-if 1.0.0",
@ -1517,9 +1524,9 @@ dependencies = [
[[package]]
name = "ethers-core"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a95c6dbf3cf0cb6e595f9a250a8c4b3a14740bd374e93f9176e3bd5318620d"
checksum = "0ebdd63c828f58aa067f40f9adcbea5e114fb1f90144b3a1e2858e0c9b1ff4e8"
dependencies = [
"arrayvec",
"bytes",
@ -1549,9 +1556,9 @@ dependencies = [
[[package]]
name = "ethers-etherscan"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04ec38aae5f36aa433a9c532ec458f85141beff52965a2ab09fc3f77a47e091b"
checksum = "b279a3d00bd219caa2f9a34451b4accbfa9a1eaafc26dcda9d572591528435f0"
dependencies = [
"ethers-core",
"getrandom",
@ -1566,9 +1573,9 @@ dependencies = [
[[package]]
name = "ethers-middleware"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27dc8ef6f63a3fe5021e8ca7a063fff5cf1faf954cf4296998ac78dad0bd5a30"
checksum = "b1e7e8632d28175352b9454bbcb604643b6ca1de4d36dc99b3f86860d75c132b"
dependencies = [
"async-trait",
"ethers-contract",
@ -1591,9 +1598,9 @@ dependencies = [
[[package]]
name = "ethers-providers"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66c690ae7f12a5a53eb91a7de40564248b8cdb6141c1e4f860d12ed23be1250"
checksum = "e46482e4d1e79b20c338fd9db9e166184eb387f0a4e7c05c5b5c0aa2e8c8900c"
dependencies = [
"async-trait",
"auto_impl",
@ -1628,9 +1635,9 @@ dependencies = [
[[package]]
name = "ethers-signers"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56536bcb4979e06a026a5edfd6bf8167368e23401e4c8a8a56345adb5ad91bd2"
checksum = "73a72ecad124e8ccd18d6a43624208cab0199e59621b1f0fa6b776b2e0529107"
dependencies = [
"async-trait",
"coins-bip32",
@ -1646,9 +1653,9 @@ dependencies = [
[[package]]
name = "ethers-solc"
version = "0.15.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86afd894e3078fbb57c24ac2ab8ddb0feeaff867da310c2f1092650a306ef4d4"
checksum = "ebe5db405d0e584aa8dae154ffebb90f2305cae588fd11d9f6b857ebe3a79294"
dependencies = [
"cfg-if 1.0.0",
"colored",
@ -2401,6 +2408,15 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b"
[[package]]
name = "iri-string"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f0f7638c1e223529f1bfdc48c8b133b9e0b434094d1d28473161ee48b235f78"
dependencies = [
"nom",
]
[[package]]
name = "itertools"
version = "0.10.3"
@ -2450,7 +2466,7 @@ dependencies = [
"ecdsa",
"elliptic-curve",
"sha2 0.10.2",
"sha3",
"sha3 0.10.1",
]
[[package]]
@ -3383,9 +3399,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]]
name = "proc-macro2"
version = "1.0.41"
version = "1.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdcc2916cde080c1876ff40292a396541241fe0072ef928cd76582e9ea5d60d2"
checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b"
dependencies = [
"unicode-ident",
]
@ -4099,6 +4115,18 @@ dependencies = [
"digest 0.10.3",
]
[[package]]
name = "sha3"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809"
dependencies = [
"block-buffer 0.9.0",
"digest 0.9.0",
"keccak",
"opaque-debug 0.3.0",
]
[[package]]
name = "sha3"
version = "0.10.1"
@ -4143,6 +4171,22 @@ version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
[[package]]
name = "siwe"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a35b706108fb328661325c3b6882f6b1d62da47d533da0b87fca36ac769877db"
dependencies = [
"hex",
"http",
"iri-string",
"k256",
"rand",
"sha3 0.9.1",
"thiserror",
"time 0.3.11",
]
[[package]]
name = "slab"
version = "0.4.7"
@ -5157,6 +5201,7 @@ dependencies = [
"sea-orm",
"serde",
"serde_json",
"siwe",
"tokio",
"tokio-stream",
"toml",

@ -9,13 +9,13 @@ name = "migration"
path = "src/lib.rs"
[dependencies]
async-std = { version = "^1", features = ["attributes", "tokio1"] }
async-std = { version = "1.12.0", features = ["attributes", "tokio1"] }
[dependencies.sea-orm-migration]
version = "^0.9.0"
version = "0.9.1"
features = [
# Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI.
# View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime.
"runtime-tokio-rustls", # `ASYNC_RUNTIME` featrure
"sqlx-mysql", # `DATABASE_DRIVER` feature
]
]

@ -1,6 +1,6 @@
CREATE TABLE users (
id SERIAL PRIMARY KEY,
chain INT,
primary_chain INT,
primary_address VARCHAR(42),
description VARCHAR(255),
email VARCHAR(320),
@ -14,7 +14,7 @@ CREATE TABLE secondary_users (
id SERIAL PRIMARY KEY,
users_id BIGINT,
secondary_address VARCHAR(42),
chain INT,
secondary_chain INT,
description VARCHAR,
email VARCHAR(320),
)

@ -6,43 +6,209 @@ pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
todo!();
manager
.create_table(
Table::create()
.table(Post::Table)
.table(User::Table)
.if_not_exists()
.col(
ColumnDef::new(Post::Id)
.integer()
ColumnDef::new(User::Id)
.big_integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(ColumnDef::new(Post::Title).string().not_null())
.col(ColumnDef::new(Post::Text).string().not_null())
.col(
ColumnDef::new(User::Address)
.string_len(42)
.not_null()
.unique_key(),
)
.col(ColumnDef::new(User::Description).string().not_null())
.col(ColumnDef::new(User::Email).string().not_null())
.to_owned(),
)
.await
.await?;
manager
.create_table(
Table::create()
.table(SecondaryUser::Table)
.if_not_exists()
.col(
ColumnDef::new(SecondaryUser::Id)
.big_integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(
ColumnDef::new(SecondaryUser::UserId)
.big_integer()
.not_null(),
)
.col(ColumnDef::new(SecondaryUser::Address).string().not_null())
.col(
ColumnDef::new(SecondaryUser::Description)
.string()
.not_null(),
)
.col(ColumnDef::new(SecondaryUser::Email).string().not_null())
.col(
ColumnDef::new(SecondaryUser::Role)
.enumeration("role", ["owner", "admin", "collaborator"])
.not_null(),
)
.to_owned(),
)
.await?;
// TODO: make sure from and to aren't backwards
manager
.create_foreign_key(
sea_query::ForeignKey::create()
.from(SecondaryUser::Table, SecondaryUser::UserId)
.to(User::Table, User::Id)
.to_owned(),
)
.await?;
manager
.create_table(
Table::create()
.table(BlockList::Table)
.if_not_exists()
.col(
ColumnDef::new(BlockList::Id)
.big_integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(ColumnDef::new(BlockList::Address).string().not_null())
.col(ColumnDef::new(BlockList::Chain).integer().not_null())
.col(ColumnDef::new(BlockList::Description).string().not_null())
.to_owned(),
)
.await?;
manager
.create_table(
Table::create()
.table(UserKeys::Table)
.if_not_exists()
.col(
ColumnDef::new(UserKeys::Id)
.big_integer()
.not_null()
.auto_increment()
.primary_key(),
)
.col(ColumnDef::new(UserKeys::UserId).big_integer().not_null())
.col(ColumnDef::new(UserKeys::ApiKey).string_len(32).not_null())
.col(ColumnDef::new(UserKeys::Description).string().not_null())
.col(
ColumnDef::new(UserKeys::PrivateTxs)
.boolean()
.default(true)
.not_null(),
)
.col(
ColumnDef::new(UserKeys::Active)
.boolean()
.default(true)
.not_null(),
)
.to_owned(),
)
.await?;
// TODO: make sure from and to aren't backwards
manager
.create_foreign_key(
sea_query::ForeignKey::create()
.from(UserKeys::Table, UserKeys::UserId)
.to(User::Table, User::Id)
.to_owned(),
)
.await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
todo!();
manager
.drop_table(Table::drop().table(Post::Table).to_owned())
.await
.drop_table(Table::drop().table(User::Table).to_owned())
.await?;
manager
.drop_table(Table::drop().table(SecondaryUser::Table).to_owned())
.await?;
manager
.drop_table(Table::drop().table(BlockList::Table).to_owned())
.await?;
manager
.drop_table(Table::drop().table(UserKeys::Table).to_owned())
.await?;
Ok(())
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum Post {
enum User {
Table,
Id,
Title,
Text,
Address,
Description,
Email,
}
/*
-- TODO: foreign keys
-- TODO: how should we store addresses?
-- TODO: creation time?
-- TODO: permissions. likely similar to infura
// TODO: creation time?
*/
#[derive(Iden)]
enum SecondaryUser {
Table,
Id,
UserId,
Address,
Description,
Email,
Role,
}
// TODO: creation time?
#[derive(Iden)]
enum BlockList {
Table,
Id,
Address,
Chain,
Description,
}
/*
-- TODO: foreign keys
-- TODO: index on api_key
-- TODO: what size for api_key
-- TODO: track active with a timestamp?
-- TODO: creation time?
-- TODO: requests_per_second INT,
-- TODO: requests_per_day INT,
-- TODO: more security features. likely similar to infura
*/
#[derive(Iden)]
enum UserKeys {
Table,
Id,
UserId,
ApiKey,
Description,
PrivateTxs,
Active,
}

@ -19,13 +19,13 @@ migration = { path = "../migration" }
anyhow = { version = "1.0.58", features = ["backtrace"] }
arc-swap = "1.5.0"
argh = "0.1.8"
axum = { version = "0.5.14", features = ["serde_json", "tokio-tungstenite", "ws"] }
axum = { version = "0.5.13", features = ["serde_json", "tokio-tungstenite", "ws"] }
axum-client-ip = "0.2.0"
counter = "0.5.6"
dashmap = "5.3.4"
derive_more = "0.99.17"
dotenv = "0.15.0"
ethers = { version = "0.15.0", features = ["rustls", "ws"] }
ethers = { version = "0.17.0", features = ["rustls", "ws"] }
fdlimit = "0.2.1"
flume = "0.10.14"
futures = { version = "0.3.21", features = ["thread-pool"] }
@ -41,12 +41,12 @@ proctitle = "0.1.1"
regex = "1.6.0"
reqwest = { version = "0.11.11", default-features = false, features = ["json", "tokio-rustls"] }
rustc-hash = "1.1.0"
# siwe = "0.4.0" # blocked by https://github.com/spruceid/siwe-rs/issues/36
siwe = "0.4.1"
sea-orm = { version = "0.9.1", features = ["macros"] }
serde = { version = "1.0.140", features = [] }
serde_json = { version = "1.0.82", default-features = false, features = ["alloc", "raw_value"] }
tokio = { version = "1.20.1", features = ["full", "tracing"] }
async-std = { version = "^1", features = ["attributes", "tokio1"] }
async-std = { version = "1.12.0", features = ["attributes", "tokio1"] }
toml = "0.5.9"
tracing = "0.1.35"
# TODO: tracing-subscriber has serde and serde_json features that we might want to use

@ -10,6 +10,7 @@ use futures::stream::FuturesUnordered;
use futures::stream::StreamExt;
use futures::Future;
use linkedhashmap::LinkedHashMap;
use migration::{Migrator, MigratorTrait};
use parking_lot::RwLock;
use redis_cell_client::bb8::ErrorSink;
use redis_cell_client::{bb8, RedisCellClient, RedisConnectionManager};
@ -289,11 +290,7 @@ impl Web3ProxyApp {
Arc<Web3ProxyApp>,
Pin<Box<dyn Future<Output = anyhow::Result<()>>>>,
)> {
// // first, we connect to mysql and make sure the latest migrations have run
// let db_pool = todo!(app_config.db_url).await;
// let connection = db_pool.get().await;
// embedded_migrations::run_with_output(&connection, &mut std::io::stdout());
// first, we connect to mysql and make sure the latest migrations have run
let db_conn = if let Some(db_url) = app_config.shared.db_url {
let mut db_opt = sea_orm::ConnectOptions::new(db_url);
@ -309,6 +306,9 @@ impl Web3ProxyApp {
let db_conn = sea_orm::Database::connect(db_opt).await?;
// TODO: if error, roll back
Migrator::up(&db_conn, None).await?;
Some(db_conn)
} else {
info!("no database");

@ -216,18 +216,19 @@ impl Web3Connection {
// TODO: don't sleep. wait for new heads subscription instead
// TODO: i think instead of atomics, we could maybe use a watch channel
sleep(Duration::from_millis(100)).await;
sleep(Duration::from_millis(200)).await;
// we could take "archive" as a parameter, but we would want a safety check on it regardless
// check common archive thresholds
// TODO: would be great if rpcs exposed this
// TODO: move this to a helper function so we can recheck on errors or as the chain grows
// TODO: move this to a helper function that checks
for block_data_limit in [u64::MAX, 90_000, 128, 64, 32] {
let mut head_block_num = new_connection.head_block.read().1;
// TODO: wait until head block is set outside the loop? if we disconnect while starting we could actually get 0 though
while head_block_num == U64::zero() {
info!(?new_connection, "no head block");
warn!(?new_connection, "no head block");
// TODO: subscribe to a channel instead of polling? subscribe to http_interval_sender?
sleep(Duration::from_secs(1)).await;