tornadocontrib/ web3-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

split mysql and anvil into their own fixtures

This commit is contained in:
Bryan Stitt 2023-07-12 14:23:39 -07:00
parent 5d207fb2c6
commit 6b5437be2c
8 changed files with 300 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
web3_proxy/tests/common/anvil.rs Normal file
View File

@ -0,0 +1,24 @@
// TODO: option to spawn in a dedicated thread?
// TODO: option to subscribe to another anvil and copy blocks
use ethers::utils::{Anvil, AnvilInstance};
use tracing::info;
/// on drop, the anvil instance will be shut down
pub struct TestAnvil {
pub instance: AnvilInstance,
}
impl TestAnvil {
pub async fn spawn(chain_id: u64) -> Self {
info!(?chain_id);
// TODO: configurable rpc and block
let instance = Anvil::new()
.chain_id(chain_id)
// .fork("https://polygon.llamarpc.com@44300000")
.spawn();
Self { instance }
}
}

214
web3_proxy/tests/common/app.rs
View File

@ -1,11 +1,8 @@
use super::{anvil::TestAnvil, mysql::TestMysql};
use ethers::{
prelude::{
rand::{self, distributions::Alphanumeric, Rng},
Http, Provider,
},
prelude::{Http, Provider},
signers::LocalWallet,
types::Address,
utils::{Anvil, AnvilInstance},
};
use hashbrown::HashMap;
use migration::sea_orm::DatabaseConnection;
@ -13,14 +10,11 @@ use parking_lot::Mutex;
use serde_json::json;
use std::{
env,
process::Command as SyncCommand,
str::FromStr,
sync::atomic::{AtomicU16, Ordering},
};
use std::{sync::Arc, time::Duration};
use tokio::{
net::TcpStream,
process::Command as AsyncCommand,
sync::{
broadcast::{self, error::SendError},
mpsc, oneshot,
@ -28,30 +22,22 @@ use tokio::{
task::JoinHandle,
time::{sleep, Instant},
};
use tracing::{info, trace, warn};
use tracing::info;
use web3_proxy::{
config::{AppConfig, TopConfig, Web3RpcConfig},
relational_db::get_migrated_db,
stats::FlushedStats,
sub_commands::ProxydSubCommand,
};
#[derive(Clone)]
pub struct DbData {
pub conn: Option<DatabaseConnection>,
pub container_name: String,
pub url: Option<String>,
}
pub struct TestApp {
/// anvil shuts down when this guard is dropped.
pub anvil: AnvilInstance,
pub anvil: TestAnvil,
/// connection to anvil.
pub anvil_provider: Provider<Http>,
/// keep track of the database so it can be stopped on drop
pub db: Option<DbData>,
pub db: Option<TestMysql>,
/// spawn handle for the proxy.
pub proxy_handle: Mutex<Option<JoinHandle<anyhow::Result<()>>>>,
@ -67,9 +53,8 @@ pub struct TestApp {
}
impl TestApp {
pub async fn spawn(chain_id: u64, setup_db: bool) -> Self {
info!(?chain_id);
pub async fn spawn(anvil: TestAnvil, db: Option<TestMysql>) -> Self {
let chain_id = anvil.instance.chain_id();
let num_workers = 2;
// TODO: move basic setup into a test fixture
@ -77,172 +62,9 @@ impl TestApp {
info!(%path);
// TODO: configurable rpc and block
let anvil = Anvil::new()
.chain_id(chain_id)
// .fork("https://polygon.llamarpc.com@44300000")
.spawn();
let anvil_provider = Provider::<Http>::try_from(anvil.instance.endpoint()).unwrap();
info!("Anvil running at `{}`", anvil.endpoint());
let anvil_provider = Provider::<Http>::try_from(anvil.endpoint()).unwrap();
// TODO: instead of starting a db every time, use a connection pool and transactions to begin/rollback
let db = if setup_db {
// sqlite doesn't seem to work. our migrations are written for mysql
// so lets use docker to start mysql
let password: String = rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(16)
.map(char::from)
.collect();
let random: String = rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(8)
.map(char::from)
.collect();
let db_container_name = format!("web3-proxy-test-{}", random);
info!(%db_container_name);
// create the db_data as soon as the url is known
// when this is dropped, the db will be stopped
let mut db_data = DbData {
conn: None,
container_name: db_container_name.clone(),
url: None,
};
let _ = AsyncCommand::new("docker")
.args([
"run",
"--name",
&db_container_name,
"--rm",
"-d",
"-e",
&format!("MYSQL_ROOT_PASSWORD={}", password),
"-e",
"MYSQL_DATABASE=web3_proxy_test",
"-p",
"0:3306",
"mysql",
])
.output()
.await
.expect("failed to start db");
// give the db a second to start
// TODO: wait until docker says it is healthy
sleep(Duration::from_secs(1)).await;
// TODO: why is this always empty?!
let docker_inspect_output = AsyncCommand::new("docker")
.args(["inspect", &db_container_name])
.output()
.await
.unwrap();
let docker_inspect_json = String::from_utf8(docker_inspect_output.stdout).unwrap();
trace!(%docker_inspect_json);
let docker_inspect_json: serde_json::Value =
serde_json::from_str(&docker_inspect_json).unwrap();
let mysql_ports = docker_inspect_json
.get(0)
.unwrap()
.get("NetworkSettings")
.unwrap()
.get("Ports")
.unwrap()
.get("3306/tcp")
.unwrap()
.get(0)
.unwrap();
trace!(?mysql_ports);
let mysql_port: u64 = mysql_ports
.get("HostPort")
.expect("unable to determine mysql port")
.as_str()
.unwrap()
.parse()
.unwrap();
let mysql_ip = mysql_ports
.get("HostIp")
.and_then(|x| x.as_str())
.expect("unable to determine mysql ip");
// let mysql_ip = "localhost";
// let mysql_ip = "127.0.0.1";
let db_url = format!(
"mysql://root:{}@{}:{}/web3_proxy_test",
password, mysql_ip, mysql_port
);
info!(%db_url, "waiting for start");
db_data.url = Some(db_url.clone());
let start = Instant::now();
let max_wait = Duration::from_secs(30);
loop {
if start.elapsed() > max_wait {
panic!("db took too long to start");
}
if TcpStream::connect(format!("{}:{}", mysql_ip, mysql_port))
.await
.is_ok()
{
break;
};
// not open wait. sleep and then try again
sleep(Duration::from_secs(1)).await;
}
// TODO: make sure mysql is actually ready for connections
sleep(Duration::from_secs(1)).await;
info!(%db_url, elapsed=%start.elapsed().as_secs_f32(), "db post is open. Migrating now...");
// try to migrate
let start = Instant::now();
let max_wait = Duration::from_secs(30);
loop {
if start.elapsed() > max_wait {
panic!("db took too long to start");
}
match get_migrated_db(db_url.clone(), 1, 1).await {
Ok(x) => {
// it worked! yey!
db_data.conn = Some(x);
break;
}
Err(err) => {
// not connected. sleep and then try again
warn!(?err, "unable to migrate db. retrying in 1 second");
sleep(Duration::from_secs(1)).await;
}
}
}
info!(%db_url, elapsed=%start.elapsed().as_secs_f32(), "db is migrated");
Some(db_data)
} else {
None
};
let db_url = db.as_ref().and_then(|x| x.url.clone());
let db_url = db.as_ref().map(|x| x.url.clone());
// make a test TopConfig
// TODO: test influx
@ -267,8 +89,8 @@ impl TestApp {
balanced_rpcs: HashMap::from([(
"anvil".to_string(),
Web3RpcConfig {
http_url: Some(anvil.endpoint()),
ws_url: Some(anvil.ws_endpoint()),
http_url: Some(anvil.instance.endpoint()),
ws_url: Some(anvil.instance.ws_endpoint()),
..Default::default()
},
)]),
@ -328,7 +150,7 @@ impl TestApp {
#[allow(unused)]
pub fn db_conn(&self) -> &DatabaseConnection {
self.db.as_ref().unwrap().conn.as_ref().unwrap()
self.db.as_ref().unwrap().conn()
}
#[allow(unused)]
@ -361,7 +183,7 @@ impl TestApp {
#[allow(unused)]
pub fn wallet(&self, id: usize) -> LocalWallet {
self.anvil.keys()[id].clone().into()
self.anvil.instance.keys()[id].clone().into()
}
}
@ -372,13 +194,3 @@ impl Drop for TestApp {
// TODO: do we care about waiting for it to stop? it will slow our tests down so we probably only care about waiting in some tests
}
}
impl Drop for DbData {
fn drop(&mut self) {
info!(%self.container_name, "killing db");
let _ = SyncCommand::new("docker")
.args(["kill", "-s", "9", &self.container_name])
.output();
}
}

2
web3_proxy/tests/common/mod.rs
View File

@ -1,8 +1,10 @@
pub mod admin_deposits;
pub mod admin_increases_balance;
pub mod anvil;
pub mod app;
pub mod create_admin;
pub mod create_user;
pub mod mysql;
pub mod referral;
pub mod rpc_key;
pub mod user_balance;

184
web3_proxy/tests/common/mysql.rs Normal file
View File

@ -0,0 +1,184 @@
use ethers::prelude::rand::{self, distributions::Alphanumeric, Rng};
use migration::sea_orm::DatabaseConnection;
use std::process::Command as SyncCommand;
use std::time::Duration;
use tokio::{
net::TcpStream,
process::Command as AsyncCommand,
time::{sleep, Instant},
};
use tracing::{info, trace, warn};
use web3_proxy::relational_db::get_migrated_db;
/// on drop, the mysql docker container will be shut down
pub struct TestMysql {
pub url: Option<String>,
pub conn: Option<DatabaseConnection>,
pub container_name: String,
}
impl TestMysql {
pub async fn spawn() -> Self {
// sqlite doesn't seem to work. our migrations are written for mysql
// so lets use docker to start mysql
let password: String = rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(16)
.map(char::from)
.collect();
let random: String = rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(8)
.map(char::from)
.collect();
let db_container_name = format!("web3-proxy-test-{}", random);
info!(%db_container_name);
// create the db_data as soon as the url is known
// when this is dropped, the db will be stopped
let mut test_mysql = Self {
conn: None,
container_name: db_container_name.clone(),
url: None,
};
let _ = AsyncCommand::new("docker")
.args([
"run",
"--name",
&db_container_name,
"--rm",
"-d",
"-e",
&format!("MYSQL_ROOT_PASSWORD={}", password),
"-e",
"MYSQL_DATABASE=web3_proxy_test",
"-p",
"0:3306",
"mysql",
])
.output()
.await
.expect("failed to start db");
// give the db a second to start
// TODO: wait until docker says it is healthy
sleep(Duration::from_secs(1)).await;
// TODO: why is this always empty?!
let docker_inspect_output = AsyncCommand::new("docker")
.args(["inspect", &db_container_name])
.output()
.await
.unwrap();
let docker_inspect_json = String::from_utf8(docker_inspect_output.stdout).unwrap();
trace!(%docker_inspect_json);
let docker_inspect_json: serde_json::Value =
serde_json::from_str(&docker_inspect_json).unwrap();
let mysql_ports = docker_inspect_json
.get(0)
.unwrap()
.get("NetworkSettings")
.unwrap()
.get("Ports")
.unwrap()
.get("3306/tcp")
.unwrap()
.get(0)
.unwrap();
trace!(?mysql_ports);
let mysql_port: u64 = mysql_ports
.get("HostPort")
.expect("unable to determine mysql port")
.as_str()
.unwrap()
.parse()
.unwrap();
let mysql_ip = mysql_ports
.get("HostIp")
.and_then(|x| x.as_str())
.expect("unable to determine mysql ip");
let db_url = format!(
"mysql://root:{}@{}:{}/web3_proxy_test",
password, mysql_ip, mysql_port
);
info!(%db_url, "waiting for start");
test_mysql.url = Some(db_url.clone());
let start = Instant::now();
let max_wait = Duration::from_secs(30);
loop {
if start.elapsed() > max_wait {
panic!("db took too long to start");
}
if TcpStream::connect(format!("{}:{}", mysql_ip, mysql_port))
.await
.is_ok()
{
break;
};
// not open wait. sleep and then try again
sleep(Duration::from_secs(1)).await;
}
// TODO: make sure mysql is actually ready for connections
sleep(Duration::from_secs(1)).await;
info!(%db_url, elapsed=%start.elapsed().as_secs_f32(), "db post is open. Migrating now...");
// try to migrate
let start = Instant::now();
let max_wait = Duration::from_secs(30);
loop {
if start.elapsed() > max_wait {
panic!("db took too long to start");
}
match get_migrated_db(db_url.clone(), 1, 1).await {
Ok(x) => {
// it worked! yey!
test_mysql.conn = Some(x);
break;
}
Err(err) => {
// not connected. sleep and then try again
warn!(?err, "unable to migrate db. retrying in 1 second");
sleep(Duration::from_secs(1)).await;
}
}
}
info!(%db_url, elapsed=%start.elapsed().as_secs_f32(), "db is migrated");
test_mysql
}
pub fn conn(&self) -> &DatabaseConnection {
self.conn.as_ref().unwrap()
}
}
impl Drop for TestMysql {
fn drop(&mut self) {
info!(%self.container_name, "killing db");
let _ = SyncCommand::new("docker")
.args(["kill", "-s", "9", &self.container_name])
.output();
}
}

22
web3_proxy/tests/test_admins.rs
View File

@ -4,8 +4,10 @@ use std::str::FromStr;
use std::time::Duration;
use crate::common::admin_increases_balance::admin_increase_balance;
use crate::common::anvil::TestAnvil;
use crate::common::create_admin::create_user_as_admin;
use crate::common::create_user::create_user;
use crate::common::mysql::TestMysql;
use crate::common::user_balance::user_get_balance;
use crate::common::TestApp;
use migration::sea_orm::prelude::Decimal;
@ -15,7 +17,11 @@ use tracing::info;
#[ignore = "under construction"]
#[test_log::test(tokio::test)]
async fn test_admin_imitate_user() {
let x = TestApp::spawn(31337, true).await;
let a: TestAnvil = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
todo!();
}
@ -24,7 +30,13 @@ async fn test_admin_imitate_user() {
#[test_log::test(tokio::test)]
async fn test_admin_grant_credits() {
info!("Starting admin grant credits test");
let x = TestApp::spawn(31337, true).await;
let a: TestAnvil = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::builder()
.timeout(Duration::from_secs(3))
.build()
@ -63,6 +75,10 @@ async fn test_admin_grant_credits() {
#[ignore = "under construction"]
#[test_log::test(tokio::test)]
async fn test_admin_change_user_tier() {
let x = TestApp::spawn(31337, true).await;
let anvil = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(anvil, Some(db)).await;
todo!();
}

11
web3_proxy/tests/test_proxy.rs
View File

@ -1,6 +1,6 @@
mod common;
use crate::common::TestApp;
use crate::common::{anvil::TestAnvil, mysql::TestMysql, TestApp};
use ethers::prelude::U256;
use http::StatusCode;
use std::time::Duration;
@ -13,7 +13,10 @@ use web3_proxy::rpcs::blockchain::ArcBlock;
#[cfg_attr(not(feature = "tests-needing-docker"), ignore)]
#[test_log::test(tokio::test)]
async fn it_migrates_the_db() {
let x = TestApp::spawn(31337, true).await;
let a = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
// we call flush stats more to be sure it works than because we expect it to save any stats
x.flush_stats().await.unwrap();
@ -21,7 +24,9 @@ async fn it_migrates_the_db() {
#[test_log::test(tokio::test)]
async fn it_starts_and_stops() {
let x = TestApp::spawn(31337, false).await;
let a = TestAnvil::spawn(31337).await;
let x = TestApp::spawn(a, None).await;
let anvil_provider = &x.anvil_provider;
let proxy_provider = &x.proxy_provider;

11
web3_proxy/tests/test_sum_credits_used.rs
View File

@ -1,8 +1,9 @@
mod common;
use crate::common::{
admin_increases_balance::admin_increase_balance, create_admin::create_user_as_admin,
create_user::create_user, rpc_key::user_get_provider, user_balance::user_get_balance, TestApp,
admin_increases_balance::admin_increase_balance, anvil::TestAnvil,
create_admin::create_user_as_admin, create_user::create_user, mysql::TestMysql,
rpc_key::user_get_provider, user_balance::user_get_balance, TestApp,
};
use ethers::prelude::U64;
use migration::sea_orm::prelude::Decimal;
@ -14,7 +15,11 @@ use web3_proxy::balance::Balance;
#[test_log::test(tokio::test)]
async fn test_sum_credits_used() {
// chain_id 999_001_999 costs $.10/CU
let x = TestApp::spawn(999_001_999, true).await;
let a = TestAnvil::spawn(999_001_999).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::builder()
.timeout(Duration::from_secs(3))

48
web3_proxy/tests/test_users.rs
View File

@ -2,8 +2,10 @@ mod common;
use crate::common::admin_deposits::get_admin_deposits;
use crate::common::admin_increases_balance::admin_increase_balance;
use crate::common::anvil::TestAnvil;
use crate::common::create_admin::create_user_as_admin;
use crate::common::create_user::create_user;
use crate::common::mysql::TestMysql;
use crate::common::referral::{
get_referral_code, get_shared_referral_codes, get_used_referral_codes, UserSharedReferralInfo,
UserUsedReferralInfo,
@ -36,7 +38,11 @@ struct LoginPostResponse {
#[cfg_attr(not(feature = "tests-needing-docker"), ignore)]
#[test_log::test(tokio::test)]
async fn test_log_in_and_out() {
let x = TestApp::spawn(31337, true).await;
let a = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::new();
@ -92,7 +98,13 @@ async fn test_log_in_and_out() {
#[test_log::test(tokio::test)]
async fn test_admin_balance_increase() {
info!("Starting admin can increase balance");
let x = TestApp::spawn(31337, true).await;
let a: TestAnvil = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::builder()
.timeout(Duration::from_secs(20))
.build()
@ -139,7 +151,13 @@ async fn test_admin_balance_increase() {
#[test_log::test(tokio::test)]
async fn test_user_balance_decreases() {
info!("Starting balance decreases with usage test");
let x = TestApp::spawn(31337, true).await;
let a: TestAnvil = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::builder()
.timeout(Duration::from_secs(20))
.build()
@ -241,7 +259,13 @@ async fn test_user_balance_decreases() {
#[test_log::test(tokio::test)]
async fn test_referral_bonus_non_concurrent() {
info!("Starting referral bonus test");
let x = TestApp::spawn(31337, true).await;
let a: TestAnvil = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::builder()
.timeout(Duration::from_secs(20))
.build()
@ -384,7 +408,13 @@ async fn test_referral_bonus_non_concurrent() {
#[test_log::test(tokio::test)]
async fn test_referral_bonus_concurrent_referrer_only() {
info!("Starting referral bonus test");
let x = TestApp::spawn(31337, true).await;
let a = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::builder()
.timeout(Duration::from_secs(20))
.build()
@ -538,7 +568,13 @@ async fn test_referral_bonus_concurrent_referrer_only() {
#[test_log::test(tokio::test)]
async fn test_referral_bonus_concurrent_referrer_and_user() {
info!("Starting referral bonus test");
let x = TestApp::spawn(31337, true).await;
let a = TestAnvil::spawn(31337).await;
let db = TestMysql::spawn().await;
let x = TestApp::spawn(a, Some(db)).await;
let r = reqwest::Client::builder()
.timeout(Duration::from_secs(20))
.build()