From de7d8919d3c52cd2c089722afd09b43f822ccc8e Mon Sep 17 00:00:00 2001 From: David Date: Wed, 7 Jun 2023 23:45:57 +0200 Subject: [PATCH] Change balance to record total spend and total deposits (#109) * lets test total deposit and total spent * removed referrer from cache for performance reasons --- entities/src/balance.rs | 13 ++- migration/src/lib.rs | 2 + .../src/m20230607_221917_total_deposits.rs | 76 +++++++++++++ web3_proxy/src/frontend/admin.rs | 6 +- web3_proxy/src/frontend/authorization.rs | 6 +- .../src/frontend/users/authentication.rs | 2 - web3_proxy/src/frontend/users/payment.rs | 8 +- web3_proxy/src/frontend/users/subuser.rs | 2 - web3_proxy/src/stats/mod.rs | 107 +++++++++++------- 9 files changed, 160 insertions(+), 62 deletions(-) create mode 100644 migration/src/m20230607_221917_total_deposits.rs diff --git a/entities/src/balance.rs b/entities/src/balance.rs index 9125442c..b8e31269 100644 --- a/entities/src/balance.rs +++ b/entities/src/balance.rs @@ -1,19 +1,20 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.6 use sea_orm::entity::prelude::*; -use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "balance")] pub struct Model { #[sea_orm(primary_key)] pub id: i32, - #[sea_orm(column_type = "Decimal(Some((20, 10)))")] - pub available_balance: Decimal, - #[sea_orm(column_type = "Decimal(Some((20, 10)))")] - pub used_balance: Decimal, #[sea_orm(unique)] pub user_id: u64, + #[sea_orm(column_type = "Decimal(Some((20, 10)))")] + pub total_spent_including_free_tier: Decimal, + #[sea_orm(column_type = "Decimal(Some((20, 10)))")] + pub total_spent_outside_free_tier: Decimal, + #[sea_orm(column_type = "Decimal(Some((20, 10)))")] + pub total_deposits: Decimal, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 91c45391..e788f77a 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -28,6 +28,7 @@ mod m20230422_172555_premium_downgrade_logic; mod m20230511_161214_remove_columns_statsv2_origin_and_method; mod m20230512_220213_allow_null_rpc_key_id_in_stats_v2; mod m20230514_114803_admin_add_credits; +mod m20230607_221917_total_deposits; pub struct Migrator; @@ -63,6 +64,7 @@ impl MigratorTrait for Migrator { Box::new(m20230511_161214_remove_columns_statsv2_origin_and_method::Migration), Box::new(m20230512_220213_allow_null_rpc_key_id_in_stats_v2::Migration), Box::new(m20230514_114803_admin_add_credits::Migration), + Box::new(m20230607_221917_total_deposits::Migration), ] } } diff --git a/migration/src/m20230607_221917_total_deposits.rs b/migration/src/m20230607_221917_total_deposits.rs new file mode 100644 index 00000000..3e544c90 --- /dev/null +++ b/migration/src/m20230607_221917_total_deposits.rs @@ -0,0 +1,76 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Balance::Table) + .add_column( + ColumnDef::new(Balance::TotalSpentOutsideFreeTier) + .decimal_len(20, 10) + .not_null() + .default(0.0), + ) + .add_column( + ColumnDef::new(Balance::TotalDeposits) + .decimal_len(20, 10) + .not_null() + .default(0.0), + ) + .rename_column(Balance::UsedBalance, Balance::TotalSpentIncludingFreeTier) + .drop_column(Balance::AvailableBalance) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Remove the column again I suppose, but this will delete data, needless to say + manager + .alter_table( + Table::alter() + .table(Balance::Table) + .rename_column(Balance::TotalSpentIncludingFreeTier, Balance::UsedBalance) + .drop_column(Balance::TotalSpentOutsideFreeTier) + .drop_column(Balance::TotalDeposits) + .add_column( + ColumnDef::new(Balance::AvailableBalance) + .decimal_len(20, 10) + .not_null() + .default(0.0), + ) + .add_column( + ColumnDef::new(Balance::UsedBalance) + .decimal_len(20, 10) + .not_null() + .default(0.0), + ) + .to_owned(), + ) + .await + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum User { + Table, + Id, +} + +#[derive(Iden)] +enum Balance { + Table, + Id, + UserId, + TotalSpentIncludingFreeTier, + TotalSpentOutsideFreeTier, + TotalDeposits, + AvailableBalance, + UsedBalance, +} diff --git a/web3_proxy/src/frontend/admin.rs b/web3_proxy/src/frontend/admin.rs index 628f9263..b2012448 100644 --- a/web3_proxy/src/frontend/admin.rs +++ b/web3_proxy/src/frontend/admin.rs @@ -120,7 +120,7 @@ pub async fn admin_increase_balance( // update balance let balance_entry = balance::ActiveModel { id: sea_orm::NotSet, - available_balance: sea_orm::Set(amount), + total_deposits: sea_orm::Set(amount), user_id: sea_orm::Set(user_entry.id), ..Default::default() }; @@ -128,8 +128,8 @@ pub async fn admin_increase_balance( .on_conflict( OnConflict::new() .values([( - balance::Column::AvailableBalance, - Expr::col(balance::Column::AvailableBalance).add(amount), + balance::Column::TotalDeposits, + Expr::col(balance::Column::TotalDeposits).add(amount), )]) .to_owned(), ) diff --git a/web3_proxy/src/frontend/authorization.rs b/web3_proxy/src/frontend/authorization.rs index 93f1f17c..63798948 100644 --- a/web3_proxy/src/frontend/authorization.rs +++ b/web3_proxy/src/frontend/authorization.rs @@ -7,7 +7,7 @@ use crate::jsonrpc::{JsonRpcForwardedResponse, JsonRpcRequest}; use crate::rpcs::one::Web3Rpc; use crate::stats::{AppStat, BackendRequests, RpcQueryStats}; use crate::user_token::UserBearerToken; -use anyhow::{Context}; +use anyhow::Context; use axum::headers::authorization::Bearer; use axum::headers::{Header, Origin, Referer, UserAgent}; use chrono::Utc; @@ -1151,7 +1151,7 @@ impl Web3ProxyApp { .one(db_replica.as_ref()) .await? { - Some(x) => x.available_balance, + Some(x) => x.total_deposits - x.total_spent_outside_free_tier, None => Decimal::default(), }; Ok(Arc::new(RwLock::new(balance))) @@ -1256,7 +1256,7 @@ impl Web3ProxyApp { .filter(balance::Column::UserId.eq(user_model.id)) .one(db_replica.as_ref()) .await? { - Some(x) => x.available_balance, + Some(x) => x.total_deposits - x.total_spent_outside_free_tier, None => Decimal::default() }; diff --git a/web3_proxy/src/frontend/users/authentication.rs b/web3_proxy/src/frontend/users/authentication.rs index 3c3321d7..cb51fb67 100644 --- a/web3_proxy/src/frontend/users/authentication.rs +++ b/web3_proxy/src/frontend/users/authentication.rs @@ -298,8 +298,6 @@ pub async fn user_login_post( // We should also create the balance entry ... let user_balance = balance::ActiveModel { user_id: sea_orm::Set(caller.id), - available_balance: sea_orm::Set(Decimal::new(0, 0)), - used_balance: sea_orm::Set(Decimal::new(0, 0)), ..Default::default() }; user_balance.insert(&txn).await?; diff --git a/web3_proxy/src/frontend/users/payment.rs b/web3_proxy/src/frontend/users/payment.rs index 58e8446d..6f3ee818 100644 --- a/web3_proxy/src/frontend/users/payment.rs +++ b/web3_proxy/src/frontend/users/payment.rs @@ -49,7 +49,7 @@ pub async fn user_balance_get( .filter(balance::Column::UserId.eq(_user.id)) .one(db_replica.as_ref()) .await? - .map(|x| x.available_balance) + .map(|x| x.total_deposits - x.total_spent_outside_free_tier) .unwrap_or_default(); let response = json!({ @@ -257,7 +257,7 @@ pub async fn user_balance_post( // create or update the balance let balance_entry = balance::ActiveModel { id: sea_orm::NotSet, - available_balance: sea_orm::Set(payment_token_amount), + total_deposits: sea_orm::Set(payment_token_amount), user_id: sea_orm::Set(recipient.id), ..Default::default() }; @@ -265,8 +265,8 @@ pub async fn user_balance_post( .on_conflict( OnConflict::new() .values([( - balance::Column::AvailableBalance, - Expr::col(balance::Column::AvailableBalance).add(payment_token_amount), + balance::Column::TotalDeposits, + Expr::col(balance::Column::TotalDeposits).add(payment_token_amount), )]) .to_owned(), ) diff --git a/web3_proxy/src/frontend/users/subuser.rs b/web3_proxy/src/frontend/users/subuser.rs index 3cb12ad0..2cea1621 100644 --- a/web3_proxy/src/frontend/users/subuser.rs +++ b/web3_proxy/src/frontend/users/subuser.rs @@ -294,8 +294,6 @@ pub async fn modify_subuser( // We should also create the balance entry ... let subuser_balance = balance::ActiveModel { user_id: sea_orm::Set(subuser.id), - available_balance: sea_orm::Set(Decimal::new(0, 0)), - used_balance: sea_orm::Set(Decimal::new(0, 0)), ..Default::default() }; subuser_balance.insert(&txn).await?; diff --git a/web3_proxy/src/stats/mod.rs b/web3_proxy/src/stats/mod.rs index c9b0cfe0..5d3cbf63 100644 --- a/web3_proxy/src/stats/mod.rs +++ b/web3_proxy/src/stats/mod.rs @@ -185,10 +185,11 @@ impl RpcQueryStats { } struct Deltas { - sender_available_balance_delta: Decimal, - sender_used_balance_delta: Decimal, + balance_used_outside_free_tier: Decimal, + balance_used_including_free_tier: Decimal, sender_bonus_applied: bool, - referrer_available_balance_delta: Decimal, + referrer_deposit_delta: Decimal, + sender_bonus_balance_deposited: Decimal, } /// A stat that we aggregate and then store in a database. @@ -370,14 +371,16 @@ impl BufferedRpcQueryStats { async fn _compute_balance_deltas( &self, + sender_balance: balance::Model, referral_objects: Option<(referee::Model, referrer::Model)>, ) -> Web3ProxyResult<(Deltas, Option<(referee::Model, referrer::Model)>)> { // Calculate Balance Only let mut deltas = Deltas { - sender_available_balance_delta: -self.sum_credits_used, - sender_used_balance_delta: self.sum_credits_used, + balance_used_outside_free_tier: Default::default(), + balance_used_including_free_tier: Default::default(), sender_bonus_applied: false, - referrer_available_balance_delta: Decimal::from(0), + referrer_deposit_delta: Default::default(), + sender_bonus_balance_deposited: Default::default(), }; // Calculate a bunch using referrals as well @@ -393,7 +396,7 @@ impl BufferedRpcQueryStats { + self.sum_credits_used) >= Decimal::from(100) { - deltas.sender_available_balance_delta += Decimal::from(100); + deltas.sender_bonus_balance_deposited += Decimal::from(10); deltas.sender_bonus_applied = true; } @@ -404,13 +407,23 @@ impl BufferedRpcQueryStats { + Months::new(12); if now <= valid_until { - deltas.referrer_available_balance_delta += - self.sum_credits_used / Decimal::new(10, 0); + deltas.referrer_deposit_delta += self.sum_credits_used / Decimal::new(10, 0); } return Ok((deltas, Some((referral_entity, referrer_code_entity)))); } + let user_balance = (sender_balance.total_deposits + deltas.sender_bonus_balance_deposited + - sender_balance.total_spent_outside_free_tier); + // Split up the component of into how much of the paid component was used, and how much of the free component was used (anything after "balance") + if user_balance >= Decimal::from(0) { + deltas.balance_used_outside_free_tier = self.sum_credits_used; + } else { + deltas.balance_used_outside_free_tier = + user_balance + deltas.sender_bonus_balance_deposited; + deltas.balance_used_including_free_tier = self.sum_credits_used; + } + Ok((deltas, None)) } @@ -425,8 +438,9 @@ impl BufferedRpcQueryStats { // Do the user updates let user_balance = balance::ActiveModel { id: sea_orm::NotSet, - available_balance: sea_orm::Set(deltas.sender_available_balance_delta), - used_balance: sea_orm::Set(deltas.sender_used_balance_delta), + total_deposits: sea_orm::Set(deltas.sender_bonus_balance_deposited), + total_spent_including_free_tier: sea_orm::Set(deltas.balance_used_including_free_tier), + total_spent_outside_free_tier: sea_orm::Set(deltas.balance_used_outside_free_tier), user_id: sea_orm::Set(sender_rpc_entity.user_id), }; @@ -435,14 +449,19 @@ impl BufferedRpcQueryStats { OnConflict::new() .values([ ( - balance::Column::AvailableBalance, - Expr::col(balance::Column::AvailableBalance) - .add(deltas.sender_available_balance_delta), + balance::Column::TotalDeposits, + Expr::col(balance::Column::TotalDeposits) + .add(deltas.sender_bonus_balance_deposited), ), ( - balance::Column::UsedBalance, - Expr::col(balance::Column::UsedBalance) - .add(deltas.sender_used_balance_delta), + balance::Column::TotalSpentIncludingFreeTier, + Expr::col(balance::Column::TotalSpentIncludingFreeTier) + .add(deltas.balance_used_including_free_tier), + ), + ( + balance::Column::TotalSpentOutsideFreeTier, + Expr::col(balance::Column::TotalSpentOutsideFreeTier) + .add(deltas.balance_used_outside_free_tier), ), ]) .to_owned(), @@ -452,7 +471,7 @@ impl BufferedRpcQueryStats { // Do the referrer_entry updates if let Some((referral_entity, referrer_code_entity)) = referral_objects { - if deltas.referrer_available_balance_delta > Decimal::from(0) { + if deltas.referrer_deposit_delta > Decimal::from(0) { let referee_entry = referee::ActiveModel { id: sea_orm::Unchanged(referral_entity.id), referral_start_date: sea_orm::Unchanged(referral_entity.referral_start_date), @@ -460,9 +479,7 @@ impl BufferedRpcQueryStats { user_id: sea_orm::Unchanged(referral_entity.user_id), credits_applied_for_referee: sea_orm::Set(deltas.sender_bonus_applied), - credits_applied_for_referrer: sea_orm::Set( - deltas.referrer_available_balance_delta, - ), + credits_applied_for_referrer: sea_orm::Set(deltas.referrer_deposit_delta), }; referee::Entity::insert(referee_entry) .on_conflict( @@ -477,7 +494,7 @@ impl BufferedRpcQueryStats { ( referee::Column::CreditsAppliedForReferrer, Expr::col(referee::Column::CreditsAppliedForReferrer) - .add(deltas.referrer_available_balance_delta), + .add(deltas.referrer_deposit_delta), ), ]) .to_owned(), @@ -487,18 +504,18 @@ impl BufferedRpcQueryStats { let user_balance = balance::ActiveModel { id: sea_orm::NotSet, - available_balance: sea_orm::Set(deltas.referrer_available_balance_delta), - used_balance: sea_orm::Set(Decimal::from(0)), + total_deposits: sea_orm::Set(deltas.referrer_deposit_delta), user_id: sea_orm::Set(referral_entity.user_id), + ..Default::default() }; let _ = balance::Entity::insert(user_balance) .on_conflict( OnConflict::new() .values([( - balance::Column::AvailableBalance, - Expr::col(balance::Column::AvailableBalance) - .add(deltas.referrer_available_balance_delta), + balance::Column::TotalDeposits, + Expr::col(balance::Column::TotalDeposits) + .add(deltas.referrer_deposit_delta), )]) .to_owned(), ) @@ -542,7 +559,9 @@ impl BufferedRpcQueryStats { let mut latest_balance = sender_latest_balance.write().await; let balance_before = *latest_balance; // Now modify the balance - *latest_balance += deltas.sender_available_balance_delta; + // TODO: Double check this (perhaps while testing...) + *latest_balance = *latest_balance - deltas.balance_used_outside_free_tier + + deltas.sender_bonus_balance_deposited; if *latest_balance < Decimal::from(0) { *latest_balance = Decimal::from(0); } @@ -568,18 +587,21 @@ impl BufferedRpcQueryStats { // ================== // Modify referrer balance // ================== - // If the referrer object is empty, we don't care about the cache, becase this will be fetched in a next request from the database - if let Some((referral_entity, _)) = referral_objects { - if let Ok(referrer_user_id) = NonZeroU64::try_from(referral_entity.user_id) { - // If the referrer object is in the cache, we just remove it from the balance cache; it will be reloaded next time - // Get all the RPC keys, delete them from cache - - // In principle, do not remove the cache for the referrer; the next reload will trigger premium - // We don't touch the RPC keys at this stage for the refferer, a payment must be paid to reset those (we want to keep things simple here) - // Anyways, the RPC keys will be updated in 5 min (600 seconds) - user_balance_cache.remove(&referrer_user_id); - } - }; + // We ignore this for performance reasons right now + // We would have to load all the RPC keys of the referrer to de-activate them + // Instead, it's fine if they wait for 60 seconds until their tier reloads + // // If the referrer object is empty, we don't care about the cache, becase this will be fetched in a next request from the database + // if let Some((referral_entity, _)) = referral_objects { + // if let Ok(referrer_user_id) = NonZeroU64::try_from(referral_entity.user_id) { + // // If the referrer object is in the cache, we just remove it from the balance cache; it will be reloaded next time + // // Get all the RPC keys, delete them from cache + // + // // In principle, do not remove the cache for the referrer; the next reload will trigger premium + // // We don't touch the RPC keys at this stage for the refferer, a payment must be paid to reset those (we want to keep things simple here) + // // Anyways, the RPC keys will be updated in 5 min (600 seconds) + // user_balance_cache.remove(&referrer_user_id); + // } + // }; Ok(()) } @@ -627,8 +649,9 @@ impl BufferedRpcQueryStats { self._get_relevant_entities(rpc_secret_key_id, &txn).await?; // Compute Changes in balance for user and referrer, incl. referral logic // - let (deltas, referral_objects): (Deltas, Option<(referee::Model, referrer::Model)>) = - self._compute_balance_deltas(referral_objects).await?; + let (deltas, referral_objects): (Deltas, Option<(referee::Model, referrer::Model)>) = self + ._compute_balance_deltas(_sender_balance, referral_objects) + .await?; // Update balances in the database self._update_balances_in_db(&deltas, &txn, &sender_rpc_entity, &referral_objects)