Change balance to record total spend and total deposits (#109)

* lets test total deposit and total spent

* removed referrer from cache for performance reasons

entities/src/balance.rs

@@ -1,19 +1,20 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.6
 
 use sea_orm::entity::prelude::*;
-use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "balance")]
 pub struct Model {
     #[sea_orm(primary_key)]
     pub id: i32,
-    #[sea_orm(column_type = "Decimal(Some((20, 10)))")]
-    pub available_balance: Decimal,
-    #[sea_orm(column_type = "Decimal(Some((20, 10)))")]
-    pub used_balance: Decimal,
     #[sea_orm(unique)]
     pub user_id: u64,
+    #[sea_orm(column_type = "Decimal(Some((20, 10)))")]
+    pub total_spent_including_free_tier: Decimal,
+    #[sea_orm(column_type = "Decimal(Some((20, 10)))")]
+    pub total_spent_outside_free_tier: Decimal,
+    #[sea_orm(column_type = "Decimal(Some((20, 10)))")]
+    pub total_deposits: Decimal,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

migration/src/lib.rs

@@ -28,6 +28,7 @@ mod m20230422_172555_premium_downgrade_logic;
 mod m20230511_161214_remove_columns_statsv2_origin_and_method;
 mod m20230512_220213_allow_null_rpc_key_id_in_stats_v2;
 mod m20230514_114803_admin_add_credits;
+mod m20230607_221917_total_deposits;
 
 pub struct Migrator;
 
@@ -63,6 +64,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20230511_161214_remove_columns_statsv2_origin_and_method::Migration),
             Box::new(m20230512_220213_allow_null_rpc_key_id_in_stats_v2::Migration),
             Box::new(m20230514_114803_admin_add_credits::Migration),
+            Box::new(m20230607_221917_total_deposits::Migration),
         ]
     }
 }

migration/src/m20230607_221917_total_deposits.rs

@@ -0,0 +1,76 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Balance::Table)
+                    .add_column(
+                        ColumnDef::new(Balance::TotalSpentOutsideFreeTier)
+                            .decimal_len(20, 10)
+                            .not_null()
+                            .default(0.0),
+                    )
+                    .add_column(
+                        ColumnDef::new(Balance::TotalDeposits)
+                            .decimal_len(20, 10)
+                            .not_null()
+                            .default(0.0),
+                    )
+                    .rename_column(Balance::UsedBalance, Balance::TotalSpentIncludingFreeTier)
+                    .drop_column(Balance::AvailableBalance)
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // Remove the column again I suppose, but this will delete data, needless to say
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Balance::Table)
+                    .rename_column(Balance::TotalSpentIncludingFreeTier, Balance::UsedBalance)
+                    .drop_column(Balance::TotalSpentOutsideFreeTier)
+                    .drop_column(Balance::TotalDeposits)
+                    .add_column(
+                        ColumnDef::new(Balance::AvailableBalance)
+                            .decimal_len(20, 10)
+                            .not_null()
+                            .default(0.0),
+                    )
+                    .add_column(
+                        ColumnDef::new(Balance::UsedBalance)
+                            .decimal_len(20, 10)
+                            .not_null()
+                            .default(0.0),
+                    )
+                    .to_owned(),
+            )
+            .await
+    }
+}
+
+/// Learn more at https://docs.rs/sea-query#iden
+#[derive(Iden)]
+enum User {
+    Table,
+    Id,
+}
+
+#[derive(Iden)]
+enum Balance {
+    Table,
+    Id,
+    UserId,
+    TotalSpentIncludingFreeTier,
+    TotalSpentOutsideFreeTier,
+    TotalDeposits,
+    AvailableBalance,
+    UsedBalance,
+}

web3_proxy/src/frontend/admin.rs

@@ -120,7 +120,7 @@ pub async fn admin_increase_balance(
     // update balance
     let balance_entry = balance::ActiveModel {
         id: sea_orm::NotSet,
-        available_balance: sea_orm::Set(amount),
+        total_deposits: sea_orm::Set(amount),
         user_id: sea_orm::Set(user_entry.id),
         ..Default::default()
     };
@@ -128,8 +128,8 @@ pub async fn admin_increase_balance(
         .on_conflict(
             OnConflict::new()
                 .values([(
-                    balance::Column::AvailableBalance,
+                    balance::Column::TotalDeposits,
-                    Expr::col(balance::Column::AvailableBalance).add(amount),
+                    Expr::col(balance::Column::TotalDeposits).add(amount),
                 )])
                 .to_owned(),
         )

web3_proxy/src/frontend/authorization.rs

@@ -7,7 +7,7 @@ use crate::jsonrpc::{JsonRpcForwardedResponse, JsonRpcRequest};
 use crate::rpcs::one::Web3Rpc;
 use crate::stats::{AppStat, BackendRequests, RpcQueryStats};
 use crate::user_token::UserBearerToken;
-use anyhow::{Context};
+use anyhow::Context;
 use axum::headers::authorization::Bearer;
 use axum::headers::{Header, Origin, Referer, UserAgent};
 use chrono::Utc;
@@ -1151,7 +1151,7 @@ impl Web3ProxyApp {
                             .one(db_replica.as_ref())
                             .await?
                         {
-                            Some(x) => x.available_balance,
+                            Some(x) => x.total_deposits - x.total_spent_outside_free_tier,
                             None => Decimal::default(),
                         };
                         Ok(Arc::new(RwLock::new(balance)))
@@ -1256,7 +1256,7 @@ impl Web3ProxyApp {
                             .filter(balance::Column::UserId.eq(user_model.id))
                             .one(db_replica.as_ref())
                             .await? {
-                            Some(x) => x.available_balance,
+                            Some(x) => x.total_deposits - x.total_spent_outside_free_tier,
                             None => Decimal::default()
                         };
 

web3_proxy/src/frontend/users/authentication.rs

@@ -298,8 +298,6 @@ pub async fn user_login_post(
             // We should also create the balance entry ...
             let user_balance = balance::ActiveModel {
                 user_id: sea_orm::Set(caller.id),
-                available_balance: sea_orm::Set(Decimal::new(0, 0)),
-                used_balance: sea_orm::Set(Decimal::new(0, 0)),
                 ..Default::default()
             };
             user_balance.insert(&txn).await?;

web3_proxy/src/frontend/users/payment.rs

@@ -49,7 +49,7 @@ pub async fn user_balance_get(
         .filter(balance::Column::UserId.eq(_user.id))
         .one(db_replica.as_ref())
         .await?
-        .map(|x| x.available_balance)
+        .map(|x| x.total_deposits - x.total_spent_outside_free_tier)
         .unwrap_or_default();
 
     let response = json!({
@@ -257,7 +257,7 @@ pub async fn user_balance_post(
             // create or update the balance
             let balance_entry = balance::ActiveModel {
                 id: sea_orm::NotSet,
-                available_balance: sea_orm::Set(payment_token_amount),
+                total_deposits: sea_orm::Set(payment_token_amount),
                 user_id: sea_orm::Set(recipient.id),
                 ..Default::default()
             };
@@ -265,8 +265,8 @@ pub async fn user_balance_post(
                 .on_conflict(
                     OnConflict::new()
                         .values([(
-                            balance::Column::AvailableBalance,
+                            balance::Column::TotalDeposits,
-                            Expr::col(balance::Column::AvailableBalance).add(payment_token_amount),
+                            Expr::col(balance::Column::TotalDeposits).add(payment_token_amount),
                         )])
                         .to_owned(),
                 )

web3_proxy/src/frontend/users/subuser.rs

@@ -294,8 +294,6 @@ pub async fn modify_subuser(
             // We should also create the balance entry ...
             let subuser_balance = balance::ActiveModel {
                 user_id: sea_orm::Set(subuser.id),
-                available_balance: sea_orm::Set(Decimal::new(0, 0)),
-                used_balance: sea_orm::Set(Decimal::new(0, 0)),
                 ..Default::default()
             };
             subuser_balance.insert(&txn).await?;

web3_proxy/src/stats/mod.rs

@@ -185,10 +185,11 @@ impl RpcQueryStats {
 }
 
 struct Deltas {
-    sender_available_balance_delta: Decimal,
+    balance_used_outside_free_tier: Decimal,
-    sender_used_balance_delta: Decimal,
+    balance_used_including_free_tier: Decimal,
     sender_bonus_applied: bool,
-    referrer_available_balance_delta: Decimal,
+    referrer_deposit_delta: Decimal,
+    sender_bonus_balance_deposited: Decimal,
 }
 
 /// A stat that we aggregate and then store in a database.
@@ -370,14 +371,16 @@ impl BufferedRpcQueryStats {
 
     async fn _compute_balance_deltas(
         &self,
+        sender_balance: balance::Model,
         referral_objects: Option<(referee::Model, referrer::Model)>,
     ) -> Web3ProxyResult<(Deltas, Option<(referee::Model, referrer::Model)>)> {
         // Calculate Balance Only
         let mut deltas = Deltas {
-            sender_available_balance_delta: -self.sum_credits_used,
+            balance_used_outside_free_tier: Default::default(),
-            sender_used_balance_delta: self.sum_credits_used,
+            balance_used_including_free_tier: Default::default(),
             sender_bonus_applied: false,
-            referrer_available_balance_delta: Decimal::from(0),
+            referrer_deposit_delta: Default::default(),
+            sender_bonus_balance_deposited: Default::default(),
         };
 
         // Calculate a bunch using referrals as well
@@ -393,7 +396,7 @@ impl BufferedRpcQueryStats {
                     + self.sum_credits_used)
                     >= Decimal::from(100)
             {
-                deltas.sender_available_balance_delta += Decimal::from(100);
+                deltas.sender_bonus_balance_deposited += Decimal::from(10);
                 deltas.sender_bonus_applied = true;
             }
 
@@ -404,13 +407,23 @@ impl BufferedRpcQueryStats {
                 + Months::new(12);
 
             if now <= valid_until {
-                deltas.referrer_available_balance_delta +=
+                deltas.referrer_deposit_delta += self.sum_credits_used / Decimal::new(10, 0);
-                    self.sum_credits_used / Decimal::new(10, 0);
             }
 
             return Ok((deltas, Some((referral_entity, referrer_code_entity))));
         }
 
+        let user_balance = (sender_balance.total_deposits + deltas.sender_bonus_balance_deposited
+            - sender_balance.total_spent_outside_free_tier);
+        // Split up the component of into how much of the paid component was used, and how much of the free component was used (anything after "balance")
+        if user_balance >= Decimal::from(0) {
+            deltas.balance_used_outside_free_tier = self.sum_credits_used;
+        } else {
+            deltas.balance_used_outside_free_tier =
+                user_balance + deltas.sender_bonus_balance_deposited;
+            deltas.balance_used_including_free_tier = self.sum_credits_used;
+        }
+
         Ok((deltas, None))
     }
 
@@ -425,8 +438,9 @@ impl BufferedRpcQueryStats {
         // Do the user updates
         let user_balance = balance::ActiveModel {
             id: sea_orm::NotSet,
-            available_balance: sea_orm::Set(deltas.sender_available_balance_delta),
+            total_deposits: sea_orm::Set(deltas.sender_bonus_balance_deposited),
-            used_balance: sea_orm::Set(deltas.sender_used_balance_delta),
+            total_spent_including_free_tier: sea_orm::Set(deltas.balance_used_including_free_tier),
+            total_spent_outside_free_tier: sea_orm::Set(deltas.balance_used_outside_free_tier),
             user_id: sea_orm::Set(sender_rpc_entity.user_id),
         };
 
@@ -435,14 +449,19 @@ impl BufferedRpcQueryStats {
                 OnConflict::new()
                     .values([
                         (
-                            balance::Column::AvailableBalance,
+                            balance::Column::TotalDeposits,
-                            Expr::col(balance::Column::AvailableBalance)
+                            Expr::col(balance::Column::TotalDeposits)
-                                .add(deltas.sender_available_balance_delta),
+                                .add(deltas.sender_bonus_balance_deposited),
                         ),
                         (
-                            balance::Column::UsedBalance,
+                            balance::Column::TotalSpentIncludingFreeTier,
-                            Expr::col(balance::Column::UsedBalance)
+                            Expr::col(balance::Column::TotalSpentIncludingFreeTier)
-                                .add(deltas.sender_used_balance_delta),
+                                .add(deltas.balance_used_including_free_tier),
+                        ),
+                        (
+                            balance::Column::TotalSpentOutsideFreeTier,
+                            Expr::col(balance::Column::TotalSpentOutsideFreeTier)
+                                .add(deltas.balance_used_outside_free_tier),
                         ),
                     ])
                     .to_owned(),
@@ -452,7 +471,7 @@ impl BufferedRpcQueryStats {
 
         // Do the referrer_entry updates
         if let Some((referral_entity, referrer_code_entity)) = referral_objects {
-            if deltas.referrer_available_balance_delta > Decimal::from(0) {
+            if deltas.referrer_deposit_delta > Decimal::from(0) {
                 let referee_entry = referee::ActiveModel {
                     id: sea_orm::Unchanged(referral_entity.id),
                     referral_start_date: sea_orm::Unchanged(referral_entity.referral_start_date),
@@ -460,9 +479,7 @@ impl BufferedRpcQueryStats {
                     user_id: sea_orm::Unchanged(referral_entity.user_id),
 
                     credits_applied_for_referee: sea_orm::Set(deltas.sender_bonus_applied),
-                    credits_applied_for_referrer: sea_orm::Set(
+                    credits_applied_for_referrer: sea_orm::Set(deltas.referrer_deposit_delta),
-                        deltas.referrer_available_balance_delta,
-                    ),
                 };
                 referee::Entity::insert(referee_entry)
                     .on_conflict(
@@ -477,7 +494,7 @@ impl BufferedRpcQueryStats {
                                 (
                                     referee::Column::CreditsAppliedForReferrer,
                                     Expr::col(referee::Column::CreditsAppliedForReferrer)
-                                        .add(deltas.referrer_available_balance_delta),
+                                        .add(deltas.referrer_deposit_delta),
                                 ),
                             ])
                             .to_owned(),
@@ -487,18 +504,18 @@ impl BufferedRpcQueryStats {
 
                 let user_balance = balance::ActiveModel {
                     id: sea_orm::NotSet,
-                    available_balance: sea_orm::Set(deltas.referrer_available_balance_delta),
+                    total_deposits: sea_orm::Set(deltas.referrer_deposit_delta),
-                    used_balance: sea_orm::Set(Decimal::from(0)),
                     user_id: sea_orm::Set(referral_entity.user_id),
+                    ..Default::default()
                 };
 
                 let _ = balance::Entity::insert(user_balance)
                     .on_conflict(
                         OnConflict::new()
                             .values([(
-                                balance::Column::AvailableBalance,
+                                balance::Column::TotalDeposits,
-                                Expr::col(balance::Column::AvailableBalance)
+                                Expr::col(balance::Column::TotalDeposits)
-                                    .add(deltas.referrer_available_balance_delta),
+                                    .add(deltas.referrer_deposit_delta),
                             )])
                             .to_owned(),
                     )
@@ -542,7 +559,9 @@ impl BufferedRpcQueryStats {
         let mut latest_balance = sender_latest_balance.write().await;
         let balance_before = *latest_balance;
         // Now modify the balance
-        *latest_balance += deltas.sender_available_balance_delta;
+        // TODO: Double check this (perhaps while testing...)
+        *latest_balance = *latest_balance - deltas.balance_used_outside_free_tier
+            + deltas.sender_bonus_balance_deposited;
         if *latest_balance < Decimal::from(0) {
             *latest_balance = Decimal::from(0);
         }
@@ -568,18 +587,21 @@ impl BufferedRpcQueryStats {
         // ==================
         // Modify referrer balance
         // ==================
-        // If the referrer object is empty, we don't care about the cache, becase this will be fetched in a next request from the database
+        // We ignore this for performance reasons right now
-        if let Some((referral_entity, _)) = referral_objects {
+        // We would have to load all the RPC keys of the referrer to de-activate them
-            if let Ok(referrer_user_id) = NonZeroU64::try_from(referral_entity.user_id) {
+        // Instead, it's fine if they wait for 60 seconds until their tier reloads
-                // If the referrer object is in the cache, we just remove it from the balance cache; it will be reloaded next time
+        // // If the referrer object is empty, we don't care about the cache, becase this will be fetched in a next request from the database
-                // Get all the RPC keys, delete them from cache
+        // if let Some((referral_entity, _)) = referral_objects {
-
+        //     if let Ok(referrer_user_id) = NonZeroU64::try_from(referral_entity.user_id) {
-                // In principle, do not remove the cache for the referrer; the next reload will trigger premium
+        //         // If the referrer object is in the cache, we just remove it from the balance cache; it will be reloaded next time
-                // We don't touch the RPC keys at this stage for the refferer, a payment must be paid to reset those (we want to keep things simple here)
+        //         // Get all the RPC keys, delete them from cache
-                // Anyways, the RPC keys will be updated in 5 min (600 seconds)
+        //
-                user_balance_cache.remove(&referrer_user_id);
+        //         // In principle, do not remove the cache for the referrer; the next reload will trigger premium
-            }
+        //         // We don't touch the RPC keys at this stage for the refferer, a payment must be paid to reset those (we want to keep things simple here)
-        };
+        //         // Anyways, the RPC keys will be updated in 5 min (600 seconds)
+        //         user_balance_cache.remove(&referrer_user_id);
+        //     }
+        // };
 
         Ok(())
     }
@@ -627,8 +649,9 @@ impl BufferedRpcQueryStats {
             self._get_relevant_entities(rpc_secret_key_id, &txn).await?;
 
         // Compute Changes in balance for user and referrer, incl. referral logic   //
-        let (deltas, referral_objects): (Deltas, Option<(referee::Model, referrer::Model)>) =
+        let (deltas, referral_objects): (Deltas, Option<(referee::Model, referrer::Model)>) = self
-            self._compute_balance_deltas(referral_objects).await?;
+            ._compute_balance_deltas(_sender_balance, referral_objects)
+            .await?;
 
         // Update balances in the database
         self._update_balances_in_db(&deltas, &txn, &sender_rpc_entity, &referral_objects)