retry providers

web3-proxy/src/connection.rs

@@ -71,7 +71,7 @@ pub struct Web3Connection {
     /// keep track of currently open requests. We sort on this
     active_requests: AtomicU32,
     /// provider is in a RwLock so that we can replace it if re-connecting
-    provider: RwLock<Arc<Web3Provider>>,
+    provider: RwLock<Option<Arc<Web3Provider>>>,
     /// rate limits are stored in a central redis so that multiple proxies can share their rate limits
     hard_limit: Option<redis_cell_client::RedisCellClient>,
     /// used for load balancing to the least loaded server
@@ -127,12 +127,14 @@ impl Web3Connection {
         // since this lock is held open over an await, we use tokio's locking
         let mut provider = self.provider.write().await;
 
+        *provider = None;
+
         // tell the block subscriber that we are at 0
         block_sender.send_async((Block::default(), rpc_id)).await?;
 
         let new_provider = Web3Provider::from_str(&self.url, http_client).await?;
 
-        *provider = Arc::new(new_provider);
+        *provider = Some(Arc::new(new_provider));
 
         Ok(())
     }
@@ -165,7 +167,7 @@ impl Web3Connection {
         let connection = Web3Connection {
             url: url_str.clone(),
             active_requests: 0.into(),
-            provider: RwLock::new(Arc::new(provider)),
+            provider: RwLock::new(Some(Arc::new(provider))),
             hard_limit,
             soft_limit,
         };
@@ -214,6 +216,11 @@ impl Web3Connection {
         self.soft_limit
     }
 
+    #[inline]
+    pub async fn has_provider(&self) -> bool {
+        self.provider.read().await.is_some()
+    }
+
     #[instrument(skip_all)]
     async fn send_block(
         self: &Arc<Self>,
@@ -246,9 +253,8 @@ impl Web3Connection {
         loop {
             info!("Watching new_heads on {}", self);
 
-            // TODO: is a RwLock of Arc the right thing here?
-            let provider = self.provider.read().await.clone();
-
+            // TODO: is a RwLock of an Option<Arc> the right thing here?
+            if let Some(provider) = self.provider.read().await.clone() {
                 match &*provider {
                     Web3Provider::Http(provider) => {
                         // there is a "watch_blocks" function, but a lot of public nodes do not support the necessary rpc endpoints
@@ -334,14 +340,14 @@ impl Web3Connection {
                         }
                     }
                 }
+            }
 
             if reconnect {
-                drop(provider);
-
                 // TODO: exponential backoff
-                warn!("new heads subscription exited. reconnecting in 10 seconds...");
-                sleep(Duration::from_secs(10)).await;
+                warn!("new heads subscription exited. Attempting to reconnect in 1 second...");
+                sleep(Duration::from_secs(1)).await;
 
+                // TODO: loop on reconnecting! do not return with a "?" here
                 self.reconnect(&block_sender, rpc_id).await?;
             } else {
                 break;
@@ -370,6 +376,12 @@ impl Web3Connection {
     }
 
     pub async fn try_request_handle(self: &Arc<Self>) -> Result<ActiveRequestHandle, Duration> {
+        // check that we are connected
+        if !self.has_provider().await {
+            // TODO: how long? use the same amount as the exponential backoff on retry
+            return Err(Duration::from_secs(1));
+        }
+
         // check rate limits
         if let Some(ratelimiter) = self.hard_limit.as_ref() {
             match ratelimiter.throttle().await {
@@ -424,9 +436,17 @@ impl ActiveRequestHandle {
         // TODO: including params in this is way too verbose
         trace!("Sending {} to {}", method, self.0);
 
-        let provider = self.0.provider.read().await.clone();
+        let mut provider = None;
 
-        let response = match &*provider {
+        while provider.is_none() {
+            // TODO: if no provider, don't unwrap. wait until there is one.
+            match self.0.provider.read().await.as_ref() {
+                None => {}
+                Some(found_provider) => provider = Some(found_provider.clone()),
+            }
+        }
+
+        let response = match &*provider.unwrap() {
             Web3Provider::Http(provider) => provider.request(method, params).await,
             Web3Provider::Ws(provider) => provider.request(method, params).await,
         };