better errors on reconnect

README.md

@@ -50,6 +50,13 @@ Flame graphs make finding slow code painless:
     $ CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph
 
 
+## GDB
+
+Run the proxy under gdb for advanced debugging:
+
+    cargo build --release && RUST_LOG=web3_proxy=debug rust-gdb --args target/debug/web3-proxy --listen-port 7503 --rpc-config-path ./config/production-eth.toml
+
+
 ## Load Testing
 
 Test the proxy:

TODO.md

@@ -6,6 +6,7 @@
   - [ ] when we receive a block, we should store it for later eth_getBlockByNumber, eth_blockNumber, and similar calls
 - [ ] eth_sendRawTransaction should return the most common result, not the first
 - [ ] if chain split detected, don't send transactions
+- [ ] if a rpc fails to connect at start, retry later instead of skipping it forever
 - [ ] endpoint for health checks. if no synced servers, give a 502 error
   - [ ] move from warp to auxm?
 - [ ] some production configs are occassionally stuck waiting at 100% cpu

web3-proxy/src/connection.rs

@@ -222,6 +222,7 @@ impl Web3Connection {
     }
 
     /// Subscribe to new blocks. If `reconnect` is true, this runs forever.
+    /// TODO: instrument with the url
     #[instrument(skip_all)]
     pub async fn subscribe_new_heads(
         self: Arc<Self>,
@@ -302,13 +303,23 @@ impl Web3Connection {
 
                     // TODO: what should this timeout be? needs to be larger than worst case block time
                     // TODO: although reconnects will make this less of an issue
-                    while let Ok(Some(new_block)) =
-                        timeout_at(Instant::now() + Duration::from_secs(300), stream.next()).await
-                    {
-                        self.send_block(Ok(new_block), &block_sender).await;
+                    loop {
+                        match timeout_at(Instant::now() + Duration::from_secs(300), stream.next())
+                            .await
+                        {
+                            Ok(Some(new_block)) => {
+                                self.send_block(Ok(new_block), &block_sender).await;
+                            }
+                            Ok(None) => {
+                                warn!("subscription ended");
+                                break;
+                            }
+                            Err(e) => {
+                                warn!("subscription ended with an error: {:?}", e);
+                                break;
+                            }
+                        }
                     }
-
-                    // TODO: re-connect!
                 }
             }