Handle RPC error about ancient blocks (#500)

oracle/config/foreign-sender.config.js

@@ -1,6 +1,6 @@
 const baseConfig = require('./base.config')
 
-const { web3Foreign, web3ForeignRedundant } = require('../src/services/web3')
+const { web3Foreign, web3ForeignRedundant, web3ForeignFallback } = require('../src/services/web3')
 
 module.exports = {
   ...baseConfig.bridgeConfig,
@@ -9,5 +9,6 @@ module.exports = {
   id: 'foreign',
   name: 'sender-foreign',
   web3: web3Foreign,
-  web3Redundant: web3ForeignRedundant
+  web3Redundant: web3ForeignRedundant,
+  web3Fallback: web3ForeignFallback
 }

oracle/config/home-sender.config.js

@@ -1,6 +1,6 @@
 const baseConfig = require('./base.config')
 
-const { web3Home, web3HomeRedundant } = require('../src/services/web3')
+const { web3Home, web3HomeRedundant, web3HomeFallback } = require('../src/services/web3')
 
 module.exports = {
   ...baseConfig.bridgeConfig,
@@ -9,5 +9,6 @@ module.exports = {
   id: 'home',
   name: 'sender-home',
   web3: web3Home,
-  web3Redundant: web3HomeRedundant
+  web3Redundant: web3HomeRedundant,
+  web3Fallback: web3HomeFallback
 }

oracle/src/sender.js

@@ -30,6 +30,8 @@ const config = require(path.join('../config/', process.argv[2]))
 
 const web3Instance = config.web3
 const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.web3Redundant : config.web3
+const { web3Fallback } = config
+
 const nonceKey = `${config.id}:nonce`
 let chainId = 0
 
@@ -128,7 +130,7 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
 
       try {
         if (isResend) {
-          const tx = await web3Instance.eth.getTransaction(job.txHash)
+          const tx = await web3Fallback.eth.getTransaction(job.txHash)
 
           if (tx && tx.blockNumber !== null) {
             logger.debug(`Transaction ${job.txHash} was successfully mined`)

oracle/src/services/HttpListProvider.js

@@ -1,10 +1,12 @@
 const fetch = require('node-fetch')
 const promiseRetry = require('promise-retry')
+const { FALLBACK_RPC_URL_SWITCH_TIMEOUT } = require('../utils/constants')
 
 // From EIP-1474 and Infura documentation
 const JSONRPC_ERROR_CODES = [-32603, -32002, -32005]
 
 const defaultOptions = {
+  name: 'main',
   requestTimeout: 0,
   retry: {
     retries: 0
@@ -30,23 +32,74 @@ function HttpListProvider(urls, options = {}) {
   this.urls = urls
   this.options = { ...defaultOptions, ...options }
   this.currentIndex = 0
+  this.lastTimeUsedPrimary = 0
+  this.logger = {
+    debug: () => {},
+    info: () => {}
+  }
+}
+
+HttpListProvider.prototype.setLogger = function(logger) {
+  this.logger = logger.child({ module: `HttpListProvider:${this.options.name}` })
 }
 
 HttpListProvider.prototype.send = async function send(payload, callback) {
+  // if fallback URL is being used for too long, switch back to the primary URL
+  if (this.currentIndex > 0 && Date.now() - this.lastTimeUsedPrimary > FALLBACK_RPC_URL_SWITCH_TIMEOUT) {
+    this.logger.info(
+      { oldURL: this.urls[this.currentIndex], newURL: this.urls[0] },
+      'Switching back to the primary JSON-RPC URL'
+    )
+    this.currentIndex = 0
+  }
+
   // save the currentIndex to avoid race condition
   const { currentIndex } = this
 
   try {
-    const [result, index] = await promiseRetry(retry => {
+    const [result, index] = await promiseRetry(
-      return trySend(payload, this.urls, currentIndex, this.options).catch(retry)
+      retry => this.trySend(payload, currentIndex).catch(retry),
-    }, this.options.retry)
+      this.options.retry
+    )
+
+    // if some of URLs failed to respond, current URL index is updated to the first URL that responded
+    if (currentIndex !== index) {
+      this.logger.info(
+        { index, oldURL: this.urls[currentIndex], newURL: this.urls[index] },
+        'Switching to fallback JSON-RPC URL'
+      )
       this.currentIndex = index
+    }
     callback(null, result)
   } catch (e) {
     callback(e)
   }
 }
 
+HttpListProvider.prototype.trySend = async function(payload, initialIndex) {
+  const errors = []
+
+  for (let count = 0; count < this.urls.length; count++) {
+    const index = (initialIndex + count) % this.urls.length
+
+    // when request is being sent to the primary URL, the corresponding time marker is updated
+    if (index === 0) {
+      this.lastTimeUsedPrimary = Date.now()
+    }
+
+    const url = this.urls[index]
+    try {
+      const result = await send(url, payload, this.options)
+      return [result, index]
+    } catch (e) {
+      this.logger.debug({ index, url, method: payload.method, error: e.message }, `JSON-RPC has failed to respond`)
+      errors.push(e)
+    }
+  }
+
+  throw new HttpListProviderError('Request failed for all urls', errors)
+}
+
 function send(url, payload, options) {
   return fetch(url, {
     headers: {
@@ -65,31 +118,16 @@ function send(url, payload, options) {
     })
     .then(response => response.json())
     .then(response => {
-      if (response.error && JSONRPC_ERROR_CODES.includes(response.error.code)) {
+      if (
+        response.error &&
+        (JSONRPC_ERROR_CODES.includes(response.error.code) || response.error.message.includes('ancient block'))
+      ) {
         throw new Error(response.error.message)
       }
       return response
     })
 }
 
-async function trySend(payload, urls, initialIndex, options) {
-  const errors = []
-
-  let index = initialIndex
-  for (let count = 0; count < urls.length; count++) {
-    const url = urls[index]
-    try {
-      const result = await send(url, payload, options)
-      return [result, index]
-    } catch (e) {
-      errors.push(e)
-    }
-    index = (index + 1) % urls.length
-  }
-
-  throw new HttpListProviderError('Request failed for all urls', errors)
-}
-
 module.exports = {
   HttpListProvider,
   HttpListProviderError,

oracle/src/services/logger.js

@@ -1,5 +1,13 @@
 const pino = require('pino')
 const path = require('path')
+const {
+  web3Home,
+  web3Foreign,
+  web3HomeFallback,
+  web3ForeignFallback,
+  web3HomeRedundant,
+  web3ForeignRedundant
+} = require('./web3')
 
 const config = process.env.NODE_ENV !== 'test' ? require(path.join('../../config/', process.argv[2])) : {}
 
@@ -15,4 +23,11 @@ const logger = pino({
       : {}
 })
 
+web3Home.currentProvider.setLogger(logger)
+web3Foreign.currentProvider.setLogger(logger)
+web3HomeFallback.currentProvider.setLogger(logger)
+web3ForeignFallback.currentProvider.setLogger(logger)
+web3HomeRedundant.currentProvider.setLogger(logger)
+web3ForeignRedundant.currentProvider.setLogger(logger)
+
 module.exports = logger

oracle/src/services/web3.js

@@ -41,15 +41,37 @@ const web3Home = new Web3(homeProvider)
 const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions)
 const web3Foreign = new Web3(foreignProvider)
 
-const redundantHomeProvider = new RedundantHttpListProvider(homeUrls, homeOptions)
+// secondary fallback providers are intended to be used in places where
-const web3HomeRedundant = new Web3(redundantHomeProvider)
+// it is more likely that RPC calls to the local non-archive nodes can fail
+// e.g. for checking status of the old transaction via eth_getTransactionByHash
+let web3HomeFallback = web3Home
+let web3ForeignFallback = web3Foreign
 
-const redundantForeignProvider = new RedundantHttpListProvider(foreignUrls, foreignOptions)
+// secondary redundant providers are intended to be used in places where
-const web3ForeignRedundant = new Web3(redundantForeignProvider)
+// the result of a single RPC request can be lost
+// e.g. for sending transactions eth_sendRawTransaction
+let web3HomeRedundant = web3Home
+let web3ForeignRedundant = web3Foreign
+
+if (homeUrls.length > 1) {
+  const provider = new HttpListProvider(homeUrls, { ...homeOptions, name: 'fallback' })
+  web3HomeFallback = new Web3(provider)
+  const redundantProvider = new RedundantHttpListProvider(homeUrls, { ...homeOptions, name: 'redundant' })
+  web3HomeRedundant = new Web3(redundantProvider)
+}
+
+if (foreignUrls.length > 1) {
+  const provider = new HttpListProvider(foreignUrls, { ...foreignOptions, name: 'fallback' })
+  web3ForeignFallback = new Web3(provider)
+  const redundantProvider = new RedundantHttpListProvider(foreignUrls, { ...foreignOptions, name: 'redundant' })
+  web3ForeignRedundant = new Web3(redundantProvider)
+}
 
 module.exports = {
   web3Home,
   web3Foreign,
   web3HomeRedundant,
-  web3ForeignRedundant
+  web3ForeignRedundant,
+  web3HomeFallback,
+  web3ForeignFallback
 }

oracle/src/utils/constants.js

@@ -24,6 +24,7 @@ module.exports = {
     MAX: 1000
   },
   TRANSACTION_RESEND_TIMEOUT: 20 * 60 * 1000,
+  FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000,
   SENDER_QUEUE_MAX_PRIORITY: 10,
   SENDER_QUEUE_SEND_PRIORITY: 5,
   SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1