From 04f66b243c9652eb4e1314bbc7a394d64120baa3 Mon Sep 17 00:00:00 2001 From: Kirill Fedoseev Date: Sun, 20 Dec 2020 01:10:47 +0300 Subject: [PATCH] Handle RPC error about ancient blocks (#500) --- oracle/config/foreign-sender.config.js | 5 +- oracle/config/home-sender.config.js | 5 +- oracle/src/sender.js | 4 +- oracle/src/services/HttpListProvider.js | 84 ++++++++++++++++++------- oracle/src/services/logger.js | 15 +++++ oracle/src/services/web3.js | 32 ++++++++-- oracle/src/utils/constants.js | 1 + 7 files changed, 113 insertions(+), 33 deletions(-) diff --git a/oracle/config/foreign-sender.config.js b/oracle/config/foreign-sender.config.js index 4234ee06..aa347c0f 100644 --- a/oracle/config/foreign-sender.config.js +++ b/oracle/config/foreign-sender.config.js @@ -1,6 +1,6 @@ const baseConfig = require('./base.config') -const { web3Foreign, web3ForeignRedundant } = require('../src/services/web3') +const { web3Foreign, web3ForeignRedundant, web3ForeignFallback } = require('../src/services/web3') module.exports = { ...baseConfig.bridgeConfig, @@ -9,5 +9,6 @@ module.exports = { id: 'foreign', name: 'sender-foreign', web3: web3Foreign, - web3Redundant: web3ForeignRedundant + web3Redundant: web3ForeignRedundant, + web3Fallback: web3ForeignFallback } diff --git a/oracle/config/home-sender.config.js b/oracle/config/home-sender.config.js index 51d9084e..ee2bd570 100644 --- a/oracle/config/home-sender.config.js +++ b/oracle/config/home-sender.config.js @@ -1,6 +1,6 @@ const baseConfig = require('./base.config') -const { web3Home, web3HomeRedundant } = require('../src/services/web3') +const { web3Home, web3HomeRedundant, web3HomeFallback } = require('../src/services/web3') module.exports = { ...baseConfig.bridgeConfig, @@ -9,5 +9,6 @@ module.exports = { id: 'home', name: 'sender-home', web3: web3Home, - web3Redundant: web3HomeRedundant + web3Redundant: web3HomeRedundant, + web3Fallback: web3HomeFallback } diff --git a/oracle/src/sender.js b/oracle/src/sender.js index 5eb51af6..172631a5 100644 --- a/oracle/src/sender.js +++ b/oracle/src/sender.js @@ -30,6 +30,8 @@ const config = require(path.join('../config/', process.argv[2])) const web3Instance = config.web3 const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.web3Redundant : config.web3 +const { web3Fallback } = config + const nonceKey = `${config.id}:nonce` let chainId = 0 @@ -128,7 +130,7 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT try { if (isResend) { - const tx = await web3Instance.eth.getTransaction(job.txHash) + const tx = await web3Fallback.eth.getTransaction(job.txHash) if (tx && tx.blockNumber !== null) { logger.debug(`Transaction ${job.txHash} was successfully mined`) diff --git a/oracle/src/services/HttpListProvider.js b/oracle/src/services/HttpListProvider.js index a21cde49..2b4c253d 100644 --- a/oracle/src/services/HttpListProvider.js +++ b/oracle/src/services/HttpListProvider.js @@ -1,10 +1,12 @@ const fetch = require('node-fetch') const promiseRetry = require('promise-retry') +const { FALLBACK_RPC_URL_SWITCH_TIMEOUT } = require('../utils/constants') // From EIP-1474 and Infura documentation const JSONRPC_ERROR_CODES = [-32603, -32002, -32005] const defaultOptions = { + name: 'main', requestTimeout: 0, retry: { retries: 0 @@ -30,23 +32,74 @@ function HttpListProvider(urls, options = {}) { this.urls = urls this.options = { ...defaultOptions, ...options } this.currentIndex = 0 + this.lastTimeUsedPrimary = 0 + this.logger = { + debug: () => {}, + info: () => {} + } +} + +HttpListProvider.prototype.setLogger = function(logger) { + this.logger = logger.child({ module: `HttpListProvider:${this.options.name}` }) } HttpListProvider.prototype.send = async function send(payload, callback) { + // if fallback URL is being used for too long, switch back to the primary URL + if (this.currentIndex > 0 && Date.now() - this.lastTimeUsedPrimary > FALLBACK_RPC_URL_SWITCH_TIMEOUT) { + this.logger.info( + { oldURL: this.urls[this.currentIndex], newURL: this.urls[0] }, + 'Switching back to the primary JSON-RPC URL' + ) + this.currentIndex = 0 + } + // save the currentIndex to avoid race condition const { currentIndex } = this try { - const [result, index] = await promiseRetry(retry => { - return trySend(payload, this.urls, currentIndex, this.options).catch(retry) - }, this.options.retry) - this.currentIndex = index + const [result, index] = await promiseRetry( + retry => this.trySend(payload, currentIndex).catch(retry), + this.options.retry + ) + + // if some of URLs failed to respond, current URL index is updated to the first URL that responded + if (currentIndex !== index) { + this.logger.info( + { index, oldURL: this.urls[currentIndex], newURL: this.urls[index] }, + 'Switching to fallback JSON-RPC URL' + ) + this.currentIndex = index + } callback(null, result) } catch (e) { callback(e) } } +HttpListProvider.prototype.trySend = async function(payload, initialIndex) { + const errors = [] + + for (let count = 0; count < this.urls.length; count++) { + const index = (initialIndex + count) % this.urls.length + + // when request is being sent to the primary URL, the corresponding time marker is updated + if (index === 0) { + this.lastTimeUsedPrimary = Date.now() + } + + const url = this.urls[index] + try { + const result = await send(url, payload, this.options) + return [result, index] + } catch (e) { + this.logger.debug({ index, url, method: payload.method, error: e.message }, `JSON-RPC has failed to respond`) + errors.push(e) + } + } + + throw new HttpListProviderError('Request failed for all urls', errors) +} + function send(url, payload, options) { return fetch(url, { headers: { @@ -65,31 +118,16 @@ function send(url, payload, options) { }) .then(response => response.json()) .then(response => { - if (response.error && JSONRPC_ERROR_CODES.includes(response.error.code)) { + if ( + response.error && + (JSONRPC_ERROR_CODES.includes(response.error.code) || response.error.message.includes('ancient block')) + ) { throw new Error(response.error.message) } return response }) } -async function trySend(payload, urls, initialIndex, options) { - const errors = [] - - let index = initialIndex - for (let count = 0; count < urls.length; count++) { - const url = urls[index] - try { - const result = await send(url, payload, options) - return [result, index] - } catch (e) { - errors.push(e) - } - index = (index + 1) % urls.length - } - - throw new HttpListProviderError('Request failed for all urls', errors) -} - module.exports = { HttpListProvider, HttpListProviderError, diff --git a/oracle/src/services/logger.js b/oracle/src/services/logger.js index a8c5d590..bf5caa4f 100644 --- a/oracle/src/services/logger.js +++ b/oracle/src/services/logger.js @@ -1,5 +1,13 @@ const pino = require('pino') const path = require('path') +const { + web3Home, + web3Foreign, + web3HomeFallback, + web3ForeignFallback, + web3HomeRedundant, + web3ForeignRedundant +} = require('./web3') const config = process.env.NODE_ENV !== 'test' ? require(path.join('../../config/', process.argv[2])) : {} @@ -15,4 +23,11 @@ const logger = pino({ : {} }) +web3Home.currentProvider.setLogger(logger) +web3Foreign.currentProvider.setLogger(logger) +web3HomeFallback.currentProvider.setLogger(logger) +web3ForeignFallback.currentProvider.setLogger(logger) +web3HomeRedundant.currentProvider.setLogger(logger) +web3ForeignRedundant.currentProvider.setLogger(logger) + module.exports = logger diff --git a/oracle/src/services/web3.js b/oracle/src/services/web3.js index d669cbfd..70b195c6 100644 --- a/oracle/src/services/web3.js +++ b/oracle/src/services/web3.js @@ -41,15 +41,37 @@ const web3Home = new Web3(homeProvider) const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions) const web3Foreign = new Web3(foreignProvider) -const redundantHomeProvider = new RedundantHttpListProvider(homeUrls, homeOptions) -const web3HomeRedundant = new Web3(redundantHomeProvider) +// secondary fallback providers are intended to be used in places where +// it is more likely that RPC calls to the local non-archive nodes can fail +// e.g. for checking status of the old transaction via eth_getTransactionByHash +let web3HomeFallback = web3Home +let web3ForeignFallback = web3Foreign -const redundantForeignProvider = new RedundantHttpListProvider(foreignUrls, foreignOptions) -const web3ForeignRedundant = new Web3(redundantForeignProvider) +// secondary redundant providers are intended to be used in places where +// the result of a single RPC request can be lost +// e.g. for sending transactions eth_sendRawTransaction +let web3HomeRedundant = web3Home +let web3ForeignRedundant = web3Foreign + +if (homeUrls.length > 1) { + const provider = new HttpListProvider(homeUrls, { ...homeOptions, name: 'fallback' }) + web3HomeFallback = new Web3(provider) + const redundantProvider = new RedundantHttpListProvider(homeUrls, { ...homeOptions, name: 'redundant' }) + web3HomeRedundant = new Web3(redundantProvider) +} + +if (foreignUrls.length > 1) { + const provider = new HttpListProvider(foreignUrls, { ...foreignOptions, name: 'fallback' }) + web3ForeignFallback = new Web3(provider) + const redundantProvider = new RedundantHttpListProvider(foreignUrls, { ...foreignOptions, name: 'redundant' }) + web3ForeignRedundant = new Web3(redundantProvider) +} module.exports = { web3Home, web3Foreign, web3HomeRedundant, - web3ForeignRedundant + web3ForeignRedundant, + web3HomeFallback, + web3ForeignFallback } diff --git a/oracle/src/utils/constants.js b/oracle/src/utils/constants.js index f3b5b177..92a76db8 100644 --- a/oracle/src/utils/constants.js +++ b/oracle/src/utils/constants.js @@ -24,6 +24,7 @@ module.exports = { MAX: 1000 }, TRANSACTION_RESEND_TIMEOUT: 20 * 60 * 1000, + FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000, SENDER_QUEUE_MAX_PRIORITY: 10, SENDER_QUEUE_SEND_PRIORITY: 5, SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1