Handle RPC error about ancient blocks (#500)

This commit is contained in:
Kirill Fedoseev 2020-12-20 01:10:47 +03:00 committed by GitHub
parent 21581b3c01
commit 04f66b243c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 113 additions and 33 deletions

@ -1,6 +1,6 @@
const baseConfig = require('./base.config') const baseConfig = require('./base.config')
const { web3Foreign, web3ForeignRedundant } = require('../src/services/web3') const { web3Foreign, web3ForeignRedundant, web3ForeignFallback } = require('../src/services/web3')
module.exports = { module.exports = {
...baseConfig.bridgeConfig, ...baseConfig.bridgeConfig,
@ -9,5 +9,6 @@ module.exports = {
id: 'foreign', id: 'foreign',
name: 'sender-foreign', name: 'sender-foreign',
web3: web3Foreign, web3: web3Foreign,
web3Redundant: web3ForeignRedundant web3Redundant: web3ForeignRedundant,
web3Fallback: web3ForeignFallback
} }

@ -1,6 +1,6 @@
const baseConfig = require('./base.config') const baseConfig = require('./base.config')
const { web3Home, web3HomeRedundant } = require('../src/services/web3') const { web3Home, web3HomeRedundant, web3HomeFallback } = require('../src/services/web3')
module.exports = { module.exports = {
...baseConfig.bridgeConfig, ...baseConfig.bridgeConfig,
@ -9,5 +9,6 @@ module.exports = {
id: 'home', id: 'home',
name: 'sender-home', name: 'sender-home',
web3: web3Home, web3: web3Home,
web3Redundant: web3HomeRedundant web3Redundant: web3HomeRedundant,
web3Fallback: web3HomeFallback
} }

@ -30,6 +30,8 @@ const config = require(path.join('../config/', process.argv[2]))
const web3Instance = config.web3 const web3Instance = config.web3
const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.web3Redundant : config.web3 const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.web3Redundant : config.web3
const { web3Fallback } = config
const nonceKey = `${config.id}:nonce` const nonceKey = `${config.id}:nonce`
let chainId = 0 let chainId = 0
@ -128,7 +130,7 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
try { try {
if (isResend) { if (isResend) {
const tx = await web3Instance.eth.getTransaction(job.txHash) const tx = await web3Fallback.eth.getTransaction(job.txHash)
if (tx && tx.blockNumber !== null) { if (tx && tx.blockNumber !== null) {
logger.debug(`Transaction ${job.txHash} was successfully mined`) logger.debug(`Transaction ${job.txHash} was successfully mined`)

@ -1,10 +1,12 @@
const fetch = require('node-fetch') const fetch = require('node-fetch')
const promiseRetry = require('promise-retry') const promiseRetry = require('promise-retry')
const { FALLBACK_RPC_URL_SWITCH_TIMEOUT } = require('../utils/constants')
// From EIP-1474 and Infura documentation // From EIP-1474 and Infura documentation
const JSONRPC_ERROR_CODES = [-32603, -32002, -32005] const JSONRPC_ERROR_CODES = [-32603, -32002, -32005]
const defaultOptions = { const defaultOptions = {
name: 'main',
requestTimeout: 0, requestTimeout: 0,
retry: { retry: {
retries: 0 retries: 0
@ -30,23 +32,74 @@ function HttpListProvider(urls, options = {}) {
this.urls = urls this.urls = urls
this.options = { ...defaultOptions, ...options } this.options = { ...defaultOptions, ...options }
this.currentIndex = 0 this.currentIndex = 0
this.lastTimeUsedPrimary = 0
this.logger = {
debug: () => {},
info: () => {}
}
}
HttpListProvider.prototype.setLogger = function(logger) {
this.logger = logger.child({ module: `HttpListProvider:${this.options.name}` })
} }
HttpListProvider.prototype.send = async function send(payload, callback) { HttpListProvider.prototype.send = async function send(payload, callback) {
// if fallback URL is being used for too long, switch back to the primary URL
if (this.currentIndex > 0 && Date.now() - this.lastTimeUsedPrimary > FALLBACK_RPC_URL_SWITCH_TIMEOUT) {
this.logger.info(
{ oldURL: this.urls[this.currentIndex], newURL: this.urls[0] },
'Switching back to the primary JSON-RPC URL'
)
this.currentIndex = 0
}
// save the currentIndex to avoid race condition // save the currentIndex to avoid race condition
const { currentIndex } = this const { currentIndex } = this
try { try {
const [result, index] = await promiseRetry(retry => { const [result, index] = await promiseRetry(
return trySend(payload, this.urls, currentIndex, this.options).catch(retry) retry => this.trySend(payload, currentIndex).catch(retry),
}, this.options.retry) this.options.retry
)
// if some of URLs failed to respond, current URL index is updated to the first URL that responded
if (currentIndex !== index) {
this.logger.info(
{ index, oldURL: this.urls[currentIndex], newURL: this.urls[index] },
'Switching to fallback JSON-RPC URL'
)
this.currentIndex = index this.currentIndex = index
}
callback(null, result) callback(null, result)
} catch (e) { } catch (e) {
callback(e) callback(e)
} }
} }
HttpListProvider.prototype.trySend = async function(payload, initialIndex) {
const errors = []
for (let count = 0; count < this.urls.length; count++) {
const index = (initialIndex + count) % this.urls.length
// when request is being sent to the primary URL, the corresponding time marker is updated
if (index === 0) {
this.lastTimeUsedPrimary = Date.now()
}
const url = this.urls[index]
try {
const result = await send(url, payload, this.options)
return [result, index]
} catch (e) {
this.logger.debug({ index, url, method: payload.method, error: e.message }, `JSON-RPC has failed to respond`)
errors.push(e)
}
}
throw new HttpListProviderError('Request failed for all urls', errors)
}
function send(url, payload, options) { function send(url, payload, options) {
return fetch(url, { return fetch(url, {
headers: { headers: {
@ -65,31 +118,16 @@ function send(url, payload, options) {
}) })
.then(response => response.json()) .then(response => response.json())
.then(response => { .then(response => {
if (response.error && JSONRPC_ERROR_CODES.includes(response.error.code)) { if (
response.error &&
(JSONRPC_ERROR_CODES.includes(response.error.code) || response.error.message.includes('ancient block'))
) {
throw new Error(response.error.message) throw new Error(response.error.message)
} }
return response return response
}) })
} }
async function trySend(payload, urls, initialIndex, options) {
const errors = []
let index = initialIndex
for (let count = 0; count < urls.length; count++) {
const url = urls[index]
try {
const result = await send(url, payload, options)
return [result, index]
} catch (e) {
errors.push(e)
}
index = (index + 1) % urls.length
}
throw new HttpListProviderError('Request failed for all urls', errors)
}
module.exports = { module.exports = {
HttpListProvider, HttpListProvider,
HttpListProviderError, HttpListProviderError,

@ -1,5 +1,13 @@
const pino = require('pino') const pino = require('pino')
const path = require('path') const path = require('path')
const {
web3Home,
web3Foreign,
web3HomeFallback,
web3ForeignFallback,
web3HomeRedundant,
web3ForeignRedundant
} = require('./web3')
const config = process.env.NODE_ENV !== 'test' ? require(path.join('../../config/', process.argv[2])) : {} const config = process.env.NODE_ENV !== 'test' ? require(path.join('../../config/', process.argv[2])) : {}
@ -15,4 +23,11 @@ const logger = pino({
: {} : {}
}) })
web3Home.currentProvider.setLogger(logger)
web3Foreign.currentProvider.setLogger(logger)
web3HomeFallback.currentProvider.setLogger(logger)
web3ForeignFallback.currentProvider.setLogger(logger)
web3HomeRedundant.currentProvider.setLogger(logger)
web3ForeignRedundant.currentProvider.setLogger(logger)
module.exports = logger module.exports = logger

@ -41,15 +41,37 @@ const web3Home = new Web3(homeProvider)
const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions) const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions)
const web3Foreign = new Web3(foreignProvider) const web3Foreign = new Web3(foreignProvider)
const redundantHomeProvider = new RedundantHttpListProvider(homeUrls, homeOptions) // secondary fallback providers are intended to be used in places where
const web3HomeRedundant = new Web3(redundantHomeProvider) // it is more likely that RPC calls to the local non-archive nodes can fail
// e.g. for checking status of the old transaction via eth_getTransactionByHash
let web3HomeFallback = web3Home
let web3ForeignFallback = web3Foreign
const redundantForeignProvider = new RedundantHttpListProvider(foreignUrls, foreignOptions) // secondary redundant providers are intended to be used in places where
const web3ForeignRedundant = new Web3(redundantForeignProvider) // the result of a single RPC request can be lost
// e.g. for sending transactions eth_sendRawTransaction
let web3HomeRedundant = web3Home
let web3ForeignRedundant = web3Foreign
if (homeUrls.length > 1) {
const provider = new HttpListProvider(homeUrls, { ...homeOptions, name: 'fallback' })
web3HomeFallback = new Web3(provider)
const redundantProvider = new RedundantHttpListProvider(homeUrls, { ...homeOptions, name: 'redundant' })
web3HomeRedundant = new Web3(redundantProvider)
}
if (foreignUrls.length > 1) {
const provider = new HttpListProvider(foreignUrls, { ...foreignOptions, name: 'fallback' })
web3ForeignFallback = new Web3(provider)
const redundantProvider = new RedundantHttpListProvider(foreignUrls, { ...foreignOptions, name: 'redundant' })
web3ForeignRedundant = new Web3(redundantProvider)
}
module.exports = { module.exports = {
web3Home, web3Home,
web3Foreign, web3Foreign,
web3HomeRedundant, web3HomeRedundant,
web3ForeignRedundant web3ForeignRedundant,
web3HomeFallback,
web3ForeignFallback
} }

@ -24,6 +24,7 @@ module.exports = {
MAX: 1000 MAX: 1000
}, },
TRANSACTION_RESEND_TIMEOUT: 20 * 60 * 1000, TRANSACTION_RESEND_TIMEOUT: 20 * 60 * 1000,
FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000,
SENDER_QUEUE_MAX_PRIORITY: 10, SENDER_QUEUE_MAX_PRIORITY: 10,
SENDER_QUEUE_SEND_PRIORITY: 5, SENDER_QUEUE_SEND_PRIORITY: 5,
SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1 SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1