tornadocontrib/tokenbridge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Try to detect unsynced node state (#592)

Browse Source
This commit is contained in:
Kirill Fedoseev 2021-08-04 00:36:50 +04:00 committed by GitHub
parent 4f5e3c47be
commit 2e6179f974
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 72 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/main.yml vendored
View File

@ -77,7 +77,7 @@ jobs:
- name: Rebuild and push updated images
run: |
function check_if_image_exists() {
curl -fsSlL -H 'Authorization: bearer ${{ github.token }}' "https://${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null
curl -fsSlL "https://${{ github.actor }}:${{ github.token }}@${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null
}
updated=()
if ! check_if_image_exists e2e ${E2E_TAG}; then updated+=("e2e"); fi
@ -104,7 +104,7 @@ jobs:
- name: Rebuild and push molecule runner e2e image
run: |
function check_if_image_exists() {
curl -fsSlL -H 'Authorization: bearer ${{ github.token }}' "https://${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null
curl -fsSlL "https://${{ github.actor }}:${{ github.token }}@${DOCKER_REGISTRY}/v2/${DOCKER_REPO}/tokenbridge-e2e-$1/manifests/$2" > /dev/null
}
if check_if_image_exists molecule_runner ${MOLECULE_RUNNER_TAG}; then
echo "Image already exists"

2
oracle/src/confirmRelay.js
View File

@ -49,7 +49,7 @@ async function initialize() {
try {
const checkHttps = checkHTTPS(ORACLE_ALLOW_HTTP_FOR_RPC, logger)
web3.currentProvider.subProvider.urls.forEach(checkHttps(chain))
web3.currentProvider.urls.forEach(checkHttps(chain))
attached = await isAttached()
if (attached) {

2
oracle/src/sender.js
View File

@ -40,7 +40,7 @@ async function initialize() {
try {
const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger)
web3.currentProvider.subProvider.urls.forEach(checkHttps(config.id))
web3.currentProvider.urls.forEach(checkHttps(config.id))
GasPrice.start(config.id)

14
oracle/src/services/HttpListProvider.js
View File

@ -41,6 +41,20 @@ function HttpListProvider(urls, options = {}) {
})
}
HttpListProvider.prototype.switchToFallbackRPC = function() {
if (this.urls.length < 2) {
return
}
const prevIndex = this.currentIndex
const newIndex = (prevIndex + 1) % this.urls.length
this.logger.info(
{ index: newIndex, oldURL: this.urls[prevIndex], newURL: this.urls[newIndex] },
'Switching to fallback JSON-RPC URL'
)
this.currentIndex = newIndex
}
HttpListProvider.prototype.send = async function send(payload, callback) {
// if fallback URL is being used for too long, switch back to the primary URL
if (this.currentIndex > 0 && Date.now() - this.lastTimeUsedPrimary > FALLBACK_RPC_URL_SWITCH_TIMEOUT) {

54
oracle/src/services/SafeEthLogsProvider.js
View File

@ -1,39 +1,35 @@
const { hexToNumber, isHexStrict } = require('web3').utils
const { onInjected } = require('./injectedLogger')
function SafeEthLogsProvider(provider) {
this.subProvider = provider
onInjected(logger => {
this.logger = logger.child({ module: 'SafeEthLogsProvider' })
})
}
const oldSend = provider.send.bind(provider)
const newSend = function(payload, callback) {
if (payload.method === 'eth_getLogs' && isHexStrict(payload.params[0].toBlock)) {
this.logger.debug('Modifying eth_getLogs request to include batch eth_blockNumber request')
SafeEthLogsProvider.prototype.send = function send(payload, callback) {
if (payload.method === 'eth_getLogs' && isHexStrict(payload.params[0].toBlock)) {
this.logger.debug('Modifying eth_getLogs request to include batch eth_blockNumber request')
const newPayload = [payload, { jsonrpc: '2.0', id: payload.id + 1, method: 'eth_blockNumber', params: [] }]
this.subProvider.send(newPayload, (err, res) => {
if (err) {
callback(err, null)
} else {
const rawLogs = res.find(({ id }) => id === payload.id)
const rawBlockNumber = res.find(({ id }) => id === payload.id + 1)
const blockNumber = hexToNumber(rawBlockNumber.result)
const toBlock = hexToNumber(payload.params[0].toBlock)
if (blockNumber < toBlock) {
this.logger.warn({ toBlock, blockNumber }, 'Returned block number is less than the specified toBlock')
callback(new Error('block number too low'), null)
const newPayload = [payload, { jsonrpc: '2.0', id: payload.id + 1, method: 'eth_blockNumber', params: [] }]
oldSend(newPayload, (err, res) => {
if (err) {
callback(err, null)
} else {
callback(null, rawLogs)
const rawLogs = res.find(({ id }) => id === payload.id)
const rawBlockNumber = res.find(({ id }) => id === payload.id + 1)
const blockNumber = hexToNumber(rawBlockNumber.result)
const toBlock = hexToNumber(payload.params[0].toBlock)
if (blockNumber < toBlock) {
this.logger.warn({ toBlock, blockNumber }, 'Returned block number is less than the specified toBlock')
callback(new Error('block number too low'), null)
} else {
callback(null, rawLogs)
}
}
}
})
} else {
this.subProvider.send(payload, callback)
})
} else {
oldSend(payload, callback)
}
}
provider.send = newSend.bind(provider)
return provider
}
module.exports = {

4
oracle/src/services/web3.js
View File

@ -38,10 +38,10 @@ const foreignOptions = {
retry: RETRY_CONFIG
}
const homeProvider = new SafeEthLogsProvider(new HttpListProvider(homeUrls, homeOptions))
const homeProvider = SafeEthLogsProvider(new HttpListProvider(homeUrls, homeOptions))
const web3Home = new Web3(homeProvider)
const foreignProvider = new SafeEthLogsProvider(new HttpListProvider(foreignUrls, foreignOptions))
const foreignProvider = SafeEthLogsProvider(new HttpListProvider(foreignUrls, foreignOptions))
const web3Foreign = new Web3(foreignProvider)
let web3ForeignArchive = null

1
oracle/src/utils/constants.js
View File

@ -25,6 +25,7 @@ module.exports = {
},
DEFAULT_TRANSACTION_RESEND_INTERVAL: 20 * 60 * 1000,
FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000,
BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT: 10,
SENDER_QUEUE_MAX_PRIORITY: 10,
SENDER_QUEUE_SEND_PRIORITY: 5,
SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1,

28
oracle/src/watcher.js
View File

@ -6,7 +6,7 @@ const logger = require('./services/logger')
const { getShutdownFlag } = require('./services/shutdownState')
const { getBlockNumber, getRequiredBlockConfirmations, getEvents } = require('./tx/web3')
const { checkHTTPS, watchdog } = require('./utils/utils')
const { EXIT_CODES } = require('./utils/constants')
const { EXIT_CODES, BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT } = require('./utils/constants')
if (process.argv.length < 3) {
logger.error('Please check the number of arguments, config file was not provided')
@ -29,12 +29,14 @@ const { getTokensState } = require('./utils/tokenState')
const { web3, bridgeContract, eventContract, startBlock, pollingInterval, chain } = config.main
const lastBlockRedisKey = `${config.id}:lastProcessedBlock`
let lastProcessedBlock = Math.max(startBlock - 1, 0)
let lastSeenBlockNumber = 0
let sameBlockNumberCounter = 0
async function initialize() {
try {
const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger)
web3.currentProvider.subProvider.urls.forEach(checkHttps(chain))
web3.currentProvider.urls.forEach(checkHttps(chain))
await getLastProcessedBlock()
connectWatcherToQueue({
@ -117,6 +119,28 @@ async function getLastBlockToProcess(web3, bridgeContract) {
getBlockNumber(web3),
getRequiredBlockConfirmations(bridgeContract)
])
if (lastBlockNumber < lastSeenBlockNumber) {
sameBlockNumberCounter = 0
logger.warn({ lastBlockNumber, lastSeenBlockNumber }, 'Received block number less than already seen block')
web3.currentProvider.switchToFallbackRPC()
} else if (lastBlockNumber === lastSeenBlockNumber) {
sameBlockNumberCounter++
if (sameBlockNumberCounter > 1) {
logger.info({ lastBlockNumber, sameBlockNumberCounter }, 'Received the same block number more than twice')
if (sameBlockNumberCounter >= BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT) {
sameBlockNumberCounter = 0
logger.warn(
{ lastBlockNumber, n: BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT },
'Received the same block number for too many times. Probably node is not synced anymore'
)
web3.currentProvider.switchToFallbackRPC()
}
}
} else {
sameBlockNumberCounter = 0
lastSeenBlockNumber = lastBlockNumber
}
return lastBlockNumber - requiredBlockConfirmations
}