Add monitor script for detecting failed AMB messages (#513)
This commit is contained in:
parent
e1536755f4
commit
894134ba26
@ -33,7 +33,7 @@
|
|||||||
mode: "0755"
|
mode: "0755"
|
||||||
|
|
||||||
- name: Upgrade pip version
|
- name: Upgrade pip version
|
||||||
shell: pip3 install --upgrade pip
|
shell: pip3 install --upgrade pip==19.3.1
|
||||||
|
|
||||||
- name: Install python docker library
|
- name: Install python docker library
|
||||||
shell: pip3 install docker docker-compose setuptools
|
shell: pip3 install docker docker-compose setuptools
|
||||||
|
@ -2,6 +2,7 @@ require('dotenv').config()
|
|||||||
const logger = require('./logger')('checkWorker3')
|
const logger = require('./logger')('checkWorker3')
|
||||||
const stuckTransfers = require('./stuckTransfers')
|
const stuckTransfers = require('./stuckTransfers')
|
||||||
const detectMediators = require('./detectMediators')
|
const detectMediators = require('./detectMediators')
|
||||||
|
const detectFailures = require('./detectFailures')
|
||||||
const { writeFile, createDir } = require('./utils/file')
|
const { writeFile, createDir } = require('./utils/file')
|
||||||
const { web3Home } = require('./utils/web3')
|
const { web3Home } = require('./utils/web3')
|
||||||
const { saveCache } = require('./utils/web3Cache')
|
const { saveCache } = require('./utils/web3Cache')
|
||||||
@ -24,11 +25,19 @@ async function checkWorker3() {
|
|||||||
logger.debug('Done')
|
logger.debug('Done')
|
||||||
} else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
|
} else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
|
||||||
createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
|
createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
|
||||||
|
|
||||||
logger.debug('calling detectMediators()')
|
logger.debug('calling detectMediators()')
|
||||||
const mediators = await detectMediators(bridgeMode)
|
const mediators = await detectMediators(bridgeMode)
|
||||||
mediators.ok = true
|
mediators.ok = true
|
||||||
mediators.health = true
|
mediators.health = true
|
||||||
writeFile(`/responses/${MONITOR_BRIDGE_NAME}/mediators.json`, mediators)
|
writeFile(`/responses/${MONITOR_BRIDGE_NAME}/mediators.json`, mediators)
|
||||||
|
|
||||||
|
logger.debug('calling detectFailures()')
|
||||||
|
const failures = await detectFailures(bridgeMode)
|
||||||
|
failures.ok = true
|
||||||
|
failures.health = true
|
||||||
|
writeFile(`/responses/${MONITOR_BRIDGE_NAME}/failures.json`, failures)
|
||||||
|
|
||||||
saveCache()
|
saveCache()
|
||||||
logger.debug('Done')
|
logger.debug('Done')
|
||||||
}
|
}
|
||||||
|
86
monitor/detectFailures.js
Normal file
86
monitor/detectFailures.js
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
require('dotenv').config()
|
||||||
|
const logger = require('./logger')('alerts')
|
||||||
|
const eventsInfo = require('./utils/events')
|
||||||
|
const { normalizeAMBMessageEvent } = require('../commons')
|
||||||
|
const { getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3')
|
||||||
|
|
||||||
|
function normalize(events) {
|
||||||
|
const requests = {}
|
||||||
|
events.forEach(event => {
|
||||||
|
const request = normalizeAMBMessageEvent(event)
|
||||||
|
request.requestTx = event.transactionHash
|
||||||
|
requests[request.messageId] = request
|
||||||
|
})
|
||||||
|
return confirmation => {
|
||||||
|
const request = requests[confirmation.returnValues.messageId] || {}
|
||||||
|
return {
|
||||||
|
...request,
|
||||||
|
status: false,
|
||||||
|
executionTx: confirmation.transactionHash,
|
||||||
|
executionBlockNumber: confirmation.blockNumber
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main(mode) {
|
||||||
|
const {
|
||||||
|
homeToForeignRequests,
|
||||||
|
homeToForeignConfirmations,
|
||||||
|
foreignToHomeConfirmations,
|
||||||
|
foreignToHomeRequests
|
||||||
|
} = await eventsInfo(mode)
|
||||||
|
const hasFailed = event => !event.returnValues.status
|
||||||
|
const cmp = (a, b) => b.executionBlockNumber - a.executionBlockNumber
|
||||||
|
const failedForeignToHomeMessages = foreignToHomeConfirmations
|
||||||
|
.filter(hasFailed)
|
||||||
|
.map(normalize(foreignToHomeRequests))
|
||||||
|
.sort(cmp)
|
||||||
|
const failedHomeToForeignMessages = homeToForeignConfirmations
|
||||||
|
.filter(hasFailed)
|
||||||
|
.map(normalize(homeToForeignRequests))
|
||||||
|
.sort(cmp)
|
||||||
|
|
||||||
|
const homeBlockNumber = await getHomeBlockNumber()
|
||||||
|
const foreignBlockNumber = await getForeignBlockNumber()
|
||||||
|
|
||||||
|
const blockRanges = [1000, 10000, 100000, 1000000]
|
||||||
|
const rangeNames = [
|
||||||
|
`last${blockRanges[0]}blocks`,
|
||||||
|
...blockRanges.slice(0, blockRanges.length - 1).map((n, i) => `last${n}to${blockRanges[i + 1]}blocks`),
|
||||||
|
`before${blockRanges[blockRanges.length - 1]}blocks`
|
||||||
|
]
|
||||||
|
|
||||||
|
const countFailures = (failedMessages, lastBlockNumber) => {
|
||||||
|
const result = {}
|
||||||
|
rangeNames.forEach(name => {
|
||||||
|
result[name] = 0
|
||||||
|
})
|
||||||
|
failedMessages.forEach(message => {
|
||||||
|
const blockAge = lastBlockNumber - message.executionBlockNumber
|
||||||
|
let rangeIndex = blockRanges.findIndex(n => n > blockAge)
|
||||||
|
if (rangeIndex === -1) {
|
||||||
|
rangeIndex = blockRanges.length
|
||||||
|
}
|
||||||
|
result[rangeNames[rangeIndex]] += 1
|
||||||
|
})
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug('Done')
|
||||||
|
|
||||||
|
return {
|
||||||
|
homeToForeign: {
|
||||||
|
total: failedHomeToForeignMessages.length,
|
||||||
|
stats: countFailures(failedHomeToForeignMessages, foreignBlockNumber),
|
||||||
|
lastFailures: failedHomeToForeignMessages.slice(0, 5)
|
||||||
|
},
|
||||||
|
foreignToHome: {
|
||||||
|
total: failedForeignToHomeMessages.length,
|
||||||
|
stats: countFailures(failedForeignToHomeMessages, homeBlockNumber),
|
||||||
|
lastFailures: failedForeignToHomeMessages.slice(0, 5)
|
||||||
|
},
|
||||||
|
lastChecked: Math.floor(Date.now() / 1000)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = main
|
@ -11,9 +11,10 @@ app.use(cors())
|
|||||||
app.get('/favicon.ico', (req, res) => res.sendStatus(204))
|
app.get('/favicon.ico', (req, res) => res.sendStatus(204))
|
||||||
app.use('/:bridgeName', bridgeRouter)
|
app.use('/:bridgeName', bridgeRouter)
|
||||||
|
|
||||||
bridgeRouter.get('/', async (req, res, next) => {
|
bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', async (req, res, next) => {
|
||||||
try {
|
try {
|
||||||
const results = await readFile(`./responses/${req.params.bridgeName}/getBalances.json`)
|
const { bridgeName, file } = req.params
|
||||||
|
const results = await readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`)
|
||||||
res.json(results)
|
res.json(results)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
// this will eventually be handled by your error handling middleware
|
// this will eventually be handled by your error handling middleware
|
||||||
@ -21,54 +22,6 @@ bridgeRouter.get('/', async (req, res, next) => {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
bridgeRouter.get('/validators', async (req, res, next) => {
|
|
||||||
try {
|
|
||||||
const results = await readFile(`./responses/${req.params.bridgeName}/validators.json`)
|
|
||||||
res.json(results)
|
|
||||||
} catch (e) {
|
|
||||||
// this will eventually be handled by your error handling middleware
|
|
||||||
next(e)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
bridgeRouter.get('/eventsStats', async (req, res, next) => {
|
|
||||||
try {
|
|
||||||
const results = await readFile(`./responses/${req.params.bridgeName}/eventsStats.json`)
|
|
||||||
res.json(results)
|
|
||||||
} catch (e) {
|
|
||||||
// this will eventually be handled by your error handling middleware
|
|
||||||
next(e)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
bridgeRouter.get('/alerts', async (req, res, next) => {
|
|
||||||
try {
|
|
||||||
const results = await readFile(`./responses/${req.params.bridgeName}/alerts.json`)
|
|
||||||
res.json(results)
|
|
||||||
} catch (e) {
|
|
||||||
next(e)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
bridgeRouter.get('/mediators', async (req, res, next) => {
|
|
||||||
try {
|
|
||||||
const results = await readFile(`./responses/${req.params.bridgeName}/mediators.json`)
|
|
||||||
res.json(results)
|
|
||||||
} catch (e) {
|
|
||||||
// this will eventually be handled by your error handling middleware
|
|
||||||
next(e)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
bridgeRouter.get('/stuckTransfers', async (req, res, next) => {
|
|
||||||
try {
|
|
||||||
const results = await readFile(`./responses/${req.params.bridgeName}/stuckTransfers.json`)
|
|
||||||
res.json(results)
|
|
||||||
} catch (e) {
|
|
||||||
next(e)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
const port = process.env.MONITOR_PORT || 3003
|
const port = process.env.MONITOR_PORT || 3003
|
||||||
app.set('port', port)
|
app.set('port', port)
|
||||||
app.listen(port, () => console.log(`Monitoring app listening on port ${port}!`))
|
app.listen(port, () => console.log(`Monitoring app listening on port ${port}!`))
|
||||||
|
Loading…
Reference in New Issue
Block a user