diff --git a/deployment/roles/common/tasks/dependencies.yml b/deployment/roles/common/tasks/dependencies.yml index 4bc82c50..0c859c3d 100644 --- a/deployment/roles/common/tasks/dependencies.yml +++ b/deployment/roles/common/tasks/dependencies.yml @@ -33,7 +33,7 @@ mode: "0755" - name: Upgrade pip version - shell: pip3 install --upgrade pip + shell: pip3 install --upgrade pip==19.3.1 - name: Install python docker library shell: pip3 install docker docker-compose setuptools diff --git a/monitor/checkWorker3.js b/monitor/checkWorker3.js index 08ad3dad..c7c27160 100644 --- a/monitor/checkWorker3.js +++ b/monitor/checkWorker3.js @@ -2,6 +2,7 @@ require('dotenv').config() const logger = require('./logger')('checkWorker3') const stuckTransfers = require('./stuckTransfers') const detectMediators = require('./detectMediators') +const detectFailures = require('./detectFailures') const { writeFile, createDir } = require('./utils/file') const { web3Home } = require('./utils/web3') const { saveCache } = require('./utils/web3Cache') @@ -24,11 +25,19 @@ async function checkWorker3() { logger.debug('Done') } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) { createDir(`/responses/${MONITOR_BRIDGE_NAME}`) + logger.debug('calling detectMediators()') const mediators = await detectMediators(bridgeMode) mediators.ok = true mediators.health = true writeFile(`/responses/${MONITOR_BRIDGE_NAME}/mediators.json`, mediators) + + logger.debug('calling detectFailures()') + const failures = await detectFailures(bridgeMode) + failures.ok = true + failures.health = true + writeFile(`/responses/${MONITOR_BRIDGE_NAME}/failures.json`, failures) + saveCache() logger.debug('Done') } diff --git a/monitor/detectFailures.js b/monitor/detectFailures.js new file mode 100644 index 00000000..81d97b4d --- /dev/null +++ b/monitor/detectFailures.js @@ -0,0 +1,86 @@ +require('dotenv').config() +const logger = require('./logger')('alerts') +const eventsInfo = require('./utils/events') +const { normalizeAMBMessageEvent } = require('../commons') +const { getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3') + +function normalize(events) { + const requests = {} + events.forEach(event => { + const request = normalizeAMBMessageEvent(event) + request.requestTx = event.transactionHash + requests[request.messageId] = request + }) + return confirmation => { + const request = requests[confirmation.returnValues.messageId] || {} + return { + ...request, + status: false, + executionTx: confirmation.transactionHash, + executionBlockNumber: confirmation.blockNumber + } + } +} + +async function main(mode) { + const { + homeToForeignRequests, + homeToForeignConfirmations, + foreignToHomeConfirmations, + foreignToHomeRequests + } = await eventsInfo(mode) + const hasFailed = event => !event.returnValues.status + const cmp = (a, b) => b.executionBlockNumber - a.executionBlockNumber + const failedForeignToHomeMessages = foreignToHomeConfirmations + .filter(hasFailed) + .map(normalize(foreignToHomeRequests)) + .sort(cmp) + const failedHomeToForeignMessages = homeToForeignConfirmations + .filter(hasFailed) + .map(normalize(homeToForeignRequests)) + .sort(cmp) + + const homeBlockNumber = await getHomeBlockNumber() + const foreignBlockNumber = await getForeignBlockNumber() + + const blockRanges = [1000, 10000, 100000, 1000000] + const rangeNames = [ + `last${blockRanges[0]}blocks`, + ...blockRanges.slice(0, blockRanges.length - 1).map((n, i) => `last${n}to${blockRanges[i + 1]}blocks`), + `before${blockRanges[blockRanges.length - 1]}blocks` + ] + + const countFailures = (failedMessages, lastBlockNumber) => { + const result = {} + rangeNames.forEach(name => { + result[name] = 0 + }) + failedMessages.forEach(message => { + const blockAge = lastBlockNumber - message.executionBlockNumber + let rangeIndex = blockRanges.findIndex(n => n > blockAge) + if (rangeIndex === -1) { + rangeIndex = blockRanges.length + } + result[rangeNames[rangeIndex]] += 1 + }) + return result + } + + logger.debug('Done') + + return { + homeToForeign: { + total: failedHomeToForeignMessages.length, + stats: countFailures(failedHomeToForeignMessages, foreignBlockNumber), + lastFailures: failedHomeToForeignMessages.slice(0, 5) + }, + foreignToHome: { + total: failedForeignToHomeMessages.length, + stats: countFailures(failedForeignToHomeMessages, homeBlockNumber), + lastFailures: failedForeignToHomeMessages.slice(0, 5) + }, + lastChecked: Math.floor(Date.now() / 1000) + } +} + +module.exports = main diff --git a/monitor/index.js b/monitor/index.js index 3aaaa820..4d245ed6 100644 --- a/monitor/index.js +++ b/monitor/index.js @@ -11,9 +11,10 @@ app.use(cors()) app.get('/favicon.ico', (req, res) => res.sendStatus(204)) app.use('/:bridgeName', bridgeRouter) -bridgeRouter.get('/', async (req, res, next) => { +bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', async (req, res, next) => { try { - const results = await readFile(`./responses/${req.params.bridgeName}/getBalances.json`) + const { bridgeName, file } = req.params + const results = await readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`) res.json(results) } catch (e) { // this will eventually be handled by your error handling middleware @@ -21,54 +22,6 @@ bridgeRouter.get('/', async (req, res, next) => { } }) -bridgeRouter.get('/validators', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/validators.json`) - res.json(results) - } catch (e) { - // this will eventually be handled by your error handling middleware - next(e) - } -}) - -bridgeRouter.get('/eventsStats', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/eventsStats.json`) - res.json(results) - } catch (e) { - // this will eventually be handled by your error handling middleware - next(e) - } -}) - -bridgeRouter.get('/alerts', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/alerts.json`) - res.json(results) - } catch (e) { - next(e) - } -}) - -bridgeRouter.get('/mediators', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/mediators.json`) - res.json(results) - } catch (e) { - // this will eventually be handled by your error handling middleware - next(e) - } -}) - -bridgeRouter.get('/stuckTransfers', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/stuckTransfers.json`) - res.json(results) - } catch (e) { - next(e) - } -}) - const port = process.env.MONITOR_PORT || 3003 app.set('port', port) app.listen(port, () => console.log(`Monitoring app listening on port ${port}!`))