Add monitor script for detecting failed AMB messages (#513)

deployment/roles/common/tasks/dependencies.yml

@@ -33,7 +33,7 @@
     mode: "0755"
 
 - name: Upgrade pip version
-  shell: pip3 install --upgrade pip
+  shell: pip3 install --upgrade pip==19.3.1
 
 - name: Install python docker library
   shell: pip3 install docker docker-compose setuptools

monitor/checkWorker3.js

@@ -2,6 +2,7 @@ require('dotenv').config()
 const logger = require('./logger')('checkWorker3')
 const stuckTransfers = require('./stuckTransfers')
 const detectMediators = require('./detectMediators')
+const detectFailures = require('./detectFailures')
 const { writeFile, createDir } = require('./utils/file')
 const { web3Home } = require('./utils/web3')
 const { saveCache } = require('./utils/web3Cache')
@@ -24,11 +25,19 @@ async function checkWorker3() {
       logger.debug('Done')
     } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
       createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
+
       logger.debug('calling detectMediators()')
       const mediators = await detectMediators(bridgeMode)
       mediators.ok = true
       mediators.health = true
       writeFile(`/responses/${MONITOR_BRIDGE_NAME}/mediators.json`, mediators)
+
+      logger.debug('calling detectFailures()')
+      const failures = await detectFailures(bridgeMode)
+      failures.ok = true
+      failures.health = true
+      writeFile(`/responses/${MONITOR_BRIDGE_NAME}/failures.json`, failures)
+
       saveCache()
       logger.debug('Done')
     }

monitor/detectFailures.js

@@ -0,0 +1,86 @@
+require('dotenv').config()
+const logger = require('./logger')('alerts')
+const eventsInfo = require('./utils/events')
+const { normalizeAMBMessageEvent } = require('../commons')
+const { getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3')
+
+function normalize(events) {
+  const requests = {}
+  events.forEach(event => {
+    const request = normalizeAMBMessageEvent(event)
+    request.requestTx = event.transactionHash
+    requests[request.messageId] = request
+  })
+  return confirmation => {
+    const request = requests[confirmation.returnValues.messageId] || {}
+    return {
+      ...request,
+      status: false,
+      executionTx: confirmation.transactionHash,
+      executionBlockNumber: confirmation.blockNumber
+    }
+  }
+}
+
+async function main(mode) {
+  const {
+    homeToForeignRequests,
+    homeToForeignConfirmations,
+    foreignToHomeConfirmations,
+    foreignToHomeRequests
+  } = await eventsInfo(mode)
+  const hasFailed = event => !event.returnValues.status
+  const cmp = (a, b) => b.executionBlockNumber - a.executionBlockNumber
+  const failedForeignToHomeMessages = foreignToHomeConfirmations
+    .filter(hasFailed)
+    .map(normalize(foreignToHomeRequests))
+    .sort(cmp)
+  const failedHomeToForeignMessages = homeToForeignConfirmations
+    .filter(hasFailed)
+    .map(normalize(homeToForeignRequests))
+    .sort(cmp)
+
+  const homeBlockNumber = await getHomeBlockNumber()
+  const foreignBlockNumber = await getForeignBlockNumber()
+
+  const blockRanges = [1000, 10000, 100000, 1000000]
+  const rangeNames = [
+    `last${blockRanges[0]}blocks`,
+    ...blockRanges.slice(0, blockRanges.length - 1).map((n, i) => `last${n}to${blockRanges[i + 1]}blocks`),
+    `before${blockRanges[blockRanges.length - 1]}blocks`
+  ]
+
+  const countFailures = (failedMessages, lastBlockNumber) => {
+    const result = {}
+    rangeNames.forEach(name => {
+      result[name] = 0
+    })
+    failedMessages.forEach(message => {
+      const blockAge = lastBlockNumber - message.executionBlockNumber
+      let rangeIndex = blockRanges.findIndex(n => n > blockAge)
+      if (rangeIndex === -1) {
+        rangeIndex = blockRanges.length
+      }
+      result[rangeNames[rangeIndex]] += 1
+    })
+    return result
+  }
+
+  logger.debug('Done')
+
+  return {
+    homeToForeign: {
+      total: failedHomeToForeignMessages.length,
+      stats: countFailures(failedHomeToForeignMessages, foreignBlockNumber),
+      lastFailures: failedHomeToForeignMessages.slice(0, 5)
+    },
+    foreignToHome: {
+      total: failedForeignToHomeMessages.length,
+      stats: countFailures(failedForeignToHomeMessages, homeBlockNumber),
+      lastFailures: failedForeignToHomeMessages.slice(0, 5)
+    },
+    lastChecked: Math.floor(Date.now() / 1000)
+  }
+}
+
+module.exports = main

monitor/index.js

@@ -11,9 +11,10 @@ app.use(cors())
 app.get('/favicon.ico', (req, res) => res.sendStatus(204))
 app.use('/:bridgeName', bridgeRouter)
 
-bridgeRouter.get('/', async (req, res, next) => {
+bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', async (req, res, next) => {
   try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/getBalances.json`)
+    const { bridgeName, file } = req.params
+    const results = await readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`)
     res.json(results)
   } catch (e) {
     // this will eventually be handled by your error handling middleware
@@ -21,54 +22,6 @@ bridgeRouter.get('/', async (req, res, next) => {
   }
 })
 
-bridgeRouter.get('/validators', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/validators.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/eventsStats', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/eventsStats.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/alerts', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/alerts.json`)
-    res.json(results)
-  } catch (e) {
-    next(e)
-  }
-})
-
-bridgeRouter.get('/mediators', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/mediators.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/stuckTransfers', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/stuckTransfers.json`)
-    res.json(results)
-  } catch (e) {
-    next(e)
-  }
-})
-
 const port = process.env.MONITOR_PORT || 3003
 app.set('port', port)
 app.listen(port, () => console.log(`Monitoring app listening on port ${port}!`))