Fix monitor metrics (#533)

monitor/alerts.js

@@ -1,18 +1,19 @@
 require('dotenv').config()
 const logger = require('./logger')('alerts')
-const eventsInfo = require('./utils/events')
 const { processedMsgNotDelivered, eventWithoutReference } = require('./utils/message')
 const { BRIDGE_MODES } = require('../commons')
-const { web3Home, web3Foreign, getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3')
+const { web3Home, web3Foreign } = require('./utils/web3')
 
-async function main() {
+async function main(eventsInfo) {
   const {
+    homeBlockNumber,
+    foreignBlockNumber,
     homeToForeignRequests,
     homeToForeignConfirmations,
     foreignToHomeConfirmations,
     foreignToHomeRequests,
     bridgeMode
-  } = await eventsInfo()
+  } = eventsInfo
 
   let xSignatures
   let xAffirmations
@@ -24,8 +25,6 @@ async function main() {
     xAffirmations = foreignToHomeConfirmations.filter(eventWithoutReference(foreignToHomeRequests))
   }
   logger.debug('building misbehavior blocks')
-  const homeBlockNumber = await getHomeBlockNumber()
-  const foreignBlockNumber = await getForeignBlockNumber()
 
   const baseRange = [false, false, false, false, false]
   const xSignaturesMisbehavior = buildRangesObject(

monitor/checkWorker2.js

@@ -1,6 +1,7 @@
 require('dotenv').config()
 const logger = require('./logger')('checkWorker2')
 const eventsStats = require('./eventsStats')
+const getEventsInfo = require('./utils/events')
 const alerts = require('./alerts')
 const { writeFile, createDir } = require('./utils/file')
 const { saveCache } = require('./utils/web3Cache')
@@ -10,8 +11,10 @@ const { MONITOR_BRIDGE_NAME } = process.env
 async function checkWorker2() {
   try {
     createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
+    logger.debug('calling getEventsInfo()')
+    const eventsInfo = await getEventsInfo()
     logger.debug('calling eventsStats()')
-    const evStats = await eventsStats()
+    const evStats = await eventsStats(eventsInfo)
     if (!evStats) throw new Error('evStats is empty: ' + JSON.stringify(evStats))
     evStats.ok =
       (evStats.onlyInHomeDeposits || evStats.home.deliveredMsgNotProcessedInForeign).length === 0 &&
@@ -22,7 +25,7 @@ async function checkWorker2() {
     writeFile(`/responses/${MONITOR_BRIDGE_NAME}/eventsStats.json`, evStats)
 
     logger.debug('calling alerts()')
-    const _alerts = await alerts()
+    const _alerts = await alerts(eventsInfo)
     if (!_alerts) throw new Error('alerts is empty: ' + JSON.stringify(_alerts))
     _alerts.ok = !_alerts.executeAffirmations.mostRecentTxHash && !_alerts.executeSignatures.mostRecentTxHash
     _alerts.health = true

monitor/eventsStats.js

@@ -1,5 +1,4 @@
 require('dotenv').config()
-const eventsInfo = require('./utils/events')
 const {
   processedMsgNotDelivered,
   deliveredMsgNotProcessed,
@@ -15,14 +14,14 @@ const {
   MONITOR_HOME_TO_FOREIGN_CHECK_SENDER
 } = process.env
 
-async function main() {
+async function main(eventsInfo) {
   const {
     homeToForeignRequests,
     homeToForeignConfirmations,
     foreignToHomeConfirmations,
     foreignToHomeRequests,
     bridgeMode
-  } = await eventsInfo()
+  } = eventsInfo
 
   if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
     return {

monitor/getBalances.js

@@ -5,7 +5,12 @@ const logger = require('./logger')('getBalances')
 const { BRIDGE_MODES } = require('../commons')
 const { web3Home, web3Foreign, getHomeBlockNumber } = require('./utils/web3')
 
-const { COMMON_HOME_BRIDGE_ADDRESS, COMMON_FOREIGN_BRIDGE_ADDRESS } = process.env
+const {
+  MONITOR_HOME_START_BLOCK,
+  MONITOR_FOREIGN_START_BLOCK,
+  COMMON_HOME_BRIDGE_ADDRESS,
+  COMMON_FOREIGN_BRIDGE_ADDRESS
+} = process.env
 
 const {
   ERC20_ABI,
@@ -20,6 +25,8 @@ const {
 
 async function main(bridgeMode, eventsInfo) {
   const {
+    homeBlockNumber,
+    foreignBlockNumber,
     homeToForeignConfirmations,
     foreignToHomeConfirmations,
     homeDelayedBlockNumber,
@@ -46,6 +53,13 @@ async function main(bridgeMode, eventsInfo) {
     ...foreignToHomeConfirmations.filter(e => e.blockNumber > homeDelayedBlockNumber).map(e => e.value)
   )
 
+  const blockRanges = {
+    startBlockHome: MONITOR_HOME_START_BLOCK,
+    endBlockHome: homeBlockNumber,
+    startBlockForeign: MONITOR_FOREIGN_START_BLOCK,
+    endBlockForeign: foreignBlockNumber
+  }
+
   if (bridgeMode === BRIDGE_MODES.ERC_TO_ERC) {
     const foreignBridge = new web3Foreign.eth.Contract(FOREIGN_ERC_TO_ERC_ABI, COMMON_FOREIGN_BRIDGE_ADDRESS)
     const erc20Address = await foreignBridge.methods.erc20token().call()
@@ -72,6 +86,7 @@ async function main(bridgeMode, eventsInfo) {
         erc20Balance: Web3Utils.fromWei(foreignErc20Balance)
       },
       balanceDiff: Number(Web3Utils.fromWei(diff)),
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else if (bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC || bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC_V1) {
@@ -94,6 +109,7 @@ async function main(bridgeMode, eventsInfo) {
         totalSupply: Web3Utils.fromWei(totalSupply)
       },
       balanceDiff: Number(Web3Utils.fromWei(diff)),
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else if (bridgeMode === BRIDGE_MODES.ERC_TO_NATIVE) {
@@ -163,12 +179,14 @@ async function main(bridgeMode, eventsInfo) {
       },
       foreign,
       balanceDiff: Number(Web3Utils.fromWei(diff)),
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
     return {
       home: {},
       foreign: {},
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else {

monitor/index.js

@@ -2,7 +2,6 @@ require('dotenv').config()
 const express = require('express')
 const cors = require('cors')
 const { readFile } = require('./utils/file')
-const { getPrometheusMetrics } = require('./prometheusMetrics')
 
 const app = express()
 const bridgeRouter = express.Router({ mergeParams: true })
@@ -25,7 +24,8 @@ bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|
 
 bridgeRouter.get('/metrics', (req, res, next) => {
   try {
-    const metrics = getPrometheusMetrics(req.params.bridgeName)
+    const { bridgeName } = req.params
+    const metrics = readFile(`./responses/${bridgeName}/metrics.txt`, false)
     res.type('text').send(metrics)
   } catch (e) {
     next(e)

monitor/metricsWorker.js

@@ -0,0 +1,20 @@
+require('dotenv').config()
+const logger = require('./logger')('metricsWorker')
+const { writeFile, createDir } = require('./utils/file')
+const getPrometheusMetrics = require('./prometheusMetrics')
+
+const { MONITOR_BRIDGE_NAME } = process.env
+
+async function metricsWorker() {
+  try {
+    createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
+    logger.debug('calling getPrometheusMetrics()')
+    const metrics = await getPrometheusMetrics(MONITOR_BRIDGE_NAME)
+    if (!metrics) throw new Error('metrics is empty: ' + JSON.stringify(metrics))
+    writeFile(`/responses/${MONITOR_BRIDGE_NAME}/metrics.txt`, metrics, { stringify: false })
+    logger.debug('Done')
+  } catch (e) {
+    logger.error(e)
+  }
+}
+metricsWorker()

monitor/package.json

@@ -4,7 +4,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js",
+    "check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js && timeout -s 9 10s node metricsWorker.js",
     "start": "node index.js",
     "check-and-start": "yarn check-all && yarn start",
     "lint": "eslint . --ignore-path ../.eslintignore",

monitor/prometheusMetrics.js

@@ -1,8 +1,10 @@
+require('dotenv').config()
+const logger = require('./logger')('getBalances')
 const { readFile } = require('./utils/file')
 
 const {
-  MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST,
+  MONITOR_HOME_START_BLOCK,
-  MONITOR_HOME_TO_FOREIGN_BLOCK_LIST,
+  MONITOR_FOREIGN_START_BLOCK,
   MONITOR_HOME_VALIDATORS_BALANCE_ENABLE,
   MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE
 } = process.env
@@ -23,7 +25,9 @@ function hasError(obj) {
   return 'error' in obj
 }
 
-function getPrometheusMetrics(bridgeName) {
+// Try to collect all metrics from JSON responses and then
+// discard all unsuccessfully retrieved ones
+async function getPrometheusMetrics(bridgeName) {
   const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json`
 
   const metrics = {}
@@ -32,22 +36,44 @@ function getPrometheusMetrics(bridgeName) {
   const balancesFile = readFile(responsePath('getBalances'))
 
   if (!hasError(balancesFile)) {
-    const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile
+    const { home, foreign, ...commonBalances } = balancesFile
-    metrics.balances_home_value = homeBalances.totalSupply
-    metrics.balances_home_txs_deposit = homeBalances.deposits
-    metrics.balances_home_txs_withdrawal = homeBalances.withdrawals
 
-    metrics.balances_foreign_value = foreignBalances.erc20Balance
+    const balanceMetrics = {
-    metrics.balances_foreign_txs_deposit = foreignBalances.deposits
+      // ERC_TO_ERC or ERC_TO_NATIVE mode
-    metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals
+      balances_home_value: home.totalSupply,
+      balances_home_txs_deposit: home.deposits,
+      balances_home_txs_withdrawal: home.withdrawals,
+      balances_foreign_value: foreign.erc20Balance,
+      balances_foreign_txs_deposit: foreign.deposits,
+      balances_foreign_txs_withdrawal: foreign.withdrawals,
 
-    metrics.balances_diff_value = commonBalances.balanceDiff
+      // Not ARBITRARY_MESSAGE mode
-    metrics.balances_diff_deposit = commonBalances.depositsDiff
+      balances_diff_value: commonBalances.balanceDiff,
-    metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff
+      balances_diff_deposit: commonBalances.depositsDiff,
-    if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) {
+      balances_diff_withdrawal: commonBalances.withdrawalDiff,
-      metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff
+
-      metrics.balances_unclaimed_value = commonBalances.unclaimedBalance
+      // MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST or MONITOR_HOME_TO_FOREIGN_BLOCK_LIST is set
+      balances_unclaimed_txs: commonBalances.unclaimedDiff,
+      balances_unclaimed_value: commonBalances.unclaimedBalance,
+
+      // ARBITRARY_MESSAGE mode
+      txs_home_out: home.toForeign,
+      txs_home_in: home.fromForeign,
+      txs_foreign_out: foreign.toHome,
+      txs_foreign_in: foreign.fromHome,
+      txs_diff_home_out_oracles: commonBalances.fromHomeToForeignDiff,
+      txs_diff_home_out_users: commonBalances.fromHomeToForeignPBUDiff,
+      txs_diff_foreign_out: commonBalances.fromForeignToHomeDiff
     }
+
+    const blockRanges = {
+      state_startblock_home: commonBalances.startBlockHome || MONITOR_HOME_START_BLOCK,
+      state_startblock_foreign: commonBalances.startBlockForeign || MONITOR_FOREIGN_START_BLOCK,
+      state_endblock_home: commonBalances.endBlockHome,
+      state_endblock_foreign: commonBalances.endBlockForeign
+    }
+
+    Object.assign(metrics, blockRanges, balanceMetrics)
   }
 
   // Validator metrics
@@ -62,7 +88,11 @@ function getPrometheusMetrics(bridgeName) {
           : Object.keys(allValidators)
 
       validatorAddressesWithBalanceCheck.forEach((addr, ind) => {
+        if (addr in allValidators) {
           metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance
+        } else {
+          logger.debug(`Nonexistent validator address ${addr}`)
+        }
       })
     }
   }
@@ -96,9 +126,12 @@ function getPrometheusMetrics(bridgeName) {
   return Object.entries(metrics).reduceRight(
     // Prometheus supports `Nan` and possibly signed `Infinity`
     // in case cast to `Number` fails
-    (acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`,
+    (acc, [key, val]) => {
+      if (typeof val === 'undefined') return acc
+      else return `${key} ${Number(val)}\n${acc}`
+    },
     ''
   )
 }
 
-module.exports = { getPrometheusMetrics }
+module.exports = getPrometheusMetrics

monitor/utils/events.js

@@ -236,13 +236,15 @@ async function main(mode) {
     foreignToHomeRequests,
     isExternalErc20,
     bridgeMode,
+    homeBlockNumber,
+    foreignBlockNumber,
     homeDelayedBlockNumber,
     foreignDelayedBlockNumber
   }
 
   if (MONITOR_CACHE_EVENTS === 'true') {
     logger.debug('saving obtained events into cache file')
-    writeFile(cacheFilePath, result, false)
+    writeFile(cacheFilePath, result, { useCwd: false })
   }
   return result
 }

monitor/utils/file.js

@@ -1,23 +1,30 @@
 const fs = require('fs')
 const path = require('path')
 
-function readFile(filePath) {
+function readFile(filePath, parseJson = true) {
   try {
     const content = fs.readFileSync(filePath)
+    if (!parseJson) return content
     const json = JSON.parse(content)
     const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked
     return Object.assign({}, json, { timeDiff })
   } catch (e) {
-    console.error(e)
+    console.error('readFlle', e)
     return {
       error: 'the bridge statistics are not available'
     }
   }
 }
 
-function writeFile(filePath, object, useCwd = true) {
+function writeFile(filePath, object, paramOptions = {}) {
+  const defaultOptions = {
+    useCwd: true,
+    stringify: true
+  }
+  const { useCwd, stringify } = Object.assign({}, defaultOptions, paramOptions)
+
   const fullPath = useCwd ? path.join(process.cwd(), filePath) : filePath
-  fs.writeFileSync(fullPath, JSON.stringify(object, null, 4))
+  fs.writeFileSync(fullPath, stringify ? JSON.stringify(object, null, 4) : object)
 }
 
 function createDir(dirPath) {