From f64f8b1c91182c89bfffc0ceeb0f79e23c998eed Mon Sep 17 00:00:00 2001 From: Leonid Tyurin Date: Fri, 26 Feb 2021 05:39:48 +0300 Subject: [PATCH] Add /metrics endpoint for prometheus support (#512) --- monitor/index.js | 14 ++++- monitor/prometheusMetrics.js | 104 +++++++++++++++++++++++++++++++++++ monitor/utils/file.js | 4 +- 3 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 monitor/prometheusMetrics.js diff --git a/monitor/index.js b/monitor/index.js index 4d245ed6..c2f3d6f3 100644 --- a/monitor/index.js +++ b/monitor/index.js @@ -2,6 +2,7 @@ require('dotenv').config() const express = require('express') const cors = require('cors') const { readFile } = require('./utils/file') +const { getPrometheusMetrics } = require('./prometheusMetrics') const app = express() const bridgeRouter = express.Router({ mergeParams: true }) @@ -11,10 +12,10 @@ app.use(cors()) app.get('/favicon.ico', (req, res) => res.sendStatus(204)) app.use('/:bridgeName', bridgeRouter) -bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', async (req, res, next) => { +bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', (req, res, next) => { try { const { bridgeName, file } = req.params - const results = await readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`) + const results = readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`) res.json(results) } catch (e) { // this will eventually be handled by your error handling middleware @@ -22,6 +23,15 @@ bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers| } }) +bridgeRouter.get('/metrics', (req, res, next) => { + try { + const metrics = getPrometheusMetrics(req.params.bridgeName) + res.type('text').send(metrics) + } catch (e) { + next(e) + } +}) + const port = process.env.MONITOR_PORT || 3003 app.set('port', port) app.listen(port, () => console.log(`Monitoring app listening on port ${port}!`)) diff --git a/monitor/prometheusMetrics.js b/monitor/prometheusMetrics.js new file mode 100644 index 00000000..af649abc --- /dev/null +++ b/monitor/prometheusMetrics.js @@ -0,0 +1,104 @@ +const { readFile } = require('./utils/file') + +const { + MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST, + MONITOR_HOME_TO_FOREIGN_BLOCK_LIST, + MONITOR_HOME_VALIDATORS_BALANCE_ENABLE, + MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE +} = process.env + +function BridgeConf(type, validatorsBalanceEnable, alertTargetFunc, failureDirection) { + this.type = type + this.validatorsBalanceEnable = validatorsBalanceEnable + this.alertTargetFunc = alertTargetFunc + this.failureDirection = failureDirection +} + +const BRIDGE_CONFS = [ + new BridgeConf('home', MONITOR_HOME_VALIDATORS_BALANCE_ENABLE, 'executeAffirmations', 'homeToForeign'), + new BridgeConf('foreign', MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE, 'executeSignatures', 'foreignToHome') +] + +function hasError(obj) { + return 'error' in obj +} + +function getPrometheusMetrics(bridgeName) { + const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json` + + const metrics = {} + + // Balance metrics + const balancesFile = readFile(responsePath('getBalances')) + + if (!hasError(balancesFile)) { + const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile + metrics.balances_home_value = homeBalances.totalSupply + metrics.balances_home_txs_deposit = homeBalances.deposits + metrics.balances_home_txs_withdrawal = homeBalances.withdrawals + + metrics.balances_foreign_value = foreignBalances.erc20Balance + metrics.balances_foreign_txs_deposit = foreignBalances.deposits + metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals + + metrics.balances_diff_value = commonBalances.balanceDiff + metrics.balances_diff_deposit = commonBalances.depositsDiff + metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff + if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) { + metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff + metrics.balances_unclaimed_value = commonBalances.unclaimedBalance + } + } + + // Validator metrics + const validatorsFile = readFile(responsePath('validators')) + + if (!hasError(validatorsFile)) { + for (const bridge of BRIDGE_CONFS) { + const allValidators = validatorsFile[bridge.type].validators + const validatorAddressesWithBalanceCheck = + typeof bridge.validatorsBalanceEnable === 'string' + ? bridge.validatorsBalanceEnable.split(' ') + : Object.keys(allValidators) + + validatorAddressesWithBalanceCheck.forEach((addr, ind) => { + metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance + }) + } + } + + // Alert metrics + const alertsFile = readFile(responsePath('alerts')) + + if (!hasError(alertsFile)) { + for (const bridge of BRIDGE_CONFS) { + Object.entries(alertsFile[bridge.alertTargetFunc].misbehavior).forEach(([period, val]) => { + metrics[`misbehavior_${bridge.type}_${period}`] = val + }) + } + } + + // Failure metrics + const failureFile = readFile(responsePath('failures')) + + if (!hasError(failureFile)) { + for (const bridge of BRIDGE_CONFS) { + const dir = bridge.failureDirection + const failures = failureFile[dir] + metrics[`failures_${dir}_total`] = failures.total + Object.entries(failures.stats).forEach(([period, count]) => { + metrics[`failures_${dir}_${period}`] = count + }) + } + } + + // Pack metrcis into a plain text + return Object.entries(metrics).reduceRight( + // Prometheus supports `Nan` and possibly signed `Infinity` + // in case cast to `Number` fails + (acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`, + '' + ) +} + +module.exports = { getPrometheusMetrics } diff --git a/monitor/utils/file.js b/monitor/utils/file.js index ad9f25f6..fc957fa8 100644 --- a/monitor/utils/file.js +++ b/monitor/utils/file.js @@ -1,9 +1,9 @@ const fs = require('fs') const path = require('path') -async function readFile(filePath) { +function readFile(filePath) { try { - const content = await fs.readFileSync(filePath) + const content = fs.readFileSync(filePath) const json = JSON.parse(content) const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked return Object.assign({}, json, { timeDiff })