tornadocontrib/tokenbridge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix monitor metrics (#533)

Browse Source
This commit is contained in:
Leonid Tyurin 2021-04-02 12:59:49 +03:00 committed by GitHub
parent dc3026e584
commit ae83c76be9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 120 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
monitor/alerts.js
View File

@ -1,18 +1,19 @@
require('dotenv').config()
const logger = require('./logger')('alerts')
const eventsInfo = require('./utils/events')
const { processedMsgNotDelivered, eventWithoutReference } = require('./utils/message')
const { BRIDGE_MODES } = require('../commons')
const { web3Home, web3Foreign, getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3')
const { web3Home, web3Foreign } = require('./utils/web3')
async function main() {
async function main(eventsInfo) {
const {
homeBlockNumber,
foreignBlockNumber,
homeToForeignRequests,
homeToForeignConfirmations,
foreignToHomeConfirmations,
foreignToHomeRequests,
bridgeMode
} = await eventsInfo()
} = eventsInfo
let xSignatures
let xAffirmations
@ -24,8 +25,6 @@ async function main() {
xAffirmations = foreignToHomeConfirmations.filter(eventWithoutReference(foreignToHomeRequests))
}
logger.debug('building misbehavior blocks')
const homeBlockNumber = await getHomeBlockNumber()
const foreignBlockNumber = await getForeignBlockNumber()
const baseRange = [false, false, false, false, false]
const xSignaturesMisbehavior = buildRangesObject(

7
monitor/checkWorker2.js
View File

@ -1,6 +1,7 @@
require('dotenv').config()
const logger = require('./logger')('checkWorker2')
const eventsStats = require('./eventsStats')
const getEventsInfo = require('./utils/events')
const alerts = require('./alerts')
const { writeFile, createDir } = require('./utils/file')
const { saveCache } = require('./utils/web3Cache')
@ -10,8 +11,10 @@ const { MONITOR_BRIDGE_NAME } = process.env
async function checkWorker2() {
try {
createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
logger.debug('calling getEventsInfo()')
const eventsInfo = await getEventsInfo()
logger.debug('calling eventsStats()')
const evStats = await eventsStats()
const evStats = await eventsStats(eventsInfo)
if (!evStats) throw new Error('evStats is empty: ' + JSON.stringify(evStats))
evStats.ok =
(evStats.onlyInHomeDeposits || evStats.home.deliveredMsgNotProcessedInForeign).length === 0 &&
@ -22,7 +25,7 @@ async function checkWorker2() {
writeFile(`/responses/${MONITOR_BRIDGE_NAME}/eventsStats.json`, evStats)
logger.debug('calling alerts()')
const _alerts = await alerts()
const _alerts = await alerts(eventsInfo)
if (!_alerts) throw new Error('alerts is empty: ' + JSON.stringify(_alerts))
_alerts.ok = !_alerts.executeAffirmations.mostRecentTxHash && !_alerts.executeSignatures.mostRecentTxHash
_alerts.health = true

5
monitor/eventsStats.js
View File

@ -1,5 +1,4 @@
require('dotenv').config()
const eventsInfo = require('./utils/events')
const {
processedMsgNotDelivered,
deliveredMsgNotProcessed,
@ -15,14 +14,14 @@ const {
MONITOR_HOME_TO_FOREIGN_CHECK_SENDER
} = process.env
async function main() {
async function main(eventsInfo) {
const {
homeToForeignRequests,
homeToForeignConfirmations,
foreignToHomeConfirmations,
foreignToHomeRequests,
bridgeMode
} = await eventsInfo()
} = eventsInfo
if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
return {

20
monitor/getBalances.js
View File

@ -5,7 +5,12 @@ const logger = require('./logger')('getBalances')
const { BRIDGE_MODES } = require('../commons')
const { web3Home, web3Foreign, getHomeBlockNumber } = require('./utils/web3')
const { COMMON_HOME_BRIDGE_ADDRESS, COMMON_FOREIGN_BRIDGE_ADDRESS } = process.env
const {
MONITOR_HOME_START_BLOCK,
MONITOR_FOREIGN_START_BLOCK,
COMMON_HOME_BRIDGE_ADDRESS,
COMMON_FOREIGN_BRIDGE_ADDRESS
} = process.env
const {
ERC20_ABI,
@ -20,6 +25,8 @@ const {
async function main(bridgeMode, eventsInfo) {
const {
homeBlockNumber,
foreignBlockNumber,
homeToForeignConfirmations,
foreignToHomeConfirmations,
homeDelayedBlockNumber,
@ -46,6 +53,13 @@ async function main(bridgeMode, eventsInfo) {
...foreignToHomeConfirmations.filter(e => e.blockNumber > homeDelayedBlockNumber).map(e => e.value)
)
const blockRanges = {
startBlockHome: MONITOR_HOME_START_BLOCK,
endBlockHome: homeBlockNumber,
startBlockForeign: MONITOR_FOREIGN_START_BLOCK,
endBlockForeign: foreignBlockNumber
}
if (bridgeMode === BRIDGE_MODES.ERC_TO_ERC) {
const foreignBridge = new web3Foreign.eth.Contract(FOREIGN_ERC_TO_ERC_ABI, COMMON_FOREIGN_BRIDGE_ADDRESS)
const erc20Address = await foreignBridge.methods.erc20token().call()
@ -72,6 +86,7 @@ async function main(bridgeMode, eventsInfo) {
erc20Balance: Web3Utils.fromWei(foreignErc20Balance)
},
balanceDiff: Number(Web3Utils.fromWei(diff)),
...blockRanges,
lastChecked: Math.floor(Date.now() / 1000)
}
} else if (bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC || bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC_V1) {
@ -94,6 +109,7 @@ async function main(bridgeMode, eventsInfo) {
totalSupply: Web3Utils.fromWei(totalSupply)
},
balanceDiff: Number(Web3Utils.fromWei(diff)),
...blockRanges,
lastChecked: Math.floor(Date.now() / 1000)
}
} else if (bridgeMode === BRIDGE_MODES.ERC_TO_NATIVE) {
@ -163,12 +179,14 @@ async function main(bridgeMode, eventsInfo) {
},
foreign,
balanceDiff: Number(Web3Utils.fromWei(diff)),
...blockRanges,
lastChecked: Math.floor(Date.now() / 1000)
}
} else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
return {
home: {},
foreign: {},
...blockRanges,
lastChecked: Math.floor(Date.now() / 1000)
}
} else {

4
monitor/index.js
View File

@ -2,7 +2,6 @@ require('dotenv').config()
const express = require('express')
const cors = require('cors')
const { readFile } = require('./utils/file')
const { getPrometheusMetrics } = require('./prometheusMetrics')
const app = express()
const bridgeRouter = express.Router({ mergeParams: true })
@ -25,7 +24,8 @@ bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|
bridgeRouter.get('/metrics', (req, res, next) => {
try {
const metrics = getPrometheusMetrics(req.params.bridgeName)
const { bridgeName } = req.params
const metrics = readFile(`./responses/${bridgeName}/metrics.txt`, false)
res.type('text').send(metrics)
} catch (e) {
next(e)

20
monitor/metricsWorker.js Normal file
View File

@ -0,0 +1,20 @@
require('dotenv').config()
const logger = require('./logger')('metricsWorker')
const { writeFile, createDir } = require('./utils/file')
const getPrometheusMetrics = require('./prometheusMetrics')
const { MONITOR_BRIDGE_NAME } = process.env
async function metricsWorker() {
try {
createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
logger.debug('calling getPrometheusMetrics()')
const metrics = await getPrometheusMetrics(MONITOR_BRIDGE_NAME)
if (!metrics) throw new Error('metrics is empty: ' + JSON.stringify(metrics))
writeFile(`/responses/${MONITOR_BRIDGE_NAME}/metrics.txt`, metrics, { stringify: false })
logger.debug('Done')
} catch (e) {
logger.error(e)
}
}
metricsWorker()

2
monitor/package.json
View File

@ -4,7 +4,7 @@
"description": "",
"main": "index.js",
"scripts": {
"check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js",
"check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js && timeout -s 9 10s node metricsWorker.js",
"start": "node index.js",
"check-and-start": "yarn check-all && yarn start",
"lint": "eslint . --ignore-path ../.eslintignore",

71
monitor/prometheusMetrics.js
View File

@ -1,8 +1,10 @@
require('dotenv').config()
const logger = require('./logger')('getBalances')
const { readFile } = require('./utils/file')
const {
MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST,
MONITOR_HOME_TO_FOREIGN_BLOCK_LIST,
MONITOR_HOME_START_BLOCK,
MONITOR_FOREIGN_START_BLOCK,
MONITOR_HOME_VALIDATORS_BALANCE_ENABLE,
MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE
} = process.env
@ -23,7 +25,9 @@ function hasError(obj) {
return 'error' in obj
}
function getPrometheusMetrics(bridgeName) {
// Try to collect all metrics from JSON responses and then
// discard all unsuccessfully retrieved ones
async function getPrometheusMetrics(bridgeName) {
const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json`
const metrics = {}
@ -32,22 +36,44 @@ function getPrometheusMetrics(bridgeName) {
const balancesFile = readFile(responsePath('getBalances'))
if (!hasError(balancesFile)) {
const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile
metrics.balances_home_value = homeBalances.totalSupply
metrics.balances_home_txs_deposit = homeBalances.deposits
metrics.balances_home_txs_withdrawal = homeBalances.withdrawals
const { home, foreign, ...commonBalances } = balancesFile
metrics.balances_foreign_value = foreignBalances.erc20Balance
metrics.balances_foreign_txs_deposit = foreignBalances.deposits
metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals
const balanceMetrics = {
// ERC_TO_ERC or ERC_TO_NATIVE mode
balances_home_value: home.totalSupply,
balances_home_txs_deposit: home.deposits,
balances_home_txs_withdrawal: home.withdrawals,
balances_foreign_value: foreign.erc20Balance,
balances_foreign_txs_deposit: foreign.deposits,
balances_foreign_txs_withdrawal: foreign.withdrawals,
metrics.balances_diff_value = commonBalances.balanceDiff
metrics.balances_diff_deposit = commonBalances.depositsDiff
metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff
if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) {
metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff
metrics.balances_unclaimed_value = commonBalances.unclaimedBalance
// Not ARBITRARY_MESSAGE mode
balances_diff_value: commonBalances.balanceDiff,
balances_diff_deposit: commonBalances.depositsDiff,
balances_diff_withdrawal: commonBalances.withdrawalDiff,
// MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST or MONITOR_HOME_TO_FOREIGN_BLOCK_LIST is set
balances_unclaimed_txs: commonBalances.unclaimedDiff,
balances_unclaimed_value: commonBalances.unclaimedBalance,
// ARBITRARY_MESSAGE mode
txs_home_out: home.toForeign,
txs_home_in: home.fromForeign,
txs_foreign_out: foreign.toHome,
txs_foreign_in: foreign.fromHome,
txs_diff_home_out_oracles: commonBalances.fromHomeToForeignDiff,
txs_diff_home_out_users: commonBalances.fromHomeToForeignPBUDiff,
txs_diff_foreign_out: commonBalances.fromForeignToHomeDiff
}
const blockRanges = {
state_startblock_home: commonBalances.startBlockHome || MONITOR_HOME_START_BLOCK,
state_startblock_foreign: commonBalances.startBlockForeign || MONITOR_FOREIGN_START_BLOCK,
state_endblock_home: commonBalances.endBlockHome,
state_endblock_foreign: commonBalances.endBlockForeign
}
Object.assign(metrics, blockRanges, balanceMetrics)
}
// Validator metrics
@ -62,7 +88,11 @@ function getPrometheusMetrics(bridgeName) {
: Object.keys(allValidators)
validatorAddressesWithBalanceCheck.forEach((addr, ind) => {
metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance
if (addr in allValidators) {
metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance
} else {
logger.debug(`Nonexistent validator address ${addr}`)
}
})
}
}
@ -96,9 +126,12 @@ function getPrometheusMetrics(bridgeName) {
return Object.entries(metrics).reduceRight(
// Prometheus supports `Nan` and possibly signed `Infinity`
// in case cast to `Number` fails
(acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`,
(acc, [key, val]) => {
if (typeof val === 'undefined') return acc
else return `${key} ${Number(val)}\n${acc}`
},
''
)
}
module.exports = { getPrometheusMetrics }
module.exports = getPrometheusMetrics

4
monitor/utils/events.js
View File

@ -236,13 +236,15 @@ async function main(mode) {
foreignToHomeRequests,
isExternalErc20,
bridgeMode,
homeBlockNumber,
foreignBlockNumber,
homeDelayedBlockNumber,
foreignDelayedBlockNumber
}
if (MONITOR_CACHE_EVENTS === 'true') {
logger.debug('saving obtained events into cache file')
writeFile(cacheFilePath, result, false)
writeFile(cacheFilePath, result, { useCwd: false })
}
return result
}

15
monitor/utils/file.js
View File

@ -1,23 +1,30 @@
const fs = require('fs')
const path = require('path')
function readFile(filePath) {
function readFile(filePath, parseJson = true) {
try {
const content = fs.readFileSync(filePath)
if (!parseJson) return content
const json = JSON.parse(content)
const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked
return Object.assign({}, json, { timeDiff })
} catch (e) {
console.error(e)
console.error('readFlle', e)
return {
error: 'the bridge statistics are not available'
}
}
}
function writeFile(filePath, object, useCwd = true) {
function writeFile(filePath, object, paramOptions = {}) {
const defaultOptions = {
useCwd: true,
stringify: true
}
const { useCwd, stringify } = Object.assign({}, defaultOptions, paramOptions)
const fullPath = useCwd ? path.join(process.cwd(), filePath) : filePath
fs.writeFileSync(fullPath, JSON.stringify(object, null, 4))
fs.writeFileSync(fullPath, stringify ? JSON.stringify(object, null, 4) : object)
}
function createDir(dirPath) {