diff --git a/.gitignore b/.gitignore index f45c5a2b..94b2a7df 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,5 @@ __pycache__ #monitor monitor/responses/* +monitor/configs/*.env !monitor/.gitkeep diff --git a/monitor/checkWorker.js b/monitor/checkWorker.js index 98c10184..c6ffdc3e 100644 --- a/monitor/checkWorker.js +++ b/monitor/checkWorker.js @@ -31,6 +31,7 @@ async function checkWorker() { const foreign = Object.assign({}, balances.foreign, events.foreign) const status = Object.assign({}, balances, events, { home }, { foreign }) if (!status) throw new Error('status is empty: ' + JSON.stringify(status)) + status.health = true writeFile(`/responses/${MONITOR_BRIDGE_NAME}/getBalances.json`, status) logger.debug('calling validators()') @@ -59,6 +60,7 @@ async function checkWorker() { } vBalances.ok = vBalances.homeOk && vBalances.foreignOk + vBalances.health = true writeFile(`/responses/${MONITOR_BRIDGE_NAME}/validators.json`, vBalances) logger.debug('Done') } catch (e) { diff --git a/monitor/checkWorker2.js b/monitor/checkWorker2.js index c05bb109..0d6811a1 100644 --- a/monitor/checkWorker2.js +++ b/monitor/checkWorker2.js @@ -16,12 +16,14 @@ async function checkWorker2() { (evStats.onlyInForeignDeposits || evStats.home.processedMsgNotDeliveredInForeign).length === 0 && (evStats.onlyInHomeWithdrawals || evStats.foreign.deliveredMsgNotProcessedInHome).length === 0 && (evStats.onlyInForeignWithdrawals || evStats.foreign.processedMsgNotDeliveredInHome).length === 0 + evStats.health = true writeFile(`/responses/${MONITOR_BRIDGE_NAME}/eventsStats.json`, evStats) logger.debug('calling alerts()') const _alerts = await alerts() if (!_alerts) throw new Error('alerts is empty: ' + JSON.stringify(_alerts)) _alerts.ok = !_alerts.executeAffirmations.mostRecentTxHash && !_alerts.executeSignatures.mostRecentTxHash + _alerts.health = true writeFile(`/responses/${MONITOR_BRIDGE_NAME}/alerts.json`, _alerts) logger.debug('Done x2') } catch (e) { diff --git a/monitor/checkWorker3.js b/monitor/checkWorker3.js index a4fd2416..f61bb42e 100644 --- a/monitor/checkWorker3.js +++ b/monitor/checkWorker3.js @@ -19,6 +19,7 @@ async function checkWorker3() { const transfers = await stuckTransfers() if (!transfers) throw new Error('transfers is empty: ' + JSON.stringify(transfers)) transfers.ok = transfers.total.length === 0 + transfers.health = true writeFile(`/responses/${MONITOR_BRIDGE_NAME}/stuckTransfers.json`, transfers) logger.debug('Done') } diff --git a/monitor/scripts/getBridgeStats.sh b/monitor/scripts/getBridgeStats.sh index 5a90b882..ccf80dbe 100755 --- a/monitor/scripts/getBridgeStats.sh +++ b/monitor/scripts/getBridgeStats.sh @@ -1,11 +1,60 @@ #!/bin/bash + +CONFIGDIR="configs" +RESPONSESDIR="responses" +IMAGETAG="latest" + cd $(dirname $0)/.. if /usr/local/bin/docker-compose ps | grep -q -i 'monitor'; then - for file in configs/*.env + tstart=`date +"%s"` + + for file in ${CONFIGDIR}/*.env do - docker run --rm --env-file $file -v $(pwd)/responses:/mono/monitor/responses poanetwork/tokenbridge-monitor:latest /bin/bash -c 'yarn check-all' + echo "${file} handling..." + + bridgename=`source ${file} && echo ${MONITOR_BRIDGE_NAME}` + reportdir=${RESPONSESDIR}"/"${bridgename} + if [ ! -d ${reportdir} ]; then + mkdir -p ${reportdir} + fi + checksumfile=${bridgename}".shasum" + rm -f ${checksumfile} + for json in alerts.json eventsStats.json getBalances.json validators.json stuckTransfers.json; do + if [ -f ${reportdir}/${json} ]; then + shasum -a 256 ${reportdir}/${json} >> ${checksumfile} + fi + done + + containername=${bridgename}"-checker" + docker container stats --no-stream ${containername} 2>/dev/null 1>&2 + if [ ! "$?" == "0" ]; then + docker run --rm --env-file $file -v $(pwd)/${RESPONSESDIR}:/mono/monitor/responses \ + --name ${containername} poanetwork/tokenbridge-monitor:${IMAGETAG} \ + /bin/bash -c 'yarn check-all' + shasum -a 256 -s -c ${checksumfile} + if [ "$?" == "0" ]; then + echo "JSON files have not been updated - the monitor is not healthy" + for json in alerts.json eventsStats.json getBalances.json validators.json stuckTransfers.json; do + if [ -f ${reportdir}/${json} ]; then + echo '{"health": false, "lastChecked": '`date +"%s"`'}' > ${reportdir}/${json} + fi + done + else + echo "JSON files have been updated - new metrics collected" + fi + else + echo "${containername} have not finished yet" >&2 + fi + + rm ${checksumfile} + echo "========================================" done + + tend=`date +"%s"` + tdiff=`expr ${tend} - ${tstart}` + echo "Total time to run: ${tdiff}" + else echo "Monitor is not running, skipping checks." -fi +fi \ No newline at end of file