Health field in the monitor reports (#334)

.gitignore

@@ -49,4 +49,5 @@ __pycache__
 
 #monitor
 monitor/responses/*
+monitor/configs/*.env
 !monitor/.gitkeep

monitor/checkWorker.js

@@ -31,6 +31,7 @@ async function checkWorker() {
     const foreign = Object.assign({}, balances.foreign, events.foreign)
     const status = Object.assign({}, balances, events, { home }, { foreign })
     if (!status) throw new Error('status is empty: ' + JSON.stringify(status))
+    status.health = true
     writeFile(`/responses/${MONITOR_BRIDGE_NAME}/getBalances.json`, status)
 
     logger.debug('calling validators()')
@@ -59,6 +60,7 @@ async function checkWorker() {
     }
 
     vBalances.ok = vBalances.homeOk && vBalances.foreignOk
+    vBalances.health = true
     writeFile(`/responses/${MONITOR_BRIDGE_NAME}/validators.json`, vBalances)
     logger.debug('Done')
   } catch (e) {

monitor/checkWorker2.js

@@ -16,12 +16,14 @@ async function checkWorker2() {
       (evStats.onlyInForeignDeposits || evStats.home.processedMsgNotDeliveredInForeign).length === 0 &&
       (evStats.onlyInHomeWithdrawals || evStats.foreign.deliveredMsgNotProcessedInHome).length === 0 &&
       (evStats.onlyInForeignWithdrawals || evStats.foreign.processedMsgNotDeliveredInHome).length === 0
+    evStats.health = true
     writeFile(`/responses/${MONITOR_BRIDGE_NAME}/eventsStats.json`, evStats)
 
     logger.debug('calling alerts()')
     const _alerts = await alerts()
     if (!_alerts) throw new Error('alerts is empty: ' + JSON.stringify(_alerts))
     _alerts.ok = !_alerts.executeAffirmations.mostRecentTxHash && !_alerts.executeSignatures.mostRecentTxHash
+    _alerts.health = true
     writeFile(`/responses/${MONITOR_BRIDGE_NAME}/alerts.json`, _alerts)
     logger.debug('Done x2')
   } catch (e) {

monitor/checkWorker3.js

@@ -19,6 +19,7 @@ async function checkWorker3() {
       const transfers = await stuckTransfers()
       if (!transfers) throw new Error('transfers is empty: ' + JSON.stringify(transfers))
       transfers.ok = transfers.total.length === 0
+      transfers.health = true
       writeFile(`/responses/${MONITOR_BRIDGE_NAME}/stuckTransfers.json`, transfers)
       logger.debug('Done')
     }

monitor/scripts/getBridgeStats.sh

@@ -1,11 +1,60 @@
 #!/bin/bash
+
+CONFIGDIR="configs"
+RESPONSESDIR="responses"
+IMAGETAG="latest"
+
 cd $(dirname $0)/..
 
 if /usr/local/bin/docker-compose ps | grep -q -i 'monitor'; then
-  for file in configs/*.env
+  tstart=`date +"%s"`
+  
+  for file in ${CONFIGDIR}/*.env
   do
-    docker run --rm --env-file $file -v $(pwd)/responses:/mono/monitor/responses poanetwork/tokenbridge-monitor:latest /bin/bash -c 'yarn check-all'
+    echo "${file} handling..."
+    
+    bridgename=`source ${file} && echo ${MONITOR_BRIDGE_NAME}`
+    reportdir=${RESPONSESDIR}"/"${bridgename}
+    if [ ! -d ${reportdir} ]; then
+      mkdir -p ${reportdir}
+    fi
+    checksumfile=${bridgename}".shasum"
+    rm -f ${checksumfile}
+    for json in alerts.json eventsStats.json getBalances.json validators.json stuckTransfers.json; do
+      if [ -f ${reportdir}/${json} ]; then
+        shasum -a 256 ${reportdir}/${json} >> ${checksumfile}
+      fi
+    done
+
+    containername=${bridgename}"-checker"
+    docker container stats --no-stream ${containername} 2>/dev/null 1>&2
+    if [ ! "$?" == "0" ]; then
+      docker run --rm --env-file $file -v $(pwd)/${RESPONSESDIR}:/mono/monitor/responses \
+        --name ${containername} poanetwork/tokenbridge-monitor:${IMAGETAG} \
+        /bin/bash -c 'yarn check-all'
+      shasum -a 256 -s -c ${checksumfile}
+      if [ "$?" == "0" ]; then
+        echo "JSON files have not been updated - the monitor is not healthy"
+        for json in alerts.json eventsStats.json getBalances.json validators.json stuckTransfers.json; do
+          if [ -f ${reportdir}/${json} ]; then
+            echo '{"health": false, "lastChecked": '`date +"%s"`'}' > ${reportdir}/${json}
+          fi
+        done
+      else
+        echo "JSON files have been updated - new metrics collected"
+      fi
+    else
+      echo "${containername} have not finished yet" >&2
+    fi
+    
+    rm ${checksumfile}
+    echo "========================================"
   done
+  
+  tend=`date +"%s"`
+  tdiff=`expr ${tend} - ${tstart}`
+  echo "Total time to run: ${tdiff}"
+
 else
   echo "Monitor is not running, skipping checks."
-fi
+fi