increment errors score

docker-compose.test.yml

@@ -54,6 +54,8 @@ services:
   redis:
     image: redis
     restart: always
+    ports:
+      - '6379:6379'
     command: [redis-server, --appendonly, 'yes']
     volumes:
       - redis:/data

src/contollers/status.js

@@ -2,11 +2,12 @@ const queue = require('../queue')
 const { netId, tornadoServiceFee, miningServiceFee, instances, rewardAccount } = require('../config')
 const { version } = require('../../package.json')
 const { redis } = require('../modules/redis')
+const { readRelayerErrors } = require('../utils')
 
 async function status(req, res) {
   const ethPrices = await redis.hgetall('prices')
   const health = await redis.hgetall('health')
-  const errors = await redis.zrevrange('errors', 0, -1)
+  const errors = await readRelayerErrors(redis)
 
   const { waiting: currentQueue } = await queue.queue.getJobCounts()
 

src/healthWatcher.js

@@ -14,7 +14,6 @@ async function main() {
     await redis.hset('health', { status: true, error: '' })
   } catch (e) {
     console.error('healthWatcher', e.message)
-    redis.zadd('errors', e.score || 0, e.message)
     await redis.hset('health', { status: false, error: e.message })
   }
 }

src/modules/redis.js

@@ -1,8 +1,8 @@
-const Redis = require('ioredis')
+const { createClient } = require('ioredis')
 const { redisUrl } = require('../config')
 
-const redis = new Redis(redisUrl)
-const redisSubscribe = new Redis(redisUrl)
+const redis = createClient(redisUrl)
+const redisSubscribe = createClient(redisUrl)
 
 module.exports = {
   redis,

src/priceWatcher.js

@@ -1,5 +1,12 @@
 const { offchainOracleAddress } = require('./config')
-const { getArgsForOracle, setSafeInterval, toChecksumAddress, toBN, RelayerError } = require('./utils')
+const {
+  getArgsForOracle,
+  setSafeInterval,
+  toChecksumAddress,
+  toBN,
+  RelayerError,
+  logRelayerError,
+} = require('./utils')
 const { redis } = require('./modules/redis')
 const web3 = require('./modules/web3')()
 
@@ -32,7 +39,7 @@ async function main() {
     await redis.hmset('prices', ethPrices)
     console.log('Wrote following prices to redis', ethPrices)
   } catch (e) {
-    redis.zadd('errors', e.score || 1, e.message)
+    await logRelayerError(redis, e.message)
     console.error('priceWatcher error', e)
   }
 }

src/treeWatcher.js

@@ -1,6 +1,6 @@
 const MerkleTree = require('fixed-merkle-tree')
 const { minerMerkleTreeHeight, torn, netId } = require('./config')
-const { poseidonHash2, toBN } = require('./utils')
+const { poseidonHash2, toBN, logRelayerError } = require('./utils')
 const resolver = require('./modules/resolver')
 const web3 = require('./modules/web3')('ws')
 const MinerABI = require('../abis/mining.abi.json')
@@ -123,7 +123,7 @@ async function init() {
     eventSubscription = contract.events.NewAccount({ fromBlock: toBlock + 1 }, processNewEvent)
     blockSubscription = web3.eth.subscribe('newBlockHeaders', processNewBlock)
   } catch (e) {
-    redis.zadd('errors', 1, e.message)
+    await logRelayerError(redis, e.message)
     console.error('error on init treeWatcher', e.message)
   }
 }

src/utils.js

@@ -125,6 +125,20 @@ class RelayerError extends Error {
   }
 }
 
+const logRelayerError = async (redis, e) => {
+  await redis.zadd('errors', 'INCR', 1, e.message)
+}
+
+const readRelayerErrors = async redis => {
+  const set = await redis.zrevrange('errors', 0, -1, 'WITHSCORES')
+  const errors = []
+  while (set.length) {
+    const [message, score] = set.splice(0, 2)
+    errors.push({ message, score })
+  }
+  return errors
+}
+
 module.exports = {
   getInstance,
   setSafeInterval,
@@ -140,4 +154,6 @@ module.exports = {
   BN,
   isAddress,
   RelayerError,
+  logRelayerError,
+  readRelayerErrors,
 }

src/worker.js

@@ -18,6 +18,7 @@ const {
   fromWei,
   toChecksumAddress,
   RelayerError,
+  logRelayerError,
 } = require('./utils')
 const { jobType, status } = require('./constants')
 const {
@@ -106,7 +107,7 @@ async function start() {
     queue.process(processJob)
     console.log('Worker started')
   } catch (e) {
-    redis.zadd('errors', e.score || 1, e.message)
+    await logRelayerError(redis, e.message)
     console.error('error on start worker', e.message)
   }
 }