Merge the develop branch to the master branch, preparation to v2.7.0-rc1

CONFIGURATION.md

@@ -44,6 +44,11 @@ ORACLE_ALWAYS_RELAY_SIGNATURES | If set to `true`, the oracle will always relay
 ORACLE_RPC_REQUEST_TIMEOUT | Timeout in milliseconds for a single RPC request. Default value is `ORACLE_*_RPC_POLLING_INTERVAL * 2`. | integer
 ORACLE_HOME_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Home sender service. Defaults to 20 minutes. | integer  
 ORACLE_FOREIGN_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Foreign sender service. Defaults to 20 minutes. | integer  
+ORACLE_SHUTDOWN_SERVICE_URL | Optional external URL to some other service/monitor/configuration manager that controls the remote shutdown process. GET request should return `application/json` message with the following schema: `{ shutdown: true/false }`. | URL
+ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL | Optional interval in milliseconds used to request the side RPC node or external shutdown service. Default is 120000. | integer
+ORACLE_SIDE_RPC_URL | Optional HTTPS URL(s) for communication with the external shutdown service or side RPC nodes, used for shutdown manager activities. Several URLs can be specified, delimited by spaces. If the connection to one of these nodes is lost the next URL is used for connection. | URL(s)
+ORACLE_SHUTDOWN_CONTRACT_ADDRESS | Optional contract address in the side chain accessible through `ORACLE_SIDE_RPC_URL`, where the method passed in `ORACLE_SHUTDOWN_CONTRACT_METHOD` is implemented. | `address`
+ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain to identify the current shutdown status. Method should return boolean. Default value is `isShutdown()`. | `function signature`
 
 
 ## UI configuration
@@ -86,5 +91,3 @@ MONITOR_HOME_TO_FOREIGN_BLOCK_LIST | File with a list of addresses, separated by
 MONITOR_HOME_TO_FOREIGN_CHECK_SENDER | If set to `true`, instructs the oracle to do an extra check for transaction origin in the block/allowance list. `false` by default. | `true` / `false`
 MONITOR_HOME_VALIDATORS_BALANCE_ENABLE | If set, defines the list of home validator addresses for which balance should be checked. | `string`
 MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE | If set, defines the list of foreign validator addresses for which balance should be checked. | `string`
-MONITOR_HOME_EXPLORER_API | The HTTPS URL of the Home network explorer API. If set, may be used as a fallback in case of the RPC node failure. | URL
-MONITOR_FOREIGN_EXPLORER_API | The HTTPS URL of the Foreign network explorer API. If set, may be used as a fallback in case of the RPC node failure. | URL

commons/abis.js

@@ -56,11 +56,6 @@ const OLD_AMB_USER_REQUEST_FOR_SIGNATURE_ABI = [
   {
     anonymous: false,
     inputs: [
-      {
-        indexed: true,
-        name: 'messageId',
-        type: 'bytes32'
-      },
       {
         indexed: false,
         name: 'encodedData',
@@ -76,11 +71,6 @@ const OLD_AMB_USER_REQUEST_FOR_AFFIRMATION_ABI = [
   {
     anonymous: false,
     inputs: [
-      {
-        indexed: true,
-        name: 'messageId',
-        type: 'bytes32'
-      },
       {
         indexed: false,
         name: 'encodedData',

deployment-e2e/molecule/multiple/tests/test_multiple.py

@@ -24,6 +24,7 @@ def test_services(host, service):
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
     ("ui_ui_1"),
     ("monitor_monitor_1")
 ])

deployment-e2e/molecule/oracle/tests/test_oracle.py

@@ -14,6 +14,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
 ])
 def test_docker_containers(host, name):
     container = host.docker(name)

e2e-commons/up.sh

@@ -42,6 +42,7 @@ startValidator () {
     fi
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle yarn manager:shutdown
 }
 
 startAMBValidator () {
@@ -52,6 +53,7 @@ startAMBValidator () {
     docker-compose $1 run $2 $3 -d oracle-amb yarn watcher:affirmation-request
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle-amb yarn manager:shutdown
 }
 
 while [ "$1" != "" ]; do
@@ -120,13 +122,7 @@ while [ "$1" != "" ]; do
   fi
 
   if [ "$1" == "alm-e2e" ]; then
-    docker-compose up -d redis rabbit
-
-    docker-compose run -d oracle-amb yarn watcher:signature-request
-    docker-compose run -d oracle-amb yarn watcher:collected-signatures
-    docker-compose run -d oracle-amb yarn watcher:affirmation-request
-    docker-compose run -d oracle-amb yarn sender:home
-    docker-compose run -d oracle-amb yarn sender:foreign
+    startAMBValidator "" "" "" "redis" "rabbit"
 
     oracle2name="-p validator2"
     oracle2Values="-e ORACLE_VALIDATOR_ADDRESS=0xdCC784657C78054aa61FbcFFd2605F32374816A4 -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5a5c3645d0f04e9eb4f27f94ed4c244a225587405b8838e7456f7781ce3a9513"

monitor/.env.example

@@ -28,6 +28,3 @@ MONITOR_HOME_TO_FOREIGN_BLOCK_LIST=
 
 # MONITOR_HOME_VALIDATORS_BALANCE_ENABLE=0x... 0x... 0x...
 # MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE=0x... 0x... 0x...
-
-# MONITOR_HOME_EXPLORER_API=
-MONITOR_FOREIGN_EXPLORER_API=https://api.bscscan.com/api?apikey=YourApiKeyToken

monitor/utils/web3Cache.js

@@ -1,15 +1,9 @@
-const fetch = require('node-fetch')
 const logger = require('../logger')('web3Cache')
 const { readCacheFile, writeCacheFile } = require('./file')
 const { web3Home, web3Foreign } = require('./web3')
 const { getPastEvents: commonGetPastEvents } = require('../../commons')
 
-const {
-  MONITOR_BRIDGE_NAME,
-  MONITOR_CACHE_EVENTS,
-  MONITOR_FOREIGN_EXPLORER_API,
-  MONITOR_HOME_EXPLORER_API
-} = process.env
+const { MONITOR_BRIDGE_NAME, MONITOR_CACHE_EVENTS } = process.env
 
 let isDirty = false
 
@@ -58,36 +52,6 @@ async function isForeignContract(address) {
   return cachedForeignIsContract[address]
 }
 
-function getPastEventsWithAPIFallback(contract, options) {
-  return commonGetPastEvents(contract, options).catch(async e => {
-    const [api, web3] =
-      options.chain === 'home' ? [MONITOR_HOME_EXPLORER_API, web3Home] : [MONITOR_FOREIGN_EXPLORER_API, web3Foreign]
-    if (api && e.message.includes('exceed maximum block range')) {
-      logger.debug('BLOCK RANGE EXCEED, using fallback to the explorer API')
-
-      const abi = contract.options.jsonInterface.find(abi => abi.type === 'event' && abi.name === options.event)
-
-      const url = new URL(api)
-      url.searchParams.append('module', 'logs')
-      url.searchParams.append('action', 'getLogs')
-      url.searchParams.append('address', contract.options.address)
-      url.searchParams.append('fromBlock', options.fromBlock)
-      url.searchParams.append('toBlock', options.toBlock || 'latest')
-      url.searchParams.append('topic0', web3.eth.abi.encodeEventSignature(abi))
-
-      const logs = await fetch(url).then(res => res.json())
-
-      return logs.result.map(log => ({
-        transactionHash: log.transactionHash,
-        blockNumber: parseInt(log.blockNumber.slice(2), 16),
-        returnValues: web3.eth.abi.decodeLog(abi.inputs, log.data, log.topics.slice(1))
-      }))
-    } else {
-      throw new Error(e)
-    }
-  })
-}
-
 async function getPastEvents(contract, options) {
   if (MONITOR_CACHE_EVENTS !== 'true') {
     return commonGetPastEvents(contract, options)
@@ -130,14 +94,14 @@ async function getPastEvents(contract, options) {
     // requested:      A...B
     // cached:    C...D
     logger.debug(`Fetching events for blocks ${fromBlock}...${toBlock}`)
-    result = await getPastEventsWithAPIFallback(contract, options)
+    result = await commonGetPastEvents(contract, options)
   } else if (fromBlock < cachedFromBlock && toBlock <= cachedToBlock) {
     // requested: A...B
     // cached:      C...D
     logger.debug(`Cache hit for blocks ${cachedFromBlock}...${toBlock}`)
     logger.debug(`Fetching events for blocks ${fromBlock}...${cachedFromBlock - 1}`)
     result = [
-      ...(await getPastEventsWithAPIFallback(contract, { ...options, toBlock: cachedFromBlock - 1 })),
+      ...(await commonGetPastEvents(contract, { ...options, toBlock: cachedFromBlock - 1 })),
       ...cachedEvents.filter(e => e.blockNumber <= toBlock)
     ]
   } else if (fromBlock < cachedFromBlock && cachedToBlock < toBlock) {
@@ -147,9 +111,9 @@ async function getPastEvents(contract, options) {
     logger.debug(`Fetching events for blocks ${fromBlock}...${cachedFromBlock - 1}`)
     logger.debug(`Fetching events for blocks ${cachedToBlock + 1}...${toBlock}`)
     result = [
-      ...(await getPastEventsWithAPIFallback(contract, { ...options, toBlock: cachedFromBlock - 1 })),
+      ...(await commonGetPastEvents(contract, { ...options, toBlock: cachedFromBlock - 1 })),
       ...cachedEvents,
-      ...(await getPastEventsWithAPIFallback(contract, { ...options, fromBlock: cachedToBlock + 1 }))
+      ...(await commonGetPastEvents(contract, { ...options, fromBlock: cachedToBlock + 1 }))
     ]
   } else if (cachedFromBlock <= fromBlock && toBlock <= cachedToBlock) {
     // requested:   A.B
@@ -163,7 +127,7 @@ async function getPastEvents(contract, options) {
     logger.debug(`Fetching events for blocks ${cachedToBlock + 1}...${toBlock}`)
     result = [
       ...cachedEvents.filter(e => e.blockNumber >= fromBlock),
-      ...(await getPastEventsWithAPIFallback(contract, { ...options, fromBlock: cachedToBlock + 1 }))
+      ...(await commonGetPastEvents(contract, { ...options, fromBlock: cachedToBlock + 1 }))
     ]
   } else {
     throw new Error(

oracle/config/base.config.js

@@ -69,7 +69,8 @@ const bridgeConfig = {
   foreignBridgeAbi: foreignAbi,
   eventFilter: {},
   validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY),
-  maxProcessingTime
+  maxProcessingTime,
+  shutdownKey: 'oracle-shutdown'
 }
 
 const homeConfig = {

oracle/config/shutdown-manager.config.js

@@ -0,0 +1,20 @@
+const baseConfig = require('./base.config')
+
+const {
+  ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL,
+  ORACLE_SHUTDOWN_SERVICE_URL,
+  ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  ORACLE_SHUTDOWN_CONTRACT_METHOD
+} = process.env
+
+module.exports = {
+  ...baseConfig.bridgeConfig,
+  id: 'shutdown-manager',
+  name: 'shutdown-manager',
+  pollingInterval: ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL || 120000,
+  checksBeforeResume: 3,
+  checksBeforeStop: 1,
+  shutdownServiceURL: ORACLE_SHUTDOWN_SERVICE_URL,
+  shutdownContractAddress: ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  shutdownMethod: (ORACLE_SHUTDOWN_CONTRACT_METHOD || 'isShutdown()').trim()
+}

oracle/docker-compose-erc-native.yml

@@ -77,6 +77,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -91,6 +97,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose-transfer.yml

@@ -61,6 +61,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -75,6 +81,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose.yml

@@ -21,12 +21,13 @@ services:
     command: [redis-server, --appendonly, 'yes']
     hostname: redis
     image: redis:4
-    networks:    
+    networks:
       - net_db_bridge_request
       - net_db_bridge_collected
       - net_db_bridge_affirmation
       - net_db_bridge_senderhome
       - net_db_bridge_senderforeign
+      - net_db_bridge_shutdown
     restart: unless-stopped
     volumes: ['~/bridge_data/redis:/data']
   bridge_request:
@@ -34,8 +35,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn watcher:signature-request
@@ -47,8 +48,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS}
     restart: unless-stopped
     entrypoint: yarn watcher:collected-signatures
@@ -60,8 +61,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS}
     restart: unless-stopped
     entrypoint: yarn watcher:affirmation-request
@@ -73,8 +74,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn sender:home
@@ -86,14 +87,25 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn sender:foreign
     networks:
       - net_db_bridge_senderforeign
       - net_rabbit_bridge_senderforeign
+  bridge_shutdown:
+    cpus: 0.1
+    mem_limit: 500m
+    image: poanetwork/tokenbridge-oracle:latest
+    env_file: ./.env
+    environment:
+      - NODE_ENV=production
+    restart: unless-stopped
+    entrypoint: yarn manager:shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -106,6 +118,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/package.json

@@ -13,6 +13,7 @@
     "sender:home": "./scripts/start-worker.sh sender home-sender",
     "sender:foreign": "./scripts/start-worker.sh sender foreign-sender",
     "confirm:transfer": "./scripts/start-worker.sh confirmRelay transfer-watcher",
+    "manager:shutdown": "./scripts/start-worker.sh shutdownManager shutdown-manager",
     "dev": "concurrently -n 'watcher:signature-request,watcher:collected-signatures,watcher:affirmation-request,watcher:transfer, sender:home,sender:foreign' -c 'red,green,yellow,blue,magenta,cyan' 'yarn watcher:signature-request' 'yarn watcher:collected-signatures' 'yarn watcher:affirmation-request' 'yarn watcher:transfer' 'yarn sender:home' 'yarn sender:foreign'",
     "test": "NODE_ENV=test mocha",
     "test:watch": "NODE_ENV=test mocha --watch --reporter=min",

oracle/src/sender.js

@@ -4,6 +4,7 @@ const { connectSenderToQueue } = require('./services/amqpClient')
 const { redis } = require('./services/redisClient')
 const GasPrice = require('./services/gasPrice')
 const logger = require('./services/logger')
+const { getShutdownFlag } = require('./services/shutdownState')
 const { sendTx } = require('./tx/sendTx')
 const { getNonce, getChainId } = require('./tx/web3')
 const {
@@ -102,6 +103,14 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
       return
     }
 
+    const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
+    if (await getShutdownFlag(logger, config.shutdownKey, true)) {
+      if (!wasShutdown) {
+        logger.info('Oracle sender was suspended via the remote shutdown process')
+      }
+      return
+    }
+
     const txArray = JSON.parse(msg.content)
     logger.debug(`Msg received with ${txArray.length} Tx to send`)
     const gasPrice = GasPrice.getPrice().toString(10)

oracle/src/services/logger.js

@@ -3,6 +3,7 @@ const path = require('path')
 const {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeFallback,
   web3ForeignFallback,
   web3HomeRedundant,
@@ -30,4 +31,8 @@ web3ForeignFallback.currentProvider.setLogger(logger)
 web3HomeRedundant.currentProvider.setLogger(logger)
 web3ForeignRedundant.currentProvider.setLogger(logger)
 
+if (web3Side) {
+  web3Side.currentProvider.setLogger(logger)
+}
+
 module.exports = logger

oracle/src/services/shutdownState.js

@@ -0,0 +1,23 @@
+const { redis } = require('./redisClient')
+
+let isShutdown = false
+async function getShutdownFlag(logger, shutdownKey, force = false) {
+  if (force) {
+    logger.debug('Reading current shutdown state from the DB')
+    isShutdown = (await redis.get(shutdownKey)) === 'true'
+    logger.debug({ isShutdown }, 'Read shutdown state from the DB')
+  }
+  return isShutdown
+}
+
+async function setShutdownFlag(logger, shutdownKey, value) {
+  logger.info({ isShutdown: value }, 'Updating current shutdown state in the DB')
+  isShutdown = value
+  await redis.set(shutdownKey, value)
+  logger.debug('Updated state in the DB')
+}
+
+module.exports = {
+  getShutdownFlag,
+  setShutdownFlag
+}

oracle/src/services/web3.js

@@ -6,6 +6,7 @@ const { RETRY_CONFIG } = require('../utils/constants')
 const {
   COMMON_HOME_RPC_URL,
   COMMON_FOREIGN_RPC_URL,
+  ORACLE_SIDE_RPC_URL,
   ORACLE_RPC_REQUEST_TIMEOUT,
   ORACLE_HOME_RPC_POLLING_INTERVAL,
   ORACLE_FOREIGN_RPC_POLLING_INTERVAL
@@ -41,6 +42,18 @@ const web3Home = new Web3(homeProvider)
 const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions)
 const web3Foreign = new Web3(foreignProvider)
 
+let web3Side = null
+if (ORACLE_SIDE_RPC_URL) {
+  const sideUrls = ORACLE_SIDE_RPC_URL.split(' ').filter(url => url.length > 0)
+  const sideOptions = {
+    requestTimeout: configuredTimeout || 2000,
+    retry: RETRY_CONFIG
+  }
+
+  const sideProvider = new HttpListProvider(sideUrls, sideOptions)
+  web3Side = new Web3(sideProvider)
+}
+
 // secondary fallback providers are intended to be used in places where
 // it is more likely that RPC calls to the local non-archive nodes can fail
 // e.g. for checking status of the old transaction via eth_getTransactionByHash
@@ -70,6 +83,7 @@ if (foreignUrls.length > 1) {
 module.exports = {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeRedundant,
   web3ForeignRedundant,
   web3HomeFallback,

oracle/src/shutdownManager.js

@@ -0,0 +1,114 @@
+const fetch = require('node-fetch')
+const path = require('path')
+
+const { EXIT_CODES } = require('./utils/constants')
+const { watchdog } = require('./utils/utils')
+const logger = require('./services/logger')
+const { redis } = require('./services/redisClient')
+const { web3Side } = require('./services/web3')
+const { getShutdownFlag, setShutdownFlag } = require('./services/shutdownState')
+
+if (process.argv.length < 3) {
+  logger.error('Please check the number of arguments, config file was not provided')
+  process.exit(EXIT_CODES.GENERAL_ERROR)
+}
+
+const config = require(path.join('../config/', process.argv[2]))
+
+if (config.shutdownContractAddress && !web3Side) {
+  logger.error(
+    'ORACLE_SHUTDOWN_CONTRACT_ADDRESS was provided but not side chain provider was registered.' +
+      ' Please, specify ORACLE_SIDE_RPC_URL as well.'
+  )
+  process.exit(EXIT_CODES.GENERAL_ERROR)
+}
+
+let shutdownCount = 0
+let okCount = 0
+
+async function fetchShutdownFlag() {
+  if (config.shutdownServiceURL) {
+    logger.debug({ url: config.shutdownServiceURL }, 'Fetching shutdown status from external URL')
+    const result = await fetch(config.shutdownServiceURL, {
+      headers: {
+        'Content-type': 'application/json'
+      },
+      method: 'GET',
+      timeout: config.requestTimeout
+    }).then(res => res.json())
+
+    if (result.shutdown === true) {
+      return true
+    }
+  }
+
+  if (config.shutdownContractAddress) {
+    const shutdownSelector = web3Side.eth.abi.encodeEventSignature(config.shutdownMethod)
+    logger.debug(
+      { contract: config.shutdownContractAddress, method: config.shutdownMethod, data: shutdownSelector },
+      'Fetching shutdown status from contract'
+    )
+    const result = await web3Side.eth.call({
+      to: config.shutdownContractAddress,
+      data: shutdownSelector
+    })
+    logger.debug({ result }, 'Obtained result from the side RPC endpoint')
+
+    if (result.length > 2 && web3Side.eth.abi.decodeParameter('bool', result)) {
+      return true
+    }
+  }
+
+  return false
+}
+
+async function checkShutdownFlag() {
+  const isShutdownFlag = await fetchShutdownFlag()
+  const isShutdown = await getShutdownFlag(logger, config.shutdownKey)
+
+  if (isShutdownFlag === true && isShutdown === false) {
+    shutdownCount += 1
+    okCount = 0
+    logger.info(
+      { shutdownCount, remainingChecks: config.checksBeforeStop - shutdownCount },
+      'Received positive shutdown flag'
+    )
+  } else if (isShutdownFlag === false && isShutdown === true) {
+    okCount += 1
+    shutdownCount = 0
+    logger.info({ okCount, remainingChecks: config.checksBeforeResume - okCount }, 'Received negative shutdown flag')
+  } else {
+    shutdownCount = 0
+    okCount = 0
+    logger.debug({ isShutdown, isShutdownFlag }, 'Received shutdown flag that is equal to the current state')
+  }
+
+  if (shutdownCount >= config.checksBeforeStop) {
+    await setShutdownFlag(logger, config.shutdownKey, true)
+  } else if (okCount >= config.checksBeforeResume) {
+    await setShutdownFlag(logger, config.shutdownKey, false)
+  }
+}
+
+async function initialize() {
+  logger.info('Starting shutdown flag watcher')
+  redis.on('connect', async () => {
+    await getShutdownFlag(logger, config.shutdownKey, true)
+    await main()
+  })
+}
+
+async function main() {
+  try {
+    await watchdog(checkShutdownFlag, config.maxProcessingTime, () => {
+      logger.fatal('Max processing time reached')
+      process.exit(EXIT_CODES.MAX_TIME_REACHED)
+    })
+  } catch (e) {
+    logger.error(e)
+  }
+
+  setTimeout(main, config.pollingInterval)
+}
+
+initialize()

oracle/src/watcher.js

@@ -5,6 +5,7 @@ const { connectWatcherToQueue, connection } = require('./services/amqpClient')
 const { getBlockNumber } = require('./tx/web3')
 const { redis } = require('./services/redisClient')
 const logger = require('./services/logger')
+const { getShutdownFlag } = require('./services/shutdownState')
 const { getRequiredBlockConfirmations, getEvents } = require('./tx/web3')
 const { checkHTTPS, watchdog } = require('./utils/utils')
 const { EXIT_CODES } = require('./utils/constants')
@@ -157,6 +158,14 @@ async function isWorkerNeeded() {
 
 async function main({ sendToQueue, sendToWorker }) {
   try {
+    const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
+    if (await getShutdownFlag(logger, config.shutdownKey, true)) {
+      if (!wasShutdown) {
+        logger.info('Oracle watcher was suspended via the remote shutdown process')
+      }
+      return
+    }
+
     await checkConditions()
 
     const lastBlockToProcess = await getLastBlockToProcess()