Add oracle service for remote shutdown (#531)

CONFIGURATION.md

@@ -44,6 +44,11 @@ ORACLE_ALWAYS_RELAY_SIGNATURES | If set to `true`, the oracle will always relay
 ORACLE_RPC_REQUEST_TIMEOUT | Timeout in milliseconds for a single RPC request. Default value is `ORACLE_*_RPC_POLLING_INTERVAL * 2`. | integer
 ORACLE_HOME_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Home sender service. Defaults to 20 minutes. | integer  
 ORACLE_FOREIGN_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Foreign sender service. Defaults to 20 minutes. | integer  
+ORACLE_SHUTDOWN_SERVICE_URL | Optional external URL to some other service/monitor/configuration manager that controls the remote shutdown process. GET request should return `application/json` message with the following schema: `{ shutdown: true/false }`. | URL
+ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL | Optional interval in milliseconds used to request the side RPC node or external shutdown service. Default is 120000. | integer
+ORACLE_SIDE_RPC_URL | Optional HTTPS URL(s) for communication with the external shutdown service or side RPC nodes, used for shutdown manager activities. Several URLs can be specified, delimited by spaces. If the connection to one of these nodes is lost the next URL is used for connection. | URL(s)
+ORACLE_SHUTDOWN_CONTRACT_ADDRESS | Optional contract address in the side chain accessible through `ORACLE_SIDE_RPC_URL`, where the method passed in `ORACLE_SHUTDOWN_CONTRACT_METHOD` is implemented. | `address`
+ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain to identify the current shutdown status. Method should return boolean. Default value is `isShutdown()`. | `function signature`
 
 
 ## UI configuration

deployment-e2e/molecule/multiple/tests/test_multiple.py

@@ -24,6 +24,7 @@ def test_services(host, service):
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
     ("ui_ui_1"),
     ("monitor_monitor_1")
 ])

deployment-e2e/molecule/oracle/tests/test_oracle.py

@@ -14,6 +14,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
 ])
 def test_docker_containers(host, name):
     container = host.docker(name)

e2e-commons/up.sh

@@ -42,6 +42,7 @@ startValidator () {
     fi
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle yarn manager:shutdown
 }
 
 startAMBValidator () {
@@ -52,6 +53,7 @@ startAMBValidator () {
     docker-compose $1 run $2 $3 -d oracle-amb yarn watcher:affirmation-request
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle-amb yarn manager:shutdown
 }
 
 while [ "$1" != "" ]; do
@@ -120,13 +122,7 @@ while [ "$1" != "" ]; do
   fi
 
   if [ "$1" == "alm-e2e" ]; then
-    docker-compose up -d redis rabbit
-
-    docker-compose run -d oracle-amb yarn watcher:signature-request
-    docker-compose run -d oracle-amb yarn watcher:collected-signatures
-    docker-compose run -d oracle-amb yarn watcher:affirmation-request
-    docker-compose run -d oracle-amb yarn sender:home
-    docker-compose run -d oracle-amb yarn sender:foreign
+    startAMBValidator "" "" "" "redis" "rabbit"
 
     oracle2name="-p validator2"
     oracle2Values="-e ORACLE_VALIDATOR_ADDRESS=0xdCC784657C78054aa61FbcFFd2605F32374816A4 -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5a5c3645d0f04e9eb4f27f94ed4c244a225587405b8838e7456f7781ce3a9513"

oracle/config/base.config.js

@@ -69,7 +69,8 @@ const bridgeConfig = {
   foreignBridgeAbi: foreignAbi,
   eventFilter: {},
   validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY),
-  maxProcessingTime
+  maxProcessingTime,
+  shutdownKey: 'oracle-shutdown'
 }
 
 const homeConfig = {

oracle/config/shutdown-manager.config.js

@@ -0,0 +1,20 @@
+const baseConfig = require('./base.config')
+
+const {
+  ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL,
+  ORACLE_SHUTDOWN_SERVICE_URL,
+  ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  ORACLE_SHUTDOWN_CONTRACT_METHOD
+} = process.env
+
+module.exports = {
+  ...baseConfig.bridgeConfig,
+  id: 'shutdown-manager',
+  name: 'shutdown-manager',
+  pollingInterval: ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL || 120000,
+  checksBeforeResume: 3,
+  checksBeforeStop: 1,
+  shutdownServiceURL: ORACLE_SHUTDOWN_SERVICE_URL,
+  shutdownContractAddress: ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  shutdownMethod: (ORACLE_SHUTDOWN_CONTRACT_METHOD || 'isShutdown()').trim()
+}

oracle/docker-compose-erc-native.yml

@@ -77,6 +77,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -91,6 +97,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose-transfer.yml

@@ -61,6 +61,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -75,6 +81,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose.yml

@@ -27,6 +27,7 @@ services:
       - net_db_bridge_affirmation
       - net_db_bridge_senderhome
       - net_db_bridge_senderforeign
+      - net_db_bridge_shutdown
     restart: unless-stopped
     volumes: ['~/bridge_data/redis:/data']
   bridge_request:
@@ -94,6 +95,17 @@ services:
     networks:
       - net_db_bridge_senderforeign
       - net_rabbit_bridge_senderforeign
+  bridge_shutdown:
+    cpus: 0.1
+    mem_limit: 500m
+    image: poanetwork/tokenbridge-oracle:latest
+    env_file: ./.env
+    environment:
+      - NODE_ENV=production
+    restart: unless-stopped
+    entrypoint: yarn manager:shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -106,6 +118,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/package.json

@@ -13,6 +13,7 @@
     "sender:home": "./scripts/start-worker.sh sender home-sender",
     "sender:foreign": "./scripts/start-worker.sh sender foreign-sender",
     "confirm:transfer": "./scripts/start-worker.sh confirmRelay transfer-watcher",
+    "manager:shutdown": "./scripts/start-worker.sh shutdownManager shutdown-manager",
     "dev": "concurrently -n 'watcher:signature-request,watcher:collected-signatures,watcher:affirmation-request,watcher:transfer, sender:home,sender:foreign' -c 'red,green,yellow,blue,magenta,cyan' 'yarn watcher:signature-request' 'yarn watcher:collected-signatures' 'yarn watcher:affirmation-request' 'yarn watcher:transfer' 'yarn sender:home' 'yarn sender:foreign'",
     "test": "NODE_ENV=test mocha",
     "test:watch": "NODE_ENV=test mocha --watch --reporter=min",

oracle/src/sender.js

@@ -4,6 +4,7 @@ const { connectSenderToQueue } = require('./services/amqpClient')
 const { redis } = require('./services/redisClient')
 const GasPrice = require('./services/gasPrice')
 const logger = require('./services/logger')
+const { getShutdownFlag } = require('./services/shutdownState')
 const { sendTx } = require('./tx/sendTx')
 const { getNonce, getChainId } = require('./tx/web3')
 const {
@@ -102,6 +103,14 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
       return
     }
 
+    const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
+    if (await getShutdownFlag(logger, config.shutdownKey, true)) {
+      if (!wasShutdown) {
+        logger.info('Oracle sender was suspended via the remote shutdown process')
+      }
+      return
+    }
+
     const txArray = JSON.parse(msg.content)
     logger.debug(`Msg received with ${txArray.length} Tx to send`)
     const gasPrice = GasPrice.getPrice().toString(10)

oracle/src/services/logger.js

@@ -3,6 +3,7 @@ const path = require('path')
 const {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeFallback,
   web3ForeignFallback,
   web3HomeRedundant,
@@ -30,4 +31,8 @@ web3ForeignFallback.currentProvider.setLogger(logger)
 web3HomeRedundant.currentProvider.setLogger(logger)
 web3ForeignRedundant.currentProvider.setLogger(logger)
 
+if (web3Side) {
+  web3Side.currentProvider.setLogger(logger)
+}
+
 module.exports = logger

oracle/src/services/shutdownState.js

@@ -0,0 +1,23 @@
+const { redis } = require('./redisClient')
+
+let isShutdown = false
+async function getShutdownFlag(logger, shutdownKey, force = false) {
+  if (force) {
+    logger.debug('Reading current shutdown state from the DB')
+    isShutdown = (await redis.get(shutdownKey)) === 'true'
+    logger.debug({ isShutdown }, 'Read shutdown state from the DB')
+  }
+  return isShutdown
+}
+
+async function setShutdownFlag(logger, shutdownKey, value) {
+  logger.info({ isShutdown: value }, 'Updating current shutdown state in the DB')
+  isShutdown = value
+  await redis.set(shutdownKey, value)
+  logger.debug('Updated state in the DB')
+}
+
+module.exports = {
+  getShutdownFlag,
+  setShutdownFlag
+}

oracle/src/services/web3.js

@@ -6,6 +6,7 @@ const { RETRY_CONFIG } = require('../utils/constants')
 const {
   COMMON_HOME_RPC_URL,
   COMMON_FOREIGN_RPC_URL,
+  ORACLE_SIDE_RPC_URL,
   ORACLE_RPC_REQUEST_TIMEOUT,
   ORACLE_HOME_RPC_POLLING_INTERVAL,
   ORACLE_FOREIGN_RPC_POLLING_INTERVAL
@@ -41,6 +42,18 @@ const web3Home = new Web3(homeProvider)
 const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions)
 const web3Foreign = new Web3(foreignProvider)
 
+let web3Side = null
+if (ORACLE_SIDE_RPC_URL) {
+  const sideUrls = ORACLE_SIDE_RPC_URL.split(' ').filter(url => url.length > 0)
+  const sideOptions = {
+    requestTimeout: configuredTimeout || 2000,
+    retry: RETRY_CONFIG
+  }
+
+  const sideProvider = new HttpListProvider(sideUrls, sideOptions)
+  web3Side = new Web3(sideProvider)
+}
+
 // secondary fallback providers are intended to be used in places where
 // it is more likely that RPC calls to the local non-archive nodes can fail
 // e.g. for checking status of the old transaction via eth_getTransactionByHash
@@ -70,6 +83,7 @@ if (foreignUrls.length > 1) {
 module.exports = {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeRedundant,
   web3ForeignRedundant,
   web3HomeFallback,

oracle/src/shutdownManager.js

@@ -0,0 +1,114 @@
+const fetch = require('node-fetch')
+const path = require('path')
+
+const { EXIT_CODES } = require('./utils/constants')
+const { watchdog } = require('./utils/utils')
+const logger = require('./services/logger')
+const { redis } = require('./services/redisClient')
+const { web3Side } = require('./services/web3')
+const { getShutdownFlag, setShutdownFlag } = require('./services/shutdownState')
+
+if (process.argv.length < 3) {
+  logger.error('Please check the number of arguments, config file was not provided')
+  process.exit(EXIT_CODES.GENERAL_ERROR)
+}
+
+const config = require(path.join('../config/', process.argv[2]))
+
+if (config.shutdownContractAddress && !web3Side) {
+  logger.error(
+    'ORACLE_SHUTDOWN_CONTRACT_ADDRESS was provided but not side chain provider was registered.' +
+      ' Please, specify ORACLE_SIDE_RPC_URL as well.'
+  )
+  process.exit(EXIT_CODES.GENERAL_ERROR)
+}
+
+let shutdownCount = 0
+let okCount = 0
+
+async function fetchShutdownFlag() {
+  if (config.shutdownServiceURL) {
+    logger.debug({ url: config.shutdownServiceURL }, 'Fetching shutdown status from external URL')
+    const result = await fetch(config.shutdownServiceURL, {
+      headers: {
+        'Content-type': 'application/json'
+      },
+      method: 'GET',
+      timeout: config.requestTimeout
+    }).then(res => res.json())
+
+    if (result.shutdown === true) {
+      return true
+    }
+  }
+
+  if (config.shutdownContractAddress) {
+    const shutdownSelector = web3Side.eth.abi.encodeEventSignature(config.shutdownMethod)
+    logger.debug(
+      { contract: config.shutdownContractAddress, method: config.shutdownMethod, data: shutdownSelector },
+      'Fetching shutdown status from contract'
+    )
+    const result = await web3Side.eth.call({
+      to: config.shutdownContractAddress,
+      data: shutdownSelector
+    })
+    logger.debug({ result }, 'Obtained result from the side RPC endpoint')
+
+    if (result.length > 2 && web3Side.eth.abi.decodeParameter('bool', result)) {
+      return true
+    }
+  }
+
+  return false
+}
+
+async function checkShutdownFlag() {
+  const isShutdownFlag = await fetchShutdownFlag()
+  const isShutdown = await getShutdownFlag(logger, config.shutdownKey)
+
+  if (isShutdownFlag === true && isShutdown === false) {
+    shutdownCount += 1
+    okCount = 0
+    logger.info(
+      { shutdownCount, remainingChecks: config.checksBeforeStop - shutdownCount },
+      'Received positive shutdown flag'
+    )
+  } else if (isShutdownFlag === false && isShutdown === true) {
+    okCount += 1
+    shutdownCount = 0
+    logger.info({ okCount, remainingChecks: config.checksBeforeResume - okCount }, 'Received negative shutdown flag')
+  } else {
+    shutdownCount = 0
+    okCount = 0
+    logger.debug({ isShutdown, isShutdownFlag }, 'Received shutdown flag that is equal to the current state')
+  }
+
+  if (shutdownCount >= config.checksBeforeStop) {
+    await setShutdownFlag(logger, config.shutdownKey, true)
+  } else if (okCount >= config.checksBeforeResume) {
+    await setShutdownFlag(logger, config.shutdownKey, false)
+  }
+}
+
+async function initialize() {
+  logger.info('Starting shutdown flag watcher')
+  redis.on('connect', async () => {
+    await getShutdownFlag(logger, config.shutdownKey, true)
+    await main()
+  })
+}
+
+async function main() {
+  try {
+    await watchdog(checkShutdownFlag, config.maxProcessingTime, () => {
+      logger.fatal('Max processing time reached')
+      process.exit(EXIT_CODES.MAX_TIME_REACHED)
+    })
+  } catch (e) {
+    logger.error(e)
+  }
+
+  setTimeout(main, config.pollingInterval)
+}
+
+initialize()

oracle/src/watcher.js

@@ -5,6 +5,7 @@ const { connectWatcherToQueue, connection } = require('./services/amqpClient')
 const { getBlockNumber } = require('./tx/web3')
 const { redis } = require('./services/redisClient')
 const logger = require('./services/logger')
+const { getShutdownFlag } = require('./services/shutdownState')
 const { getRequiredBlockConfirmations, getEvents } = require('./tx/web3')
 const { checkHTTPS, watchdog } = require('./utils/utils')
 const { EXIT_CODES } = require('./utils/constants')
@@ -157,6 +158,14 @@ async function isWorkerNeeded() {
 
 async function main({ sendToQueue, sendToWorker }) {
   try {
+    const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
+    if (await getShutdownFlag(logger, config.shutdownKey, true)) {
+      if (!wasShutdown) {
+        logger.info('Oracle watcher was suspended via the remote shutdown process')
+      }
+      return
+    }
+
     await checkConditions()
 
     const lastBlockToProcess = await getLastBlockToProcess()