Merge the develop branch to the master branch, preparation to v2.7.0-rc1

CONFIGURATION.md

@@ -42,6 +42,13 @@ ORACLE_HOME_TO_FOREIGN_BLOCK_LIST | Filename with a list of addresses, separated
 ORACLE_HOME_TO_FOREIGN_CHECK_SENDER | If set to `true`, instructs the oracle to do an extra check for transaction origin in the block/allowance list. `false` by default. | `true` / `false`
 ORACLE_ALWAYS_RELAY_SIGNATURES | If set to `true`, the oracle will always relay signatures even if it was not the last who finilized the signatures collecting process. The default is `false`. | `true` / `false`
 ORACLE_RPC_REQUEST_TIMEOUT | Timeout in milliseconds for a single RPC request. Default value is `ORACLE_*_RPC_POLLING_INTERVAL * 2`. | integer
+ORACLE_HOME_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Home sender service. Defaults to 20 minutes. | integer  
+ORACLE_FOREIGN_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Foreign sender service. Defaults to 20 minutes. | integer  
+ORACLE_SHUTDOWN_SERVICE_URL | Optional external URL to some other service/monitor/configuration manager that controls the remote shutdown process. GET request should return `application/json` message with the following schema: `{ shutdown: true/false }`. | URL
+ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL | Optional interval in milliseconds used to request the side RPC node or external shutdown service. Default is 120000. | integer
+ORACLE_SIDE_RPC_URL | Optional HTTPS URL(s) for communication with the external shutdown service or side RPC nodes, used for shutdown manager activities. Several URLs can be specified, delimited by spaces. If the connection to one of these nodes is lost the next URL is used for connection. | URL(s)
+ORACLE_SHUTDOWN_CONTRACT_ADDRESS | Optional contract address in the side chain accessible through `ORACLE_SIDE_RPC_URL`, where the method passed in `ORACLE_SHUTDOWN_CONTRACT_METHOD` is implemented. | `address`
+ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain to identify the current shutdown status. Method should return boolean. Default value is `isShutdown()`. | `function signature`
 
 
 ## UI configuration

alm/Dockerfile

@@ -19,7 +19,7 @@ COPY --from=contracts /mono/contracts/build ./contracts/build
 COPY commons/package.json ./commons/
 COPY alm/package.json ./alm/
 COPY yarn.lock .
-RUN NOYARNPOSTINSTALL=1 yarn install --frozen-lockfile --production
+RUN NOYARNPOSTINSTALL=1 yarn install --frozen-lockfile
 
 COPY ./commons ./commons
 COPY ./alm ./alm

alm/package.json

@@ -58,6 +58,7 @@
     ]
   },
   "devDependencies": {
-    "eslint-plugin-prettier": "^3.1.3"
+    "eslint-plugin-prettier": "^3.1.3",
+    "node-fetch": "^2.6.1"
   }
 }

alm/scripts/createSnapshots.js

@@ -3,6 +3,8 @@ const { BRIDGE_VALIDATORS_ABI, HOME_AMB_ABI } = require('commons')
 const path = require('path')
 require('dotenv').config()
 const Web3 = require('web3')
+const fetch = require('node-fetch')
+const { URL } = require('url')
 
 const fs = require('fs')
 
@@ -10,7 +12,9 @@ const {
   COMMON_HOME_RPC_URL,
   COMMON_HOME_BRIDGE_ADDRESS,
   COMMON_FOREIGN_RPC_URL,
-  COMMON_FOREIGN_BRIDGE_ADDRESS
+  COMMON_FOREIGN_BRIDGE_ADDRESS,
+  ALM_FOREIGN_EXPLORER_API,
+  ALM_HOME_EXPLORER_API
 } = process.env
 
 const generateSnapshot = async (side, url, bridgeAddress) => {
@@ -19,6 +23,31 @@ const generateSnapshot = async (side, url, bridgeAddress) => {
   const snapshot = {}
 
   const web3 = new Web3(new Web3.providers.HttpProvider(url))
+  const api = side === 'home' ? ALM_HOME_EXPLORER_API : ALM_FOREIGN_EXPLORER_API
+
+  const getPastEventsWithFallback = (contract, eventName, options) =>
+    contract.getPastEvents(eventName, options).catch(async e => {
+      if (e.message.includes('exceed maximum block range')) {
+        const abi = contract.options.jsonInterface.find(abi => abi.type === 'event' && abi.name === eventName)
+
+        const url = new URL(api)
+        url.searchParams.append('module', 'logs')
+        url.searchParams.append('action', 'getLogs')
+        url.searchParams.append('address', contract.options.address)
+        url.searchParams.append('fromBlock', options.fromBlock)
+        url.searchParams.append('toBlock', options.toBlock || 'latest')
+        url.searchParams.append('topic0', web3.eth.abi.encodeEventSignature(abi))
+
+        const logs = await fetch(url).then(res => res.json())
+
+        return logs.result.map(log => ({
+          transactionHash: log.transactionHash,
+          blockNumber: parseInt(log.blockNumber.slice(2), 16),
+          returnValues: web3.eth.abi.decodeLog(abi.inputs, log.data, log.topics.slice(1))
+        }))
+      }
+      throw e
+    })
 
   const currentBlockNumber = await web3.eth.getBlockNumber()
   snapshot.snapshotBlockNumber = currentBlockNumber
@@ -29,10 +58,14 @@ const generateSnapshot = async (side, url, bridgeAddress) => {
   const bridgeContract = new web3.eth.Contract(HOME_AMB_ABI, bridgeAddress)
 
   // Save RequiredBlockConfirmationChanged events
-  let requiredBlockConfirmationChangedEvents = await bridgeContract.getPastEvents('RequiredBlockConfirmationChanged', {
-    fromBlock: 0,
-    toBlock: currentBlockNumber
-  })
+  let requiredBlockConfirmationChangedEvents = await getPastEventsWithFallback(
+    bridgeContract,
+    'RequiredBlockConfirmationChanged',
+    {
+      fromBlock: 0,
+      toBlock: currentBlockNumber
+    }
+  )
 
   // In case RequiredBlockConfirmationChanged was not emitted during initialization in early versions of AMB
   // manually generate an event for this. Example Sokol - Kovan bridge
@@ -59,10 +92,14 @@ const generateSnapshot = async (side, url, bridgeAddress) => {
   const validatorContract = new web3.eth.Contract(BRIDGE_VALIDATORS_ABI, validatorAddress)
 
   // Save RequiredSignaturesChanged events
-  const RequiredSignaturesChangedEvents = await validatorContract.getPastEvents('RequiredSignaturesChanged', {
-    fromBlock: 0,
-    toBlock: currentBlockNumber
-  })
+  const RequiredSignaturesChangedEvents = await getPastEventsWithFallback(
+    validatorContract,
+    'RequiredSignaturesChanged',
+    {
+      fromBlock: 0,
+      toBlock: currentBlockNumber
+    }
+  )
   snapshot.RequiredSignaturesChanged = RequiredSignaturesChangedEvents.map(e => ({
     blockNumber: e.blockNumber,
     returnValues: {
@@ -71,7 +108,7 @@ const generateSnapshot = async (side, url, bridgeAddress) => {
   }))
 
   // Save ValidatorAdded events
-  const validatorAddedEvents = await validatorContract.getPastEvents('ValidatorAdded', {
+  const validatorAddedEvents = await getPastEventsWithFallback(validatorContract, 'ValidatorAdded', {
     fromBlock: 0,
     toBlock: currentBlockNumber
   })
@@ -85,7 +122,7 @@ const generateSnapshot = async (side, url, bridgeAddress) => {
   }))
 
   // Save ValidatorRemoved events
-  const validatorRemovedEvents = await validatorContract.getPastEvents('ValidatorRemoved', {
+  const validatorRemovedEvents = await getPastEventsWithFallback(validatorContract, 'ValidatorRemoved', {
     fromBlock: 0,
     toBlock: currentBlockNumber
   })

alm/src/components/ManualExecutionButton.tsx

@@ -6,6 +6,7 @@ import {
   DOUBLE_EXECUTION_ATTEMPT_ERROR,
   EXECUTION_FAILED_ERROR,
   EXECUTION_OUT_OF_GAS_ERROR,
+  FOREIGN_EXPLORER_API,
   INCORRECT_CHAIN_ERROR,
   VALIDATOR_CONFIRMATION_STATUS
 } from '../config/constants'
@@ -92,7 +93,13 @@ export const ManualExecutionButton = ({
         })
         .on('error', async (e: Error, receipt: TransactionReceipt) => {
           if (e.message.includes('Transaction has been reverted by the EVM')) {
-            const successExecutionData = await getSuccessExecutionData(bridge, 'RelayedMessage', library, messageId)
+            const successExecutionData = await getSuccessExecutionData(
+              bridge,
+              'RelayedMessage',
+              library,
+              messageId,
+              FOREIGN_EXPLORER_API
+            )
             if (successExecutionData) {
               setExecutionData(successExecutionData)
               setError(DOUBLE_EXECUTION_ATTEMPT_ERROR)

alm/src/hooks/useBlockConfirmations.ts

@@ -4,6 +4,8 @@ import { useStateProvider } from '../state/StateProvider'
 import { Contract } from 'web3-eth-contract'
 import { getRequiredBlockConfirmations } from '../utils/contract'
 import { foreignSnapshotProvider, homeSnapshotProvider, SnapshotProvider } from '../services/SnapshotProvider'
+import Web3 from 'web3'
+import { FOREIGN_EXPLORER_API, HOME_EXPLORER_API } from '../config/constants'
 
 export interface UseBlockConfirmationsParams {
   fromHome: boolean
@@ -19,9 +21,11 @@ export const useBlockConfirmations = ({ receipt, fromHome }: UseBlockConfirmatio
     contract: Contract,
     receipt: TransactionReceipt,
     setResult: Function,
-    snapshotProvider: SnapshotProvider
+    snapshotProvider: SnapshotProvider,
+    web3: Web3,
+    api: string
   ) => {
-    const result = await getRequiredBlockConfirmations(contract, receipt.blockNumber, snapshotProvider)
+    const result = await getRequiredBlockConfirmations(contract, receipt.blockNumber, snapshotProvider, web3, api)
     setResult(result)
   }
 
@@ -29,10 +33,12 @@ export const useBlockConfirmations = ({ receipt, fromHome }: UseBlockConfirmatio
     () => {
       const bridgeContract = fromHome ? home.bridgeContract : foreign.bridgeContract
       const snapshotProvider = fromHome ? homeSnapshotProvider : foreignSnapshotProvider
-      if (!bridgeContract || !receipt) return
-      callRequireBlockConfirmations(bridgeContract, receipt, setBlockConfirmations, snapshotProvider)
+      const web3 = fromHome ? home.web3 : foreign.web3
+      const api = fromHome ? HOME_EXPLORER_API : FOREIGN_EXPLORER_API
+      if (!bridgeContract || !receipt || !web3) return
+      callRequireBlockConfirmations(bridgeContract, receipt, setBlockConfirmations, snapshotProvider, web3, api)
     },
-    [home.bridgeContract, foreign.bridgeContract, receipt, fromHome]
+    [home.bridgeContract, foreign.bridgeContract, receipt, fromHome, home.web3, foreign.web3]
   )
 
   return {

alm/src/hooks/useMessageConfirmations.ts

@@ -11,9 +11,6 @@ import {
   VALIDATOR_CONFIRMATION_STATUS
 } from '../config/constants'
 import { homeBlockNumberProvider, foreignBlockNumberProvider } from '../services/BlockNumberProvider'
-import { checkSignaturesWaitingForBLocks } from '../utils/signatureWaitingForBlocks'
-import { getCollectedSignaturesEvent } from '../utils/getCollectedSignaturesEvent'
-import { checkWaitingBlocksForExecution } from '../utils/executionWaitingForBlocks'
 import { getConfirmationsForTx } from '../utils/getConfirmationsForTx'
 import { getFinalizationEvent } from '../utils/getFinalizationEvent'
 import {
@@ -64,11 +61,11 @@ export const useMessageConfirmations = ({
   blockConfirmations
 }: useMessageConfirmationsParams) => {
   const { home, foreign } = useStateProvider()
-  const [confirmations, setConfirmations] = useState([])
+  const [confirmations, setConfirmations] = useState<ConfirmationParam[]>([])
   const [status, setStatus] = useState(CONFIRMATIONS_STATUS.UNDEFINED)
   const [waitingBlocks, setWaitingBlocks] = useState(false)
   const [waitingBlocksResolved, setWaitingBlocksResolved] = useState(false)
-  const [signatureCollected, setSignatureCollected] = useState(false)
+  const [signatureCollected, setSignatureCollected] = useState<boolean | string[]>(false)
   const [executionEventsFetched, setExecutionEventsFetched] = useState(false)
   const [collectedSignaturesEvent, setCollectedSignaturesEvent] = useState<Maybe<EventData>>(null)
   const [executionData, setExecutionData] = useState<ExecutionData>({
@@ -85,12 +82,10 @@ export const useMessageConfirmations = ({
   const [pendingConfirmations, setPendingConfirmations] = useState(false)
   const [pendingExecution, setPendingExecution] = useState(false)
 
-  const existsConfirmation = (confirmationArray: ConfirmationParam[]) => {
-    const filteredList = confirmationArray.filter(
+  const existsConfirmation = (confirmationArray: ConfirmationParam[]) =>
+    confirmationArray.some(
       c => c.status !== VALIDATOR_CONFIRMATION_STATUS.UNDEFINED && c.status !== VALIDATOR_CONFIRMATION_STATUS.WAITING
     )
-    return filteredList.length > 0
-  }
 
   // start watching blocks at the start
   useEffect(
@@ -108,129 +103,154 @@ export const useMessageConfirmations = ({
     () => {
       if (!receipt || !blockConfirmations || waitingBlocksResolved) return
 
-      const subscriptions: Array<number> = []
-
-      const unsubscribe = () => {
-        subscriptions.forEach(s => {
-          clearTimeout(s)
-        })
-      }
+      let timeoutId: number
 
       const blockProvider = fromHome ? homeBlockNumberProvider : foreignBlockNumberProvider
       const interval = fromHome ? HOME_RPC_POLLING_INTERVAL : FOREIGN_RPC_POLLING_INTERVAL
       const targetBlock = receipt.blockNumber + blockConfirmations
+      const validatorsWaiting = validatorList.map(validator => ({
+        validator,
+        status: VALIDATOR_CONFIRMATION_STATUS.WAITING,
+        txHash: '',
+        timestamp: 0
+      }))
 
-      checkSignaturesWaitingForBLocks(
-        targetBlock,
-        setWaitingBlocks,
-        setWaitingBlocksResolved,
-        validatorList,
-        setConfirmations,
-        blockProvider,
-        interval,
-        subscriptions
-      )
+      const checkSignaturesWaitingForBLocks = () => {
+        const currentBlock = blockProvider.get()
 
-      return () => {
-        unsubscribe()
+        if (currentBlock && currentBlock >= targetBlock) {
+          setWaitingBlocksResolved(true)
+          setWaitingBlocks(false)
+        } else if (currentBlock) {
+          setWaitingBlocks(true)
+          setConfirmations(validatorsWaiting)
+          timeoutId = setTimeout(checkSignaturesWaitingForBLocks, interval)
+        } else {
+          timeoutId = setTimeout(checkSignaturesWaitingForBLocks, 500)
+        }
       }
+
+      checkSignaturesWaitingForBLocks()
+
+      return () => clearTimeout(timeoutId)
     },
-    [
-      blockConfirmations,
-      foreign.web3,
-      fromHome,
-      validatorList,
-      home.web3,
-      receipt,
-      setConfirmations,
-      waitingBlocksResolved
-    ]
+    [blockConfirmations, fromHome, receipt, validatorList, waitingBlocksResolved]
   )
 
   // The collected signature event is only fetched once the signatures are collected on tx from home to foreign, to calculate if
   // the execution tx on the foreign network is waiting for block confirmations
   // This is executed if the message is in Home to Foreign direction only
+  const hasCollectedSignatures = !!signatureCollected // true or string[]
   useEffect(
     () => {
-      if (!fromHome || !receipt || !home.web3 || !signatureCollected) return
+      if (!fromHome || !receipt || !home.web3 || !home.bridgeContract || !hasCollectedSignatures) return
 
-      const subscriptions: Array<number> = []
+      let timeoutId: number
+      let isCancelled = false
 
-      const unsubscribe = () => {
-        subscriptions.forEach(s => {
-          clearTimeout(s)
-        })
+      const messageHash = home.web3.utils.soliditySha3Raw(message.data)
+      const contract = home.bridgeContract
+
+      const getCollectedSignaturesEvent = async (fromBlock: number, toBlock: number) => {
+        const currentBlock = homeBlockNumberProvider.get()
+
+        if (currentBlock) {
+          // prevent errors if the toBlock parameter is bigger than the latest
+          const securedToBlock = toBlock >= currentBlock ? currentBlock : toBlock
+          const events = await contract.getPastEvents('CollectedSignatures', {
+            fromBlock,
+            toBlock: securedToBlock
+          })
+          const event = events.find(e => e.returnValues.messageHash === messageHash)
+          if (event) {
+            setCollectedSignaturesEvent(event)
+          } else if (!isCancelled) {
+            timeoutId = setTimeout(() => getCollectedSignaturesEvent(securedToBlock, securedToBlock + BLOCK_RANGE), 500)
+          }
+        } else if (!isCancelled) {
+          timeoutId = setTimeout(() => getCollectedSignaturesEvent(fromBlock, toBlock), 500)
+        }
       }
 
-      const fromBlock = receipt.blockNumber
-      const toBlock = fromBlock + BLOCK_RANGE
-      const messageHash = home.web3.utils.soliditySha3Raw(message.data)
-
-      getCollectedSignaturesEvent(
-        home.web3,
-        home.bridgeContract,
-        fromBlock,
-        toBlock,
-        messageHash,
-        setCollectedSignaturesEvent,
-        subscriptions
-      )
+      getCollectedSignaturesEvent(receipt.blockNumber, receipt.blockNumber + BLOCK_RANGE)
 
       return () => {
-        unsubscribe()
+        clearTimeout(timeoutId)
+        isCancelled = true
       }
     },
-    [fromHome, home.bridgeContract, home.web3, message.data, receipt, signatureCollected]
+    [fromHome, home.bridgeContract, home.web3, message.data, receipt, hasCollectedSignatures]
   )
 
   // Check if the responsible validator is waiting for block confirmations to execute the message on foreign network
   // This is executed if the message is in Home to Foreign direction only
   useEffect(
     () => {
-      if (!fromHome || !home.web3 || !receipt || !collectedSignaturesEvent || !blockConfirmations) return
+      if (!fromHome || !home.web3 || !collectedSignaturesEvent || !blockConfirmations) return
       if (waitingBlocksForExecutionResolved) return
 
-      const subscriptions: Array<number> = []
-
-      const unsubscribe = () => {
-        subscriptions.forEach(s => {
-          clearTimeout(s)
-        })
-      }
+      let timeoutId: number
 
       const targetBlock = collectedSignaturesEvent.blockNumber + blockConfirmations
 
-      checkWaitingBlocksForExecution(
-        homeBlockNumberProvider,
-        HOME_RPC_POLLING_INTERVAL,
-        targetBlock,
-        collectedSignaturesEvent,
-        setWaitingBlocksForExecution,
-        setWaitingBlocksForExecutionResolved,
-        setExecutionData,
-        subscriptions
-      )
+      const checkWaitingBlocksForExecution = () => {
+        const currentBlock = homeBlockNumberProvider.get()
 
-      return () => {
-        unsubscribe()
+        if (currentBlock && currentBlock >= targetBlock) {
+          const undefinedExecutionState = {
+            status: VALIDATOR_CONFIRMATION_STATUS.UNDEFINED,
+            validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay,
+            txHash: '',
+            timestamp: 0,
+            executionResult: false
+          }
+          setExecutionData(
+            (data: any) =>
+              data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED ||
+              data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING
+                ? undefinedExecutionState
+                : data
+          )
+          setWaitingBlocksForExecutionResolved(true)
+          setWaitingBlocksForExecution(false)
+        } else if (currentBlock) {
+          setWaitingBlocksForExecution(true)
+          const waitingExecutionState = {
+            status: VALIDATOR_CONFIRMATION_STATUS.WAITING,
+            validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay,
+            txHash: '',
+            timestamp: 0,
+            executionResult: false
+          }
+          setExecutionData(
+            (data: any) =>
+              data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED ||
+              data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING
+                ? waitingExecutionState
+                : data
+          )
+          timeoutId = setTimeout(() => checkWaitingBlocksForExecution(), HOME_RPC_POLLING_INTERVAL)
+        } else {
+          timeoutId = setTimeout(() => checkWaitingBlocksForExecution(), 500)
+        }
       }
+
+      checkWaitingBlocksForExecution()
+
+      return () => clearTimeout(timeoutId)
     },
-    [collectedSignaturesEvent, fromHome, blockConfirmations, home.web3, receipt, waitingBlocksForExecutionResolved]
+    [collectedSignaturesEvent, fromHome, blockConfirmations, home.web3, waitingBlocksForExecutionResolved]
   )
 
   // Checks if validators verified the message
   // To avoid making extra requests, this is only executed when validators finished waiting for blocks confirmations
   useEffect(
     () => {
-      if (!waitingBlocksResolved || !homeStartBlock || !requiredSignatures) return
+      if (!waitingBlocksResolved || !homeStartBlock || !requiredSignatures || !home.web3 || !home.bridgeContract) return
+      if (!validatorList || !validatorList.length) return
 
-      const subscriptions: Array<number> = []
-
-      const unsubscribe = () => {
-        subscriptions.forEach(s => {
-          clearTimeout(s)
-        })
-      }
+      let timeoutId: number
+      let isCancelled = false
 
       getConfirmationsForTx(
         message.data,
@@ -241,8 +261,8 @@ export const useMessageConfirmations = ({
         setConfirmations,
         requiredSignatures,
         setSignatureCollected,
-        waitingBlocksResolved,
-        subscriptions,
+        id => (timeoutId = id),
+        () => isCancelled,
         homeStartBlock,
         getValidatorFailedTransactionsForMessage,
         setFailedConfirmations,
@@ -252,7 +272,8 @@ export const useMessageConfirmations = ({
       )
 
       return () => {
-        unsubscribe()
+        clearTimeout(timeoutId)
+        isCancelled = true
       }
     },
     [
@@ -263,8 +284,7 @@ export const useMessageConfirmations = ({
       home.bridgeContract,
       requiredSignatures,
       waitingBlocksResolved,
-      homeStartBlock,
-      setConfirmations
+      homeStartBlock
     ]
   )
 
@@ -274,32 +294,23 @@ export const useMessageConfirmations = ({
   useEffect(
     () => {
       if ((fromHome && !waitingBlocksForExecutionResolved) || (!fromHome && !waitingBlocksResolved)) return
-      const startBlock = fromHome ? foreignStartBlock : homeStartBlock
-      if (!startBlock) return
 
-      const subscriptions: Array<number> = []
-
-      const unsubscribe = () => {
-        subscriptions.forEach(s => {
-          clearTimeout(s)
-        })
-      }
-
-      const contractEvent = fromHome ? 'RelayedMessage' : 'AffirmationCompleted'
       const bridgeContract = fromHome ? foreign.bridgeContract : home.bridgeContract
-      const providedWeb3 = fromHome ? foreign.web3 : home.web3
-      const interval = fromHome ? FOREIGN_RPC_POLLING_INTERVAL : HOME_RPC_POLLING_INTERVAL
+      const web3 = fromHome ? foreign.web3 : home.web3
+      const startBlock = fromHome ? foreignStartBlock : homeStartBlock
+      if (!startBlock || !bridgeContract || !web3) return
+
+      let timeoutId: number
+      let isCancelled = false
 
       getFinalizationEvent(
         fromHome,
         bridgeContract,
-        contractEvent,
-        providedWeb3,
+        web3,
         setExecutionData,
-        waitingBlocksResolved,
         message,
-        interval,
-        subscriptions,
+        id => (timeoutId = id),
+        () => isCancelled,
         startBlock,
         collectedSignaturesEvent,
         getExecutionFailedTransactionForMessage,
@@ -310,7 +321,8 @@ export const useMessageConfirmations = ({
       )
 
       return () => {
-        unsubscribe()
+        clearTimeout(timeoutId)
+        isCancelled = true
       }
     },
     [

alm/src/hooks/useTransactionFinder.ts

@@ -11,40 +11,23 @@ export const useTransactionFinder = ({ txHash, web3 }: { txHash: string; web3: M
     () => {
       if (!txHash || !web3) return
 
-      const subscriptions: number[] = []
+      let timeoutId: number
 
-      const unsubscribe = () => {
-        subscriptions.forEach(s => {
-          clearTimeout(s)
-        })
-      }
-
-      const getReceipt = async (
-        web3: Web3,
-        txHash: string,
-        setReceipt: Function,
-        setStatus: Function,
-        subscriptions: number[]
-      ) => {
+      const getReceipt = async () => {
         const txReceipt = await web3.eth.getTransactionReceipt(txHash)
         setReceipt(txReceipt)
 
         if (!txReceipt) {
           setStatus(TRANSACTION_STATUS.NOT_FOUND)
-          const timeoutId = setTimeout(
-            () => getReceipt(web3, txHash, setReceipt, setStatus, subscriptions),
-            HOME_RPC_POLLING_INTERVAL
-          )
-          subscriptions.push(timeoutId)
+          timeoutId = setTimeout(getReceipt, HOME_RPC_POLLING_INTERVAL)
         } else {
           setStatus(TRANSACTION_STATUS.FOUND)
         }
       }
 
-      getReceipt(web3, txHash, setReceipt, setStatus, subscriptions)
-      return () => {
-        unsubscribe()
-      }
+      getReceipt()
+
+      return () => clearTimeout(timeoutId)
     },
     [txHash, web3]
   )

alm/src/hooks/useTransactionStatus.ts

@@ -31,19 +31,14 @@ export const useTransactionStatus = ({
 
   useEffect(
     () => {
-      const subscriptions: Array<number> = []
+      if (!chainId || !txHash || !home.chainId || !foreign.chainId || !home.web3 || !foreign.web3) return
+      const isHome = chainId === home.chainId
+      const web3 = isHome ? home.web3 : foreign.web3
 
-      const unsubscribe = () => {
-        subscriptions.forEach(s => {
-          clearTimeout(s)
-        })
-      }
+      let timeoutId: number
 
       const getReceipt = async () => {
-        if (!chainId || !txHash || !home.chainId || !foreign.chainId || !home.web3 || !foreign.web3) return
         setLoading(true)
-        const isHome = chainId === home.chainId
-        const web3 = isHome ? home.web3 : foreign.web3
 
         let txReceipt
 
@@ -59,8 +54,7 @@ export const useTransactionStatus = ({
           setStatus(TRANSACTION_STATUS.NOT_FOUND)
           setDescription(getTransactionStatusDescription(TRANSACTION_STATUS.NOT_FOUND))
           setMessages([{ id: txHash, data: '' }])
-          const timeoutId = setTimeout(() => getReceipt(), HOME_RPC_POLLING_INTERVAL)
-          subscriptions.push(timeoutId)
+          timeoutId = setTimeout(() => getReceipt(), HOME_RPC_POLLING_INTERVAL)
         } else {
           const blockNumber = txReceipt.blockNumber
           const block = await getBlock(web3, blockNumber)
@@ -70,9 +64,9 @@ export const useTransactionStatus = ({
           if (txReceipt.status) {
             let bridgeMessages: Array<MessageObject>
             if (isHome) {
-              bridgeMessages = getHomeMessagesFromReceipt(txReceipt, home.web3, home.bridgeAddress)
+              bridgeMessages = getHomeMessagesFromReceipt(txReceipt, web3, home.bridgeAddress)
             } else {
-              bridgeMessages = getForeignMessagesFromReceipt(txReceipt, foreign.web3, foreign.bridgeAddress)
+              bridgeMessages = getForeignMessagesFromReceipt(txReceipt, web3, foreign.bridgeAddress)
             }
 
             if (bridgeMessages.length === 0) {
@@ -98,14 +92,9 @@ export const useTransactionStatus = ({
         setLoading(false)
       }
 
-      // unsubscribe from previous txHash
-      unsubscribe()
-
       getReceipt()
-      return () => {
-        // unsubscribe when unmount component
-        unsubscribe()
-      }
+
+      return () => clearTimeout(timeoutId)
     },
     [
       txHash,

alm/src/hooks/useValidatorContract.ts

@@ -6,6 +6,7 @@ import { BRIDGE_VALIDATORS_ABI } from '../abis'
 import { useStateProvider } from '../state/StateProvider'
 import { TransactionReceipt } from 'web3-eth'
 import { foreignSnapshotProvider, homeSnapshotProvider, SnapshotProvider } from '../services/SnapshotProvider'
+import { FOREIGN_EXPLORER_API, HOME_EXPLORER_API } from '../config/constants'
 
 export interface useValidatorContractParams {
   fromHome: boolean
@@ -30,10 +31,12 @@ export const useValidatorContract = ({ receipt, fromHome }: useValidatorContract
     contract: Maybe<Contract>,
     receipt: TransactionReceipt,
     setResult: Function,
-    snapshotProvider: SnapshotProvider
+    snapshotProvider: SnapshotProvider,
+    web3: Web3,
+    api: string
   ) => {
     if (!contract) return
-    const result = await getRequiredSignatures(contract, receipt.blockNumber, snapshotProvider)
+    const result = await getRequiredSignatures(contract, receipt.blockNumber, snapshotProvider, web3, api)
     setResult(result)
   }
 
@@ -41,32 +44,35 @@ export const useValidatorContract = ({ receipt, fromHome }: useValidatorContract
     contract: Maybe<Contract>,
     receipt: TransactionReceipt,
     setResult: Function,
-    snapshotProvider: SnapshotProvider
+    snapshotProvider: SnapshotProvider,
+    web3: Web3,
+    api: string
   ) => {
     if (!contract) return
-    const result = await getValidatorList(contract, receipt.blockNumber, snapshotProvider)
+    const result = await getValidatorList(contract, receipt.blockNumber, snapshotProvider, web3, api)
     setResult(result)
   }
 
+  const web3 = fromHome ? home.web3 : foreign.web3
+  const api = fromHome ? HOME_EXPLORER_API : FOREIGN_EXPLORER_API
+  const bridgeContract = fromHome ? home.bridgeContract : foreign.bridgeContract
+  const snapshotProvider = fromHome ? homeSnapshotProvider : foreignSnapshotProvider
+
   useEffect(
     () => {
-      const web3 = fromHome ? home.web3 : foreign.web3
-      const bridgeContract = fromHome ? home.bridgeContract : foreign.bridgeContract
-
       if (!web3 || !bridgeContract) return
       callValidatorContract(bridgeContract, web3, setValidatorContract)
     },
-    [home.web3, foreign.web3, home.bridgeContract, foreign.bridgeContract, fromHome]
+    [web3, bridgeContract]
   )
 
   useEffect(
     () => {
-      if (!receipt) return
-      const snapshotProvider = fromHome ? homeSnapshotProvider : foreignSnapshotProvider
-      callRequiredSignatures(validatorContract, receipt, setRequiredSignatures, snapshotProvider)
-      callValidatorList(validatorContract, receipt, setValidatorList, snapshotProvider)
+      if (!web3 || !receipt) return
+      callRequiredSignatures(validatorContract, receipt, setRequiredSignatures, snapshotProvider, web3, api)
+      callValidatorList(validatorContract, receipt, setValidatorList, snapshotProvider, web3, api)
     },
-    [validatorContract, receipt, fromHome]
+    [validatorContract, receipt, web3, snapshotProvider, api]
   )
 
   return {

alm/src/utils/__tests__/contracts.test.ts

@@ -16,7 +16,7 @@ describe('getRequiredBlockConfirmations', () => {
 
   test('Should call requiredBlockConfirmations method if no events present', async () => {
     const contract = ({
-      getPastEvents: () => {
+      getPastEvents: async () => {
         return []
       },
       methods: methodsBuilder('1')
@@ -37,7 +37,7 @@ describe('getRequiredBlockConfirmations', () => {
   })
   test('Should not call to get events if block number was included in the snapshot', async () => {
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => []),
+      getPastEvents: jest.fn().mockImplementation(async () => []),
       methods: methodsBuilder('3')
     } as unknown) as Contract
 
@@ -64,7 +64,7 @@ describe('getRequiredBlockConfirmations', () => {
   })
   test('Should call to get events if block number was not included in the snapshot', async () => {
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => [
+      getPastEvents: jest.fn().mockImplementation(async () => [
         {
           blockNumber: 9,
           returnValues: {
@@ -102,7 +102,7 @@ describe('getRequiredBlockConfirmations', () => {
   })
   test('Should use the most updated event', async () => {
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => [
+      getPastEvents: jest.fn().mockImplementation(async () => [
         {
           blockNumber: 9,
           returnValues: {
@@ -141,7 +141,7 @@ describe('getRequiredBlockConfirmations', () => {
 describe('getRequiredSignatures', () => {
   test('Should not call to get events if block number was included in the snapshot', async () => {
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => [])
+      getPastEvents: jest.fn().mockImplementation(async () => [])
     } as unknown) as Contract
 
     const snapshotProvider = ({
@@ -173,7 +173,7 @@ describe('getRequiredSignatures', () => {
   })
   test('Should call to get events if block number is higher than the snapshot block number', async () => {
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => [
+      getPastEvents: jest.fn().mockImplementation(async () => [
         {
           blockNumber: 15,
           returnValues: {
@@ -216,7 +216,7 @@ describe('getRequiredSignatures', () => {
   })
   test('Should use the most updated event before the block number', async () => {
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => [
+      getPastEvents: jest.fn().mockImplementation(async () => [
         {
           blockNumber: 15,
           returnValues: {
@@ -270,7 +270,7 @@ describe('getValidatorList', () => {
   test('Should return the current validator list if no events found', async () => {
     const currentValidators = [validator1, validator2, validator3]
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => []),
+      getPastEvents: jest.fn().mockImplementation(async () => []),
       methods: methodsBuilder(currentValidators)
     } as unknown) as Contract
 
@@ -301,7 +301,7 @@ describe('getValidatorList', () => {
   test('If validator was added later from snapshot it should not include it', async () => {
     const currentValidators = [validator1, validator2, validator3]
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => []),
+      getPastEvents: jest.fn().mockImplementation(async () => []),
       methods: methodsBuilder(currentValidators)
     } as unknown) as Contract
 
@@ -340,7 +340,7 @@ describe('getValidatorList', () => {
   test('If validator was added later from chain it should not include it', async () => {
     const currentValidators = [validator1, validator2, validator3]
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(event => {
+      getPastEvents: jest.fn().mockImplementation(async event => {
         if (event === 'ValidatorAdded') {
           return [
             {
@@ -385,7 +385,7 @@ describe('getValidatorList', () => {
   test('If validator was removed later from snapshot it should include it', async () => {
     const currentValidators = [validator1, validator2]
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(() => []),
+      getPastEvents: jest.fn().mockImplementation(async () => []),
       methods: methodsBuilder(currentValidators)
     } as unknown) as Contract
 
@@ -424,7 +424,7 @@ describe('getValidatorList', () => {
   test('If validator was removed later from chain it should include it', async () => {
     const currentValidators = [validator1, validator2]
     const contract = ({
-      getPastEvents: jest.fn().mockImplementation(event => {
+      getPastEvents: jest.fn().mockImplementation(async event => {
         if (event === 'ValidatorRemoved') {
           return [
             {

alm/src/utils/__tests__/getConfirmationsForTx.test.ts

@@ -34,7 +34,7 @@ const bridgeContract = {
   }
 } as Contract
 const requiredSignatures = 2
-const waitingBlocksResolved = true
+const isCancelled = () => false
 let subscriptions: Array<number> = []
 const timestamp = 1594045859
 const getFailedTransactions = (): Promise<APITransaction[]> => Promise.resolve([])
@@ -94,8 +94,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,
@@ -177,8 +177,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,
@@ -241,8 +241,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,
@@ -340,8 +340,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,
@@ -453,8 +453,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,
@@ -557,8 +557,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,
@@ -678,8 +678,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,
@@ -746,8 +746,8 @@ describe('getConfirmationsForTx', () => {
       setResult,
       requiredSignatures,
       setSignatureCollected,
-      waitingBlocksResolved,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       getFailedTransactions,
       setFailedConfirmations,

alm/src/utils/__tests__/getFinalizationEvent.test.ts

@@ -4,7 +4,6 @@ import Web3 from 'web3'
 import { getFinalizationEvent } from '../getFinalizationEvent'
 import { VALIDATOR_CONFIRMATION_STATUS } from '../../config/constants'
 
-const eventName = 'RelayedMessage'
 const timestamp = 1594045859
 const validator1 = '0x45b96809336A8b714BFbdAB3E4B5e0fe5d839908'
 const txHash = '0xdab36c9210e7e45fb82af10ffe4960461e41661dce0c9cd36b2843adaa1df156'
@@ -20,12 +19,11 @@ const web3 = ({
     toChecksumAddress: (a: string) => a
   }
 } as unknown) as Web3
-const waitingBlocksResolved = true
 const message = {
   id: '0x123',
   data: '0x123456789'
 }
-const interval = 10000
+const isCancelled = () => false
 let subscriptions: Array<number> = []
 
 const event = {
@@ -50,7 +48,7 @@ beforeEach(() => {
 describe('getFinalizationEvent', () => {
   test('should get finalization event and not try to get failed or pending transactions', async () => {
     const contract = ({
-      getPastEvents: () => {
+      getPastEvents: async () => {
         return [event]
       }
     } as unknown) as Contract
@@ -66,13 +64,11 @@ describe('getFinalizationEvent', () => {
     await getFinalizationEvent(
       true,
       contract,
-      eventName,
       web3,
       setResult,
-      waitingBlocksResolved,
       message,
-      interval,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       collectedSignaturesEvent,
       getFailedExecution,
@@ -102,7 +98,7 @@ describe('getFinalizationEvent', () => {
   })
   test('should retry to get finalization event and not try to get failed or pending transactions if foreign to home transaction', async () => {
     const contract = ({
-      getPastEvents: () => {
+      getPastEvents: async () => {
         return []
       }
     } as unknown) as Contract
@@ -118,13 +114,11 @@ describe('getFinalizationEvent', () => {
     await getFinalizationEvent(
       true,
       contract,
-      eventName,
       web3,
       setResult,
-      waitingBlocksResolved,
       message,
-      interval,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       collectedSignaturesEvent,
       getFailedExecution,
@@ -147,7 +141,7 @@ describe('getFinalizationEvent', () => {
   })
   test('should retry to get finalization event and try to get failed and pending transactions if home to foreign transaction', async () => {
     const contract = ({
-      getPastEvents: () => {
+      getPastEvents: async () => {
         return []
       },
       options: {
@@ -170,13 +164,11 @@ describe('getFinalizationEvent', () => {
     await getFinalizationEvent(
       true,
       contract,
-      eventName,
       web3,
       setResult,
-      waitingBlocksResolved,
       message,
-      interval,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       collectedSignaturesEvent,
       getFailedExecution,
@@ -199,7 +191,7 @@ describe('getFinalizationEvent', () => {
   })
   test('should retry to get finalization event and not to try to get failed transaction if pending transactions found if home to foreign transaction', async () => {
     const contract = ({
-      getPastEvents: () => {
+      getPastEvents: async () => {
         return []
       },
       options: {
@@ -222,13 +214,11 @@ describe('getFinalizationEvent', () => {
     await getFinalizationEvent(
       true,
       contract,
-      eventName,
       web3,
       setResult,
-      waitingBlocksResolved,
       message,
-      interval,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       collectedSignaturesEvent,
       getFailedExecution,
@@ -258,7 +248,7 @@ describe('getFinalizationEvent', () => {
   })
   test('should retry to get finalization event even if failed transaction found if home to foreign transaction', async () => {
     const contract = ({
-      getPastEvents: () => {
+      getPastEvents: async () => {
         return []
       },
       options: {
@@ -281,13 +271,11 @@ describe('getFinalizationEvent', () => {
     await getFinalizationEvent(
       true,
       contract,
-      eventName,
       web3,
       setResult,
-      waitingBlocksResolved,
       message,
-      interval,
-      subscriptions,
+      subscriptions.push.bind(subscriptions),
+      isCancelled,
       timestamp,
       collectedSignaturesEvent,
       getFailedExecution,

alm/src/utils/contract.ts

@@ -1,18 +1,33 @@
 import { Contract } from 'web3-eth-contract'
 import { EventData } from 'web3-eth-contract'
 import { SnapshotProvider } from '../services/SnapshotProvider'
+import { getLogs } from './explorer'
+import Web3 from 'web3'
+
+const getPastEventsWithFallback = (
+  api: string,
+  web3: Web3 | null,
+  contract: Contract,
+  eventName: string,
+  options: any
+) =>
+  contract
+    .getPastEvents(eventName, options)
+    .catch(() => (api && web3 ? getLogs(api, web3, contract, eventName, options) : []))
 
 export const getRequiredBlockConfirmations = async (
   contract: Contract,
   blockNumber: number,
-  snapshotProvider: SnapshotProvider
+  snapshotProvider: SnapshotProvider,
+  web3: Web3 | null = null,
+  api: string = ''
 ) => {
   const eventsFromSnapshot = snapshotProvider.requiredBlockConfirmationEvents(blockNumber)
   const snapshotBlockNumber = snapshotProvider.snapshotBlockNumber()
 
   let contractEvents: EventData[] = []
   if (blockNumber > snapshotBlockNumber) {
-    contractEvents = await contract.getPastEvents('RequiredBlockConfirmationChanged', {
+    contractEvents = await getPastEventsWithFallback(api, web3, contract, 'RequiredBlockConfirmationChanged', {
       fromBlock: snapshotBlockNumber + 1,
       toBlock: blockNumber
     })
@@ -38,14 +53,16 @@ export const getValidatorAddress = (contract: Contract) => contract.methods.vali
 export const getRequiredSignatures = async (
   contract: Contract,
   blockNumber: number,
-  snapshotProvider: SnapshotProvider
+  snapshotProvider: SnapshotProvider,
+  web3: Web3 | null = null,
+  api: string = ''
 ) => {
   const eventsFromSnapshot = snapshotProvider.requiredSignaturesEvents(blockNumber)
   const snapshotBlockNumber = snapshotProvider.snapshotBlockNumber()
 
   let contractEvents: EventData[] = []
   if (blockNumber > snapshotBlockNumber) {
-    contractEvents = await contract.getPastEvents('RequiredSignaturesChanged', {
+    contractEvents = await getPastEventsWithFallback(api, web3, contract, 'RequiredSignaturesChanged', {
       fromBlock: snapshotBlockNumber + 1,
       toBlock: blockNumber
     })
@@ -59,7 +76,13 @@ export const getRequiredSignatures = async (
   return parseInt(requiredSignatures)
 }
 
-export const getValidatorList = async (contract: Contract, blockNumber: number, snapshotProvider: SnapshotProvider) => {
+export const getValidatorList = async (
+  contract: Contract,
+  blockNumber: number,
+  snapshotProvider: SnapshotProvider,
+  web3: Web3 | null = null,
+  api: string = ''
+) => {
   const addedEventsFromSnapshot = snapshotProvider.validatorAddedEvents(blockNumber)
   const removedEventsFromSnapshot = snapshotProvider.validatorRemovedEvents(blockNumber)
   const snapshotBlockNumber = snapshotProvider.snapshotBlockNumber()
@@ -67,10 +90,10 @@ export const getValidatorList = async (contract: Contract, blockNumber: number,
   const fromBlock = snapshotBlockNumber > blockNumber ? snapshotBlockNumber + 1 : blockNumber
   const [currentList, added, removed] = await Promise.all([
     contract.methods.validatorList().call(),
-    contract.getPastEvents('ValidatorAdded', {
+    getPastEventsWithFallback(api, web3, contract, 'ValidatorAdded', {
       fromBlock
     }),
-    contract.getPastEvents('ValidatorRemoved', {
+    getPastEventsWithFallback(api, web3, contract, 'ValidatorRemoved', {
       fromBlock
     })
   ])

alm/src/utils/executionWaitingForBlocks.ts

@@ -1,70 +0,0 @@
-import { BlockNumberProvider } from '../services/BlockNumberProvider'
-import { VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
-import { EventData } from 'web3-eth-contract'
-
-export const checkWaitingBlocksForExecution = async (
-  blockProvider: BlockNumberProvider,
-  interval: number,
-  targetBlock: number,
-  collectedSignaturesEvent: EventData,
-  setWaitingBlocksForExecution: Function,
-  setWaitingBlocksForExecutionResolved: Function,
-  setExecutionData: Function,
-  subscriptions: number[]
-) => {
-  const currentBlock = blockProvider.get()
-
-  if (currentBlock && currentBlock >= targetBlock) {
-    const undefinedExecutionState = {
-      status: VALIDATOR_CONFIRMATION_STATUS.UNDEFINED,
-      validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay,
-      txHash: '',
-      timestamp: 0,
-      executionResult: false
-    }
-    setExecutionData(
-      (data: any) =>
-        data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED || data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING
-          ? undefinedExecutionState
-          : data
-    )
-    setWaitingBlocksForExecutionResolved(true)
-    setWaitingBlocksForExecution(false)
-  } else {
-    let nextInterval = interval
-    if (!currentBlock) {
-      nextInterval = 500
-    } else {
-      setWaitingBlocksForExecution(true)
-      const waitingExecutionState = {
-        status: VALIDATOR_CONFIRMATION_STATUS.WAITING,
-        validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay,
-        txHash: '',
-        timestamp: 0,
-        executionResult: false
-      }
-      setExecutionData(
-        (data: any) =>
-          data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED ||
-          data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING
-            ? waitingExecutionState
-            : data
-      )
-    }
-    const timeoutId = setTimeout(
-      () =>
-        checkWaitingBlocksForExecution(
-          blockProvider,
-          interval,
-          targetBlock,
-          collectedSignaturesEvent,
-          setWaitingBlocksForExecution,
-          setWaitingBlocksForExecutionResolved,
-          setExecutionData,
-          subscriptions
-        ),
-      nextInterval
-    )
-    subscriptions.push(timeoutId)
-  }
-}

alm/src/utils/explorer.ts

@@ -7,6 +7,9 @@ import {
   MAX_TX_SEARCH_BLOCK_RANGE,
   SUBMIT_SIGNATURE_HASH
 } from '../config/constants'
+import { AbiItem } from 'web3-utils'
+import Web3 from 'web3'
+import { Contract } from 'web3-eth-contract'
 
 export interface APITransaction {
   timeStamp: string
@@ -47,10 +50,15 @@ export interface GetTransactionParams extends GetPendingTransactionParams {
 }
 
 export const fetchAccountTransactions = async ({ account, startBlock, endBlock, api }: AccountTransactionsParams) => {
-  const params = `module=account&action=txlist&address=${account}&filterby=from&startblock=${startBlock}&endblock=${endBlock}`
-  const url = api.includes('blockscout') ? `${api}?${params}` : `${api}&${params}`
+  const url = new URL(api)
+  url.searchParams.append('module', 'account')
+  url.searchParams.append('action', 'txlist')
+  url.searchParams.append('address', account)
+  url.searchParams.append('filterby', 'from')
+  url.searchParams.append('startblock', startBlock.toString())
+  url.searchParams.append('endblock', endBlock.toString())
 
-  const result = await fetch(url).then(res => res.json())
+  const result = await fetch(url.toString()).then(res => res.json())
 
   if (result.message === 'No transactions found') {
     return []
@@ -66,10 +74,13 @@ export const fetchPendingTransactions = async ({
   if (!api.includes('blockscout')) {
     return []
   }
-  const url = `${api}?module=account&action=pendingtxlist&address=${account}`
+  const url = new URL(api)
+  url.searchParams.append('module', 'account')
+  url.searchParams.append('action', 'pendingtxlist')
+  url.searchParams.append('address', account)
 
   try {
-    const result = await fetch(url).then(res => res.json())
+    const result = await fetch(url.toString()).then(res => res.json())
     if (result.status === '0') {
       return []
     }
@@ -85,9 +96,13 @@ export const getClosestBlockByTimestamp = async (api: string, timestamp: number)
     throw new Error('Blockscout does not support getblocknobytime')
   }
 
-  const url = `${api}&module=block&action=getblocknobytime&timestamp=${timestamp}&closest=before`
+  const url = new URL(api)
+  url.searchParams.append('module', 'block')
+  url.searchParams.append('action', 'getblocknobytime')
+  url.searchParams.append('timestamp', timestamp.toString())
+  url.searchParams.append('closest', 'before')
 
-  const blockNumber = await fetch(url).then(res => res.json())
+  const blockNumber = await fetch(url.toString()).then(res => res.json())
 
   return parseInt(blockNumber.result)
 }
@@ -144,6 +159,41 @@ export const getAccountTransactions = async ({
   return transactionsCache[key].transactions
 }
 
+export const getLogs = async (
+  api: string,
+  web3: Web3,
+  contract: Contract,
+  event: string,
+  options: { fromBlock: number; toBlock: number | 'latest'; topics: (string | null)[] }
+) => {
+  const abi = contract.options.jsonInterface.find((abi: AbiItem) => abi.type === 'event' && abi.name === event)!
+
+  const url = new URL(api)
+  url.searchParams.append('module', 'logs')
+  url.searchParams.append('action', 'getLogs')
+  url.searchParams.append('address', contract.options.address)
+  url.searchParams.append('fromBlock', options.fromBlock.toString())
+  url.searchParams.append('toBlock', (options.toBlock || 'latest').toString())
+
+  const topics = [web3.eth.abi.encodeEventSignature(abi), ...(options.topics || [])]
+  for (let i = 0; i < topics.length; i++) {
+    if (topics[i] !== null) {
+      url.searchParams.append(`topic${i}`, topics[i] as string)
+      for (let j = 0; j < i; j++) {
+        url.searchParams.append(`topic${j}_${i}_opr`, 'and')
+      }
+    }
+  }
+
+  const logs = await fetch(url.toString()).then(res => res.json())
+
+  return logs.result.map((log: any) => ({
+    transactionHash: log.transactionHash,
+    blockNumber: parseInt(log.blockNumber.slice(2), 16),
+    returnValues: web3.eth.abi.decodeLog(abi.inputs!, log.data, log.topics.slice(1))
+  }))
+}
+
 const filterReceiver = (to: string) => (tx: APITransaction) => tx.to.toLowerCase() === to.toLowerCase()
 
 export const getFailedTransactions = async (

alm/src/utils/getCollectedSignaturesEvent.ts

@@ -1,52 +0,0 @@
-import Web3 from 'web3'
-import { Contract, EventData } from 'web3-eth-contract'
-import { homeBlockNumberProvider } from '../services/BlockNumberProvider'
-import { BLOCK_RANGE } from '../config/constants'
-
-export const getCollectedSignaturesEvent = async (
-  web3: Maybe<Web3>,
-  contract: Maybe<Contract>,
-  fromBlock: number,
-  toBlock: number,
-  messageHash: string,
-  setCollectedSignaturesEvent: Function,
-  subscriptions: number[]
-) => {
-  if (!web3 || !contract) return
-  const currentBlock = homeBlockNumberProvider.get()
-
-  let events: EventData[] = []
-  let securedToBlock = toBlock
-  if (currentBlock) {
-    // prevent errors if the toBlock parameter is bigger than the latest
-    securedToBlock = toBlock >= currentBlock ? currentBlock : toBlock
-    events = await contract.getPastEvents('CollectedSignatures', {
-      fromBlock,
-      toBlock: securedToBlock
-    })
-  }
-
-  const filteredEvents = events.filter(e => e.returnValues.messageHash === messageHash)
-
-  if (filteredEvents.length) {
-    const event = filteredEvents[0]
-    setCollectedSignaturesEvent(event)
-  } else {
-    const newFromBlock = currentBlock ? securedToBlock : fromBlock
-    const newToBlock = currentBlock ? toBlock + BLOCK_RANGE : toBlock
-    const timeoutId = setTimeout(
-      () =>
-        getCollectedSignaturesEvent(
-          web3,
-          contract,
-          newFromBlock,
-          newToBlock,
-          messageHash,
-          setCollectedSignaturesEvent,
-          subscriptions
-        ),
-      500
-    )
-    subscriptions.push(timeoutId)
-  }
-}

alm/src/utils/getConfirmationsForTx.ts

@@ -29,15 +29,15 @@ const mergeConfirmations = (oldConfirmations: BasicConfirmationParam[], newConfi
 
 export const getConfirmationsForTx = async (
   messageData: string,
-  web3: Maybe<Web3>,
+  web3: Web3,
   validatorList: string[],
-  bridgeContract: Maybe<Contract>,
+  bridgeContract: Contract,
   fromHome: boolean,
   setResult: Function,
   requiredSignatures: number,
   setSignatureCollected: Function,
-  waitingBlocksResolved: boolean,
-  subscriptions: number[],
+  setTimeoutId: (timeoutId: number) => void,
+  isCancelled: () => boolean,
   startBlock: number,
   getFailedTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>,
   setFailedConfirmations: Function,
@@ -45,8 +45,6 @@ export const getConfirmationsForTx = async (
   setPendingConfirmations: Function,
   getSuccessTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>
 ) => {
-  if (!web3 || !validatorList || !validatorList.length || !bridgeContract || !waitingBlocksResolved) return
-
   const confirmationContractMethod = fromHome ? getMessagesSigned : getAffirmationsSigned
 
   const hashMsg = web3.utils.soliditySha3Raw(messageData)
@@ -144,28 +142,30 @@ export const getConfirmationsForTx = async (
     (!hasEnoughSignatures && missingConfirmations.length > 0) ||
     successConfirmationWithTxFound.length < successConfirmationWithData.length
   ) {
-    const timeoutId = setTimeout(
-      () =>
-        getConfirmationsForTx(
-          messageData,
-          web3,
-          validatorList,
-          bridgeContract,
-          fromHome,
-          setResult,
-          requiredSignatures,
-          setSignatureCollected,
-          waitingBlocksResolved,
-          subscriptions,
-          startBlock,
-          getFailedTransactions,
-          setFailedConfirmations,
-          getPendingTransactions,
-          setPendingConfirmations,
-          getSuccessTransactions
-        ),
-      HOME_RPC_POLLING_INTERVAL
-    )
-    subscriptions.push(timeoutId)
+    if (!isCancelled()) {
+      const timeoutId = setTimeout(
+        () =>
+          getConfirmationsForTx(
+            messageData,
+            web3,
+            validatorList,
+            bridgeContract,
+            fromHome,
+            setResult,
+            requiredSignatures,
+            setSignatureCollected,
+            setTimeoutId,
+            isCancelled,
+            startBlock,
+            getFailedTransactions,
+            setFailedConfirmations,
+            getPendingTransactions,
+            setPendingConfirmations,
+            getSuccessTransactions
+          ),
+        HOME_RPC_POLLING_INTERVAL
+      )
+      setTimeoutId(timeoutId)
+    }
   }
 }

alm/src/utils/getFinalizationEvent.ts

@@ -1,16 +1,47 @@
 import { Contract, EventData } from 'web3-eth-contract'
 import Web3 from 'web3'
-import { CACHE_KEY_EXECUTION_FAILED, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
+import {
+  CACHE_KEY_EXECUTION_FAILED,
+  FOREIGN_EXPLORER_API,
+  FOREIGN_RPC_POLLING_INTERVAL,
+  HOME_EXPLORER_API,
+  HOME_RPC_POLLING_INTERVAL,
+  VALIDATOR_CONFIRMATION_STATUS
+} from '../config/constants'
 import { ExecutionData } from '../hooks/useMessageConfirmations'
-import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer'
+import {
+  APIPendingTransaction,
+  APITransaction,
+  GetTransactionParams,
+  GetPendingTransactionParams,
+  getLogs
+} from './explorer'
 import { getBlock, MessageObject } from './web3'
 import validatorsCache from '../services/ValidatorsCache'
 import { foreignBlockNumberProvider, homeBlockNumberProvider } from '../services/BlockNumberProvider'
 
-export const getSuccessExecutionData = async (contract: Contract, eventName: string, web3: Web3, messageId: string) => {
+const getPastEventsWithFallback = (api: string, web3: Web3, contract: Contract, eventName: string, options: any) =>
+  contract.getPastEvents(eventName, options).catch(
+    () =>
+      api
+        ? getLogs(api, web3, contract, eventName, {
+            fromBlock: options.fromBlock,
+            toBlock: options.toBlock,
+            topics: [null, null, options.filter.messageId]
+          })
+        : []
+  )
+
+export const getSuccessExecutionData = async (
+  contract: Contract,
+  eventName: string,
+  web3: Web3,
+  messageId: string,
+  api: string = ''
+) => {
   // Since it filters by the message id, only one event will be fetched
   // so there is no need to limit the range of the block to reduce the network traffic
-  const events: EventData[] = await contract.getPastEvents(eventName, {
+  const events: EventData[] = await getPastEventsWithFallback(api, web3, contract, eventName, {
     fromBlock: 0,
     toBlock: 'latest',
     filter: {
@@ -40,14 +71,12 @@ export const getSuccessExecutionData = async (contract: Contract, eventName: str
 
 export const getFinalizationEvent = async (
   fromHome: boolean,
-  contract: Maybe<Contract>,
-  eventName: string,
-  web3: Maybe<Web3>,
+  contract: Contract,
+  web3: Web3,
   setResult: React.Dispatch<React.SetStateAction<ExecutionData>>,
-  waitingBlocksResolved: boolean,
   message: MessageObject,
-  interval: number,
-  subscriptions: number[],
+  setTimeoutId: (timeoutId: number) => void,
+  isCancelled: () => boolean,
   startBlock: number,
   collectedSignaturesEvent: Maybe<EventData>,
   getFailedExecution: (args: GetTransactionParams) => Promise<APITransaction[]>,
@@ -56,8 +85,11 @@ export const getFinalizationEvent = async (
   setPendingExecution: Function,
   setExecutionEventsFetched: Function
 ) => {
-  if (!contract || !web3 || !waitingBlocksResolved) return
-  const successExecutionData = await getSuccessExecutionData(contract, eventName, web3, message.id)
+  const eventName = fromHome ? 'RelayedMessage' : 'AffirmationCompleted'
+  const api = fromHome ? FOREIGN_EXPLORER_API : HOME_EXPLORER_API
+
+  const successExecutionData = await getSuccessExecutionData(contract, eventName, web3, message.id, api)
+
   if (successExecutionData) {
     setResult(successExecutionData)
   } else {
@@ -120,28 +152,28 @@ export const getFinalizationEvent = async (
       }
     }
 
-    const timeoutId = setTimeout(
-      () =>
-        getFinalizationEvent(
-          fromHome,
-          contract,
-          eventName,
-          web3,
-          setResult,
-          waitingBlocksResolved,
-          message,
-          interval,
-          subscriptions,
-          startBlock,
-          collectedSignaturesEvent,
-          getFailedExecution,
-          setFailedExecution,
-          getPendingExecution,
-          setPendingExecution,
-          setExecutionEventsFetched
-        ),
-      interval
-    )
-    subscriptions.push(timeoutId)
+    if (!isCancelled()) {
+      const timeoutId = setTimeout(
+        () =>
+          getFinalizationEvent(
+            fromHome,
+            contract,
+            web3,
+            setResult,
+            message,
+            setTimeoutId,
+            isCancelled,
+            startBlock,
+            collectedSignaturesEvent,
+            getFailedExecution,
+            setFailedExecution,
+            getPendingExecution,
+            setPendingExecution,
+            setExecutionEventsFetched
+          ),
+        fromHome ? FOREIGN_RPC_POLLING_INTERVAL : HOME_RPC_POLLING_INTERVAL
+      )
+      setTimeoutId(timeoutId)
+    }
   }
 }

alm/src/utils/signatureWaitingForBlocks.ts

@@ -1,49 +0,0 @@
-import { VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
-import { BlockNumberProvider } from '../services/BlockNumberProvider'
-
-export const checkSignaturesWaitingForBLocks = async (
-  targetBlock: number,
-  setWaitingStatus: Function,
-  setWaitingBlocksResolved: Function,
-  validatorList: string[],
-  setConfirmations: Function,
-  blockProvider: BlockNumberProvider,
-  interval: number,
-  subscriptions: number[]
-) => {
-  const currentBlock = blockProvider.get()
-
-  if (currentBlock && currentBlock >= targetBlock) {
-    setWaitingBlocksResolved(true)
-    setWaitingStatus(false)
-  } else {
-    let nextInterval = interval
-    if (!currentBlock) {
-      nextInterval = 500
-    } else {
-      const validatorsWaiting = validatorList.map(validator => {
-        return {
-          validator,
-          status: VALIDATOR_CONFIRMATION_STATUS.WAITING
-        }
-      })
-      setWaitingStatus(true)
-      setConfirmations(validatorsWaiting)
-    }
-    const timeoutId = setTimeout(
-      () =>
-        checkSignaturesWaitingForBLocks(
-          targetBlock,
-          setWaitingStatus,
-          setWaitingBlocksResolved,
-          validatorList,
-          setConfirmations,
-          blockProvider,
-          interval,
-          subscriptions
-        ),
-      nextInterval
-    )
-    subscriptions.push(timeoutId)
-  }
-}

deployment-e2e/molecule/multiple/tests/test_multiple.py

@@ -24,6 +24,7 @@ def test_services(host, service):
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
     ("ui_ui_1"),
     ("monitor_monitor_1")
 ])

deployment-e2e/molecule/oracle/tests/test_oracle.py

@@ -14,6 +14,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     ("oracle_bridge_affirmation_1"),
     ("oracle_bridge_senderhome_1"),
     ("oracle_bridge_senderforeign_1"),
+    ("oracle_bridge_shutdown_1"),
 ])
 def test_docker_containers(host, name):
     container = host.docker(name)

e2e-commons/up.sh

@@ -42,6 +42,7 @@ startValidator () {
     fi
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle yarn manager:shutdown
 }
 
 startAMBValidator () {
@@ -52,6 +53,7 @@ startAMBValidator () {
     docker-compose $1 run $2 $3 -d oracle-amb yarn watcher:affirmation-request
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:home
     docker-compose $1 run $2 $3 -d oracle-amb yarn sender:foreign
+    docker-compose $1 run $2 $3 -d oracle-amb yarn manager:shutdown
 }
 
 while [ "$1" != "" ]; do
@@ -120,13 +122,7 @@ while [ "$1" != "" ]; do
   fi
 
   if [ "$1" == "alm-e2e" ]; then
-    docker-compose up -d redis rabbit
-
-    docker-compose run -d oracle-amb yarn watcher:signature-request
-    docker-compose run -d oracle-amb yarn watcher:collected-signatures
-    docker-compose run -d oracle-amb yarn watcher:affirmation-request
-    docker-compose run -d oracle-amb yarn sender:home
-    docker-compose run -d oracle-amb yarn sender:foreign
+    startAMBValidator "" "" "" "redis" "rabbit"
 
     oracle2name="-p validator2"
     oracle2Values="-e ORACLE_VALIDATOR_ADDRESS=0xdCC784657C78054aa61FbcFFd2605F32374816A4 -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5a5c3645d0f04e9eb4f27f94ed4c244a225587405b8838e7456f7781ce3a9513"

monitor/alerts.js

@@ -1,18 +1,19 @@
 require('dotenv').config()
 const logger = require('./logger')('alerts')
-const eventsInfo = require('./utils/events')
 const { processedMsgNotDelivered, eventWithoutReference } = require('./utils/message')
 const { BRIDGE_MODES } = require('../commons')
-const { web3Home, web3Foreign, getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3')
+const { web3Home, web3Foreign } = require('./utils/web3')
 
-async function main() {
+async function main(eventsInfo) {
   const {
+    homeBlockNumber,
+    foreignBlockNumber,
     homeToForeignRequests,
     homeToForeignConfirmations,
     foreignToHomeConfirmations,
     foreignToHomeRequests,
     bridgeMode
-  } = await eventsInfo()
+  } = eventsInfo
 
   let xSignatures
   let xAffirmations
@@ -24,8 +25,6 @@ async function main() {
     xAffirmations = foreignToHomeConfirmations.filter(eventWithoutReference(foreignToHomeRequests))
   }
   logger.debug('building misbehavior blocks')
-  const homeBlockNumber = await getHomeBlockNumber()
-  const foreignBlockNumber = await getForeignBlockNumber()
 
   const baseRange = [false, false, false, false, false]
   const xSignaturesMisbehavior = buildRangesObject(

monitor/checkWorker2.js

@@ -1,6 +1,7 @@
 require('dotenv').config()
 const logger = require('./logger')('checkWorker2')
 const eventsStats = require('./eventsStats')
+const getEventsInfo = require('./utils/events')
 const alerts = require('./alerts')
 const { writeFile, createDir } = require('./utils/file')
 const { saveCache } = require('./utils/web3Cache')
@@ -10,8 +11,10 @@ const { MONITOR_BRIDGE_NAME } = process.env
 async function checkWorker2() {
   try {
     createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
+    logger.debug('calling getEventsInfo()')
+    const eventsInfo = await getEventsInfo()
     logger.debug('calling eventsStats()')
-    const evStats = await eventsStats()
+    const evStats = await eventsStats(eventsInfo)
     if (!evStats) throw new Error('evStats is empty: ' + JSON.stringify(evStats))
     evStats.ok =
       (evStats.onlyInHomeDeposits || evStats.home.deliveredMsgNotProcessedInForeign).length === 0 &&
@@ -22,7 +25,7 @@ async function checkWorker2() {
     writeFile(`/responses/${MONITOR_BRIDGE_NAME}/eventsStats.json`, evStats)
 
     logger.debug('calling alerts()')
-    const _alerts = await alerts()
+    const _alerts = await alerts(eventsInfo)
     if (!_alerts) throw new Error('alerts is empty: ' + JSON.stringify(_alerts))
     _alerts.ok = !_alerts.executeAffirmations.mostRecentTxHash && !_alerts.executeSignatures.mostRecentTxHash
     _alerts.health = true

monitor/eventsStats.js

@@ -1,5 +1,4 @@
 require('dotenv').config()
-const eventsInfo = require('./utils/events')
 const {
   processedMsgNotDelivered,
   deliveredMsgNotProcessed,
@@ -15,14 +14,14 @@ const {
   MONITOR_HOME_TO_FOREIGN_CHECK_SENDER
 } = process.env
 
-async function main() {
+async function main(eventsInfo) {
   const {
     homeToForeignRequests,
     homeToForeignConfirmations,
     foreignToHomeConfirmations,
     foreignToHomeRequests,
     bridgeMode
-  } = await eventsInfo()
+  } = eventsInfo
 
   if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
     return {

monitor/getBalances.js

@@ -5,7 +5,12 @@ const logger = require('./logger')('getBalances')
 const { BRIDGE_MODES } = require('../commons')
 const { web3Home, web3Foreign, getHomeBlockNumber } = require('./utils/web3')
 
-const { COMMON_HOME_BRIDGE_ADDRESS, COMMON_FOREIGN_BRIDGE_ADDRESS } = process.env
+const {
+  MONITOR_HOME_START_BLOCK,
+  MONITOR_FOREIGN_START_BLOCK,
+  COMMON_HOME_BRIDGE_ADDRESS,
+  COMMON_FOREIGN_BRIDGE_ADDRESS
+} = process.env
 
 const {
   ERC20_ABI,
@@ -20,6 +25,8 @@ const {
 
 async function main(bridgeMode, eventsInfo) {
   const {
+    homeBlockNumber,
+    foreignBlockNumber,
     homeToForeignConfirmations,
     foreignToHomeConfirmations,
     homeDelayedBlockNumber,
@@ -46,6 +53,13 @@ async function main(bridgeMode, eventsInfo) {
     ...foreignToHomeConfirmations.filter(e => e.blockNumber > homeDelayedBlockNumber).map(e => e.value)
   )
 
+  const blockRanges = {
+    startBlockHome: MONITOR_HOME_START_BLOCK,
+    endBlockHome: homeBlockNumber,
+    startBlockForeign: MONITOR_FOREIGN_START_BLOCK,
+    endBlockForeign: foreignBlockNumber
+  }
+
   if (bridgeMode === BRIDGE_MODES.ERC_TO_ERC) {
     const foreignBridge = new web3Foreign.eth.Contract(FOREIGN_ERC_TO_ERC_ABI, COMMON_FOREIGN_BRIDGE_ADDRESS)
     const erc20Address = await foreignBridge.methods.erc20token().call()
@@ -72,6 +86,7 @@ async function main(bridgeMode, eventsInfo) {
         erc20Balance: Web3Utils.fromWei(foreignErc20Balance)
       },
       balanceDiff: Number(Web3Utils.fromWei(diff)),
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else if (bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC || bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC_V1) {
@@ -94,6 +109,7 @@ async function main(bridgeMode, eventsInfo) {
         totalSupply: Web3Utils.fromWei(totalSupply)
       },
       balanceDiff: Number(Web3Utils.fromWei(diff)),
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else if (bridgeMode === BRIDGE_MODES.ERC_TO_NATIVE) {
@@ -163,12 +179,14 @@ async function main(bridgeMode, eventsInfo) {
       },
       foreign,
       balanceDiff: Number(Web3Utils.fromWei(diff)),
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
     return {
       home: {},
       foreign: {},
+      ...blockRanges,
       lastChecked: Math.floor(Date.now() / 1000)
     }
   } else {

monitor/index.js

@@ -2,7 +2,6 @@ require('dotenv').config()
 const express = require('express')
 const cors = require('cors')
 const { readFile } = require('./utils/file')
-const { getPrometheusMetrics } = require('./prometheusMetrics')
 
 const app = express()
 const bridgeRouter = express.Router({ mergeParams: true })
@@ -25,7 +24,8 @@ bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|
 
 bridgeRouter.get('/metrics', (req, res, next) => {
   try {
-    const metrics = getPrometheusMetrics(req.params.bridgeName)
+    const { bridgeName } = req.params
+    const metrics = readFile(`./responses/${bridgeName}/metrics.txt`, false)
     res.type('text').send(metrics)
   } catch (e) {
     next(e)

monitor/metricsWorker.js

@@ -0,0 +1,20 @@
+require('dotenv').config()
+const logger = require('./logger')('metricsWorker')
+const { writeFile, createDir } = require('./utils/file')
+const getPrometheusMetrics = require('./prometheusMetrics')
+
+const { MONITOR_BRIDGE_NAME } = process.env
+
+async function metricsWorker() {
+  try {
+    createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
+    logger.debug('calling getPrometheusMetrics()')
+    const metrics = await getPrometheusMetrics(MONITOR_BRIDGE_NAME)
+    if (!metrics) throw new Error('metrics is empty: ' + JSON.stringify(metrics))
+    writeFile(`/responses/${MONITOR_BRIDGE_NAME}/metrics.txt`, metrics, { stringify: false })
+    logger.debug('Done')
+  } catch (e) {
+    logger.error(e)
+  }
+}
+metricsWorker()

monitor/package.json

@@ -4,7 +4,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js",
+    "check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js && timeout -s 9 10s node metricsWorker.js",
     "start": "node index.js",
     "check-and-start": "yarn check-all && yarn start",
     "lint": "eslint . --ignore-path ../.eslintignore",

monitor/prometheusMetrics.js

@@ -1,8 +1,10 @@
+require('dotenv').config()
+const logger = require('./logger')('getBalances')
 const { readFile } = require('./utils/file')
 
 const {
-  MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST,
-  MONITOR_HOME_TO_FOREIGN_BLOCK_LIST,
+  MONITOR_HOME_START_BLOCK,
+  MONITOR_FOREIGN_START_BLOCK,
   MONITOR_HOME_VALIDATORS_BALANCE_ENABLE,
   MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE
 } = process.env
@@ -23,7 +25,9 @@ function hasError(obj) {
   return 'error' in obj
 }
 
-function getPrometheusMetrics(bridgeName) {
+// Try to collect all metrics from JSON responses and then
+// discard all unsuccessfully retrieved ones
+async function getPrometheusMetrics(bridgeName) {
   const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json`
 
   const metrics = {}
@@ -32,22 +36,44 @@ function getPrometheusMetrics(bridgeName) {
   const balancesFile = readFile(responsePath('getBalances'))
 
   if (!hasError(balancesFile)) {
-    const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile
-    metrics.balances_home_value = homeBalances.totalSupply
-    metrics.balances_home_txs_deposit = homeBalances.deposits
-    metrics.balances_home_txs_withdrawal = homeBalances.withdrawals
+    const { home, foreign, ...commonBalances } = balancesFile
 
-    metrics.balances_foreign_value = foreignBalances.erc20Balance
-    metrics.balances_foreign_txs_deposit = foreignBalances.deposits
-    metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals
+    const balanceMetrics = {
+      // ERC_TO_ERC or ERC_TO_NATIVE mode
+      balances_home_value: home.totalSupply,
+      balances_home_txs_deposit: home.deposits,
+      balances_home_txs_withdrawal: home.withdrawals,
+      balances_foreign_value: foreign.erc20Balance,
+      balances_foreign_txs_deposit: foreign.deposits,
+      balances_foreign_txs_withdrawal: foreign.withdrawals,
 
-    metrics.balances_diff_value = commonBalances.balanceDiff
-    metrics.balances_diff_deposit = commonBalances.depositsDiff
-    metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff
-    if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) {
-      metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff
-      metrics.balances_unclaimed_value = commonBalances.unclaimedBalance
+      // Not ARBITRARY_MESSAGE mode
+      balances_diff_value: commonBalances.balanceDiff,
+      balances_diff_deposit: commonBalances.depositsDiff,
+      balances_diff_withdrawal: commonBalances.withdrawalDiff,
+
+      // MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST or MONITOR_HOME_TO_FOREIGN_BLOCK_LIST is set
+      balances_unclaimed_txs: commonBalances.unclaimedDiff,
+      balances_unclaimed_value: commonBalances.unclaimedBalance,
+
+      // ARBITRARY_MESSAGE mode
+      txs_home_out: home.toForeign,
+      txs_home_in: home.fromForeign,
+      txs_foreign_out: foreign.toHome,
+      txs_foreign_in: foreign.fromHome,
+      txs_diff_home_out_oracles: commonBalances.fromHomeToForeignDiff,
+      txs_diff_home_out_users: commonBalances.fromHomeToForeignPBUDiff,
+      txs_diff_foreign_out: commonBalances.fromForeignToHomeDiff
     }
+
+    const blockRanges = {
+      state_startblock_home: commonBalances.startBlockHome || MONITOR_HOME_START_BLOCK,
+      state_startblock_foreign: commonBalances.startBlockForeign || MONITOR_FOREIGN_START_BLOCK,
+      state_endblock_home: commonBalances.endBlockHome,
+      state_endblock_foreign: commonBalances.endBlockForeign
+    }
+
+    Object.assign(metrics, blockRanges, balanceMetrics)
   }
 
   // Validator metrics
@@ -62,7 +88,11 @@ function getPrometheusMetrics(bridgeName) {
           : Object.keys(allValidators)
 
       validatorAddressesWithBalanceCheck.forEach((addr, ind) => {
-        metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance
+        if (addr in allValidators) {
+          metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance
+        } else {
+          logger.debug(`Nonexistent validator address ${addr}`)
+        }
       })
     }
   }
@@ -96,9 +126,12 @@ function getPrometheusMetrics(bridgeName) {
   return Object.entries(metrics).reduceRight(
     // Prometheus supports `Nan` and possibly signed `Infinity`
     // in case cast to `Number` fails
-    (acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`,
+    (acc, [key, val]) => {
+      if (typeof val === 'undefined') return acc
+      else return `${key} ${Number(val)}\n${acc}`
+    },
     ''
   )
 }
 
-module.exports = { getPrometheusMetrics }
+module.exports = getPrometheusMetrics

monitor/utils/events.js

@@ -236,13 +236,15 @@ async function main(mode) {
     foreignToHomeRequests,
     isExternalErc20,
     bridgeMode,
+    homeBlockNumber,
+    foreignBlockNumber,
     homeDelayedBlockNumber,
     foreignDelayedBlockNumber
   }
 
   if (MONITOR_CACHE_EVENTS === 'true') {
     logger.debug('saving obtained events into cache file')
-    writeFile(cacheFilePath, result, false)
+    writeFile(cacheFilePath, result, { useCwd: false })
   }
   return result
 }

monitor/utils/file.js

@@ -1,23 +1,30 @@
 const fs = require('fs')
 const path = require('path')
 
-function readFile(filePath) {
+function readFile(filePath, parseJson = true) {
   try {
     const content = fs.readFileSync(filePath)
+    if (!parseJson) return content
     const json = JSON.parse(content)
     const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked
     return Object.assign({}, json, { timeDiff })
   } catch (e) {
-    console.error(e)
+    console.error('readFlle', e)
     return {
       error: 'the bridge statistics are not available'
     }
   }
 }
 
-function writeFile(filePath, object, useCwd = true) {
+function writeFile(filePath, object, paramOptions = {}) {
+  const defaultOptions = {
+    useCwd: true,
+    stringify: true
+  }
+  const { useCwd, stringify } = Object.assign({}, defaultOptions, paramOptions)
+
   const fullPath = useCwd ? path.join(process.cwd(), filePath) : filePath
-  fs.writeFileSync(fullPath, JSON.stringify(object, null, 4))
+  fs.writeFileSync(fullPath, stringify ? JSON.stringify(object, null, 4) : object)
 }
 
 function createDir(dirPath) {

oracle/.env.example

@@ -14,12 +14,14 @@ COMMON_HOME_GAS_PRICE_SPEED_TYPE=standard
 COMMON_HOME_GAS_PRICE_FALLBACK=1000000000
 ORACLE_HOME_GAS_PRICE_UPDATE_INTERVAL=600000
 COMMON_HOME_GAS_PRICE_FACTOR=1
+ORACLE_HOME_TX_RESEND_INTERVAL=300000
 
 COMMON_FOREIGN_GAS_PRICE_SUPPLIER_URL=https://gasprice.poa.network/
 COMMON_FOREIGN_GAS_PRICE_SPEED_TYPE=standard
 COMMON_FOREIGN_GAS_PRICE_FALLBACK=1000000000
 ORACLE_FOREIGN_GAS_PRICE_UPDATE_INTERVAL=600000
 COMMON_FOREIGN_GAS_PRICE_FACTOR=1
+ORACLE_FOREIGN_TX_RESEND_INTERVAL=1200000
 
 ORACLE_QUEUE_URL=amqp://rabbit
 ORACLE_REDIS_URL=redis://redis

oracle/config/base.config.js

@@ -69,7 +69,8 @@ const bridgeConfig = {
   foreignBridgeAbi: foreignAbi,
   eventFilter: {},
   validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY),
-  maxProcessingTime
+  maxProcessingTime,
+  shutdownKey: 'oracle-shutdown'
 }
 
 const homeConfig = {

oracle/config/foreign-sender.config.js

@@ -1,7 +1,10 @@
 const baseConfig = require('./base.config')
 
+const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants')
 const { web3Foreign, web3ForeignRedundant, web3ForeignFallback } = require('../src/services/web3')
 
+const { ORACLE_FOREIGN_TX_RESEND_INTERVAL } = process.env
+
 module.exports = {
   ...baseConfig.bridgeConfig,
   queue: 'foreign-prioritized',
@@ -10,5 +13,6 @@ module.exports = {
   name: 'sender-foreign',
   web3: web3Foreign,
   web3Redundant: web3ForeignRedundant,
-  web3Fallback: web3ForeignFallback
+  web3Fallback: web3ForeignFallback,
+  resendInterval: parseInt(ORACLE_FOREIGN_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL
 }

oracle/config/home-sender.config.js

@@ -1,7 +1,10 @@
 const baseConfig = require('./base.config')
 
+const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants')
 const { web3Home, web3HomeRedundant, web3HomeFallback } = require('../src/services/web3')
 
+const { ORACLE_HOME_TX_RESEND_INTERVAL } = process.env
+
 module.exports = {
   ...baseConfig.bridgeConfig,
   queue: 'home-prioritized',
@@ -10,5 +13,6 @@ module.exports = {
   name: 'sender-home',
   web3: web3Home,
   web3Redundant: web3HomeRedundant,
-  web3Fallback: web3HomeFallback
+  web3Fallback: web3HomeFallback,
+  resendInterval: parseInt(ORACLE_HOME_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL
 }

oracle/config/shutdown-manager.config.js

@@ -0,0 +1,20 @@
+const baseConfig = require('./base.config')
+
+const {
+  ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL,
+  ORACLE_SHUTDOWN_SERVICE_URL,
+  ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  ORACLE_SHUTDOWN_CONTRACT_METHOD
+} = process.env
+
+module.exports = {
+  ...baseConfig.bridgeConfig,
+  id: 'shutdown-manager',
+  name: 'shutdown-manager',
+  pollingInterval: ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL || 120000,
+  checksBeforeResume: 3,
+  checksBeforeStop: 1,
+  shutdownServiceURL: ORACLE_SHUTDOWN_SERVICE_URL,
+  shutdownContractAddress: ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
+  shutdownMethod: (ORACLE_SHUTDOWN_CONTRACT_METHOD || 'isShutdown()').trim()
+}

oracle/docker-compose-erc-native.yml

@@ -77,6 +77,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -91,6 +97,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose-transfer.yml

@@ -61,6 +61,12 @@ services:
     networks:
       - net_db_bridge_request
       - net_rabbit_bridge_request
+  bridge_shutdown:
+    extends:
+      file: docker-compose.yml
+      service: bridge_shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -75,6 +81,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/docker-compose.yml

@@ -21,12 +21,13 @@ services:
     command: [redis-server, --appendonly, 'yes']
     hostname: redis
     image: redis:4
-    networks:    
+    networks:
       - net_db_bridge_request
       - net_db_bridge_collected
       - net_db_bridge_affirmation
       - net_db_bridge_senderhome
       - net_db_bridge_senderforeign
+      - net_db_bridge_shutdown
     restart: unless-stopped
     volumes: ['~/bridge_data/redis:/data']
   bridge_request:
@@ -34,8 +35,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn watcher:signature-request
@@ -47,8 +48,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS}
     restart: unless-stopped
     entrypoint: yarn watcher:collected-signatures
@@ -60,8 +61,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS}
     restart: unless-stopped
     entrypoint: yarn watcher:affirmation-request
@@ -73,8 +74,8 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn sender:home
@@ -86,14 +87,25 @@ services:
     mem_limit: 500m
     image: poanetwork/tokenbridge-oracle:latest
     env_file: ./.env
-    environment: 
-      - NODE_ENV=production 
+    environment:
+      - NODE_ENV=production
       - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}
     restart: unless-stopped
     entrypoint: yarn sender:foreign
     networks:
       - net_db_bridge_senderforeign
       - net_rabbit_bridge_senderforeign
+  bridge_shutdown:
+    cpus: 0.1
+    mem_limit: 500m
+    image: poanetwork/tokenbridge-oracle:latest
+    env_file: ./.env
+    environment:
+      - NODE_ENV=production
+    restart: unless-stopped
+    entrypoint: yarn manager:shutdown
+    networks:
+      - net_db_bridge_shutdown
 
 networks:
   net_db_bridge_request:
@@ -106,6 +118,8 @@ networks:
     driver: bridge
   net_db_bridge_senderforeign:
     driver: bridge
+  net_db_bridge_shutdown:
+    driver: bridge
   net_rabbit_bridge_request:
     driver: bridge
   net_rabbit_bridge_collected:

oracle/package.json

@@ -13,6 +13,7 @@
     "sender:home": "./scripts/start-worker.sh sender home-sender",
     "sender:foreign": "./scripts/start-worker.sh sender foreign-sender",
     "confirm:transfer": "./scripts/start-worker.sh confirmRelay transfer-watcher",
+    "manager:shutdown": "./scripts/start-worker.sh shutdownManager shutdown-manager",
     "dev": "concurrently -n 'watcher:signature-request,watcher:collected-signatures,watcher:affirmation-request,watcher:transfer, sender:home,sender:foreign' -c 'red,green,yellow,blue,magenta,cyan' 'yarn watcher:signature-request' 'yarn watcher:collected-signatures' 'yarn watcher:affirmation-request' 'yarn watcher:transfer' 'yarn sender:home' 'yarn sender:foreign'",
     "test": "NODE_ENV=test mocha",
     "test:watch": "NODE_ENV=test mocha --watch --reporter=min",

oracle/src/sender.js

@@ -4,6 +4,7 @@ const { connectSenderToQueue } = require('./services/amqpClient')
 const { redis } = require('./services/redisClient')
 const GasPrice = require('./services/gasPrice')
 const logger = require('./services/logger')
+const { getShutdownFlag } = require('./services/shutdownState')
 const { sendTx } = require('./tx/sendTx')
 const { getNonce, getChainId } = require('./tx/web3')
 const {
@@ -47,6 +48,7 @@ async function initialize() {
     connectSenderToQueue({
       queueName: config.queue,
       oldQueueName: config.oldQueue,
+      resendInterval: config.resendInterval,
       cb: options => {
         if (config.maxProcessingTime) {
           return watchdog(() => main(options), config.maxProcessingTime, () => {
@@ -101,6 +103,14 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
       return
     }
 
+    const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
+    if (await getShutdownFlag(logger, config.shutdownKey, true)) {
+      if (!wasShutdown) {
+        logger.info('Oracle sender was suspended via the remote shutdown process')
+      }
+      return
+    }
+
     const txArray = JSON.parse(msg.content)
     logger.debug(`Msg received with ${txArray.length} Tx to send`)
     const gasPrice = GasPrice.getPrice().toString(10)
@@ -207,7 +217,7 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
       await scheduleForRetry(failedTx, msg.properties.headers['x-retries'])
     }
     if (resendJobs.length) {
-      logger.info(`Sending ${resendJobs.length} Tx Delayed Resend Requests to Queue`)
+      logger.info({ delay: config.resendInterval }, `Sending ${resendJobs.length} Tx Delayed Resend Requests to Queue`)
       await scheduleTransactionResend(resendJobs)
     }
     ackMsg(msg)

oracle/src/services/amqpClient.js

@@ -5,7 +5,6 @@ const connection = require('amqp-connection-manager').connect(process.env.ORACLE
 const logger = require('./logger')
 const { getRetrySequence } = require('../utils/utils')
 const {
-  TRANSACTION_RESEND_TIMEOUT,
   SENDER_QUEUE_MAX_PRIORITY,
   SENDER_QUEUE_SEND_PRIORITY,
   SENDER_QUEUE_CHECK_STATUS_PRIORITY
@@ -48,7 +47,7 @@ function connectWatcherToQueue({ queueName, workerQueue, cb }) {
   cb({ sendToQueue, sendToWorker, channel: channelWrapper })
 }
 
-function connectSenderToQueue({ queueName, oldQueueName, cb }) {
+function connectSenderToQueue({ queueName, oldQueueName, cb, resendInterval }) {
   const deadLetterExchange = `${queueName}-retry`
 
   async function resendMessagesToNewQueue(channel) {
@@ -97,7 +96,8 @@ function connectSenderToQueue({ queueName, oldQueueName, cb }) {
             channelWrapper,
             channel,
             queueName,
-            deadLetterExchange
+            deadLetterExchange,
+            delay: resendInterval
           })
         }
       })
@@ -164,13 +164,13 @@ async function generateRetry({ data, msgRetries, channelWrapper, channel, queueN
   })
 }
 
-async function generateTransactionResend({ data, channelWrapper, channel, queueName, deadLetterExchange }) {
-  const retryQueue = `${queueName}-check-tx-status`
+async function generateTransactionResend({ data, channelWrapper, channel, queueName, deadLetterExchange, delay }) {
+  const retryQueue = `${queueName}-check-tx-status-${delay}`
   await channel.assertQueue(retryQueue, {
     durable: true,
     deadLetterExchange,
-    messageTtl: TRANSACTION_RESEND_TIMEOUT,
-    expires: TRANSACTION_RESEND_TIMEOUT * 10,
+    messageTtl: delay,
+    expires: delay * 10,
     maxPriority: SENDER_QUEUE_MAX_PRIORITY
   })
   await channelWrapper.sendToQueue(retryQueue, data, {

oracle/src/services/logger.js

@@ -3,6 +3,7 @@ const path = require('path')
 const {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeFallback,
   web3ForeignFallback,
   web3HomeRedundant,
@@ -30,4 +31,8 @@ web3ForeignFallback.currentProvider.setLogger(logger)
 web3HomeRedundant.currentProvider.setLogger(logger)
 web3ForeignRedundant.currentProvider.setLogger(logger)
 
+if (web3Side) {
+  web3Side.currentProvider.setLogger(logger)
+}
+
 module.exports = logger

oracle/src/services/shutdownState.js

@@ -0,0 +1,23 @@
+const { redis } = require('./redisClient')
+
+let isShutdown = false
+async function getShutdownFlag(logger, shutdownKey, force = false) {
+  if (force) {
+    logger.debug('Reading current shutdown state from the DB')
+    isShutdown = (await redis.get(shutdownKey)) === 'true'
+    logger.debug({ isShutdown }, 'Read shutdown state from the DB')
+  }
+  return isShutdown
+}
+
+async function setShutdownFlag(logger, shutdownKey, value) {
+  logger.info({ isShutdown: value }, 'Updating current shutdown state in the DB')
+  isShutdown = value
+  await redis.set(shutdownKey, value)
+  logger.debug('Updated state in the DB')
+}
+
+module.exports = {
+  getShutdownFlag,
+  setShutdownFlag
+}

oracle/src/services/web3.js

@@ -6,6 +6,7 @@ const { RETRY_CONFIG } = require('../utils/constants')
 const {
   COMMON_HOME_RPC_URL,
   COMMON_FOREIGN_RPC_URL,
+  ORACLE_SIDE_RPC_URL,
   ORACLE_RPC_REQUEST_TIMEOUT,
   ORACLE_HOME_RPC_POLLING_INTERVAL,
   ORACLE_FOREIGN_RPC_POLLING_INTERVAL
@@ -41,6 +42,18 @@ const web3Home = new Web3(homeProvider)
 const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions)
 const web3Foreign = new Web3(foreignProvider)
 
+let web3Side = null
+if (ORACLE_SIDE_RPC_URL) {
+  const sideUrls = ORACLE_SIDE_RPC_URL.split(' ').filter(url => url.length > 0)
+  const sideOptions = {
+    requestTimeout: configuredTimeout || 2000,
+    retry: RETRY_CONFIG
+  }
+
+  const sideProvider = new HttpListProvider(sideUrls, sideOptions)
+  web3Side = new Web3(sideProvider)
+}
+
 // secondary fallback providers are intended to be used in places where
 // it is more likely that RPC calls to the local non-archive nodes can fail
 // e.g. for checking status of the old transaction via eth_getTransactionByHash
@@ -70,6 +83,7 @@ if (foreignUrls.length > 1) {
 module.exports = {
   web3Home,
   web3Foreign,
+  web3Side,
   web3HomeRedundant,
   web3ForeignRedundant,
   web3HomeFallback,

oracle/src/shutdownManager.js

@@ -0,0 +1,114 @@
+const fetch = require('node-fetch')
+const path = require('path')
+
+const { EXIT_CODES } = require('./utils/constants')
+const { watchdog } = require('./utils/utils')
+const logger = require('./services/logger')
+const { redis } = require('./services/redisClient')
+const { web3Side } = require('./services/web3')
+const { getShutdownFlag, setShutdownFlag } = require('./services/shutdownState')
+
+if (process.argv.length < 3) {
+  logger.error('Please check the number of arguments, config file was not provided')
+  process.exit(EXIT_CODES.GENERAL_ERROR)
+}
+
+const config = require(path.join('../config/', process.argv[2]))
+
+if (config.shutdownContractAddress && !web3Side) {
+  logger.error(
+    'ORACLE_SHUTDOWN_CONTRACT_ADDRESS was provided but not side chain provider was registered.' +
+      ' Please, specify ORACLE_SIDE_RPC_URL as well.'
+  )
+  process.exit(EXIT_CODES.GENERAL_ERROR)
+}
+
+let shutdownCount = 0
+let okCount = 0
+
+async function fetchShutdownFlag() {
+  if (config.shutdownServiceURL) {
+    logger.debug({ url: config.shutdownServiceURL }, 'Fetching shutdown status from external URL')
+    const result = await fetch(config.shutdownServiceURL, {
+      headers: {
+        'Content-type': 'application/json'
+      },
+      method: 'GET',
+      timeout: config.requestTimeout
+    }).then(res => res.json())
+
+    if (result.shutdown === true) {
+      return true
+    }
+  }
+
+  if (config.shutdownContractAddress) {
+    const shutdownSelector = web3Side.eth.abi.encodeEventSignature(config.shutdownMethod)
+    logger.debug(
+      { contract: config.shutdownContractAddress, method: config.shutdownMethod, data: shutdownSelector },
+      'Fetching shutdown status from contract'
+    )
+    const result = await web3Side.eth.call({
+      to: config.shutdownContractAddress,
+      data: shutdownSelector
+    })
+    logger.debug({ result }, 'Obtained result from the side RPC endpoint')
+
+    if (result.length > 2 && web3Side.eth.abi.decodeParameter('bool', result)) {
+      return true
+    }
+  }
+
+  return false
+}
+
+async function checkShutdownFlag() {
+  const isShutdownFlag = await fetchShutdownFlag()
+  const isShutdown = await getShutdownFlag(logger, config.shutdownKey)
+
+  if (isShutdownFlag === true && isShutdown === false) {
+    shutdownCount += 1
+    okCount = 0
+    logger.info(
+      { shutdownCount, remainingChecks: config.checksBeforeStop - shutdownCount },
+      'Received positive shutdown flag'
+    )
+  } else if (isShutdownFlag === false && isShutdown === true) {
+    okCount += 1
+    shutdownCount = 0
+    logger.info({ okCount, remainingChecks: config.checksBeforeResume - okCount }, 'Received negative shutdown flag')
+  } else {
+    shutdownCount = 0
+    okCount = 0
+    logger.debug({ isShutdown, isShutdownFlag }, 'Received shutdown flag that is equal to the current state')
+  }
+
+  if (shutdownCount >= config.checksBeforeStop) {
+    await setShutdownFlag(logger, config.shutdownKey, true)
+  } else if (okCount >= config.checksBeforeResume) {
+    await setShutdownFlag(logger, config.shutdownKey, false)
+  }
+}
+
+async function initialize() {
+  logger.info('Starting shutdown flag watcher')
+  redis.on('connect', async () => {
+    await getShutdownFlag(logger, config.shutdownKey, true)
+    await main()
+  })
+}
+
+async function main() {
+  try {
+    await watchdog(checkShutdownFlag, config.maxProcessingTime, () => {
+      logger.fatal('Max processing time reached')
+      process.exit(EXIT_CODES.MAX_TIME_REACHED)
+    })
+  } catch (e) {
+    logger.error(e)
+  }
+
+  setTimeout(main, config.pollingInterval)
+}
+
+initialize()

oracle/src/utils/constants.js

@@ -23,7 +23,7 @@ module.exports = {
     MIN: 1,
     MAX: 1000
   },
-  TRANSACTION_RESEND_TIMEOUT: 20 * 60 * 1000,
+  DEFAULT_TRANSACTION_RESEND_INTERVAL: 20 * 60 * 1000,
   FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000,
   SENDER_QUEUE_MAX_PRIORITY: 10,
   SENDER_QUEUE_SEND_PRIORITY: 5,

oracle/src/watcher.js

@@ -5,6 +5,7 @@ const { connectWatcherToQueue, connection } = require('./services/amqpClient')
 const { getBlockNumber } = require('./tx/web3')
 const { redis } = require('./services/redisClient')
 const logger = require('./services/logger')
+const { getShutdownFlag } = require('./services/shutdownState')
 const { getRequiredBlockConfirmations, getEvents } = require('./tx/web3')
 const { checkHTTPS, watchdog } = require('./utils/utils')
 const { EXIT_CODES } = require('./utils/constants')
@@ -157,6 +158,14 @@ async function isWorkerNeeded() {
 
 async function main({ sendToQueue, sendToWorker }) {
   try {
+    const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
+    if (await getShutdownFlag(logger, config.shutdownKey, true)) {
+      if (!wasShutdown) {
+        logger.info('Oracle watcher was suspended via the remote shutdown process')
+      }
+      return
+    }
+
     await checkConditions()
 
     const lastBlockToProcess = await getLastBlockToProcess()

yarn.lock

@@ -13953,6 +13953,11 @@ node-fetch@^2.1.2, node-fetch@^2.3.0, node-fetch@^2.5.0:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
   integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
 
+node-fetch@^2.6.1:
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
+  integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
+
 node-forge@0.7.5:
   version "0.7.5"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.5.tgz#6c152c345ce11c52f465c2abd957e8639cd674df"