Merge the develop branch to the master branch, preparation to v2.7.0-rc0 (#515)

This merge contains the following set of changes:
  * [Improvement] Add /metrics endpoint for prometheus support (#512)
  * [Improvement] Add monitor script for detecting failed AMB messages (#513)
  * [Improvement] Improve performance of BS requests in ALM (#516)
  * [Fix] Add pretty error messages for manual execution transaction reverts (#511)
  * [Fix] Fix resend of stuck pending transactions (#514)

alm/src/components/ConfirmationsContainer.tsx

@@ -39,10 +39,17 @@ export interface ConfirmationsContainerParams {
   message: MessageObject
   receipt: Maybe<TransactionReceipt>
   fromHome: boolean
-  timestamp: number
+  homeStartBlock: Maybe<number>
+  foreignStartBlock: Maybe<number>
 }
 
-export const ConfirmationsContainer = ({ message, receipt, fromHome, timestamp }: ConfirmationsContainerParams) => {
+export const ConfirmationsContainer = ({
+  message,
+  receipt,
+  fromHome,
+  homeStartBlock,
+  foreignStartBlock
+}: ConfirmationsContainerParams) => {
   const {
     home: { name: homeName },
     foreign: { name: foreignName }
@@ -62,7 +69,8 @@ export const ConfirmationsContainer = ({ message, receipt, fromHome, timestamp }
     message,
     receipt,
     fromHome,
-    timestamp,
+    homeStartBlock,
+    foreignStartBlock,
     requiredSignatures,
     validatorList,
     blockConfirmations

alm/src/components/ExecutionConfirmation.tsx

@@ -36,10 +36,12 @@ export const ExecutionConfirmation = ({
   const availableManualExecution =
     !isHome &&
     (executionData.status === VALIDATOR_CONFIRMATION_STATUS.WAITING ||
+      executionData.status === VALIDATOR_CONFIRMATION_STATUS.FAILED ||
       (executionData.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED &&
         executionEventsFetched &&
         !!executionData.validator))
   const requiredManualExecution = availableManualExecution && ALM_HOME_TO_FOREIGN_MANUAL_EXECUTION
+  const showAgeColumn = !requiredManualExecution || executionData.status === VALIDATOR_CONFIRMATION_STATUS.FAILED
   const windowWidth = useWindowWidth()
 
   const txExplorerLink = getExplorerTxUrl(executionData.txHash, isHome)
@@ -71,7 +73,7 @@ export const ExecutionConfirmation = ({
           <tr>
             <th>{requiredManualExecution ? 'Execution info' : 'Executed by'}</th>
             <th className="text-center">Status</th>
-            {!requiredManualExecution && <th className="text-center">Age</th>}
+            {showAgeColumn && <th className="text-center">Age</th>}
             {availableManualExecution && <th className="text-center">Actions</th>}
           </tr>
         </Thead>
@@ -87,7 +89,7 @@ export const ExecutionConfirmation = ({
               )}
             </td>
             <StatusTd className="text-center">{getExecutionStatusElement(executionData.status)}</StatusTd>
-            {!requiredManualExecution && (
+            {showAgeColumn && (
               <AgeTd className="text-center">
                 {executionData.timestamp > 0 ? (
                   <ExplorerTxLink href={txExplorerLink} target="_blank">

alm/src/components/ManualExecutionButton.tsx

@@ -2,9 +2,17 @@ import React, { useState, useEffect } from 'react'
 import styled from 'styled-components'
 import { InjectedConnector } from '@web3-react/injected-connector'
 import { useWeb3React } from '@web3-react/core'
-import { INCORRECT_CHAIN_ERROR, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
+import {
+  DOUBLE_EXECUTION_ATTEMPT_ERROR,
+  EXECUTION_FAILED_ERROR,
+  EXECUTION_OUT_OF_GAS_ERROR,
+  INCORRECT_CHAIN_ERROR,
+  VALIDATOR_CONFIRMATION_STATUS
+} from '../config/constants'
 import { useStateProvider } from '../state/StateProvider'
 import { signatureToVRS, packSignatures } from '../utils/signatures'
+import { getSuccessExecutionData } from '../utils/getFinalizationEvent'
+import { TransactionReceipt } from 'web3-eth'
 
 const StyledButton = styled.button`
   color: var(--button-color);
@@ -61,7 +69,9 @@ export const ManualExecutionButton = ({
       if (!library || !foreign.bridgeContract || !signatureCollected || !signatureCollected.length) return
 
       const signatures = packSignatures(signatureCollected.map(signatureToVRS))
-      const data = foreign.bridgeContract.methods.executeSignatures(messageData, signatures).encodeABI()
+      const messageId = messageData.slice(0, 66)
+      const bridge = foreign.bridgeContract
+      const data = bridge.methods.executeSignatures(messageData, signatures).encodeABI()
       setManualExecution(false)
 
       library.eth
@@ -80,7 +90,27 @@ export const ManualExecutionButton = ({
           })
           setPendingExecution(true)
         })
-        .on('error', (e: Error) => setError(e.message))
+        .on('error', async (e: Error, receipt: TransactionReceipt) => {
+          if (e.message.includes('Transaction has been reverted by the EVM')) {
+            const successExecutionData = await getSuccessExecutionData(bridge, 'RelayedMessage', library, messageId)
+            if (successExecutionData) {
+              setExecutionData(successExecutionData)
+              setError(DOUBLE_EXECUTION_ATTEMPT_ERROR)
+            } else {
+              const { gas } = await library.eth.getTransaction(receipt.transactionHash)
+              setExecutionData({
+                status: VALIDATOR_CONFIRMATION_STATUS.FAILED,
+                validator: account,
+                txHash: receipt.transactionHash,
+                timestamp: Math.floor(new Date().getTime() / 1000.0),
+                executionResult: false
+              })
+              setError(gas === receipt.gasUsed ? EXECUTION_OUT_OF_GAS_ERROR : EXECUTION_FAILED_ERROR)
+            }
+          } else {
+            setError(e.message)
+          }
+        })
     },
     [
       manualExecution,

alm/src/components/StatusContainer.tsx

@@ -10,6 +10,7 @@ import { ExplorerTxLink } from './commons/ExplorerTxLink'
 import { ConfirmationsContainer } from './ConfirmationsContainer'
 import { TransactionReceipt } from 'web3-eth'
 import { BackButton } from './commons/BackButton'
+import { useClosestBlock } from '../hooks/useClosestBlock'
 
 export interface StatusContainerParam {
   onBackToMain: () => void
@@ -23,12 +24,15 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar
   const { chainId, txHash, messageIdParam } = useParams()
   const validChainId = chainId === home.chainId.toString() || chainId === foreign.chainId.toString()
   const validParameters = validChainId && validTxHash(txHash)
+  const isHome = chainId === home.chainId.toString()
 
   const { messages, receipt, status, description, timestamp, loading } = useTransactionStatus({
     txHash: validParameters ? txHash : '',
     chainId: validParameters ? parseInt(chainId) : 0,
     receiptParam
   })
+  const homeStartBlock = useClosestBlock(true, isHome, receipt, timestamp)
+  const foreignStartBlock = useClosestBlock(false, isHome, receipt, timestamp)
 
   const selectedMessageId = messageIdParam === undefined || messages[messageIdParam] === undefined ? -1 : messageIdParam
 
@@ -64,7 +68,6 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar
   const displayReference = multiMessageSelected ? messages[selectedMessageId].id : txHash
   const formattedMessageId = formatTxHash(displayReference)
 
-  const isHome = chainId === home.chainId.toString()
   const txExplorerLink = getExplorerTxUrl(txHash, isHome)
   const displayExplorerLink = status !== TRANSACTION_STATUS.NOT_FOUND
 
@@ -101,7 +104,13 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar
       )}
       {displayMessageSelector && <MessageSelector messages={messages} onMessageSelected={onMessageSelected} />}
       {displayConfirmations && (
-        <ConfirmationsContainer message={messageToConfirm} receipt={receipt} fromHome={isHome} timestamp={timestamp} />
+        <ConfirmationsContainer
+          message={messageToConfirm}
+          receipt={receipt}
+          fromHome={isHome}
+          homeStartBlock={homeStartBlock}
+          foreignStartBlock={foreignStartBlock}
+        />
       )}
       <BackButton onBackToMain={onBackToMain} />
     </div>

alm/src/components/commons/ErrorAlert.tsx

@@ -2,6 +2,7 @@ import React from 'react'
 import styled from 'styled-components'
 import { InfoIcon } from './InfoIcon'
 import { CloseIcon } from './CloseIcon'
+import { ExplorerTxLink } from './ExplorerTxLink'
 
 const StyledErrorAlert = styled.div`
   border: 1px solid var(--failed-color);
@@ -15,17 +16,33 @@ const CloseIconContainer = styled.div`
 `
 
 const TextContainer = styled.div`
+  white-space: pre-wrap;
   flex-direction: column;
 `
 
-export const ErrorAlert = ({ onClick, error }: { onClick: () => void; error: string }) => (
+export const ErrorAlert = ({ onClick, error }: { onClick: () => void; error: string }) => {
-  <div className="row is-center">
+  const errorArray = error.split('%link')
-    <StyledErrorAlert className="col-10 is-vertical-align row">
+  const text = errorArray[0]
-      <InfoIcon color="var(--failed-color)" />
+  let link
-      <TextContainer className="col-10">{error}</TextContainer>
+  if (errorArray.length > 1) {
-      <CloseIconContainer className="col-1 is-vertical-align is-center" onClick={onClick}>
+    link = (
-        <CloseIcon color="var(--failed-color)" />
+      <ExplorerTxLink href={errorArray[1]} target="_blank" rel="noopener noreferrer">
-      </CloseIconContainer>
+        {errorArray[1]}
-    </StyledErrorAlert>
+      </ExplorerTxLink>
-  </div>
+    )
-)
+  }
+  return (
+    <div className="row is-center">
+      <StyledErrorAlert className="col-10 is-vertical-align row">
+        <InfoIcon color="var(--failed-color)" />
+        <TextContainer className="col-10">
+          {text}
+          {link}
+        </TextContainer>
+        <CloseIconContainer className="col-1 is-vertical-align is-center" onClick={onClick}>
+          <CloseIcon color="var(--failed-color)" />
+        </CloseIconContainer>
+      </StyledErrorAlert>
+    </div>
+  )
+}

alm/src/config/constants.ts

@@ -18,9 +18,8 @@ export const ALM_HOME_TO_FOREIGN_MANUAL_EXECUTION: boolean =
 
 export const HOME_RPC_POLLING_INTERVAL: number = 5000
 export const FOREIGN_RPC_POLLING_INTERVAL: number = 5000
-export const BLOCK_RANGE: number = 50
+export const BLOCK_RANGE: number = 500
-export const ONE_DAY_TIMESTAMP: number = 86400
+export const MAX_TX_SEARCH_BLOCK_RANGE: number = 10000
-export const THREE_DAYS_TIMESTAMP: number = 259200
 
 export const EXECUTE_AFFIRMATION_HASH = 'e7a2c01f'
 export const SUBMIT_SIGNATURE_HASH = '630cea8e'
@@ -66,3 +65,12 @@ export const VALIDATOR_CONFIRMATION_STATUS = {
 export const SEARCHING_TX = 'Searching Transaction...'
 
 export const INCORRECT_CHAIN_ERROR = `Incorrect chain chosen. Switch to ${FOREIGN_NETWORK_NAME} in the wallet.`
+
+export const DOUBLE_EXECUTION_ATTEMPT_ERROR = `Your execution transaction has been reverted.
+However, the execution completed successfully in the transaction sent by a different party.`
+
+export const EXECUTION_FAILED_ERROR = `Your execution transaction has been reverted.
+Please, contact the support by messaging on %linkhttps://forum.poa.network/c/support`
+
+export const EXECUTION_OUT_OF_GAS_ERROR = `Your execution transaction has been reverted due to Out-of-Gas error.
+Please, resend the transaction and provide more gas to it.`

alm/src/hooks/useClosestBlock.ts

@@ -0,0 +1,68 @@
+import { useEffect, useState } from 'react'
+import { TransactionReceipt } from 'web3-eth'
+import { useStateProvider } from '../state/StateProvider'
+import { FOREIGN_EXPLORER_API, HOME_EXPLORER_API } from '../config/constants'
+import { getClosestBlockByTimestamp } from '../utils/explorer'
+
+export function useClosestBlock(
+  searchHome: boolean,
+  fromHome: boolean,
+  receipt: Maybe<TransactionReceipt>,
+  timestamp: number
+) {
+  const { home, foreign } = useStateProvider()
+  const [blockNumber, setBlockNumber] = useState<number | null>(null)
+
+  useEffect(
+    () => {
+      if (!receipt || blockNumber || !timestamp) return
+
+      if (fromHome === searchHome) {
+        setBlockNumber(receipt.blockNumber)
+        return
+      }
+
+      const web3 = searchHome ? home.web3 : foreign.web3
+      if (!web3) return
+
+      const getBlock = async () => {
+        // try to fast-fetch closest block number from the chain explorer
+        try {
+          const api = searchHome ? HOME_EXPLORER_API : FOREIGN_EXPLORER_API
+          setBlockNumber(await getClosestBlockByTimestamp(api, timestamp))
+          return
+        } catch {}
+
+        const lastBlock = await web3.eth.getBlock('latest')
+        if (lastBlock.timestamp <= timestamp) {
+          setBlockNumber(lastBlock.number)
+          return
+        }
+
+        const oldBlock = await web3.eth.getBlock(Math.max(lastBlock.number - 10000, 1))
+        const blockDiff = lastBlock.number - oldBlock.number
+        const timeDiff = (lastBlock.timestamp as number) - (oldBlock.timestamp as number)
+        const averageBlockTime = timeDiff / blockDiff
+        let currentBlock = lastBlock
+
+        let prevBlockDiff = Infinity
+        while (true) {
+          const timeDiff = (currentBlock.timestamp as number) - timestamp
+          const blockDiff = Math.ceil(timeDiff / averageBlockTime)
+          if (Math.abs(blockDiff) < 5 || Math.abs(blockDiff) >= Math.abs(prevBlockDiff)) {
+            setBlockNumber(currentBlock.number - blockDiff - 5)
+            break
+          }
+
+          prevBlockDiff = blockDiff
+          currentBlock = await web3.eth.getBlock(currentBlock.number - blockDiff)
+        }
+      }
+
+      getBlock()
+    },
+    [blockNumber, foreign.web3, fromHome, home.web3, receipt, searchHome, timestamp]
+  )
+
+  return blockNumber
+}

alm/src/hooks/useMessageConfirmations.ts

@@ -28,7 +28,8 @@ export interface useMessageConfirmationsParams {
   message: MessageObject
   receipt: Maybe<TransactionReceipt>
   fromHome: boolean
-  timestamp: number
+  homeStartBlock: Maybe<number>
+  foreignStartBlock: Maybe<number>
   requiredSignatures: number
   validatorList: string[]
   blockConfirmations: number
@@ -56,7 +57,8 @@ export const useMessageConfirmations = ({
   message,
   receipt,
   fromHome,
-  timestamp,
+  homeStartBlock,
+  foreignStartBlock,
   requiredSignatures,
   validatorList,
   blockConfirmations
@@ -90,6 +92,17 @@ export const useMessageConfirmations = ({
     return filteredList.length > 0
   }
 
+  // start watching blocks at the start
+  useEffect(
+    () => {
+      if (!home.web3 || !foreign.web3) return
+
+      homeBlockNumberProvider.start(home.web3)
+      foreignBlockNumberProvider.start(foreign.web3)
+    },
+    [foreign.web3, home.web3]
+  )
+
   // Check if the validators are waiting for block confirmations to verify the message
   useEffect(
     () => {
@@ -105,9 +118,6 @@ export const useMessageConfirmations = ({
 
       const blockProvider = fromHome ? homeBlockNumberProvider : foreignBlockNumberProvider
       const interval = fromHome ? HOME_RPC_POLLING_INTERVAL : FOREIGN_RPC_POLLING_INTERVAL
-      const web3 = fromHome ? home.web3 : foreign.web3
-      blockProvider.start(web3)
-
       const targetBlock = receipt.blockNumber + blockConfirmations
 
       checkSignaturesWaitingForBLocks(
@@ -123,7 +133,6 @@ export const useMessageConfirmations = ({
 
       return () => {
         unsubscribe()
-        blockProvider.stop()
       }
     },
     [
@@ -153,8 +162,6 @@ export const useMessageConfirmations = ({
         })
       }
 
-      homeBlockNumberProvider.start(home.web3)
-
       const fromBlock = receipt.blockNumber
       const toBlock = fromBlock + BLOCK_RANGE
       const messageHash = home.web3.utils.soliditySha3Raw(message.data)
@@ -171,7 +178,6 @@ export const useMessageConfirmations = ({
 
       return () => {
         unsubscribe()
-        homeBlockNumberProvider.stop()
       }
     },
     [fromHome, home.bridgeContract, home.web3, message.data, receipt, signatureCollected]
@@ -192,7 +198,6 @@ export const useMessageConfirmations = ({
         })
       }
 
-      homeBlockNumberProvider.start(home.web3)
       const targetBlock = collectedSignaturesEvent.blockNumber + blockConfirmations
 
       checkWaitingBlocksForExecution(
@@ -208,7 +213,6 @@ export const useMessageConfirmations = ({
 
       return () => {
         unsubscribe()
-        homeBlockNumberProvider.stop()
       }
     },
     [collectedSignaturesEvent, fromHome, blockConfirmations, home.web3, receipt, waitingBlocksForExecutionResolved]
@@ -218,7 +222,7 @@ export const useMessageConfirmations = ({
   // To avoid making extra requests, this is only executed when validators finished waiting for blocks confirmations
   useEffect(
     () => {
-      if (!waitingBlocksResolved || !timestamp || !requiredSignatures) return
+      if (!waitingBlocksResolved || !homeStartBlock || !requiredSignatures) return
 
       const subscriptions: Array<number> = []
 
@@ -239,7 +243,7 @@ export const useMessageConfirmations = ({
         setSignatureCollected,
         waitingBlocksResolved,
         subscriptions,
-        timestamp,
+        homeStartBlock,
         getValidatorFailedTransactionsForMessage,
         setFailedConfirmations,
         getValidatorPendingTransactionsForMessage,
@@ -259,7 +263,7 @@ export const useMessageConfirmations = ({
       home.bridgeContract,
       requiredSignatures,
       waitingBlocksResolved,
-      timestamp,
+      homeStartBlock,
       setConfirmations
     ]
   )
@@ -270,6 +274,8 @@ export const useMessageConfirmations = ({
   useEffect(
     () => {
       if ((fromHome && !waitingBlocksForExecutionResolved) || (!fromHome && !waitingBlocksResolved)) return
+      const startBlock = fromHome ? foreignStartBlock : homeStartBlock
+      if (!startBlock) return
 
       const subscriptions: Array<number> = []
 
@@ -285,6 +291,7 @@ export const useMessageConfirmations = ({
       const interval = fromHome ? FOREIGN_RPC_POLLING_INTERVAL : HOME_RPC_POLLING_INTERVAL
 
       getFinalizationEvent(
+        fromHome,
         bridgeContract,
         contractEvent,
         providedWeb3,
@@ -293,7 +300,7 @@ export const useMessageConfirmations = ({
         message,
         interval,
         subscriptions,
-        timestamp,
+        startBlock,
         collectedSignaturesEvent,
         getExecutionFailedTransactionForMessage,
         setFailedExecution,
@@ -315,8 +322,9 @@ export const useMessageConfirmations = ({
       home.web3,
       waitingBlocksResolved,
       waitingBlocksForExecutionResolved,
-      timestamp,
+      collectedSignaturesEvent,
-      collectedSignaturesEvent
+      foreignStartBlock,
+      homeStartBlock
     ]
   )
 
@@ -328,6 +336,9 @@ export const useMessageConfirmations = ({
           ? CONFIRMATIONS_STATUS.SUCCESS
           : CONFIRMATIONS_STATUS.SUCCESS_MESSAGE_FAILED
         setStatus(newStatus)
+
+        foreignBlockNumberProvider.stop()
+        homeBlockNumberProvider.stop()
       } else if (signatureCollected) {
         if (fromHome) {
           if (waitingBlocksForExecution) {

alm/src/services/BlockNumberProvider.ts

@@ -1,6 +1,6 @@
 import Web3 from 'web3'
 import differenceInMilliseconds from 'date-fns/differenceInMilliseconds'
-import { HOME_RPC_POLLING_INTERVAL } from '../config/constants'
+import { FOREIGN_RPC_POLLING_INTERVAL, HOME_RPC_POLLING_INTERVAL } from '../config/constants'
 
 export class BlockNumberProvider {
   private running: number
@@ -61,4 +61,4 @@ export class BlockNumberProvider {
 }
 
 export const homeBlockNumberProvider = new BlockNumberProvider(HOME_RPC_POLLING_INTERVAL)
-export const foreignBlockNumberProvider = new BlockNumberProvider(HOME_RPC_POLLING_INTERVAL)
+export const foreignBlockNumberProvider = new BlockNumberProvider(FOREIGN_RPC_POLLING_INTERVAL)

alm/src/utils/__tests__/explorer.test.ts

@@ -17,19 +17,33 @@ const otherAddress = '0xD4075FB57fCf038bFc702c915Ef9592534bED5c1'
 
 describe('getFailedTransactions', () => {
   test('should only return failed transactions', async () => {
-    const transactions = [{ isError: '0' }, { isError: '1' }, { isError: '0' }, { isError: '1' }, { isError: '1' }]
+    const to = otherAddress
+    const transactions = [
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '1', to }
+    ]
 
     const fetchAccountTransactions = jest.fn().mockImplementation(() => transactions)
-    const result = await getFailedTransactions('', '', 0, 1, '', fetchAccountTransactions)
+    const result = await getFailedTransactions('', to, 0, 1, '', fetchAccountTransactions)
     expect(result.length).toEqual(3)
   })
 })
 describe('getSuccessTransactions', () => {
   test('should only return success transactions', async () => {
-    const transactions = [{ isError: '0' }, { isError: '1' }, { isError: '0' }, { isError: '1' }, { isError: '1' }]
+    const to = otherAddress
+    const transactions = [
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '1', to }
+    ]
 
     const fetchAccountTransactions = jest.fn().mockImplementation(() => transactions)
-    const result = await getSuccessTransactions('', '', 0, 1, '', fetchAccountTransactions)
+    const result = await getSuccessTransactions('', to, 0, 1, '', fetchAccountTransactions)
     expect(result.length).toEqual(2)
   })
 })
@@ -74,8 +88,8 @@ describe('getExecutionFailedTransactionForMessage', () => {
         account: '',
         to: '',
         messageData,
-        startTimestamp: 0,
+        startBlock: 0,
-        endTimestamp: 1
+        endBlock: 1
       },
       fetchAccountTransactions
     )

alm/src/utils/__tests__/getFinalizationEvent.test.ts

@@ -64,6 +64,7 @@ describe('getFinalizationEvent', () => {
     const setExecutionEventsFetched = jest.fn()
 
     await getFinalizationEvent(
+      true,
       contract,
       eventName,
       web3,
@@ -115,6 +116,7 @@ describe('getFinalizationEvent', () => {
     const setExecutionEventsFetched = jest.fn()
 
     await getFinalizationEvent(
+      true,
       contract,
       eventName,
       web3,
@@ -166,6 +168,7 @@ describe('getFinalizationEvent', () => {
     const setExecutionEventsFetched = jest.fn()
 
     await getFinalizationEvent(
+      true,
       contract,
       eventName,
       web3,
@@ -217,6 +220,7 @@ describe('getFinalizationEvent', () => {
     const setExecutionEventsFetched = jest.fn()
 
     await getFinalizationEvent(
+      true,
       contract,
       eventName,
       web3,
@@ -275,6 +279,7 @@ describe('getFinalizationEvent', () => {
     const setExecutionEventsFetched = jest.fn()
 
     await getFinalizationEvent(
+      true,
       contract,
       eventName,
       web3,

alm/src/utils/executionWaitingForBlocks.ts

@@ -30,7 +30,6 @@ export const checkWaitingBlocksForExecution = async (
     )
     setWaitingBlocksForExecutionResolved(true)
     setWaitingBlocksForExecution(false)
-    blockProvider.stop()
   } else {
     let nextInterval = interval
     if (!currentBlock) {

alm/src/utils/explorer.ts

@@ -1,8 +1,10 @@
 import {
+  BLOCK_RANGE,
   EXECUTE_AFFIRMATION_HASH,
   EXECUTE_SIGNATURES_HASH,
   FOREIGN_EXPLORER_API,
   HOME_EXPLORER_API,
+  MAX_TX_SEARCH_BLOCK_RANGE,
   SUBMIT_SIGNATURE_HASH
 } from '../config/constants'
 
@@ -12,6 +14,7 @@ export interface APITransaction {
   input: string
   to: string
   hash: string
+  blockNumber: string
 }
 
 export interface APIPendingTransaction {
@@ -27,106 +30,42 @@ export interface PendingTransactionsParams {
 
 export interface AccountTransactionsParams {
   account: string
-  to: string
+  startBlock: number
-  startTimestamp: number
+  endBlock: number
-  endTimestamp: number
   api: string
 }
 
-export interface GetFailedTransactionParams {
-  account: string
-  to: string
-  messageData: string
-  startTimestamp: number
-  endTimestamp: number
-}
-
 export interface GetPendingTransactionParams {
   account: string
   to: string
   messageData: string
 }
 
-export const fetchAccountTransactionsFromBlockscout = async ({
+export interface GetTransactionParams extends GetPendingTransactionParams {
-  account,
+  startBlock: number
-  to,
+  endBlock: number
-  startTimestamp,
-  endTimestamp,
-  api
-}: AccountTransactionsParams): Promise<APITransaction[]> => {
-  const url = `${api}?module=account&action=txlist&address=${account}&filterby=from=${account}&to=${to}&starttimestamp=${startTimestamp}&endtimestamp=${endTimestamp}`
-
-  try {
-    const result = await fetch(url).then(res => res.json())
-    if (result.status === '0') {
-      return []
-    }
-
-    return result.result
-  } catch (e) {
-    console.log(e)
-    return []
-  }
 }
 
-export const getBlockByTimestampUrl = (api: string, timestamp: number) =>
+export const fetchAccountTransactions = async ({ account, startBlock, endBlock, api }: AccountTransactionsParams) => {
-  `${api}&module=block&action=getblocknobytime&timestamp=${timestamp}&closest=before`
+  const params = `module=account&action=txlist&address=${account}&filterby=from&startblock=${startBlock}&endblock=${endBlock}`
+  const url = api.includes('blockscout') ? `${api}?${params}` : `${api}&${params}`
 
-export const fetchAccountTransactionsFromEtherscan = async ({
+  const result = await fetch(url).then(res => res.json())
-  account,
-  to,
-  startTimestamp,
-  endTimestamp,
-  api
-}: AccountTransactionsParams): Promise<APITransaction[]> => {
-  const startBlockUrl = getBlockByTimestampUrl(api, startTimestamp)
-  const endBlockUrl = getBlockByTimestampUrl(api, endTimestamp)
-  let fromBlock = 0
-  let toBlock = 9999999999999
-  try {
-    const [fromBlockResult, toBlockResult] = await Promise.all([
-      fetch(startBlockUrl).then(res => res.json()),
-      fetch(endBlockUrl).then(res => res.json())
-    ])
 
-    if (fromBlockResult.status !== '0') {
+  if (result.message === 'No transactions found') {
-      fromBlock = parseInt(fromBlockResult.result)
-    }
-
-    if (toBlockResult.status !== '0') {
-      toBlock = parseInt(toBlockResult.result)
-    }
-  } catch (e) {
-    console.log(e)
     return []
   }
 
-  const url = `${api}&module=account&action=txlist&address=${account}&startblock=${fromBlock}&endblock=${toBlock}`
+  return result.result
-
-  try {
-    const result = await fetch(url).then(res => res.json())
-
-    if (result.status === '0') {
-      return []
-    }
-
-    const toAddressLowerCase = to.toLowerCase()
-    const transactions: APITransaction[] = result.result
-    return transactions.filter(t => t.to.toLowerCase() === toAddressLowerCase)
-  } catch (e) {
-    console.log(e)
-    return []
-  }
-}
-
-export const fetchAccountTransactions = (api: string) => {
-  return api.includes('blockscout') ? fetchAccountTransactionsFromBlockscout : fetchAccountTransactionsFromEtherscan
 }
 
 export const fetchPendingTransactions = async ({
   account,
   api
 }: PendingTransactionsParams): Promise<APIPendingTransaction[]> => {
+  if (!api.includes('blockscout')) {
+    return []
+  }
   const url = `${api}?module=account&action=pendingtxlist&address=${account}`
 
   try {
@@ -141,30 +80,96 @@ export const fetchPendingTransactions = async ({
   }
 }
 
+export const getClosestBlockByTimestamp = async (api: string, timestamp: number): Promise<number> => {
+  if (api.includes('blockscout')) {
+    throw new Error('Blockscout does not support getblocknobytime')
+  }
+
+  const url = `${api}&module=block&action=getblocknobytime&timestamp=${timestamp}&closest=before`
+
+  const blockNumber = await fetch(url).then(res => res.json())
+
+  return parseInt(blockNumber.result)
+}
+
+// fast version of fetchAccountTransactions
+// sequentially fetches transactions in small batches
+// caches the result
+const transactionsCache: { [key: string]: { lastBlock: number; transactions: APITransaction[] } } = {}
+export const getAccountTransactions = async ({
+  account,
+  startBlock,
+  endBlock,
+  api
+}: AccountTransactionsParams): Promise<APITransaction[]> => {
+  const key = `${account}-${startBlock}-${api}`
+
+  // initialize empty cache if it doesn't exist yet
+  if (!transactionsCache[key]) {
+    transactionsCache[key] = { lastBlock: startBlock - 1, transactions: [] }
+  }
+
+  // if cache contains events up to block X,
+  // new batch is fetched for range [X + 1, X + 1 + BLOCK_RANGE]
+  const newStartBlock = transactionsCache[key].lastBlock + 1
+  const newEndBlock = newStartBlock + BLOCK_RANGE
+
+  // search for new transactions only if max allowed block range is not yet exceeded
+  if (newEndBlock <= startBlock + MAX_TX_SEARCH_BLOCK_RANGE) {
+    const newTransactions = await fetchAccountTransactions({
+      account,
+      startBlock: newStartBlock,
+      endBlock: newEndBlock,
+      api
+    })
+
+    const transactions = transactionsCache[key].transactions.concat(...newTransactions)
+
+    // cache updated transactions list
+    transactionsCache[key].transactions = transactions
+
+    // enbBlock is assumed to be the current block number of the chain
+    // if the whole range is finalized, last block can be safely updated to the end of the range
+    // this works even if there are no transactions in the list
+    if (newEndBlock < endBlock) {
+      transactionsCache[key].lastBlock = newEndBlock
+    } else if (transactions.length > 0) {
+      transactionsCache[key].lastBlock = parseInt(transactions[transactions.length - 1].blockNumber, 10)
+    }
+
+    return transactions
+  }
+
+  console.warn(`Reached max transaction searching range, returning previously cached transactions for ${account}`)
+  return transactionsCache[key].transactions
+}
+
+const filterReceiver = (to: string) => (tx: APITransaction) => tx.to.toLowerCase() === to.toLowerCase()
+
 export const getFailedTransactions = async (
   account: string,
   to: string,
-  startTimestamp: number,
+  startBlock: number,
-  endTimestamp: number,
+  endBlock: number,
   api: string,
-  fetchAccountTransactions: (args: AccountTransactionsParams) => Promise<APITransaction[]>
+  getAccountTransactionsMethod = getAccountTransactions
 ): Promise<APITransaction[]> => {
-  const transactions = await fetchAccountTransactions({ account, to, startTimestamp, endTimestamp, api })
+  const transactions = await getAccountTransactionsMethod({ account, startBlock, endBlock, api })
 
-  return transactions.filter(t => t.isError !== '0')
+  return transactions.filter(t => t.isError !== '0').filter(filterReceiver(to))
 }
 
 export const getSuccessTransactions = async (
   account: string,
   to: string,
-  startTimestamp: number,
+  startBlock: number,
-  endTimestamp: number,
+  endBlock: number,
   api: string,
-  fetchAccountTransactions: (args: AccountTransactionsParams) => Promise<APITransaction[]>
+  getAccountTransactionsMethod = getAccountTransactions
 ): Promise<APITransaction[]> => {
-  const transactions = await fetchAccountTransactions({ account, to, startTimestamp, endTimestamp, api })
+  const transactions = await getAccountTransactionsMethod({ account, startBlock, endBlock, api })
 
-  return transactions.filter(t => t.isError === '0')
+  return transactions.filter(t => t.isError === '0').filter(filterReceiver(to))
 }
 
 export const filterValidatorSignatureTransaction = (
@@ -183,17 +188,10 @@ export const getValidatorFailedTransactionsForMessage = async ({
   account,
   to,
   messageData,
-  startTimestamp,
+  startBlock,
-  endTimestamp
+  endBlock
-}: GetFailedTransactionParams): Promise<APITransaction[]> => {
+}: GetTransactionParams): Promise<APITransaction[]> => {
-  const failedTransactions = await getFailedTransactions(
+  const failedTransactions = await getFailedTransactions(account, to, startBlock, endBlock, HOME_EXPLORER_API)
-    account,
-    to,
-    startTimestamp,
-    endTimestamp,
-    HOME_EXPLORER_API,
-    fetchAccountTransactionsFromBlockscout
-  )
 
   return filterValidatorSignatureTransaction(failedTransactions, messageData)
 }
@@ -202,33 +200,19 @@ export const getValidatorSuccessTransactionsForMessage = async ({
   account,
   to,
   messageData,
-  startTimestamp,
+  startBlock,
-  endTimestamp
+  endBlock
-}: GetFailedTransactionParams): Promise<APITransaction[]> => {
+}: GetTransactionParams): Promise<APITransaction[]> => {
-  const transactions = await getSuccessTransactions(
+  const transactions = await getSuccessTransactions(account, to, startBlock, endBlock, HOME_EXPLORER_API)
-    account,
-    to,
-    startTimestamp,
-    endTimestamp,
-    HOME_EXPLORER_API,
-    fetchAccountTransactionsFromBlockscout
-  )
 
   return filterValidatorSignatureTransaction(transactions, messageData)
 }
 
 export const getExecutionFailedTransactionForMessage = async (
-  { account, to, messageData, startTimestamp, endTimestamp }: GetFailedTransactionParams,
+  { account, to, messageData, startBlock, endBlock }: GetTransactionParams,
   getFailedTransactionsMethod = getFailedTransactions
 ): Promise<APITransaction[]> => {
-  const failedTransactions = await getFailedTransactionsMethod(
+  const failedTransactions = await getFailedTransactionsMethod(account, to, startBlock, endBlock, FOREIGN_EXPLORER_API)
-    account,
-    to,
-    startTimestamp,
-    endTimestamp,
-    FOREIGN_EXPLORER_API,
-    fetchAccountTransactions(FOREIGN_EXPLORER_API)
-  )
 
   const messageDataValue = messageData.replace('0x', '')
   return failedTransactions.filter(t => t.input.includes(EXECUTE_SIGNATURES_HASH) && t.input.includes(messageDataValue))

alm/src/utils/getCollectedSignaturesEvent.ts

@@ -31,7 +31,6 @@ export const getCollectedSignaturesEvent = async (
   if (filteredEvents.length) {
     const event = filteredEvents[0]
     setCollectedSignaturesEvent(event)
-    homeBlockNumberProvider.stop()
   } else {
     const newFromBlock = currentBlock ? securedToBlock : fromBlock
     const newToBlock = currentBlock ? toBlock + BLOCK_RANGE : toBlock

alm/src/utils/getConfirmationsForTx.ts

@@ -1,12 +1,7 @@
 import Web3 from 'web3'
 import { Contract } from 'web3-eth-contract'
 import { HOME_RPC_POLLING_INTERVAL, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
-import {
+import { GetTransactionParams, APITransaction, APIPendingTransaction, GetPendingTransactionParams } from './explorer'
-  GetFailedTransactionParams,
-  APITransaction,
-  APIPendingTransaction,
-  GetPendingTransactionParams
-} from './explorer'
 import { getAffirmationsSigned, getMessagesSigned } from './contract'
 import {
   getValidatorConfirmation,
@@ -43,12 +38,12 @@ export const getConfirmationsForTx = async (
   setSignatureCollected: Function,
   waitingBlocksResolved: boolean,
   subscriptions: number[],
-  timestamp: number,
+  startBlock: number,
-  getFailedTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>,
+  getFailedTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>,
   setFailedConfirmations: Function,
   getPendingTransactions: (args: GetPendingTransactionParams) => Promise<APIPendingTransaction[]>,
   setPendingConfirmations: Function,
-  getSuccessTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>
+  getSuccessTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>
 ) => {
   if (!web3 || !validatorList || !validatorList.length || !bridgeContract || !waitingBlocksResolved) return
 
@@ -102,7 +97,7 @@ export const getConfirmationsForTx = async (
   // Check if confirmation failed
   const validatorFailedConfirmationsChecks = await Promise.all(
     undefinedConfirmations.map(
-      getValidatorFailedTransaction(bridgeContract, messageData, timestamp, getFailedTransactions)
+      getValidatorFailedTransaction(bridgeContract, messageData, startBlock, getFailedTransactions)
     )
   )
   const validatorFailedConfirmations = validatorFailedConfirmationsChecks.filter(
@@ -136,7 +131,7 @@ export const getConfirmationsForTx = async (
   // get transactions from success signatures
   const successConfirmationWithData = await Promise.all(
     successConfirmations.map(
-      getSuccessExecutionTransaction(web3, bridgeContract, fromHome, messageData, timestamp, getSuccessTransactions)
+      getSuccessExecutionTransaction(web3, bridgeContract, fromHome, messageData, startBlock, getSuccessTransactions)
     )
   )
 
@@ -162,7 +157,7 @@ export const getConfirmationsForTx = async (
           setSignatureCollected,
           waitingBlocksResolved,
           subscriptions,
-          timestamp,
+          startBlock,
           getFailedTransactions,
           setFailedConfirmations,
           getPendingTransactions,

alm/src/utils/getFinalizationEvent.ts

@@ -1,41 +1,20 @@
 import { Contract, EventData } from 'web3-eth-contract'
 import Web3 from 'web3'
-import { CACHE_KEY_EXECUTION_FAILED, THREE_DAYS_TIMESTAMP, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
+import { CACHE_KEY_EXECUTION_FAILED, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
 import { ExecutionData } from '../hooks/useMessageConfirmations'
-import {
+import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer'
-  APIPendingTransaction,
-  APITransaction,
-  GetFailedTransactionParams,
-  GetPendingTransactionParams
-} from './explorer'
 import { getBlock, MessageObject } from './web3'
 import validatorsCache from '../services/ValidatorsCache'
+import { foreignBlockNumberProvider, homeBlockNumberProvider } from '../services/BlockNumberProvider'
 
-export const getFinalizationEvent = async (
+export const getSuccessExecutionData = async (contract: Contract, eventName: string, web3: Web3, messageId: string) => {
-  contract: Maybe<Contract>,
-  eventName: string,
-  web3: Maybe<Web3>,
-  setResult: React.Dispatch<React.SetStateAction<ExecutionData>>,
-  waitingBlocksResolved: boolean,
-  message: MessageObject,
-  interval: number,
-  subscriptions: number[],
-  timestamp: number,
-  collectedSignaturesEvent: Maybe<EventData>,
-  getFailedExecution: (args: GetFailedTransactionParams) => Promise<APITransaction[]>,
-  setFailedExecution: Function,
-  getPendingExecution: (args: GetPendingTransactionParams) => Promise<APIPendingTransaction[]>,
-  setPendingExecution: Function,
-  setExecutionEventsFetched: Function
-) => {
-  if (!contract || !web3 || !waitingBlocksResolved) return
   // Since it filters by the message id, only one event will be fetched
   // so there is no need to limit the range of the block to reduce the network traffic
   const events: EventData[] = await contract.getPastEvents(eventName, {
     fromBlock: 0,
     toBlock: 'latest',
     filter: {
-      messageId: message.id
+      messageId
     }
   })
   if (events.length > 0) {
@@ -48,13 +27,39 @@ export const getFinalizationEvent = async (
     const blockTimestamp = typeof block.timestamp === 'string' ? parseInt(block.timestamp) : block.timestamp
     const validatorAddress = web3.utils.toChecksumAddress(txReceipt.from)
 
-    setResult({
+    return {
       status: VALIDATOR_CONFIRMATION_STATUS.SUCCESS,
       validator: validatorAddress,
       txHash: event.transactionHash,
       timestamp: blockTimestamp,
       executionResult: event.returnValues.status
-    })
+    }
+  }
+  return null
+}
+
+export const getFinalizationEvent = async (
+  fromHome: boolean,
+  contract: Maybe<Contract>,
+  eventName: string,
+  web3: Maybe<Web3>,
+  setResult: React.Dispatch<React.SetStateAction<ExecutionData>>,
+  waitingBlocksResolved: boolean,
+  message: MessageObject,
+  interval: number,
+  subscriptions: number[],
+  startBlock: number,
+  collectedSignaturesEvent: Maybe<EventData>,
+  getFailedExecution: (args: GetTransactionParams) => Promise<APITransaction[]>,
+  setFailedExecution: Function,
+  getPendingExecution: (args: GetPendingTransactionParams) => Promise<APIPendingTransaction[]>,
+  setPendingExecution: Function,
+  setExecutionEventsFetched: Function
+) => {
+  if (!contract || !web3 || !waitingBlocksResolved) return
+  const successExecutionData = await getSuccessExecutionData(contract, eventName, web3, message.id)
+  if (successExecutionData) {
+    setResult(successExecutionData)
   } else {
     setExecutionEventsFetched(true)
     // If event is defined, it means it is a message from Home to Foreign
@@ -84,14 +89,15 @@ export const getFinalizationEvent = async (
       } else {
         const validatorExecutionCacheKey = `${CACHE_KEY_EXECUTION_FAILED}${validator}-${message.id}`
         const failedFromCache = validatorsCache.get(validatorExecutionCacheKey)
+        const blockProvider = fromHome ? foreignBlockNumberProvider : homeBlockNumberProvider
 
         if (!failedFromCache) {
           const failedTransactions = await getFailedExecution({
             account: validator,
             to: contract.options.address,
             messageData: message.data,
-            startTimestamp: timestamp,
+            startBlock,
-            endTimestamp: timestamp + THREE_DAYS_TIMESTAMP
+            endBlock: blockProvider.get() || 0
           })
 
           if (failedTransactions.length > 0) {
@@ -117,6 +123,7 @@ export const getFinalizationEvent = async (
     const timeoutId = setTimeout(
       () =>
         getFinalizationEvent(
+          fromHome,
           contract,
           eventName,
           web3,
@@ -125,7 +132,7 @@ export const getFinalizationEvent = async (
           message,
           interval,
           subscriptions,
-          timestamp,
+          startBlock,
           collectedSignaturesEvent,
           getFailedExecution,
           setFailedExecution,

alm/src/utils/signatureWaitingForBlocks.ts

@@ -16,7 +16,6 @@ export const checkSignaturesWaitingForBLocks = async (
   if (currentBlock && currentBlock >= targetBlock) {
     setWaitingBlocksResolved(true)
     setWaitingStatus(false)
-    blockProvider.stop()
   } else {
     let nextInterval = interval
     if (!currentBlock) {

alm/src/utils/validatorConfirmationHelpers.ts

@@ -2,18 +2,9 @@ import Web3 from 'web3'
 import { Contract } from 'web3-eth-contract'
 import { BasicConfirmationParam, ConfirmationParam } from '../hooks/useMessageConfirmations'
 import validatorsCache from '../services/ValidatorsCache'
-import {
+import { CACHE_KEY_FAILED, CACHE_KEY_SUCCESS, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
-  CACHE_KEY_FAILED,
+import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer'
-  CACHE_KEY_SUCCESS,
+import { homeBlockNumberProvider } from '../services/BlockNumberProvider'
-  ONE_DAY_TIMESTAMP,
-  VALIDATOR_CONFIRMATION_STATUS
-} from '../config/constants'
-import {
-  APIPendingTransaction,
-  APITransaction,
-  GetFailedTransactionParams,
-  GetPendingTransactionParams
-} from './explorer'
 
 export const getValidatorConfirmation = (
   web3: Web3,
@@ -50,8 +41,8 @@ export const getSuccessExecutionTransaction = (
   bridgeContract: Contract,
   fromHome: boolean,
   messageData: string,
-  timestamp: number,
+  startBlock: number,
-  getSuccessTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>
+  getSuccessTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>
 ) => async (validatorData: BasicConfirmationParam): Promise<ConfirmationParam> => {
   const { validator } = validatorData
   const validatorCacheKey = `${CACHE_KEY_SUCCESS}${validatorData.validator}-${messageData}`
@@ -65,8 +56,8 @@ export const getSuccessExecutionTransaction = (
     account: validatorData.validator,
     to: bridgeContract.options.address,
     messageData,
-    startTimestamp: timestamp,
+    startBlock,
-    endTimestamp: timestamp + ONE_DAY_TIMESTAMP
+    endBlock: homeBlockNumberProvider.get() || 0
   })
 
   let txHashTimestamp = 0
@@ -98,8 +89,8 @@ export const getSuccessExecutionTransaction = (
 export const getValidatorFailedTransaction = (
   bridgeContract: Contract,
   messageData: string,
-  timestamp: number,
+  startBlock: number,
-  getFailedTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>
+  getFailedTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>
 ) => async (validatorData: BasicConfirmationParam): Promise<ConfirmationParam> => {
   const validatorCacheKey = `${CACHE_KEY_FAILED}${validatorData.validator}-${messageData}`
   const failedFromCache = validatorsCache.getData(validatorCacheKey)
@@ -112,8 +103,8 @@ export const getValidatorFailedTransaction = (
     account: validatorData.validator,
     to: bridgeContract.options.address,
     messageData,
-    startTimestamp: timestamp,
+    startBlock,
-    endTimestamp: timestamp + ONE_DAY_TIMESTAMP
+    endBlock: homeBlockNumberProvider.get() || 0
   })
   const newStatus =
     failedTransactions.length > 0 ? VALIDATOR_CONFIRMATION_STATUS.FAILED : VALIDATOR_CONFIRMATION_STATUS.UNDEFINED

deployment/roles/common/tasks/dependencies.yml

@@ -33,7 +33,7 @@
     mode: "0755"
 
 - name: Upgrade pip version
-  shell: pip3 install --upgrade pip
+  shell: pip3 install --upgrade pip==19.3.1
 
 - name: Install python docker library
   shell: pip3 install docker docker-compose setuptools

monitor/checkWorker3.js

@@ -2,6 +2,7 @@ require('dotenv').config()
 const logger = require('./logger')('checkWorker3')
 const stuckTransfers = require('./stuckTransfers')
 const detectMediators = require('./detectMediators')
+const detectFailures = require('./detectFailures')
 const { writeFile, createDir } = require('./utils/file')
 const { web3Home } = require('./utils/web3')
 const { saveCache } = require('./utils/web3Cache')
@@ -24,11 +25,19 @@ async function checkWorker3() {
       logger.debug('Done')
     } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
       createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
+
       logger.debug('calling detectMediators()')
       const mediators = await detectMediators(bridgeMode)
       mediators.ok = true
       mediators.health = true
       writeFile(`/responses/${MONITOR_BRIDGE_NAME}/mediators.json`, mediators)
+
+      logger.debug('calling detectFailures()')
+      const failures = await detectFailures(bridgeMode)
+      failures.ok = true
+      failures.health = true
+      writeFile(`/responses/${MONITOR_BRIDGE_NAME}/failures.json`, failures)
+
       saveCache()
       logger.debug('Done')
     }

monitor/detectFailures.js

@@ -0,0 +1,86 @@
+require('dotenv').config()
+const logger = require('./logger')('alerts')
+const eventsInfo = require('./utils/events')
+const { normalizeAMBMessageEvent } = require('../commons')
+const { getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3')
+
+function normalize(events) {
+  const requests = {}
+  events.forEach(event => {
+    const request = normalizeAMBMessageEvent(event)
+    request.requestTx = event.transactionHash
+    requests[request.messageId] = request
+  })
+  return confirmation => {
+    const request = requests[confirmation.returnValues.messageId] || {}
+    return {
+      ...request,
+      status: false,
+      executionTx: confirmation.transactionHash,
+      executionBlockNumber: confirmation.blockNumber
+    }
+  }
+}
+
+async function main(mode) {
+  const {
+    homeToForeignRequests,
+    homeToForeignConfirmations,
+    foreignToHomeConfirmations,
+    foreignToHomeRequests
+  } = await eventsInfo(mode)
+  const hasFailed = event => !event.returnValues.status
+  const cmp = (a, b) => b.executionBlockNumber - a.executionBlockNumber
+  const failedForeignToHomeMessages = foreignToHomeConfirmations
+    .filter(hasFailed)
+    .map(normalize(foreignToHomeRequests))
+    .sort(cmp)
+  const failedHomeToForeignMessages = homeToForeignConfirmations
+    .filter(hasFailed)
+    .map(normalize(homeToForeignRequests))
+    .sort(cmp)
+
+  const homeBlockNumber = await getHomeBlockNumber()
+  const foreignBlockNumber = await getForeignBlockNumber()
+
+  const blockRanges = [1000, 10000, 100000, 1000000]
+  const rangeNames = [
+    `last${blockRanges[0]}blocks`,
+    ...blockRanges.slice(0, blockRanges.length - 1).map((n, i) => `last${n}to${blockRanges[i + 1]}blocks`),
+    `before${blockRanges[blockRanges.length - 1]}blocks`
+  ]
+
+  const countFailures = (failedMessages, lastBlockNumber) => {
+    const result = {}
+    rangeNames.forEach(name => {
+      result[name] = 0
+    })
+    failedMessages.forEach(message => {
+      const blockAge = lastBlockNumber - message.executionBlockNumber
+      let rangeIndex = blockRanges.findIndex(n => n > blockAge)
+      if (rangeIndex === -1) {
+        rangeIndex = blockRanges.length
+      }
+      result[rangeNames[rangeIndex]] += 1
+    })
+    return result
+  }
+
+  logger.debug('Done')
+
+  return {
+    homeToForeign: {
+      total: failedHomeToForeignMessages.length,
+      stats: countFailures(failedHomeToForeignMessages, foreignBlockNumber),
+      lastFailures: failedHomeToForeignMessages.slice(0, 5)
+    },
+    foreignToHome: {
+      total: failedForeignToHomeMessages.length,
+      stats: countFailures(failedForeignToHomeMessages, homeBlockNumber),
+      lastFailures: failedForeignToHomeMessages.slice(0, 5)
+    },
+    lastChecked: Math.floor(Date.now() / 1000)
+  }
+}
+
+module.exports = main

monitor/index.js

@@ -2,6 +2,7 @@ require('dotenv').config()
 const express = require('express')
 const cors = require('cors')
 const { readFile } = require('./utils/file')
+const { getPrometheusMetrics } = require('./prometheusMetrics')
 
 const app = express()
 const bridgeRouter = express.Router({ mergeParams: true })
@@ -11,9 +12,10 @@ app.use(cors())
 app.get('/favicon.ico', (req, res) => res.sendStatus(204))
 app.use('/:bridgeName', bridgeRouter)
 
-bridgeRouter.get('/', async (req, res, next) => {
+bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', (req, res, next) => {
   try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/getBalances.json`)
+    const { bridgeName, file } = req.params
+    const results = readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`)
     res.json(results)
   } catch (e) {
     // this will eventually be handled by your error handling middleware
@@ -21,49 +23,10 @@ bridgeRouter.get('/', async (req, res, next) => {
   }
 })
 
-bridgeRouter.get('/validators', async (req, res, next) => {
+bridgeRouter.get('/metrics', (req, res, next) => {
   try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/validators.json`)
+    const metrics = getPrometheusMetrics(req.params.bridgeName)
-    res.json(results)
+    res.type('text').send(metrics)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/eventsStats', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/eventsStats.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/alerts', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/alerts.json`)
-    res.json(results)
-  } catch (e) {
-    next(e)
-  }
-})
-
-bridgeRouter.get('/mediators', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/mediators.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/stuckTransfers', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/stuckTransfers.json`)
-    res.json(results)
   } catch (e) {
     next(e)
   }

monitor/prometheusMetrics.js

@@ -0,0 +1,104 @@
+const { readFile } = require('./utils/file')
+
+const {
+  MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST,
+  MONITOR_HOME_TO_FOREIGN_BLOCK_LIST,
+  MONITOR_HOME_VALIDATORS_BALANCE_ENABLE,
+  MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE
+} = process.env
+
+function BridgeConf(type, validatorsBalanceEnable, alertTargetFunc, failureDirection) {
+  this.type = type
+  this.validatorsBalanceEnable = validatorsBalanceEnable
+  this.alertTargetFunc = alertTargetFunc
+  this.failureDirection = failureDirection
+}
+
+const BRIDGE_CONFS = [
+  new BridgeConf('home', MONITOR_HOME_VALIDATORS_BALANCE_ENABLE, 'executeAffirmations', 'homeToForeign'),
+  new BridgeConf('foreign', MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE, 'executeSignatures', 'foreignToHome')
+]
+
+function hasError(obj) {
+  return 'error' in obj
+}
+
+function getPrometheusMetrics(bridgeName) {
+  const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json`
+
+  const metrics = {}
+
+  // Balance metrics
+  const balancesFile = readFile(responsePath('getBalances'))
+
+  if (!hasError(balancesFile)) {
+    const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile
+    metrics.balances_home_value = homeBalances.totalSupply
+    metrics.balances_home_txs_deposit = homeBalances.deposits
+    metrics.balances_home_txs_withdrawal = homeBalances.withdrawals
+
+    metrics.balances_foreign_value = foreignBalances.erc20Balance
+    metrics.balances_foreign_txs_deposit = foreignBalances.deposits
+    metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals
+
+    metrics.balances_diff_value = commonBalances.balanceDiff
+    metrics.balances_diff_deposit = commonBalances.depositsDiff
+    metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff
+    if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) {
+      metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff
+      metrics.balances_unclaimed_value = commonBalances.unclaimedBalance
+    }
+  }
+
+  // Validator metrics
+  const validatorsFile = readFile(responsePath('validators'))
+
+  if (!hasError(validatorsFile)) {
+    for (const bridge of BRIDGE_CONFS) {
+      const allValidators = validatorsFile[bridge.type].validators
+      const validatorAddressesWithBalanceCheck =
+        typeof bridge.validatorsBalanceEnable === 'string'
+          ? bridge.validatorsBalanceEnable.split(' ')
+          : Object.keys(allValidators)
+
+      validatorAddressesWithBalanceCheck.forEach((addr, ind) => {
+        metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance
+      })
+    }
+  }
+
+  // Alert metrics
+  const alertsFile = readFile(responsePath('alerts'))
+
+  if (!hasError(alertsFile)) {
+    for (const bridge of BRIDGE_CONFS) {
+      Object.entries(alertsFile[bridge.alertTargetFunc].misbehavior).forEach(([period, val]) => {
+        metrics[`misbehavior_${bridge.type}_${period}`] = val
+      })
+    }
+  }
+
+  // Failure metrics
+  const failureFile = readFile(responsePath('failures'))
+
+  if (!hasError(failureFile)) {
+    for (const bridge of BRIDGE_CONFS) {
+      const dir = bridge.failureDirection
+      const failures = failureFile[dir]
+      metrics[`failures_${dir}_total`] = failures.total
+      Object.entries(failures.stats).forEach(([period, count]) => {
+        metrics[`failures_${dir}_${period}`] = count
+      })
+    }
+  }
+
+  // Pack metrcis into a plain text
+  return Object.entries(metrics).reduceRight(
+    // Prometheus supports `Nan` and possibly signed `Infinity`
+    // in case cast to `Number` fails
+    (acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`,
+    ''
+  )
+}
+
+module.exports = { getPrometheusMetrics }

monitor/utils/file.js

@@ -1,9 +1,9 @@
 const fs = require('fs')
 const path = require('path')
 
-async function readFile(filePath) {
+function readFile(filePath) {
   try {
-    const content = await fs.readFileSync(filePath)
+    const content = fs.readFileSync(filePath)
     const json = JSON.parse(content)
     const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked
     return Object.assign({}, json, { timeDiff })

oracle/src/sender.js

@@ -173,15 +173,18 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
           `Tx Failed for event Tx ${job.transactionReference}.`,
           e.message
         )
-        if (!e.message.toLowerCase().includes('transaction with the same hash was already imported')) {
+
-          if (isResend) {
+        const message = e.message.toLowerCase()
-            resendJobs.push(job)
+        if (isResend || message.includes('transaction with the same hash was already imported')) {
-          } else {
+          resendJobs.push(job)
-            failedTx.push(job)
+        } else {
-          }
+          // if initial transaction sending has failed not due to the same hash error
+          // send it to the failed tx queue
+          // this will result in the sooner resend attempt than if using resendJobs
+          failedTx.push(job)
         }
 
-        if (e.message.toLowerCase().includes('insufficient funds')) {
+        if (message.includes('insufficient funds')) {
           insufficientFunds = true
           const currentBalance = await web3Instance.eth.getBalance(ORACLE_VALIDATOR_ADDRESS)
           minimumBalance = gasLimit.multipliedBy(gasPrice)