Merge the develop branch to the master branch, preparation to v2.7.0-rc0 (#515)

This merge contains the following set of changes: * [Improvement] Add /metrics endpoint for prometheus support (#512) * [Improvement] Add monitor script for detecting failed AMB messages (#513) * [Improvement] Improve performance of BS requests in ALM (#516) * [Fix] Add pretty error messages for manual execution transaction reverts (#511) * [Fix] Fix resend of stuck pending transactions (#514)
2021-02-25 20:42:51 -06:00 · 2021-02-25 20:42:51 -06:00 · f93ab330cc
commit f93ab330cc
parent 0451d6e373 f64f8b1c91
25 changed files with 599 additions and 288 deletions
--- a/alm/src/components/ConfirmationsContainer.tsx
+++ b/alm/src/components/ConfirmationsContainer.tsx
@ -39,10 +39,17 @@ export interface ConfirmationsContainerParams {
  message: MessageObject
  receipt: Maybe<TransactionReceipt>
  fromHome: boolean
-  timestamp: number
+  homeStartBlock: Maybe<number>
+  foreignStartBlock: Maybe<number>
 }

-export const ConfirmationsContainer = ({ message, receipt, fromHome, timestamp }: ConfirmationsContainerParams) => {
+export const ConfirmationsContainer = ({
+  message,
+  receipt,
+  fromHome,
+  homeStartBlock,
+  foreignStartBlock
+}: ConfirmationsContainerParams) => {
  const {
    home: { name: homeName },
    foreign: { name: foreignName }
@ -62,7 +69,8 @@ export const ConfirmationsContainer = ({ message, receipt, fromHome, timestamp }
    message,
    receipt,
    fromHome,
-    timestamp,
+    homeStartBlock,
+    foreignStartBlock,
    requiredSignatures,
    validatorList,
    blockConfirmations
--- a/alm/src/components/ExecutionConfirmation.tsx
+++ b/alm/src/components/ExecutionConfirmation.tsx
@ -36,10 +36,12 @@ export const ExecutionConfirmation = ({
  const availableManualExecution =
    !isHome &&
    (executionData.status === VALIDATOR_CONFIRMATION_STATUS.WAITING ||
+      executionData.status === VALIDATOR_CONFIRMATION_STATUS.FAILED ||
      (executionData.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED &&
        executionEventsFetched &&
        !!executionData.validator))
  const requiredManualExecution = availableManualExecution && ALM_HOME_TO_FOREIGN_MANUAL_EXECUTION
+  const showAgeColumn = !requiredManualExecution || executionData.status === VALIDATOR_CONFIRMATION_STATUS.FAILED
  const windowWidth = useWindowWidth()

  const txExplorerLink = getExplorerTxUrl(executionData.txHash, isHome)
@ -71,7 +73,7 @@ export const ExecutionConfirmation = ({
          <tr>
            <th>{requiredManualExecution ? 'Execution info' : 'Executed by'}</th>
            <th className="text-center">Status</th>
-            {!requiredManualExecution && <th className="text-center">Age</th>}
+            {showAgeColumn && <th className="text-center">Age</th>}
            {availableManualExecution && <th className="text-center">Actions</th>}
          </tr>
        </Thead>
@ -87,7 +89,7 @@ export const ExecutionConfirmation = ({
              )}
            </td>
            <StatusTd className="text-center">{getExecutionStatusElement(executionData.status)}</StatusTd>
-            {!requiredManualExecution && (
+            {showAgeColumn && (
              <AgeTd className="text-center">
                {executionData.timestamp > 0 ? (
                  <ExplorerTxLink href={txExplorerLink} target="_blank">
--- a/alm/src/components/ManualExecutionButton.tsx
+++ b/alm/src/components/ManualExecutionButton.tsx
@ -2,9 +2,17 @@ import React, { useState, useEffect } from 'react'
 import styled from 'styled-components'
 import { InjectedConnector } from '@web3-react/injected-connector'
 import { useWeb3React } from '@web3-react/core'
-import { INCORRECT_CHAIN_ERROR, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
+import {
+  DOUBLE_EXECUTION_ATTEMPT_ERROR,
+  EXECUTION_FAILED_ERROR,
+  EXECUTION_OUT_OF_GAS_ERROR,
+  INCORRECT_CHAIN_ERROR,
+  VALIDATOR_CONFIRMATION_STATUS
+} from '../config/constants'
 import { useStateProvider } from '../state/StateProvider'
 import { signatureToVRS, packSignatures } from '../utils/signatures'
+import { getSuccessExecutionData } from '../utils/getFinalizationEvent'
+import { TransactionReceipt } from 'web3-eth'

 const StyledButton = styled.button`
  color: var(--button-color);
@ -61,7 +69,9 @@ export const ManualExecutionButton = ({
      if (!library || !foreign.bridgeContract || !signatureCollected || !signatureCollected.length) return

      const signatures = packSignatures(signatureCollected.map(signatureToVRS))
-      const data = foreign.bridgeContract.methods.executeSignatures(messageData, signatures).encodeABI()
+      const messageId = messageData.slice(0, 66)
+      const bridge = foreign.bridgeContract
+      const data = bridge.methods.executeSignatures(messageData, signatures).encodeABI()
      setManualExecution(false)

      library.eth
@ -80,7 +90,27 @@ export const ManualExecutionButton = ({
          })
          setPendingExecution(true)
        })
-        .on('error', (e: Error) => setError(e.message))
+        .on('error', async (e: Error, receipt: TransactionReceipt) => {
+          if (e.message.includes('Transaction has been reverted by the EVM')) {
+            const successExecutionData = await getSuccessExecutionData(bridge, 'RelayedMessage', library, messageId)
+            if (successExecutionData) {
+              setExecutionData(successExecutionData)
+              setError(DOUBLE_EXECUTION_ATTEMPT_ERROR)
+            } else {
+              const { gas } = await library.eth.getTransaction(receipt.transactionHash)
+              setExecutionData({
+                status: VALIDATOR_CONFIRMATION_STATUS.FAILED,
+                validator: account,
+                txHash: receipt.transactionHash,
+                timestamp: Math.floor(new Date().getTime() / 1000.0),
+                executionResult: false
+              })
+              setError(gas === receipt.gasUsed ? EXECUTION_OUT_OF_GAS_ERROR : EXECUTION_FAILED_ERROR)
+            }
+          } else {
+            setError(e.message)
+          }
+        })
    },
    [
      manualExecution,
--- a/alm/src/components/StatusContainer.tsx
+++ b/alm/src/components/StatusContainer.tsx
@ -10,6 +10,7 @@ import { ExplorerTxLink } from './commons/ExplorerTxLink'
 import { ConfirmationsContainer } from './ConfirmationsContainer'
 import { TransactionReceipt } from 'web3-eth'
 import { BackButton } from './commons/BackButton'
+import { useClosestBlock } from '../hooks/useClosestBlock'

 export interface StatusContainerParam {
  onBackToMain: () => void
@ -23,12 +24,15 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar
  const { chainId, txHash, messageIdParam } = useParams()
  const validChainId = chainId === home.chainId.toString() || chainId === foreign.chainId.toString()
  const validParameters = validChainId && validTxHash(txHash)
+  const isHome = chainId === home.chainId.toString()

  const { messages, receipt, status, description, timestamp, loading } = useTransactionStatus({
    txHash: validParameters ? txHash : '',
    chainId: validParameters ? parseInt(chainId) : 0,
    receiptParam
  })
+  const homeStartBlock = useClosestBlock(true, isHome, receipt, timestamp)
+  const foreignStartBlock = useClosestBlock(false, isHome, receipt, timestamp)

  const selectedMessageId = messageIdParam === undefined || messages[messageIdParam] === undefined ? -1 : messageIdParam

@ -64,7 +68,6 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar
  const displayReference = multiMessageSelected ? messages[selectedMessageId].id : txHash
  const formattedMessageId = formatTxHash(displayReference)

-  const isHome = chainId === home.chainId.toString()
  const txExplorerLink = getExplorerTxUrl(txHash, isHome)
  const displayExplorerLink = status !== TRANSACTION_STATUS.NOT_FOUND

@ -101,7 +104,13 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar
      )}
      {displayMessageSelector && <MessageSelector messages={messages} onMessageSelected={onMessageSelected} />}
      {displayConfirmations && (
-        <ConfirmationsContainer message={messageToConfirm} receipt={receipt} fromHome={isHome} timestamp={timestamp} />
+        <ConfirmationsContainer
+          message={messageToConfirm}
+          receipt={receipt}
+          fromHome={isHome}
+          homeStartBlock={homeStartBlock}
+          foreignStartBlock={foreignStartBlock}
+        />
      )}
      <BackButton onBackToMain={onBackToMain} />
    </div>
--- a/alm/src/components/commons/ErrorAlert.tsx
+++ b/alm/src/components/commons/ErrorAlert.tsx
@ -2,6 +2,7 @@ import React from 'react'
 import styled from 'styled-components'
 import { InfoIcon } from './InfoIcon'
 import { CloseIcon } from './CloseIcon'
+import { ExplorerTxLink } from './ExplorerTxLink'

 const StyledErrorAlert = styled.div`
  border: 1px solid var(--failed-color);
@ -15,17 +16,33 @@ const CloseIconContainer = styled.div`
 `

 const TextContainer = styled.div`
+  white-space: pre-wrap;
  flex-direction: column;
 `

-export const ErrorAlert = ({ onClick, error }: { onClick: () => void; error: string }) => (
+export const ErrorAlert = ({ onClick, error }: { onClick: () => void; error: string }) => {
+  const errorArray = error.split('%link')
+  const text = errorArray[0]
+  let link
+  if (errorArray.length > 1) {
+    link = (
+      <ExplorerTxLink href={errorArray[1]} target="_blank" rel="noopener noreferrer">
+        {errorArray[1]}
+      </ExplorerTxLink>
+    )
+  }
+  return (
    <div className="row is-center">
      <StyledErrorAlert className="col-10 is-vertical-align row">
        <InfoIcon color="var(--failed-color)" />
-      <TextContainer className="col-10">{error}</TextContainer>
+        <TextContainer className="col-10">
+          {text}
+          {link}
+        </TextContainer>
        <CloseIconContainer className="col-1 is-vertical-align is-center" onClick={onClick}>
          <CloseIcon color="var(--failed-color)" />
        </CloseIconContainer>
      </StyledErrorAlert>
    </div>
-)
+  )
+}
--- a/alm/src/config/constants.ts
+++ b/alm/src/config/constants.ts
@ -18,9 +18,8 @@ export const ALM_HOME_TO_FOREIGN_MANUAL_EXECUTION: boolean =

 export const HOME_RPC_POLLING_INTERVAL: number = 5000
 export const FOREIGN_RPC_POLLING_INTERVAL: number = 5000
-export const BLOCK_RANGE: number = 50
-export const ONE_DAY_TIMESTAMP: number = 86400
-export const THREE_DAYS_TIMESTAMP: number = 259200
+export const BLOCK_RANGE: number = 500
+export const MAX_TX_SEARCH_BLOCK_RANGE: number = 10000

 export const EXECUTE_AFFIRMATION_HASH = 'e7a2c01f'
 export const SUBMIT_SIGNATURE_HASH = '630cea8e'
@ -66,3 +65,12 @@ export const VALIDATOR_CONFIRMATION_STATUS = {
 export const SEARCHING_TX = 'Searching Transaction...'

 export const INCORRECT_CHAIN_ERROR = `Incorrect chain chosen. Switch to ${FOREIGN_NETWORK_NAME} in the wallet.`
+
+export const DOUBLE_EXECUTION_ATTEMPT_ERROR = `Your execution transaction has been reverted.
+However, the execution completed successfully in the transaction sent by a different party.`
+
+export const EXECUTION_FAILED_ERROR = `Your execution transaction has been reverted.
+Please, contact the support by messaging on %linkhttps://forum.poa.network/c/support`
+
+export const EXECUTION_OUT_OF_GAS_ERROR = `Your execution transaction has been reverted due to Out-of-Gas error.
+Please, resend the transaction and provide more gas to it.`
--- a/alm/src/hooks/useClosestBlock.ts
+++ b/alm/src/hooks/useClosestBlock.ts
@ -0,0 +1,68 @@
+import { useEffect, useState } from 'react'
+import { TransactionReceipt } from 'web3-eth'
+import { useStateProvider } from '../state/StateProvider'
+import { FOREIGN_EXPLORER_API, HOME_EXPLORER_API } from '../config/constants'
+import { getClosestBlockByTimestamp } from '../utils/explorer'
+
+export function useClosestBlock(
+  searchHome: boolean,
+  fromHome: boolean,
+  receipt: Maybe<TransactionReceipt>,
+  timestamp: number
+) {
+  const { home, foreign } = useStateProvider()
+  const [blockNumber, setBlockNumber] = useState<number | null>(null)
+
+  useEffect(
+    () => {
+      if (!receipt || blockNumber || !timestamp) return
+
+      if (fromHome === searchHome) {
+        setBlockNumber(receipt.blockNumber)
+        return
+      }
+
+      const web3 = searchHome ? home.web3 : foreign.web3
+      if (!web3) return
+
+      const getBlock = async () => {
+        // try to fast-fetch closest block number from the chain explorer
+        try {
+          const api = searchHome ? HOME_EXPLORER_API : FOREIGN_EXPLORER_API
+          setBlockNumber(await getClosestBlockByTimestamp(api, timestamp))
+          return
+        } catch {}
+
+        const lastBlock = await web3.eth.getBlock('latest')
+        if (lastBlock.timestamp <= timestamp) {
+          setBlockNumber(lastBlock.number)
+          return
+        }
+
+        const oldBlock = await web3.eth.getBlock(Math.max(lastBlock.number - 10000, 1))
+        const blockDiff = lastBlock.number - oldBlock.number
+        const timeDiff = (lastBlock.timestamp as number) - (oldBlock.timestamp as number)
+        const averageBlockTime = timeDiff / blockDiff
+        let currentBlock = lastBlock
+
+        let prevBlockDiff = Infinity
+        while (true) {
+          const timeDiff = (currentBlock.timestamp as number) - timestamp
+          const blockDiff = Math.ceil(timeDiff / averageBlockTime)
+          if (Math.abs(blockDiff) < 5 || Math.abs(blockDiff) >= Math.abs(prevBlockDiff)) {
+            setBlockNumber(currentBlock.number - blockDiff - 5)
+            break
+          }
+
+          prevBlockDiff = blockDiff
+          currentBlock = await web3.eth.getBlock(currentBlock.number - blockDiff)
+        }
+      }
+
+      getBlock()
+    },
+    [blockNumber, foreign.web3, fromHome, home.web3, receipt, searchHome, timestamp]
+  )
+
+  return blockNumber
+}
--- a/alm/src/hooks/useMessageConfirmations.ts
+++ b/alm/src/hooks/useMessageConfirmations.ts
@ -28,7 +28,8 @@ export interface useMessageConfirmationsParams {
  message: MessageObject
  receipt: Maybe<TransactionReceipt>
  fromHome: boolean
-  timestamp: number
+  homeStartBlock: Maybe<number>
+  foreignStartBlock: Maybe<number>
  requiredSignatures: number
  validatorList: string[]
  blockConfirmations: number
@ -56,7 +57,8 @@ export const useMessageConfirmations = ({
  message,
  receipt,
  fromHome,
-  timestamp,
+  homeStartBlock,
+  foreignStartBlock,
  requiredSignatures,
  validatorList,
  blockConfirmations
@ -90,6 +92,17 @@ export const useMessageConfirmations = ({
    return filteredList.length > 0
  }

+  // start watching blocks at the start
+  useEffect(
+    () => {
+      if (!home.web3 || !foreign.web3) return
+
+      homeBlockNumberProvider.start(home.web3)
+      foreignBlockNumberProvider.start(foreign.web3)
+    },
+    [foreign.web3, home.web3]
+  )
+
  // Check if the validators are waiting for block confirmations to verify the message
  useEffect(
    () => {
@ -105,9 +118,6 @@ export const useMessageConfirmations = ({

      const blockProvider = fromHome ? homeBlockNumberProvider : foreignBlockNumberProvider
      const interval = fromHome ? HOME_RPC_POLLING_INTERVAL : FOREIGN_RPC_POLLING_INTERVAL
-      const web3 = fromHome ? home.web3 : foreign.web3
-      blockProvider.start(web3)
-
      const targetBlock = receipt.blockNumber + blockConfirmations

      checkSignaturesWaitingForBLocks(
@ -123,7 +133,6 @@ export const useMessageConfirmations = ({

      return () => {
        unsubscribe()
-        blockProvider.stop()
      }
    },
    [
@ -153,8 +162,6 @@ export const useMessageConfirmations = ({
        })
      }

-      homeBlockNumberProvider.start(home.web3)
-
      const fromBlock = receipt.blockNumber
      const toBlock = fromBlock + BLOCK_RANGE
      const messageHash = home.web3.utils.soliditySha3Raw(message.data)
@ -171,7 +178,6 @@ export const useMessageConfirmations = ({

      return () => {
        unsubscribe()
-        homeBlockNumberProvider.stop()
      }
    },
    [fromHome, home.bridgeContract, home.web3, message.data, receipt, signatureCollected]
@ -192,7 +198,6 @@ export const useMessageConfirmations = ({
        })
      }

-      homeBlockNumberProvider.start(home.web3)
      const targetBlock = collectedSignaturesEvent.blockNumber + blockConfirmations

      checkWaitingBlocksForExecution(
@ -208,7 +213,6 @@ export const useMessageConfirmations = ({

      return () => {
        unsubscribe()
-        homeBlockNumberProvider.stop()
      }
    },
    [collectedSignaturesEvent, fromHome, blockConfirmations, home.web3, receipt, waitingBlocksForExecutionResolved]
@ -218,7 +222,7 @@ export const useMessageConfirmations = ({
  // To avoid making extra requests, this is only executed when validators finished waiting for blocks confirmations
  useEffect(
    () => {
-      if (!waitingBlocksResolved || !timestamp || !requiredSignatures) return
+      if (!waitingBlocksResolved || !homeStartBlock || !requiredSignatures) return

      const subscriptions: Array<number> = []

@ -239,7 +243,7 @@ export const useMessageConfirmations = ({
        setSignatureCollected,
        waitingBlocksResolved,
        subscriptions,
-        timestamp,
+        homeStartBlock,
        getValidatorFailedTransactionsForMessage,
        setFailedConfirmations,
        getValidatorPendingTransactionsForMessage,
@ -259,7 +263,7 @@ export const useMessageConfirmations = ({
      home.bridgeContract,
      requiredSignatures,
      waitingBlocksResolved,
-      timestamp,
+      homeStartBlock,
      setConfirmations
    ]
  )
@ -270,6 +274,8 @@ export const useMessageConfirmations = ({
  useEffect(
    () => {
      if ((fromHome && !waitingBlocksForExecutionResolved) || (!fromHome && !waitingBlocksResolved)) return
+      const startBlock = fromHome ? foreignStartBlock : homeStartBlock
+      if (!startBlock) return

      const subscriptions: Array<number> = []

@ -285,6 +291,7 @@ export const useMessageConfirmations = ({
      const interval = fromHome ? FOREIGN_RPC_POLLING_INTERVAL : HOME_RPC_POLLING_INTERVAL

      getFinalizationEvent(
+        fromHome,
        bridgeContract,
        contractEvent,
        providedWeb3,
@ -293,7 +300,7 @@ export const useMessageConfirmations = ({
        message,
        interval,
        subscriptions,
-        timestamp,
+        startBlock,
        collectedSignaturesEvent,
        getExecutionFailedTransactionForMessage,
        setFailedExecution,
@ -315,8 +322,9 @@ export const useMessageConfirmations = ({
      home.web3,
      waitingBlocksResolved,
      waitingBlocksForExecutionResolved,
-      timestamp,
-      collectedSignaturesEvent
+      collectedSignaturesEvent,
+      foreignStartBlock,
+      homeStartBlock
    ]
  )

@ -328,6 +336,9 @@ export const useMessageConfirmations = ({
          ? CONFIRMATIONS_STATUS.SUCCESS
          : CONFIRMATIONS_STATUS.SUCCESS_MESSAGE_FAILED
        setStatus(newStatus)
+
+        foreignBlockNumberProvider.stop()
+        homeBlockNumberProvider.stop()
      } else if (signatureCollected) {
        if (fromHome) {
          if (waitingBlocksForExecution) {
--- a/alm/src/services/BlockNumberProvider.ts
+++ b/alm/src/services/BlockNumberProvider.ts
@ -1,6 +1,6 @@
 import Web3 from 'web3'
 import differenceInMilliseconds from 'date-fns/differenceInMilliseconds'
-import { HOME_RPC_POLLING_INTERVAL } from '../config/constants'
+import { FOREIGN_RPC_POLLING_INTERVAL, HOME_RPC_POLLING_INTERVAL } from '../config/constants'

 export class BlockNumberProvider {
  private running: number
@ -61,4 +61,4 @@ export class BlockNumberProvider {
 }

 export const homeBlockNumberProvider = new BlockNumberProvider(HOME_RPC_POLLING_INTERVAL)
-export const foreignBlockNumberProvider = new BlockNumberProvider(HOME_RPC_POLLING_INTERVAL)
+export const foreignBlockNumberProvider = new BlockNumberProvider(FOREIGN_RPC_POLLING_INTERVAL)
--- a/alm/src/utils/tests/explorer.test.ts
+++ b/alm/src/utils/tests/explorer.test.ts
@ -17,19 +17,33 @@ const otherAddress = '0xD4075FB57fCf038bFc702c915Ef9592534bED5c1'

 describe('getFailedTransactions', () => {
  test('should only return failed transactions', async () => {
-    const transactions = [{ isError: '0' }, { isError: '1' }, { isError: '0' }, { isError: '1' }, { isError: '1' }]
+    const to = otherAddress
+    const transactions = [
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '1', to }
+    ]

    const fetchAccountTransactions = jest.fn().mockImplementation(() => transactions)
-    const result = await getFailedTransactions('', '', 0, 1, '', fetchAccountTransactions)
+    const result = await getFailedTransactions('', to, 0, 1, '', fetchAccountTransactions)
    expect(result.length).toEqual(3)
  })
 })
 describe('getSuccessTransactions', () => {
  test('should only return success transactions', async () => {
-    const transactions = [{ isError: '0' }, { isError: '1' }, { isError: '0' }, { isError: '1' }, { isError: '1' }]
+    const to = otherAddress
+    const transactions = [
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '0', to },
+      { isError: '1', to },
+      { isError: '1', to }
+    ]

    const fetchAccountTransactions = jest.fn().mockImplementation(() => transactions)
-    const result = await getSuccessTransactions('', '', 0, 1, '', fetchAccountTransactions)
+    const result = await getSuccessTransactions('', to, 0, 1, '', fetchAccountTransactions)
    expect(result.length).toEqual(2)
  })
 })
@ -74,8 +88,8 @@ describe('getExecutionFailedTransactionForMessage', () => {
        account: '',
        to: '',
        messageData,
-        startTimestamp: 0,
-        endTimestamp: 1
+        startBlock: 0,
+        endBlock: 1
      },
      fetchAccountTransactions
    )
--- a/alm/src/utils/tests/getFinalizationEvent.test.ts
+++ b/alm/src/utils/tests/getFinalizationEvent.test.ts
@ -64,6 +64,7 @@ describe('getFinalizationEvent', () => {
    const setExecutionEventsFetched = jest.fn()

    await getFinalizationEvent(
+      true,
      contract,
      eventName,
      web3,
@ -115,6 +116,7 @@ describe('getFinalizationEvent', () => {
    const setExecutionEventsFetched = jest.fn()

    await getFinalizationEvent(
+      true,
      contract,
      eventName,
      web3,
@ -166,6 +168,7 @@ describe('getFinalizationEvent', () => {
    const setExecutionEventsFetched = jest.fn()

    await getFinalizationEvent(
+      true,
      contract,
      eventName,
      web3,
@ -217,6 +220,7 @@ describe('getFinalizationEvent', () => {
    const setExecutionEventsFetched = jest.fn()

    await getFinalizationEvent(
+      true,
      contract,
      eventName,
      web3,
@ -275,6 +279,7 @@ describe('getFinalizationEvent', () => {
    const setExecutionEventsFetched = jest.fn()

    await getFinalizationEvent(
+      true,
      contract,
      eventName,
      web3,
--- a/alm/src/utils/executionWaitingForBlocks.ts
+++ b/alm/src/utils/executionWaitingForBlocks.ts
@ -30,7 +30,6 @@ export const checkWaitingBlocksForExecution = async (
    )
    setWaitingBlocksForExecutionResolved(true)
    setWaitingBlocksForExecution(false)
-    blockProvider.stop()
  } else {
    let nextInterval = interval
    if (!currentBlock) {
--- a/alm/src/utils/explorer.ts
+++ b/alm/src/utils/explorer.ts
@ -1,8 +1,10 @@
 import {
+  BLOCK_RANGE,
  EXECUTE_AFFIRMATION_HASH,
  EXECUTE_SIGNATURES_HASH,
  FOREIGN_EXPLORER_API,
  HOME_EXPLORER_API,
+  MAX_TX_SEARCH_BLOCK_RANGE,
  SUBMIT_SIGNATURE_HASH
 } from '../config/constants'

@ -12,6 +14,7 @@ export interface APITransaction {
  input: string
  to: string
  hash: string
+  blockNumber: string
 }

 export interface APIPendingTransaction {
@ -27,106 +30,42 @@ export interface PendingTransactionsParams {

 export interface AccountTransactionsParams {
  account: string
-  to: string
-  startTimestamp: number
-  endTimestamp: number
+  startBlock: number
+  endBlock: number
  api: string
 }

-export interface GetFailedTransactionParams {
-  account: string
-  to: string
-  messageData: string
-  startTimestamp: number
-  endTimestamp: number
-}
-
 export interface GetPendingTransactionParams {
  account: string
  to: string
  messageData: string
 }

-export const fetchAccountTransactionsFromBlockscout = async ({
-  account,
-  to,
-  startTimestamp,
-  endTimestamp,
-  api
-}: AccountTransactionsParams): Promise<APITransaction[]> => {
-  const url = `${api}?module=account&action=txlist&address=${account}&filterby=from=${account}&to=${to}&starttimestamp=${startTimestamp}&endtimestamp=${endTimestamp}`
+export interface GetTransactionParams extends GetPendingTransactionParams {
+  startBlock: number
+  endBlock: number
+}
+
+export const fetchAccountTransactions = async ({ account, startBlock, endBlock, api }: AccountTransactionsParams) => {
+  const params = `module=account&action=txlist&address=${account}&filterby=from&startblock=${startBlock}&endblock=${endBlock}`
+  const url = api.includes('blockscout') ? `${api}?${params}` : `${api}&${params}`

-  try {
  const result = await fetch(url).then(res => res.json())
-    if (result.status === '0') {
+
+  if (result.message === 'No transactions found') {
    return []
  }

  return result.result
-  } catch (e) {
-    console.log(e)
-    return []
-  }
-}
-
-export const getBlockByTimestampUrl = (api: string, timestamp: number) =>
-  `${api}&module=block&action=getblocknobytime&timestamp=${timestamp}&closest=before`
-
-export const fetchAccountTransactionsFromEtherscan = async ({
-  account,
-  to,
-  startTimestamp,
-  endTimestamp,
-  api
-}: AccountTransactionsParams): Promise<APITransaction[]> => {
-  const startBlockUrl = getBlockByTimestampUrl(api, startTimestamp)
-  const endBlockUrl = getBlockByTimestampUrl(api, endTimestamp)
-  let fromBlock = 0
-  let toBlock = 9999999999999
-  try {
-    const [fromBlockResult, toBlockResult] = await Promise.all([
-      fetch(startBlockUrl).then(res => res.json()),
-      fetch(endBlockUrl).then(res => res.json())
-    ])
-
-    if (fromBlockResult.status !== '0') {
-      fromBlock = parseInt(fromBlockResult.result)
-    }
-
-    if (toBlockResult.status !== '0') {
-      toBlock = parseInt(toBlockResult.result)
-    }
-  } catch (e) {
-    console.log(e)
-    return []
-  }
-
-  const url = `${api}&module=account&action=txlist&address=${account}&startblock=${fromBlock}&endblock=${toBlock}`
-
-  try {
-    const result = await fetch(url).then(res => res.json())
-
-    if (result.status === '0') {
-      return []
-    }
-
-    const toAddressLowerCase = to.toLowerCase()
-    const transactions: APITransaction[] = result.result
-    return transactions.filter(t => t.to.toLowerCase() === toAddressLowerCase)
-  } catch (e) {
-    console.log(e)
-    return []
-  }
-}
-
-export const fetchAccountTransactions = (api: string) => {
-  return api.includes('blockscout') ? fetchAccountTransactionsFromBlockscout : fetchAccountTransactionsFromEtherscan
 }

 export const fetchPendingTransactions = async ({
  account,
  api
 }: PendingTransactionsParams): Promise<APIPendingTransaction[]> => {
+  if (!api.includes('blockscout')) {
+    return []
+  }
  const url = `${api}?module=account&action=pendingtxlist&address=${account}`

  try {
@ -141,30 +80,96 @@ export const fetchPendingTransactions = async ({
  }
 }

+export const getClosestBlockByTimestamp = async (api: string, timestamp: number): Promise<number> => {
+  if (api.includes('blockscout')) {
+    throw new Error('Blockscout does not support getblocknobytime')
+  }
+
+  const url = `${api}&module=block&action=getblocknobytime&timestamp=${timestamp}&closest=before`
+
+  const blockNumber = await fetch(url).then(res => res.json())
+
+  return parseInt(blockNumber.result)
+}
+
+// fast version of fetchAccountTransactions
+// sequentially fetches transactions in small batches
+// caches the result
+const transactionsCache: { [key: string]: { lastBlock: number; transactions: APITransaction[] } } = {}
+export const getAccountTransactions = async ({
+  account,
+  startBlock,
+  endBlock,
+  api
+}: AccountTransactionsParams): Promise<APITransaction[]> => {
+  const key = `${account}-${startBlock}-${api}`
+
+  // initialize empty cache if it doesn't exist yet
+  if (!transactionsCache[key]) {
+    transactionsCache[key] = { lastBlock: startBlock - 1, transactions: [] }
+  }
+
+  // if cache contains events up to block X,
+  // new batch is fetched for range [X + 1, X + 1 + BLOCK_RANGE]
+  const newStartBlock = transactionsCache[key].lastBlock + 1
+  const newEndBlock = newStartBlock + BLOCK_RANGE
+
+  // search for new transactions only if max allowed block range is not yet exceeded
+  if (newEndBlock <= startBlock + MAX_TX_SEARCH_BLOCK_RANGE) {
+    const newTransactions = await fetchAccountTransactions({
+      account,
+      startBlock: newStartBlock,
+      endBlock: newEndBlock,
+      api
+    })
+
+    const transactions = transactionsCache[key].transactions.concat(...newTransactions)
+
+    // cache updated transactions list
+    transactionsCache[key].transactions = transactions
+
+    // enbBlock is assumed to be the current block number of the chain
+    // if the whole range is finalized, last block can be safely updated to the end of the range
+    // this works even if there are no transactions in the list
+    if (newEndBlock < endBlock) {
+      transactionsCache[key].lastBlock = newEndBlock
+    } else if (transactions.length > 0) {
+      transactionsCache[key].lastBlock = parseInt(transactions[transactions.length - 1].blockNumber, 10)
+    }
+
+    return transactions
+  }
+
+  console.warn(`Reached max transaction searching range, returning previously cached transactions for ${account}`)
+  return transactionsCache[key].transactions
+}
+
+const filterReceiver = (to: string) => (tx: APITransaction) => tx.to.toLowerCase() === to.toLowerCase()
+
 export const getFailedTransactions = async (
  account: string,
  to: string,
-  startTimestamp: number,
-  endTimestamp: number,
+  startBlock: number,
+  endBlock: number,
  api: string,
-  fetchAccountTransactions: (args: AccountTransactionsParams) => Promise<APITransaction[]>
+  getAccountTransactionsMethod = getAccountTransactions
 ): Promise<APITransaction[]> => {
-  const transactions = await fetchAccountTransactions({ account, to, startTimestamp, endTimestamp, api })
+  const transactions = await getAccountTransactionsMethod({ account, startBlock, endBlock, api })

-  return transactions.filter(t => t.isError !== '0')
+  return transactions.filter(t => t.isError !== '0').filter(filterReceiver(to))
 }

 export const getSuccessTransactions = async (
  account: string,
  to: string,
-  startTimestamp: number,
-  endTimestamp: number,
+  startBlock: number,
+  endBlock: number,
  api: string,
-  fetchAccountTransactions: (args: AccountTransactionsParams) => Promise<APITransaction[]>
+  getAccountTransactionsMethod = getAccountTransactions
 ): Promise<APITransaction[]> => {
-  const transactions = await fetchAccountTransactions({ account, to, startTimestamp, endTimestamp, api })
+  const transactions = await getAccountTransactionsMethod({ account, startBlock, endBlock, api })

-  return transactions.filter(t => t.isError === '0')
+  return transactions.filter(t => t.isError === '0').filter(filterReceiver(to))
 }

 export const filterValidatorSignatureTransaction = (
@ -183,17 +188,10 @@ export const getValidatorFailedTransactionsForMessage = async ({
  account,
  to,
  messageData,
-  startTimestamp,
-  endTimestamp
-}: GetFailedTransactionParams): Promise<APITransaction[]> => {
-  const failedTransactions = await getFailedTransactions(
-    account,
-    to,
-    startTimestamp,
-    endTimestamp,
-    HOME_EXPLORER_API,
-    fetchAccountTransactionsFromBlockscout
-  )
+  startBlock,
+  endBlock
+}: GetTransactionParams): Promise<APITransaction[]> => {
+  const failedTransactions = await getFailedTransactions(account, to, startBlock, endBlock, HOME_EXPLORER_API)

  return filterValidatorSignatureTransaction(failedTransactions, messageData)
 }
@ -202,33 +200,19 @@ export const getValidatorSuccessTransactionsForMessage = async ({
  account,
  to,
  messageData,
-  startTimestamp,
-  endTimestamp
-}: GetFailedTransactionParams): Promise<APITransaction[]> => {
-  const transactions = await getSuccessTransactions(
-    account,
-    to,
-    startTimestamp,
-    endTimestamp,
-    HOME_EXPLORER_API,
-    fetchAccountTransactionsFromBlockscout
-  )
+  startBlock,
+  endBlock
+}: GetTransactionParams): Promise<APITransaction[]> => {
+  const transactions = await getSuccessTransactions(account, to, startBlock, endBlock, HOME_EXPLORER_API)

  return filterValidatorSignatureTransaction(transactions, messageData)
 }

 export const getExecutionFailedTransactionForMessage = async (
-  { account, to, messageData, startTimestamp, endTimestamp }: GetFailedTransactionParams,
+  { account, to, messageData, startBlock, endBlock }: GetTransactionParams,
  getFailedTransactionsMethod = getFailedTransactions
 ): Promise<APITransaction[]> => {
-  const failedTransactions = await getFailedTransactionsMethod(
-    account,
-    to,
-    startTimestamp,
-    endTimestamp,
-    FOREIGN_EXPLORER_API,
-    fetchAccountTransactions(FOREIGN_EXPLORER_API)
-  )
+  const failedTransactions = await getFailedTransactionsMethod(account, to, startBlock, endBlock, FOREIGN_EXPLORER_API)

  const messageDataValue = messageData.replace('0x', '')
  return failedTransactions.filter(t => t.input.includes(EXECUTE_SIGNATURES_HASH) && t.input.includes(messageDataValue))
--- a/alm/src/utils/getCollectedSignaturesEvent.ts
+++ b/alm/src/utils/getCollectedSignaturesEvent.ts
@ -31,7 +31,6 @@ export const getCollectedSignaturesEvent = async (
  if (filteredEvents.length) {
    const event = filteredEvents[0]
    setCollectedSignaturesEvent(event)
-    homeBlockNumberProvider.stop()
  } else {
    const newFromBlock = currentBlock ? securedToBlock : fromBlock
    const newToBlock = currentBlock ? toBlock + BLOCK_RANGE : toBlock
--- a/alm/src/utils/getConfirmationsForTx.ts
+++ b/alm/src/utils/getConfirmationsForTx.ts
@ -1,12 +1,7 @@
 import Web3 from 'web3'
 import { Contract } from 'web3-eth-contract'
 import { HOME_RPC_POLLING_INTERVAL, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
-import {
-  GetFailedTransactionParams,
-  APITransaction,
-  APIPendingTransaction,
-  GetPendingTransactionParams
-} from './explorer'
+import { GetTransactionParams, APITransaction, APIPendingTransaction, GetPendingTransactionParams } from './explorer'
 import { getAffirmationsSigned, getMessagesSigned } from './contract'
 import {
  getValidatorConfirmation,
@ -43,12 +38,12 @@ export const getConfirmationsForTx = async (
  setSignatureCollected: Function,
  waitingBlocksResolved: boolean,
  subscriptions: number[],
-  timestamp: number,
-  getFailedTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>,
+  startBlock: number,
+  getFailedTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>,
  setFailedConfirmations: Function,
  getPendingTransactions: (args: GetPendingTransactionParams) => Promise<APIPendingTransaction[]>,
  setPendingConfirmations: Function,
-  getSuccessTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>
+  getSuccessTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>
 ) => {
  if (!web3 || !validatorList || !validatorList.length || !bridgeContract || !waitingBlocksResolved) return

@ -102,7 +97,7 @@ export const getConfirmationsForTx = async (
  // Check if confirmation failed
  const validatorFailedConfirmationsChecks = await Promise.all(
    undefinedConfirmations.map(
-      getValidatorFailedTransaction(bridgeContract, messageData, timestamp, getFailedTransactions)
+      getValidatorFailedTransaction(bridgeContract, messageData, startBlock, getFailedTransactions)
    )
  )
  const validatorFailedConfirmations = validatorFailedConfirmationsChecks.filter(
@ -136,7 +131,7 @@ export const getConfirmationsForTx = async (
  // get transactions from success signatures
  const successConfirmationWithData = await Promise.all(
    successConfirmations.map(
-      getSuccessExecutionTransaction(web3, bridgeContract, fromHome, messageData, timestamp, getSuccessTransactions)
+      getSuccessExecutionTransaction(web3, bridgeContract, fromHome, messageData, startBlock, getSuccessTransactions)
    )
  )

@ -162,7 +157,7 @@ export const getConfirmationsForTx = async (
          setSignatureCollected,
          waitingBlocksResolved,
          subscriptions,
-          timestamp,
+          startBlock,
          getFailedTransactions,
          setFailedConfirmations,
          getPendingTransactions,
--- a/alm/src/utils/getFinalizationEvent.ts
+++ b/alm/src/utils/getFinalizationEvent.ts
@ -1,41 +1,20 @@
 import { Contract, EventData } from 'web3-eth-contract'
 import Web3 from 'web3'
-import { CACHE_KEY_EXECUTION_FAILED, THREE_DAYS_TIMESTAMP, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
+import { CACHE_KEY_EXECUTION_FAILED, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
 import { ExecutionData } from '../hooks/useMessageConfirmations'
-import {
-  APIPendingTransaction,
-  APITransaction,
-  GetFailedTransactionParams,
-  GetPendingTransactionParams
-} from './explorer'
+import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer'
 import { getBlock, MessageObject } from './web3'
 import validatorsCache from '../services/ValidatorsCache'
+import { foreignBlockNumberProvider, homeBlockNumberProvider } from '../services/BlockNumberProvider'

-export const getFinalizationEvent = async (
-  contract: Maybe<Contract>,
-  eventName: string,
-  web3: Maybe<Web3>,
-  setResult: React.Dispatch<React.SetStateAction<ExecutionData>>,
-  waitingBlocksResolved: boolean,
-  message: MessageObject,
-  interval: number,
-  subscriptions: number[],
-  timestamp: number,
-  collectedSignaturesEvent: Maybe<EventData>,
-  getFailedExecution: (args: GetFailedTransactionParams) => Promise<APITransaction[]>,
-  setFailedExecution: Function,
-  getPendingExecution: (args: GetPendingTransactionParams) => Promise<APIPendingTransaction[]>,
-  setPendingExecution: Function,
-  setExecutionEventsFetched: Function
-) => {
-  if (!contract || !web3 || !waitingBlocksResolved) return
+export const getSuccessExecutionData = async (contract: Contract, eventName: string, web3: Web3, messageId: string) => {
  // Since it filters by the message id, only one event will be fetched
  // so there is no need to limit the range of the block to reduce the network traffic
  const events: EventData[] = await contract.getPastEvents(eventName, {
    fromBlock: 0,
    toBlock: 'latest',
    filter: {
-      messageId: message.id
+      messageId
    }
  })
  if (events.length > 0) {
@ -48,13 +27,39 @@ export const getFinalizationEvent = async (
    const blockTimestamp = typeof block.timestamp === 'string' ? parseInt(block.timestamp) : block.timestamp
    const validatorAddress = web3.utils.toChecksumAddress(txReceipt.from)

-    setResult({
+    return {
      status: VALIDATOR_CONFIRMATION_STATUS.SUCCESS,
      validator: validatorAddress,
      txHash: event.transactionHash,
      timestamp: blockTimestamp,
      executionResult: event.returnValues.status
-    })
+    }
+  }
+  return null
+}
+
+export const getFinalizationEvent = async (
+  fromHome: boolean,
+  contract: Maybe<Contract>,
+  eventName: string,
+  web3: Maybe<Web3>,
+  setResult: React.Dispatch<React.SetStateAction<ExecutionData>>,
+  waitingBlocksResolved: boolean,
+  message: MessageObject,
+  interval: number,
+  subscriptions: number[],
+  startBlock: number,
+  collectedSignaturesEvent: Maybe<EventData>,
+  getFailedExecution: (args: GetTransactionParams) => Promise<APITransaction[]>,
+  setFailedExecution: Function,
+  getPendingExecution: (args: GetPendingTransactionParams) => Promise<APIPendingTransaction[]>,
+  setPendingExecution: Function,
+  setExecutionEventsFetched: Function
+) => {
+  if (!contract || !web3 || !waitingBlocksResolved) return
+  const successExecutionData = await getSuccessExecutionData(contract, eventName, web3, message.id)
+  if (successExecutionData) {
+    setResult(successExecutionData)
  } else {
    setExecutionEventsFetched(true)
    // If event is defined, it means it is a message from Home to Foreign
@ -84,14 +89,15 @@ export const getFinalizationEvent = async (
      } else {
        const validatorExecutionCacheKey = `${CACHE_KEY_EXECUTION_FAILED}${validator}-${message.id}`
        const failedFromCache = validatorsCache.get(validatorExecutionCacheKey)
+        const blockProvider = fromHome ? foreignBlockNumberProvider : homeBlockNumberProvider

        if (!failedFromCache) {
          const failedTransactions = await getFailedExecution({
            account: validator,
            to: contract.options.address,
            messageData: message.data,
-            startTimestamp: timestamp,
-            endTimestamp: timestamp + THREE_DAYS_TIMESTAMP
+            startBlock,
+            endBlock: blockProvider.get() || 0
          })

          if (failedTransactions.length > 0) {
@ -117,6 +123,7 @@ export const getFinalizationEvent = async (
    const timeoutId = setTimeout(
      () =>
        getFinalizationEvent(
+          fromHome,
          contract,
          eventName,
          web3,
@ -125,7 +132,7 @@ export const getFinalizationEvent = async (
          message,
          interval,
          subscriptions,
-          timestamp,
+          startBlock,
          collectedSignaturesEvent,
          getFailedExecution,
          setFailedExecution,
--- a/alm/src/utils/signatureWaitingForBlocks.ts
+++ b/alm/src/utils/signatureWaitingForBlocks.ts
@ -16,7 +16,6 @@ export const checkSignaturesWaitingForBLocks = async (
  if (currentBlock && currentBlock >= targetBlock) {
    setWaitingBlocksResolved(true)
    setWaitingStatus(false)
-    blockProvider.stop()
  } else {
    let nextInterval = interval
    if (!currentBlock) {
--- a/alm/src/utils/validatorConfirmationHelpers.ts
+++ b/alm/src/utils/validatorConfirmationHelpers.ts
@ -2,18 +2,9 @@ import Web3 from 'web3'
 import { Contract } from 'web3-eth-contract'
 import { BasicConfirmationParam, ConfirmationParam } from '../hooks/useMessageConfirmations'
 import validatorsCache from '../services/ValidatorsCache'
-import {
-  CACHE_KEY_FAILED,
-  CACHE_KEY_SUCCESS,
-  ONE_DAY_TIMESTAMP,
-  VALIDATOR_CONFIRMATION_STATUS
-} from '../config/constants'
-import {
-  APIPendingTransaction,
-  APITransaction,
-  GetFailedTransactionParams,
-  GetPendingTransactionParams
-} from './explorer'
+import { CACHE_KEY_FAILED, CACHE_KEY_SUCCESS, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants'
+import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer'
+import { homeBlockNumberProvider } from '../services/BlockNumberProvider'

 export const getValidatorConfirmation = (
  web3: Web3,
@ -50,8 +41,8 @@ export const getSuccessExecutionTransaction = (
  bridgeContract: Contract,
  fromHome: boolean,
  messageData: string,
-  timestamp: number,
-  getSuccessTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>
+  startBlock: number,
+  getSuccessTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>
 ) => async (validatorData: BasicConfirmationParam): Promise<ConfirmationParam> => {
  const { validator } = validatorData
  const validatorCacheKey = `${CACHE_KEY_SUCCESS}${validatorData.validator}-${messageData}`
@ -65,8 +56,8 @@ export const getSuccessExecutionTransaction = (
    account: validatorData.validator,
    to: bridgeContract.options.address,
    messageData,
-    startTimestamp: timestamp,
-    endTimestamp: timestamp + ONE_DAY_TIMESTAMP
+    startBlock,
+    endBlock: homeBlockNumberProvider.get() || 0
  })

  let txHashTimestamp = 0
@ -98,8 +89,8 @@ export const getSuccessExecutionTransaction = (
 export const getValidatorFailedTransaction = (
  bridgeContract: Contract,
  messageData: string,
-  timestamp: number,
-  getFailedTransactions: (args: GetFailedTransactionParams) => Promise<APITransaction[]>
+  startBlock: number,
+  getFailedTransactions: (args: GetTransactionParams) => Promise<APITransaction[]>
 ) => async (validatorData: BasicConfirmationParam): Promise<ConfirmationParam> => {
  const validatorCacheKey = `${CACHE_KEY_FAILED}${validatorData.validator}-${messageData}`
  const failedFromCache = validatorsCache.getData(validatorCacheKey)
@ -112,8 +103,8 @@ export const getValidatorFailedTransaction = (
    account: validatorData.validator,
    to: bridgeContract.options.address,
    messageData,
-    startTimestamp: timestamp,
-    endTimestamp: timestamp + ONE_DAY_TIMESTAMP
+    startBlock,
+    endBlock: homeBlockNumberProvider.get() || 0
  })
  const newStatus =
    failedTransactions.length > 0 ? VALIDATOR_CONFIRMATION_STATUS.FAILED : VALIDATOR_CONFIRMATION_STATUS.UNDEFINED
--- a/deployment/roles/common/tasks/dependencies.yml
+++ b/deployment/roles/common/tasks/dependencies.yml
@ -33,7 +33,7 @@
    mode: "0755"

 - name: Upgrade pip version
-  shell: pip3 install --upgrade pip
+  shell: pip3 install --upgrade pip==19.3.1

 - name: Install python docker library
  shell: pip3 install docker docker-compose setuptools
--- a/monitor/checkWorker3.js
+++ b/monitor/checkWorker3.js
@ -2,6 +2,7 @@ require('dotenv').config()
 const logger = require('./logger')('checkWorker3')
 const stuckTransfers = require('./stuckTransfers')
 const detectMediators = require('./detectMediators')
+const detectFailures = require('./detectFailures')
 const { writeFile, createDir } = require('./utils/file')
 const { web3Home } = require('./utils/web3')
 const { saveCache } = require('./utils/web3Cache')
@ -24,11 +25,19 @@ async function checkWorker3() {
      logger.debug('Done')
    } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) {
      createDir(`/responses/${MONITOR_BRIDGE_NAME}`)
+
      logger.debug('calling detectMediators()')
      const mediators = await detectMediators(bridgeMode)
      mediators.ok = true
      mediators.health = true
      writeFile(`/responses/${MONITOR_BRIDGE_NAME}/mediators.json`, mediators)
+
+      logger.debug('calling detectFailures()')
+      const failures = await detectFailures(bridgeMode)
+      failures.ok = true
+      failures.health = true
+      writeFile(`/responses/${MONITOR_BRIDGE_NAME}/failures.json`, failures)
+
      saveCache()
      logger.debug('Done')
    }
--- a/monitor/detectFailures.js
+++ b/monitor/detectFailures.js
@ -0,0 +1,86 @@
+require('dotenv').config()
+const logger = require('./logger')('alerts')
+const eventsInfo = require('./utils/events')
+const { normalizeAMBMessageEvent } = require('../commons')
+const { getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3')
+
+function normalize(events) {
+  const requests = {}
+  events.forEach(event => {
+    const request = normalizeAMBMessageEvent(event)
+    request.requestTx = event.transactionHash
+    requests[request.messageId] = request
+  })
+  return confirmation => {
+    const request = requests[confirmation.returnValues.messageId] || {}
+    return {
+      ...request,
+      status: false,
+      executionTx: confirmation.transactionHash,
+      executionBlockNumber: confirmation.blockNumber
+    }
+  }
+}
+
+async function main(mode) {
+  const {
+    homeToForeignRequests,
+    homeToForeignConfirmations,
+    foreignToHomeConfirmations,
+    foreignToHomeRequests
+  } = await eventsInfo(mode)
+  const hasFailed = event => !event.returnValues.status
+  const cmp = (a, b) => b.executionBlockNumber - a.executionBlockNumber
+  const failedForeignToHomeMessages = foreignToHomeConfirmations
+    .filter(hasFailed)
+    .map(normalize(foreignToHomeRequests))
+    .sort(cmp)
+  const failedHomeToForeignMessages = homeToForeignConfirmations
+    .filter(hasFailed)
+    .map(normalize(homeToForeignRequests))
+    .sort(cmp)
+
+  const homeBlockNumber = await getHomeBlockNumber()
+  const foreignBlockNumber = await getForeignBlockNumber()
+
+  const blockRanges = [1000, 10000, 100000, 1000000]
+  const rangeNames = [
+    `last${blockRanges[0]}blocks`,
+    ...blockRanges.slice(0, blockRanges.length - 1).map((n, i) => `last${n}to${blockRanges[i + 1]}blocks`),
+    `before${blockRanges[blockRanges.length - 1]}blocks`
+  ]
+
+  const countFailures = (failedMessages, lastBlockNumber) => {
+    const result = {}
+    rangeNames.forEach(name => {
+      result[name] = 0
+    })
+    failedMessages.forEach(message => {
+      const blockAge = lastBlockNumber - message.executionBlockNumber
+      let rangeIndex = blockRanges.findIndex(n => n > blockAge)
+      if (rangeIndex === -1) {
+        rangeIndex = blockRanges.length
+      }
+      result[rangeNames[rangeIndex]] += 1
+    })
+    return result
+  }
+
+  logger.debug('Done')
+
+  return {
+    homeToForeign: {
+      total: failedHomeToForeignMessages.length,
+      stats: countFailures(failedHomeToForeignMessages, foreignBlockNumber),
+      lastFailures: failedHomeToForeignMessages.slice(0, 5)
+    },
+    foreignToHome: {
+      total: failedForeignToHomeMessages.length,
+      stats: countFailures(failedForeignToHomeMessages, homeBlockNumber),
+      lastFailures: failedForeignToHomeMessages.slice(0, 5)
+    },
+    lastChecked: Math.floor(Date.now() / 1000)
+  }
+}
+
+module.exports = main
--- a/monitor/index.js
+++ b/monitor/index.js
@ -2,6 +2,7 @@ require('dotenv').config()
 const express = require('express')
 const cors = require('cors')
 const { readFile } = require('./utils/file')
+const { getPrometheusMetrics } = require('./prometheusMetrics')

 const app = express()
 const bridgeRouter = express.Router({ mergeParams: true })
@ -11,9 +12,10 @@ app.use(cors())
 app.get('/favicon.ico', (req, res) => res.sendStatus(204))
 app.use('/:bridgeName', bridgeRouter)

-bridgeRouter.get('/', async (req, res, next) => {
+bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', (req, res, next) => {
  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/getBalances.json`)
+    const { bridgeName, file } = req.params
+    const results = readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`)
    res.json(results)
  } catch (e) {
    // this will eventually be handled by your error handling middleware
@ -21,49 +23,10 @@ bridgeRouter.get('/', async (req, res, next) => {
  }
 })

-bridgeRouter.get('/validators', async (req, res, next) => {
+bridgeRouter.get('/metrics', (req, res, next) => {
  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/validators.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/eventsStats', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/eventsStats.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/alerts', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/alerts.json`)
-    res.json(results)
-  } catch (e) {
-    next(e)
-  }
-})
-
-bridgeRouter.get('/mediators', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/mediators.json`)
-    res.json(results)
-  } catch (e) {
-    // this will eventually be handled by your error handling middleware
-    next(e)
-  }
-})
-
-bridgeRouter.get('/stuckTransfers', async (req, res, next) => {
-  try {
-    const results = await readFile(`./responses/${req.params.bridgeName}/stuckTransfers.json`)
-    res.json(results)
+    const metrics = getPrometheusMetrics(req.params.bridgeName)
+    res.type('text').send(metrics)
  } catch (e) {
    next(e)
  }
--- a/monitor/prometheusMetrics.js
+++ b/monitor/prometheusMetrics.js
@ -0,0 +1,104 @@
+const { readFile } = require('./utils/file')
+
+const {
+  MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST,
+  MONITOR_HOME_TO_FOREIGN_BLOCK_LIST,
+  MONITOR_HOME_VALIDATORS_BALANCE_ENABLE,
+  MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE
+} = process.env
+
+function BridgeConf(type, validatorsBalanceEnable, alertTargetFunc, failureDirection) {
+  this.type = type
+  this.validatorsBalanceEnable = validatorsBalanceEnable
+  this.alertTargetFunc = alertTargetFunc
+  this.failureDirection = failureDirection
+}
+
+const BRIDGE_CONFS = [
+  new BridgeConf('home', MONITOR_HOME_VALIDATORS_BALANCE_ENABLE, 'executeAffirmations', 'homeToForeign'),
+  new BridgeConf('foreign', MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE, 'executeSignatures', 'foreignToHome')
+]
+
+function hasError(obj) {
+  return 'error' in obj
+}
+
+function getPrometheusMetrics(bridgeName) {
+  const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json`
+
+  const metrics = {}
+
+  // Balance metrics
+  const balancesFile = readFile(responsePath('getBalances'))
+
+  if (!hasError(balancesFile)) {
+    const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile
+    metrics.balances_home_value = homeBalances.totalSupply
+    metrics.balances_home_txs_deposit = homeBalances.deposits
+    metrics.balances_home_txs_withdrawal = homeBalances.withdrawals
+
+    metrics.balances_foreign_value = foreignBalances.erc20Balance
+    metrics.balances_foreign_txs_deposit = foreignBalances.deposits
+    metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals
+
+    metrics.balances_diff_value = commonBalances.balanceDiff
+    metrics.balances_diff_deposit = commonBalances.depositsDiff
+    metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff
+    if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) {
+      metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff
+      metrics.balances_unclaimed_value = commonBalances.unclaimedBalance
+    }
+  }
+
+  // Validator metrics
+  const validatorsFile = readFile(responsePath('validators'))
+
+  if (!hasError(validatorsFile)) {
+    for (const bridge of BRIDGE_CONFS) {
+      const allValidators = validatorsFile[bridge.type].validators
+      const validatorAddressesWithBalanceCheck =
+        typeof bridge.validatorsBalanceEnable === 'string'
+          ? bridge.validatorsBalanceEnable.split(' ')
+          : Object.keys(allValidators)
+
+      validatorAddressesWithBalanceCheck.forEach((addr, ind) => {
+        metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance
+      })
+    }
+  }
+
+  // Alert metrics
+  const alertsFile = readFile(responsePath('alerts'))
+
+  if (!hasError(alertsFile)) {
+    for (const bridge of BRIDGE_CONFS) {
+      Object.entries(alertsFile[bridge.alertTargetFunc].misbehavior).forEach(([period, val]) => {
+        metrics[`misbehavior_${bridge.type}_${period}`] = val
+      })
+    }
+  }
+
+  // Failure metrics
+  const failureFile = readFile(responsePath('failures'))
+
+  if (!hasError(failureFile)) {
+    for (const bridge of BRIDGE_CONFS) {
+      const dir = bridge.failureDirection
+      const failures = failureFile[dir]
+      metrics[`failures_${dir}_total`] = failures.total
+      Object.entries(failures.stats).forEach(([period, count]) => {
+        metrics[`failures_${dir}_${period}`] = count
+      })
+    }
+  }
+
+  // Pack metrcis into a plain text
+  return Object.entries(metrics).reduceRight(
+    // Prometheus supports `Nan` and possibly signed `Infinity`
+    // in case cast to `Number` fails
+    (acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`,
+    ''
+  )
+}
+
+module.exports = { getPrometheusMetrics }
--- a/monitor/utils/file.js
+++ b/monitor/utils/file.js
@ -1,9 +1,9 @@
 const fs = require('fs')
 const path = require('path')

-async function readFile(filePath) {
+function readFile(filePath) {
  try {
-    const content = await fs.readFileSync(filePath)
+    const content = fs.readFileSync(filePath)
    const json = JSON.parse(content)
    const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked
    return Object.assign({}, json, { timeDiff })
--- a/oracle/src/sender.js
+++ b/oracle/src/sender.js
@ -173,15 +173,18 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
          `Tx Failed for event Tx ${job.transactionReference}.`,
          e.message
        )
-        if (!e.message.toLowerCase().includes('transaction with the same hash was already imported')) {
-          if (isResend) {
+
+        const message = e.message.toLowerCase()
+        if (isResend || message.includes('transaction with the same hash was already imported')) {
          resendJobs.push(job)
        } else {
+          // if initial transaction sending has failed not due to the same hash error
+          // send it to the failed tx queue
+          // this will result in the sooner resend attempt than if using resendJobs
          failedTx.push(job)
        }
-        }

-        if (e.message.toLowerCase().includes('insufficient funds')) {
+        if (message.includes('insufficient funds')) {
          insufficientFunds = true
          const currentBalance = await web3Instance.eth.getBalance(ORACLE_VALIDATOR_ADDRESS)
          minimumBalance = gasLimit.multipliedBy(gasPrice)