diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 58fda638..bbb26d1e 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -42,6 +42,13 @@ ORACLE_HOME_TO_FOREIGN_BLOCK_LIST | Filename with a list of addresses, separated ORACLE_HOME_TO_FOREIGN_CHECK_SENDER | If set to `true`, instructs the oracle to do an extra check for transaction origin in the block/allowance list. `false` by default. | `true` / `false` ORACLE_ALWAYS_RELAY_SIGNATURES | If set to `true`, the oracle will always relay signatures even if it was not the last who finilized the signatures collecting process. The default is `false`. | `true` / `false` ORACLE_RPC_REQUEST_TIMEOUT | Timeout in milliseconds for a single RPC request. Default value is `ORACLE_*_RPC_POLLING_INTERVAL * 2`. | integer +ORACLE_HOME_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Home sender service. Defaults to 20 minutes. | integer +ORACLE_FOREIGN_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Foreign sender service. Defaults to 20 minutes. | integer +ORACLE_SHUTDOWN_SERVICE_URL | Optional external URL to some other service/monitor/configuration manager that controls the remote shutdown process. GET request should return `application/json` message with the following schema: `{ shutdown: true/false }`. | URL +ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL | Optional interval in milliseconds used to request the side RPC node or external shutdown service. Default is 120000. | integer +ORACLE_SIDE_RPC_URL | Optional HTTPS URL(s) for communication with the external shutdown service or side RPC nodes, used for shutdown manager activities. Several URLs can be specified, delimited by spaces. If the connection to one of these nodes is lost the next URL is used for connection. | URL(s) +ORACLE_SHUTDOWN_CONTRACT_ADDRESS | Optional contract address in the side chain accessible through `ORACLE_SIDE_RPC_URL`, where the method passed in `ORACLE_SHUTDOWN_CONTRACT_METHOD` is implemented. | `address` +ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain to identify the current shutdown status. Method should return boolean. Default value is `isShutdown()`. | `function signature` ## UI configuration diff --git a/alm/Dockerfile b/alm/Dockerfile index 66c11784..db2810c4 100644 --- a/alm/Dockerfile +++ b/alm/Dockerfile @@ -19,7 +19,7 @@ COPY --from=contracts /mono/contracts/build ./contracts/build COPY commons/package.json ./commons/ COPY alm/package.json ./alm/ COPY yarn.lock . -RUN NOYARNPOSTINSTALL=1 yarn install --frozen-lockfile --production +RUN NOYARNPOSTINSTALL=1 yarn install --frozen-lockfile COPY ./commons ./commons COPY ./alm ./alm diff --git a/alm/package.json b/alm/package.json index 41ede67e..7fed33df 100644 --- a/alm/package.json +++ b/alm/package.json @@ -58,6 +58,7 @@ ] }, "devDependencies": { - "eslint-plugin-prettier": "^3.1.3" + "eslint-plugin-prettier": "^3.1.3", + "node-fetch": "^2.6.1" } } diff --git a/alm/scripts/createSnapshots.js b/alm/scripts/createSnapshots.js index 604c72d4..b57e8953 100644 --- a/alm/scripts/createSnapshots.js +++ b/alm/scripts/createSnapshots.js @@ -3,6 +3,8 @@ const { BRIDGE_VALIDATORS_ABI, HOME_AMB_ABI } = require('commons') const path = require('path') require('dotenv').config() const Web3 = require('web3') +const fetch = require('node-fetch') +const { URL } = require('url') const fs = require('fs') @@ -10,7 +12,9 @@ const { COMMON_HOME_RPC_URL, COMMON_HOME_BRIDGE_ADDRESS, COMMON_FOREIGN_RPC_URL, - COMMON_FOREIGN_BRIDGE_ADDRESS + COMMON_FOREIGN_BRIDGE_ADDRESS, + ALM_FOREIGN_EXPLORER_API, + ALM_HOME_EXPLORER_API } = process.env const generateSnapshot = async (side, url, bridgeAddress) => { @@ -19,6 +23,31 @@ const generateSnapshot = async (side, url, bridgeAddress) => { const snapshot = {} const web3 = new Web3(new Web3.providers.HttpProvider(url)) + const api = side === 'home' ? ALM_HOME_EXPLORER_API : ALM_FOREIGN_EXPLORER_API + + const getPastEventsWithFallback = (contract, eventName, options) => + contract.getPastEvents(eventName, options).catch(async e => { + if (e.message.includes('exceed maximum block range')) { + const abi = contract.options.jsonInterface.find(abi => abi.type === 'event' && abi.name === eventName) + + const url = new URL(api) + url.searchParams.append('module', 'logs') + url.searchParams.append('action', 'getLogs') + url.searchParams.append('address', contract.options.address) + url.searchParams.append('fromBlock', options.fromBlock) + url.searchParams.append('toBlock', options.toBlock || 'latest') + url.searchParams.append('topic0', web3.eth.abi.encodeEventSignature(abi)) + + const logs = await fetch(url).then(res => res.json()) + + return logs.result.map(log => ({ + transactionHash: log.transactionHash, + blockNumber: parseInt(log.blockNumber.slice(2), 16), + returnValues: web3.eth.abi.decodeLog(abi.inputs, log.data, log.topics.slice(1)) + })) + } + throw e + }) const currentBlockNumber = await web3.eth.getBlockNumber() snapshot.snapshotBlockNumber = currentBlockNumber @@ -29,10 +58,14 @@ const generateSnapshot = async (side, url, bridgeAddress) => { const bridgeContract = new web3.eth.Contract(HOME_AMB_ABI, bridgeAddress) // Save RequiredBlockConfirmationChanged events - let requiredBlockConfirmationChangedEvents = await bridgeContract.getPastEvents('RequiredBlockConfirmationChanged', { - fromBlock: 0, - toBlock: currentBlockNumber - }) + let requiredBlockConfirmationChangedEvents = await getPastEventsWithFallback( + bridgeContract, + 'RequiredBlockConfirmationChanged', + { + fromBlock: 0, + toBlock: currentBlockNumber + } + ) // In case RequiredBlockConfirmationChanged was not emitted during initialization in early versions of AMB // manually generate an event for this. Example Sokol - Kovan bridge @@ -59,10 +92,14 @@ const generateSnapshot = async (side, url, bridgeAddress) => { const validatorContract = new web3.eth.Contract(BRIDGE_VALIDATORS_ABI, validatorAddress) // Save RequiredSignaturesChanged events - const RequiredSignaturesChangedEvents = await validatorContract.getPastEvents('RequiredSignaturesChanged', { - fromBlock: 0, - toBlock: currentBlockNumber - }) + const RequiredSignaturesChangedEvents = await getPastEventsWithFallback( + validatorContract, + 'RequiredSignaturesChanged', + { + fromBlock: 0, + toBlock: currentBlockNumber + } + ) snapshot.RequiredSignaturesChanged = RequiredSignaturesChangedEvents.map(e => ({ blockNumber: e.blockNumber, returnValues: { @@ -71,7 +108,7 @@ const generateSnapshot = async (side, url, bridgeAddress) => { })) // Save ValidatorAdded events - const validatorAddedEvents = await validatorContract.getPastEvents('ValidatorAdded', { + const validatorAddedEvents = await getPastEventsWithFallback(validatorContract, 'ValidatorAdded', { fromBlock: 0, toBlock: currentBlockNumber }) @@ -85,7 +122,7 @@ const generateSnapshot = async (side, url, bridgeAddress) => { })) // Save ValidatorRemoved events - const validatorRemovedEvents = await validatorContract.getPastEvents('ValidatorRemoved', { + const validatorRemovedEvents = await getPastEventsWithFallback(validatorContract, 'ValidatorRemoved', { fromBlock: 0, toBlock: currentBlockNumber }) diff --git a/alm/src/components/ManualExecutionButton.tsx b/alm/src/components/ManualExecutionButton.tsx index 5a7d1f90..3034013d 100644 --- a/alm/src/components/ManualExecutionButton.tsx +++ b/alm/src/components/ManualExecutionButton.tsx @@ -6,6 +6,7 @@ import { DOUBLE_EXECUTION_ATTEMPT_ERROR, EXECUTION_FAILED_ERROR, EXECUTION_OUT_OF_GAS_ERROR, + FOREIGN_EXPLORER_API, INCORRECT_CHAIN_ERROR, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' @@ -92,7 +93,13 @@ export const ManualExecutionButton = ({ }) .on('error', async (e: Error, receipt: TransactionReceipt) => { if (e.message.includes('Transaction has been reverted by the EVM')) { - const successExecutionData = await getSuccessExecutionData(bridge, 'RelayedMessage', library, messageId) + const successExecutionData = await getSuccessExecutionData( + bridge, + 'RelayedMessage', + library, + messageId, + FOREIGN_EXPLORER_API + ) if (successExecutionData) { setExecutionData(successExecutionData) setError(DOUBLE_EXECUTION_ATTEMPT_ERROR) diff --git a/alm/src/hooks/useBlockConfirmations.ts b/alm/src/hooks/useBlockConfirmations.ts index 470cfc23..b9427e68 100644 --- a/alm/src/hooks/useBlockConfirmations.ts +++ b/alm/src/hooks/useBlockConfirmations.ts @@ -4,6 +4,8 @@ import { useStateProvider } from '../state/StateProvider' import { Contract } from 'web3-eth-contract' import { getRequiredBlockConfirmations } from '../utils/contract' import { foreignSnapshotProvider, homeSnapshotProvider, SnapshotProvider } from '../services/SnapshotProvider' +import Web3 from 'web3' +import { FOREIGN_EXPLORER_API, HOME_EXPLORER_API } from '../config/constants' export interface UseBlockConfirmationsParams { fromHome: boolean @@ -19,9 +21,11 @@ export const useBlockConfirmations = ({ receipt, fromHome }: UseBlockConfirmatio contract: Contract, receipt: TransactionReceipt, setResult: Function, - snapshotProvider: SnapshotProvider + snapshotProvider: SnapshotProvider, + web3: Web3, + api: string ) => { - const result = await getRequiredBlockConfirmations(contract, receipt.blockNumber, snapshotProvider) + const result = await getRequiredBlockConfirmations(contract, receipt.blockNumber, snapshotProvider, web3, api) setResult(result) } @@ -29,10 +33,12 @@ export const useBlockConfirmations = ({ receipt, fromHome }: UseBlockConfirmatio () => { const bridgeContract = fromHome ? home.bridgeContract : foreign.bridgeContract const snapshotProvider = fromHome ? homeSnapshotProvider : foreignSnapshotProvider - if (!bridgeContract || !receipt) return - callRequireBlockConfirmations(bridgeContract, receipt, setBlockConfirmations, snapshotProvider) + const web3 = fromHome ? home.web3 : foreign.web3 + const api = fromHome ? HOME_EXPLORER_API : FOREIGN_EXPLORER_API + if (!bridgeContract || !receipt || !web3) return + callRequireBlockConfirmations(bridgeContract, receipt, setBlockConfirmations, snapshotProvider, web3, api) }, - [home.bridgeContract, foreign.bridgeContract, receipt, fromHome] + [home.bridgeContract, foreign.bridgeContract, receipt, fromHome, home.web3, foreign.web3] ) return { diff --git a/alm/src/hooks/useMessageConfirmations.ts b/alm/src/hooks/useMessageConfirmations.ts index a7a204fc..84efc2f9 100644 --- a/alm/src/hooks/useMessageConfirmations.ts +++ b/alm/src/hooks/useMessageConfirmations.ts @@ -11,9 +11,6 @@ import { VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' import { homeBlockNumberProvider, foreignBlockNumberProvider } from '../services/BlockNumberProvider' -import { checkSignaturesWaitingForBLocks } from '../utils/signatureWaitingForBlocks' -import { getCollectedSignaturesEvent } from '../utils/getCollectedSignaturesEvent' -import { checkWaitingBlocksForExecution } from '../utils/executionWaitingForBlocks' import { getConfirmationsForTx } from '../utils/getConfirmationsForTx' import { getFinalizationEvent } from '../utils/getFinalizationEvent' import { @@ -64,11 +61,11 @@ export const useMessageConfirmations = ({ blockConfirmations }: useMessageConfirmationsParams) => { const { home, foreign } = useStateProvider() - const [confirmations, setConfirmations] = useState([]) + const [confirmations, setConfirmations] = useState([]) const [status, setStatus] = useState(CONFIRMATIONS_STATUS.UNDEFINED) const [waitingBlocks, setWaitingBlocks] = useState(false) const [waitingBlocksResolved, setWaitingBlocksResolved] = useState(false) - const [signatureCollected, setSignatureCollected] = useState(false) + const [signatureCollected, setSignatureCollected] = useState(false) const [executionEventsFetched, setExecutionEventsFetched] = useState(false) const [collectedSignaturesEvent, setCollectedSignaturesEvent] = useState>(null) const [executionData, setExecutionData] = useState({ @@ -85,12 +82,10 @@ export const useMessageConfirmations = ({ const [pendingConfirmations, setPendingConfirmations] = useState(false) const [pendingExecution, setPendingExecution] = useState(false) - const existsConfirmation = (confirmationArray: ConfirmationParam[]) => { - const filteredList = confirmationArray.filter( + const existsConfirmation = (confirmationArray: ConfirmationParam[]) => + confirmationArray.some( c => c.status !== VALIDATOR_CONFIRMATION_STATUS.UNDEFINED && c.status !== VALIDATOR_CONFIRMATION_STATUS.WAITING ) - return filteredList.length > 0 - } // start watching blocks at the start useEffect( @@ -108,129 +103,154 @@ export const useMessageConfirmations = ({ () => { if (!receipt || !blockConfirmations || waitingBlocksResolved) return - const subscriptions: Array = [] - - const unsubscribe = () => { - subscriptions.forEach(s => { - clearTimeout(s) - }) - } + let timeoutId: number const blockProvider = fromHome ? homeBlockNumberProvider : foreignBlockNumberProvider const interval = fromHome ? HOME_RPC_POLLING_INTERVAL : FOREIGN_RPC_POLLING_INTERVAL const targetBlock = receipt.blockNumber + blockConfirmations + const validatorsWaiting = validatorList.map(validator => ({ + validator, + status: VALIDATOR_CONFIRMATION_STATUS.WAITING, + txHash: '', + timestamp: 0 + })) - checkSignaturesWaitingForBLocks( - targetBlock, - setWaitingBlocks, - setWaitingBlocksResolved, - validatorList, - setConfirmations, - blockProvider, - interval, - subscriptions - ) + const checkSignaturesWaitingForBLocks = () => { + const currentBlock = blockProvider.get() - return () => { - unsubscribe() + if (currentBlock && currentBlock >= targetBlock) { + setWaitingBlocksResolved(true) + setWaitingBlocks(false) + } else if (currentBlock) { + setWaitingBlocks(true) + setConfirmations(validatorsWaiting) + timeoutId = setTimeout(checkSignaturesWaitingForBLocks, interval) + } else { + timeoutId = setTimeout(checkSignaturesWaitingForBLocks, 500) + } } + + checkSignaturesWaitingForBLocks() + + return () => clearTimeout(timeoutId) }, - [ - blockConfirmations, - foreign.web3, - fromHome, - validatorList, - home.web3, - receipt, - setConfirmations, - waitingBlocksResolved - ] + [blockConfirmations, fromHome, receipt, validatorList, waitingBlocksResolved] ) // The collected signature event is only fetched once the signatures are collected on tx from home to foreign, to calculate if // the execution tx on the foreign network is waiting for block confirmations // This is executed if the message is in Home to Foreign direction only + const hasCollectedSignatures = !!signatureCollected // true or string[] useEffect( () => { - if (!fromHome || !receipt || !home.web3 || !signatureCollected) return + if (!fromHome || !receipt || !home.web3 || !home.bridgeContract || !hasCollectedSignatures) return - const subscriptions: Array = [] + let timeoutId: number + let isCancelled = false - const unsubscribe = () => { - subscriptions.forEach(s => { - clearTimeout(s) - }) + const messageHash = home.web3.utils.soliditySha3Raw(message.data) + const contract = home.bridgeContract + + const getCollectedSignaturesEvent = async (fromBlock: number, toBlock: number) => { + const currentBlock = homeBlockNumberProvider.get() + + if (currentBlock) { + // prevent errors if the toBlock parameter is bigger than the latest + const securedToBlock = toBlock >= currentBlock ? currentBlock : toBlock + const events = await contract.getPastEvents('CollectedSignatures', { + fromBlock, + toBlock: securedToBlock + }) + const event = events.find(e => e.returnValues.messageHash === messageHash) + if (event) { + setCollectedSignaturesEvent(event) + } else if (!isCancelled) { + timeoutId = setTimeout(() => getCollectedSignaturesEvent(securedToBlock, securedToBlock + BLOCK_RANGE), 500) + } + } else if (!isCancelled) { + timeoutId = setTimeout(() => getCollectedSignaturesEvent(fromBlock, toBlock), 500) + } } - const fromBlock = receipt.blockNumber - const toBlock = fromBlock + BLOCK_RANGE - const messageHash = home.web3.utils.soliditySha3Raw(message.data) - - getCollectedSignaturesEvent( - home.web3, - home.bridgeContract, - fromBlock, - toBlock, - messageHash, - setCollectedSignaturesEvent, - subscriptions - ) + getCollectedSignaturesEvent(receipt.blockNumber, receipt.blockNumber + BLOCK_RANGE) return () => { - unsubscribe() + clearTimeout(timeoutId) + isCancelled = true } }, - [fromHome, home.bridgeContract, home.web3, message.data, receipt, signatureCollected] + [fromHome, home.bridgeContract, home.web3, message.data, receipt, hasCollectedSignatures] ) // Check if the responsible validator is waiting for block confirmations to execute the message on foreign network // This is executed if the message is in Home to Foreign direction only useEffect( () => { - if (!fromHome || !home.web3 || !receipt || !collectedSignaturesEvent || !blockConfirmations) return + if (!fromHome || !home.web3 || !collectedSignaturesEvent || !blockConfirmations) return if (waitingBlocksForExecutionResolved) return - const subscriptions: Array = [] - - const unsubscribe = () => { - subscriptions.forEach(s => { - clearTimeout(s) - }) - } + let timeoutId: number const targetBlock = collectedSignaturesEvent.blockNumber + blockConfirmations - checkWaitingBlocksForExecution( - homeBlockNumberProvider, - HOME_RPC_POLLING_INTERVAL, - targetBlock, - collectedSignaturesEvent, - setWaitingBlocksForExecution, - setWaitingBlocksForExecutionResolved, - setExecutionData, - subscriptions - ) + const checkWaitingBlocksForExecution = () => { + const currentBlock = homeBlockNumberProvider.get() - return () => { - unsubscribe() + if (currentBlock && currentBlock >= targetBlock) { + const undefinedExecutionState = { + status: VALIDATOR_CONFIRMATION_STATUS.UNDEFINED, + validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay, + txHash: '', + timestamp: 0, + executionResult: false + } + setExecutionData( + (data: any) => + data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED || + data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING + ? undefinedExecutionState + : data + ) + setWaitingBlocksForExecutionResolved(true) + setWaitingBlocksForExecution(false) + } else if (currentBlock) { + setWaitingBlocksForExecution(true) + const waitingExecutionState = { + status: VALIDATOR_CONFIRMATION_STATUS.WAITING, + validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay, + txHash: '', + timestamp: 0, + executionResult: false + } + setExecutionData( + (data: any) => + data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED || + data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING + ? waitingExecutionState + : data + ) + timeoutId = setTimeout(() => checkWaitingBlocksForExecution(), HOME_RPC_POLLING_INTERVAL) + } else { + timeoutId = setTimeout(() => checkWaitingBlocksForExecution(), 500) + } } + + checkWaitingBlocksForExecution() + + return () => clearTimeout(timeoutId) }, - [collectedSignaturesEvent, fromHome, blockConfirmations, home.web3, receipt, waitingBlocksForExecutionResolved] + [collectedSignaturesEvent, fromHome, blockConfirmations, home.web3, waitingBlocksForExecutionResolved] ) // Checks if validators verified the message // To avoid making extra requests, this is only executed when validators finished waiting for blocks confirmations useEffect( () => { - if (!waitingBlocksResolved || !homeStartBlock || !requiredSignatures) return + if (!waitingBlocksResolved || !homeStartBlock || !requiredSignatures || !home.web3 || !home.bridgeContract) return + if (!validatorList || !validatorList.length) return - const subscriptions: Array = [] - - const unsubscribe = () => { - subscriptions.forEach(s => { - clearTimeout(s) - }) - } + let timeoutId: number + let isCancelled = false getConfirmationsForTx( message.data, @@ -241,8 +261,8 @@ export const useMessageConfirmations = ({ setConfirmations, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + id => (timeoutId = id), + () => isCancelled, homeStartBlock, getValidatorFailedTransactionsForMessage, setFailedConfirmations, @@ -252,7 +272,8 @@ export const useMessageConfirmations = ({ ) return () => { - unsubscribe() + clearTimeout(timeoutId) + isCancelled = true } }, [ @@ -263,8 +284,7 @@ export const useMessageConfirmations = ({ home.bridgeContract, requiredSignatures, waitingBlocksResolved, - homeStartBlock, - setConfirmations + homeStartBlock ] ) @@ -274,32 +294,23 @@ export const useMessageConfirmations = ({ useEffect( () => { if ((fromHome && !waitingBlocksForExecutionResolved) || (!fromHome && !waitingBlocksResolved)) return - const startBlock = fromHome ? foreignStartBlock : homeStartBlock - if (!startBlock) return - const subscriptions: Array = [] - - const unsubscribe = () => { - subscriptions.forEach(s => { - clearTimeout(s) - }) - } - - const contractEvent = fromHome ? 'RelayedMessage' : 'AffirmationCompleted' const bridgeContract = fromHome ? foreign.bridgeContract : home.bridgeContract - const providedWeb3 = fromHome ? foreign.web3 : home.web3 - const interval = fromHome ? FOREIGN_RPC_POLLING_INTERVAL : HOME_RPC_POLLING_INTERVAL + const web3 = fromHome ? foreign.web3 : home.web3 + const startBlock = fromHome ? foreignStartBlock : homeStartBlock + if (!startBlock || !bridgeContract || !web3) return + + let timeoutId: number + let isCancelled = false getFinalizationEvent( fromHome, bridgeContract, - contractEvent, - providedWeb3, + web3, setExecutionData, - waitingBlocksResolved, message, - interval, - subscriptions, + id => (timeoutId = id), + () => isCancelled, startBlock, collectedSignaturesEvent, getExecutionFailedTransactionForMessage, @@ -310,7 +321,8 @@ export const useMessageConfirmations = ({ ) return () => { - unsubscribe() + clearTimeout(timeoutId) + isCancelled = true } }, [ diff --git a/alm/src/hooks/useTransactionFinder.ts b/alm/src/hooks/useTransactionFinder.ts index 80611d62..a92f12e2 100644 --- a/alm/src/hooks/useTransactionFinder.ts +++ b/alm/src/hooks/useTransactionFinder.ts @@ -11,40 +11,23 @@ export const useTransactionFinder = ({ txHash, web3 }: { txHash: string; web3: M () => { if (!txHash || !web3) return - const subscriptions: number[] = [] + let timeoutId: number - const unsubscribe = () => { - subscriptions.forEach(s => { - clearTimeout(s) - }) - } - - const getReceipt = async ( - web3: Web3, - txHash: string, - setReceipt: Function, - setStatus: Function, - subscriptions: number[] - ) => { + const getReceipt = async () => { const txReceipt = await web3.eth.getTransactionReceipt(txHash) setReceipt(txReceipt) if (!txReceipt) { setStatus(TRANSACTION_STATUS.NOT_FOUND) - const timeoutId = setTimeout( - () => getReceipt(web3, txHash, setReceipt, setStatus, subscriptions), - HOME_RPC_POLLING_INTERVAL - ) - subscriptions.push(timeoutId) + timeoutId = setTimeout(getReceipt, HOME_RPC_POLLING_INTERVAL) } else { setStatus(TRANSACTION_STATUS.FOUND) } } - getReceipt(web3, txHash, setReceipt, setStatus, subscriptions) - return () => { - unsubscribe() - } + getReceipt() + + return () => clearTimeout(timeoutId) }, [txHash, web3] ) diff --git a/alm/src/hooks/useTransactionStatus.ts b/alm/src/hooks/useTransactionStatus.ts index 44763f54..9d03581a 100644 --- a/alm/src/hooks/useTransactionStatus.ts +++ b/alm/src/hooks/useTransactionStatus.ts @@ -31,19 +31,14 @@ export const useTransactionStatus = ({ useEffect( () => { - const subscriptions: Array = [] + if (!chainId || !txHash || !home.chainId || !foreign.chainId || !home.web3 || !foreign.web3) return + const isHome = chainId === home.chainId + const web3 = isHome ? home.web3 : foreign.web3 - const unsubscribe = () => { - subscriptions.forEach(s => { - clearTimeout(s) - }) - } + let timeoutId: number const getReceipt = async () => { - if (!chainId || !txHash || !home.chainId || !foreign.chainId || !home.web3 || !foreign.web3) return setLoading(true) - const isHome = chainId === home.chainId - const web3 = isHome ? home.web3 : foreign.web3 let txReceipt @@ -59,8 +54,7 @@ export const useTransactionStatus = ({ setStatus(TRANSACTION_STATUS.NOT_FOUND) setDescription(getTransactionStatusDescription(TRANSACTION_STATUS.NOT_FOUND)) setMessages([{ id: txHash, data: '' }]) - const timeoutId = setTimeout(() => getReceipt(), HOME_RPC_POLLING_INTERVAL) - subscriptions.push(timeoutId) + timeoutId = setTimeout(() => getReceipt(), HOME_RPC_POLLING_INTERVAL) } else { const blockNumber = txReceipt.blockNumber const block = await getBlock(web3, blockNumber) @@ -70,9 +64,9 @@ export const useTransactionStatus = ({ if (txReceipt.status) { let bridgeMessages: Array if (isHome) { - bridgeMessages = getHomeMessagesFromReceipt(txReceipt, home.web3, home.bridgeAddress) + bridgeMessages = getHomeMessagesFromReceipt(txReceipt, web3, home.bridgeAddress) } else { - bridgeMessages = getForeignMessagesFromReceipt(txReceipt, foreign.web3, foreign.bridgeAddress) + bridgeMessages = getForeignMessagesFromReceipt(txReceipt, web3, foreign.bridgeAddress) } if (bridgeMessages.length === 0) { @@ -98,14 +92,9 @@ export const useTransactionStatus = ({ setLoading(false) } - // unsubscribe from previous txHash - unsubscribe() - getReceipt() - return () => { - // unsubscribe when unmount component - unsubscribe() - } + + return () => clearTimeout(timeoutId) }, [ txHash, diff --git a/alm/src/hooks/useValidatorContract.ts b/alm/src/hooks/useValidatorContract.ts index 8557883f..cc232647 100644 --- a/alm/src/hooks/useValidatorContract.ts +++ b/alm/src/hooks/useValidatorContract.ts @@ -6,6 +6,7 @@ import { BRIDGE_VALIDATORS_ABI } from '../abis' import { useStateProvider } from '../state/StateProvider' import { TransactionReceipt } from 'web3-eth' import { foreignSnapshotProvider, homeSnapshotProvider, SnapshotProvider } from '../services/SnapshotProvider' +import { FOREIGN_EXPLORER_API, HOME_EXPLORER_API } from '../config/constants' export interface useValidatorContractParams { fromHome: boolean @@ -30,10 +31,12 @@ export const useValidatorContract = ({ receipt, fromHome }: useValidatorContract contract: Maybe, receipt: TransactionReceipt, setResult: Function, - snapshotProvider: SnapshotProvider + snapshotProvider: SnapshotProvider, + web3: Web3, + api: string ) => { if (!contract) return - const result = await getRequiredSignatures(contract, receipt.blockNumber, snapshotProvider) + const result = await getRequiredSignatures(contract, receipt.blockNumber, snapshotProvider, web3, api) setResult(result) } @@ -41,32 +44,35 @@ export const useValidatorContract = ({ receipt, fromHome }: useValidatorContract contract: Maybe, receipt: TransactionReceipt, setResult: Function, - snapshotProvider: SnapshotProvider + snapshotProvider: SnapshotProvider, + web3: Web3, + api: string ) => { if (!contract) return - const result = await getValidatorList(contract, receipt.blockNumber, snapshotProvider) + const result = await getValidatorList(contract, receipt.blockNumber, snapshotProvider, web3, api) setResult(result) } + const web3 = fromHome ? home.web3 : foreign.web3 + const api = fromHome ? HOME_EXPLORER_API : FOREIGN_EXPLORER_API + const bridgeContract = fromHome ? home.bridgeContract : foreign.bridgeContract + const snapshotProvider = fromHome ? homeSnapshotProvider : foreignSnapshotProvider + useEffect( () => { - const web3 = fromHome ? home.web3 : foreign.web3 - const bridgeContract = fromHome ? home.bridgeContract : foreign.bridgeContract - if (!web3 || !bridgeContract) return callValidatorContract(bridgeContract, web3, setValidatorContract) }, - [home.web3, foreign.web3, home.bridgeContract, foreign.bridgeContract, fromHome] + [web3, bridgeContract] ) useEffect( () => { - if (!receipt) return - const snapshotProvider = fromHome ? homeSnapshotProvider : foreignSnapshotProvider - callRequiredSignatures(validatorContract, receipt, setRequiredSignatures, snapshotProvider) - callValidatorList(validatorContract, receipt, setValidatorList, snapshotProvider) + if (!web3 || !receipt) return + callRequiredSignatures(validatorContract, receipt, setRequiredSignatures, snapshotProvider, web3, api) + callValidatorList(validatorContract, receipt, setValidatorList, snapshotProvider, web3, api) }, - [validatorContract, receipt, fromHome] + [validatorContract, receipt, web3, snapshotProvider, api] ) return { diff --git a/alm/src/utils/__tests__/contracts.test.ts b/alm/src/utils/__tests__/contracts.test.ts index 7296658d..94bc02b6 100644 --- a/alm/src/utils/__tests__/contracts.test.ts +++ b/alm/src/utils/__tests__/contracts.test.ts @@ -16,7 +16,7 @@ describe('getRequiredBlockConfirmations', () => { test('Should call requiredBlockConfirmations method if no events present', async () => { const contract = ({ - getPastEvents: () => { + getPastEvents: async () => { return [] }, methods: methodsBuilder('1') @@ -37,7 +37,7 @@ describe('getRequiredBlockConfirmations', () => { }) test('Should not call to get events if block number was included in the snapshot', async () => { const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => []), + getPastEvents: jest.fn().mockImplementation(async () => []), methods: methodsBuilder('3') } as unknown) as Contract @@ -64,7 +64,7 @@ describe('getRequiredBlockConfirmations', () => { }) test('Should call to get events if block number was not included in the snapshot', async () => { const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => [ + getPastEvents: jest.fn().mockImplementation(async () => [ { blockNumber: 9, returnValues: { @@ -102,7 +102,7 @@ describe('getRequiredBlockConfirmations', () => { }) test('Should use the most updated event', async () => { const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => [ + getPastEvents: jest.fn().mockImplementation(async () => [ { blockNumber: 9, returnValues: { @@ -141,7 +141,7 @@ describe('getRequiredBlockConfirmations', () => { describe('getRequiredSignatures', () => { test('Should not call to get events if block number was included in the snapshot', async () => { const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => []) + getPastEvents: jest.fn().mockImplementation(async () => []) } as unknown) as Contract const snapshotProvider = ({ @@ -173,7 +173,7 @@ describe('getRequiredSignatures', () => { }) test('Should call to get events if block number is higher than the snapshot block number', async () => { const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => [ + getPastEvents: jest.fn().mockImplementation(async () => [ { blockNumber: 15, returnValues: { @@ -216,7 +216,7 @@ describe('getRequiredSignatures', () => { }) test('Should use the most updated event before the block number', async () => { const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => [ + getPastEvents: jest.fn().mockImplementation(async () => [ { blockNumber: 15, returnValues: { @@ -270,7 +270,7 @@ describe('getValidatorList', () => { test('Should return the current validator list if no events found', async () => { const currentValidators = [validator1, validator2, validator3] const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => []), + getPastEvents: jest.fn().mockImplementation(async () => []), methods: methodsBuilder(currentValidators) } as unknown) as Contract @@ -301,7 +301,7 @@ describe('getValidatorList', () => { test('If validator was added later from snapshot it should not include it', async () => { const currentValidators = [validator1, validator2, validator3] const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => []), + getPastEvents: jest.fn().mockImplementation(async () => []), methods: methodsBuilder(currentValidators) } as unknown) as Contract @@ -340,7 +340,7 @@ describe('getValidatorList', () => { test('If validator was added later from chain it should not include it', async () => { const currentValidators = [validator1, validator2, validator3] const contract = ({ - getPastEvents: jest.fn().mockImplementation(event => { + getPastEvents: jest.fn().mockImplementation(async event => { if (event === 'ValidatorAdded') { return [ { @@ -385,7 +385,7 @@ describe('getValidatorList', () => { test('If validator was removed later from snapshot it should include it', async () => { const currentValidators = [validator1, validator2] const contract = ({ - getPastEvents: jest.fn().mockImplementation(() => []), + getPastEvents: jest.fn().mockImplementation(async () => []), methods: methodsBuilder(currentValidators) } as unknown) as Contract @@ -424,7 +424,7 @@ describe('getValidatorList', () => { test('If validator was removed later from chain it should include it', async () => { const currentValidators = [validator1, validator2] const contract = ({ - getPastEvents: jest.fn().mockImplementation(event => { + getPastEvents: jest.fn().mockImplementation(async event => { if (event === 'ValidatorRemoved') { return [ { diff --git a/alm/src/utils/__tests__/getConfirmationsForTx.test.ts b/alm/src/utils/__tests__/getConfirmationsForTx.test.ts index 3d21a46e..689bb8d1 100644 --- a/alm/src/utils/__tests__/getConfirmationsForTx.test.ts +++ b/alm/src/utils/__tests__/getConfirmationsForTx.test.ts @@ -34,7 +34,7 @@ const bridgeContract = { } } as Contract const requiredSignatures = 2 -const waitingBlocksResolved = true +const isCancelled = () => false let subscriptions: Array = [] const timestamp = 1594045859 const getFailedTransactions = (): Promise => Promise.resolve([]) @@ -94,8 +94,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, @@ -177,8 +177,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, @@ -241,8 +241,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, @@ -340,8 +340,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, @@ -453,8 +453,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, @@ -557,8 +557,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, @@ -678,8 +678,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, @@ -746,8 +746,8 @@ describe('getConfirmationsForTx', () => { setResult, requiredSignatures, setSignatureCollected, - waitingBlocksResolved, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, getFailedTransactions, setFailedConfirmations, diff --git a/alm/src/utils/__tests__/getFinalizationEvent.test.ts b/alm/src/utils/__tests__/getFinalizationEvent.test.ts index bbce1486..ae9c4100 100644 --- a/alm/src/utils/__tests__/getFinalizationEvent.test.ts +++ b/alm/src/utils/__tests__/getFinalizationEvent.test.ts @@ -4,7 +4,6 @@ import Web3 from 'web3' import { getFinalizationEvent } from '../getFinalizationEvent' import { VALIDATOR_CONFIRMATION_STATUS } from '../../config/constants' -const eventName = 'RelayedMessage' const timestamp = 1594045859 const validator1 = '0x45b96809336A8b714BFbdAB3E4B5e0fe5d839908' const txHash = '0xdab36c9210e7e45fb82af10ffe4960461e41661dce0c9cd36b2843adaa1df156' @@ -20,12 +19,11 @@ const web3 = ({ toChecksumAddress: (a: string) => a } } as unknown) as Web3 -const waitingBlocksResolved = true const message = { id: '0x123', data: '0x123456789' } -const interval = 10000 +const isCancelled = () => false let subscriptions: Array = [] const event = { @@ -50,7 +48,7 @@ beforeEach(() => { describe('getFinalizationEvent', () => { test('should get finalization event and not try to get failed or pending transactions', async () => { const contract = ({ - getPastEvents: () => { + getPastEvents: async () => { return [event] } } as unknown) as Contract @@ -66,13 +64,11 @@ describe('getFinalizationEvent', () => { await getFinalizationEvent( true, contract, - eventName, web3, setResult, - waitingBlocksResolved, message, - interval, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, collectedSignaturesEvent, getFailedExecution, @@ -102,7 +98,7 @@ describe('getFinalizationEvent', () => { }) test('should retry to get finalization event and not try to get failed or pending transactions if foreign to home transaction', async () => { const contract = ({ - getPastEvents: () => { + getPastEvents: async () => { return [] } } as unknown) as Contract @@ -118,13 +114,11 @@ describe('getFinalizationEvent', () => { await getFinalizationEvent( true, contract, - eventName, web3, setResult, - waitingBlocksResolved, message, - interval, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, collectedSignaturesEvent, getFailedExecution, @@ -147,7 +141,7 @@ describe('getFinalizationEvent', () => { }) test('should retry to get finalization event and try to get failed and pending transactions if home to foreign transaction', async () => { const contract = ({ - getPastEvents: () => { + getPastEvents: async () => { return [] }, options: { @@ -170,13 +164,11 @@ describe('getFinalizationEvent', () => { await getFinalizationEvent( true, contract, - eventName, web3, setResult, - waitingBlocksResolved, message, - interval, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, collectedSignaturesEvent, getFailedExecution, @@ -199,7 +191,7 @@ describe('getFinalizationEvent', () => { }) test('should retry to get finalization event and not to try to get failed transaction if pending transactions found if home to foreign transaction', async () => { const contract = ({ - getPastEvents: () => { + getPastEvents: async () => { return [] }, options: { @@ -222,13 +214,11 @@ describe('getFinalizationEvent', () => { await getFinalizationEvent( true, contract, - eventName, web3, setResult, - waitingBlocksResolved, message, - interval, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, collectedSignaturesEvent, getFailedExecution, @@ -258,7 +248,7 @@ describe('getFinalizationEvent', () => { }) test('should retry to get finalization event even if failed transaction found if home to foreign transaction', async () => { const contract = ({ - getPastEvents: () => { + getPastEvents: async () => { return [] }, options: { @@ -281,13 +271,11 @@ describe('getFinalizationEvent', () => { await getFinalizationEvent( true, contract, - eventName, web3, setResult, - waitingBlocksResolved, message, - interval, - subscriptions, + subscriptions.push.bind(subscriptions), + isCancelled, timestamp, collectedSignaturesEvent, getFailedExecution, diff --git a/alm/src/utils/contract.ts b/alm/src/utils/contract.ts index f5578c01..7e9824e5 100644 --- a/alm/src/utils/contract.ts +++ b/alm/src/utils/contract.ts @@ -1,18 +1,33 @@ import { Contract } from 'web3-eth-contract' import { EventData } from 'web3-eth-contract' import { SnapshotProvider } from '../services/SnapshotProvider' +import { getLogs } from './explorer' +import Web3 from 'web3' + +const getPastEventsWithFallback = ( + api: string, + web3: Web3 | null, + contract: Contract, + eventName: string, + options: any +) => + contract + .getPastEvents(eventName, options) + .catch(() => (api && web3 ? getLogs(api, web3, contract, eventName, options) : [])) export const getRequiredBlockConfirmations = async ( contract: Contract, blockNumber: number, - snapshotProvider: SnapshotProvider + snapshotProvider: SnapshotProvider, + web3: Web3 | null = null, + api: string = '' ) => { const eventsFromSnapshot = snapshotProvider.requiredBlockConfirmationEvents(blockNumber) const snapshotBlockNumber = snapshotProvider.snapshotBlockNumber() let contractEvents: EventData[] = [] if (blockNumber > snapshotBlockNumber) { - contractEvents = await contract.getPastEvents('RequiredBlockConfirmationChanged', { + contractEvents = await getPastEventsWithFallback(api, web3, contract, 'RequiredBlockConfirmationChanged', { fromBlock: snapshotBlockNumber + 1, toBlock: blockNumber }) @@ -38,14 +53,16 @@ export const getValidatorAddress = (contract: Contract) => contract.methods.vali export const getRequiredSignatures = async ( contract: Contract, blockNumber: number, - snapshotProvider: SnapshotProvider + snapshotProvider: SnapshotProvider, + web3: Web3 | null = null, + api: string = '' ) => { const eventsFromSnapshot = snapshotProvider.requiredSignaturesEvents(blockNumber) const snapshotBlockNumber = snapshotProvider.snapshotBlockNumber() let contractEvents: EventData[] = [] if (blockNumber > snapshotBlockNumber) { - contractEvents = await contract.getPastEvents('RequiredSignaturesChanged', { + contractEvents = await getPastEventsWithFallback(api, web3, contract, 'RequiredSignaturesChanged', { fromBlock: snapshotBlockNumber + 1, toBlock: blockNumber }) @@ -59,7 +76,13 @@ export const getRequiredSignatures = async ( return parseInt(requiredSignatures) } -export const getValidatorList = async (contract: Contract, blockNumber: number, snapshotProvider: SnapshotProvider) => { +export const getValidatorList = async ( + contract: Contract, + blockNumber: number, + snapshotProvider: SnapshotProvider, + web3: Web3 | null = null, + api: string = '' +) => { const addedEventsFromSnapshot = snapshotProvider.validatorAddedEvents(blockNumber) const removedEventsFromSnapshot = snapshotProvider.validatorRemovedEvents(blockNumber) const snapshotBlockNumber = snapshotProvider.snapshotBlockNumber() @@ -67,10 +90,10 @@ export const getValidatorList = async (contract: Contract, blockNumber: number, const fromBlock = snapshotBlockNumber > blockNumber ? snapshotBlockNumber + 1 : blockNumber const [currentList, added, removed] = await Promise.all([ contract.methods.validatorList().call(), - contract.getPastEvents('ValidatorAdded', { + getPastEventsWithFallback(api, web3, contract, 'ValidatorAdded', { fromBlock }), - contract.getPastEvents('ValidatorRemoved', { + getPastEventsWithFallback(api, web3, contract, 'ValidatorRemoved', { fromBlock }) ]) diff --git a/alm/src/utils/executionWaitingForBlocks.ts b/alm/src/utils/executionWaitingForBlocks.ts deleted file mode 100644 index a221f311..00000000 --- a/alm/src/utils/executionWaitingForBlocks.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { BlockNumberProvider } from '../services/BlockNumberProvider' -import { VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' -import { EventData } from 'web3-eth-contract' - -export const checkWaitingBlocksForExecution = async ( - blockProvider: BlockNumberProvider, - interval: number, - targetBlock: number, - collectedSignaturesEvent: EventData, - setWaitingBlocksForExecution: Function, - setWaitingBlocksForExecutionResolved: Function, - setExecutionData: Function, - subscriptions: number[] -) => { - const currentBlock = blockProvider.get() - - if (currentBlock && currentBlock >= targetBlock) { - const undefinedExecutionState = { - status: VALIDATOR_CONFIRMATION_STATUS.UNDEFINED, - validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay, - txHash: '', - timestamp: 0, - executionResult: false - } - setExecutionData( - (data: any) => - data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED || data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING - ? undefinedExecutionState - : data - ) - setWaitingBlocksForExecutionResolved(true) - setWaitingBlocksForExecution(false) - } else { - let nextInterval = interval - if (!currentBlock) { - nextInterval = 500 - } else { - setWaitingBlocksForExecution(true) - const waitingExecutionState = { - status: VALIDATOR_CONFIRMATION_STATUS.WAITING, - validator: collectedSignaturesEvent.returnValues.authorityResponsibleForRelay, - txHash: '', - timestamp: 0, - executionResult: false - } - setExecutionData( - (data: any) => - data.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED || - data.status === VALIDATOR_CONFIRMATION_STATUS.WAITING - ? waitingExecutionState - : data - ) - } - const timeoutId = setTimeout( - () => - checkWaitingBlocksForExecution( - blockProvider, - interval, - targetBlock, - collectedSignaturesEvent, - setWaitingBlocksForExecution, - setWaitingBlocksForExecutionResolved, - setExecutionData, - subscriptions - ), - nextInterval - ) - subscriptions.push(timeoutId) - } -} diff --git a/alm/src/utils/explorer.ts b/alm/src/utils/explorer.ts index 07293efc..6b1f53bf 100644 --- a/alm/src/utils/explorer.ts +++ b/alm/src/utils/explorer.ts @@ -7,6 +7,9 @@ import { MAX_TX_SEARCH_BLOCK_RANGE, SUBMIT_SIGNATURE_HASH } from '../config/constants' +import { AbiItem } from 'web3-utils' +import Web3 from 'web3' +import { Contract } from 'web3-eth-contract' export interface APITransaction { timeStamp: string @@ -47,10 +50,15 @@ export interface GetTransactionParams extends GetPendingTransactionParams { } export const fetchAccountTransactions = async ({ account, startBlock, endBlock, api }: AccountTransactionsParams) => { - const params = `module=account&action=txlist&address=${account}&filterby=from&startblock=${startBlock}&endblock=${endBlock}` - const url = api.includes('blockscout') ? `${api}?${params}` : `${api}&${params}` + const url = new URL(api) + url.searchParams.append('module', 'account') + url.searchParams.append('action', 'txlist') + url.searchParams.append('address', account) + url.searchParams.append('filterby', 'from') + url.searchParams.append('startblock', startBlock.toString()) + url.searchParams.append('endblock', endBlock.toString()) - const result = await fetch(url).then(res => res.json()) + const result = await fetch(url.toString()).then(res => res.json()) if (result.message === 'No transactions found') { return [] @@ -66,10 +74,13 @@ export const fetchPendingTransactions = async ({ if (!api.includes('blockscout')) { return [] } - const url = `${api}?module=account&action=pendingtxlist&address=${account}` + const url = new URL(api) + url.searchParams.append('module', 'account') + url.searchParams.append('action', 'pendingtxlist') + url.searchParams.append('address', account) try { - const result = await fetch(url).then(res => res.json()) + const result = await fetch(url.toString()).then(res => res.json()) if (result.status === '0') { return [] } @@ -85,9 +96,13 @@ export const getClosestBlockByTimestamp = async (api: string, timestamp: number) throw new Error('Blockscout does not support getblocknobytime') } - const url = `${api}&module=block&action=getblocknobytime×tamp=${timestamp}&closest=before` + const url = new URL(api) + url.searchParams.append('module', 'block') + url.searchParams.append('action', 'getblocknobytime') + url.searchParams.append('timestamp', timestamp.toString()) + url.searchParams.append('closest', 'before') - const blockNumber = await fetch(url).then(res => res.json()) + const blockNumber = await fetch(url.toString()).then(res => res.json()) return parseInt(blockNumber.result) } @@ -144,6 +159,41 @@ export const getAccountTransactions = async ({ return transactionsCache[key].transactions } +export const getLogs = async ( + api: string, + web3: Web3, + contract: Contract, + event: string, + options: { fromBlock: number; toBlock: number | 'latest'; topics: (string | null)[] } +) => { + const abi = contract.options.jsonInterface.find((abi: AbiItem) => abi.type === 'event' && abi.name === event)! + + const url = new URL(api) + url.searchParams.append('module', 'logs') + url.searchParams.append('action', 'getLogs') + url.searchParams.append('address', contract.options.address) + url.searchParams.append('fromBlock', options.fromBlock.toString()) + url.searchParams.append('toBlock', (options.toBlock || 'latest').toString()) + + const topics = [web3.eth.abi.encodeEventSignature(abi), ...(options.topics || [])] + for (let i = 0; i < topics.length; i++) { + if (topics[i] !== null) { + url.searchParams.append(`topic${i}`, topics[i] as string) + for (let j = 0; j < i; j++) { + url.searchParams.append(`topic${j}_${i}_opr`, 'and') + } + } + } + + const logs = await fetch(url.toString()).then(res => res.json()) + + return logs.result.map((log: any) => ({ + transactionHash: log.transactionHash, + blockNumber: parseInt(log.blockNumber.slice(2), 16), + returnValues: web3.eth.abi.decodeLog(abi.inputs!, log.data, log.topics.slice(1)) + })) +} + const filterReceiver = (to: string) => (tx: APITransaction) => tx.to.toLowerCase() === to.toLowerCase() export const getFailedTransactions = async ( diff --git a/alm/src/utils/getCollectedSignaturesEvent.ts b/alm/src/utils/getCollectedSignaturesEvent.ts deleted file mode 100644 index 003a6095..00000000 --- a/alm/src/utils/getCollectedSignaturesEvent.ts +++ /dev/null @@ -1,52 +0,0 @@ -import Web3 from 'web3' -import { Contract, EventData } from 'web3-eth-contract' -import { homeBlockNumberProvider } from '../services/BlockNumberProvider' -import { BLOCK_RANGE } from '../config/constants' - -export const getCollectedSignaturesEvent = async ( - web3: Maybe, - contract: Maybe, - fromBlock: number, - toBlock: number, - messageHash: string, - setCollectedSignaturesEvent: Function, - subscriptions: number[] -) => { - if (!web3 || !contract) return - const currentBlock = homeBlockNumberProvider.get() - - let events: EventData[] = [] - let securedToBlock = toBlock - if (currentBlock) { - // prevent errors if the toBlock parameter is bigger than the latest - securedToBlock = toBlock >= currentBlock ? currentBlock : toBlock - events = await contract.getPastEvents('CollectedSignatures', { - fromBlock, - toBlock: securedToBlock - }) - } - - const filteredEvents = events.filter(e => e.returnValues.messageHash === messageHash) - - if (filteredEvents.length) { - const event = filteredEvents[0] - setCollectedSignaturesEvent(event) - } else { - const newFromBlock = currentBlock ? securedToBlock : fromBlock - const newToBlock = currentBlock ? toBlock + BLOCK_RANGE : toBlock - const timeoutId = setTimeout( - () => - getCollectedSignaturesEvent( - web3, - contract, - newFromBlock, - newToBlock, - messageHash, - setCollectedSignaturesEvent, - subscriptions - ), - 500 - ) - subscriptions.push(timeoutId) - } -} diff --git a/alm/src/utils/getConfirmationsForTx.ts b/alm/src/utils/getConfirmationsForTx.ts index 015793d0..54374efb 100644 --- a/alm/src/utils/getConfirmationsForTx.ts +++ b/alm/src/utils/getConfirmationsForTx.ts @@ -29,15 +29,15 @@ const mergeConfirmations = (oldConfirmations: BasicConfirmationParam[], newConfi export const getConfirmationsForTx = async ( messageData: string, - web3: Maybe, + web3: Web3, validatorList: string[], - bridgeContract: Maybe, + bridgeContract: Contract, fromHome: boolean, setResult: Function, requiredSignatures: number, setSignatureCollected: Function, - waitingBlocksResolved: boolean, - subscriptions: number[], + setTimeoutId: (timeoutId: number) => void, + isCancelled: () => boolean, startBlock: number, getFailedTransactions: (args: GetTransactionParams) => Promise, setFailedConfirmations: Function, @@ -45,8 +45,6 @@ export const getConfirmationsForTx = async ( setPendingConfirmations: Function, getSuccessTransactions: (args: GetTransactionParams) => Promise ) => { - if (!web3 || !validatorList || !validatorList.length || !bridgeContract || !waitingBlocksResolved) return - const confirmationContractMethod = fromHome ? getMessagesSigned : getAffirmationsSigned const hashMsg = web3.utils.soliditySha3Raw(messageData) @@ -144,28 +142,30 @@ export const getConfirmationsForTx = async ( (!hasEnoughSignatures && missingConfirmations.length > 0) || successConfirmationWithTxFound.length < successConfirmationWithData.length ) { - const timeoutId = setTimeout( - () => - getConfirmationsForTx( - messageData, - web3, - validatorList, - bridgeContract, - fromHome, - setResult, - requiredSignatures, - setSignatureCollected, - waitingBlocksResolved, - subscriptions, - startBlock, - getFailedTransactions, - setFailedConfirmations, - getPendingTransactions, - setPendingConfirmations, - getSuccessTransactions - ), - HOME_RPC_POLLING_INTERVAL - ) - subscriptions.push(timeoutId) + if (!isCancelled()) { + const timeoutId = setTimeout( + () => + getConfirmationsForTx( + messageData, + web3, + validatorList, + bridgeContract, + fromHome, + setResult, + requiredSignatures, + setSignatureCollected, + setTimeoutId, + isCancelled, + startBlock, + getFailedTransactions, + setFailedConfirmations, + getPendingTransactions, + setPendingConfirmations, + getSuccessTransactions + ), + HOME_RPC_POLLING_INTERVAL + ) + setTimeoutId(timeoutId) + } } } diff --git a/alm/src/utils/getFinalizationEvent.ts b/alm/src/utils/getFinalizationEvent.ts index 3406007e..81c0a6d7 100644 --- a/alm/src/utils/getFinalizationEvent.ts +++ b/alm/src/utils/getFinalizationEvent.ts @@ -1,16 +1,47 @@ import { Contract, EventData } from 'web3-eth-contract' import Web3 from 'web3' -import { CACHE_KEY_EXECUTION_FAILED, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' +import { + CACHE_KEY_EXECUTION_FAILED, + FOREIGN_EXPLORER_API, + FOREIGN_RPC_POLLING_INTERVAL, + HOME_EXPLORER_API, + HOME_RPC_POLLING_INTERVAL, + VALIDATOR_CONFIRMATION_STATUS +} from '../config/constants' import { ExecutionData } from '../hooks/useMessageConfirmations' -import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer' +import { + APIPendingTransaction, + APITransaction, + GetTransactionParams, + GetPendingTransactionParams, + getLogs +} from './explorer' import { getBlock, MessageObject } from './web3' import validatorsCache from '../services/ValidatorsCache' import { foreignBlockNumberProvider, homeBlockNumberProvider } from '../services/BlockNumberProvider' -export const getSuccessExecutionData = async (contract: Contract, eventName: string, web3: Web3, messageId: string) => { +const getPastEventsWithFallback = (api: string, web3: Web3, contract: Contract, eventName: string, options: any) => + contract.getPastEvents(eventName, options).catch( + () => + api + ? getLogs(api, web3, contract, eventName, { + fromBlock: options.fromBlock, + toBlock: options.toBlock, + topics: [null, null, options.filter.messageId] + }) + : [] + ) + +export const getSuccessExecutionData = async ( + contract: Contract, + eventName: string, + web3: Web3, + messageId: string, + api: string = '' +) => { // Since it filters by the message id, only one event will be fetched // so there is no need to limit the range of the block to reduce the network traffic - const events: EventData[] = await contract.getPastEvents(eventName, { + const events: EventData[] = await getPastEventsWithFallback(api, web3, contract, eventName, { fromBlock: 0, toBlock: 'latest', filter: { @@ -40,14 +71,12 @@ export const getSuccessExecutionData = async (contract: Contract, eventName: str export const getFinalizationEvent = async ( fromHome: boolean, - contract: Maybe, - eventName: string, - web3: Maybe, + contract: Contract, + web3: Web3, setResult: React.Dispatch>, - waitingBlocksResolved: boolean, message: MessageObject, - interval: number, - subscriptions: number[], + setTimeoutId: (timeoutId: number) => void, + isCancelled: () => boolean, startBlock: number, collectedSignaturesEvent: Maybe, getFailedExecution: (args: GetTransactionParams) => Promise, @@ -56,8 +85,11 @@ export const getFinalizationEvent = async ( setPendingExecution: Function, setExecutionEventsFetched: Function ) => { - if (!contract || !web3 || !waitingBlocksResolved) return - const successExecutionData = await getSuccessExecutionData(contract, eventName, web3, message.id) + const eventName = fromHome ? 'RelayedMessage' : 'AffirmationCompleted' + const api = fromHome ? FOREIGN_EXPLORER_API : HOME_EXPLORER_API + + const successExecutionData = await getSuccessExecutionData(contract, eventName, web3, message.id, api) + if (successExecutionData) { setResult(successExecutionData) } else { @@ -120,28 +152,28 @@ export const getFinalizationEvent = async ( } } - const timeoutId = setTimeout( - () => - getFinalizationEvent( - fromHome, - contract, - eventName, - web3, - setResult, - waitingBlocksResolved, - message, - interval, - subscriptions, - startBlock, - collectedSignaturesEvent, - getFailedExecution, - setFailedExecution, - getPendingExecution, - setPendingExecution, - setExecutionEventsFetched - ), - interval - ) - subscriptions.push(timeoutId) + if (!isCancelled()) { + const timeoutId = setTimeout( + () => + getFinalizationEvent( + fromHome, + contract, + web3, + setResult, + message, + setTimeoutId, + isCancelled, + startBlock, + collectedSignaturesEvent, + getFailedExecution, + setFailedExecution, + getPendingExecution, + setPendingExecution, + setExecutionEventsFetched + ), + fromHome ? FOREIGN_RPC_POLLING_INTERVAL : HOME_RPC_POLLING_INTERVAL + ) + setTimeoutId(timeoutId) + } } } diff --git a/alm/src/utils/signatureWaitingForBlocks.ts b/alm/src/utils/signatureWaitingForBlocks.ts deleted file mode 100644 index 51db374d..00000000 --- a/alm/src/utils/signatureWaitingForBlocks.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' -import { BlockNumberProvider } from '../services/BlockNumberProvider' - -export const checkSignaturesWaitingForBLocks = async ( - targetBlock: number, - setWaitingStatus: Function, - setWaitingBlocksResolved: Function, - validatorList: string[], - setConfirmations: Function, - blockProvider: BlockNumberProvider, - interval: number, - subscriptions: number[] -) => { - const currentBlock = blockProvider.get() - - if (currentBlock && currentBlock >= targetBlock) { - setWaitingBlocksResolved(true) - setWaitingStatus(false) - } else { - let nextInterval = interval - if (!currentBlock) { - nextInterval = 500 - } else { - const validatorsWaiting = validatorList.map(validator => { - return { - validator, - status: VALIDATOR_CONFIRMATION_STATUS.WAITING - } - }) - setWaitingStatus(true) - setConfirmations(validatorsWaiting) - } - const timeoutId = setTimeout( - () => - checkSignaturesWaitingForBLocks( - targetBlock, - setWaitingStatus, - setWaitingBlocksResolved, - validatorList, - setConfirmations, - blockProvider, - interval, - subscriptions - ), - nextInterval - ) - subscriptions.push(timeoutId) - } -} diff --git a/deployment-e2e/molecule/multiple/tests/test_multiple.py b/deployment-e2e/molecule/multiple/tests/test_multiple.py index 7feebd25..d24117ad 100644 --- a/deployment-e2e/molecule/multiple/tests/test_multiple.py +++ b/deployment-e2e/molecule/multiple/tests/test_multiple.py @@ -24,6 +24,7 @@ def test_services(host, service): ("oracle_bridge_affirmation_1"), ("oracle_bridge_senderhome_1"), ("oracle_bridge_senderforeign_1"), + ("oracle_bridge_shutdown_1"), ("ui_ui_1"), ("monitor_monitor_1") ]) diff --git a/deployment-e2e/molecule/oracle/tests/test_oracle.py b/deployment-e2e/molecule/oracle/tests/test_oracle.py index 5ac88b31..147ff81a 100644 --- a/deployment-e2e/molecule/oracle/tests/test_oracle.py +++ b/deployment-e2e/molecule/oracle/tests/test_oracle.py @@ -14,6 +14,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( ("oracle_bridge_affirmation_1"), ("oracle_bridge_senderhome_1"), ("oracle_bridge_senderforeign_1"), + ("oracle_bridge_shutdown_1"), ]) def test_docker_containers(host, name): container = host.docker(name) diff --git a/e2e-commons/up.sh b/e2e-commons/up.sh index ace0de63..ad309637 100755 --- a/e2e-commons/up.sh +++ b/e2e-commons/up.sh @@ -42,6 +42,7 @@ startValidator () { fi docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:home docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:foreign + docker-compose $1 run $2 $3 -d oracle yarn manager:shutdown } startAMBValidator () { @@ -52,6 +53,7 @@ startAMBValidator () { docker-compose $1 run $2 $3 -d oracle-amb yarn watcher:affirmation-request docker-compose $1 run $2 $3 -d oracle-amb yarn sender:home docker-compose $1 run $2 $3 -d oracle-amb yarn sender:foreign + docker-compose $1 run $2 $3 -d oracle-amb yarn manager:shutdown } while [ "$1" != "" ]; do @@ -120,13 +122,7 @@ while [ "$1" != "" ]; do fi if [ "$1" == "alm-e2e" ]; then - docker-compose up -d redis rabbit - - docker-compose run -d oracle-amb yarn watcher:signature-request - docker-compose run -d oracle-amb yarn watcher:collected-signatures - docker-compose run -d oracle-amb yarn watcher:affirmation-request - docker-compose run -d oracle-amb yarn sender:home - docker-compose run -d oracle-amb yarn sender:foreign + startAMBValidator "" "" "" "redis" "rabbit" oracle2name="-p validator2" oracle2Values="-e ORACLE_VALIDATOR_ADDRESS=0xdCC784657C78054aa61FbcFFd2605F32374816A4 -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5a5c3645d0f04e9eb4f27f94ed4c244a225587405b8838e7456f7781ce3a9513" diff --git a/monitor/alerts.js b/monitor/alerts.js index 597e0cff..66ed24bb 100644 --- a/monitor/alerts.js +++ b/monitor/alerts.js @@ -1,18 +1,19 @@ require('dotenv').config() const logger = require('./logger')('alerts') -const eventsInfo = require('./utils/events') const { processedMsgNotDelivered, eventWithoutReference } = require('./utils/message') const { BRIDGE_MODES } = require('../commons') -const { web3Home, web3Foreign, getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3') +const { web3Home, web3Foreign } = require('./utils/web3') -async function main() { +async function main(eventsInfo) { const { + homeBlockNumber, + foreignBlockNumber, homeToForeignRequests, homeToForeignConfirmations, foreignToHomeConfirmations, foreignToHomeRequests, bridgeMode - } = await eventsInfo() + } = eventsInfo let xSignatures let xAffirmations @@ -24,8 +25,6 @@ async function main() { xAffirmations = foreignToHomeConfirmations.filter(eventWithoutReference(foreignToHomeRequests)) } logger.debug('building misbehavior blocks') - const homeBlockNumber = await getHomeBlockNumber() - const foreignBlockNumber = await getForeignBlockNumber() const baseRange = [false, false, false, false, false] const xSignaturesMisbehavior = buildRangesObject( diff --git a/monitor/checkWorker2.js b/monitor/checkWorker2.js index 06778c55..3fbce3b5 100644 --- a/monitor/checkWorker2.js +++ b/monitor/checkWorker2.js @@ -1,6 +1,7 @@ require('dotenv').config() const logger = require('./logger')('checkWorker2') const eventsStats = require('./eventsStats') +const getEventsInfo = require('./utils/events') const alerts = require('./alerts') const { writeFile, createDir } = require('./utils/file') const { saveCache } = require('./utils/web3Cache') @@ -10,8 +11,10 @@ const { MONITOR_BRIDGE_NAME } = process.env async function checkWorker2() { try { createDir(`/responses/${MONITOR_BRIDGE_NAME}`) + logger.debug('calling getEventsInfo()') + const eventsInfo = await getEventsInfo() logger.debug('calling eventsStats()') - const evStats = await eventsStats() + const evStats = await eventsStats(eventsInfo) if (!evStats) throw new Error('evStats is empty: ' + JSON.stringify(evStats)) evStats.ok = (evStats.onlyInHomeDeposits || evStats.home.deliveredMsgNotProcessedInForeign).length === 0 && @@ -22,7 +25,7 @@ async function checkWorker2() { writeFile(`/responses/${MONITOR_BRIDGE_NAME}/eventsStats.json`, evStats) logger.debug('calling alerts()') - const _alerts = await alerts() + const _alerts = await alerts(eventsInfo) if (!_alerts) throw new Error('alerts is empty: ' + JSON.stringify(_alerts)) _alerts.ok = !_alerts.executeAffirmations.mostRecentTxHash && !_alerts.executeSignatures.mostRecentTxHash _alerts.health = true diff --git a/monitor/eventsStats.js b/monitor/eventsStats.js index 2d5ac542..4f506a52 100644 --- a/monitor/eventsStats.js +++ b/monitor/eventsStats.js @@ -1,5 +1,4 @@ require('dotenv').config() -const eventsInfo = require('./utils/events') const { processedMsgNotDelivered, deliveredMsgNotProcessed, @@ -15,14 +14,14 @@ const { MONITOR_HOME_TO_FOREIGN_CHECK_SENDER } = process.env -async function main() { +async function main(eventsInfo) { const { homeToForeignRequests, homeToForeignConfirmations, foreignToHomeConfirmations, foreignToHomeRequests, bridgeMode - } = await eventsInfo() + } = eventsInfo if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) { return { diff --git a/monitor/getBalances.js b/monitor/getBalances.js index 806cd045..0d32efaf 100644 --- a/monitor/getBalances.js +++ b/monitor/getBalances.js @@ -5,7 +5,12 @@ const logger = require('./logger')('getBalances') const { BRIDGE_MODES } = require('../commons') const { web3Home, web3Foreign, getHomeBlockNumber } = require('./utils/web3') -const { COMMON_HOME_BRIDGE_ADDRESS, COMMON_FOREIGN_BRIDGE_ADDRESS } = process.env +const { + MONITOR_HOME_START_BLOCK, + MONITOR_FOREIGN_START_BLOCK, + COMMON_HOME_BRIDGE_ADDRESS, + COMMON_FOREIGN_BRIDGE_ADDRESS +} = process.env const { ERC20_ABI, @@ -20,6 +25,8 @@ const { async function main(bridgeMode, eventsInfo) { const { + homeBlockNumber, + foreignBlockNumber, homeToForeignConfirmations, foreignToHomeConfirmations, homeDelayedBlockNumber, @@ -46,6 +53,13 @@ async function main(bridgeMode, eventsInfo) { ...foreignToHomeConfirmations.filter(e => e.blockNumber > homeDelayedBlockNumber).map(e => e.value) ) + const blockRanges = { + startBlockHome: MONITOR_HOME_START_BLOCK, + endBlockHome: homeBlockNumber, + startBlockForeign: MONITOR_FOREIGN_START_BLOCK, + endBlockForeign: foreignBlockNumber + } + if (bridgeMode === BRIDGE_MODES.ERC_TO_ERC) { const foreignBridge = new web3Foreign.eth.Contract(FOREIGN_ERC_TO_ERC_ABI, COMMON_FOREIGN_BRIDGE_ADDRESS) const erc20Address = await foreignBridge.methods.erc20token().call() @@ -72,6 +86,7 @@ async function main(bridgeMode, eventsInfo) { erc20Balance: Web3Utils.fromWei(foreignErc20Balance) }, balanceDiff: Number(Web3Utils.fromWei(diff)), + ...blockRanges, lastChecked: Math.floor(Date.now() / 1000) } } else if (bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC || bridgeMode === BRIDGE_MODES.NATIVE_TO_ERC_V1) { @@ -94,6 +109,7 @@ async function main(bridgeMode, eventsInfo) { totalSupply: Web3Utils.fromWei(totalSupply) }, balanceDiff: Number(Web3Utils.fromWei(diff)), + ...blockRanges, lastChecked: Math.floor(Date.now() / 1000) } } else if (bridgeMode === BRIDGE_MODES.ERC_TO_NATIVE) { @@ -163,12 +179,14 @@ async function main(bridgeMode, eventsInfo) { }, foreign, balanceDiff: Number(Web3Utils.fromWei(diff)), + ...blockRanges, lastChecked: Math.floor(Date.now() / 1000) } } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) { return { home: {}, foreign: {}, + ...blockRanges, lastChecked: Math.floor(Date.now() / 1000) } } else { diff --git a/monitor/index.js b/monitor/index.js index c2f3d6f3..0a8fa45d 100644 --- a/monitor/index.js +++ b/monitor/index.js @@ -2,7 +2,6 @@ require('dotenv').config() const express = require('express') const cors = require('cors') const { readFile } = require('./utils/file') -const { getPrometheusMetrics } = require('./prometheusMetrics') const app = express() const bridgeRouter = express.Router({ mergeParams: true }) @@ -25,7 +24,8 @@ bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers| bridgeRouter.get('/metrics', (req, res, next) => { try { - const metrics = getPrometheusMetrics(req.params.bridgeName) + const { bridgeName } = req.params + const metrics = readFile(`./responses/${bridgeName}/metrics.txt`, false) res.type('text').send(metrics) } catch (e) { next(e) diff --git a/monitor/metricsWorker.js b/monitor/metricsWorker.js new file mode 100644 index 00000000..66027c76 --- /dev/null +++ b/monitor/metricsWorker.js @@ -0,0 +1,20 @@ +require('dotenv').config() +const logger = require('./logger')('metricsWorker') +const { writeFile, createDir } = require('./utils/file') +const getPrometheusMetrics = require('./prometheusMetrics') + +const { MONITOR_BRIDGE_NAME } = process.env + +async function metricsWorker() { + try { + createDir(`/responses/${MONITOR_BRIDGE_NAME}`) + logger.debug('calling getPrometheusMetrics()') + const metrics = await getPrometheusMetrics(MONITOR_BRIDGE_NAME) + if (!metrics) throw new Error('metrics is empty: ' + JSON.stringify(metrics)) + writeFile(`/responses/${MONITOR_BRIDGE_NAME}/metrics.txt`, metrics, { stringify: false }) + logger.debug('Done') + } catch (e) { + logger.error(e) + } +} +metricsWorker() diff --git a/monitor/package.json b/monitor/package.json index 55583cf3..b81c8cdf 100644 --- a/monitor/package.json +++ b/monitor/package.json @@ -4,7 +4,7 @@ "description": "", "main": "index.js", "scripts": { - "check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js", + "check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js && timeout -s 9 5m node checkWorker3.js && timeout -s 9 10s node metricsWorker.js", "start": "node index.js", "check-and-start": "yarn check-all && yarn start", "lint": "eslint . --ignore-path ../.eslintignore", diff --git a/monitor/prometheusMetrics.js b/monitor/prometheusMetrics.js index af649abc..02e19279 100644 --- a/monitor/prometheusMetrics.js +++ b/monitor/prometheusMetrics.js @@ -1,8 +1,10 @@ +require('dotenv').config() +const logger = require('./logger')('getBalances') const { readFile } = require('./utils/file') const { - MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST, - MONITOR_HOME_TO_FOREIGN_BLOCK_LIST, + MONITOR_HOME_START_BLOCK, + MONITOR_FOREIGN_START_BLOCK, MONITOR_HOME_VALIDATORS_BALANCE_ENABLE, MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE } = process.env @@ -23,7 +25,9 @@ function hasError(obj) { return 'error' in obj } -function getPrometheusMetrics(bridgeName) { +// Try to collect all metrics from JSON responses and then +// discard all unsuccessfully retrieved ones +async function getPrometheusMetrics(bridgeName) { const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json` const metrics = {} @@ -32,22 +36,44 @@ function getPrometheusMetrics(bridgeName) { const balancesFile = readFile(responsePath('getBalances')) if (!hasError(balancesFile)) { - const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile - metrics.balances_home_value = homeBalances.totalSupply - metrics.balances_home_txs_deposit = homeBalances.deposits - metrics.balances_home_txs_withdrawal = homeBalances.withdrawals + const { home, foreign, ...commonBalances } = balancesFile - metrics.balances_foreign_value = foreignBalances.erc20Balance - metrics.balances_foreign_txs_deposit = foreignBalances.deposits - metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals + const balanceMetrics = { + // ERC_TO_ERC or ERC_TO_NATIVE mode + balances_home_value: home.totalSupply, + balances_home_txs_deposit: home.deposits, + balances_home_txs_withdrawal: home.withdrawals, + balances_foreign_value: foreign.erc20Balance, + balances_foreign_txs_deposit: foreign.deposits, + balances_foreign_txs_withdrawal: foreign.withdrawals, - metrics.balances_diff_value = commonBalances.balanceDiff - metrics.balances_diff_deposit = commonBalances.depositsDiff - metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff - if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) { - metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff - metrics.balances_unclaimed_value = commonBalances.unclaimedBalance + // Not ARBITRARY_MESSAGE mode + balances_diff_value: commonBalances.balanceDiff, + balances_diff_deposit: commonBalances.depositsDiff, + balances_diff_withdrawal: commonBalances.withdrawalDiff, + + // MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST or MONITOR_HOME_TO_FOREIGN_BLOCK_LIST is set + balances_unclaimed_txs: commonBalances.unclaimedDiff, + balances_unclaimed_value: commonBalances.unclaimedBalance, + + // ARBITRARY_MESSAGE mode + txs_home_out: home.toForeign, + txs_home_in: home.fromForeign, + txs_foreign_out: foreign.toHome, + txs_foreign_in: foreign.fromHome, + txs_diff_home_out_oracles: commonBalances.fromHomeToForeignDiff, + txs_diff_home_out_users: commonBalances.fromHomeToForeignPBUDiff, + txs_diff_foreign_out: commonBalances.fromForeignToHomeDiff } + + const blockRanges = { + state_startblock_home: commonBalances.startBlockHome || MONITOR_HOME_START_BLOCK, + state_startblock_foreign: commonBalances.startBlockForeign || MONITOR_FOREIGN_START_BLOCK, + state_endblock_home: commonBalances.endBlockHome, + state_endblock_foreign: commonBalances.endBlockForeign + } + + Object.assign(metrics, blockRanges, balanceMetrics) } // Validator metrics @@ -62,7 +88,11 @@ function getPrometheusMetrics(bridgeName) { : Object.keys(allValidators) validatorAddressesWithBalanceCheck.forEach((addr, ind) => { - metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance + if (addr in allValidators) { + metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance + } else { + logger.debug(`Nonexistent validator address ${addr}`) + } }) } } @@ -96,9 +126,12 @@ function getPrometheusMetrics(bridgeName) { return Object.entries(metrics).reduceRight( // Prometheus supports `Nan` and possibly signed `Infinity` // in case cast to `Number` fails - (acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`, + (acc, [key, val]) => { + if (typeof val === 'undefined') return acc + else return `${key} ${Number(val)}\n${acc}` + }, '' ) } -module.exports = { getPrometheusMetrics } +module.exports = getPrometheusMetrics diff --git a/monitor/utils/events.js b/monitor/utils/events.js index 5d03ec5e..ac312d5a 100644 --- a/monitor/utils/events.js +++ b/monitor/utils/events.js @@ -236,13 +236,15 @@ async function main(mode) { foreignToHomeRequests, isExternalErc20, bridgeMode, + homeBlockNumber, + foreignBlockNumber, homeDelayedBlockNumber, foreignDelayedBlockNumber } if (MONITOR_CACHE_EVENTS === 'true') { logger.debug('saving obtained events into cache file') - writeFile(cacheFilePath, result, false) + writeFile(cacheFilePath, result, { useCwd: false }) } return result } diff --git a/monitor/utils/file.js b/monitor/utils/file.js index fc957fa8..8247c5af 100644 --- a/monitor/utils/file.js +++ b/monitor/utils/file.js @@ -1,23 +1,30 @@ const fs = require('fs') const path = require('path') -function readFile(filePath) { +function readFile(filePath, parseJson = true) { try { const content = fs.readFileSync(filePath) + if (!parseJson) return content const json = JSON.parse(content) const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked return Object.assign({}, json, { timeDiff }) } catch (e) { - console.error(e) + console.error('readFlle', e) return { error: 'the bridge statistics are not available' } } } -function writeFile(filePath, object, useCwd = true) { +function writeFile(filePath, object, paramOptions = {}) { + const defaultOptions = { + useCwd: true, + stringify: true + } + const { useCwd, stringify } = Object.assign({}, defaultOptions, paramOptions) + const fullPath = useCwd ? path.join(process.cwd(), filePath) : filePath - fs.writeFileSync(fullPath, JSON.stringify(object, null, 4)) + fs.writeFileSync(fullPath, stringify ? JSON.stringify(object, null, 4) : object) } function createDir(dirPath) { diff --git a/oracle/.env.example b/oracle/.env.example index d3301b5e..d927d24c 100644 --- a/oracle/.env.example +++ b/oracle/.env.example @@ -14,12 +14,14 @@ COMMON_HOME_GAS_PRICE_SPEED_TYPE=standard COMMON_HOME_GAS_PRICE_FALLBACK=1000000000 ORACLE_HOME_GAS_PRICE_UPDATE_INTERVAL=600000 COMMON_HOME_GAS_PRICE_FACTOR=1 +ORACLE_HOME_TX_RESEND_INTERVAL=300000 COMMON_FOREIGN_GAS_PRICE_SUPPLIER_URL=https://gasprice.poa.network/ COMMON_FOREIGN_GAS_PRICE_SPEED_TYPE=standard COMMON_FOREIGN_GAS_PRICE_FALLBACK=1000000000 ORACLE_FOREIGN_GAS_PRICE_UPDATE_INTERVAL=600000 COMMON_FOREIGN_GAS_PRICE_FACTOR=1 +ORACLE_FOREIGN_TX_RESEND_INTERVAL=1200000 ORACLE_QUEUE_URL=amqp://rabbit ORACLE_REDIS_URL=redis://redis diff --git a/oracle/config/base.config.js b/oracle/config/base.config.js index 5808b5dd..372b6868 100644 --- a/oracle/config/base.config.js +++ b/oracle/config/base.config.js @@ -69,7 +69,8 @@ const bridgeConfig = { foreignBridgeAbi: foreignAbi, eventFilter: {}, validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY), - maxProcessingTime + maxProcessingTime, + shutdownKey: 'oracle-shutdown' } const homeConfig = { diff --git a/oracle/config/foreign-sender.config.js b/oracle/config/foreign-sender.config.js index aa347c0f..97b1a154 100644 --- a/oracle/config/foreign-sender.config.js +++ b/oracle/config/foreign-sender.config.js @@ -1,7 +1,10 @@ const baseConfig = require('./base.config') +const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants') const { web3Foreign, web3ForeignRedundant, web3ForeignFallback } = require('../src/services/web3') +const { ORACLE_FOREIGN_TX_RESEND_INTERVAL } = process.env + module.exports = { ...baseConfig.bridgeConfig, queue: 'foreign-prioritized', @@ -10,5 +13,6 @@ module.exports = { name: 'sender-foreign', web3: web3Foreign, web3Redundant: web3ForeignRedundant, - web3Fallback: web3ForeignFallback + web3Fallback: web3ForeignFallback, + resendInterval: parseInt(ORACLE_FOREIGN_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL } diff --git a/oracle/config/home-sender.config.js b/oracle/config/home-sender.config.js index ee2bd570..d442c03d 100644 --- a/oracle/config/home-sender.config.js +++ b/oracle/config/home-sender.config.js @@ -1,7 +1,10 @@ const baseConfig = require('./base.config') +const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants') const { web3Home, web3HomeRedundant, web3HomeFallback } = require('../src/services/web3') +const { ORACLE_HOME_TX_RESEND_INTERVAL } = process.env + module.exports = { ...baseConfig.bridgeConfig, queue: 'home-prioritized', @@ -10,5 +13,6 @@ module.exports = { name: 'sender-home', web3: web3Home, web3Redundant: web3HomeRedundant, - web3Fallback: web3HomeFallback + web3Fallback: web3HomeFallback, + resendInterval: parseInt(ORACLE_HOME_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL } diff --git a/oracle/config/shutdown-manager.config.js b/oracle/config/shutdown-manager.config.js new file mode 100644 index 00000000..cbbe3747 --- /dev/null +++ b/oracle/config/shutdown-manager.config.js @@ -0,0 +1,20 @@ +const baseConfig = require('./base.config') + +const { + ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL, + ORACLE_SHUTDOWN_SERVICE_URL, + ORACLE_SHUTDOWN_CONTRACT_ADDRESS, + ORACLE_SHUTDOWN_CONTRACT_METHOD +} = process.env + +module.exports = { + ...baseConfig.bridgeConfig, + id: 'shutdown-manager', + name: 'shutdown-manager', + pollingInterval: ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL || 120000, + checksBeforeResume: 3, + checksBeforeStop: 1, + shutdownServiceURL: ORACLE_SHUTDOWN_SERVICE_URL, + shutdownContractAddress: ORACLE_SHUTDOWN_CONTRACT_ADDRESS, + shutdownMethod: (ORACLE_SHUTDOWN_CONTRACT_METHOD || 'isShutdown()').trim() +} diff --git a/oracle/docker-compose-erc-native.yml b/oracle/docker-compose-erc-native.yml index ed1a5d59..a5abc9e1 100644 --- a/oracle/docker-compose-erc-native.yml +++ b/oracle/docker-compose-erc-native.yml @@ -77,6 +77,12 @@ services: networks: - net_db_bridge_request - net_rabbit_bridge_request + bridge_shutdown: + extends: + file: docker-compose.yml + service: bridge_shutdown + networks: + - net_db_bridge_shutdown networks: net_db_bridge_request: @@ -91,6 +97,8 @@ networks: driver: bridge net_db_bridge_senderforeign: driver: bridge + net_db_bridge_shutdown: + driver: bridge net_rabbit_bridge_request: driver: bridge net_rabbit_bridge_collected: diff --git a/oracle/docker-compose-transfer.yml b/oracle/docker-compose-transfer.yml index eabba1c3..e3bbdb85 100644 --- a/oracle/docker-compose-transfer.yml +++ b/oracle/docker-compose-transfer.yml @@ -61,6 +61,12 @@ services: networks: - net_db_bridge_request - net_rabbit_bridge_request + bridge_shutdown: + extends: + file: docker-compose.yml + service: bridge_shutdown + networks: + - net_db_bridge_shutdown networks: net_db_bridge_request: @@ -75,6 +81,8 @@ networks: driver: bridge net_db_bridge_senderforeign: driver: bridge + net_db_bridge_shutdown: + driver: bridge net_rabbit_bridge_request: driver: bridge net_rabbit_bridge_collected: diff --git a/oracle/docker-compose.yml b/oracle/docker-compose.yml index 73cc6c8d..6417e53a 100644 --- a/oracle/docker-compose.yml +++ b/oracle/docker-compose.yml @@ -21,12 +21,13 @@ services: command: [redis-server, --appendonly, 'yes'] hostname: redis image: redis:4 - networks: + networks: - net_db_bridge_request - net_db_bridge_collected - net_db_bridge_affirmation - net_db_bridge_senderhome - net_db_bridge_senderforeign + - net_db_bridge_shutdown restart: unless-stopped volumes: ['~/bridge_data/redis:/data'] bridge_request: @@ -34,8 +35,8 @@ services: mem_limit: 500m image: poanetwork/tokenbridge-oracle:latest env_file: ./.env - environment: - - NODE_ENV=production + environment: + - NODE_ENV=production - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY} restart: unless-stopped entrypoint: yarn watcher:signature-request @@ -47,8 +48,8 @@ services: mem_limit: 500m image: poanetwork/tokenbridge-oracle:latest env_file: ./.env - environment: - - NODE_ENV=production + environment: + - NODE_ENV=production - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS} restart: unless-stopped entrypoint: yarn watcher:collected-signatures @@ -60,8 +61,8 @@ services: mem_limit: 500m image: poanetwork/tokenbridge-oracle:latest env_file: ./.env - environment: - - NODE_ENV=production + environment: + - NODE_ENV=production - ORACLE_VALIDATOR_ADDRESS=${ORACLE_VALIDATOR_ADDRESS} restart: unless-stopped entrypoint: yarn watcher:affirmation-request @@ -73,8 +74,8 @@ services: mem_limit: 500m image: poanetwork/tokenbridge-oracle:latest env_file: ./.env - environment: - - NODE_ENV=production + environment: + - NODE_ENV=production - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY} restart: unless-stopped entrypoint: yarn sender:home @@ -86,14 +87,25 @@ services: mem_limit: 500m image: poanetwork/tokenbridge-oracle:latest env_file: ./.env - environment: - - NODE_ENV=production + environment: + - NODE_ENV=production - ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY} restart: unless-stopped entrypoint: yarn sender:foreign networks: - net_db_bridge_senderforeign - net_rabbit_bridge_senderforeign + bridge_shutdown: + cpus: 0.1 + mem_limit: 500m + image: poanetwork/tokenbridge-oracle:latest + env_file: ./.env + environment: + - NODE_ENV=production + restart: unless-stopped + entrypoint: yarn manager:shutdown + networks: + - net_db_bridge_shutdown networks: net_db_bridge_request: @@ -106,6 +118,8 @@ networks: driver: bridge net_db_bridge_senderforeign: driver: bridge + net_db_bridge_shutdown: + driver: bridge net_rabbit_bridge_request: driver: bridge net_rabbit_bridge_collected: diff --git a/oracle/package.json b/oracle/package.json index 77c2a600..76eb2f18 100644 --- a/oracle/package.json +++ b/oracle/package.json @@ -13,6 +13,7 @@ "sender:home": "./scripts/start-worker.sh sender home-sender", "sender:foreign": "./scripts/start-worker.sh sender foreign-sender", "confirm:transfer": "./scripts/start-worker.sh confirmRelay transfer-watcher", + "manager:shutdown": "./scripts/start-worker.sh shutdownManager shutdown-manager", "dev": "concurrently -n 'watcher:signature-request,watcher:collected-signatures,watcher:affirmation-request,watcher:transfer, sender:home,sender:foreign' -c 'red,green,yellow,blue,magenta,cyan' 'yarn watcher:signature-request' 'yarn watcher:collected-signatures' 'yarn watcher:affirmation-request' 'yarn watcher:transfer' 'yarn sender:home' 'yarn sender:foreign'", "test": "NODE_ENV=test mocha", "test:watch": "NODE_ENV=test mocha --watch --reporter=min", diff --git a/oracle/src/sender.js b/oracle/src/sender.js index 659d0b5f..fa8d13e3 100644 --- a/oracle/src/sender.js +++ b/oracle/src/sender.js @@ -4,6 +4,7 @@ const { connectSenderToQueue } = require('./services/amqpClient') const { redis } = require('./services/redisClient') const GasPrice = require('./services/gasPrice') const logger = require('./services/logger') +const { getShutdownFlag } = require('./services/shutdownState') const { sendTx } = require('./tx/sendTx') const { getNonce, getChainId } = require('./tx/web3') const { @@ -47,6 +48,7 @@ async function initialize() { connectSenderToQueue({ queueName: config.queue, oldQueueName: config.oldQueue, + resendInterval: config.resendInterval, cb: options => { if (config.maxProcessingTime) { return watchdog(() => main(options), config.maxProcessingTime, () => { @@ -101,6 +103,14 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT return } + const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false) + if (await getShutdownFlag(logger, config.shutdownKey, true)) { + if (!wasShutdown) { + logger.info('Oracle sender was suspended via the remote shutdown process') + } + return + } + const txArray = JSON.parse(msg.content) logger.debug(`Msg received with ${txArray.length} Tx to send`) const gasPrice = GasPrice.getPrice().toString(10) @@ -207,7 +217,7 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT await scheduleForRetry(failedTx, msg.properties.headers['x-retries']) } if (resendJobs.length) { - logger.info(`Sending ${resendJobs.length} Tx Delayed Resend Requests to Queue`) + logger.info({ delay: config.resendInterval }, `Sending ${resendJobs.length} Tx Delayed Resend Requests to Queue`) await scheduleTransactionResend(resendJobs) } ackMsg(msg) diff --git a/oracle/src/services/amqpClient.js b/oracle/src/services/amqpClient.js index 86bbb19a..588d95e4 100644 --- a/oracle/src/services/amqpClient.js +++ b/oracle/src/services/amqpClient.js @@ -5,7 +5,6 @@ const connection = require('amqp-connection-manager').connect(process.env.ORACLE const logger = require('./logger') const { getRetrySequence } = require('../utils/utils') const { - TRANSACTION_RESEND_TIMEOUT, SENDER_QUEUE_MAX_PRIORITY, SENDER_QUEUE_SEND_PRIORITY, SENDER_QUEUE_CHECK_STATUS_PRIORITY @@ -48,7 +47,7 @@ function connectWatcherToQueue({ queueName, workerQueue, cb }) { cb({ sendToQueue, sendToWorker, channel: channelWrapper }) } -function connectSenderToQueue({ queueName, oldQueueName, cb }) { +function connectSenderToQueue({ queueName, oldQueueName, cb, resendInterval }) { const deadLetterExchange = `${queueName}-retry` async function resendMessagesToNewQueue(channel) { @@ -97,7 +96,8 @@ function connectSenderToQueue({ queueName, oldQueueName, cb }) { channelWrapper, channel, queueName, - deadLetterExchange + deadLetterExchange, + delay: resendInterval }) } }) @@ -164,13 +164,13 @@ async function generateRetry({ data, msgRetries, channelWrapper, channel, queueN }) } -async function generateTransactionResend({ data, channelWrapper, channel, queueName, deadLetterExchange }) { - const retryQueue = `${queueName}-check-tx-status` +async function generateTransactionResend({ data, channelWrapper, channel, queueName, deadLetterExchange, delay }) { + const retryQueue = `${queueName}-check-tx-status-${delay}` await channel.assertQueue(retryQueue, { durable: true, deadLetterExchange, - messageTtl: TRANSACTION_RESEND_TIMEOUT, - expires: TRANSACTION_RESEND_TIMEOUT * 10, + messageTtl: delay, + expires: delay * 10, maxPriority: SENDER_QUEUE_MAX_PRIORITY }) await channelWrapper.sendToQueue(retryQueue, data, { diff --git a/oracle/src/services/logger.js b/oracle/src/services/logger.js index bf5caa4f..48cc80d6 100644 --- a/oracle/src/services/logger.js +++ b/oracle/src/services/logger.js @@ -3,6 +3,7 @@ const path = require('path') const { web3Home, web3Foreign, + web3Side, web3HomeFallback, web3ForeignFallback, web3HomeRedundant, @@ -30,4 +31,8 @@ web3ForeignFallback.currentProvider.setLogger(logger) web3HomeRedundant.currentProvider.setLogger(logger) web3ForeignRedundant.currentProvider.setLogger(logger) +if (web3Side) { + web3Side.currentProvider.setLogger(logger) +} + module.exports = logger diff --git a/oracle/src/services/shutdownState.js b/oracle/src/services/shutdownState.js new file mode 100644 index 00000000..a1313de3 --- /dev/null +++ b/oracle/src/services/shutdownState.js @@ -0,0 +1,23 @@ +const { redis } = require('./redisClient') + +let isShutdown = false +async function getShutdownFlag(logger, shutdownKey, force = false) { + if (force) { + logger.debug('Reading current shutdown state from the DB') + isShutdown = (await redis.get(shutdownKey)) === 'true' + logger.debug({ isShutdown }, 'Read shutdown state from the DB') + } + return isShutdown +} + +async function setShutdownFlag(logger, shutdownKey, value) { + logger.info({ isShutdown: value }, 'Updating current shutdown state in the DB') + isShutdown = value + await redis.set(shutdownKey, value) + logger.debug('Updated state in the DB') +} + +module.exports = { + getShutdownFlag, + setShutdownFlag +} diff --git a/oracle/src/services/web3.js b/oracle/src/services/web3.js index 70b195c6..9d1d1458 100644 --- a/oracle/src/services/web3.js +++ b/oracle/src/services/web3.js @@ -6,6 +6,7 @@ const { RETRY_CONFIG } = require('../utils/constants') const { COMMON_HOME_RPC_URL, COMMON_FOREIGN_RPC_URL, + ORACLE_SIDE_RPC_URL, ORACLE_RPC_REQUEST_TIMEOUT, ORACLE_HOME_RPC_POLLING_INTERVAL, ORACLE_FOREIGN_RPC_POLLING_INTERVAL @@ -41,6 +42,18 @@ const web3Home = new Web3(homeProvider) const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions) const web3Foreign = new Web3(foreignProvider) +let web3Side = null +if (ORACLE_SIDE_RPC_URL) { + const sideUrls = ORACLE_SIDE_RPC_URL.split(' ').filter(url => url.length > 0) + const sideOptions = { + requestTimeout: configuredTimeout || 2000, + retry: RETRY_CONFIG + } + + const sideProvider = new HttpListProvider(sideUrls, sideOptions) + web3Side = new Web3(sideProvider) +} + // secondary fallback providers are intended to be used in places where // it is more likely that RPC calls to the local non-archive nodes can fail // e.g. for checking status of the old transaction via eth_getTransactionByHash @@ -70,6 +83,7 @@ if (foreignUrls.length > 1) { module.exports = { web3Home, web3Foreign, + web3Side, web3HomeRedundant, web3ForeignRedundant, web3HomeFallback, diff --git a/oracle/src/shutdownManager.js b/oracle/src/shutdownManager.js new file mode 100644 index 00000000..ba99ec31 --- /dev/null +++ b/oracle/src/shutdownManager.js @@ -0,0 +1,114 @@ +const fetch = require('node-fetch') +const path = require('path') + +const { EXIT_CODES } = require('./utils/constants') +const { watchdog } = require('./utils/utils') +const logger = require('./services/logger') +const { redis } = require('./services/redisClient') +const { web3Side } = require('./services/web3') +const { getShutdownFlag, setShutdownFlag } = require('./services/shutdownState') + +if (process.argv.length < 3) { + logger.error('Please check the number of arguments, config file was not provided') + process.exit(EXIT_CODES.GENERAL_ERROR) +} + +const config = require(path.join('../config/', process.argv[2])) + +if (config.shutdownContractAddress && !web3Side) { + logger.error( + 'ORACLE_SHUTDOWN_CONTRACT_ADDRESS was provided but not side chain provider was registered.' + + ' Please, specify ORACLE_SIDE_RPC_URL as well.' + ) + process.exit(EXIT_CODES.GENERAL_ERROR) +} + +let shutdownCount = 0 +let okCount = 0 + +async function fetchShutdownFlag() { + if (config.shutdownServiceURL) { + logger.debug({ url: config.shutdownServiceURL }, 'Fetching shutdown status from external URL') + const result = await fetch(config.shutdownServiceURL, { + headers: { + 'Content-type': 'application/json' + }, + method: 'GET', + timeout: config.requestTimeout + }).then(res => res.json()) + + if (result.shutdown === true) { + return true + } + } + + if (config.shutdownContractAddress) { + const shutdownSelector = web3Side.eth.abi.encodeEventSignature(config.shutdownMethod) + logger.debug( + { contract: config.shutdownContractAddress, method: config.shutdownMethod, data: shutdownSelector }, + 'Fetching shutdown status from contract' + ) + const result = await web3Side.eth.call({ + to: config.shutdownContractAddress, + data: shutdownSelector + }) + logger.debug({ result }, 'Obtained result from the side RPC endpoint') + + if (result.length > 2 && web3Side.eth.abi.decodeParameter('bool', result)) { + return true + } + } + + return false +} + +async function checkShutdownFlag() { + const isShutdownFlag = await fetchShutdownFlag() + const isShutdown = await getShutdownFlag(logger, config.shutdownKey) + + if (isShutdownFlag === true && isShutdown === false) { + shutdownCount += 1 + okCount = 0 + logger.info( + { shutdownCount, remainingChecks: config.checksBeforeStop - shutdownCount }, + 'Received positive shutdown flag' + ) + } else if (isShutdownFlag === false && isShutdown === true) { + okCount += 1 + shutdownCount = 0 + logger.info({ okCount, remainingChecks: config.checksBeforeResume - okCount }, 'Received negative shutdown flag') + } else { + shutdownCount = 0 + okCount = 0 + logger.debug({ isShutdown, isShutdownFlag }, 'Received shutdown flag that is equal to the current state') + } + + if (shutdownCount >= config.checksBeforeStop) { + await setShutdownFlag(logger, config.shutdownKey, true) + } else if (okCount >= config.checksBeforeResume) { + await setShutdownFlag(logger, config.shutdownKey, false) + } +} + +async function initialize() { + logger.info('Starting shutdown flag watcher') + redis.on('connect', async () => { + await getShutdownFlag(logger, config.shutdownKey, true) + await main() + }) +} + +async function main() { + try { + await watchdog(checkShutdownFlag, config.maxProcessingTime, () => { + logger.fatal('Max processing time reached') + process.exit(EXIT_CODES.MAX_TIME_REACHED) + }) + } catch (e) { + logger.error(e) + } + + setTimeout(main, config.pollingInterval) +} + +initialize() diff --git a/oracle/src/utils/constants.js b/oracle/src/utils/constants.js index 92a76db8..a99c300c 100644 --- a/oracle/src/utils/constants.js +++ b/oracle/src/utils/constants.js @@ -23,7 +23,7 @@ module.exports = { MIN: 1, MAX: 1000 }, - TRANSACTION_RESEND_TIMEOUT: 20 * 60 * 1000, + DEFAULT_TRANSACTION_RESEND_INTERVAL: 20 * 60 * 1000, FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000, SENDER_QUEUE_MAX_PRIORITY: 10, SENDER_QUEUE_SEND_PRIORITY: 5, diff --git a/oracle/src/watcher.js b/oracle/src/watcher.js index 76cae4b8..1127f86d 100644 --- a/oracle/src/watcher.js +++ b/oracle/src/watcher.js @@ -5,6 +5,7 @@ const { connectWatcherToQueue, connection } = require('./services/amqpClient') const { getBlockNumber } = require('./tx/web3') const { redis } = require('./services/redisClient') const logger = require('./services/logger') +const { getShutdownFlag } = require('./services/shutdownState') const { getRequiredBlockConfirmations, getEvents } = require('./tx/web3') const { checkHTTPS, watchdog } = require('./utils/utils') const { EXIT_CODES } = require('./utils/constants') @@ -157,6 +158,14 @@ async function isWorkerNeeded() { async function main({ sendToQueue, sendToWorker }) { try { + const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false) + if (await getShutdownFlag(logger, config.shutdownKey, true)) { + if (!wasShutdown) { + logger.info('Oracle watcher was suspended via the remote shutdown process') + } + return + } + await checkConditions() const lastBlockToProcess = await getLastBlockToProcess() diff --git a/yarn.lock b/yarn.lock index f6dc08e6..06f3fdd9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13953,6 +13953,11 @@ node-fetch@^2.1.2, node-fetch@^2.3.0, node-fetch@^2.5.0: resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== +node-fetch@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" + integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== + node-forge@0.7.5: version "0.7.5" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.5.tgz#6c152c345ce11c52f465c2abd957e8639cd674df"