Add oracle service for remote shutdown (#531)

This commit is contained in:
Kirill Fedoseev 2021-04-12 23:38:42 +03:00 committed by GitHub
parent ae83c76be9
commit f95beee5dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 248 additions and 19 deletions

@ -44,6 +44,11 @@ ORACLE_ALWAYS_RELAY_SIGNATURES | If set to `true`, the oracle will always relay
ORACLE_RPC_REQUEST_TIMEOUT | Timeout in milliseconds for a single RPC request. Default value is `ORACLE_*_RPC_POLLING_INTERVAL * 2`. | integer ORACLE_RPC_REQUEST_TIMEOUT | Timeout in milliseconds for a single RPC request. Default value is `ORACLE_*_RPC_POLLING_INTERVAL * 2`. | integer
ORACLE_HOME_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Home sender service. Defaults to 20 minutes. | integer ORACLE_HOME_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Home sender service. Defaults to 20 minutes. | integer
ORACLE_FOREIGN_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Foreign sender service. Defaults to 20 minutes. | integer ORACLE_FOREIGN_TX_RESEND_INTERVAL | Interval in milliseconds for automatic resending of stuck transactions for Foreign sender service. Defaults to 20 minutes. | integer
ORACLE_SHUTDOWN_SERVICE_URL | Optional external URL to some other service/monitor/configuration manager that controls the remote shutdown process. GET request should return `application/json` message with the following schema: `{ shutdown: true/false }`. | URL
ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL | Optional interval in milliseconds used to request the side RPC node or external shutdown service. Default is 120000. | integer
ORACLE_SIDE_RPC_URL | Optional HTTPS URL(s) for communication with the external shutdown service or side RPC nodes, used for shutdown manager activities. Several URLs can be specified, delimited by spaces. If the connection to one of these nodes is lost the next URL is used for connection. | URL(s)
ORACLE_SHUTDOWN_CONTRACT_ADDRESS | Optional contract address in the side chain accessible through `ORACLE_SIDE_RPC_URL`, where the method passed in `ORACLE_SHUTDOWN_CONTRACT_METHOD` is implemented. | `address`
ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain to identify the current shutdown status. Method should return boolean. Default value is `isShutdown()`. | `function signature`
## UI configuration ## UI configuration

@ -24,6 +24,7 @@ def test_services(host, service):
("oracle_bridge_affirmation_1"), ("oracle_bridge_affirmation_1"),
("oracle_bridge_senderhome_1"), ("oracle_bridge_senderhome_1"),
("oracle_bridge_senderforeign_1"), ("oracle_bridge_senderforeign_1"),
("oracle_bridge_shutdown_1"),
("ui_ui_1"), ("ui_ui_1"),
("monitor_monitor_1") ("monitor_monitor_1")
]) ])

@ -14,6 +14,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
("oracle_bridge_affirmation_1"), ("oracle_bridge_affirmation_1"),
("oracle_bridge_senderhome_1"), ("oracle_bridge_senderhome_1"),
("oracle_bridge_senderforeign_1"), ("oracle_bridge_senderforeign_1"),
("oracle_bridge_shutdown_1"),
]) ])
def test_docker_containers(host, name): def test_docker_containers(host, name):
container = host.docker(name) container = host.docker(name)

@ -42,6 +42,7 @@ startValidator () {
fi fi
docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:home docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:home
docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:foreign docker-compose $1 run $2 $3 -d oracle-erc20-native yarn sender:foreign
docker-compose $1 run $2 $3 -d oracle yarn manager:shutdown
} }
startAMBValidator () { startAMBValidator () {
@ -52,6 +53,7 @@ startAMBValidator () {
docker-compose $1 run $2 $3 -d oracle-amb yarn watcher:affirmation-request docker-compose $1 run $2 $3 -d oracle-amb yarn watcher:affirmation-request
docker-compose $1 run $2 $3 -d oracle-amb yarn sender:home docker-compose $1 run $2 $3 -d oracle-amb yarn sender:home
docker-compose $1 run $2 $3 -d oracle-amb yarn sender:foreign docker-compose $1 run $2 $3 -d oracle-amb yarn sender:foreign
docker-compose $1 run $2 $3 -d oracle-amb yarn manager:shutdown
} }
while [ "$1" != "" ]; do while [ "$1" != "" ]; do
@ -120,13 +122,7 @@ while [ "$1" != "" ]; do
fi fi
if [ "$1" == "alm-e2e" ]; then if [ "$1" == "alm-e2e" ]; then
docker-compose up -d redis rabbit startAMBValidator "" "" "" "redis" "rabbit"
docker-compose run -d oracle-amb yarn watcher:signature-request
docker-compose run -d oracle-amb yarn watcher:collected-signatures
docker-compose run -d oracle-amb yarn watcher:affirmation-request
docker-compose run -d oracle-amb yarn sender:home
docker-compose run -d oracle-amb yarn sender:foreign
oracle2name="-p validator2" oracle2name="-p validator2"
oracle2Values="-e ORACLE_VALIDATOR_ADDRESS=0xdCC784657C78054aa61FbcFFd2605F32374816A4 -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5a5c3645d0f04e9eb4f27f94ed4c244a225587405b8838e7456f7781ce3a9513" oracle2Values="-e ORACLE_VALIDATOR_ADDRESS=0xdCC784657C78054aa61FbcFFd2605F32374816A4 -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=5a5c3645d0f04e9eb4f27f94ed4c244a225587405b8838e7456f7781ce3a9513"

@ -69,7 +69,8 @@ const bridgeConfig = {
foreignBridgeAbi: foreignAbi, foreignBridgeAbi: foreignAbi,
eventFilter: {}, eventFilter: {},
validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY), validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY),
maxProcessingTime maxProcessingTime,
shutdownKey: 'oracle-shutdown'
} }
const homeConfig = { const homeConfig = {

@ -0,0 +1,20 @@
const baseConfig = require('./base.config')
const {
ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL,
ORACLE_SHUTDOWN_SERVICE_URL,
ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
ORACLE_SHUTDOWN_CONTRACT_METHOD
} = process.env
module.exports = {
...baseConfig.bridgeConfig,
id: 'shutdown-manager',
name: 'shutdown-manager',
pollingInterval: ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL || 120000,
checksBeforeResume: 3,
checksBeforeStop: 1,
shutdownServiceURL: ORACLE_SHUTDOWN_SERVICE_URL,
shutdownContractAddress: ORACLE_SHUTDOWN_CONTRACT_ADDRESS,
shutdownMethod: (ORACLE_SHUTDOWN_CONTRACT_METHOD || 'isShutdown()').trim()
}

@ -77,6 +77,12 @@ services:
networks: networks:
- net_db_bridge_request - net_db_bridge_request
- net_rabbit_bridge_request - net_rabbit_bridge_request
bridge_shutdown:
extends:
file: docker-compose.yml
service: bridge_shutdown
networks:
- net_db_bridge_shutdown
networks: networks:
net_db_bridge_request: net_db_bridge_request:
@ -91,6 +97,8 @@ networks:
driver: bridge driver: bridge
net_db_bridge_senderforeign: net_db_bridge_senderforeign:
driver: bridge driver: bridge
net_db_bridge_shutdown:
driver: bridge
net_rabbit_bridge_request: net_rabbit_bridge_request:
driver: bridge driver: bridge
net_rabbit_bridge_collected: net_rabbit_bridge_collected:

@ -61,6 +61,12 @@ services:
networks: networks:
- net_db_bridge_request - net_db_bridge_request
- net_rabbit_bridge_request - net_rabbit_bridge_request
bridge_shutdown:
extends:
file: docker-compose.yml
service: bridge_shutdown
networks:
- net_db_bridge_shutdown
networks: networks:
net_db_bridge_request: net_db_bridge_request:
@ -75,6 +81,8 @@ networks:
driver: bridge driver: bridge
net_db_bridge_senderforeign: net_db_bridge_senderforeign:
driver: bridge driver: bridge
net_db_bridge_shutdown:
driver: bridge
net_rabbit_bridge_request: net_rabbit_bridge_request:
driver: bridge driver: bridge
net_rabbit_bridge_collected: net_rabbit_bridge_collected:

@ -27,6 +27,7 @@ services:
- net_db_bridge_affirmation - net_db_bridge_affirmation
- net_db_bridge_senderhome - net_db_bridge_senderhome
- net_db_bridge_senderforeign - net_db_bridge_senderforeign
- net_db_bridge_shutdown
restart: unless-stopped restart: unless-stopped
volumes: ['~/bridge_data/redis:/data'] volumes: ['~/bridge_data/redis:/data']
bridge_request: bridge_request:
@ -94,6 +95,17 @@ services:
networks: networks:
- net_db_bridge_senderforeign - net_db_bridge_senderforeign
- net_rabbit_bridge_senderforeign - net_rabbit_bridge_senderforeign
bridge_shutdown:
cpus: 0.1
mem_limit: 500m
image: poanetwork/tokenbridge-oracle:latest
env_file: ./.env
environment:
- NODE_ENV=production
restart: unless-stopped
entrypoint: yarn manager:shutdown
networks:
- net_db_bridge_shutdown
networks: networks:
net_db_bridge_request: net_db_bridge_request:
@ -106,6 +118,8 @@ networks:
driver: bridge driver: bridge
net_db_bridge_senderforeign: net_db_bridge_senderforeign:
driver: bridge driver: bridge
net_db_bridge_shutdown:
driver: bridge
net_rabbit_bridge_request: net_rabbit_bridge_request:
driver: bridge driver: bridge
net_rabbit_bridge_collected: net_rabbit_bridge_collected:

@ -13,6 +13,7 @@
"sender:home": "./scripts/start-worker.sh sender home-sender", "sender:home": "./scripts/start-worker.sh sender home-sender",
"sender:foreign": "./scripts/start-worker.sh sender foreign-sender", "sender:foreign": "./scripts/start-worker.sh sender foreign-sender",
"confirm:transfer": "./scripts/start-worker.sh confirmRelay transfer-watcher", "confirm:transfer": "./scripts/start-worker.sh confirmRelay transfer-watcher",
"manager:shutdown": "./scripts/start-worker.sh shutdownManager shutdown-manager",
"dev": "concurrently -n 'watcher:signature-request,watcher:collected-signatures,watcher:affirmation-request,watcher:transfer, sender:home,sender:foreign' -c 'red,green,yellow,blue,magenta,cyan' 'yarn watcher:signature-request' 'yarn watcher:collected-signatures' 'yarn watcher:affirmation-request' 'yarn watcher:transfer' 'yarn sender:home' 'yarn sender:foreign'", "dev": "concurrently -n 'watcher:signature-request,watcher:collected-signatures,watcher:affirmation-request,watcher:transfer, sender:home,sender:foreign' -c 'red,green,yellow,blue,magenta,cyan' 'yarn watcher:signature-request' 'yarn watcher:collected-signatures' 'yarn watcher:affirmation-request' 'yarn watcher:transfer' 'yarn sender:home' 'yarn sender:foreign'",
"test": "NODE_ENV=test mocha", "test": "NODE_ENV=test mocha",
"test:watch": "NODE_ENV=test mocha --watch --reporter=min", "test:watch": "NODE_ENV=test mocha --watch --reporter=min",

@ -4,6 +4,7 @@ const { connectSenderToQueue } = require('./services/amqpClient')
const { redis } = require('./services/redisClient') const { redis } = require('./services/redisClient')
const GasPrice = require('./services/gasPrice') const GasPrice = require('./services/gasPrice')
const logger = require('./services/logger') const logger = require('./services/logger')
const { getShutdownFlag } = require('./services/shutdownState')
const { sendTx } = require('./tx/sendTx') const { sendTx } = require('./tx/sendTx')
const { getNonce, getChainId } = require('./tx/web3') const { getNonce, getChainId } = require('./tx/web3')
const { const {
@ -102,6 +103,14 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT
return return
} }
const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
if (await getShutdownFlag(logger, config.shutdownKey, true)) {
if (!wasShutdown) {
logger.info('Oracle sender was suspended via the remote shutdown process')
}
return
}
const txArray = JSON.parse(msg.content) const txArray = JSON.parse(msg.content)
logger.debug(`Msg received with ${txArray.length} Tx to send`) logger.debug(`Msg received with ${txArray.length} Tx to send`)
const gasPrice = GasPrice.getPrice().toString(10) const gasPrice = GasPrice.getPrice().toString(10)

@ -3,6 +3,7 @@ const path = require('path')
const { const {
web3Home, web3Home,
web3Foreign, web3Foreign,
web3Side,
web3HomeFallback, web3HomeFallback,
web3ForeignFallback, web3ForeignFallback,
web3HomeRedundant, web3HomeRedundant,
@ -30,4 +31,8 @@ web3ForeignFallback.currentProvider.setLogger(logger)
web3HomeRedundant.currentProvider.setLogger(logger) web3HomeRedundant.currentProvider.setLogger(logger)
web3ForeignRedundant.currentProvider.setLogger(logger) web3ForeignRedundant.currentProvider.setLogger(logger)
if (web3Side) {
web3Side.currentProvider.setLogger(logger)
}
module.exports = logger module.exports = logger

@ -0,0 +1,23 @@
const { redis } = require('./redisClient')
let isShutdown = false
async function getShutdownFlag(logger, shutdownKey, force = false) {
if (force) {
logger.debug('Reading current shutdown state from the DB')
isShutdown = (await redis.get(shutdownKey)) === 'true'
logger.debug({ isShutdown }, 'Read shutdown state from the DB')
}
return isShutdown
}
async function setShutdownFlag(logger, shutdownKey, value) {
logger.info({ isShutdown: value }, 'Updating current shutdown state in the DB')
isShutdown = value
await redis.set(shutdownKey, value)
logger.debug('Updated state in the DB')
}
module.exports = {
getShutdownFlag,
setShutdownFlag
}

@ -6,6 +6,7 @@ const { RETRY_CONFIG } = require('../utils/constants')
const { const {
COMMON_HOME_RPC_URL, COMMON_HOME_RPC_URL,
COMMON_FOREIGN_RPC_URL, COMMON_FOREIGN_RPC_URL,
ORACLE_SIDE_RPC_URL,
ORACLE_RPC_REQUEST_TIMEOUT, ORACLE_RPC_REQUEST_TIMEOUT,
ORACLE_HOME_RPC_POLLING_INTERVAL, ORACLE_HOME_RPC_POLLING_INTERVAL,
ORACLE_FOREIGN_RPC_POLLING_INTERVAL ORACLE_FOREIGN_RPC_POLLING_INTERVAL
@ -41,6 +42,18 @@ const web3Home = new Web3(homeProvider)
const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions) const foreignProvider = new HttpListProvider(foreignUrls, foreignOptions)
const web3Foreign = new Web3(foreignProvider) const web3Foreign = new Web3(foreignProvider)
let web3Side = null
if (ORACLE_SIDE_RPC_URL) {
const sideUrls = ORACLE_SIDE_RPC_URL.split(' ').filter(url => url.length > 0)
const sideOptions = {
requestTimeout: configuredTimeout || 2000,
retry: RETRY_CONFIG
}
const sideProvider = new HttpListProvider(sideUrls, sideOptions)
web3Side = new Web3(sideProvider)
}
// secondary fallback providers are intended to be used in places where // secondary fallback providers are intended to be used in places where
// it is more likely that RPC calls to the local non-archive nodes can fail // it is more likely that RPC calls to the local non-archive nodes can fail
// e.g. for checking status of the old transaction via eth_getTransactionByHash // e.g. for checking status of the old transaction via eth_getTransactionByHash
@ -70,6 +83,7 @@ if (foreignUrls.length > 1) {
module.exports = { module.exports = {
web3Home, web3Home,
web3Foreign, web3Foreign,
web3Side,
web3HomeRedundant, web3HomeRedundant,
web3ForeignRedundant, web3ForeignRedundant,
web3HomeFallback, web3HomeFallback,

@ -0,0 +1,114 @@
const fetch = require('node-fetch')
const path = require('path')
const { EXIT_CODES } = require('./utils/constants')
const { watchdog } = require('./utils/utils')
const logger = require('./services/logger')
const { redis } = require('./services/redisClient')
const { web3Side } = require('./services/web3')
const { getShutdownFlag, setShutdownFlag } = require('./services/shutdownState')
if (process.argv.length < 3) {
logger.error('Please check the number of arguments, config file was not provided')
process.exit(EXIT_CODES.GENERAL_ERROR)
}
const config = require(path.join('../config/', process.argv[2]))
if (config.shutdownContractAddress && !web3Side) {
logger.error(
'ORACLE_SHUTDOWN_CONTRACT_ADDRESS was provided but not side chain provider was registered.' +
' Please, specify ORACLE_SIDE_RPC_URL as well.'
)
process.exit(EXIT_CODES.GENERAL_ERROR)
}
let shutdownCount = 0
let okCount = 0
async function fetchShutdownFlag() {
if (config.shutdownServiceURL) {
logger.debug({ url: config.shutdownServiceURL }, 'Fetching shutdown status from external URL')
const result = await fetch(config.shutdownServiceURL, {
headers: {
'Content-type': 'application/json'
},
method: 'GET',
timeout: config.requestTimeout
}).then(res => res.json())
if (result.shutdown === true) {
return true
}
}
if (config.shutdownContractAddress) {
const shutdownSelector = web3Side.eth.abi.encodeEventSignature(config.shutdownMethod)
logger.debug(
{ contract: config.shutdownContractAddress, method: config.shutdownMethod, data: shutdownSelector },
'Fetching shutdown status from contract'
)
const result = await web3Side.eth.call({
to: config.shutdownContractAddress,
data: shutdownSelector
})
logger.debug({ result }, 'Obtained result from the side RPC endpoint')
if (result.length > 2 && web3Side.eth.abi.decodeParameter('bool', result)) {
return true
}
}
return false
}
async function checkShutdownFlag() {
const isShutdownFlag = await fetchShutdownFlag()
const isShutdown = await getShutdownFlag(logger, config.shutdownKey)
if (isShutdownFlag === true && isShutdown === false) {
shutdownCount += 1
okCount = 0
logger.info(
{ shutdownCount, remainingChecks: config.checksBeforeStop - shutdownCount },
'Received positive shutdown flag'
)
} else if (isShutdownFlag === false && isShutdown === true) {
okCount += 1
shutdownCount = 0
logger.info({ okCount, remainingChecks: config.checksBeforeResume - okCount }, 'Received negative shutdown flag')
} else {
shutdownCount = 0
okCount = 0
logger.debug({ isShutdown, isShutdownFlag }, 'Received shutdown flag that is equal to the current state')
}
if (shutdownCount >= config.checksBeforeStop) {
await setShutdownFlag(logger, config.shutdownKey, true)
} else if (okCount >= config.checksBeforeResume) {
await setShutdownFlag(logger, config.shutdownKey, false)
}
}
async function initialize() {
logger.info('Starting shutdown flag watcher')
redis.on('connect', async () => {
await getShutdownFlag(logger, config.shutdownKey, true)
await main()
})
}
async function main() {
try {
await watchdog(checkShutdownFlag, config.maxProcessingTime, () => {
logger.fatal('Max processing time reached')
process.exit(EXIT_CODES.MAX_TIME_REACHED)
})
} catch (e) {
logger.error(e)
}
setTimeout(main, config.pollingInterval)
}
initialize()

@ -5,6 +5,7 @@ const { connectWatcherToQueue, connection } = require('./services/amqpClient')
const { getBlockNumber } = require('./tx/web3') const { getBlockNumber } = require('./tx/web3')
const { redis } = require('./services/redisClient') const { redis } = require('./services/redisClient')
const logger = require('./services/logger') const logger = require('./services/logger')
const { getShutdownFlag } = require('./services/shutdownState')
const { getRequiredBlockConfirmations, getEvents } = require('./tx/web3') const { getRequiredBlockConfirmations, getEvents } = require('./tx/web3')
const { checkHTTPS, watchdog } = require('./utils/utils') const { checkHTTPS, watchdog } = require('./utils/utils')
const { EXIT_CODES } = require('./utils/constants') const { EXIT_CODES } = require('./utils/constants')
@ -157,6 +158,14 @@ async function isWorkerNeeded() {
async function main({ sendToQueue, sendToWorker }) { async function main({ sendToQueue, sendToWorker }) {
try { try {
const wasShutdown = await getShutdownFlag(logger, config.shutdownKey, false)
if (await getShutdownFlag(logger, config.shutdownKey, true)) {
if (!wasShutdown) {
logger.info('Oracle watcher was suspended via the remote shutdown process')
}
return
}
await checkConditions() await checkConditions()
const lastBlockToProcess = await getLastBlockToProcess() const lastBlockToProcess = await getLastBlockToProcess()