Merge branch 'develop' into combined-bridge-oracle

This merge contains the following set of changes:
  * [Oracle, Improvement] Periodic check for RPC sync state (#656)

CONFIGURATION.md

@@ -56,9 +56,11 @@ ORACLE_JSONRPC_ERROR_CODES | Override default JSON rpc error codes that can trig
 ORACLE_HOME_EVENTS_REPROCESSING | If set to `true`, home events happened in the past will be refetched and processed once again, to ensure that nothing was missed on the first pass. | `bool`
 ORACLE_HOME_EVENTS_REPROCESSING_BATCH_SIZE | Batch size for one `eth_getLogs` request when reprocessing old logs in the home chain. Defaults to `1000` | `integer`
 ORACLE_HOME_EVENTS_REPROCESSING_BLOCK_DELAY | Block confirmations number, after which old logs are being reprocessed in the home chain. Defaults to `500` | `integer`
+ORACLE_HOME_RPC_SYNC_STATE_CHECK_INTERVAL | Interval for checking JSON RPC sync state, by requesting the latest block number. Oracle will switch to the fallback JSON RPC in case sync process is stuck | `integer`
 ORACLE_FOREIGN_EVENTS_REPROCESSING  | If set to `true`, foreign events happened in the past will be refetched and processed once again, to ensure that nothing was missed on the first pass. | `bool`
 ORACLE_FOREIGN_EVENTS_REPROCESSING_BATCH_SIZE | Batch size for one `eth_getLogs` request when reprocessing old logs in the foreign chain. Defaults to `500` | `integer`
 ORACLE_FOREIGN_EVENTS_REPROCESSING_BLOCK_DELAY | Block confirmations number, after which old logs are being reprocessed in the foreign chain. Defaults to `250` | `integer`
+ORACLE_FOREIGN_RPC_SYNC_STATE_CHECK_INTERVAL | Interval for checking JSON RPC sync state, by requesting the latest block number. Oracle will switch to the fallback JSON RPC in case sync process is stuck | `integer`
 
 
 ## Monitor configuration

deployment/hosts.yml.example

@@ -5,7 +5,8 @@ sokol-kovan:
       hosts:
         127.0.0.1:
           ansible_user: ubuntu
-          ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+          TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+          AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
           #syslog_server_port: "udp://127.0.0.1:514"
     monitor:
       hosts:

deployment/roles/oracle/defaults/main.yml

@@ -1,6 +1,3 @@
 ---
 bridge_path: "/home/{{ compose_service_user }}/bridge"
-ORACLE_ALLOW_HTTP_FOR_RPC: no
-ORACLE_QUEUE_URL: amqp://rabbit
-ORACLE_REDIS_URL: redis://redis
 keyfile_path: "/root/.key"

deployment/roles/oracle/tasks/logging.yml

@@ -2,8 +2,6 @@
 - include_tasks: logging_by_syslog.yml
   with_items:
     - docker-compose
-    - docker-compose-transfer
-    - docker-compose-amb
   loop_control:
     loop_var: file
 

deployment/roles/oracle/tasks/post_config.yml

@@ -1,39 +1,59 @@
 ---
-- name: Get blocks
+- name: Get blocks for TokenBridge
   become_user: "{{ compose_service_user }}"
   shell: docker-compose run --rm --entrypoint "node scripts/getValidatorStartBlocks.js" bridge_affirmation
   args:
     chdir: "{{ bridge_path }}/oracle"
-  register: BLOCKS
+  register: TBBLOCKS
-  when: (ORACLE_HOME_START_BLOCK is not defined) or (ORACLE_FOREIGN_START_BLOCK is not defined)
+  when: (TB_HOME_START_BLOCK is not defined) or (TB_FOREIGN_START_BLOCK is not defined)
 
-- name: Write blocks
+- name: Get blocks for AMB
+  become_user: "{{ compose_service_user }}"
+  shell: docker-compose run --rm --entrypoint "node scripts/getValidatorStartBlocks.js" bridge_amb_affirmation
+  args:
+    chdir: "{{ bridge_path }}/oracle"
+  register: AMBBLOCKS
+  when: (AMB_HOME_START_BLOCK is not defined) or (AMB_FOREIGN_START_BLOCK is not defined)
+
+- name: Write blocks for TokenBridge
   blockinfile:
     path: "{{ bridge_path }}/oracle/.env"
     marker: "## {mark} Calculated by scripts/getValidatorStartBlocks.js"
     block: |
-      ORACLE_HOME_START_BLOCK={{ (BLOCKS.stdout | from_json).homeStartBlock }}
+      TB_HOME_START_BLOCK={{ (TBBLOCKS.stdout | from_json).homeStartBlock }}
-      ORACLE_FOREIGN_START_BLOCK={{ (BLOCKS.stdout | from_json).foreignStartBlock }}
+      TB_FOREIGN_START_BLOCK={{ (TBBLOCKS.stdout | from_json).foreignStartBlock }}
-  when: (ORACLE_HOME_START_BLOCK is not defined) or (ORACLE_FOREIGN_START_BLOCK is not defined)
+  when: (TB_HOME_START_BLOCK is not defined) or (TB_FOREIGN_START_BLOCK is not defined)
 
-- name: Get validator address
+- name: Write blocks for AMB
+  blockinfile:
+    path: "{{ bridge_path }}/oracle/.env"
+    marker: "## {mark} Calculated by scripts/getValidatorStartBlocks.js"
+    block: |
+      AMB_HOME_START_BLOCK={{ (AMBBLOCKS.stdout | from_json).homeStartBlock }}
+      AMB_FOREIGN_START_BLOCK={{ (AMBBLOCKS.stdout | from_json).foreignStartBlock }}
+  when: (AMB_HOME_START_BLOCK is not defined) or (AMB_FOREIGN_START_BLOCK is not defined)
+
+- name: Get validator address for TokenBridge
   become_user: "{{ compose_service_user }}"
-  shell: docker-compose run --rm -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY="{{ ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY }}" --entrypoint "node scripts/privateKeyToAddress.js" bridge_affirmation
+  shell: docker-compose run --rm -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY="{{ TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY }}" --entrypoint "node scripts/privateKeyToAddress.js" bridge_affirmation
   args:
     chdir: "{{ bridge_path }}/oracle"
-  register: VADDRESS
+  register: TBVADDRESS
 
-- name: Set ORACLE_VALIDATOR_ADDRESS variable
+- name: Set TB_ORACLE_VALIDATOR_ADDRESS variable
   set_fact:
-    ORACLE_VALIDATOR_ADDRESS: "{{ VADDRESS.stdout }}"
+    TB_ORACLE_VALIDATOR_ADDRESS: "{{ TBVADDRESS.stdout }}"
 
-- name: Extend docker compose file for erc to native
+- name: Get validator address for AMB
-  set_fact: composefileoverride="-f docker-compose-transfer.yml"
+  become_user: "{{ compose_service_user }}"
-  when: ORACLE_BRIDGE_MODE == "ERC_TO_NATIVE"
+  shell: docker-compose run --rm -e ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY="{{ AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY }}" --entrypoint "node scripts/privateKeyToAddress.js" bridge_affirmation
+  args:
+    chdir: "{{ bridge_path }}/oracle"
+  register: AMBVADDRESS
 
-- name: Extend docker compose file for amb
+- name: Set AMB_ORACLE_VALIDATOR_ADDRESS variable
-  set_fact: composefileoverride="-f docker-compose-amb.yml"
+  set_fact:
-  when: ORACLE_BRIDGE_MODE == "ARBITRARY_MESSAGE"
+    AMB_ORACLE_VALIDATOR_ADDRESS: "{{ AMBVADDRESS.stdout }}"
 
 - name: Install .key config
   template:

deployment/roles/oracle/tasks/pre_config.yml

@@ -12,12 +12,8 @@
     owner: "{{ compose_service_user }}"
     mode: '0640'
 
-- name: Copy docker-compose files
+- name: Copy docker-compose file
   copy:
-    src: ../../../../oracle/{{ item }}
+    src: ../../../../oracle/docker-compose-combined.yml
-    dest: "{{ bridge_path }}/oracle/"
+    dest: "{{ bridge_path }}/oracle/docker-compose.yml"
     mode: '0755'
-  with_items:
-    - docker-compose.yml
-    - docker-compose-transfer.yml
-    - docker-compose-amb.yml

deployment/roles/oracle/templates/.env.j2

@@ -1,21 +1,28 @@
 ## General settings
-ORACLE_BRIDGE_MODE={{ ORACLE_BRIDGE_MODE }}
 {% if ORACLE_LOG_LEVEL | default('') != '' %}
 ORACLE_LOG_LEVEL={{ ORACLE_LOG_LEVEL }}
 {% endif %}
 
 ## Home contract
 COMMON_HOME_RPC_URL={{ COMMON_HOME_RPC_URL }}
-COMMON_HOME_BRIDGE_ADDRESS={{ COMMON_HOME_BRIDGE_ADDRESS }}
+TB_HOME_BRIDGE_ADDRESS={{ TB_HOME_BRIDGE_ADDRESS }}
+AMB_HOME_BRIDGE_ADDRESS={{ AMB_HOME_BRIDGE_ADDRESS }}
 ORACLE_HOME_RPC_POLLING_INTERVAL={{ ORACLE_HOME_RPC_POLLING_INTERVAL }}
+{% if ORACLE_HOME_RPC_BLOCK_POLLING_LIMIT | default('') != '' %}
+ORACLE_HOME_RPC_BLOCK_POLLING_LIMIT={{ ORACLE_HOME_RPC_BLOCK_POLLING_LIMIT }}
+{% endif %}
 
 ## Foreign contract
 COMMON_FOREIGN_RPC_URL={{ COMMON_FOREIGN_RPC_URL }}
 {% if ORACLE_FOREIGN_ARCHIVE_RPC_URL | default('') != '' %}
 ORACLE_FOREIGN_ARCHIVE_RPC_URL={{ ORACLE_FOREIGN_ARCHIVE_RPC_URL }}
 {% endif %}
-COMMON_FOREIGN_BRIDGE_ADDRESS={{ COMMON_FOREIGN_BRIDGE_ADDRESS }}
+TB_FOREIGN_BRIDGE_ADDRESS={{ TB_FOREIGN_BRIDGE_ADDRESS }}
+AMB_FOREIGN_BRIDGE_ADDRESS={{ AMB_FOREIGN_BRIDGE_ADDRESS }}
 ORACLE_FOREIGN_RPC_POLLING_INTERVAL={{ ORACLE_FOREIGN_RPC_POLLING_INTERVAL }}
+{% if ORACLE_FOREIGN_RPC_BLOCK_POLLING_LIMIT | default('') != '' %}
+ORACLE_FOREIGN_RPC_BLOCK_POLLING_LIMIT={{ ORACLE_FOREIGN_RPC_BLOCK_POLLING_LIMIT }}
+{% endif %}
 
 {% if ORACLE_TX_REDUNDANCY | default('') != '' %}
 ORACLE_TX_REDUNDANCY={{ ORACLE_TX_REDUNDANCY }}
@@ -52,8 +59,10 @@ COMMON_FOREIGN_GAS_PRICE_FACTOR={{ COMMON_FOREIGN_GAS_PRICE_FACTOR }}
 
 ## Transport configuration
 ORACLE_ALLOW_HTTP_FOR_RPC={{ "yes" if ORACLE_ALLOW_HTTP_FOR_RPC else "no" }}
-ORACLE_QUEUE_URL={{ ORACLE_QUEUE_URL }}
+TB_QUEUE_URL={{ TB_QUEUE_URL }}
-ORACLE_REDIS_URL={{ ORACLE_REDIS_URL }}
+TB_REDIS_URL={{ TB_REDIS_URL }}
+AMB_QUEUE_URL={{ AMB_QUEUE_URL }}
+AMB_REDIS_URL={{ AMB_REDIS_URL }}
 {% if ORACLE_FOREIGN_TX_RESEND_INTERVAL | default('') != '' %}
 ORACLE_FOREIGN_TX_RESEND_INTERVAL={{ ORACLE_FOREIGN_TX_RESEND_INTERVAL }}
 {% endif %}
@@ -61,26 +70,50 @@ ORACLE_FOREIGN_TX_RESEND_INTERVAL={{ ORACLE_FOREIGN_TX_RESEND_INTERVAL }}
 ORACLE_HOME_TX_RESEND_INTERVAL={{ ORACLE_HOME_TX_RESEND_INTERVAL }}
 {% endif %}
 
-## Emergency shutdown configuration
+## Emergency shutdown configuration for TokenBridge
-{% if ORACLE_SHUTDOWN_SERVICE_URL | default('') != '' %}
+{% if TB_SHUTDOWN_SERVICE_URL | default('') != '' %}
-ORACLE_SHUTDOWN_SERVICE_URL={{ ORACLE_SHUTDOWN_SERVICE_URL }}
+TB_SHUTDOWN_SERVICE_URL={{ TB_SHUTDOWN_SERVICE_URL }}
 {% endif %}
-{% if ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL | default('') != '' %}
+{% if TB_SHUTDOWN_SERVICE_POLLING_INTERVAL | default('') != '' %}
-ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL={{ ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL }}
+TB_SHUTDOWN_SERVICE_POLLING_INTERVAL={{ TB_SHUTDOWN_SERVICE_POLLING_INTERVAL }}
 {% endif %}
-{% if ORACLE_SIDE_RPC_URL | default('') != '' %}
+{% if TB_SIDE_RPC_URL | default('') != '' %}
-ORACLE_SIDE_RPC_URL={{ ORACLE_SIDE_RPC_URL }}
+TB_SIDE_RPC_URL={{ TB_SIDE_RPC_URL }}
 {% endif %}
-{% if ORACLE_SHUTDOWN_CONTRACT_ADDRESS | default('') != '' %}
+{% if TB_SHUTDOWN_CONTRACT_ADDRESS | default('') != '' %}
-ORACLE_SHUTDOWN_CONTRACT_ADDRESS={{ ORACLE_SHUTDOWN_CONTRACT_ADDRESS }}
+TB_SHUTDOWN_CONTRACT_ADDRESS={{ TB_SHUTDOWN_CONTRACT_ADDRESS }}
 {% endif %}
-{% if ORACLE_SHUTDOWN_CONTRACT_METHOD | default('') != '' %}
+{% if TB_SHUTDOWN_CONTRACT_METHOD | default('') != '' %}
-ORACLE_SHUTDOWN_CONTRACT_METHOD={{ ORACLE_SHUTDOWN_CONTRACT_METHOD }}
+TB_SHUTDOWN_CONTRACT_METHOD={{ TB_SHUTDOWN_CONTRACT_METHOD }}
 {% endif %}
 
-{% if ORACLE_HOME_START_BLOCK | default('') != '' %}
+## Emergency shutdown configuration for AMB
-ORACLE_HOME_START_BLOCK={{ ORACLE_HOME_START_BLOCK }}
+{% if AMB_SHUTDOWN_SERVICE_URL | default('') != '' %}
+AMB_SHUTDOWN_SERVICE_URL={{ AMB_SHUTDOWN_SERVICE_URL }}
 {% endif %}
-{% if ORACLE_FOREIGN_START_BLOCK | default('') != '' %}
+{% if AMB_SHUTDOWN_SERVICE_POLLING_INTERVAL | default('') != '' %}
-ORACLE_FOREIGN_START_BLOCK={{ ORACLE_FOREIGN_START_BLOCK }}
+AMB_SHUTDOWN_SERVICE_POLLING_INTERVAL={{ AMB_SHUTDOWN_SERVICE_POLLING_INTERVAL }}
 {% endif %}
+{% if AMB_SIDE_RPC_URL | default('') != '' %}
+AMB_SIDE_RPC_URL={{ AMB_SIDE_RPC_URL }}
+{% endif %}
+{% if AMB_SHUTDOWN_CONTRACT_ADDRESS | default('') != '' %}
+AMB_SHUTDOWN_CONTRACT_ADDRESS={{ AMB_SHUTDOWN_CONTRACT_ADDRESS }}
+{% endif %}
+{% if AMB_SHUTDOWN_CONTRACT_METHOD | default('') != '' %}
+AMB_SHUTDOWN_CONTRACT_METHOD={{ AMB_SHUTDOWN_CONTRACT_METHOD }}
+{% endif %}
+
+{% if TB_HOME_START_BLOCK | default('') != '' %}
+TB_HOME_START_BLOCK={{ TB_HOME_START_BLOCK }}
+{% endif %}
+{% if TB_FOREIGN_START_BLOCK | default('') != '' %}
+TB_FOREIGN_START_BLOCK={{ TB_FOREIGN_START_BLOCK }}
+{% endif %}
+
+{% if AMB_HOME_START_BLOCK | default('') != '' %}
+AMB_HOME_START_BLOCK={{ AMB_HOME_START_BLOCK }}
+{% endif %}
+{% if AMB_FOREIGN_START_BLOCK | default('') != '' %}
+AMB_FOREIGN_START_BLOCK={{ AMB_FOREIGN_START_BLOCK }}
+{% endif %}

deployment/roles/oracle/templates/key.j2

@@ -1,3 +1,6 @@
 ## Validator-specific options                                                                                                                                                                                     
-ORACLE_VALIDATOR_ADDRESS={{ ORACLE_VALIDATOR_ADDRESS }} 
+TB_ORACLE_VALIDATOR_ADDRESS={{ TB_ORACLE_VALIDATOR_ADDRESS }} 
-ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY={{ ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY }} 
+TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY={{ TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY }} 
+
+AMB_ORACLE_VALIDATOR_ADDRESS={{ AMB_ORACLE_VALIDATOR_ADDRESS }} 
+AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY={{ AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY }} 

deployment/roles/oracle/templates/poabridge.j2

@@ -14,19 +14,26 @@ WORKDIR="{{ '/home/' + compose_service_user | default('poadocker') + '/' + bridg
 
 #Getting path to private key file and variable name for parsing key file
 keyfile="{{ keyfile_path }}"
-vaddr="ORACLE_VALIDATOR_ADDRESS="
+tbvaddr="TB_ORACLE_VALIDATOR_ADDRESS="
-vkey="ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY="
+tbvkey="TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY="
-composefileoverride="{{ composefileoverride | default('') }}"
+ambvaddr="AMB_ORACLE_VALIDATOR_ADDRESS="
+ambvkey="AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY="
 
 #Parsing file content and add key to variable
 while read -r line
 do
     case $line in
-        $vaddr*)
+        $tbvaddr*)
-            vaddr=${line#$vaddr}
+            tbvaddr=${line#$tbvaddr}
             ;;
-        $vkey*)
+        $tbvkey*)
-            vkey=${line#$vkey}
+            tbvkey=${line#$tbvkey}
+            ;;
+        $ambvaddr*)
+            ambvaddr=${line#$ambvaddr}
+            ;;
+        $ambvkey*)
+            ambvkey=${line#$ambvkey}
             ;;
     esac
 done < $keyfile
@@ -34,22 +41,29 @@ done < $keyfile
 start(){
     echo "Starting bridge.."
     cd $WORKDIR
-    sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose $composefileoverride down -v
+    sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose down -v
-    sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose $composefileoverride rm -fv
+    sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose rm -fv
-    sudo -u "{{ compose_service_user }}" ORACLE_VALIDATOR_ADDRESS=$vaddr ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=$vkey /usr/local/bin/docker-compose $composefileoverride up --detach
+    sudo -u "{{ compose_service_user }}" \
+      TB_ORACLE_VALIDATOR_ADDRESS=$tbvaddr \
+      TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=$tbvkey \
+      AMB_ORACLE_VALIDATOR_ADDRESS=$ambvaddr \
+      AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=$ambvkey \
+      /usr/local/bin/docker-compose up --detach
 }
 
 stop(){
     echo "Stopping bridge.."
     cd $WORKDIR
-    sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose $composefileoverride  down -v
+    sudo -u "{{ compose_service_user }}" \
+      /usr/local/bin/docker-compose down -v
     sleep 2
 }
 
 status(){
     echo "Bridge status:"
     cd $WORKDIR
-    sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose $composefileoverride  ps
+    sudo -u "{{ compose_service_user }}" \
+      /usr/local/bin/docker-compose ps
 }
 
 

oracle/config/base.config.js

@@ -7,7 +7,15 @@ const {
   HOME_AMB_ABI,
   FOREIGN_AMB_ABI
 } = require('../../commons')
-const { web3Home, web3Foreign } = require('../src/services/web3')
+const {
+  web3Home,
+  web3Foreign,
+  web3HomeRedundant,
+  web3HomeFallback,
+  web3ForeignRedundant,
+  web3ForeignFallback,
+  web3ForeignArchive
+} = require('../src/services/web3')
 const { add0xPrefix, privateKeyToAddress } = require('../src/utils/utils')
 const { EXIT_CODES } = require('../src/utils/constants')
 
@@ -27,9 +35,11 @@ const {
   ORACLE_HOME_EVENTS_REPROCESSING,
   ORACLE_HOME_EVENTS_REPROCESSING_BATCH_SIZE,
   ORACLE_HOME_EVENTS_REPROCESSING_BLOCK_DELAY,
+  ORACLE_HOME_RPC_SYNC_STATE_CHECK_INTERVAL,
   ORACLE_FOREIGN_EVENTS_REPROCESSING,
   ORACLE_FOREIGN_EVENTS_REPROCESSING_BATCH_SIZE,
-  ORACLE_FOREIGN_EVENTS_REPROCESSING_BLOCK_DELAY
+  ORACLE_FOREIGN_EVENTS_REPROCESSING_BLOCK_DELAY,
+  ORACLE_FOREIGN_RPC_SYNC_STATE_CHECK_INTERVAL
 } = process.env
 
 let homeAbi
@@ -63,9 +73,12 @@ const homeConfig = {
   bridgeAddress: COMMON_HOME_BRIDGE_ADDRESS,
   bridgeABI: homeAbi,
   pollingInterval: parseInt(ORACLE_HOME_RPC_POLLING_INTERVAL, 10),
+  syncCheckInterval: parseInt(ORACLE_HOME_RPC_SYNC_STATE_CHECK_INTERVAL, 10) || 60000,
   startBlock: parseInt(ORACLE_HOME_START_BLOCK, 10) || 0,
   blockPollingLimit: parseInt(ORACLE_HOME_RPC_BLOCK_POLLING_LIMIT, 10),
   web3: web3Home,
+  web3Redundant: web3HomeRedundant,
+  web3Fallback: web3HomeFallback,
   bridgeContract: homeContract,
   eventContract: homeContract,
   reprocessingOptions: {
@@ -81,9 +94,13 @@ const foreignConfig = {
   bridgeAddress: COMMON_FOREIGN_BRIDGE_ADDRESS,
   bridgeABI: foreignAbi,
   pollingInterval: parseInt(ORACLE_FOREIGN_RPC_POLLING_INTERVAL, 10),
+  syncCheckInterval: parseInt(ORACLE_FOREIGN_RPC_SYNC_STATE_CHECK_INTERVAL, 10) || 60000,
   startBlock: parseInt(ORACLE_FOREIGN_START_BLOCK, 10) || 0,
   blockPollingLimit: parseInt(ORACLE_FOREIGN_RPC_BLOCK_POLLING_LIMIT, 10),
   web3: web3Foreign,
+  web3Redundant: web3ForeignRedundant,
+  web3Fallback: web3ForeignFallback,
+  web3Archive: web3ForeignArchive || web3Foreign,
   bridgeContract: foreignContract,
   eventContract: foreignContract,
   reprocessingOptions: {

oracle/config/foreign-sender.config.js

@@ -1,17 +1,14 @@
 const baseConfig = require('./base.config')
 
 const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants')
-const { web3Foreign, web3ForeignRedundant, web3ForeignFallback } = require('../src/services/web3')
 
 const { ORACLE_FOREIGN_TX_RESEND_INTERVAL } = process.env
 
 module.exports = {
   ...baseConfig,
+  main: baseConfig.foreign,
   queue: 'foreign-prioritized',
   id: 'foreign',
   name: 'sender-foreign',
-  web3: web3Foreign,
-  web3Redundant: web3ForeignRedundant,
-  web3Fallback: web3ForeignFallback,
   resendInterval: parseInt(ORACLE_FOREIGN_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL
 }

oracle/config/home-sender.config.js

@@ -1,17 +1,14 @@
 const baseConfig = require('./base.config')
 
 const { DEFAULT_TRANSACTION_RESEND_INTERVAL } = require('../src/utils/constants')
-const { web3Home, web3HomeRedundant, web3HomeFallback } = require('../src/services/web3')
 
 const { ORACLE_HOME_TX_RESEND_INTERVAL } = process.env
 
 module.exports = {
   ...baseConfig,
+  main: baseConfig.home,
   queue: 'home-prioritized',
   id: 'home',
   name: 'sender-home',
-  web3: web3Home,
-  web3Redundant: web3HomeRedundant,
-  web3Fallback: web3HomeFallback,
   resendInterval: parseInt(ORACLE_HOME_TX_RESEND_INTERVAL, 10) || DEFAULT_TRANSACTION_RESEND_INTERVAL
 }

oracle/config/information-request-watcher.config.js

@@ -1,11 +1,9 @@
 const baseConfig = require('./base.config')
-const { web3ForeignArchive } = require('../src/services/web3')
 
 const id = `${baseConfig.id}-information-request`
 
 module.exports = {
   ...baseConfig,
-  web3ForeignArchive: web3ForeignArchive || baseConfig.foreign.web3,
   main: baseConfig.home,
   event: 'UserRequestForInformation',
   sender: 'home',

oracle/docker-compose-combined.yml

@@ -0,0 +1,334 @@
+---
+version: '2.4'
+networks:
+  net_db_bridge_green: {driver: bridge}
+  net_db_bridge_red: {driver: bridge}
+  net_rabbit_bridge_green: {driver: bridge}
+  net_rabbit_bridge_red: {driver: bridge}
+  net_ne_bridge_red: {driver: bridge}
+  net_ne_bridge_green: {driver: bridge}
+  net_db_bridge_amb_green: {driver: bridge}
+  net_db_bridge_amb_red: {driver: bridge}
+  net_rabbit_bridge_amb_green: {driver: bridge}
+  net_rabbit_bridge_amb_red: {driver: bridge}
+  net_ne_bridge_amb_red: {driver: bridge}
+  net_ne_bridge_amb_green: {driver: bridge}
+services:
+  bridge_affirmation:
+    cpus: 0.1
+    entrypoint: yarn watcher:affirmation-request
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ERC_TO_NATIVE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${TB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${TB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${TB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${TB_REDIS_URL}',
+                  'ORACLE_FOREIGN_START_BLOCK=${TB_FOREIGN_START_BLOCK}',
+                  'ORACLE_VALIDATOR_ADDRESS=${TB_ORACLE_VALIDATOR_ADDRESS}']
+    image: poanetwork/tokenbridge-oracle:latest
+    mem_limit: 500m
+    networks: [net_db_bridge_green, 
+               net_rabbit_bridge_green,
+               net_ne_bridge_green]
+    restart: unless-stopped
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    depends_on:
+      - redis
+      - rabbit  
+  bridge_shutdown:
+    cpus: 0.1
+    mem_limit: 500m
+    image: poanetwork/tokenbridge-oracle:latest
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ERC_TO_NATIVE,
+                  'ORACLE_REDIS_URL=${TB_REDIS_URL}',
+                  'ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL=${TB_SHUTDOWN_SERVICE_POLLING_INTERVAL}',
+                  'ORACLE_SIDE_RPC_URL=${TB_SIDE_RPC_URL}',
+                  'ORACLE_SHUTDOWN_CONTRACT_ADDRESS=${TB_SHUTDOWN_CONTRACT_ADDRESS}',
+                  'ORACLE_SHUTDOWN_CONTRACT_METHOD=${TB_SHUTDOWN_CONTRACT_METHOD}']
+    restart: unless-stopped
+    entrypoint: yarn manager:shutdown
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    networks: [net_db_bridge_green]
+    depends_on:
+      - redis
+  bridge_request:
+    cpus: 0.1
+    entrypoint: yarn watcher:signature-request
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ERC_TO_NATIVE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${TB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${TB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${TB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${TB_REDIS_URL}',
+                  'ORACLE_HOME_START_BLOCK=${TB_HOME_START_BLOCK}', 
+                  'ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}']
+    image: poanetwork/tokenbridge-oracle:latest
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_db_bridge_red, 
+               net_rabbit_bridge_red,
+               net_ne_bridge_red]
+    restart: unless-stopped
+    depends_on:
+      - redis
+      - rabbit  
+  bridge_senderhome:
+    cpus: 0.1
+    entrypoint: yarn sender:home
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ERC_TO_NATIVE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${TB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${TB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${TB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${TB_REDIS_URL}',
+                  'ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${TB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}']
+    image: poanetwork/tokenbridge-oracle:latest
+    mem_limit: 500m
+    networks: [net_db_bridge_red,
+               net_rabbit_bridge_red,
+               net_ne_bridge_red]
+    restart: unless-stopped
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}    
+    depends_on:
+      - redis
+      - rabbit  
+  bridge_transfer:
+    cpus: 0.1
+    entrypoint: yarn watcher:transfer
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ERC_TO_NATIVE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${TB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${TB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${TB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${TB_REDIS_URL}',
+                  'ORACLE_FOREIGN_START_BLOCK=${TB_FOREIGN_START_BLOCK}',
+                  'ORACLE_VALIDATOR_ADDRESS=${TB_ORACLE_VALIDATOR_ADDRESS}']
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    image: poanetwork/tokenbridge-oracle:latest
+    mem_limit: 500m
+    networks: [net_db_bridge_green, 
+               net_rabbit_bridge_green,
+               net_ne_bridge_green]
+    restart: unless-stopped
+    depends_on:
+      - redis
+      - rabbit  
+  rabbit:
+    cpus: 0.3
+    environment: [RABBITMQ_NODENAME=node@rabbit]
+    hostname: rabbit
+    image: rabbitmq:3
+    mem_limit: 500m
+    networks: [net_rabbit_bridge_red,
+               net_rabbit_bridge_green]
+    restart: unless-stopped
+    volumes: ['~/bridge_data/rabbitmq:/var/lib/rabbitmq/mnesia']
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+  redis:
+    command: [redis-server, --appendonly, 'yes']
+    cpus: 0.1
+    hostname: redis
+    image: redis:4
+    mem_limit: 500m
+    networks: [net_db_bridge_red,
+               net_db_bridge_green]
+    restart: unless-stopped
+    volumes: ['~/bridge_data/redis:/data']
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+
+  ne:
+    container_name: ne
+    image: nethermind/nethermind:latest
+    networks: [net_ne_bridge_green,
+               net_ne_bridge_red,
+               net_ne_bridge_amb_green,
+               net_ne_bridge_amb_red]
+    command:
+      --config xdai
+      --baseDbPath /nethermind
+      --JsonRpc.Enabled true
+      --JsonRpc.Host 0.0.0.0
+      --Init.StoreReceipts false
+      --TxPool.Size 8192
+      --Pruning.Mode Hybrid
+      --Pruning.CacheMb 1024
+      --Sync.FastSync true
+      --Sync.FastBlocks true
+      --Sync.DownloadBodiesInFastSync false
+      --Sync.DownloadReceiptsInFastSync false
+      --Sync.DownloadHeadersInFastSync false
+    volumes:
+      - ~/ne-data/logs:/nethermind/logs
+      - ~/ne-data/nethermind_db:/nethermind/nethermind_db
+    expose:
+      - "8545"
+    ports:
+      - "30304:30303/tcp"
+      - "30304:30303/udp"
+    restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "1"
+  bridge_amb_affirmation:
+    cpus: 0.1
+    entrypoint: yarn watcher:affirmation-request
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ARBITRARY_MESSAGE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${AMB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${AMB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${AMB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${AMB_REDIS_URL}',
+                  'ORACLE_FOREIGN_START_BLOCK=${AMB_FOREIGN_START_BLOCK}',
+                  'ORACLE_VALIDATOR_ADDRESS=${AMB_ORACLE_VALIDATOR_ADDRESS}']
+    image: poanetwork/tokenbridge-oracle:latest
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_db_bridge_amb_green, 
+               net_rabbit_bridge_amb_green,
+               net_ne_bridge_amb_green]
+    restart: unless-stopped
+    depends_on:
+      - redis-amb
+      - rabbit-amb
+  bridge_amb_information:
+    cpus: 0.1
+    entrypoint: yarn watcher:information-request
+    env_file: ./.env
+    environment: [NODE_ENV=production, 
+                  ORACLE_BRIDGE_MODE=ARBITRARY_MESSAGE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${AMB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${AMB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${AMB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${AMB_REDIS_URL}',
+                  'ORACLE_HOME_START_BLOCK=${AMB_HOME_START_BLOCK}',
+                  'ORACLE_VALIDATOR_ADDRESS=${AMB_ORACLE_VALIDATOR_ADDRESS}']
+    image: poanetwork/tokenbridge-oracle:latest
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_db_bridge_amb_green,
+               net_rabbit_bridge_amb_green,
+               net_ne_bridge_amb_green]
+    restart: unless-stopped
+    depends_on:
+      - redis-amb
+      - rabbit-amb
+  bridge_amb_request:
+    cpus: 0.1
+    entrypoint: yarn watcher:signature-request
+    env_file: ./.env
+    environment: [NODE_ENV=production, 
+                  ORACLE_BRIDGE_MODE=ARBITRARY_MESSAGE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${AMB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${AMB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${AMB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${AMB_REDIS_URL}', 
+                  'ORACLE_HOME_START_BLOCK=${AMB_HOME_START_BLOCK}',
+                  'ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}']
+    image: poanetwork/tokenbridge-oracle:latest
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_db_bridge_amb_red, 
+               net_rabbit_bridge_amb_red,
+               net_ne_bridge_amb_red]
+    restart: unless-stopped
+    depends_on:
+      - redis-amb
+      - rabbit-amb
+  bridge_amb_shutdown:
+    cpus: 0.1
+    entrypoint: yarn manager:shutdown
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ARBITRARY_MESSAGE,
+                  'ORACLE_REDIS_URL=${AMB_REDIS_URL}',
+                  'ORACLE_SHUTDOWN_SERVICE_POLLING_INTERVAL=${AMB_SHUTDOWN_SERVICE_POLLING_INTERVAL}',
+                  'ORACLE_SIDE_RPC_URL=${AMB_SIDE_RPC_URL}',
+                  'ORACLE_SHUTDOWN_CONTRACT_ADDRESS=${AMB_SHUTDOWN_CONTRACT_ADDRESS}',
+                  'ORACLE_SHUTDOWN_CONTRACT_METHOD=${AMB_SHUTDOWN_CONTRACT_METHOD}']
+    image: poanetwork/tokenbridge-oracle:latest
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_db_bridge_amb_green]
+    restart: unless-stopped
+    depends_on:
+      - redis-amb
+  bridge_amb_senderhome:
+    cpus: 0.1
+    entrypoint: yarn sender:home
+    env_file: ./.env
+    environment: [NODE_ENV=production,
+                  ORACLE_BRIDGE_MODE=ARBITRARY_MESSAGE,
+                  'COMMON_HOME_BRIDGE_ADDRESS=${AMB_HOME_BRIDGE_ADDRESS}',
+                  'COMMON_FOREIGN_BRIDGE_ADDRESS=${AMB_FOREIGN_BRIDGE_ADDRESS}',
+                  'ORACLE_QUEUE_URL=${AMB_QUEUE_URL}',
+                  'ORACLE_REDIS_URL=${AMB_REDIS_URL}',
+                  'ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY=${AMB_ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY}']
+    image: poanetwork/tokenbridge-oracle:latest
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_db_bridge_amb_red, 
+               net_rabbit_bridge_amb_red,
+               net_ne_bridge_amb_red]
+    restart: unless-stopped
+    depends_on:
+      - redis-amb
+      - rabbit-amb
+  rabbit-amb:
+    cpus: 0.3
+    environment: [RABBITMQ_NODENAME=node@rabbit-amb]
+    hostname: rabbit-amb
+    image: rabbitmq:3
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_rabbit_bridge_amb_red,
+               net_rabbit_bridge_amb_green]
+    restart: unless-stopped
+    volumes: ['~/amb_bridge_data/rabbitmq:/var/lib/rabbitmq/mnesia']
+  redis-amb:
+    command: [redis-server, --appendonly, 'yes']
+    cpus: 0.1
+    hostname: redis-amb
+    image: redis:4
+    logging:
+      driver: syslog
+      options: {tag: '{{.Name}}/{{.ID}}'}
+    mem_limit: 500m
+    networks: [net_db_bridge_amb_red, 
+               net_db_bridge_amb_green]
+    restart: unless-stopped
+    volumes: ['~/amb_bridge_data/redis:/data']

oracle/src/events/processAMBInformationRequests/index.js

@@ -35,11 +35,13 @@ Object.keys(asyncCalls).forEach(method => {
 })
 
 function processInformationRequestsBuilder(config) {
-  const { home, foreign, web3ForeignArchive } = config
+  const { home, foreign } = config
 
   let validatorContract = null
   let blockFinder = null
 
+  foreign.web3Archive.currentProvider.startSyncStateChecker(foreign.syncCheckInterval)
+
   return async function processInformationRequests(informationRequests) {
     const txToSend = []
 
@@ -49,13 +51,15 @@ function processInformationRequestsBuilder(config) {
 
     if (blockFinder === null) {
       rootLogger.debug('Initializing block finder')
-      blockFinder = await makeBlockFinder('foreign', foreign.web3)
+      blockFinder = await makeBlockFinder('foreign', foreign.web3Archive)
     }
 
+    // latest foreign block is requested from an archive RPC, to ensure that it is synced with the network
+    // block confirmations can be requested from the regular JSON RPC
     const foreignBlockNumber =
-      (await getBlockNumber(foreign.web3)) - (await getRequiredBlockConfirmations(foreign.bridgeContract))
+      (await getBlockNumber(foreign.web3Archive)) - (await getRequiredBlockConfirmations(foreign.bridgeContract))
     const homeBlock = await getBlock(home.web3, informationRequests[0].blockNumber)
-    const lastForeignBlock = await getBlock(foreign.web3, foreignBlockNumber)
+    const lastForeignBlock = await getBlock(foreign.web3Archive, foreignBlockNumber)
 
     if (homeBlock.timestamp > lastForeignBlock.timestamp) {
       rootLogger.debug(
@@ -85,7 +89,7 @@ function processInformationRequestsBuilder(config) {
         logger.info({ requestSelector, method: asyncCallMethod, data }, 'Processing async request')
 
         const call = asyncCalls[asyncCallMethod]
-        let [status, result] = await call(web3ForeignArchive, data, foreignClosestBlock).catch(e => {
+        let [status, result] = await call(foreign.web3Archive, data, foreignClosestBlock).catch(e => {
           if (e instanceof HttpListProviderError) {
             throw e
           }

oracle/src/sender.js

@@ -32,8 +32,8 @@ if (process.argv.length < 3) {
 
 const config = require(path.join('../config/', process.argv[2]))
 
-const { web3, web3Fallback } = config
+const { web3, web3Fallback, syncCheckInterval } = config.main
-const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.web3Redundant : web3
+const web3Redundant = ORACLE_TX_REDUNDANCY === 'true' ? config.main.web3Redundant : web3
 
 const nonceKey = `${config.id}:nonce`
 let chainId = 0
@@ -43,6 +43,7 @@ async function initialize() {
     const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger)
 
     web3.currentProvider.urls.forEach(checkHttps(config.id))
+    web3.currentProvider.startSyncStateChecker(syncCheckInterval)
 
     GasPrice.start(config.id, web3)
 

oracle/src/services/HttpListProvider.js

@@ -1,5 +1,6 @@
 const fetch = require('node-fetch')
 const promiseRetry = require('promise-retry')
+const { utils } = require('web3')
 const { FALLBACK_RPC_URL_SWITCH_TIMEOUT } = require('../utils/constants')
 
 const { onInjected } = require('./injectedLogger')
@@ -39,19 +40,54 @@ function HttpListProvider(urls, options = {}) {
   this.options = { ...defaultOptions, ...options }
   this.currentIndex = 0
   this.lastTimeUsedPrimary = 0
+  this.latestBlock = 0
+  this.syncStateCheckerIntervalId = 0
 
   onInjected(logger => {
     this.logger = logger.child({ module: `HttpListProvider:${this.options.name}` })
   })
 }
 
-HttpListProvider.prototype.switchToFallbackRPC = function() {
+HttpListProvider.prototype.startSyncStateChecker = function(syncCheckInterval) {
-  if (this.urls.length < 2) {
+  if (this.urls.length > 1 && syncCheckInterval > 0 && this.syncStateCheckerIntervalId === 0) {
+    this.syncStateCheckerIntervalId = setInterval(this.checkLatestBlock.bind(this), syncCheckInterval)
+  }
+}
+
+HttpListProvider.prototype.checkLatestBlock = function() {
+  const payload = { jsonrpc: '2.0', id: 1, method: 'eth_blockNumber', params: [] }
+  this.send(payload, (error, result) => {
+    if (error) {
+      this.logger.warn({ oldBlock: this.latestBlock }, 'Failed to request latest block from all RPC urls')
+    } else if (result.error) {
+      this.logger.warn(
+        { oldBlock: this.latestBlock, error: result.error.message },
+        'Failed to make eth_blockNumber request due to unknown error, switching to fallback RPC'
+      )
+      this.switchToFallbackRPC()
+    } else {
+      const blockNumber = utils.hexToNumber(result.result)
+      if (blockNumber > this.latestBlock) {
+        this.logger.debug({ oldBlock: this.latestBlock, newBlock: blockNumber }, 'Updating latest block number')
+        this.latestBlock = blockNumber
+      } else {
+        this.logger.warn(
+          { oldBlock: this.latestBlock, newBlock: blockNumber },
+          'Latest block on the node was not updated since last request, switching to fallback RPC'
+        )
+        this.switchToFallbackRPC()
+      }
+    }
+  })
+}
+
+HttpListProvider.prototype.switchToFallbackRPC = function(index) {
+  const prevIndex = this.currentIndex
+  const newIndex = index || (prevIndex + 1) % this.urls.length
+  if (this.urls.length < 2 || prevIndex === newIndex) {
     return
   }
 
-  const prevIndex = this.currentIndex
-  const newIndex = (prevIndex + 1) % this.urls.length
   this.logger.info(
     { index: newIndex, oldURL: this.urls[prevIndex], newURL: this.urls[newIndex] },
     'Switching to fallback JSON-RPC URL'
@@ -80,11 +116,7 @@ HttpListProvider.prototype.send = async function send(payload, callback) {
 
     // if some of URLs failed to respond, current URL index is updated to the first URL that responded
     if (currentIndex !== index) {
-      this.logger.info(
+      this.switchToFallbackRPC(index)
-        { index, oldURL: this.urls[currentIndex], newURL: this.urls[index] },
-        'Switching to fallback JSON-RPC URL'
-      )
-      this.currentIndex = index
     }
     callback(null, result)
   } catch (e) {

oracle/src/utils/constants.js

@@ -27,7 +27,6 @@ module.exports = {
   MIN_GAS_PRICE_BUMP_FACTOR: 0.1,
   DEFAULT_TRANSACTION_RESEND_INTERVAL: 20 * 60 * 1000,
   FALLBACK_RPC_URL_SWITCH_TIMEOUT: 60 * 60 * 1000,
-  BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT: 10,
   SENDER_QUEUE_MAX_PRIORITY: 10,
   SENDER_QUEUE_SEND_PRIORITY: 5,
   SENDER_QUEUE_CHECK_STATUS_PRIORITY: 1,

oracle/src/watcher.js

@@ -6,11 +6,7 @@ const logger = require('./services/logger')
 const { getShutdownFlag } = require('./services/shutdownState')
 const { getBlockNumber, getRequiredBlockConfirmations, getEvents } = require('./tx/web3')
 const { checkHTTPS, watchdog } = require('./utils/utils')
-const {
+const { EXIT_CODES, MAX_HISTORY_BLOCK_TO_REPROCESS } = require('./utils/constants')
-  EXIT_CODES,
-  BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT,
-  MAX_HISTORY_BLOCK_TO_REPROCESS
-} = require('./utils/constants')
 
 if (process.argv.length < 3) {
   logger.error('Please check the number of arguments, config file was not provided')
@@ -38,21 +34,21 @@ const {
   pollingInterval,
   chain,
   reprocessingOptions,
-  blockPollingLimit
+  blockPollingLimit,
+  syncCheckInterval
 } = config.main
 const lastBlockRedisKey = `${config.id}:lastProcessedBlock`
 const lastReprocessedBlockRedisKey = `${config.id}:lastReprocessedBlock`
 const seenEventsRedisKey = `${config.id}:seenEvents`
 let lastProcessedBlock = Math.max(startBlock - 1, 0)
 let lastReprocessedBlock
-let lastSeenBlockNumber = 0
-let sameBlockNumberCounter = 0
 
 async function initialize() {
   try {
     const checkHttps = checkHTTPS(process.env.ORACLE_ALLOW_HTTP_FOR_RPC, logger)
 
     web3.currentProvider.urls.forEach(checkHttps(chain))
+    web3.currentProvider.startSyncStateChecker(syncCheckInterval)
 
     await getLastProcessedBlock()
     await getLastReprocessedBlock()
@@ -225,28 +221,6 @@ async function getLastBlockToProcess(web3, bridgeContract) {
     getBlockNumber(web3),
     getRequiredBlockConfirmations(bridgeContract)
   ])
-
-  if (lastBlockNumber < lastSeenBlockNumber) {
-    sameBlockNumberCounter = 0
-    logger.warn({ lastBlockNumber, lastSeenBlockNumber }, 'Received block number less than already seen block')
-    web3.currentProvider.switchToFallbackRPC()
-  } else if (lastBlockNumber === lastSeenBlockNumber) {
-    sameBlockNumberCounter++
-    if (sameBlockNumberCounter > 1) {
-      logger.info({ lastBlockNumber, sameBlockNumberCounter }, 'Received the same block number more than twice')
-      if (sameBlockNumberCounter >= BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT) {
-        sameBlockNumberCounter = 0
-        logger.warn(
-          { lastBlockNumber, n: BLOCK_NUMBER_PROGRESS_ITERATIONS_LIMIT },
-          'Received the same block number for too many times. Probably node is not synced anymore'
-        )
-        web3.currentProvider.switchToFallbackRPC()
-      }
-    }
-  } else {
-    sameBlockNumberCounter = 0
-    lastSeenBlockNumber = lastBlockNumber
-  }
   return lastBlockNumber - requiredBlockConfirmations
 }