Allow to store validator keystore file in the docker swarm secret

CONFIGURATION.md

@@ -53,6 +53,8 @@ ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain
 ORACLE_FOREIGN_RPC_BLOCK_POLLING_LIMIT | Max length for the block range used in `eth_getLogs` requests for polling contract events for the Foreign chain. Infinite, if not provided. | `integer`
 ORACLE_HOME_RPC_BLOCK_POLLING_LIMIT | Max length for the block range used in `eth_getLogs` requests for polling contract events for the Home chain. Infinite, if not provided. | `integer`
 ORACLE_JSONRPC_ERROR_CODES | Override default JSON rpc error codes that can trigger RPC fallback to the next URL from the list (or a retry in case of a single RPC URL). Default is `-32603,-32002,-32005`. Should be a comma-separated list of negative integers. | `string`
+ORACLE_VALIDATOR_KEYSTORE_PATH | Path to the keystore v3 json file with the encrypted validator key. | `string`  
+ORACLE_VALIDATOR_KEYSTORE_PASSWORD | Password from the provided keystore file, oracle won't startup properly, if the provided password is invalid | `string`
 
 
 ## Monitor configuration

deployment/hosts.yml.example

@@ -7,6 +7,13 @@ sokol-kovan:
           ansible_user: ubuntu
           ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
           #syslog_server_port: "udp://127.0.0.1:514"
+    oracle_swarm:
+      hosts:
+        127.0.0.1:
+          ansible_user: ubuntu
+          ORACLE_VALIDATOR_KEYSTORE_PATH: "/path/to/keystore.json"
+          ORACLE_VALIDATOR_KEYSTORE_PASSWORD: "12345678"
+          #syslog_server_port: "udp://127.0.0.1:514"
     monitor:
       hosts:
         127.0.0.1:

deployment/roles/common/files/daemon.json

@@ -1,4 +1,4 @@
 {
-    "live-restore": true,
+    "live-restore": false,
     "no-new-privileges": true
 }

deployment/roles/common/tasks/dependencies.yml

@@ -31,6 +31,7 @@
     owner: "root"
     group: "root"
     mode: "0755"
+  when: skip_compose is undefined
 
 - name: Upgrade pip version
   shell: pip3 install --upgrade pip==19.3.1
@@ -44,6 +45,7 @@
     comment: user to run docker-compose
     group: docker
     createhome: yes
+  when: skip_compose is undefined
 
 - name: Install auditd
   apt:

deployment/roles/oracle_swarm/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+bridge_path: "/home/{{ compose_service_user }}/bridge"
+bridge_data_path: "/home/{{ compose_service_user }}/bridge_data"
+ORACLE_ALLOW_HTTP_FOR_RPC: no
+ORACLE_QUEUE_URL: amqp://rabbit
+ORACLE_REDIS_URL: redis://redis
+keyfile_path: "/root/.key"
+keystore_path: "/root/.keystore.json"
+oracle_image: poanetwork/tokenbridge-oracle:latest

deployment/roles/oracle_swarm/files/modify_to_use_syslog.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+
+from yaml import safe_load, safe_dump
+from argparse import ArgumentParser
+from os.path import basename
+import sys
+
+parser = ArgumentParser()
+parser.add_argument('composefile', type=str, nargs=1, metavar='compose-file', help='docker-compose.yml')
+parser.add_argument('-d', action='store_true', help='output result instead of writing the file', dest='debug')
+
+if basename(sys.argv[0]) == "ipykernel_launcher.py":
+    args = parser.parse_args(['docker-compose.yml'])
+else:
+    args = parser.parse_args()
+    
+file_to_operate = args.composefile[0]
+
+with open(file_to_operate) as composefile:
+    composecnt=composefile.read()
+yml = safe_load(composecnt)
+for i in yml['services']:
+    yml['services'][i]['logging'] = {'driver': 'syslog','options': {'tag': '{{.Name}}/{{.ID}}'}}
+if args.debug or (basename(sys.argv[0]) == "ipykernel_launcher.py"):
+    print(safe_dump(yml))
+else:
+    with open(file_to_operate, 'w') as composefile:
+        safe_dump(yml, composefile, explicit_start=True)

deployment/roles/oracle_swarm/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common, skip_repo: true, skip_compose: true }

deployment/roles/oracle_swarm/tasks/jumpbox.yml

@@ -0,0 +1,6 @@
+---
+- name: Pull the containers images
+  shell: docker pull {{ oracle_image }}
+  args:
+    chdir: "{{ bridge_path }}/oracle"
+  when: skip_pull is undefined

deployment/roles/oracle_swarm/tasks/logging.yml

@@ -0,0 +1,47 @@
+---
+- include_tasks: logging_by_syslog.yml
+  with_items:
+    - docker-compose
+  loop_control:
+    loop_var: file
+
+- name: Set the oracle's containers local logs configuration file
+  template:
+    src: 31-oracle-docker.conf.j2
+    dest: /etc/rsyslog.d/31-oracle-docker.conf
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Set the redis container local logs configuration file
+  template:
+    src: 32-redis-docker.conf.j2
+    dest: /etc/rsyslog.d/32-redis-docker.conf
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Set the rabbit MQ container local logs configuration file
+  template:
+    src: 33-rabbit-docker.conf.j2
+    dest: /etc/rsyslog.d/33-rabbit-docker.conf
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Set the log configuration file to send container logs to remote server
+  template:
+    src: 36-oracle-remote-logging.conf.j2
+    dest: /etc/rsyslog.d/36-oracle-remote-logging.conf
+    owner: root
+    group: root
+    mode: 0644
+  when: syslog_server_port is defined
+
+- name: Discarding unwanted messages in rsyslog
+  blockinfile:
+    path: /etc/rsyslog.conf
+    insertbefore: "# Where to place spool and state files"
+    marker: "#{mark} add string to discarding unwanted messages"
+    content: ':msg, contains, "ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY" ~'
+  notify: restart rsyslog

deployment/roles/oracle_swarm/tasks/logging_by_syslog.yml

@@ -0,0 +1,5 @@
+---
+- name: Change logging facility to forward logs to syslog
+  script: modify_to_use_syslog.py "{{ bridge_path }}/oracle/{{ file }}.yml"
+  args:
+    executable: python3

deployment/roles/oracle_swarm/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- include_tasks: pre_config.yml
+- include_tasks: logging.yml
+- include_tasks: jumpbox.yml
+- include_tasks: post_config.yml
+- include_tasks: servinstall.yml

deployment/roles/oracle_swarm/tasks/post_config.yml

@@ -0,0 +1,41 @@
+---
+- name: Get blocks
+  become_user: "{{ compose_service_user }}"
+  shell: docker run --env-file .env --rm {{ oracle_image }} scripts/getValidatorStartBlocks.js
+  args:
+    chdir: "{{ bridge_path }}/oracle"
+  register: BLOCKS
+  when: (ORACLE_HOME_START_BLOCK is not defined) or (ORACLE_FOREIGN_START_BLOCK is not defined)
+
+- name: Write blocks
+  blockinfile:
+    path: "{{ bridge_path }}/oracle/.env"
+    marker: "## {mark} Calculated by scripts/getValidatorStartBlocks.js"
+    block: |
+      ORACLE_HOME_START_BLOCK={{ (BLOCKS.stdout | from_json).homeStartBlock }}
+      ORACLE_FOREIGN_START_BLOCK={{ (BLOCKS.stdout | from_json).foreignStartBlock }}
+  when: (ORACLE_HOME_START_BLOCK is not defined) or (ORACLE_FOREIGN_START_BLOCK is not defined)
+
+- name: Copy keystore file
+  copy:
+    src: "{{ ORACLE_VALIDATOR_KEYSTORE_PATH }}"
+    dest: "{{ keystore_path }}"
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Create swarm secret
+  shell: docker secret create oracle_keystore {{ keystore_path }}
+
+- name: Remove unencrypted keystore file
+  file:
+    path: "{{ keystore_path }}"
+    state: absent
+
+- name: Install .key config
+  template:
+    src: key.j2
+    dest: "{{ keyfile_path }}"
+    owner: root
+    group: root
+    mode: 0600

deployment/roles/oracle_swarm/tasks/pre_config.yml

@@ -0,0 +1,41 @@
+---
+- name: Leave swarm cluster
+  become_user: "{{ compose_service_user }}"
+  shell: docker swarm leave --force
+  ignore_errors: true
+
+- name: Init docker swarm
+  become_user: "{{ compose_service_user }}"
+  shell: docker swarm init --autolock
+  register: swarm_init
+
+- name: Print unlock token
+  debug: var=swarm_init.stdout_lines
+
+- name: Create oracle directory
+  file:
+    path: "{{ bridge_path }}/oracle"
+    state: directory
+    mode: '0755'
+
+- name: Create rabbitmq directory
+  file:
+    path: "{{ bridge_data_path }}/{{ item }}"
+    state: directory
+    mode: '0775'
+  loop:
+    - rabbitmq
+    - redis
+
+- name: Install .env config
+  template:
+    src: .env.j2
+    dest: "{{ bridge_path }}/oracle/.env"
+    owner: "{{ compose_service_user }}"
+    mode: '0640'
+
+- name: Install docker-compose file
+  template:
+    src: docker-compose.yml.j2
+    dest: "{{ bridge_path }}/oracle/docker-compose.yml"
+    mode: '0755'

deployment/roles/oracle_swarm/tasks/servinstall.yml

@@ -0,0 +1,19 @@
+# This role creates a poabridge service which is designed to manage docker-compose bridge deployment.
+# /etc/init.d/poabridge start, status, stop, restart - does what the services usually do in such cases.
+---
+- name: "Set poabridge service"
+  template:
+    src: poabridge.j2
+    dest: "/etc/init.d/poabridge"
+    owner: root
+    mode: 755
+
+- name: "Enable the service"
+  service:
+    name: "poabridge"
+    state: started
+    enabled: yes
+    use: service
+
+- name: Start the service
+  shell: service poabridge start

deployment/roles/oracle_swarm/templates/31-oracle-docker.conf.j2

@@ -0,0 +1,11 @@
+$FileCreateMode 0644
+template(name="DockerLogFileName_Oracle" type="list") {
+   constant(value="/var/log/docker/")
+   property(name="syslogtag" securepath="replace" regex.type="ERE" regex.submatch="1" regex.expression="bridge_(.*)\\/[a-zA-Z0-9]+\\[")
+   constant(value="/docker.log")
+}
+
+if $programname startswith 'oracle_bridge_' then \
+  ?DockerLogFileName_Oracle
+
+$FileCreateMode 0600

deployment/roles/oracle_swarm/templates/32-redis-docker.conf.j2

@@ -0,0 +1,11 @@
+$FileCreateMode 0644
+template(name="DockerLogFileName_Redis" type="list") {
+   constant(value="/var/log/docker/")
+   property(name="syslogtag" securepath="replace" regex.type="ERE" regex.submatch="1" regex.expression="oracle_(.*redis.*)\\/[a-zA-Z0-9]+\\[")
+   constant(value="/docker.log")
+}
+
+if $programname contains 'oracle' and $programname contains 'redis' then \
+  ?DockerLogFileName_Redis
+
+$FileCreateMode 0600

deployment/roles/oracle_swarm/templates/33-rabbit-docker.conf.j2

@@ -0,0 +1,11 @@
+$FileCreateMode 0644
+template(name="DockerLogFileName_Rabbit" type="list") {
+   constant(value="/var/log/docker/")
+   property(name="syslogtag" securepath="replace" regex.type="ERE" regex.submatch="1" regex.expression="oracle_(.*rabbit.*)\\/[a-zA-Z0-9]+\\[")
+   constant(value="/docker.log")
+}
+
+if $programname contains 'oracle' and $programname contains 'rabbit' then \
+  ?DockerLogFileName_Rabbit
+
+$FileCreateMode 0600

deployment/roles/oracle_swarm/templates/36-oracle-remote-logging.conf.j2

@@ -0,0 +1,15 @@
+if $programname startswith 'oracle_bridge_' then {
+action(
+   type="omfwd"
+   protocol="{{ syslog_server_port.split(":")[0] }}"
+   target="{{ (syslog_server_port.split(":")[1])[2:] }}"
+   port="{{ syslog_server_port.split(":")[2] }}"
+   template="RemoteForwardFormat"
+   queue.SpoolDirectory="/var/spool/rsyslog"
+   queue.FileName="remote"
+   queue.MaxDiskSpace="1g"
+   queue.SaveOnShutdown="on"
+   queue.Type="LinkedList"
+   ResendLastMSGOnReconnect="on"
+   )
+}

deployment/roles/oracle_swarm/templates/docker-compose.yml.j2

@@ -0,0 +1,130 @@
+version: '3.9'
+x-deploy: &x-deploy
+  resources:
+    limits:
+      cpus: '0.3'
+      memory: 500M
+    reservations:
+      memory: 100M
+x-keystore-access: &x-keystore-access
+  environment:
+    ORACLE_VALIDATOR_KEYSTORE_PATH: /run/secrets/oracle_keystore
+    ORACLE_VALIDATOR_KEYSTORE_PASSWORD:
+  secrets:
+    - oracle_keystore
+services:
+  rabbit:
+    image: rabbitmq:3
+    hostname: rabbit
+    deploy: *x-deploy
+    environment: [ 'RABBITMQ_NODENAME=node@rabbit' ]
+    networks:
+      - net_rabbit_bridge_request
+      - net_rabbit_bridge_collected
+      - net_rabbit_bridge_affirmation
+      - net_rabbit_bridge_senderhome
+      - net_rabbit_bridge_senderforeign
+    volumes: [ '{{ bridge_data_path }}/rabbitmq:/var/lib/rabbitmq/mnesia' ]
+  redis:
+    image: redis:4
+    hostname: redis
+    deploy: *x-deploy
+    command: [ redis-server, --appendonly, 'yes' ]
+    networks:
+      - net_db_bridge_request
+      - net_db_bridge_collected
+      - net_db_bridge_affirmation
+      - net_db_bridge_senderhome
+      - net_db_bridge_senderforeign
+      - net_db_bridge_shutdown
+    volumes: [ '{{ bridge_data_path }}/redis:/data' ]
+  bridge_request:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    <<: *x-keystore-access
+    entrypoint: yarn watcher:signature-request
+    networks:
+      - net_db_bridge_request
+      - net_rabbit_bridge_request
+  bridge_collected:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    entrypoint: yarn watcher:collected-signatures
+    networks:
+      - net_db_bridge_collected
+      - net_rabbit_bridge_collected
+  bridge_affirmation:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    entrypoint: yarn watcher:affirmation-request
+    networks:
+      - net_db_bridge_affirmation
+      - net_rabbit_bridge_affirmation
+  bridge_senderhome:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    <<: *x-keystore-access
+    entrypoint: yarn sender:home
+    networks:
+      - net_db_bridge_senderhome
+      - net_rabbit_bridge_senderhome
+  bridge_senderforeign:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    <<: *x-keystore-access
+    entrypoint: yarn sender:foreign
+    networks:
+      - net_db_bridge_senderforeign
+      - net_rabbit_bridge_senderforeign
+  bridge_shutdown:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    entrypoint: yarn manager:shutdown
+    networks:
+      - net_db_bridge_shutdown
+{% if ORACLE_BRIDGE_MODE == "ERC_TO_NATIVE" %}
+  bridge_transfer:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    entrypoint: yarn watcher:transfer
+    networks:
+      - net_db_bridge_transfer
+      - net_rabbit_bridge_transfer
+{% endif %}
+{% if ORACLE_BRIDGE_MODE == "ARBITRARY_MESSAGE" %}
+  bridge_information:
+    image: {{ oracle_image }}
+    deploy: *x-deploy
+    env_file: ./.env
+    entrypoint: yarn watcher:information-request
+    networks:
+      - net_db_bridge_information
+      - net_rabbit_bridge_information
+{% endif %}
+networks:
+  net_db_bridge_request:
+  net_db_bridge_collected:
+  net_db_bridge_affirmation:
+  net_db_bridge_senderhome:
+  net_db_bridge_senderforeign:
+  net_db_bridge_shutdown:
+  net_db_bridge_transfer:
+  net_db_bridge_information:
+  net_rabbit_bridge_request:
+  net_rabbit_bridge_collected:
+  net_rabbit_bridge_affirmation:
+  net_rabbit_bridge_senderhome:
+  net_rabbit_bridge_senderforeign:
+  net_rabbit_bridge_transfer:
+  net_rabbit_bridge_information:
+
+secrets:
+  oracle_keystore:
+    external: true

deployment/roles/oracle_swarm/templates/key.j2

@@ -0,0 +1,2 @@
+## Validator-specific options
+ORACLE_VALIDATOR_KEYSTORE_PASSWORD={{ ORACLE_VALIDATOR_KEYSTORE_PASSWORD }}

deployment/roles/oracle_swarm/templates/poabridge.j2

@@ -0,0 +1,66 @@
+#! /bin/bash
+
+### BEGIN INIT INFO
+# Provides:          poabridge
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start daemon at boot time
+# Description:       Enable service provided by daemon.
+### END INIT INFO
+
+WORKDIR="{{ '/home/' + compose_service_user | default('poadocker') + '/' + bridge_path + '/oracle' if bridge_path[:1] != "/" else bridge_path + '/oracle' }}"
+
+#Getting path to private key file and variable name for parsing key file
+source {{ keyfile_path }}
+
+start(){
+    echo "Starting bridge.."
+    cd $WORKDIR
+    sudo -u "{{ compose_service_user }}" docker stack rm oracle
+    sudo -u "{{ compose_service_user }}" "ORACLE_VALIDATOR_KEYSTORE_PASSWORD=$ORACLE_VALIDATOR_KEYSTORE_PASSWORD" docker stack deploy oracle -c docker-compose.yml
+}
+
+stop(){
+    echo "Stopping bridge.."
+    cd $WORKDIR
+    sudo -u "{{ compose_service_user }}" docker stack rm oracle
+    sleep 2
+}
+
+status(){
+    echo "Bridge status:"
+    cd $WORKDIR
+    sudo -u "{{ compose_service_user }}" docker service ls
+}
+
+
+case "$1" in
+
+  start)
+      start
+    ;;
+
+  stop)
+      stop
+    ;;
+
+  status)
+      status
+    ;;
+
+  restart)
+    echo "Restarting bridge.."
+    stop
+    start
+    ;;
+
+  *)
+    echo $"Usage: $0 {start|stop|restart|status}"
+    exit 1
+    ;;
+
+esac
+
+exit 0

deployment/site.yml

@@ -4,6 +4,11 @@
   become: true
   roles:
     - { role: oracle }
+- name: Install Oracle as a Docker Swarm service
+  hosts: oracle_swarm
+  become: true
+  roles:
+    - { role: oracle_swarm }
 - name: Install Monitor
   hosts: monitor
   become: true

oracle/config/base.config.js

@@ -8,13 +8,15 @@ const {
   FOREIGN_AMB_ABI
 } = require('../../commons')
 const { web3Home, web3Foreign } = require('../src/services/web3')
-const { add0xPrefix, privateKeyToAddress } = require('../src/utils/utils')
+const { add0xPrefix, privateKeyToAddress, loadKeystore } = require('../src/utils/utils')
 const { EXIT_CODES } = require('../src/utils/constants')
 
 const {
   ORACLE_BRIDGE_MODE,
   ORACLE_VALIDATOR_ADDRESS,
   ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY,
+  ORACLE_VALIDATOR_KEYSTORE_PATH,
+  ORACLE_VALIDATOR_KEYSTORE_PASSWORD,
   ORACLE_MAX_PROCESSING_TIME,
   COMMON_HOME_BRIDGE_ADDRESS,
   COMMON_FOREIGN_BRIDGE_ADDRESS,
@@ -81,6 +83,7 @@ const maxProcessingTime =
   parseInt(ORACLE_MAX_PROCESSING_TIME, 10) || 4 * Math.max(homeConfig.pollingInterval, foreignConfig.pollingInterval)
 
 let validatorPrivateKey
+let validatorAddress = ORACLE_VALIDATOR_ADDRESS
 if (ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY) {
   validatorPrivateKey = add0xPrefix(ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY)
   const derived = privateKeyToAddress(validatorPrivateKey)
@@ -90,12 +93,22 @@ if (ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY) {
     )
     process.exit(EXIT_CODES.INCOMPATIBILITY)
   }
+  validatorAddress = derived
+} else if (ORACLE_VALIDATOR_KEYSTORE_PATH) {
+  try {
+    const keystore = loadKeystore(ORACLE_VALIDATOR_KEYSTORE_PATH, ORACLE_VALIDATOR_KEYSTORE_PASSWORD)
+    validatorPrivateKey = keystore.privateKey
+    validatorAddress = keystore.address
+  } catch (e) {
+    console.error(`Can't load keystore file: ${e.message}`)
+    process.exit(EXIT_CODES.INCOMPATIBILITY)
+  }
 }
 
 module.exports = {
   eventFilter: {},
   validatorPrivateKey,
-  validatorAddress: ORACLE_VALIDATOR_ADDRESS || privateKeyToAddress(validatorPrivateKey),
+  validatorAddress,
   maxProcessingTime,
   shutdownKey: 'oracle-shutdown',
   home: homeConfig,

oracle/src/utils/utils.js

@@ -99,6 +99,11 @@ function privateKeyToAddress(privateKey) {
   return privateKey ? new Web3().eth.accounts.privateKeyToAccount(add0xPrefix(privateKey)).address : null
 }
 
+function loadKeystore(keystorePath, password) {
+  const keystore = JSON.parse(fs.readFileSync(keystorePath).toString())
+  return new Web3().eth.accounts.wallet.decrypt(keystore, password)[0]
+}
+
 function isGasPriceError(e) {
   const message = e.message.toLowerCase()
   return message.includes('replacement transaction underpriced')
@@ -195,5 +200,6 @@ module.exports = {
   getRetrySequence,
   promiseAny,
   readAccessListFile,
-  zipToObject
+  zipToObject,
+  loadKeystore
 }