From 3f27b98b8b96a54676e6ef2c82fa6f4357a7c0c3 Mon Sep 17 00:00:00 2001 From: Przemyslaw Rzad Date: Tue, 9 Jul 2019 19:26:40 +0200 Subject: [PATCH] Monitor deployment playbook (#133) * Introduced monitor deployment playbook * Timeouts * dai deployment block * wetc deployment blocks * Corrected path * Typo. * Added monitor to execution readme. * Monitor port. * Check on start * Timeouts * check-and-start --- deployment/CONFIGURATION.md | 6 +- deployment/EXECUTION.md | 1 + deployment/group_vars/dai.yml | 9 +++ deployment/group_vars/example.yml | 9 +++ deployment/group_vars/wetc.yml | 9 +++ deployment/hosts.yml.example | 4 + deployment/roles/monitor/meta/main.yml | 2 + deployment/roles/monitor/tasks/jumpbox.yml | 4 + deployment/roles/monitor/tasks/main.yml | 3 + deployment/roles/monitor/tasks/pre_config.yml | 4 + .../roles/monitor/tasks/servinstall.yml | 17 +++++ deployment/roles/monitor/templates/.env.j2 | 12 +++ .../monitor/templates/tokenbridge-monitor.j2 | 76 +++++++++++++++++++ deployment/site.yml | 5 ++ monitor/Dockerfile | 2 +- monitor/docker-compose.yml | 2 +- monitor/package.json | 3 +- 17 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 deployment/roles/monitor/meta/main.yml create mode 100644 deployment/roles/monitor/tasks/jumpbox.yml create mode 100644 deployment/roles/monitor/tasks/main.yml create mode 100644 deployment/roles/monitor/tasks/pre_config.yml create mode 100644 deployment/roles/monitor/tasks/servinstall.yml create mode 100644 deployment/roles/monitor/templates/.env.j2 create mode 100644 deployment/roles/monitor/templates/tokenbridge-monitor.j2 diff --git a/deployment/CONFIGURATION.md b/deployment/CONFIGURATION.md index 3ff6f74a..0852221d 100644 --- a/deployment/CONFIGURATION.md +++ b/deployment/CONFIGURATION.md @@ -42,9 +42,13 @@ cp hosts.yml.example hosts.yml : ansible_user: #syslog_server_port: "://:" + monitor: + hosts: + : + ansible_user: ``` -The config above would install the Oracle on ``, UI on ``, and both Oracle and UI on ``. +The config above would install the Oracle on ``, UI on ``, and both Oracle, UI and Monitor on ``. Example config for installing only UI: ```yaml diff --git a/deployment/EXECUTION.md b/deployment/EXECUTION.md index 4264185e..59796886 100644 --- a/deployment/EXECUTION.md +++ b/deployment/EXECUTION.md @@ -45,6 +45,7 @@ Component | Service Name --- | --- Oracle | poabridge UI | tokenbridge-ui +Monitor | tokenbridge-monitor Use the default `SysVinit` commands to `start`, `stop`, `restart`, and `rebuild` the service and to check the `status` of the service. diff --git a/deployment/group_vars/dai.yml b/deployment/group_vars/dai.yml index 5f05fc86..d5bbb31f 100644 --- a/deployment/group_vars/dai.yml +++ b/deployment/group_vars/dai.yml @@ -39,3 +39,12 @@ UI_HOME_EXPLORER_ADDRESS_TEMPLATE: https://blockscout.com/poa/dai/address/%s UI_FOREIGN_EXPLORER_ADDRESS_TEMPLATE: https://blockscout.com/eth/mainnet/address/%s UI_HOME_GAS_PRICE_FALLBACK: 1000000000 UI_FOREIGN_GAS_PRICE_FALLBACK: 10000000000 + +## Monitor +MONITOR_PORT: 3003 +MONITOR_HOME_DEPLOYMENT_BLOCK: 759 +MONITOR_FOREIGN_DEPLOYMENT_BLOCK: 6478417 +MONITOR_GAS_LIMIT: 300000 +MONITOR_GAS_PRICE_FALLBACK: 21 +MONITOR_LEFT_TX_THRESHOLD: 100 +MONITOR_CRON_SCHEDULE: "* * * * *" diff --git a/deployment/group_vars/example.yml b/deployment/group_vars/example.yml index deb33f69..2588a5c1 100644 --- a/deployment/group_vars/example.yml +++ b/deployment/group_vars/example.yml @@ -39,3 +39,12 @@ UI_HOME_EXPLORER_ADDRESS_TEMPLATE: https://blockscout.com/poa/sokol/address/%s UI_FOREIGN_EXPLORER_ADDRESS_TEMPLATE: https://blockscout.com/eth/kovan/address/%s UI_HOME_GAS_PRICE_FALLBACK: 1000000000 UI_FOREIGN_GAS_PRICE_FALLBACK: 1000000000 + +## Monitor +MONITOR_PORT: 3003 +MONITOR_HOME_DEPLOYMENT_BLOCK: 0 +MONITOR_FOREIGN_DEPLOYMENT_BLOCK: 0 +MONITOR_GAS_LIMIT: 300000 +MONITOR_GAS_PRICE_FALLBACK: 21 +MONITOR_LEFT_TX_THRESHOLD: 100 +MONITOR_CRON_SCHEDULE: "* * * * *" diff --git a/deployment/group_vars/wetc.yml b/deployment/group_vars/wetc.yml index 4b704737..ad1f3d3d 100644 --- a/deployment/group_vars/wetc.yml +++ b/deployment/group_vars/wetc.yml @@ -39,3 +39,12 @@ UI_HOME_EXPLORER_ADDRESS_TEMPLATE: https://blockscout.com/etc/mainnet/address/%s UI_FOREIGN_EXPLORER_ADDRESS_TEMPLATE: https://blockscout.com/eth/mainnet/address/%s UI_HOME_GAS_PRICE_FALLBACK: 15000000000 UI_FOREIGN_GAS_PRICE_FALLBACK: 10000000000 + +## Monitor +MONITOR_PORT: 3003 +MONITOR_HOME_DEPLOYMENT_BLOCK: 7703292 +MONITOR_FOREIGN_DEPLOYMENT_BLOCK: 7412459 +MONITOR_GAS_LIMIT: 300000 +MONITOR_GAS_PRICE_FALLBACK: 21 +MONITOR_LEFT_TX_THRESHOLD: 100 +MONITOR_CRON_SCHEDULE: "* * * * *" diff --git a/deployment/hosts.yml.example b/deployment/hosts.yml.example index 142d2d04..c8806cb0 100644 --- a/deployment/hosts.yml.example +++ b/deployment/hosts.yml.example @@ -11,3 +11,7 @@ sokol-kovan: 127.0.0.1: ansible_user: ubuntu #syslog_server_port: "udp://127.0.0.1:514" + monitor: + hosts: + 127.0.0.1: + ansible_user: ubuntu diff --git a/deployment/roles/monitor/meta/main.yml b/deployment/roles/monitor/meta/main.yml new file mode 100644 index 00000000..9711b330 --- /dev/null +++ b/deployment/roles/monitor/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - role: common diff --git a/deployment/roles/monitor/tasks/jumpbox.yml b/deployment/roles/monitor/tasks/jumpbox.yml new file mode 100644 index 00000000..0cd81157 --- /dev/null +++ b/deployment/roles/monitor/tasks/jumpbox.yml @@ -0,0 +1,4 @@ +- name: Launch container + shell: docker-compose up -d + args: + chdir: "{{ bridge_path }}/monitor" diff --git a/deployment/roles/monitor/tasks/main.yml b/deployment/roles/monitor/tasks/main.yml new file mode 100644 index 00000000..9606ad58 --- /dev/null +++ b/deployment/roles/monitor/tasks/main.yml @@ -0,0 +1,3 @@ +- include_tasks: pre_config.yml +- include_tasks: jumpbox.yml +- include_tasks: servinstall.yml diff --git a/deployment/roles/monitor/tasks/pre_config.yml b/deployment/roles/monitor/tasks/pre_config.yml new file mode 100644 index 00000000..1dfb250b --- /dev/null +++ b/deployment/roles/monitor/tasks/pre_config.yml @@ -0,0 +1,4 @@ +- name: Install .env config + template: + src: .env.j2 + dest: "{{ bridge_path }}/monitor/.env" diff --git a/deployment/roles/monitor/tasks/servinstall.yml b/deployment/roles/monitor/tasks/servinstall.yml new file mode 100644 index 00000000..4694d61a --- /dev/null +++ b/deployment/roles/monitor/tasks/servinstall.yml @@ -0,0 +1,17 @@ +# This role creates a tokenbridge-monitor service which is designed to manage docker-compose monitor deployment. +# /etc/init.d/tokenbridge-monitor start, status, stop, restart - does what the services usually do in such cases. +# /etc/init.d/tokenbridge-monitor rebuild - builds a new monitor deployment from scratch. +--- +- name: "Set the service" + template: + src: tokenbridge-monitor.j2 + dest: "/etc/init.d/tokenbridge-monitor" + owner: root + mode: 755 + +- name: "Start/Enable the service" + service: + name: "tokenbridge-monitor" + state: started + enabled: yes + use: service diff --git a/deployment/roles/monitor/templates/.env.j2 b/deployment/roles/monitor/templates/.env.j2 new file mode 100644 index 00000000..010c33e7 --- /dev/null +++ b/deployment/roles/monitor/templates/.env.j2 @@ -0,0 +1,12 @@ +HOME_RPC_URL={{ HOME_RPC_URL }} +FOREIGN_RPC_URL={{ FOREIGN_RPC_URL }} +HOME_BRIDGE_ADDRESS={{ HOME_BRIDGE_ADDRESS }} +FOREIGN_BRIDGE_ADDRESS={{ FOREIGN_BRIDGE_ADDRESS }} +HOME_DEPLOYMENT_BLOCK={{ MONITOR_HOME_DEPLOYMENT_BLOCK }} +FOREIGN_DEPLOYMENT_BLOCK={{ MONITOR_FOREIGN_DEPLOYMENT_BLOCK }} +GAS_PRICE_SPEED_TYPE={{ HOME_GAS_PRICE_SPEED_TYPE }} +GAS_LIMIT={{ MONITOR_GAS_LIMIT }} +GAS_PRICE_FALLBACK={{ MONITOR_GAS_PRICE_FALLBACK }} +LEFT_TX_THRESHOLD={{ MONITOR_LEFT_TX_THRESHOLD }} +PORT={{ MONITOR_PORT }} +CRON_SCHEDULE={{ MONITOR_CRON_SCHEDULE }} diff --git a/deployment/roles/monitor/templates/tokenbridge-monitor.j2 b/deployment/roles/monitor/templates/tokenbridge-monitor.j2 new file mode 100644 index 00000000..8f3fe8b2 --- /dev/null +++ b/deployment/roles/monitor/templates/tokenbridge-monitor.j2 @@ -0,0 +1,76 @@ +#! /bin/bash + +### BEGIN INIT INFO +# Provides: tokenbridge-monitor +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start daemon at boot time +# Description: Enable service provided by daemon. +### END INIT INFO + +WORKDIR="{{ '/home/' + compose_service_user | default('poadocker') + '/' + bridge_path + '/monitor' if bridge_path[:1] != "/" else bridge_path + '/monitor' }}" + +start(){ + echo "Starting TokenBridge Monitor.." + cd $WORKDIR + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose down -v + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose rm -fv + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose up --detach +} + +stop(){ + echo "Stopping TokenBridge Monitor.." + cd $WORKDIR + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose down -v + sleep 2 +} + +status(){ + echo "TokenBridge Monitor status:" + cd $WORKDIR + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose ps +} + +rebuild(){ + echo "Rebuild TokenBridge Monitor.." + cd $WORKDIR + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose down -v + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose rm -fv + sudo -u "{{ compose_service_user }}" /usr/local/bin/docker-compose up --detach --force-recreate --no-deps --build +} + + +case "$1" in + + start) + start + ;; + + stop) + stop + ;; + + status) + status + ;; + + restart) + echo "Restarting TokenBridge Monitor.." + stop + start + ;; + + rebuild) + rebuild + ;; + + *) + echo $"Usage: $0 {start|stop|restart|rebuild|status}" + exit 1 + ;; + +esac + +exit 0 diff --git a/deployment/site.yml b/deployment/site.yml index d8055746..6901b4ba 100644 --- a/deployment/site.yml +++ b/deployment/site.yml @@ -8,3 +8,8 @@ become: true roles: - { role: ui } +- name: Install Monitor + hosts: monitor + become: true + roles: + - { role: monitor } diff --git a/monitor/Dockerfile b/monitor/Dockerfile index 8c0505fb..23139bdc 100644 --- a/monitor/Dockerfile +++ b/monitor/Dockerfile @@ -14,4 +14,4 @@ RUN mv ./contracts/build ./ && rm -rf ./contracts/* ./contracts/.[!.]* && mv ./b COPY ./monitor ./monitor WORKDIR /mono/monitor CMD echo "To start the monitor run:" \ - "yarn start" + "yarn check-and-start" diff --git a/monitor/docker-compose.yml b/monitor/docker-compose.yml index 12c01c52..1c94da8d 100644 --- a/monitor/docker-compose.yml +++ b/monitor/docker-compose.yml @@ -10,4 +10,4 @@ services: environment: - NODE_ENV=production restart: unless-stopped - entrypoint: yarn start + entrypoint: "yarn check-and-start" diff --git a/monitor/package.json b/monitor/package.json index 833d0e9e..f3471340 100644 --- a/monitor/package.json +++ b/monitor/package.json @@ -4,8 +4,9 @@ "description": "", "main": "index.js", "scripts": { - "check-all": "node checkWorker.js && node checkWorker2.js", + "check-all": "timeout -s 9 5m node checkWorker.js && timeout -s 9 5m node checkWorker2.js", "start": "node index.js", + "check-and-start": "yarn check-all && yarn start", "lint": "eslint . --ignore-path ../.eslintignore", "lint:fix": "eslint . --fix" },