From 776bcd149f10d5c6775cca54197dd16a633e2c2f Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Tue, 24 Jan 2023 00:05:31 -0800 Subject: [PATCH] send panics to pagerduty --- Cargo.lock | 191 ++++++++++++++++++ web3_proxy/Cargo.toml | 2 + web3_proxy/src/bin/web3_proxy_cli/main.rs | 81 +++++++- .../src/bin/web3_proxy_cli/sentryd/mod.rs | 4 - 4 files changed, 272 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b131edf9..47dab722 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -948,6 +948,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.3" @@ -1834,6 +1844,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -2045,6 +2070,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a329e22866dd78b35d2c639a4a23d7b950aeae300dfd79f4fb19f74055c2404" +dependencies = [ + "libc", + "windows", +] + [[package]] name = "getrandom" version = "0.2.6" @@ -2344,6 +2379,19 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.46" @@ -2819,6 +2867,24 @@ dependencies = [ "getrandom", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -3031,6 +3097,51 @@ dependencies = [ "syn", ] +[[package]] +name = "openssl" +version = "0.10.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_info" version = "3.5.1" @@ -3078,6 +3189,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "pagerduty-rs" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd10bab2b6df910bbe6c4987d76aa4221235103d9a9c000cfabcee6a6abc8f7a" +dependencies = [ + "reqwest", + "serde", + "time 0.3.17", + "url", +] + [[package]] name = "parity-scale-codec" version = "3.1.2" @@ -3711,10 +3834,12 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -3724,6 +3849,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", + "tokio-native-tls", "tokio-rustls", "tower-service", "url", @@ -3943,6 +4069,15 @@ dependencies = [ "syn", ] +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + [[package]] name = "scheduled-thread-pool" version = "0.2.6" @@ -4154,6 +4289,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645926f31b250a2dca3c232496c2d898d91036e45ca0e97e0e2390c54e11be36" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.14" @@ -5018,6 +5176,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.23.4" @@ -5427,6 +5595,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -5583,6 +5757,7 @@ dependencies = [ "fdlimit", "flume", "futures", + "gethostname", "glob", "handlebars", "hashbrown 0.13.2", @@ -5597,6 +5772,7 @@ dependencies = [ "notify", "num", "num-traits", + "pagerduty-rs", "parking_lot 0.12.1", "proctitle", "redis-rate-limiter", @@ -5679,6 +5855,21 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + [[package]] name = "windows-sys" version = "0.36.1" diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml index df8b73c6..2f1089b9 100644 --- a/web3_proxy/Cargo.toml +++ b/web3_proxy/Cargo.toml @@ -39,6 +39,7 @@ ethers = { version = "1.0.2", default-features = false, features = ["rustls", "w fdlimit = "0.2.1" flume = "0.10.14" futures = { version = "0.3.25", features = ["thread-pool"] } +gethostname = "0.4.1" glob = "0.3.1" handlebars = "4.3.6" hashbrown = { version = "0.13.2", features = ["serde"] } @@ -52,6 +53,7 @@ moka = { version = "0.9.6", default-features = false, features = ["future"] } notify = "5.0.0" num = "0.4.0" num-traits = "0.2.15" +pagerduty-rs = { version = "0.1.6", features = ["async", "sync"] } parking_lot = { version = "0.12.1", features = ["arc_lock"] } proctitle = "0.1.1" regex = "1.7.1" diff --git a/web3_proxy/src/bin/web3_proxy_cli/main.rs b/web3_proxy/src/bin/web3_proxy_cli/main.rs index 08f80ff5..7a3a3ee9 100644 --- a/web3_proxy/src/bin/web3_proxy_cli/main.rs +++ b/web3_proxy/src/bin/web3_proxy_cli/main.rs @@ -8,6 +8,7 @@ mod create_user; mod daemon; mod drop_migration_lock; mod list_user_tier; +mod pagerduty; mod rpc_accounting; mod sentryd; mod transfer_key; @@ -17,9 +18,13 @@ mod user_import; use anyhow::Context; use argh::FromArgs; use ethers::types::U256; -use log::{info, warn}; +use gethostname::gethostname; +use log::{error, info, warn}; +use pagerduty_rs::eventsv2sync::EventsV2 as PagerdutySyncEventsV2; +use pagerduty_rs::types::{AlertTrigger, AlertTriggerPayload}; +use pagerduty_rs::{eventsv2async::EventsV2 as PagerdutyAsyncEventsV2, types::Event}; use std::{ - fs, + fs, panic, path::Path, sync::atomic::{self, AtomicUsize}, }; @@ -71,6 +76,7 @@ enum SubCommand { CountUsers(count_users::CountUsersSubCommand), CreateUser(create_user::CreateUserSubCommand), DropMigrationLock(drop_migration_lock::DropMigrationLockSubCommand), + Pagerduty(pagerduty::PagerdutySubCommand), Proxyd(daemon::ProxydSubCommand), RpcAccounting(rpc_accounting::RpcAccountingSubCommand), Sentryd(sentryd::SentrydSubCommand), @@ -191,6 +197,70 @@ fn main() -> anyhow::Result<()> { info!("{}", APP_USER_AGENT); + // optionally connect to pagerduty + // TODO: fix this nested result + let (pagerduty_async, pagerduty_sync) = if let Ok(pagerduty_key) = + std::env::var("PAGERDUTY_INTEGRATION_KEY") + { + let pagerduty_async = + PagerdutyAsyncEventsV2::new(pagerduty_key.clone(), Some(APP_USER_AGENT.to_string()))?; + let pagerduty_sync = + PagerdutySyncEventsV2::new(pagerduty_key, Some(APP_USER_AGENT.to_string()))?; + + (Some(pagerduty_async), Some(pagerduty_sync)) + } else { + info!("No PAGERDUTY_INTEGRATION_KEY"); + + (None, None) + }; + + // panic handler that sends to pagerduty + // TODO: there is a `pagerduty_panic` module that looks like it would work with minor tweaks, but ethers-rs panics when a websocket exit and that would fire too many alerts + + if let Some(pagerduty_sync) = pagerduty_sync { + let client = top_config + .as_ref() + .map(|top_config| format!("web3-proxy chain #{}", top_config.app.chain_id)) + .unwrap_or_else(|| format!("web3-proxy w/o chain")); + + let client_url = top_config + .as_ref() + .and_then(|x| x.app.redirect_public_url.clone()); + + panic::set_hook(Box::new(move |x| { + let hostname = gethostname().into_string().unwrap_or("unknown".to_string()); + let panic_msg = format!("{} {:?}", x, x); + + error!("sending panic to pagerduty: {}", panic_msg); + + let payload = AlertTriggerPayload { + severity: pagerduty_rs::types::Severity::Error, + summary: panic_msg.clone(), + source: hostname, + timestamp: None, + component: None, + group: Some("web3-proxy".to_string()), + class: Some("panic".to_string()), + custom_details: None::<()>, + }; + + let event = Event::AlertTrigger(AlertTrigger { + payload, + dedup_key: None, + images: None, + links: None, + client: Some(client.clone()), + client_url: client_url.clone(), + }); + + if let Err(err) = pagerduty_sync.event(event) { + error!("Failed sending panic to pagerduty: {}", err); + } + })); + } else { + info!("No pagerduty key. Using default panic handler"); + } + // set up tokio's async runtime let mut rt_builder = runtime::Builder::new_multi_thread(); @@ -286,6 +356,13 @@ fn main() -> anyhow::Result<()> { x.main(&db_conn).await } + SubCommand::Pagerduty(x) => { + if cli_config.sentry_url.is_none() { + warn!("sentry_url is not set! Logs will only show in this console"); + } + + x.main(pagerduty_async, top_config).await + } SubCommand::Sentryd(x) => { if cli_config.sentry_url.is_none() { warn!("sentry_url is not set! Logs will only show in this console"); diff --git a/web3_proxy/src/bin/web3_proxy_cli/sentryd/mod.rs b/web3_proxy/src/bin/web3_proxy_cli/sentryd/mod.rs index 51d0df7a..8199bfc3 100644 --- a/web3_proxy/src/bin/web3_proxy_cli/sentryd/mod.rs +++ b/web3_proxy/src/bin/web3_proxy_cli/sentryd/mod.rs @@ -13,10 +13,6 @@ use tokio::time::{interval, MissedTickBehavior}; /// Loop healthchecks and send pager duty alerts if any fail #[argh(subcommand, name = "sentryd")] pub struct SentrydSubCommand { - #[argh(positional)] - /// a descriptive name for this node (probably the hostname) - location: String, - #[argh(positional)] /// the main (HTTP only) web3-proxy being checked. web3_proxy: String,