web3-proxy/web3_proxy/src/pagerduty.rs

203 lines
5.6 KiB
Rust
Raw Normal View History

use crate::config::TopConfig;
use pagerduty_rs::eventsv2sync::EventsV2 as PagerdutySyncEventsV2;
use pagerduty_rs::types::{AlertTrigger, AlertTriggerPayload, Event};
use serde::Serialize;
2023-03-08 00:44:27 +03:00
use std::backtrace::Backtrace;
2023-01-25 11:27:07 +03:00
use std::{
collections::hash_map::DefaultHasher,
hash::{Hash, Hasher},
panic::PanicInfo,
2023-01-25 11:27:07 +03:00
};
2023-01-24 14:12:36 +03:00
use time::OffsetDateTime;
use tracing::{debug, error, warn};
2023-01-24 14:12:36 +03:00
/*
let client = top_config
.as_ref()
.map(|top_config| format!("web3-proxy chain #{}", top_config.app.chain_id))
.unwrap_or_else(|| format!("web3-proxy w/o chain"));
let client_url = top_config
.as_ref()
.and_then(|x| x.app.redirect_public_url.clone());
panic::set_hook(Box::new(move |x| {
let hostname = hostname.get().into_string().unwrap_or("unknown".to_string());
let panic_msg = format!("{} {:?}", x, x);
if panic_msg.starts_with("panicked at 'WS Server panic") {
info!("Underlying library {}", panic_msg);
} else {
error!("sending panic to pagerduty: {}", panic_msg);
let mut s = DefaultHasher::new();
panic_msg.hash(&mut s);
panic_msg.hash(&mut s);
let dedup_key = s.finish().to_string();
let payload = AlertTriggerPayload {
severity: pagerduty_rs::types::Severity::Error,
summary: panic_msg,
source: hostname,
timestamp: None,
component: None,
group: Some("web3-proxy".to_string()),
class: Some("panic".to_string()),
custom_details: None::<()>,
};
let event = Event::AlertTrigger(AlertTrigger {
payload,
dedup_key: Some(dedup_key),
images: None,
links: None,
client: Some(client.clone()),
client_url: client_url.clone(),
});
if let Err(err) = pagerduty_sync.event(event) {
error!("Failed sending panic to pagerduty: {}", err);
}
}
}));
*/
pub fn panic_handler(
top_config: Option<TopConfig>,
pagerduty_sync: &PagerdutySyncEventsV2,
panic_info: &PanicInfo,
) {
let summary = format!("{}", panic_info);
2023-03-08 00:44:27 +03:00
let backtrace = Backtrace::force_capture();
2023-03-08 00:46:47 +03:00
// TODO: try to send to sentry and then put the sentry link into the page
2023-03-08 00:44:27 +03:00
let details = format!("{:#?}\n{:#?}", panic_info, backtrace);
if summary.starts_with("panicked at 'WS Server panic") {
// the ethers-rs library panics when websockets disconnect. this isn't a panic we care about reporting
debug!("Underlying library {}", details);
return;
}
let class = Some("panic".to_string());
let alert = if let Some(top_config) = top_config {
pagerduty_alert_for_config(
class,
None,
Some(details),
pagerduty_rs::types::Severity::Critical,
summary,
None,
top_config,
)
} else {
pagerduty_alert(
None,
class,
None,
None,
None,
Some(details),
pagerduty_rs::types::Severity::Critical,
None,
summary,
None,
)
};
let event = Event::AlertTrigger(alert);
if let Err(err) = pagerduty_sync.event(event) {
2023-06-29 07:30:00 +03:00
error!(?err, "Failed sending alert to pagerduty!");
}
}
pub fn pagerduty_alert_for_config<T: Serialize>(
2023-01-24 14:12:36 +03:00
class: Option<String>,
component: Option<String>,
custom_details: Option<T>,
severity: pagerduty_rs::types::Severity,
summary: String,
timestamp: Option<OffsetDateTime>,
top_config: TopConfig,
) -> AlertTrigger<T> {
let chain_id = top_config.app.chain_id;
let client_url = top_config.app.redirect_public_url;
2023-01-24 14:12:36 +03:00
pagerduty_alert(
Some(chain_id),
class,
None,
2023-01-24 14:12:36 +03:00
client_url,
component,
custom_details,
severity,
None,
summary,
timestamp,
)
}
#[allow(clippy::too_many_arguments)]
2023-01-24 14:12:36 +03:00
pub fn pagerduty_alert<T: Serialize>(
chain_id: Option<u64>,
class: Option<String>,
client: Option<String>,
2023-01-24 14:12:36 +03:00
client_url: Option<String>,
component: Option<String>,
custom_details: Option<T>,
severity: pagerduty_rs::types::Severity,
source: Option<String>,
summary: String,
timestamp: Option<OffsetDateTime>,
) -> AlertTrigger<T> {
let client = client.unwrap_or_else(|| "web3-proxy".to_string());
let group = chain_id.map(|x| format!("chain #{}", x));
2023-01-24 14:12:36 +03:00
let source = source.unwrap_or_else(|| {
hostname::get()
.unwrap()
.into_string()
.unwrap_or_else(|err| {
2023-06-29 07:30:00 +03:00
warn!(?err, "unable to handle hostname");
"unknown".to_string()
})
});
2023-01-24 14:12:36 +03:00
2023-01-25 11:27:07 +03:00
let mut s = DefaultHasher::new();
2023-01-25 12:00:28 +03:00
// TODO: include severity here?
2023-01-25 11:27:07 +03:00
summary.hash(&mut s);
client.hash(&mut s);
client_url.hash(&mut s);
component.hash(&mut s);
group.hash(&mut s);
class.hash(&mut s);
let dedup_key = s.finish().to_string();
2023-01-24 14:12:36 +03:00
let payload = AlertTriggerPayload {
severity,
summary,
source,
timestamp,
component,
group,
class,
custom_details,
};
AlertTrigger {
payload,
2023-01-25 11:27:07 +03:00
dedup_key: Some(dedup_key),
2023-01-24 14:12:36 +03:00
images: None,
links: None,
client: Some(client),
client_url,
2023-01-24 14:12:36 +03:00
}
}