don't send pagerduty alerts for websocket panics

This commit is contained in:
Bryan Stitt 2023-01-24 09:38:12 -08:00
parent 106dec294f
commit 522678e394
2 changed files with 27 additions and 21 deletions

@ -319,6 +319,8 @@ These are not yet ordered. There might be duplicates. We might not actually need
- this will let us easily wait for a new head or a new synced connection
- [x] broadcast transactions to more servers
- [x] send sentryd errors to pagerduty
- [x] improve handling of unknown methods
- [x] don't send pagerduty alerts for websocket panics
- [-] proxy mode for benchmarking all backends
- [-] proxy mode for sending to multiple backends
- [-] let users choose a % of reverts to log (or maybe x/second). someone like curve logging all reverts will be a BIG database very quickly

@ -231,30 +231,34 @@ fn main() -> anyhow::Result<()> {
let hostname = gethostname().into_string().unwrap_or("unknown".to_string());
let panic_msg = format!("{} {:?}", x, x);
error!("sending panic to pagerduty: {}", panic_msg);
if panic_msg.starts_with("panicked at 'WS Server panic") {
info!("Underlying library {}", panic_msg);
} else {
error!("sending panic to pagerduty: {}", panic_msg);
let payload = AlertTriggerPayload {
severity: pagerduty_rs::types::Severity::Error,
summary: panic_msg.clone(),
source: hostname,
timestamp: None,
component: None,
group: Some("web3-proxy".to_string()),
class: Some("panic".to_string()),
custom_details: None::<()>,
};
let payload = AlertTriggerPayload {
severity: pagerduty_rs::types::Severity::Error,
summary: panic_msg.clone(),
source: hostname,
timestamp: None,
component: None,
group: Some("web3-proxy".to_string()),
class: Some("panic".to_string()),
custom_details: None::<()>,
};
let event = Event::AlertTrigger(AlertTrigger {
payload,
dedup_key: None,
images: None,
links: None,
client: Some(client.clone()),
client_url: client_url.clone(),
});
let event = Event::AlertTrigger(AlertTrigger {
payload,
dedup_key: None,
images: None,
links: None,
client: Some(client.clone()),
client_url: client_url.clone(),
});
if let Err(err) = pagerduty_sync.event(event) {
error!("Failed sending panic to pagerduty: {}", err);
if let Err(err) = pagerduty_sync.event(event) {
error!("Failed sending panic to pagerduty: {}", err);
}
}
}));
}