teamcraft/
notify.rs

1//! Error notifications to a Mattermost incoming webhook
2//! (`MATTERMOST_ERROR_WEBHOOK`). At most one message per minute is sent;
3//! further errors in that window are counted and reported with the next one.
4
5use std::{
6    sync::{
7        atomic::{AtomicU64, Ordering},
8        Mutex, OnceLock,
9    },
10    time::{Duration, Instant},
11};
12
13use serde_json::json;
14
15use crate::config::Config;
16
17const MIN_INTERVAL: Duration = Duration::from_secs(60);
18
19static NOTIFIER: OnceLock<Notifier> = OnceLock::new();
20
21struct Notifier {
22    url: String,
23    instance: String,
24    public_url: String,
25    http: reqwest::Client,
26    last_sent: Mutex<Option<Instant>>,
27    suppressed: AtomicU64,
28}
29
30/// Installs the notifier if a webhook is configured. Safe to call once.
31pub fn init(config: &Config) {
32    let Some(url) = config.mattermost_error_webhook.clone() else {
33        return;
34    };
35    let http = match reqwest::Client::builder()
36        .timeout(Duration::from_secs(5))
37        .build()
38    {
39        Ok(c) => c,
40        Err(e) => {
41            tracing::warn!(error = %e, "cannot create HTTP client for Mattermost notifications");
42            return;
43        }
44    };
45    let _ = NOTIFIER.set(Notifier {
46        url,
47        instance: config.instance_name.clone(),
48        public_url: config.public_url.clone(),
49        http,
50        last_sent: Mutex::new(None),
51        suppressed: AtomicU64::new(0),
52    });
53}
54
55impl Notifier {
56    /// Whether a message may go out at `now`: at most one per
57    /// [`MIN_INTERVAL`]. Returns the number of errors suppressed since the
58    /// last message (and starts counting anew), or `None` when this error is
59    /// only counted.
60    fn admit(&self, now: Instant) -> Option<u64> {
61        let mut last = self.last_sent.lock().unwrap_or_else(|e| e.into_inner());
62        if last.is_some_and(|t| now.saturating_duration_since(t) < MIN_INTERVAL) {
63            self.suppressed.fetch_add(1, Ordering::Relaxed);
64            return None;
65        }
66        *last = Some(now);
67        Some(self.suppressed.swap(0, Ordering::Relaxed))
68    }
69
70    /// The Mattermost message for an error.
71    fn text(&self, message: &str, suppressed: u64) -> String {
72        let mut text = format!(
73            "#### :rotating_light: {} error\n`{}`\n```\n{}\n```",
74            self.instance,
75            self.public_url,
76            truncate(message, 3000)
77        );
78        if suppressed > 0 {
79            text.push_str(&format!(
80                "\n_{suppressed} further error(s) were suppressed during the last minute._"
81            ));
82        }
83        text
84    }
85}
86
87/// Reports an internal error. Never blocks and never fails.
88pub fn report_error(message: &str) {
89    let Some(n) = NOTIFIER.get() else { return };
90    let Ok(runtime) = tokio::runtime::Handle::try_current() else {
91        return;
92    };
93    let Some(suppressed) = n.admit(Instant::now()) else {
94        return;
95    };
96    let text = n.text(message, suppressed);
97    let request = n
98        .http
99        .post(&n.url)
100        .json(&json!({ "username": "TeamCraft", "text": text }));
101    runtime.spawn(async move {
102        if let Err(e) = request.send().await.and_then(|r| r.error_for_status()) {
103            tracing::warn!(error = %e, "sending the Mattermost error notification failed");
104        }
105    });
106}
107
108fn truncate(s: &str, max: usize) -> String {
109    if s.chars().count() <= max {
110        s.to_string()
111    } else {
112        let mut t: String = s.chars().take(max).collect();
113        t.push('…');
114        t
115    }
116}
117
118#[cfg(test)]
119mod tests {
120    use super::*;
121
122    #[test]
123    fn truncation() {
124        assert_eq!(truncate("abc", 5), "abc");
125        assert_eq!(truncate("abcdef", 3), "abc…");
126    }
127
128    #[test]
129    fn report_without_notifier_is_a_noop() {
130        report_error("nothing happens");
131    }
132
133    fn notifier() -> Notifier {
134        Notifier {
135            url: "http://127.0.0.1:9/hooks/x".into(),
136            instance: "TeamCraft Test".into(),
137            public_url: "https://plm.example.org/".into(),
138            http: reqwest::Client::new(),
139            last_sent: Mutex::new(None),
140            suppressed: AtomicU64::new(0),
141        }
142    }
143
144    #[test]
145    fn one_message_per_minute_and_the_rest_counted() {
146        let n = notifier();
147        let t0 = Instant::now();
148        assert_eq!(n.admit(t0), Some(0), "the first error goes out");
149        assert_eq!(n.admit(t0 + Duration::from_secs(1)), None);
150        assert_eq!(n.admit(t0 + Duration::from_secs(59)), None);
151        // the minute is over: the next message reports the two in between
152        assert_eq!(n.admit(t0 + MIN_INTERVAL), Some(2));
153        assert_eq!(n.admit(t0 + MIN_INTERVAL + Duration::from_secs(1)), None);
154        assert_eq!(
155            n.admit(t0 + MIN_INTERVAL * 3),
156            Some(1),
157            "counting starts anew"
158        );
159        // a clock that seems to go backwards does not send twice
160        assert_eq!(n.admit(t0), None);
161    }
162
163    #[test]
164    fn the_message_names_instance_and_address_and_is_bounded() {
165        let n = notifier();
166        let text = n.text("database error: boom", 0);
167        assert!(text.starts_with("#### :rotating_light: TeamCraft Test error\n"));
168        assert!(text.contains("`https://plm.example.org/`"));
169        assert!(text.contains("```\ndatabase error: boom\n```"));
170        assert!(!text.contains("suppressed"));
171        let text = n.text("x", 3);
172        assert!(text.ends_with("_3 further error(s) were suppressed during the last minute._"));
173        // a huge error text is cut at 3000 characters (multi-byte safe)
174        let text = n.text(&"ü".repeat(10_000), 0);
175        assert_eq!(text.matches('ü').count(), 3000);
176        assert!(text.contains("ü…\n```"));
177    }
178}
teamcraft/notify.rs

teamcraft/
notify.rs
