diff --git a/build/pluto/prometheus/alertmanager.nix b/build/pluto/prometheus/alertmanager.nix index d63a5fb5..6f9dd199 100644 --- a/build/pluto/prometheus/alertmanager.nix +++ b/build/pluto/prometheus/alertmanager.nix @@ -1,4 +1,9 @@ -{ config, ... }: +{ + config, + lib, + pkgs, + ... +}: { services.prometheus = { @@ -30,7 +35,7 @@ routes = [ { - receiver = "go-neb"; + receiver = "matrix"; group_wait = "30s"; match.severity = "warning"; } @@ -42,11 +47,15 @@ name = "ignore"; } { - name = "go-neb"; + name = "matrix"; webhook_configs = [ { - url = "${config.services.go-neb.baseUrl}:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U"; + url = "http://localhost:${toString config.services.matrix-alertmanager.port}/alerts"; send_resolved = true; + http_config.basic_auth = { + username = "alertmanager"; + password_file = config.age.secrets."matrix-alertmanager-secret".path; + }; } ]; } @@ -88,83 +97,38 @@ }; }; - age.secrets.alertmanager-matrix-forwarder = { - file = ../../secrets/alertmanager-matrix-forwarder.age; - owner = config.systemd.services.go-neb.serviceConfig.User; - }; - - # Create user so that we can set the ownership of the key to - # it. DynamicUser will not take full effect as a result of this. - users.users.go-neb = { - isSystemUser = true; - group = "go-neb"; + # access token + age.secrets."matrix-alertmanager-token".file = ../../secrets/matrix-alertmanager-token.age; + # webhook secret + age.secrets."matrix-alertmanager-secret" = { + file = ../../secrets/matrix-alertmanager-secret.age; + owner = "alertmanager"; }; - users.groups.go-neb = { }; - systemd.services.go-neb.serviceConfig.SupplementaryGroups = [ "keys" ]; - - nixpkgs.config.permittedInsecurePackages = [ "olm-3.2.16" ]; - - services.go-neb = { + services.matrix-alertmanager = { enable = true; - bindAddress = "localhost:4050"; - baseUrl = "http://localhost"; - secretFile = config.age.secrets.alertmanager-matrix-forwarder.path; - config = { - clients = [ - { - UserId = "@bot:nixos.org"; - AccessToken = "$CHANGEME"; - HomeServerUrl = "https://matrix.nixos.org"; - Sync = true; - AutoJoinRooms = true; - DisplayName = "Bot"; - } + package = pkgs.matrix-alertmanager.overrideAttrs (oldAttrs: { + patches = oldAttrs.patches or [ ] ++ [ + ./matrix-alertmanager-linkfix.patch ]; - services = [ - { - ID = "alertmanager_service"; - Type = "alertmanager"; - UserId = "@bot:nixos.org"; - Config = { - webhook_url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U"; - rooms = { - # infra-alerts:nixos.org - "!QLQqibtFaVtDgurUAE:nixos.org" = { - text_template = '' - {{range .Alerts -}} [{{ .Status }}] {{index .Labels "alertname" }}: {{index .Annotations "description"}} {{ end -}} - ''; + }); + tokenFile = config.age.secrets.matrix-alertmanager-token.path; + secretFile = config.age.secrets.matrix-alertmanager-secret.path; + homeserverUrl = "https://matrix.nixos.org"; + matrixUser = "@bot:nixos.org"; + matrixRooms = [ + { + receivers = [ "matrix" ]; + roomId = "!QLQqibtFaVtDgurUAE:nixos.org"; + } + ]; + }; - # $$severity otherwise envsubst replaces $severity with an empty string - html_template = '' - {{range .Alerts -}} - {{ $$severity := index .Labels "severity" }} - {{ if eq .Status "firing" }} - {{ if eq $$severity "critical"}} - [FIRING - CRITICAL] - {{ else if eq $$severity "warning"}} - [FIRING - WARNING] - {{ else }} - [FIRING - {{ $$severity }}] - {{ end }} - {{ else }} - [RESOLVED] - {{ end }} - {{ index .Labels "alertname"}}: {{ index .Annotations "summary"}} - ( - {{ if .Annotations.grafana }} - 📈 Grafana, - {{ end }} - 🔥 Prometheus, - 🔕 Silence - )
- {{end -}}''; - msg_type = "m.text"; # Must be either `m.text` or `m.notice` - }; - }; - }; - } - ]; - }; + systemd.services.matrix-alertmanager.environment = { + ALERT_LINKS = lib.concatStringsSep "|" [ + "📈 Grafana:{annotations.grafana}" + "🔥 Prometheus:{generatorURL}" + "🔕 Silence:https://alerts.nixos.org/#/silences/new?filter={labels.alertname}" + ]; }; } diff --git a/build/pluto/prometheus/matrix-alertmanager-linkfix.patch b/build/pluto/prometheus/matrix-alertmanager-linkfix.patch new file mode 100644 index 00000000..60441a47 --- /dev/null +++ b/build/pluto/prometheus/matrix-alertmanager-linkfix.patch @@ -0,0 +1,31 @@ +diff --git a/src/utils.js b/src/utils.js +index f71935f..a1e16f9 100644 +--- a/src/utils.js ++++ b/src/utils.js +@@ -76,6 +76,8 @@ const utils = { + } + // Add custom links if configured + if (process.env.ALERT_LINKS) { ++ let links = [] ++ + const linkConfigs = process.env.ALERT_LINKS.split('|') + for (let linkConfig of linkConfigs) { + const firstColonIndex = linkConfig.indexOf(':') +@@ -105,11 +107,15 @@ const utils = { + return encodeURIComponent(data.labels[labelName] || '') + }) + url = url.replace(/{annotations\.([^}]+)}/g, (match, annotationName) => { +- return encodeURIComponent(data.annotations[annotationName] || '') ++ return data.annotations[annotationName] || '' + }) +- parts.push('
', name, '') ++ links.push('', name.trim(), ' ') + } + } ++ ++ if (links.length >= 0) { ++ parts.push(...links) ++ } + } else { + // Fallback to the original message if no custom links configured + parts.push('
Alert link') diff --git a/build/secrets.nix b/build/secrets.nix index d6b57086..20b8590d 100644 --- a/build/secrets.nix +++ b/build/secrets.nix @@ -2,7 +2,6 @@ let keys = import ../ssh-keys.nix; secrets = with keys; { - alertmanager-matrix-forwarder = [ machines.pluto ]; alertmanager-oauth2-proxy-env = [ machines.pluto ]; fastly-exporter-env = [ machines.pluto ]; grafana-secret-key = [ machines.pluto ]; @@ -10,6 +9,8 @@ let hydra-github-client-secret = [ machines.mimas ]; hydra-mirror-aws-credentials = [ machines.pluto ]; hydra-mirror-git-credentials = [ machines.pluto ]; + matrix-alertmanager-secret = [ machines.pluto ]; + matrix-alertmanager-token = [ machines.pluto ]; owncast-admin-password = [ machines.pluto ]; pluto-backup-secret = [ machines.pluto ]; pluto-backup-ssh-key = [ machines.pluto ]; diff --git a/build/secrets/alertmanager-matrix-forwarder.age b/build/secrets/alertmanager-matrix-forwarder.age deleted file mode 100644 index bb5e2910..00000000 Binary files a/build/secrets/alertmanager-matrix-forwarder.age and /dev/null differ diff --git a/build/secrets/matrix-alertmanager-secret.age b/build/secrets/matrix-alertmanager-secret.age new file mode 100644 index 00000000..44a63f36 --- /dev/null +++ b/build/secrets/matrix-alertmanager-secret.age @@ -0,0 +1,20 @@ +age-encryption.org/v1 +-> ssh-ed25519 s9hT2g sHIWxz0EAvyP3maXx/y5O1RjfG1/8SS+3K3O7hj7o1Y +Aa/GaMwxE4qoIl0xjM9nqc4c7XmsFUsU+xca2T3WJq0 +-> ssh-ed25519 Gr9EaQ Kn4W4fOUB4X98aaqxTonGb6gP/e8kqs8+Nz0saU0wCw +JWmJwzIEwFtYW97pt92ASmK2lr+H1Vt6BwqVayBgeL8 +-> ssh-ed25519 3ENwVg h4qBVOqVQhmRco/7sgtHnFiODIE0CEOKo3euSBwPKHU +qxLAejAX+zbC2gVPOKV+y/1SccAQnZCZKOP8DgpF8JI +-> ssh-rsa MuWD+w +cqSb00aSm+AgrzOKsHMiC7jH62SyZ9Bc60HpMnAnbAay7tIlevSQKaFPIvfvpjQT +d8VsDDnotynBgfQ5PJ5DFBNVFhWs1TEqG1Oh/tPP5UUBmheuT2eGrrX1dpU/TP1O +XcQ3Q39TPAG+Uvd3HG7vVlf/plStkc9zhlP55RUebng0zj3VNuTwEqP7QzaLGfWT +xxTmX9iEBrvankU50BSu/Gf4ukKwhohBeJCFeBnhDBB0xP3QmOZDhKq7THtCXIvf +RefwIFgrThrZaQ5dXPTwm7pBHqpGdFEXVci8PlEIzMv5NU493sUxYKgozbz5js2P +6ISGM5yykk6k6+0G/0eYkA +-> ssh-ed25519 92bXiA 4g6lh/KyvFR8UIOw87x2Cn2haBnh/a7npANLYDt+wUg +Cv3WvAB1NQme9iH/V4+u48iLxtq6RdLr2S6Jkbj6kcI +-> ssh-ed25519 Y121Gw 3Kdx++jQNfRR8AowsBAqK3KLY41BTNO6rzYqO1RpGRg +UB9oOD5IMXn3cBG61uGSnSQPeHDSOGaG5NobVr56hSY +--- Psr5uomAhtbIM8ib6wXSZh8hKrEzF339YMe4AAGM6ak +zLzt_!EIcddAnCơ}3o.]v'yH20 i4,\ \ No newline at end of file diff --git a/build/secrets/matrix-alertmanager-token.age b/build/secrets/matrix-alertmanager-token.age new file mode 100644 index 00000000..a276df3c --- /dev/null +++ b/build/secrets/matrix-alertmanager-token.age @@ -0,0 +1,20 @@ +age-encryption.org/v1 +-> ssh-ed25519 s9hT2g i17vHLBACJz3KhippQJNfGfZ88vTFTcoDHULVWa5aRI +QcgiuyqDYpXqqdF8eTO1kq5SEbVYQjqhd6fqXt9t73M +-> ssh-ed25519 Gr9EaQ +oefOEIaq+cGGkuxJLg+/G6n3Mo7pxS0w6jxmC5q3G8 +VJJBC6sZ4WV/uwKkAEMYDZUmKkoJmX2NdwMCNMuORXI +-> ssh-ed25519 3ENwVg hdNCwExI1ZLmqQtRTUvehgpYKqhp3vo7IzXYIB1lyRU +I4f3yf3pNV1NCL0nEIR/i1tLguGt66z1ekBPzhamw8w +-> ssh-rsa MuWD+w +VF4EsI0rbzWJ7sJ388gqQc5QZSY6GegbBOrIUd74S4mpYwnTme/LzF5cbXCthaZM +eF7cHDYYN0Dw10oACxGKqWMJgIQ5S3MbVCJArLR9Dnsd0h5HXIBgysXdxmWFVUtv +GaaKUD0RmRaW97/OOLy+dSEwSsF+AZ9n/zfZU3xNkFP2C95Fa8+pdmRqL8iC+RuL +KdEyjqSwLIulxu5UsYgMHscBgoZBsaPts0Et+eFw1qAQ2VFDxphXkrrbbzuuwWRA +raxx2Zflib2uJB7MomfDCwIey8A+IH0y0NAuqXcSFvSiUPP2qCyzcgzrMBdV/pbt +/3JANnE9OVoI1kpekjwTQA +-> ssh-ed25519 92bXiA IR0urBRAiiHu1XGMg6KsLdT+haot5HkaxYHop3LcZ0A +nAOHJC0ge+wA2IBJtPnZvDMJ32mdaKMp9OJ5djtzlSE +-> ssh-ed25519 Y121Gw dA6Kk/YcDU67YSI0iDTFz4c+bxIJxOY6VTzP0uuxQyU +aRfS3KUbFTJAyKlsFQUzkaZ67jJArPOdkBI7iGB3SIo +--- 9z895yYMnkL/6P1N0l/9wn/RO6Hheqd5dallnvNgv3s +BBpGBf柙Z`A݂8I@{SMXIe-=xoM>9?$ \ No newline at end of file