matrix-alertmanager: migrate from go-neb

build/pluto/prometheus/alertmanager.nix

@@ -1,4 +1,9 @@
-{ config, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 
 {
   services.prometheus = {
@@ -30,7 +35,7 @@
 
           routes = [
             {
-              receiver = "go-neb";
+              receiver = "matrix";
               group_wait = "30s";
               match.severity = "warning";
             }
@@ -42,11 +47,15 @@
             name = "ignore";
           }
           {
-            name = "go-neb";
+            name = "matrix";
             webhook_configs = [
               {
-                url = "${config.services.go-neb.baseUrl}:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
+                url = "http://localhost:${toString config.services.matrix-alertmanager.port}/alerts";
                 send_resolved = true;
+                http_config.basic_auth = {
+                  username = "alertmanager";
+                  password_file = config.age.secrets."matrix-alertmanager-secret".path;
+                };
               }
             ];
           }
@@ -88,83 +97,38 @@
     };
   };
 
-  age.secrets.alertmanager-matrix-forwarder = {
-    file = ../../secrets/alertmanager-matrix-forwarder.age;
-    owner = config.systemd.services.go-neb.serviceConfig.User;
-  };
-
-  # Create user so that we can set the ownership of the key to
-  # it. DynamicUser will not take full effect as a result of this.
-  users.users.go-neb = {
-    isSystemUser = true;
-    group = "go-neb";
+  # access token
+  age.secrets."matrix-alertmanager-token".file = ../../secrets/matrix-alertmanager-token.age;
+  # webhook secret
+  age.secrets."matrix-alertmanager-secret" = {
+    file = ../../secrets/matrix-alertmanager-secret.age;
+    owner = "alertmanager";
   };
-  users.groups.go-neb = { };
 
-  systemd.services.go-neb.serviceConfig.SupplementaryGroups = [ "keys" ];
-
-  nixpkgs.config.permittedInsecurePackages = [ "olm-3.2.16" ];
-
-  services.go-neb = {
+  services.matrix-alertmanager = {
     enable = true;
-    bindAddress = "localhost:4050";
-    baseUrl = "http://localhost";
-    secretFile = config.age.secrets.alertmanager-matrix-forwarder.path;
-    config = {
-      clients = [
-        {
-          UserId = "@bot:nixos.org";
-          AccessToken = "$CHANGEME";
-          HomeServerUrl = "https://matrix.nixos.org";
-          Sync = true;
-          AutoJoinRooms = true;
-          DisplayName = "Bot";
-        }
+    package = pkgs.matrix-alertmanager.overrideAttrs (oldAttrs: {
+      patches = oldAttrs.patches or [ ] ++ [
+        ./matrix-alertmanager-linkfix.patch
       ];
-      services = [
-        {
-          ID = "alertmanager_service";
-          Type = "alertmanager";
-          UserId = "@bot:nixos.org";
-          Config = {
-            webhook_url = "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U";
-            rooms = {
-              # infra-alerts:nixos.org
-              "!QLQqibtFaVtDgurUAE:nixos.org" = {
-                text_template = ''
-                  {{range .Alerts -}} [{{ .Status }}] {{index .Labels "alertname" }}: {{index .Annotations "description"}} {{ end -}}
-                '';
+    });
+    tokenFile = config.age.secrets.matrix-alertmanager-token.path;
+    secretFile = config.age.secrets.matrix-alertmanager-secret.path;
+    homeserverUrl = "https://matrix.nixos.org";
+    matrixUser = "@bot:nixos.org";
+    matrixRooms = [
+      {
+        receivers = [ "matrix" ];
+        roomId = "!QLQqibtFaVtDgurUAE:nixos.org";
+      }
+    ];
+  };
 
-                # $$severity otherwise envsubst replaces $severity with an empty string
-                html_template = ''
-                  {{range .Alerts -}}
-                    {{ $$severity := index .Labels "severity" }}
-                    {{ if eq .Status "firing" }}
-                      {{ if eq $$severity "critical"}}
-                        <font color='red'><b>[FIRING - CRITICAL]</b></font>
-                      {{ else if eq $$severity "warning"}}
-                        <font color='orange'><b>[FIRING - WARNING]</b></font>
-                      {{ else }}
-                        <b>[FIRING - {{ $$severity }}]</b>
-                      {{ end }}
-                    {{ else }}
-                      <font color='green'><b>[RESOLVED]</b></font>
-                    {{ end }}
-                    {{ index .Labels "alertname"}}: {{ index .Annotations "summary"}}
-                    (
-                      {{ if .Annotations.grafana }}
-                        <a href="{{ index .Annotations "grafana" }}">📈 Grafana</a>,
-                      {{ end }}
-                      <a href="{{ .GeneratorURL }}">🔥 Prometheus</a>,
-                      <a href="{{ .SilenceURL }}">🔕 Silence</a>
-                    )<br/>
-                  {{end -}}'';
-                msg_type = "m.text"; # Must be either `m.text` or `m.notice`
-              };
-            };
-          };
-        }
-      ];
-    };
+  systemd.services.matrix-alertmanager.environment = {
+    ALERT_LINKS = lib.concatStringsSep "|" [
+      "📈 Grafana:{annotations.grafana}"
+      "🔥 Prometheus:{generatorURL}"
+      "🔕 Silence:https://alerts.nixos.org/#/silences/new?filter={labels.alertname}"
+    ];
   };
 }

build/pluto/prometheus/matrix-alertmanager-linkfix.patch

@@ -0,0 +1,31 @@
+diff --git a/src/utils.js b/src/utils.js
+index f71935f..a1e16f9 100644
+--- a/src/utils.js
++++ b/src/utils.js
+@@ -76,6 +76,8 @@ const utils = {
+         }
+         // Add custom links if configured
+         if (process.env.ALERT_LINKS) {
++            let links = []
++
+             const linkConfigs = process.env.ALERT_LINKS.split('|')
+             for (let linkConfig of linkConfigs) {
+                 const firstColonIndex = linkConfig.indexOf(':')
+@@ -105,11 +107,15 @@ const utils = {
+                         return encodeURIComponent(data.labels[labelName] || '')
+                     })
+                     url = url.replace(/{annotations\.([^}]+)}/g, (match, annotationName) => {
+-                        return encodeURIComponent(data.annotations[annotationName] || '')
++                        return data.annotations[annotationName] || ''
+                     })
+-                    parts.push('<br><a href="', url, '">', name, '</a>')
++                    links.push('<a href="', url.trim(), '">', name.trim(), '</a>&nbsp;')
+                 }
+             }
++
++            if (links.length >= 0) {
++                parts.push(...links)
++            }
+         } else {
+             // Fallback to the original message if no custom links configured
+             parts.push('<br><a href="', data.generatorURL,'">Alert link</a>')

build/secrets.nix

@@ -2,14 +2,15 @@ let
   keys = import ../ssh-keys.nix;
 
   secrets = with keys; {
-    alertmanager-matrix-forwarder = [ machines.pluto ];
     alertmanager-oauth2-proxy-env = [ machines.pluto ];
     fastly-exporter-env = [ machines.pluto ];
     grafana-secret-key = [ machines.pluto ];
     hydra-aws-credentials = [ machines.mimas ];
     hydra-github-client-secret = [ machines.mimas ];
     hydra-mirror-aws-credentials = [ machines.pluto ];
     hydra-mirror-git-credentials = [ machines.pluto ];
+    matrix-alertmanager-secret = [ machines.pluto ];
+    matrix-alertmanager-token = [ machines.pluto ];
     owncast-admin-password = [ machines.pluto ];
     pluto-backup-secret = [ machines.pluto ];
     pluto-backup-ssh-key = [ machines.pluto ];

build/secrets/matrix-alertmanager-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 s9hT2g sHIWxz0EAvyP3maXx/y5O1RjfG1/8SS+3K3O7hj7o1Y
+Aa/GaMwxE4qoIl0xjM9nqc4c7XmsFUsU+xca2T3WJq0
+-> ssh-ed25519 Gr9EaQ Kn4W4fOUB4X98aaqxTonGb6gP/e8kqs8+Nz0saU0wCw
+JWmJwzIEwFtYW97pt92ASmK2lr+H1Vt6BwqVayBgeL8
+-> ssh-ed25519 3ENwVg h4qBVOqVQhmRco/7sgtHnFiODIE0CEOKo3euSBwPKHU
+qxLAejAX+zbC2gVPOKV+y/1SccAQnZCZKOP8DgpF8JI
+-> ssh-rsa MuWD+w
+cqSb00aSm+AgrzOKsHMiC7jH62SyZ9Bc60HpMnAnbAay7tIlevSQKaFPIvfvpjQT
+d8VsDDnotynBgfQ5PJ5DFBNVFhWs1TEqG1Oh/tPP5UUBmheuT2eGrrX1dpU/TP1O
+XcQ3Q39TPAG+Uvd3HG7vVlf/plStkc9zhlP55RUebng0zj3VNuTwEqP7QzaLGfWT
+xxTmX9iEBrvankU50BSu/Gf4ukKwhohBeJCFeBnhDBB0xP3QmOZDhKq7THtCXIvf
+RefwIFgrThrZaQ5dXPTwm7pBHqpGdFEXVci8PlEIzMv5NU493sUxYKgozbz5js2P
+6ISGM5yykk6k6+0G/0eYkA
+-> ssh-ed25519 92bXiA 4g6lh/KyvFR8UIOw87x2Cn2haBnh/a7npANLYDt+wUg
+Cv3WvAB1NQme9iH/V4+u48iLxtq6RdLr2S6Jkbj6kcI
+-> ssh-ed25519 Y121Gw 3Kdx++jQNfRR8AowsBAqK3KLY41BTNO6rzYqO1RpGRg
+UB9oOD5IMXn3cBG61uGSnSQPeHDSOGaG5NobVr56hSY
+--- Psr5uomAhtbIM8ib6wXSZh8hKrEzF339YMe4AAGM6ak
+z�Lzt���_!E�Icd�dA�nC�ơ}3���o�.]v'��yH2�0�i4�,�\��

build/secrets/matrix-alertmanager-token.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 s9hT2g i17vHLBACJz3KhippQJNfGfZ88vTFTcoDHULVWa5aRI
+QcgiuyqDYpXqqdF8eTO1kq5SEbVYQjqhd6fqXt9t73M
+-> ssh-ed25519 Gr9EaQ +oefOEIaq+cGGkuxJLg+/G6n3Mo7pxS0w6jxmC5q3G8
+VJJBC6sZ4WV/uwKkAEMYDZUmKkoJmX2NdwMCNMuORXI
+-> ssh-ed25519 3ENwVg hdNCwExI1ZLmqQtRTUvehgpYKqhp3vo7IzXYIB1lyRU
+I4f3yf3pNV1NCL0nEIR/i1tLguGt66z1ekBPzhamw8w
+-> ssh-rsa MuWD+w
+VF4EsI0rbzWJ7sJ388gqQc5QZSY6GegbBOrIUd74S4mpYwnTme/LzF5cbXCthaZM
+eF7cHDYYN0Dw10oACxGKqWMJgIQ5S3MbVCJArLR9Dnsd0h5HXIBgysXdxmWFVUtv
+GaaKUD0RmRaW97/OOLy+dSEwSsF+AZ9n/zfZU3xNkFP2C95Fa8+pdmRqL8iC+RuL
+KdEyjqSwLIulxu5UsYgMHscBgoZBsaPts0Et+eFw1qAQ2VFDxphXkrrbbzuuwWRA
+raxx2Zflib2uJB7MomfDCwIey8A+IH0y0NAuqXcSFvSiUPP2qCyzcgzrMBdV/pbt
+/3JANnE9OVoI1kpekjwTQA
+-> ssh-ed25519 92bXiA IR0urBRAiiHu1XGMg6KsLdT+haot5HkaxYHop3LcZ0A
+nAOHJC0ge+wA2IBJtPnZvDMJ32mdaKMp9OJ5djtzlSE
+-> ssh-ed25519 Y121Gw dA6Kk/YcDU67YSI0iDTFz4c+bxIJxOY6VTzP0uuxQyU
+aRfS3KUbFTJAyKlsFQUzkaZ67jJArPOdkBI7iGB3SIo
+--- 9z895yYMnkL/6P1N0l/9wn/RO6Hheqd5dallnvNgv3s
+BBpG�Bf��柙Z���`�A݂8�I�@{�S���MXI�e�-�=��x��o�M�>��9?���$�