add alert

modules/services/grafana-dashboards/alerts.yml

@@ -0,0 +1,97 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: default
+      folder: ALARM
+      interval: 5m
+      rules:
+        - uid: ddin0kv0wnj0gd
+          title: Systemd Units State
+          condition: B
+          data:
+            - refId: D
+              relativeTimeRange:
+                from: 86400
+                to: 0
+              datasourceUid: PBFE396EC0B189D67
+              model:
+                datasource:
+                    type: prometheus
+                    uid: PBFE396EC0B189D67
+                expr: node_systemd_units{instance="127.0.0.1:9001",job="scrapema",state="failed"}
+                format: time_series
+                interval: ""
+                intervalFactor: 1
+                intervalMs: 15000
+                legendFormat: Failed
+                maxDataPoints: 43200
+                refId: D
+                step: 240
+            - refId: A
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params: []
+                      reducer:
+                        params: []
+                        type: avg
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: D
+                hide: false
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: A
+                settings:
+                    mode: dropNN
+                type: reduce
+            - refId: B
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params: []
+                      reducer:
+                        params: []
+                        type: avg
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: A
+                hide: false
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: B
+                type: threshold
+          dashboardUid: rYdddlPWk
+          panelId: 298
+          noDataState: NoData
+          execErrState: Error
+          for: 5m
+          annotations:
+            __dashboardUid__: rYdddlPWk
+            __panelId__: "298"
+          labels: {}
+          isPaused: false
+

modules/services/grafana-dashboards/notification-policies.yml

@@ -0,0 +1,7 @@
+apiVersion: 1
+policies:
+    - orgId: 1
+      receiver: discord
+      group_by:
+        - grafana_folder
+        - alertname

modules/services/grafana.nix

@@ -4,6 +4,12 @@ let
   fqdn = "grafana.home.arpa";
 in
 {
+  age.secrets.grafana-contact-points = {
+    name = "contact-points.yml";
+    owner = "grafana";
+    file = ../../secrets/grafana-contact-points.age;
+  };
+
   services.grafana = {
     enable = true;
     settings.server = {
@@ -12,6 +18,9 @@ in
       http_addr = "127.0.0.1";
     };
 
+    provision.alerting.contactPoints.path = config.age.secrets.grafana-contact-points.path;
+    provision.alerting.policies.path = ./grafana-dashboards/notification-policies.yml; 
+    provision.alerting.templates.path = ./grafana-dashboards/alerts.yml;
     provision.datasources.settings = {
      datasources =
        [

secrets/grafana-contact-points.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 IV3DkQ KeRBStynFOCooZSVXuTl/5ezdHbiUtFi2n/rKBw2inI
+atLbCeW3O13klIVFjrDmpDCNBMCgaOISMVCKOcsVb+I
+-> ssh-ed25519 DCzi1A Nkv5PRjae74rWPBKJ+i3gW9FLNoa/HGHCg9/22EAP20
+MHVw5X747PS0O+RDScukc3kH2oE7OkuTlv4X2tN4SDs
+--- RDFOrBjghqvyf6rMRiloSF2poVmkHJU1vMRqmDXzu2k
+BÛÀ"<22>U
+(.Óø“yù踇‚Ÿ68É”™Úâ5”à­Z]NÊzÂü3à¥áâ@ºÍÇ?ù×7I
+Rɧ07$<24>Ù6"ø…Ïóû”2…³—®Ö<£ÕHÊ¢´§&¡R¢=i¬nQ`Y€0Â<30>—·/£lüs<C3BC>>îÖPocfMÛѺ²íw,2çéê	8£i““TÍeR
<04>t’<74>»µéÐÒ9ÿßœöšî”’<0C>MT5,•¥™$»Ï˜*>ÜßõÎ ~¢ÊZÂN<C382>

secrets/secrets.nix

@@ -24,4 +24,5 @@ in
   "step-ca-key.age".publicKeys = [ nix-test-vm server kop ];
   "syncthing-key.age".publicKeys = [ server kop ];
   "syncthing-cert.age".publicKeys = [ server kop ];
+  "grafana-contact-points.age".publicKeys = [ server kop];
 }