add alert

modules/services/grafana-dashboards/alerts.yml

@@ -0,0 +1,97 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: default
+      folder: ALARM
+      interval: 5m
+      rules:
+        - uid: ddin0kv0wnj0gd
+          title: Systemd Units State
+          condition: B
+          data:
+            - refId: D
+              relativeTimeRange:
+                from: 86400
+                to: 0
+              datasourceUid: PBFE396EC0B189D67
+              model:
+                datasource:
+                    type: prometheus
+                    uid: PBFE396EC0B189D67
+                expr: node_systemd_units{instance="127.0.0.1:9001",job="scrapema",state="failed"}
+                format: time_series
+                interval: ""
+                intervalFactor: 1
+                intervalMs: 15000
+                legendFormat: Failed
+                maxDataPoints: 43200
+                refId: D
+                step: 240
+            - refId: A
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params: []
+                      reducer:
+                        params: []
+                        type: avg
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: D
+                hide: false
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: A
+                settings:
+                    mode: dropNN
+                type: reduce
+            - refId: B
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params: []
+                      reducer:
+                        params: []
+                        type: avg
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: A
+                hide: false
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: B
+                type: threshold
+          dashboardUid: rYdddlPWk
+          panelId: 298
+          noDataState: NoData
+          execErrState: Error
+          for: 5m
+          annotations:
+            __dashboardUid__: rYdddlPWk
+            __panelId__: "298"
+          labels: {}
+          isPaused: false
+

modules/services/grafana-dashboards/notification-policies.yml

@@ -0,0 +1,7 @@
+apiVersion: 1
+policies:
+    - orgId: 1
+      receiver: discord
+      group_by:
+        - grafana_folder
+        - alertname

modules/services/grafana.nix

@@ -4,6 +4,12 @@ let
   fqdn = "grafana.home.arpa";
 in
 {
+  age.secrets.grafana-contact-points = {
+    name = "contact-points.yml";
+    owner = "grafana";
+    file = ../../secrets/grafana-contact-points.age;
+  };
+
   services.grafana = {
     enable = true;
     settings.server = {
@@ -12,6 +18,9 @@ in
       http_addr = "127.0.0.1";
     };
 
+    provision.alerting.contactPoints.path = config.age.secrets.grafana-contact-points.path;
+    provision.alerting.policies.path = ./grafana-dashboards/notification-policies.yml; 
+    provision.alerting.templates.path = ./grafana-dashboards/alerts.yml;
     provision.datasources.settings = {
      datasources =
        [