From 50e66db24bd72b312306b7c5cba427744ce34b7a Mon Sep 17 00:00:00 2001 From: Kopatz <7265381+Kropatz@users.noreply.github.com> Date: Sat, 13 Apr 2024 22:14:49 +0200 Subject: [PATCH] add alert --- .../services/grafana-dashboards/alerts.yml | 97 +++++++++++++++++++ .../notification-policies.yml | 7 ++ modules/services/grafana.nix | 9 ++ secrets/grafana-contact-points.age | 9 ++ secrets/secrets.nix | 1 + 5 files changed, 123 insertions(+) create mode 100644 modules/services/grafana-dashboards/alerts.yml create mode 100644 modules/services/grafana-dashboards/notification-policies.yml create mode 100644 secrets/grafana-contact-points.age diff --git a/modules/services/grafana-dashboards/alerts.yml b/modules/services/grafana-dashboards/alerts.yml new file mode 100644 index 0000000..b01bada --- /dev/null +++ b/modules/services/grafana-dashboards/alerts.yml @@ -0,0 +1,97 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: default + folder: ALARM + interval: 5m + rules: + - uid: ddin0kv0wnj0gd + title: Systemd Units State + condition: B + data: + - refId: D + relativeTimeRange: + from: 86400 + to: 0 + datasourceUid: PBFE396EC0B189D67 + model: + datasource: + type: prometheus + uid: PBFE396EC0B189D67 + expr: node_systemd_units{instance="127.0.0.1:9001",job="scrapema",state="failed"} + format: time_series + interval: "" + intervalFactor: 1 + intervalMs: 15000 + legendFormat: Failed + maxDataPoints: 43200 + refId: D + step: 240 + - refId: A + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: D + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: A + settings: + mode: dropNN + type: reduce + - refId: B + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: A + hide: false + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + dashboardUid: rYdddlPWk + panelId: 298 + noDataState: NoData + execErrState: Error + for: 5m + annotations: + __dashboardUid__: rYdddlPWk + __panelId__: "298" + labels: {} + isPaused: false + diff --git a/modules/services/grafana-dashboards/notification-policies.yml b/modules/services/grafana-dashboards/notification-policies.yml new file mode 100644 index 0000000..67c4b3e --- /dev/null +++ b/modules/services/grafana-dashboards/notification-policies.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +policies: + - orgId: 1 + receiver: discord + group_by: + - grafana_folder + - alertname diff --git a/modules/services/grafana.nix b/modules/services/grafana.nix index 6871493..356117c 100644 --- a/modules/services/grafana.nix +++ b/modules/services/grafana.nix @@ -4,6 +4,12 @@ let fqdn = "grafana.home.arpa"; in { + age.secrets.grafana-contact-points = { + name = "contact-points.yml"; + owner = "grafana"; + file = ../../secrets/grafana-contact-points.age; + }; + services.grafana = { enable = true; settings.server = { @@ -12,6 +18,9 @@ in http_addr = "127.0.0.1"; }; + provision.alerting.contactPoints.path = config.age.secrets.grafana-contact-points.path; + provision.alerting.policies.path = ./grafana-dashboards/notification-policies.yml; + provision.alerting.templates.path = ./grafana-dashboards/alerts.yml; provision.datasources.settings = { datasources = [ diff --git a/secrets/grafana-contact-points.age b/secrets/grafana-contact-points.age new file mode 100644 index 0000000..dea2c70 --- /dev/null +++ b/secrets/grafana-contact-points.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> ssh-ed25519 IV3DkQ KeRBStynFOCooZSVXuTl/5ezdHbiUtFi2n/rKBw2inI +atLbCeW3O13klIVFjrDmpDCNBMCgaOISMVCKOcsVb+I +-> ssh-ed25519 DCzi1A Nkv5PRjae74rWPBKJ+i3gW9FLNoa/HGHCg9/22EAP20 +MHVw5X747PS0O+RDScukc3kH2oE7OkuTlv4X2tN4SDs +--- RDFOrBjghqvyf6rMRiloSF2poVmkHJU1vMRqmDXzu2k +B"U (.y685Z]Nz3@?7I +Rɧ07$6"2PocfMºw,2 8iTeRt9ߜ MT5,$Ϙ* > ~ZN Ԗ j/QKI$ ? 9N:H +T9&y]{pO=jh[[)+o0\d͡&&@f 70jr1JiYG4 O^~9EK[VK_0|0x=p4W]lܳ-%@ZkBp \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index ef10587..310b339 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -24,4 +24,5 @@ in "step-ca-key.age".publicKeys = [ nix-test-vm server kop ]; "syncthing-key.age".publicKeys = [ server kop ]; "syncthing-cert.age".publicKeys = [ server kop ]; + "grafana-contact-points.age".publicKeys = [ server kop]; }