Merge pull request #957 from lsst-it/IT-6303_alertmanager_tuning

(fleet/prometheus-alerts) add pvc alert

Author: csilva-cl

fleet/lib/kube-prometheus-stack/overlays/antu/values.yaml

@@ -192,7 +192,7 @@ alertmanager:
         - site
       group_wait: 30s
       group_interval: 5m
-      repeat_interval: 24h
+      repeat_interval: 120h
       receiver: blackhole
       routes:
         - receiver: blackhole

fleet/lib/prometheus-alerts/README.md

@@ -13,20 +13,10 @@ file with the `rules.namespace` key.
 
 ## Prometheus rule AURA standards
 
-* `summary` annotation: The `summary` annotation is used to be able to describe a
-  group of alerts incomming. This annotation DOES NOT contain any templated
-  variables and provides a simple single sentence summary of what the alert is
-  about. For example "Disk space full in 24h". When a cluster triggers several
-  alerts, it can be hany to group these alerts into a single notification, this
-  is when the `summary` can be used.
-* `discription` annotation: This provides a detailed overview of the alert
-  specifically to this instance of the alert. It MAY contain templated variables
-  to enrich the message.
-* `receiver` label: The receiver label is used by alertmanager to decide on the
-  routing of the notification for the alert. It exists out of `,` seperated list
-  of receivers, pre- and suffixed with `,` to make regex matching easier in the
-  alertmanager. For example: `,slack,squadcast,email,` The receivers are defined
-  in the alertmanager configuration.
-  Currently (20240503) the following receivers are configured:
-   * `slack-test`
-   * `squadcast-test`
+* `summary` annotation: This annotation MAY contain a templated variable to differentiate between hosts, pods, clusters, etc. and provides a simple single sentence summary of what the alert is about. For example, "Disk space full in acme.lsst.org". When a cluster triggers several alerts, it can be helpful to group these alerts into a single notification. A distinctive summary, it is also useful as a title for Jira tickets.
+* `description` annotation: This provides a detailed overview of the alert specifically to this instance of the alert. It MAY contain templated variables to enrich the message.
+* routing label: Rubin uses labels to route alerts. The label is used by alertmanager to determine the routing of the notification for the alert. By default, all alerts should be routed to Squadcast. The escalation and notification will be handled by Squadcast API.
+
+  Currently (20250616) the following receivers are configured:
+* `gnocpush`: Requires label `gnoc: "true"`
+* `squadcast-alertmanager`: Requires label `prod: "true"`. In most cases this should be the label of the alert.

fleet/lib/prometheus-alerts/rules/prometheusrule-k8s.yaml

@@ -0,0 +1,20 @@
+groups:
+  - name: k8s.rules
+    rules:
+      - alert: PVCLowFreeSpace
+        annotations:
+          summary: PVC {{ $labels.persistentvolumeclaim }} is low on free space
+          description: >
+            PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }}
+            has less than 20% free space.
+        expr: |
+          (kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"}
+          / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"}) < 0.20
+          and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"} > 0
+        for: 2m
+        labels:
+          prod: "true"
+          severity: warning
+          node_name: '{{ $labels.prom_cluster }}'
+          device: null
+          service_name: null

fleet/lib/prometheus-alerts/rules/prometheusrule-net.yaml

@@ -4,7 +4,7 @@ groups:
       - alert: HostDown
         annotations:
           summary: Host {{ $labels.instance }} is down
-          description: Host {{ $labels.instance }} is down. Maybe it is on fire??? 🗑🔥
+          description: Host {{ $labels.instance }} is down. Maybe it is on fire???
         expr: probe_success != 1
         for: 1m
         labels: