(fleet/prometheus-alerts) add pvc alert

Author: csilva-cl

fleet/lib/kube-prometheus-stack/overlays/antu/values.yaml

@@ -192,7 +192,7 @@ alertmanager:
         - site
       group_wait: 30s
       group_interval: 5m
-      repeat_interval: 24h
+      repeat_interval: 8760h
       receiver: blackhole
       routes:
         - receiver: blackhole

fleet/lib/prometheus-alerts/README.md

@@ -27,6 +27,7 @@ file with the `rules.namespace` key.
   of receivers, pre- and suffixed with `,` to make regex matching easier in the
   alertmanager. For example: `,slack,squadcast,email,` The receivers are defined
   in the alertmanager configuration.
-  Currently (20240503) the following receivers are configured:
-   * `slack-test`
-   * `squadcast-test`
+  
+  Currently (20250616) the following receivers are configured:
+   * `gnocpush`: Requires label `gnoc = "true"`
+   * `squadcast-alertmanager`: Requires label `prod = "true"`

fleet/lib/prometheus-alerts/rules/prometheusrule-k8s.yaml

@@ -0,0 +1,20 @@
+groups:
+  - name: k8s.rules
+    rules:
+      - alert: PVCLowFreeSpace
+        annotations:
+          summary: PVC {{ $labels.persistentvolumeclaim }} is low on free space
+          description: >
+            PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }}
+            has less than 20% free space.
+        expr: |
+          (kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"}
+          / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"}) < 0.20
+          and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"} > 0
+        for: 2m
+        labels:
+          prod: "true"
+          severity: warning
+          node_name: '{{ $labels.prom_cluster }}'
+          device: null
+          service_name: null

fleet/lib/prometheus-alerts/rules/prometheusrule-net.yaml

@@ -4,7 +4,7 @@ groups:
       - alert: HostDown
         annotations:
           summary: Host {{ $labels.instance }} is down
-          description: Host {{ $labels.instance }} is down. Maybe it is on fire??? 🗑🔥
+          description: Host {{ $labels.instance }} is down. Maybe it is on fire???
         expr: probe_success != 1
         for: 1m
         labels: