k3s: Pod-Monitoring in Grafana (Task 6.5)

kube-state-metrics (NodePort 30808) und ServiceAccount prometheus-infra (nur
nodes/metrics) im Cluster; Prometheus auf infra liest kubelet/cAdvisor per TLS
gegen die k3s-CA. Ziele per file_sd aus kubectl get nodes, cAdvisor auf die
nötigen Reihen begrenzt. Dashboard "k3s – Pods" (Vorgabe nex), Alerts K3s*
nur mit notify. Playbook monitoring-k3s.yml.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

deploy/prometheus/alerts.yml

@@ -47,3 +47,33 @@ groups:
         for: 15m
         labels: {severity: warning}
         annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}
+
+  # k3s (Task 6.5): nur melden, der Healer greift nicht ein (config/healer-rules.yaml, Aktion notify)
+  - name: knaster-k3s
+    rules:
+      - alert: K3sNodeNotReady
+        expr: kube_node_status_condition{condition="Ready", status="true"} == 0
+        for: 5m
+        labels: {severity: high}
+        annotations: {summary: "k3s-Knoten {{ $labels.node }} nicht bereit"}
+
+      - alert: K3sPodCrashLooping
+        expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) == 1
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) startet immer wieder neu"}
+
+      - alert: K3sDeploymentUnavailable
+        expr: max by (namespace, deployment) (kube_deployment_spec_replicas) > max by (namespace, deployment) (kube_deployment_status_replicas_available)
+        for: 15m
+        labels: {severity: warning}
+        annotations: {summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} hat zu wenige bereite Pods"}
+
+      - alert: K3sContainerMemoryNearLimit
+        expr: >
+          max by (namespace, pod, container) (container_memory_working_set_bytes{job="cadvisor"})
+          / on (namespace, pod, container)
+          max by (namespace, pod, container) (kube_pod_container_resource_limits{resource="memory"}) > 0.9
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) über 90 % des Speicherlimits"}

deploy/prometheus/prometheus.yml

@@ -66,3 +66,39 @@ scrape_configs:
       - {source_labels: [__address__], target_label: __param_target}
       - {source_labels: [__param_target], target_label: instance}
       - {target_label: __address__, replacement: "pve-exporter:9221"}
+
+  # k3s (Task 6.5): kubelet direkt auf jedem Knoten (10250), TLS gegen die k3s-CA, Token des ServiceAccounts
+  # prometheus-infra (nur nodes/metrics). Ziele per file_sd aus `kubectl get nodes` (monitoring-targets.py auf ops).
+  # infra hat 4 GB RAM: von kubelet und cAdvisor nur die Reihen, die Dashboard „k3s – Pods“ und Alerts brauchen.
+  - job_name: kubelet
+    scheme: https
+    authorization: {credentials_file: /etc/knaster-k3s/token}
+    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
+    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: kubelet_running_pods|kubelet_running_containers|kubelet_volume_stats_(used|capacity)_bytes
+        action: keep
+
+  - job_name: cadvisor
+    scheme: https
+    metrics_path: /metrics/cadvisor
+    authorization: {credentials_file: /etc/knaster-k3s/token}
+    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
+    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_oom_events_total|container_network_(receive|transmit)_bytes_total
+        action: keep
+      # nur Pods (keine systemd-Slices, keine Docker-Container der Compose-Stacks)
+      - source_labels: [pod]
+        regex: ""
+        action: drop
+      # CPU/Speicher je Container: Pod-Summe und Pause-Container verwerfen (Netz gibt es nur je Pod)
+      - source_labels: [__name__, container]
+        regex: "container_(cpu|memory|oom)[^;]*;(POD)?"
+        action: drop
+
+  - job_name: kube-state-metrics
+    honor_labels: true   # namespace/pod/node kommen aus den Metriken selbst
+    file_sd_configs: [{files: [/etc/knaster-sd/kube-state-metrics.json]}]