k3s: Pod-Monitoring in Grafana (Task 6.5)

kube-state-metrics (NodePort 30808) und ServiceAccount prometheus-infra (nur nodes/metrics) im Cluster; Prometheus auf infra liest kubelet/cAdvisor per TLS gegen die k3s-CA. Ziele per file_sd aus kubectl get nodes, cAdvisor auf die nötigen Reihen begrenzt. Dashboard "k3s – Pods" (Vorgabe nex), Alerts K3s* nur mit notify. Playbook monitoring-k3s.yml. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 16:31:03 +02:00
parent abf4a75063
commit 4ac251715e
17 changed files with 1271 additions and 13 deletions
@@ -70,6 +70,8 @@ services:
      - ../../prometheus:/etc/prometheus:ro
      # file_sd: Ziele der GPU-VMs (DHCP, IPs wechseln), erzeugt auf ops aus `tofu output -json gpu_vms`
      - ${KNASTER_STATE_DIR:-.}/file_sd:/etc/knaster-sd:ro
+      # Token und CA für kubelet/cAdvisor in k3s (Task 6.5), nur dieser Unterordner der Geheimnisse
+      - ${KNASTER_STATE_DIR:-.}/secrets/k3s:/etc/knaster-k3s:ro
      - prometheus:/prometheus
    extra_hosts: ["host.docker.internal:host-gateway"]   # node-exporter des LXC läuft im Host-Netz
    ports: ["9090:9090"]
@@ -0,0 +1,824 @@
+{
+  "uid": "k3s-pods",
+  "title": "k3s – Pods",
+  "tags": [
+    "knaster",
+    "k3s"
+  ],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": false,
+  "refresh": "30s",
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "templating": {
+    "list": [
+      {
+        "name": "namespace",
+        "label": "Namespace",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": {
+          "query": "label_values(kube_namespace_status_phase, namespace)",
+          "refId": "ns"
+        },
+        "definition": "label_values(kube_namespace_status_phase, namespace)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "sort": 1,
+        "current": {
+          "selected": true,
+          "text": [
+            "nex"
+          ],
+          "value": [
+            "nex"
+          ]
+        }
+      }
+    ]
+  },
+  "annotations": {
+    "list": []
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Pods laufend",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"}) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Pods nicht bereit",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 4,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(kube_pod_status_ready{namespace=~\"$namespace\", condition=\"false\"}) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Neustarts (24 h)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 8,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "round(sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[24h]))) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Speicher (Arbeitssatz)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": null,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 5,
+      "type": "stat",
+      "title": "CPU (Kerne)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 16,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"cadvisor\", namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": null,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "k3s-Knoten bereit",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 20,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 5
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "Speicher je Pod",
+      "description": "Arbeitssatz (container_memory_working_set_bytes), danach richtet sich das OOM-Limit",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (pod) (container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})",
+          "legendFormat": "{{pod}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 8,
+      "type": "timeseries",
+      "title": "CPU je Pod",
+      "description": "Kerne, Mittel über 5 min",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{job=\"cadvisor\", namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": "{{pod}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 9,
+      "type": "table",
+      "title": "Pods",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 13,
+        "w": 24,
+        "h": 9
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (pod, node) (kube_pod_info{namespace=~\"$namespace\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "B",
+          "expr": "sum by (pod) (kube_pod_container_status_restarts_total{namespace=~\"$namespace\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "C",
+          "expr": "sum by (pod) (container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "D",
+          "expr": "sum by (pod) (kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "E",
+          "expr": "sum by (pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value #A": true
+            },
+            "indexByName": {
+              "pod": 0,
+              "node": 1,
+              "Value #B": 2,
+              "Value #C": 3,
+              "Value #D": 4,
+              "Value #E": 5
+            },
+            "renameByName": {
+              "pod": "Pod",
+              "node": "Knoten",
+              "Value #B": "Neustarts",
+              "Value #C": "Speicher",
+              "Value #D": "Request",
+              "Value #E": "Limit"
+            }
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Speicher|Request|Limit"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "bytes"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Neustarts"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "transparent",
+                      "value": null
+                    },
+                    {
+                      "color": "orange",
+                      "value": 1
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": [
+          {
+            "displayName": "Speicher",
+            "desc": true
+          }
+        ]
+      }
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "Pods je Knoten (alle Namespaces)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 22,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (node) (kube_pod_info)",
+          "legendFormat": "{{node}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "Speicher der Pods je Knoten (alle Namespaces)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 22,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (instance) (container_memory_working_set_bytes{job=\"cadvisor\"})",
+          "legendFormat": "{{instance}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 12,
+      "type": "bargauge",
+      "title": "Belegung der Volumes (PVC)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 30,
+        "w": 24,
+        "h": 6
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"})",
+          "legendFormat": "{{persistentvolumeclaim}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.8
+              },
+              {
+                "color": "red",
+                "value": 0.9
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    }
+  ]
+}
@@ -4,6 +4,7 @@
 #   node-exporter-full.json  ID 1860  „Node Exporter Full“ Revision 45
 #   nvidia-gpu.json          ID 14574 „Nvidia GPU Metrics“ Revision 15 (für utkuozdemir/nvidia_gpu_exporter)
 #   knaster-uebersicht.json  eigenes Dashboard (Startseite, für den Kiosk in Task 5.3)
+#   k3s-pods.json            eigenes Dashboard (Task 6.5): Pods je Namespace (Vorgabe nex), kube-state-metrics + cAdvisor
 apiVersion: 1
 providers:
  - name: knaster
@@ -11,6 +11,7 @@ Ausgerollt von ops mit `kubectl` (Rolle `k3s-apps`, Playbook `playbooks/k3s.yml`
 | `nex-searxng/` | NEX-Websuche SearXNG, NodePort 30880 (Playbook `playbooks/nex-searxng-k3s.yml`), Einstellungen aus dem nex-Repo |
 | `portainer/` | Portainer CE 2.45.1 auf der Dienst-VM k3s, Datenbank von appsrv08 übernommen (Playbook `playbooks/portainer-k3s.yml` mit Rollback BE→CE), NodePort 30779 |
 | `nex/` | Namespace nex für Agenten im Cluster: Brücke zu den NEX-Diensten auf appsrv08 (Services ohne Selector), Zugang für nex-admin (ServiceAccount, Role), Pull-Secret Gitea (Playbook `playbooks/nex-k3s.yml`); nex-Spec 2026-10-19 |
+| `monitoring/` | kube-state-metrics v2.19.1 (NodePort 30808) und ServiceAccount `prometheus-infra` (nur `nodes/metrics`) für Prometheus auf infra (Playbook `playbooks/monitoring-k3s.yml`, Task 6.5) |
 | `rustdesk/` | RustDesk-Server hbbs/hbbr, Service LoadBalancer 21115–21119 auf jeder Knoten-Adresse (Playbook `playbooks/rustdesk-k3s.yml`), Schlüssel im Secret `rustdesk-keys` |

 Portainer: Umgebung „Kubernetes → Agent“, Adresse `<IP eines Knotens>:30778`.
@@ -0,0 +1,88 @@
+# kube-state-metrics v2.19.1 (Task 6.5): Zustand der Objekte (Pods, Deployments, Knoten, PVCs) für Prometheus auf infra.
+# NodePort 30808 auf jedem Knoten; Prometheus fragt genau ein Ziel ab (file_sd, monitoring-targets.py).
+# Nur die Ressourcen, die Dashboard und Alerts brauchen (infra hat 4 GB RAM).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+rules:
+  - apiGroups: [""]
+    resources: [nodes, pods, namespaces, persistentvolumeclaims]
+    verbs: [list, watch]
+  - apiGroups: [apps]
+    resources: [deployments, daemonsets, statefulsets, replicasets]
+    verbs: [list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+  labels: {app: kube-state-metrics}
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: kube-state-metrics}
+  template:
+    metadata:
+      labels: {app: kube-state-metrics}
+    spec:
+      serviceAccountName: kube-state-metrics
+      automountServiceAccountToken: true
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: kube-state-metrics
+          image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1
+          args:
+            - --resources=nodes,pods,namespaces,persistentvolumeclaims,deployments,daemonsets,statefulsets,replicasets
+            - --port=8080
+            - --telemetry-port=8081
+          ports:
+            - {name: metrics, containerPort: 8080}
+            - {name: telemetry, containerPort: 8081}
+          readinessProbe:
+            httpGet: {path: /readyz, port: 8081}
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: {path: /livez, port: 8080}
+            initialDelaySeconds: 5
+          resources:
+            requests: {cpu: 10m, memory: 64Mi}
+            limits: {memory: 192Mi}
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: {drop: [ALL]}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  type: NodePort
+  selector: {app: kube-state-metrics}
+  ports:
+    - {name: metrics, port: 8080, targetPort: metrics, nodePort: 30808}
@@ -0,0 +1,40 @@
+# Lesezugang für Prometheus auf infra (Task 6.5): kubelet /metrics und /metrics/cadvisor auf Port 10250.
+# Der kubelet prüft das Token per SubjectAccessReview gegen nodes/metrics; mehr darf der ServiceAccount nicht.
+# Das Token (Secret prometheus-infra-token) legt playbooks/monitoring-k3s.yml auf ops ab, die Rolle
+# infra-stack kopiert es nach /opt/knaster/infra/secrets/k3s/.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-infra
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: prometheus-infra-token
+  namespace: kube-system
+  annotations:
+    kubernetes.io/service-account.name: prometheus-infra
+type: kubernetes.io/service-account-token
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-infra
+rules:
+  - apiGroups: [""]
+    resources: [nodes/metrics]
+    verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-infra
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-infra
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-infra
+    namespace: kube-system
@@ -47,3 +47,33 @@ groups:
        for: 15m
        labels: {severity: warning}
        annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}
+
+  # k3s (Task 6.5): nur melden, der Healer greift nicht ein (config/healer-rules.yaml, Aktion notify)
+  - name: knaster-k3s
+    rules:
+      - alert: K3sNodeNotReady
+        expr: kube_node_status_condition{condition="Ready", status="true"} == 0
+        for: 5m
+        labels: {severity: high}
+        annotations: {summary: "k3s-Knoten {{ $labels.node }} nicht bereit"}
+
+      - alert: K3sPodCrashLooping
+        expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) == 1
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) startet immer wieder neu"}
+
+      - alert: K3sDeploymentUnavailable
+        expr: max by (namespace, deployment) (kube_deployment_spec_replicas) > max by (namespace, deployment) (kube_deployment_status_replicas_available)
+        for: 15m
+        labels: {severity: warning}
+        annotations: {summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} hat zu wenige bereite Pods"}
+
+      - alert: K3sContainerMemoryNearLimit
+        expr: >
+          max by (namespace, pod, container) (container_memory_working_set_bytes{job="cadvisor"})
+          / on (namespace, pod, container)
+          max by (namespace, pod, container) (kube_pod_container_resource_limits{resource="memory"}) > 0.9
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) über 90 % des Speicherlimits"}
@@ -66,3 +66,39 @@ scrape_configs:
      - {source_labels: [__address__], target_label: __param_target}
      - {source_labels: [__param_target], target_label: instance}
      - {target_label: __address__, replacement: "pve-exporter:9221"}
+
+  # k3s (Task 6.5): kubelet direkt auf jedem Knoten (10250), TLS gegen die k3s-CA, Token des ServiceAccounts
+  # prometheus-infra (nur nodes/metrics). Ziele per file_sd aus `kubectl get nodes` (monitoring-targets.py auf ops).
+  # infra hat 4 GB RAM: von kubelet und cAdvisor nur die Reihen, die Dashboard „k3s – Pods“ und Alerts brauchen.
+  - job_name: kubelet
+    scheme: https
+    authorization: {credentials_file: /etc/knaster-k3s/token}
+    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
+    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: kubelet_running_pods|kubelet_running_containers|kubelet_volume_stats_(used|capacity)_bytes
+        action: keep
+
+  - job_name: cadvisor
+    scheme: https
+    metrics_path: /metrics/cadvisor
+    authorization: {credentials_file: /etc/knaster-k3s/token}
+    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
+    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_oom_events_total|container_network_(receive|transmit)_bytes_total
+        action: keep
+      # nur Pods (keine systemd-Slices, keine Docker-Container der Compose-Stacks)
+      - source_labels: [pod]
+        regex: ""
+        action: drop
+      # CPU/Speicher je Container: Pod-Summe und Pause-Container verwerfen (Netz gibt es nur je Pod)
+      - source_labels: [__name__, container]
+        regex: "container_(cpu|memory|oom)[^;]*;(POD)?"
+        action: drop
+
+  - job_name: kube-state-metrics
+    honor_labels: true   # namespace/pod/node kommen aus den Metriken selbst
+    file_sd_configs: [{files: [/etc/knaster-sd/kube-state-metrics.json]}]