k3s: Pod-Monitoring in Grafana (Task 6.5)

kube-state-metrics (NodePort 30808) und ServiceAccount prometheus-infra (nur
nodes/metrics) im Cluster; Prometheus auf infra liest kubelet/cAdvisor per TLS
gegen die k3s-CA. Ziele per file_sd aus kubectl get nodes, cAdvisor auf die
nötigen Reihen begrenzt. Dashboard "k3s – Pods" (Vorgabe nex), Alerts K3s*
nur mit notify. Playbook monitoring-k3s.yml.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

config/healer-rules.yaml

@@ -51,3 +51,24 @@ rules:
     match: {alertname: DiskAlmostFull}
     ladder:
       - {action: notify, params: {text: "Platte auf {instance} ({mountpoint}) fast voll", severity: warning}, attempts: 1}
+
+  # k3s (Task 6.5): nur melden; Pods startet Kubernetes selbst neu
+  - id: k3s-node-not-ready
+    match: {alertname: K3sNodeNotReady}
+    ladder:
+      - {action: notify, params: {text: "k3s-Knoten {node} nicht bereit – VM und k3s-agent prüfen", severity: high}, attempts: 1}
+
+  - id: k3s-pod-crashloop
+    match: {alertname: K3sPodCrashLooping}
+    ladder:
+      - {action: notify, params: {text: "Pod {namespace}/{pod} ({container}) startet immer wieder neu", severity: warning}, attempts: 1}
+
+  - id: k3s-deployment-unavailable
+    match: {alertname: K3sDeploymentUnavailable}
+    ladder:
+      - {action: notify, params: {text: "Deployment {namespace}/{deployment} hat zu wenige bereite Pods", severity: warning}, attempts: 1}
+
+  - id: k3s-memory-near-limit
+    match: {alertname: K3sContainerMemoryNearLimit}
+    ladder:
+      - {action: notify, params: {text: "{namespace}/{pod} ({container}) über 90 % des Speicherlimits", severity: warning}, attempts: 1}

core/tests/test_artifacts.py

@@ -39,13 +39,18 @@ def test_healer_labels_available_on_alerts():
     assert set(OPS_LABELS) == {"docker_host", "container", "pve_node", "vmid"}
 
 
-def _monitoring_targets() -> dict:
+def _targets_module():
     import importlib.util
 
     spec = importlib.util.spec_from_file_location(
         "monitoring_targets", REPO_ROOT / "platform/proxmox/scripts/monitoring-targets.py")
     mod = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(mod)
+    return mod
+
+
+def _monitoring_targets() -> dict:
+    mod = _targets_module()
     registry = yaml.safe_load((REPO_ROOT / "sites/knaster/registry.yaml").read_text(encoding="utf-8"))
     gpu_vms = {  # Form wie `tofu output -json gpu_vms`
         "gpu-pve44": {"node": "pve44", "vmid": 244, "ipv4": "192.168.100.186"},
@@ -67,6 +72,55 @@ def test_monitoring_targets_from_tofu_output():
     assert t["node"][2]["labels"]["slot"] == "none"   # gpu-pve52 trägt noch keinen Slot
 
 
+def _k3s_node(name: str, ip: str | None, ready: str = "True") -> dict:
+    addresses = [{"type": "Hostname", "address": name}] + ([{"type": "InternalIP", "address": ip}] if ip else [])
+    return {"metadata": {"name": name},
+            "status": {"addresses": addresses, "conditions": [{"type": "Ready", "status": ready}]}}
+
+
+def test_k3s_targets_from_kubectl_nodes():
+    nodes = {"items": [  # Form wie `kubectl get nodes -o json`
+        _k3s_node("gpu-pve52", "192.168.100.174"),
+        _k3s_node("k3s", "192.168.100.124"),
+        _k3s_node("gpu-pve44", "192.168.100.186", ready="False"),
+        _k3s_node("gpu-pve46", None),
+    ]}
+    t = _targets_module().build_k3s_targets(nodes)
+    assert [(e["targets"][0], e["labels"]["instance"]) for e in t["kubelet"]] == [
+        ("192.168.100.186:10250", "gpu-pve44"), ("192.168.100.174:10250", "gpu-pve52"),
+        ("192.168.100.124:10250", "k3s")]   # gpu-pve46 ohne IP fehlt
+    assert t["kube-state-metrics"] == [
+        {"targets": ["192.168.100.124:30808"], "labels": {"instance": "kube-state-metrics"}}]   # Dienst-VM bevorzugt
+    t = _targets_module().build_k3s_targets({"items": [nodes["items"][0], nodes["items"][2]]})
+    assert t["kube-state-metrics"][0]["targets"] == ["192.168.100.174:30808"]   # erster bereiter Knoten
+    assert _targets_module().build_k3s_targets({}) == {"kubelet": [], "kube-state-metrics": []}
+
+
+def test_k3s_jobs_use_generated_targets_and_nodeport():
+    prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text())
+    jobs = {j["job_name"]: j for j in prom["scrape_configs"]}
+    mod = _targets_module()
+    written = set(mod.build_k3s_targets({}))
+    for job, sd in {"kubelet": "kubelet", "cadvisor": "kubelet", "kube-state-metrics": "kube-state-metrics"}.items():
+        assert jobs[job]["file_sd_configs"] == [{"files": [f"/etc/knaster-sd/{sd}.json"]}]
+        assert sd in written
+    for job in ("kubelet", "cadvisor"):
+        assert jobs[job]["scheme"] == "https" and jobs[job]["tls_config"]["ca_file"].startswith("/etc/knaster-k3s/")
+    docs = list(yaml.safe_load_all((DEPLOY / "k8s/monitoring/kube-state-metrics.yaml").read_text()))
+    svc = next(d for d in docs if d["kind"] == "Service")
+    assert svc["spec"]["ports"][0]["nodePort"] == mod.KSM_NODEPORT
+
+
+DASHBOARDS = sorted((DEPLOY / "grafana/dashboards").glob("*.json"))
+
+
+@pytest.mark.parametrize("path", DASHBOARDS, ids=[p.name for p in DASHBOARDS])
+def test_grafana_dashboards_parse_with_unique_uid(path):
+    d = json.loads(path.read_text(encoding="utf-8"))
+    assert d["uid"] and d["title"] and d["panels"]
+    assert [x for x in DASHBOARDS if json.loads(x.read_text(encoding="utf-8"))["uid"] == d["uid"]] == [path]
+
+
 def test_alertmanager_targets_router_endpoint():
     am = yaml.safe_load((DEPLOY / "alertmanager/alertmanager.yml").read_text())
     url = am["receivers"][0]["webhook_configs"][0]["url"]

deploy/compose/infra/docker-compose.yml

@@ -70,6 +70,8 @@ services:
       - ../../prometheus:/etc/prometheus:ro
       # file_sd: Ziele der GPU-VMs (DHCP, IPs wechseln), erzeugt auf ops aus `tofu output -json gpu_vms`
       - ${KNASTER_STATE_DIR:-.}/file_sd:/etc/knaster-sd:ro
+      # Token und CA für kubelet/cAdvisor in k3s (Task 6.5), nur dieser Unterordner der Geheimnisse
+      - ${KNASTER_STATE_DIR:-.}/secrets/k3s:/etc/knaster-k3s:ro
       - prometheus:/prometheus
     extra_hosts: ["host.docker.internal:host-gateway"]   # node-exporter des LXC läuft im Host-Netz
     ports: ["9090:9090"]

deploy/grafana/dashboards/k3s-pods.json

@@ -0,0 +1,824 @@
+{
+  "uid": "k3s-pods",
+  "title": "k3s – Pods",
+  "tags": [
+    "knaster",
+    "k3s"
+  ],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": false,
+  "refresh": "30s",
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "templating": {
+    "list": [
+      {
+        "name": "namespace",
+        "label": "Namespace",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": {
+          "query": "label_values(kube_namespace_status_phase, namespace)",
+          "refId": "ns"
+        },
+        "definition": "label_values(kube_namespace_status_phase, namespace)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "sort": 1,
+        "current": {
+          "selected": true,
+          "text": [
+            "nex"
+          ],
+          "value": [
+            "nex"
+          ]
+        }
+      }
+    ]
+  },
+  "annotations": {
+    "list": []
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Pods laufend",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"}) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Pods nicht bereit",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 4,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(kube_pod_status_ready{namespace=~\"$namespace\", condition=\"false\"}) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Neustarts (24 h)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 8,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "round(sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[24h]))) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Speicher (Arbeitssatz)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": null,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 5,
+      "type": "stat",
+      "title": "CPU (Kerne)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 16,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"cadvisor\", namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": null,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "k3s-Knoten bereit",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 20,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 5
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "Speicher je Pod",
+      "description": "Arbeitssatz (container_memory_working_set_bytes), danach richtet sich das OOM-Limit",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (pod) (container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})",
+          "legendFormat": "{{pod}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 8,
+      "type": "timeseries",
+      "title": "CPU je Pod",
+      "description": "Kerne, Mittel über 5 min",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{job=\"cadvisor\", namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": "{{pod}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 9,
+      "type": "table",
+      "title": "Pods",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 13,
+        "w": 24,
+        "h": 9
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (pod, node) (kube_pod_info{namespace=~\"$namespace\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "B",
+          "expr": "sum by (pod) (kube_pod_container_status_restarts_total{namespace=~\"$namespace\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "C",
+          "expr": "sum by (pod) (container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "D",
+          "expr": "sum by (pod) (kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "E",
+          "expr": "sum by (pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})",
+          "legendFormat": "",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value #A": true
+            },
+            "indexByName": {
+              "pod": 0,
+              "node": 1,
+              "Value #B": 2,
+              "Value #C": 3,
+              "Value #D": 4,
+              "Value #E": 5
+            },
+            "renameByName": {
+              "pod": "Pod",
+              "node": "Knoten",
+              "Value #B": "Neustarts",
+              "Value #C": "Speicher",
+              "Value #D": "Request",
+              "Value #E": "Limit"
+            }
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "Speicher|Request|Limit"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "bytes"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Neustarts"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "transparent",
+                      "value": null
+                    },
+                    {
+                      "color": "orange",
+                      "value": 1
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": [
+          {
+            "displayName": "Speicher",
+            "desc": true
+          }
+        ]
+      }
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "Pods je Knoten (alle Namespaces)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 22,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (node) (kube_pod_info)",
+          "legendFormat": "{{node}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "Speicher der Pods je Knoten (alle Namespaces)",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 22,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum by (instance) (container_memory_working_set_bytes{job=\"cadvisor\"})",
+          "legendFormat": "{{instance}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {
+            "fillOpacity": 10,
+            "lineWidth": 1,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 12,
+      "type": "bargauge",
+      "title": "Belegung der Volumes (PVC)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 30,
+        "w": 24,
+        "h": 6
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"})",
+          "legendFormat": "{{persistentvolumeclaim}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.8
+              },
+              {
+                "color": "red",
+                "value": 0.9
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      }
+    }
+  ]
+}

deploy/grafana/provisioning/dashboards/knaster.yml

@@ -4,6 +4,7 @@
 #   node-exporter-full.json  ID 1860  „Node Exporter Full“ Revision 45
 #   nvidia-gpu.json          ID 14574 „Nvidia GPU Metrics“ Revision 15 (für utkuozdemir/nvidia_gpu_exporter)
 #   knaster-uebersicht.json  eigenes Dashboard (Startseite, für den Kiosk in Task 5.3)
+#   k3s-pods.json            eigenes Dashboard (Task 6.5): Pods je Namespace (Vorgabe nex), kube-state-metrics + cAdvisor
 apiVersion: 1
 providers:
   - name: knaster

deploy/k8s/README.md

@@ -11,6 +11,7 @@ Ausgerollt von ops mit `kubectl` (Rolle `k3s-apps`, Playbook `playbooks/k3s.yml`
 | `nex-searxng/` | NEX-Websuche SearXNG, NodePort 30880 (Playbook `playbooks/nex-searxng-k3s.yml`), Einstellungen aus dem nex-Repo |
 | `portainer/` | Portainer CE 2.45.1 auf der Dienst-VM k3s, Datenbank von appsrv08 übernommen (Playbook `playbooks/portainer-k3s.yml` mit Rollback BE→CE), NodePort 30779 |
 | `nex/` | Namespace nex für Agenten im Cluster: Brücke zu den NEX-Diensten auf appsrv08 (Services ohne Selector), Zugang für nex-admin (ServiceAccount, Role), Pull-Secret Gitea (Playbook `playbooks/nex-k3s.yml`); nex-Spec 2026-10-19 |
+| `monitoring/` | kube-state-metrics v2.19.1 (NodePort 30808) und ServiceAccount `prometheus-infra` (nur `nodes/metrics`) für Prometheus auf infra (Playbook `playbooks/monitoring-k3s.yml`, Task 6.5) |
 | `rustdesk/` | RustDesk-Server hbbs/hbbr, Service LoadBalancer 21115–21119 auf jeder Knoten-Adresse (Playbook `playbooks/rustdesk-k3s.yml`), Schlüssel im Secret `rustdesk-keys` |
 
 Portainer: Umgebung „Kubernetes → Agent“, Adresse `<IP eines Knotens>:30778`.

deploy/k8s/monitoring/kube-state-metrics.yaml

@@ -0,0 +1,88 @@
+# kube-state-metrics v2.19.1 (Task 6.5): Zustand der Objekte (Pods, Deployments, Knoten, PVCs) für Prometheus auf infra.
+# NodePort 30808 auf jedem Knoten; Prometheus fragt genau ein Ziel ab (file_sd, monitoring-targets.py).
+# Nur die Ressourcen, die Dashboard und Alerts brauchen (infra hat 4 GB RAM).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+rules:
+  - apiGroups: [""]
+    resources: [nodes, pods, namespaces, persistentvolumeclaims]
+    verbs: [list, watch]
+  - apiGroups: [apps]
+    resources: [deployments, daemonsets, statefulsets, replicasets]
+    verbs: [list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+  labels: {app: kube-state-metrics}
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: kube-state-metrics}
+  template:
+    metadata:
+      labels: {app: kube-state-metrics}
+    spec:
+      serviceAccountName: kube-state-metrics
+      automountServiceAccountToken: true
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: kube-state-metrics
+          image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1
+          args:
+            - --resources=nodes,pods,namespaces,persistentvolumeclaims,deployments,daemonsets,statefulsets,replicasets
+            - --port=8080
+            - --telemetry-port=8081
+          ports:
+            - {name: metrics, containerPort: 8080}
+            - {name: telemetry, containerPort: 8081}
+          readinessProbe:
+            httpGet: {path: /readyz, port: 8081}
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: {path: /livez, port: 8080}
+            initialDelaySeconds: 5
+          resources:
+            requests: {cpu: 10m, memory: 64Mi}
+            limits: {memory: 192Mi}
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: {drop: [ALL]}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  type: NodePort
+  selector: {app: kube-state-metrics}
+  ports:
+    - {name: metrics, port: 8080, targetPort: metrics, nodePort: 30808}

deploy/k8s/monitoring/prometheus-access.yaml

@@ -0,0 +1,40 @@
+# Lesezugang für Prometheus auf infra (Task 6.5): kubelet /metrics und /metrics/cadvisor auf Port 10250.
+# Der kubelet prüft das Token per SubjectAccessReview gegen nodes/metrics; mehr darf der ServiceAccount nicht.
+# Das Token (Secret prometheus-infra-token) legt playbooks/monitoring-k3s.yml auf ops ab, die Rolle
+# infra-stack kopiert es nach /opt/knaster/infra/secrets/k3s/.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-infra
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: prometheus-infra-token
+  namespace: kube-system
+  annotations:
+    kubernetes.io/service-account.name: prometheus-infra
+type: kubernetes.io/service-account-token
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-infra
+rules:
+  - apiGroups: [""]
+    resources: [nodes/metrics]
+    verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-infra
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-infra
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-infra
+    namespace: kube-system

deploy/prometheus/alerts.yml

@@ -47,3 +47,33 @@ groups:
         for: 15m
         labels: {severity: warning}
         annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}
+
+  # k3s (Task 6.5): nur melden, der Healer greift nicht ein (config/healer-rules.yaml, Aktion notify)
+  - name: knaster-k3s
+    rules:
+      - alert: K3sNodeNotReady
+        expr: kube_node_status_condition{condition="Ready", status="true"} == 0
+        for: 5m
+        labels: {severity: high}
+        annotations: {summary: "k3s-Knoten {{ $labels.node }} nicht bereit"}
+
+      - alert: K3sPodCrashLooping
+        expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) == 1
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) startet immer wieder neu"}
+
+      - alert: K3sDeploymentUnavailable
+        expr: max by (namespace, deployment) (kube_deployment_spec_replicas) > max by (namespace, deployment) (kube_deployment_status_replicas_available)
+        for: 15m
+        labels: {severity: warning}
+        annotations: {summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} hat zu wenige bereite Pods"}
+
+      - alert: K3sContainerMemoryNearLimit
+        expr: >
+          max by (namespace, pod, container) (container_memory_working_set_bytes{job="cadvisor"})
+          / on (namespace, pod, container)
+          max by (namespace, pod, container) (kube_pod_container_resource_limits{resource="memory"}) > 0.9
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) über 90 % des Speicherlimits"}

deploy/prometheus/prometheus.yml

@@ -66,3 +66,39 @@ scrape_configs:
       - {source_labels: [__address__], target_label: __param_target}
       - {source_labels: [__param_target], target_label: instance}
       - {target_label: __address__, replacement: "pve-exporter:9221"}
+
+  # k3s (Task 6.5): kubelet direkt auf jedem Knoten (10250), TLS gegen die k3s-CA, Token des ServiceAccounts
+  # prometheus-infra (nur nodes/metrics). Ziele per file_sd aus `kubectl get nodes` (monitoring-targets.py auf ops).
+  # infra hat 4 GB RAM: von kubelet und cAdvisor nur die Reihen, die Dashboard „k3s – Pods“ und Alerts brauchen.
+  - job_name: kubelet
+    scheme: https
+    authorization: {credentials_file: /etc/knaster-k3s/token}
+    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
+    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: kubelet_running_pods|kubelet_running_containers|kubelet_volume_stats_(used|capacity)_bytes
+        action: keep
+
+  - job_name: cadvisor
+    scheme: https
+    metrics_path: /metrics/cadvisor
+    authorization: {credentials_file: /etc/knaster-k3s/token}
+    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
+    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
+    metric_relabel_configs:
+      - source_labels: [__name__]
+        regex: container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_oom_events_total|container_network_(receive|transmit)_bytes_total
+        action: keep
+      # nur Pods (keine systemd-Slices, keine Docker-Container der Compose-Stacks)
+      - source_labels: [pod]
+        regex: ""
+        action: drop
+      # CPU/Speicher je Container: Pod-Summe und Pause-Container verwerfen (Netz gibt es nur je Pod)
+      - source_labels: [__name__, container]
+        regex: "container_(cpu|memory|oom)[^;]*;(POD)?"
+        action: drop
+
+  - job_name: kube-state-metrics
+    honor_labels: true   # namespace/pod/node kommen aus den Metriken selbst
+    file_sd_configs: [{files: [/etc/knaster-sd/kube-state-metrics.json]}]

docs/BETRIEB.md

@@ -129,7 +129,9 @@ Prometheus liest die Dateien in `/opt/knaster/infra/file_sd` selbst neu, ein Neu
 
 **Alerts:** Alertmanager meldet an den Router (`/v1/healer/alertmanager`, Healer im Dry-Run). Der Receiver für n8n `knaster-notify` ist in `deploy/alertmanager/alertmanager.yml` nur vorbereitet (auskommentiert). Test: `docker exec knaster-infra-alertmanager-1 amtool --alertmanager.url=http://localhost:9093 alert add test severity=warning` und `… alert query`.
 
-**Dashboards** (Ordner `knaster`, Startseite „knaster – Übersicht“): Proxmox via Prometheus (grafana.com 10347), Node Exporter Full (1860), Nvidia GPU Metrics (14574), knaster – Übersicht (eigenes).
+**Dashboards** (Ordner `knaster`, Startseite „knaster – Übersicht“): Proxmox via Prometheus (grafana.com 10347), Node Exporter Full (1860), Nvidia GPU Metrics (14574), knaster – Übersicht (eigenes), k3s – Pods (eigenes, Task 6.5).
+
+**k3s-Pods (Task 6.5, seit 19.10.):** Prometheus fragt drei Jobs ab. `kubelet` und `cadvisor` gehen direkt an Port 10250 jedes k3s-Knotens (TLS gegen die k3s-CA, Token des ServiceAccounts `prometheus-infra`, darf nur `nodes/metrics` lesen). `kube-state-metrics` geht an NodePort 30808 (bevorzugt auf der Dienst-VM k3s). Die Ziele erzeugt `monitoring-targets.py` aus `kubectl get nodes` auf ops (`kubelet.json`, `kube-state-metrics.json`), sie werden mit `playbooks/monitoring-targets.yml` aufgefrischt wie die der GPU-VMs. Token und CA: auf ops `/root/.config/knaster/k3s-prometheus.token` und `k3s-ca.crt`, im LXC `/opt/knaster/infra/secrets/k3s/` (nur dieser Ordner ist in Prometheus eingehängt). Von cAdvisor bleiben nur CPU, Speicher, OOM und Netz je Pod (`metric_relabel_configs`), weil infra nur 4 GB hat. Alerts `K3s*` (Knoten nicht bereit, CrashLoop, Deployment ohne bereite Pods, Speicher > 90 % des Limits) melden nur, der Healer greift nicht ein. Ausrollen: `ansible-playbook playbooks/monitoring-k3s.yml` (Cluster-Teil, dann Infra-Stack).
 
 **Achtung HA-Umzug und ZFS-Seiten-Cache (gefunden 12.10.2026 bei der HA-Probe)**
 Nach `zfs receive` (Replikation) liefert ein Knoten für Dateien, die dort früher per mmap geöffnet waren, weiter den alten Inhalt aus seinem Seiten-Cache (ZFS 2.4.4, Kernel 7.0.14; Beleg auf pve44: `zfs diff <snap>` leer, `written` 0, md5 der Datei trotzdem ungleich der im Snapshot). Folge beim ersten Umzug nach pve48: dockerd las veraltete bbolt-Datenbanken, brach mit `panic: invalid freelist page` ab, danach fehlten Netzwerk und Volume-Labels. Betroffen ist jeder replizierte LXC mit mmap-Nutzern (Docker, Prometheus-TSDB, später `grp-test`), sobald er auf einen Knoten zurückkehrt, auf dem er schon lief.
@@ -221,7 +223,8 @@ Die GPU-VMs bilden mit der Dienst-VM `k3s` einen k3s-Cluster (v1.36.5+k3s1). Ste
 | Server-Adresse | per DHCP (heute 192.168.100.124, MAC `bc:24:11:a5:d5:52`); Agents nutzen den Namen `k3s.knaster` aus `/etc/hosts` (Rolle k3s-agent). Ändert sich die Adresse: `refresh-vm-access.sh`, dann `playbooks/k3s.yml` |
 | Token und kubeconfig | nur auf ops: `/root/.config/knaster/k3s.env`, `/root/.kube/config`; `kubectl` liegt auf ops |
 | Manifeste | `deploy/k8s/` (RuntimeClass `nvidia`, NVIDIA Device Plugin v0.20.1, Portainer Agent 2.45.2, `nex-speech/`, `nex-searxng/`, `rustdesk/`) |
-| Playbooks | `playbooks/k3s.yml` (Server, Agents, Grundausstattung), `playbooks/nex-speech-k3s.yml`, `playbooks/nex-searxng-k3s.yml`, `playbooks/rustdesk-k3s.yml` (Umzüge) |
+| Playbooks | `playbooks/k3s.yml` (Server, Agents, Grundausstattung), `playbooks/nex-speech-k3s.yml`, `playbooks/nex-searxng-k3s.yml`, `playbooks/rustdesk-k3s.yml` (Umzüge), `playbooks/monitoring-k3s.yml` (Pod-Monitoring) |
+| Monitoring | kube-state-metrics v2.19.1 in `kube-system` (NodePort 30808), Prometheus auf infra liest kubelet/cAdvisor; Grafana-Dashboard „k3s – Pods“ (Abschnitt 7). `kubectl top` zeigt weiter den metrics-server von k3s |
 | GPU in Pods | `runtimeClassName: nvidia` + `resources.limits: {nvidia.com/gpu: 1}`; Knoten-Labels `knaster.gpu=true`, `knaster.gpu.m4000=1`, `knaster.gpu.p2000=1` |
 | Portainer | Umgebung „Kubernetes → Agent“ mit Adresse `<IP eines Knotens>:30778`, z. B. `192.168.100.124:30778` (Agent-Pod läuft im Namespace `portainer`) |
 

docs/superpowers/plans/2026-10-12-aufbau-cluster-knaster.md

@@ -382,15 +382,17 @@ Ersetzt PLAN §8 „1 Data Plane“.
     - `test_worker_registry_slot_falls_back_to_tier0_when_list_empty`
     - `test_worker_registry_slot_falls_back_when_list_unreachable`
   - Prüfung: `make test` PASS. Live tagsüber geht eine tier1-Anfrage an tier0, nachts an GODE-PC-05.
-- [ ] **6.5 Pod-Monitoring (k3s in Grafana)** · **Planner:** „Proxmox: Grafana-Dashboard für den Cluster“ · **Files:** Create `deploy/k8s/kube-state-metrics/`, `platform/proxmox/ansible/playbooks/kube-state-metrics-k3s.yml`, `deploy/grafana/dashboards/k3s-pods.json`; Modify `deploy/prometheus/prometheus.yml`, `platform/proxmox/scripts/monitoring-targets.py`, Rolle `infra-stack` (Token), `core/tests/test_artifacts.py`, BETRIEB §7/§14
+- [ ] **6.5 Pod-Monitoring (k3s in Grafana)** · **Planner:** „Proxmox: Grafana-Dashboard für den Cluster“ · **Files:** Create `deploy/k8s/monitoring/`, `platform/proxmox/ansible/playbooks/monitoring-k3s.yml`, `deploy/grafana/dashboards/k3s-pods.json`; Modify `deploy/prometheus/prometheus.yml`, `deploy/prometheus/alerts.yml`, `config/healer-rules.yaml`, `platform/proxmox/scripts/monitoring-targets.py`, Rolle `infra-stack` (Token, Ziele), `deploy/compose/infra/docker-compose.yml`, `core/tests/test_artifacts.py`, BETRIEB §7/§14
   Ausgangslage 19.10.: Grafana 12.4 und Prometheus laufen auf infra mit Dashboards für Proxmox, Knoten, GPU und knaster; für k3s gibt es kein Ziel. Zustand und Speicher der Pods zeigt bisher nur nex-admin über die Metrics-API (metrics-server, nur Momentwerte, keine Grafana-Datenquelle).
-  - [ ] kube-state-metrics als Deployment im Namespace `kube-system` (Image gepinnt), Service als NodePort. Liefert Zustand: Bereitschaft, Neustarts, Requests/Limits, Deployments.
+  - [x] (19.10.) kube-state-metrics v2.19.1 im Namespace `kube-system` (`deploy/k8s/monitoring/kube-state-metrics.yaml`, nur die nötigen Ressourcen), Service NodePort 30808.
-  - [ ] ServiceAccount `prometheus-infra` mit ClusterRole nur zum Lesen (`nodes`, `nodes/metrics`, `nodes/proxy`), langlebiges Token als Secret. Das Token liegt wie beim pve-exporter nur im LXC unter `/opt/knaster/infra/secrets/k3s-prometheus.token`, kopiert von der Rolle `infra-stack`; die k3s-CA dazu (`/var/lib/rancher/k3s/server/tls/server-ca.crt`), damit kein `insecure_skip_verify` nötig ist. Kubelet-Zertifikate der Agents gegen diese CA prüfen, sonst bewusst entscheiden.
+  - [x] (19.10.) ServiceAccount `prometheus-infra`, ClusterRole nur `get nodes/metrics`, langlebiges Token-Secret. Token und CA legt `playbooks/monitoring-k3s.yml` auf ops ab (`/root/.config/knaster/k3s-prometheus.token`, `k3s-ca.crt`), die Rolle `infra-stack` kopiert sie nach `/opt/knaster/infra/secrets/k3s/` (nur dieser Ordner in Prometheus eingehängt). Die kubelet-Zertifikate sind von `k3s-server-ca` signiert und tragen die Knoten-IP (geprüft auf gpu-pve52), daher TLS ohne `insecure_skip_verify`.
-  - [ ] Ziele per file_sd statt fester Adressen, weil die k3s-VM (.124) und gpu-pve44/46/48 noch keine Reservierung haben: `monitoring-targets.py` schreibt zusätzlich `kubelet.json` (alle fünf k3s-Knoten, Port 10250, Pfade `/metrics/cadvisor` und `/metrics`) und `kube-state-metrics.json` (NodePort auf jedem Knoten, Prometheus nimmt den ersten erreichbaren bzw. dedupliziert per `instance`).
+  - [x] (19.10.) Ziele per file_sd aus `kubectl get nodes` (statt tofu, so ist die k3s-VM dabei): `monitoring-targets.py --k3s-nodes` schreibt `kubelet.json` (alle Knoten, 10250) und `kube-state-metrics.json` (ein Ziel, NodePort auf der Dienst-VM k3s, sonst erster bereiter Knoten; `instance` fest, damit ein Wechsel keine neuen Reihen erzeugt).
-  - [ ] Prometheus-Jobs `kubelet`, `cadvisor`, `kube-state-metrics` in `deploy/prometheus/prometheus.yml`. infra hat nur 4 GB: für cAdvisor `metric_relabel_configs`, die nur `container_cpu_usage_seconds_total`, `container_memory_working_set_bytes`, `container_memory_rss` und Netzwerk behalten und `container=""`/Pause-Container verwerfen. Danach Speicher von Prometheus eine Woche beobachten.
+  - [x] (19.10.) Prometheus-Jobs `kubelet`, `cadvisor`, `kube-state-metrics`. cAdvisor behält nur CPU, Arbeitssatz, RSS, OOM und Netz und verwirft Reihen ohne Pod sowie Pod-Summen/Pause-Container.
-  - [ ] ufw in den GPU-VMs (gpu-pve52 lässt heute nur 8178/8179 von .29 zu): 10250 und den NodePort nur von `.188` freigeben (Rolle `k3s-agent` bzw. `gpu-exporters`).
+  - [ ] Speicher von Prometheus eine Woche beobachten (infra hat 4 GB).
-  - [ ] Dashboard „k3s – Pods“ (Grundlage Community-Dashboard, z. B. grafana.com 15661/15760, angepasst auf die vorhandenen Metriken) mit Namespace-Auswahl, Vorgabe `nex`: Pods je Knoten, Neustarts, CPU/Speicher gegen Requests/Limits. In die Kiosk-Playlist (5.3) aufnehmen.
+  - [x] ufw: Die Rolle `k3s-agent` erlaubt 10250 und 30000–32767 schon seit 15.10. aus dem LAN, keine Änderung nötig.
-  - [ ] Alerts in `deploy/prometheus/alerts.yml`: Pod in CrashLoopBackOff > 10 min, Deployment nicht verfügbar, Speicher > 90 % des Limits. Nur melden, kein Healer-Eingriff (der heilt nur VMs).
+  - [x] (19.10.) Dashboard „k3s – Pods“ (`deploy/grafana/dashboards/k3s-pods.json`, eigenes statt Community, passend zu den behaltenen Metriken): Namespace-Auswahl mit Vorgabe `nex`, Kennzahlen, Speicher/CPU je Pod, Tabelle mit Knoten, Neustarts, Speicher gegen Request/Limit, Pods und Speicher je Knoten, PVC-Belegung.
+  - [ ] Dashboard in die Kiosk-Playlist (5.3) aufnehmen, sobald es die Playlist gibt.
+  - [x] (19.10.) Alerts `K3sNodeNotReady`, `K3sPodCrashLooping` (> 10 min), `K3sDeploymentUnavailable` (> 15 min), `K3sContainerMemoryNearLimit` (> 90 %, 10 min); Healer-Regeln nur `notify`.
   - Prüfung: `make test` PASS (Struktur von prometheus.yml/alerts.yml). Auf infra zeigt `api/v1/targets` die Jobs `kubelet`, `cadvisor`, `kube-state-metrics` mit allen fünf Knoten `up`. Grafana zeigt für `nex` dieselben Pods und ungefähr denselben Speicher wie `kubectl top pods -n nex`. Ein absichtlich kaputtes Test-Deployment löst den CrashLoop-Alert aus.
 
 ---

platform/proxmox/ansible/playbooks/monitoring-k3s.yml

@@ -0,0 +1,74 @@
+# Pod-Monitoring (Task 6.5): kube-state-metrics und Lesezugang für Prometheus in k3s, danach der Infra-Stack
+# (Prometheus-Jobs kubelet/cadvisor/kube-state-metrics, file_sd-Ziele, Dashboard „k3s – Pods“). Läuft auf ops:
+#   ansible-playbook playbooks/monitoring-k3s.yml
+# Token und CA landen auf ops unter /root/.config/knaster (0600), die Rolle infra-stack kopiert sie nach infra.
+- name: k3s – kube-state-metrics und Zugang für Prometheus
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    manifests: "{{ playbook_dir }}/../../../../deploy/k8s/monitoring"
+    token_file: /root/.config/knaster/k3s-prometheus.token
+    ca_file: /root/.config/knaster/k3s-ca.crt
+  tasks:
+    - name: Manifeste anwenden
+      ansible.builtin.command: kubectl apply -f {{ manifests }}/{{ item }}
+      loop: [prometheus-access.yaml, kube-state-metrics.yaml]
+      register: apply
+      changed_when: "'configured' in apply.stdout or 'created' in apply.stdout"
+
+    - name: Warten, bis kube-state-metrics bereit ist
+      ansible.builtin.command: kubectl -n kube-system rollout status deployment/kube-state-metrics --timeout=180s
+      changed_when: false
+
+    - name: Token des ServiceAccounts prometheus-infra lesen (Secret wird vom Controller befüllt)
+      ansible.builtin.shell: >
+        set -o pipefail;
+        kubectl -n kube-system get secret prometheus-infra-token -o go-template='{{ "{{" }}.data.token{{ "}}" }}' | base64 -d
+      args: {executable: /bin/bash}
+      register: token
+      until: token.stdout | length > 0
+      retries: 10
+      delay: 2
+      changed_when: false
+      no_log: true
+
+    - name: Token als Datei auf ops ablegen (600, nie ausgeben)
+      ansible.builtin.copy:
+        content: "{{ token.stdout }}"
+        dest: "{{ token_file }}"
+        mode: "0600"
+      no_log: true
+
+    # Die kubelet-Zertifikate sind von k3s-server-ca signiert und tragen die Knoten-IP (19.10. geprüft).
+    - name: CA des Clusters lesen
+      ansible.builtin.shell: >
+        set -o pipefail;
+        kubectl -n kube-system get secret prometheus-infra-token -o go-template='{{ "{{" }}index .data "ca.crt"{{ "}}" }}' | base64 -d
+      args: {executable: /bin/bash}
+      register: ca
+      changed_when: false
+
+    - name: CA als Datei auf ops ablegen
+      ansible.builtin.copy:
+        content: "{{ ca.stdout }}\n"
+        dest: "{{ ca_file }}"
+        mode: "0600"
+
+    - name: Zugang prüfen (cAdvisor eines Knotens mit Token und CA)
+      ansible.builtin.shell: >
+        set -o pipefail;
+        ip=$(kubectl get node k3s -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}');
+        curl -fsS -m 10 --cacert {{ ca_file }} -H "Authorization: Bearer $(cat {{ token_file }})"
+        https://$ip:10250/metrics/cadvisor | grep -c '^container_memory_working_set_bytes'
+      args: {executable: /bin/bash}
+      register: probe
+      changed_when: false
+
+    - name: Ergebnis
+      ansible.builtin.debug:
+        msg: "cAdvisor auf k3s erreichbar, {{ probe.stdout }} Reihen container_memory_working_set_bytes"
+
+- name: Infra-LXC – Prometheus, Ziele und Dashboard
+  hosts: infra
+  roles: [infra-stack]

platform/proxmox/ansible/roles/infra-stack/defaults/main.yml

@@ -24,6 +24,10 @@ infra_secrets: [POSTGRES_PASSWORD, KNASTER_API_TOKEN, GRAFANA_ADMIN_PASSWORD, GR
 # Lese-Token monitoring@pve!prom (PVEAuditor) auf ops
 infra_pve_exporter_env: /root/.config/knaster/pve-exporter.env
 
+# Lesezugang zum kubelet für Prometheus (Task 6.5), auf ops von playbooks/monitoring-k3s.yml abgelegt
+infra_k3s_token: /root/.config/knaster/k3s-prometheus.token
+infra_k3s_ca: /root/.config/knaster/k3s-ca.crt
+
 # file_sd: auf ops aus `tofu output -json gpu_vms` erzeugt (nicht im Repo)
 infra_tofu_dir: "{{ playbook_dir }}/../../tofu"
 infra_targets_dir: "{{ playbook_dir }}/../monitoring-targets.generated"

platform/proxmox/ansible/roles/infra-stack/tasks/main.yml

@@ -17,6 +17,7 @@
     - {path: "{{ infra_repo_dir }}", mode: "0755"}
     - {path: "{{ infra_state_dir }}", mode: "0700"}
     - {path: "{{ infra_state_dir }}/secrets", mode: "0700"}
+    - {path: "{{ infra_state_dir }}/secrets/k3s", mode: "0755"}
 
 # Nur die Teile, die Build und Konfiguration brauchen. Nie .git, .venv, tofu-State oder .env-Dateien.
 - name: Repo-Auszug von ops abgleichen
@@ -87,6 +88,30 @@
     mode: "0600"
   no_log: true
 
+# Task 6.5: Token des ServiceAccounts prometheus-infra und CA des Clusters (von playbooks/monitoring-k3s.yml
+# auf ops abgelegt). Lesbar für nobody (Prometheus im Container). Fehlen sie, bleiben die k3s-Jobs down.
+- name: k3s-Zugang für Prometheus vorhanden? (auf ops)
+  ansible.builtin.stat:
+    path: "{{ item }}"
+  loop: ["{{ infra_k3s_token }}", "{{ infra_k3s_ca }}"]
+  delegate_to: localhost
+  register: k3s_access
+
+- name: k3s-Zugang für Prometheus kopieren
+  ansible.builtin.copy:
+    src: "{{ item.src }}"
+    dest: "{{ infra_state_dir }}/secrets/k3s/{{ item.dest }}"
+    owner: "65534"
+    group: "65534"
+    mode: "0400"
+  loop:
+    - {src: "{{ infra_k3s_token }}", dest: token}
+    - {src: "{{ infra_k3s_ca }}", dest: ca.crt}
+  loop_control:
+    label: "{{ item.dest }}"
+  when: k3s_access.results | map(attribute='stat.exists') | min
+  no_log: true
+
 - name: file_sd-Ziele
   ansible.builtin.include_tasks: targets.yml
 

platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml

@@ -1,11 +1,24 @@
 # file_sd-Ziele der GPU-VMs (node, gpu, llamacpp). Auch einzeln aufrufbar: playbooks/monitoring-targets.yml
 # (nach refresh-vm-access.sh, wenn sich DHCP-Adressen geändert haben). Prometheus liest die Dateien selbst neu.
+# k3s (Task 6.5): kubelet und kube-state-metrics aus `kubectl get nodes` auf ops. Ist der Cluster nicht
+# erreichbar, bleiben diese Listen leer und der Rest läuft weiter.
+- name: k3s-Knoten lesen (auf ops)
+  ansible.builtin.shell: mkdir -p {{ infra_targets_dir }} && kubectl get nodes -o json > {{ infra_targets_dir }}/k3s-nodes.json
+  args:
+    executable: /bin/bash
+  delegate_to: localhost
+  run_once: true
+  register: k3s_nodes
+  failed_when: false
+  changed_when: false
+
 - name: file_sd-Ziele aus tofu output erzeugen (auf ops)
   ansible.builtin.shell: |
     set -euo pipefail
     tofu output -json gpu_vms | python3 {{ infra_repo_src }}/platform/proxmox/scripts/monitoring-targets.py \
       --registry {{ infra_repo_src }}/sites/{{ knaster_site }}/registry.yaml --out {{ infra_targets_dir }} \
-      --llamacpp '{{ dict(_llamacpp_hosts | zip(_llamacpp_hosts | map('extract', hostvars, 'llamacpp_profiles'))) | to_json }}'
+      --llamacpp '{{ dict(_llamacpp_hosts | zip(_llamacpp_hosts | map('extract', hostvars, 'llamacpp_profiles'))) | to_json }}' \
+      {{ ('--k3s-nodes ' ~ infra_targets_dir ~ '/k3s-nodes.json') if k3s_nodes.rc == 0 else '' }}
   args:
     chdir: "{{ infra_tofu_dir }}"
     executable: /bin/bash
@@ -27,4 +40,4 @@
     src: "{{ infra_targets_dir }}/{{ item }}.json"
     dest: "{{ infra_state_dir }}/file_sd/{{ item }}.json"
     mode: "0644"
-  loop: [node, gpu, llamacpp]
+  loop: [node, gpu, llamacpp, kubelet, kube-state-metrics]

platform/proxmox/scripts/monitoring-targets.py

@@ -18,6 +18,11 @@ Registry (die kennt je Slot nur eine VM, die zweite Embedding-Instanz fehlt dort
 `--llamacpp '{"gpu-pve44": ["embed"], ...}'`. Die Slots eines Dienstes kommen aus der Registry
 (engine llamacpp, gleicher Container). Labels zusätzlich docker_host (= VM) und container, damit der
 Healer die tatsächliche VM trifft. Diese Slots zählen auch für node/gpu der VM.
+
+k3s (Task 6.5): Mit `--k3s-nodes DATEI` (Ausgabe von `kubectl get nodes -o json`) entstehen zusätzlich
+DIR/kubelet.json (jeder Knoten, Port 10250, für die Jobs kubelet und cadvisor) und
+DIR/kube-state-metrics.json (ein Ziel: NodePort auf der Dienst-VM k3s, sonst auf dem ersten bereiten Knoten).
+Ohne die Option bleiben beide Listen leer.
 """
 
 from __future__ import annotations
@@ -35,6 +40,9 @@ LLAMACPP_PROFILES = {
     "embed": [("llamacpp-embed", 8081), ("llamacpp-rerank", 8082)],
     "chat": [("llamacpp-chat", 8080)],
 }
+KUBELET_PORT = 10250
+KSM_NODEPORT = 30808   # deploy/k8s/monitoring/kube-state-metrics.yaml
+K3S_SERVER = "k3s"     # Dienst-VM mit der Steuerung (HA), bevorzugtes Ziel für kube-state-metrics
 
 
 def slots_by_vmid(registry: dict) -> dict[str, list[str]]:
@@ -89,11 +97,41 @@ def build_targets(gpu_vms: dict, registry: dict,
     return result
 
 
+def _internal_ip(node: dict) -> str | None:
+    return next((a["address"] for a in node.get("status", {}).get("addresses", [])
+                 if a.get("type") == "InternalIP"), None)
+
+
+def _ready(node: dict) -> bool:
+    return any(c.get("type") == "Ready" and c.get("status") == "True"
+               for c in node.get("status", {}).get("conditions", []))
+
+
+def build_k3s_targets(nodes: dict) -> dict[str, list[dict]]:
+    """Ziele aus `kubectl get nodes -o json`. instance ist der Knotenname (bleibt bei IP-Wechsel gleich)."""
+    kubelet: list[dict] = []
+    ready: list[tuple[str, str]] = []
+    for node in sorted(nodes.get("items", []), key=lambda n: n["metadata"]["name"]):
+        name, ip = node["metadata"]["name"], _internal_ip(node)
+        if not ip:
+            print(f"Warnung: k3s-Knoten {name} ohne InternalIP, übersprungen", file=sys.stderr)
+            continue
+        kubelet.append({"targets": [f"{ip}:{KUBELET_PORT}"], "labels": {"instance": name, "node": name}})
+        if _ready(node):
+            ready.append((name, ip))
+    ksm: list[dict] = []
+    if ready:
+        _, ip = next((r for r in ready if r[0] == K3S_SERVER), ready[0])
+        ksm.append({"targets": [f"{ip}:{KSM_NODEPORT}"], "labels": {"instance": "kube-state-metrics"}})
+    return {"kubelet": kubelet, "kube-state-metrics": ksm}
+
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     ap.add_argument("--registry", required=True, type=Path)
     ap.add_argument("--out", required=True, type=Path)
     ap.add_argument("--llamacpp", default="{}", help='JSON {"<vm>": ["embed"|"chat", ...]} aus dem Inventar')
+    ap.add_argument("--k3s-nodes", type=Path, help="Ausgabe von `kubectl get nodes -o json` (Task 6.5)")
     args = ap.parse_args()
 
     gpu_vms = json.load(sys.stdin)
@@ -103,6 +141,8 @@ def main() -> int:
     if unknown:
         ap.error(f"unbekannte llama.cpp-Profile: {sorted(unknown)}")
     targets = build_targets(gpu_vms, registry, llamacpp)
+    k3s_nodes = json.loads(args.k3s_nodes.read_text(encoding="utf-8")) if args.k3s_nodes else {}
+    targets.update(build_k3s_targets(k3s_nodes))
     args.out.mkdir(parents=True, exist_ok=True)
     for job, entries in targets.items():
         path = args.out / f"{job}.json"