diff --git a/config/healer-rules.yaml b/config/healer-rules.yaml index b29bc77..f1aff71 100644 --- a/config/healer-rules.yaml +++ b/config/healer-rules.yaml @@ -51,3 +51,24 @@ rules: match: {alertname: DiskAlmostFull} ladder: - {action: notify, params: {text: "Platte auf {instance} ({mountpoint}) fast voll", severity: warning}, attempts: 1} + + # k3s (Task 6.5): nur melden; Pods startet Kubernetes selbst neu + - id: k3s-node-not-ready + match: {alertname: K3sNodeNotReady} + ladder: + - {action: notify, params: {text: "k3s-Knoten {node} nicht bereit – VM und k3s-agent prüfen", severity: high}, attempts: 1} + + - id: k3s-pod-crashloop + match: {alertname: K3sPodCrashLooping} + ladder: + - {action: notify, params: {text: "Pod {namespace}/{pod} ({container}) startet immer wieder neu", severity: warning}, attempts: 1} + + - id: k3s-deployment-unavailable + match: {alertname: K3sDeploymentUnavailable} + ladder: + - {action: notify, params: {text: "Deployment {namespace}/{deployment} hat zu wenige bereite Pods", severity: warning}, attempts: 1} + + - id: k3s-memory-near-limit + match: {alertname: K3sContainerMemoryNearLimit} + ladder: + - {action: notify, params: {text: "{namespace}/{pod} ({container}) über 90 % des Speicherlimits", severity: warning}, attempts: 1} diff --git a/core/tests/test_artifacts.py b/core/tests/test_artifacts.py index a55e5ec..02513a4 100644 --- a/core/tests/test_artifacts.py +++ b/core/tests/test_artifacts.py @@ -39,13 +39,18 @@ def test_healer_labels_available_on_alerts(): assert set(OPS_LABELS) == {"docker_host", "container", "pve_node", "vmid"} -def _monitoring_targets() -> dict: +def _targets_module(): import importlib.util spec = importlib.util.spec_from_file_location( "monitoring_targets", REPO_ROOT / "platform/proxmox/scripts/monitoring-targets.py") mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) + return mod + + +def _monitoring_targets() -> dict: + mod = _targets_module() registry = yaml.safe_load((REPO_ROOT / "sites/knaster/registry.yaml").read_text(encoding="utf-8")) gpu_vms = { # Form wie `tofu output -json gpu_vms` "gpu-pve44": {"node": "pve44", "vmid": 244, "ipv4": "192.168.100.186"}, @@ -67,6 +72,55 @@ def test_monitoring_targets_from_tofu_output(): assert t["node"][2]["labels"]["slot"] == "none" # gpu-pve52 trägt noch keinen Slot +def _k3s_node(name: str, ip: str | None, ready: str = "True") -> dict: + addresses = [{"type": "Hostname", "address": name}] + ([{"type": "InternalIP", "address": ip}] if ip else []) + return {"metadata": {"name": name}, + "status": {"addresses": addresses, "conditions": [{"type": "Ready", "status": ready}]}} + + +def test_k3s_targets_from_kubectl_nodes(): + nodes = {"items": [ # Form wie `kubectl get nodes -o json` + _k3s_node("gpu-pve52", "192.168.100.174"), + _k3s_node("k3s", "192.168.100.124"), + _k3s_node("gpu-pve44", "192.168.100.186", ready="False"), + _k3s_node("gpu-pve46", None), + ]} + t = _targets_module().build_k3s_targets(nodes) + assert [(e["targets"][0], e["labels"]["instance"]) for e in t["kubelet"]] == [ + ("192.168.100.186:10250", "gpu-pve44"), ("192.168.100.174:10250", "gpu-pve52"), + ("192.168.100.124:10250", "k3s")] # gpu-pve46 ohne IP fehlt + assert t["kube-state-metrics"] == [ + {"targets": ["192.168.100.124:30808"], "labels": {"instance": "kube-state-metrics"}}] # Dienst-VM bevorzugt + t = _targets_module().build_k3s_targets({"items": [nodes["items"][0], nodes["items"][2]]}) + assert t["kube-state-metrics"][0]["targets"] == ["192.168.100.174:30808"] # erster bereiter Knoten + assert _targets_module().build_k3s_targets({}) == {"kubelet": [], "kube-state-metrics": []} + + +def test_k3s_jobs_use_generated_targets_and_nodeport(): + prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text()) + jobs = {j["job_name"]: j for j in prom["scrape_configs"]} + mod = _targets_module() + written = set(mod.build_k3s_targets({})) + for job, sd in {"kubelet": "kubelet", "cadvisor": "kubelet", "kube-state-metrics": "kube-state-metrics"}.items(): + assert jobs[job]["file_sd_configs"] == [{"files": [f"/etc/knaster-sd/{sd}.json"]}] + assert sd in written + for job in ("kubelet", "cadvisor"): + assert jobs[job]["scheme"] == "https" and jobs[job]["tls_config"]["ca_file"].startswith("/etc/knaster-k3s/") + docs = list(yaml.safe_load_all((DEPLOY / "k8s/monitoring/kube-state-metrics.yaml").read_text())) + svc = next(d for d in docs if d["kind"] == "Service") + assert svc["spec"]["ports"][0]["nodePort"] == mod.KSM_NODEPORT + + +DASHBOARDS = sorted((DEPLOY / "grafana/dashboards").glob("*.json")) + + +@pytest.mark.parametrize("path", DASHBOARDS, ids=[p.name for p in DASHBOARDS]) +def test_grafana_dashboards_parse_with_unique_uid(path): + d = json.loads(path.read_text(encoding="utf-8")) + assert d["uid"] and d["title"] and d["panels"] + assert [x for x in DASHBOARDS if json.loads(x.read_text(encoding="utf-8"))["uid"] == d["uid"]] == [path] + + def test_alertmanager_targets_router_endpoint(): am = yaml.safe_load((DEPLOY / "alertmanager/alertmanager.yml").read_text()) url = am["receivers"][0]["webhook_configs"][0]["url"] diff --git a/deploy/compose/infra/docker-compose.yml b/deploy/compose/infra/docker-compose.yml index 71451ed..538af7a 100644 --- a/deploy/compose/infra/docker-compose.yml +++ b/deploy/compose/infra/docker-compose.yml @@ -70,6 +70,8 @@ services: - ../../prometheus:/etc/prometheus:ro # file_sd: Ziele der GPU-VMs (DHCP, IPs wechseln), erzeugt auf ops aus `tofu output -json gpu_vms` - ${KNASTER_STATE_DIR:-.}/file_sd:/etc/knaster-sd:ro + # Token und CA für kubelet/cAdvisor in k3s (Task 6.5), nur dieser Unterordner der Geheimnisse + - ${KNASTER_STATE_DIR:-.}/secrets/k3s:/etc/knaster-k3s:ro - prometheus:/prometheus extra_hosts: ["host.docker.internal:host-gateway"] # node-exporter des LXC läuft im Host-Netz ports: ["9090:9090"] diff --git a/deploy/grafana/dashboards/k3s-pods.json b/deploy/grafana/dashboards/k3s-pods.json new file mode 100644 index 0000000..db3ee01 --- /dev/null +++ b/deploy/grafana/dashboards/k3s-pods.json @@ -0,0 +1,824 @@ +{ + "uid": "k3s-pods", + "title": "k3s – Pods", + "tags": [ + "knaster", + "k3s" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": false, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(kube_namespace_status_phase, namespace)", + "refId": "ns" + }, + "definition": "label_values(kube_namespace_status_phase, namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "sort": 1, + "current": { + "selected": true, + "text": [ + "nex" + ], + "value": [ + "nex" + ] + } + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Pods laufend", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"}) or vector(0)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 2, + "type": "stat", + "title": "Pods nicht bereit", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 4, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum(kube_pod_status_ready{namespace=~\"$namespace\", condition=\"false\"}) or vector(0)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Neustarts (24 h)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "round(sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[24h]))) or vector(0)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Speicher (Arbeitssatz)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "CPU (Kerne)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"cadvisor\", namespace=~\"$namespace\"}[5m]))", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 6, + "type": "stat", + "title": "k3s-Knoten bereit", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 20, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Speicher je Pod", + "description": "Arbeitssatz (container_memory_working_set_bytes), danach richtet sich das OOM-Limit", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (pod) (container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "CPU je Pod", + "description": "Kerne, Mittel über 5 min", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{job=\"cadvisor\", namespace=~\"$namespace\"}[5m]))", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "table", + "title": "Pods", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 13, + "w": 24, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "max by (pod, node) (kube_pod_info{namespace=~\"$namespace\"})", + "legendFormat": "", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "B", + "expr": "sum by (pod) (kube_pod_container_status_restarts_total{namespace=~\"$namespace\"})", + "legendFormat": "", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "C", + "expr": "sum by (pod) (container_memory_working_set_bytes{job=\"cadvisor\", namespace=~\"$namespace\"})", + "legendFormat": "", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "D", + "expr": "sum by (pod) (kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})", + "legendFormat": "", + "format": "table", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "E", + "expr": "sum by (pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})", + "legendFormat": "", + "format": "table", + "instant": true + } + ], + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value #A": true + }, + "indexByName": { + "pod": 0, + "node": 1, + "Value #B": 2, + "Value #C": 3, + "Value #D": 4, + "Value #E": 5 + }, + "renameByName": { + "pod": "Pod", + "node": "Knoten", + "Value #B": "Neustarts", + "Value #C": "Speicher", + "Value #D": "Request", + "Value #E": "Limit" + } + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Speicher|Request|Limit" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Neustarts" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Speicher", + "desc": true + } + ] + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Pods je Knoten (alle Namespaces)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 22, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (node) (kube_pod_info)", + "legendFormat": "{{node}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Speicher der Pods je Knoten (alle Namespaces)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 22, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (instance) (container_memory_working_set_bytes{job=\"cadvisor\"})", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "bargauge", + "title": "Belegung der Volumes (PVC)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 30, + "w": 24, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "max by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"})", + "legendFormat": "{{persistentvolumeclaim}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + } + ] +} diff --git a/deploy/grafana/provisioning/dashboards/knaster.yml b/deploy/grafana/provisioning/dashboards/knaster.yml index 68e4f17..47c606c 100644 --- a/deploy/grafana/provisioning/dashboards/knaster.yml +++ b/deploy/grafana/provisioning/dashboards/knaster.yml @@ -4,6 +4,7 @@ # node-exporter-full.json ID 1860 „Node Exporter Full“ Revision 45 # nvidia-gpu.json ID 14574 „Nvidia GPU Metrics“ Revision 15 (für utkuozdemir/nvidia_gpu_exporter) # knaster-uebersicht.json eigenes Dashboard (Startseite, für den Kiosk in Task 5.3) +# k3s-pods.json eigenes Dashboard (Task 6.5): Pods je Namespace (Vorgabe nex), kube-state-metrics + cAdvisor apiVersion: 1 providers: - name: knaster diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index 925b8d5..0a3a63a 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -11,6 +11,7 @@ Ausgerollt von ops mit `kubectl` (Rolle `k3s-apps`, Playbook `playbooks/k3s.yml` | `nex-searxng/` | NEX-Websuche SearXNG, NodePort 30880 (Playbook `playbooks/nex-searxng-k3s.yml`), Einstellungen aus dem nex-Repo | | `portainer/` | Portainer CE 2.45.1 auf der Dienst-VM k3s, Datenbank von appsrv08 übernommen (Playbook `playbooks/portainer-k3s.yml` mit Rollback BE→CE), NodePort 30779 | | `nex/` | Namespace nex für Agenten im Cluster: Brücke zu den NEX-Diensten auf appsrv08 (Services ohne Selector), Zugang für nex-admin (ServiceAccount, Role), Pull-Secret Gitea (Playbook `playbooks/nex-k3s.yml`); nex-Spec 2026-10-19 | +| `monitoring/` | kube-state-metrics v2.19.1 (NodePort 30808) und ServiceAccount `prometheus-infra` (nur `nodes/metrics`) für Prometheus auf infra (Playbook `playbooks/monitoring-k3s.yml`, Task 6.5) | | `rustdesk/` | RustDesk-Server hbbs/hbbr, Service LoadBalancer 21115–21119 auf jeder Knoten-Adresse (Playbook `playbooks/rustdesk-k3s.yml`), Schlüssel im Secret `rustdesk-keys` | Portainer: Umgebung „Kubernetes → Agent“, Adresse `:30778`. diff --git a/deploy/k8s/monitoring/kube-state-metrics.yaml b/deploy/k8s/monitoring/kube-state-metrics.yaml new file mode 100644 index 0000000..d5b5a04 --- /dev/null +++ b/deploy/k8s/monitoring/kube-state-metrics.yaml @@ -0,0 +1,88 @@ +# kube-state-metrics v2.19.1 (Task 6.5): Zustand der Objekte (Pods, Deployments, Knoten, PVCs) für Prometheus auf infra. +# NodePort 30808 auf jedem Knoten; Prometheus fragt genau ein Ziel ab (file_sd, monitoring-targets.py). +# Nur die Ressourcen, die Dashboard und Alerts brauchen (infra hat 4 GB RAM). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics +rules: + - apiGroups: [""] + resources: [nodes, pods, namespaces, persistentvolumeclaims] + verbs: [list, watch] + - apiGroups: [apps] + resources: [deployments, daemonsets, statefulsets, replicasets] + verbs: [list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: + - kind: ServiceAccount + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: kube-system + labels: {app: kube-state-metrics} +spec: + replicas: 1 + selector: + matchLabels: {app: kube-state-metrics} + template: + metadata: + labels: {app: kube-state-metrics} + spec: + serviceAccountName: kube-state-metrics + automountServiceAccountToken: true + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: {type: RuntimeDefault} + containers: + - name: kube-state-metrics + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.19.1 + args: + - --resources=nodes,pods,namespaces,persistentvolumeclaims,deployments,daemonsets,statefulsets,replicasets + - --port=8080 + - --telemetry-port=8081 + ports: + - {name: metrics, containerPort: 8080} + - {name: telemetry, containerPort: 8081} + readinessProbe: + httpGet: {path: /readyz, port: 8081} + initialDelaySeconds: 5 + livenessProbe: + httpGet: {path: /livez, port: 8080} + initialDelaySeconds: 5 + resources: + requests: {cpu: 10m, memory: 64Mi} + limits: {memory: 192Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-state-metrics + namespace: kube-system +spec: + type: NodePort + selector: {app: kube-state-metrics} + ports: + - {name: metrics, port: 8080, targetPort: metrics, nodePort: 30808} diff --git a/deploy/k8s/monitoring/prometheus-access.yaml b/deploy/k8s/monitoring/prometheus-access.yaml new file mode 100644 index 0000000..ebd20ce --- /dev/null +++ b/deploy/k8s/monitoring/prometheus-access.yaml @@ -0,0 +1,40 @@ +# Lesezugang für Prometheus auf infra (Task 6.5): kubelet /metrics und /metrics/cadvisor auf Port 10250. +# Der kubelet prüft das Token per SubjectAccessReview gegen nodes/metrics; mehr darf der ServiceAccount nicht. +# Das Token (Secret prometheus-infra-token) legt playbooks/monitoring-k3s.yml auf ops ab, die Rolle +# infra-stack kopiert es nach /opt/knaster/infra/secrets/k3s/. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-infra + namespace: kube-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: prometheus-infra-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: prometheus-infra +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-infra +rules: + - apiGroups: [""] + resources: [nodes/metrics] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-infra +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-infra +subjects: + - kind: ServiceAccount + name: prometheus-infra + namespace: kube-system diff --git a/deploy/prometheus/alerts.yml b/deploy/prometheus/alerts.yml index 6d3b0ed..066923c 100644 --- a/deploy/prometheus/alerts.yml +++ b/deploy/prometheus/alerts.yml @@ -47,3 +47,33 @@ groups: for: 15m labels: {severity: warning} annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"} + + # k3s (Task 6.5): nur melden, der Healer greift nicht ein (config/healer-rules.yaml, Aktion notify) + - name: knaster-k3s + rules: + - alert: K3sNodeNotReady + expr: kube_node_status_condition{condition="Ready", status="true"} == 0 + for: 5m + labels: {severity: high} + annotations: {summary: "k3s-Knoten {{ $labels.node }} nicht bereit"} + + - alert: K3sPodCrashLooping + expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) == 1 + for: 10m + labels: {severity: warning} + annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) startet immer wieder neu"} + + - alert: K3sDeploymentUnavailable + expr: max by (namespace, deployment) (kube_deployment_spec_replicas) > max by (namespace, deployment) (kube_deployment_status_replicas_available) + for: 15m + labels: {severity: warning} + annotations: {summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} hat zu wenige bereite Pods"} + + - alert: K3sContainerMemoryNearLimit + expr: > + max by (namespace, pod, container) (container_memory_working_set_bytes{job="cadvisor"}) + / on (namespace, pod, container) + max by (namespace, pod, container) (kube_pod_container_resource_limits{resource="memory"}) > 0.9 + for: 10m + labels: {severity: warning} + annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) über 90 % des Speicherlimits"} diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml index 4fbae03..d4a356e 100644 --- a/deploy/prometheus/prometheus.yml +++ b/deploy/prometheus/prometheus.yml @@ -66,3 +66,39 @@ scrape_configs: - {source_labels: [__address__], target_label: __param_target} - {source_labels: [__param_target], target_label: instance} - {target_label: __address__, replacement: "pve-exporter:9221"} + + # k3s (Task 6.5): kubelet direkt auf jedem Knoten (10250), TLS gegen die k3s-CA, Token des ServiceAccounts + # prometheus-infra (nur nodes/metrics). Ziele per file_sd aus `kubectl get nodes` (monitoring-targets.py auf ops). + # infra hat 4 GB RAM: von kubelet und cAdvisor nur die Reihen, die Dashboard „k3s – Pods“ und Alerts brauchen. + - job_name: kubelet + scheme: https + authorization: {credentials_file: /etc/knaster-k3s/token} + tls_config: {ca_file: /etc/knaster-k3s/ca.crt} + file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}] + metric_relabel_configs: + - source_labels: [__name__] + regex: kubelet_running_pods|kubelet_running_containers|kubelet_volume_stats_(used|capacity)_bytes + action: keep + + - job_name: cadvisor + scheme: https + metrics_path: /metrics/cadvisor + authorization: {credentials_file: /etc/knaster-k3s/token} + tls_config: {ca_file: /etc/knaster-k3s/ca.crt} + file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}] + metric_relabel_configs: + - source_labels: [__name__] + regex: container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_oom_events_total|container_network_(receive|transmit)_bytes_total + action: keep + # nur Pods (keine systemd-Slices, keine Docker-Container der Compose-Stacks) + - source_labels: [pod] + regex: "" + action: drop + # CPU/Speicher je Container: Pod-Summe und Pause-Container verwerfen (Netz gibt es nur je Pod) + - source_labels: [__name__, container] + regex: "container_(cpu|memory|oom)[^;]*;(POD)?" + action: drop + + - job_name: kube-state-metrics + honor_labels: true # namespace/pod/node kommen aus den Metriken selbst + file_sd_configs: [{files: [/etc/knaster-sd/kube-state-metrics.json]}] diff --git a/docs/BETRIEB.md b/docs/BETRIEB.md index c665db6..92be092 100644 --- a/docs/BETRIEB.md +++ b/docs/BETRIEB.md @@ -129,7 +129,9 @@ Prometheus liest die Dateien in `/opt/knaster/infra/file_sd` selbst neu, ein Neu **Alerts:** Alertmanager meldet an den Router (`/v1/healer/alertmanager`, Healer im Dry-Run). Der Receiver für n8n `knaster-notify` ist in `deploy/alertmanager/alertmanager.yml` nur vorbereitet (auskommentiert). Test: `docker exec knaster-infra-alertmanager-1 amtool --alertmanager.url=http://localhost:9093 alert add test severity=warning` und `… alert query`. -**Dashboards** (Ordner `knaster`, Startseite „knaster – Übersicht“): Proxmox via Prometheus (grafana.com 10347), Node Exporter Full (1860), Nvidia GPU Metrics (14574), knaster – Übersicht (eigenes). +**Dashboards** (Ordner `knaster`, Startseite „knaster – Übersicht“): Proxmox via Prometheus (grafana.com 10347), Node Exporter Full (1860), Nvidia GPU Metrics (14574), knaster – Übersicht (eigenes), k3s – Pods (eigenes, Task 6.5). + +**k3s-Pods (Task 6.5, seit 19.10.):** Prometheus fragt drei Jobs ab. `kubelet` und `cadvisor` gehen direkt an Port 10250 jedes k3s-Knotens (TLS gegen die k3s-CA, Token des ServiceAccounts `prometheus-infra`, darf nur `nodes/metrics` lesen). `kube-state-metrics` geht an NodePort 30808 (bevorzugt auf der Dienst-VM k3s). Die Ziele erzeugt `monitoring-targets.py` aus `kubectl get nodes` auf ops (`kubelet.json`, `kube-state-metrics.json`), sie werden mit `playbooks/monitoring-targets.yml` aufgefrischt wie die der GPU-VMs. Token und CA: auf ops `/root/.config/knaster/k3s-prometheus.token` und `k3s-ca.crt`, im LXC `/opt/knaster/infra/secrets/k3s/` (nur dieser Ordner ist in Prometheus eingehängt). Von cAdvisor bleiben nur CPU, Speicher, OOM und Netz je Pod (`metric_relabel_configs`), weil infra nur 4 GB hat. Alerts `K3s*` (Knoten nicht bereit, CrashLoop, Deployment ohne bereite Pods, Speicher > 90 % des Limits) melden nur, der Healer greift nicht ein. Ausrollen: `ansible-playbook playbooks/monitoring-k3s.yml` (Cluster-Teil, dann Infra-Stack). **Achtung HA-Umzug und ZFS-Seiten-Cache (gefunden 12.10.2026 bei der HA-Probe)** Nach `zfs receive` (Replikation) liefert ein Knoten für Dateien, die dort früher per mmap geöffnet waren, weiter den alten Inhalt aus seinem Seiten-Cache (ZFS 2.4.4, Kernel 7.0.14; Beleg auf pve44: `zfs diff ` leer, `written` 0, md5 der Datei trotzdem ungleich der im Snapshot). Folge beim ersten Umzug nach pve48: dockerd las veraltete bbolt-Datenbanken, brach mit `panic: invalid freelist page` ab, danach fehlten Netzwerk und Volume-Labels. Betroffen ist jeder replizierte LXC mit mmap-Nutzern (Docker, Prometheus-TSDB, später `grp-test`), sobald er auf einen Knoten zurückkehrt, auf dem er schon lief. @@ -221,7 +223,8 @@ Die GPU-VMs bilden mit der Dienst-VM `k3s` einen k3s-Cluster (v1.36.5+k3s1). Ste | Server-Adresse | per DHCP (heute 192.168.100.124, MAC `bc:24:11:a5:d5:52`); Agents nutzen den Namen `k3s.knaster` aus `/etc/hosts` (Rolle k3s-agent). Ändert sich die Adresse: `refresh-vm-access.sh`, dann `playbooks/k3s.yml` | | Token und kubeconfig | nur auf ops: `/root/.config/knaster/k3s.env`, `/root/.kube/config`; `kubectl` liegt auf ops | | Manifeste | `deploy/k8s/` (RuntimeClass `nvidia`, NVIDIA Device Plugin v0.20.1, Portainer Agent 2.45.2, `nex-speech/`, `nex-searxng/`, `rustdesk/`) | -| Playbooks | `playbooks/k3s.yml` (Server, Agents, Grundausstattung), `playbooks/nex-speech-k3s.yml`, `playbooks/nex-searxng-k3s.yml`, `playbooks/rustdesk-k3s.yml` (Umzüge) | +| Playbooks | `playbooks/k3s.yml` (Server, Agents, Grundausstattung), `playbooks/nex-speech-k3s.yml`, `playbooks/nex-searxng-k3s.yml`, `playbooks/rustdesk-k3s.yml` (Umzüge), `playbooks/monitoring-k3s.yml` (Pod-Monitoring) | +| Monitoring | kube-state-metrics v2.19.1 in `kube-system` (NodePort 30808), Prometheus auf infra liest kubelet/cAdvisor; Grafana-Dashboard „k3s – Pods“ (Abschnitt 7). `kubectl top` zeigt weiter den metrics-server von k3s | | GPU in Pods | `runtimeClassName: nvidia` + `resources.limits: {nvidia.com/gpu: 1}`; Knoten-Labels `knaster.gpu=true`, `knaster.gpu.m4000=1`, `knaster.gpu.p2000=1` | | Portainer | Umgebung „Kubernetes → Agent“ mit Adresse `:30778`, z. B. `192.168.100.124:30778` (Agent-Pod läuft im Namespace `portainer`) | diff --git a/docs/superpowers/plans/2026-10-12-aufbau-cluster-knaster.md b/docs/superpowers/plans/2026-10-12-aufbau-cluster-knaster.md index dca54dc..1523136 100644 --- a/docs/superpowers/plans/2026-10-12-aufbau-cluster-knaster.md +++ b/docs/superpowers/plans/2026-10-12-aufbau-cluster-knaster.md @@ -382,15 +382,17 @@ Ersetzt PLAN §8 „1 Data Plane“. - `test_worker_registry_slot_falls_back_to_tier0_when_list_empty` - `test_worker_registry_slot_falls_back_when_list_unreachable` - Prüfung: `make test` PASS. Live tagsüber geht eine tier1-Anfrage an tier0, nachts an GODE-PC-05. -- [ ] **6.5 Pod-Monitoring (k3s in Grafana)** · **Planner:** „Proxmox: Grafana-Dashboard für den Cluster“ · **Files:** Create `deploy/k8s/kube-state-metrics/`, `platform/proxmox/ansible/playbooks/kube-state-metrics-k3s.yml`, `deploy/grafana/dashboards/k3s-pods.json`; Modify `deploy/prometheus/prometheus.yml`, `platform/proxmox/scripts/monitoring-targets.py`, Rolle `infra-stack` (Token), `core/tests/test_artifacts.py`, BETRIEB §7/§14 +- [ ] **6.5 Pod-Monitoring (k3s in Grafana)** · **Planner:** „Proxmox: Grafana-Dashboard für den Cluster“ · **Files:** Create `deploy/k8s/monitoring/`, `platform/proxmox/ansible/playbooks/monitoring-k3s.yml`, `deploy/grafana/dashboards/k3s-pods.json`; Modify `deploy/prometheus/prometheus.yml`, `deploy/prometheus/alerts.yml`, `config/healer-rules.yaml`, `platform/proxmox/scripts/monitoring-targets.py`, Rolle `infra-stack` (Token, Ziele), `deploy/compose/infra/docker-compose.yml`, `core/tests/test_artifacts.py`, BETRIEB §7/§14 Ausgangslage 19.10.: Grafana 12.4 und Prometheus laufen auf infra mit Dashboards für Proxmox, Knoten, GPU und knaster; für k3s gibt es kein Ziel. Zustand und Speicher der Pods zeigt bisher nur nex-admin über die Metrics-API (metrics-server, nur Momentwerte, keine Grafana-Datenquelle). - - [ ] kube-state-metrics als Deployment im Namespace `kube-system` (Image gepinnt), Service als NodePort. Liefert Zustand: Bereitschaft, Neustarts, Requests/Limits, Deployments. - - [ ] ServiceAccount `prometheus-infra` mit ClusterRole nur zum Lesen (`nodes`, `nodes/metrics`, `nodes/proxy`), langlebiges Token als Secret. Das Token liegt wie beim pve-exporter nur im LXC unter `/opt/knaster/infra/secrets/k3s-prometheus.token`, kopiert von der Rolle `infra-stack`; die k3s-CA dazu (`/var/lib/rancher/k3s/server/tls/server-ca.crt`), damit kein `insecure_skip_verify` nötig ist. Kubelet-Zertifikate der Agents gegen diese CA prüfen, sonst bewusst entscheiden. - - [ ] Ziele per file_sd statt fester Adressen, weil die k3s-VM (.124) und gpu-pve44/46/48 noch keine Reservierung haben: `monitoring-targets.py` schreibt zusätzlich `kubelet.json` (alle fünf k3s-Knoten, Port 10250, Pfade `/metrics/cadvisor` und `/metrics`) und `kube-state-metrics.json` (NodePort auf jedem Knoten, Prometheus nimmt den ersten erreichbaren bzw. dedupliziert per `instance`). - - [ ] Prometheus-Jobs `kubelet`, `cadvisor`, `kube-state-metrics` in `deploy/prometheus/prometheus.yml`. infra hat nur 4 GB: für cAdvisor `metric_relabel_configs`, die nur `container_cpu_usage_seconds_total`, `container_memory_working_set_bytes`, `container_memory_rss` und Netzwerk behalten und `container=""`/Pause-Container verwerfen. Danach Speicher von Prometheus eine Woche beobachten. - - [ ] ufw in den GPU-VMs (gpu-pve52 lässt heute nur 8178/8179 von .29 zu): 10250 und den NodePort nur von `.188` freigeben (Rolle `k3s-agent` bzw. `gpu-exporters`). - - [ ] Dashboard „k3s – Pods“ (Grundlage Community-Dashboard, z. B. grafana.com 15661/15760, angepasst auf die vorhandenen Metriken) mit Namespace-Auswahl, Vorgabe `nex`: Pods je Knoten, Neustarts, CPU/Speicher gegen Requests/Limits. In die Kiosk-Playlist (5.3) aufnehmen. - - [ ] Alerts in `deploy/prometheus/alerts.yml`: Pod in CrashLoopBackOff > 10 min, Deployment nicht verfügbar, Speicher > 90 % des Limits. Nur melden, kein Healer-Eingriff (der heilt nur VMs). + - [x] (19.10.) kube-state-metrics v2.19.1 im Namespace `kube-system` (`deploy/k8s/monitoring/kube-state-metrics.yaml`, nur die nötigen Ressourcen), Service NodePort 30808. + - [x] (19.10.) ServiceAccount `prometheus-infra`, ClusterRole nur `get nodes/metrics`, langlebiges Token-Secret. Token und CA legt `playbooks/monitoring-k3s.yml` auf ops ab (`/root/.config/knaster/k3s-prometheus.token`, `k3s-ca.crt`), die Rolle `infra-stack` kopiert sie nach `/opt/knaster/infra/secrets/k3s/` (nur dieser Ordner in Prometheus eingehängt). Die kubelet-Zertifikate sind von `k3s-server-ca` signiert und tragen die Knoten-IP (geprüft auf gpu-pve52), daher TLS ohne `insecure_skip_verify`. + - [x] (19.10.) Ziele per file_sd aus `kubectl get nodes` (statt tofu, so ist die k3s-VM dabei): `monitoring-targets.py --k3s-nodes` schreibt `kubelet.json` (alle Knoten, 10250) und `kube-state-metrics.json` (ein Ziel, NodePort auf der Dienst-VM k3s, sonst erster bereiter Knoten; `instance` fest, damit ein Wechsel keine neuen Reihen erzeugt). + - [x] (19.10.) Prometheus-Jobs `kubelet`, `cadvisor`, `kube-state-metrics`. cAdvisor behält nur CPU, Arbeitssatz, RSS, OOM und Netz und verwirft Reihen ohne Pod sowie Pod-Summen/Pause-Container. + - [ ] Speicher von Prometheus eine Woche beobachten (infra hat 4 GB). + - [x] ufw: Die Rolle `k3s-agent` erlaubt 10250 und 30000–32767 schon seit 15.10. aus dem LAN, keine Änderung nötig. + - [x] (19.10.) Dashboard „k3s – Pods“ (`deploy/grafana/dashboards/k3s-pods.json`, eigenes statt Community, passend zu den behaltenen Metriken): Namespace-Auswahl mit Vorgabe `nex`, Kennzahlen, Speicher/CPU je Pod, Tabelle mit Knoten, Neustarts, Speicher gegen Request/Limit, Pods und Speicher je Knoten, PVC-Belegung. + - [ ] Dashboard in die Kiosk-Playlist (5.3) aufnehmen, sobald es die Playlist gibt. + - [x] (19.10.) Alerts `K3sNodeNotReady`, `K3sPodCrashLooping` (> 10 min), `K3sDeploymentUnavailable` (> 15 min), `K3sContainerMemoryNearLimit` (> 90 %, 10 min); Healer-Regeln nur `notify`. - Prüfung: `make test` PASS (Struktur von prometheus.yml/alerts.yml). Auf infra zeigt `api/v1/targets` die Jobs `kubelet`, `cadvisor`, `kube-state-metrics` mit allen fünf Knoten `up`. Grafana zeigt für `nex` dieselben Pods und ungefähr denselben Speicher wie `kubectl top pods -n nex`. Ein absichtlich kaputtes Test-Deployment löst den CrashLoop-Alert aus. --- diff --git a/platform/proxmox/ansible/playbooks/monitoring-k3s.yml b/platform/proxmox/ansible/playbooks/monitoring-k3s.yml new file mode 100644 index 0000000..4a22523 --- /dev/null +++ b/platform/proxmox/ansible/playbooks/monitoring-k3s.yml @@ -0,0 +1,74 @@ +# Pod-Monitoring (Task 6.5): kube-state-metrics und Lesezugang für Prometheus in k3s, danach der Infra-Stack +# (Prometheus-Jobs kubelet/cadvisor/kube-state-metrics, file_sd-Ziele, Dashboard „k3s – Pods“). Läuft auf ops: +# ansible-playbook playbooks/monitoring-k3s.yml +# Token und CA landen auf ops unter /root/.config/knaster (0600), die Rolle infra-stack kopiert sie nach infra. +- name: k3s – kube-state-metrics und Zugang für Prometheus + hosts: localhost + connection: local + gather_facts: false + vars: + manifests: "{{ playbook_dir }}/../../../../deploy/k8s/monitoring" + token_file: /root/.config/knaster/k3s-prometheus.token + ca_file: /root/.config/knaster/k3s-ca.crt + tasks: + - name: Manifeste anwenden + ansible.builtin.command: kubectl apply -f {{ manifests }}/{{ item }} + loop: [prometheus-access.yaml, kube-state-metrics.yaml] + register: apply + changed_when: "'configured' in apply.stdout or 'created' in apply.stdout" + + - name: Warten, bis kube-state-metrics bereit ist + ansible.builtin.command: kubectl -n kube-system rollout status deployment/kube-state-metrics --timeout=180s + changed_when: false + + - name: Token des ServiceAccounts prometheus-infra lesen (Secret wird vom Controller befüllt) + ansible.builtin.shell: > + set -o pipefail; + kubectl -n kube-system get secret prometheus-infra-token -o go-template='{{ "{{" }}.data.token{{ "}}" }}' | base64 -d + args: {executable: /bin/bash} + register: token + until: token.stdout | length > 0 + retries: 10 + delay: 2 + changed_when: false + no_log: true + + - name: Token als Datei auf ops ablegen (600, nie ausgeben) + ansible.builtin.copy: + content: "{{ token.stdout }}" + dest: "{{ token_file }}" + mode: "0600" + no_log: true + + # Die kubelet-Zertifikate sind von k3s-server-ca signiert und tragen die Knoten-IP (19.10. geprüft). + - name: CA des Clusters lesen + ansible.builtin.shell: > + set -o pipefail; + kubectl -n kube-system get secret prometheus-infra-token -o go-template='{{ "{{" }}index .data "ca.crt"{{ "}}" }}' | base64 -d + args: {executable: /bin/bash} + register: ca + changed_when: false + + - name: CA als Datei auf ops ablegen + ansible.builtin.copy: + content: "{{ ca.stdout }}\n" + dest: "{{ ca_file }}" + mode: "0600" + + - name: Zugang prüfen (cAdvisor eines Knotens mit Token und CA) + ansible.builtin.shell: > + set -o pipefail; + ip=$(kubectl get node k3s -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'); + curl -fsS -m 10 --cacert {{ ca_file }} -H "Authorization: Bearer $(cat {{ token_file }})" + https://$ip:10250/metrics/cadvisor | grep -c '^container_memory_working_set_bytes' + args: {executable: /bin/bash} + register: probe + changed_when: false + + - name: Ergebnis + ansible.builtin.debug: + msg: "cAdvisor auf k3s erreichbar, {{ probe.stdout }} Reihen container_memory_working_set_bytes" + +- name: Infra-LXC – Prometheus, Ziele und Dashboard + hosts: infra + roles: [infra-stack] diff --git a/platform/proxmox/ansible/roles/infra-stack/defaults/main.yml b/platform/proxmox/ansible/roles/infra-stack/defaults/main.yml index 866008a..9757f1e 100644 --- a/platform/proxmox/ansible/roles/infra-stack/defaults/main.yml +++ b/platform/proxmox/ansible/roles/infra-stack/defaults/main.yml @@ -24,6 +24,10 @@ infra_secrets: [POSTGRES_PASSWORD, KNASTER_API_TOKEN, GRAFANA_ADMIN_PASSWORD, GR # Lese-Token monitoring@pve!prom (PVEAuditor) auf ops infra_pve_exporter_env: /root/.config/knaster/pve-exporter.env +# Lesezugang zum kubelet für Prometheus (Task 6.5), auf ops von playbooks/monitoring-k3s.yml abgelegt +infra_k3s_token: /root/.config/knaster/k3s-prometheus.token +infra_k3s_ca: /root/.config/knaster/k3s-ca.crt + # file_sd: auf ops aus `tofu output -json gpu_vms` erzeugt (nicht im Repo) infra_tofu_dir: "{{ playbook_dir }}/../../tofu" infra_targets_dir: "{{ playbook_dir }}/../monitoring-targets.generated" diff --git a/platform/proxmox/ansible/roles/infra-stack/tasks/main.yml b/platform/proxmox/ansible/roles/infra-stack/tasks/main.yml index c815eca..9032f36 100644 --- a/platform/proxmox/ansible/roles/infra-stack/tasks/main.yml +++ b/platform/proxmox/ansible/roles/infra-stack/tasks/main.yml @@ -17,6 +17,7 @@ - {path: "{{ infra_repo_dir }}", mode: "0755"} - {path: "{{ infra_state_dir }}", mode: "0700"} - {path: "{{ infra_state_dir }}/secrets", mode: "0700"} + - {path: "{{ infra_state_dir }}/secrets/k3s", mode: "0755"} # Nur die Teile, die Build und Konfiguration brauchen. Nie .git, .venv, tofu-State oder .env-Dateien. - name: Repo-Auszug von ops abgleichen @@ -87,6 +88,30 @@ mode: "0600" no_log: true +# Task 6.5: Token des ServiceAccounts prometheus-infra und CA des Clusters (von playbooks/monitoring-k3s.yml +# auf ops abgelegt). Lesbar für nobody (Prometheus im Container). Fehlen sie, bleiben die k3s-Jobs down. +- name: k3s-Zugang für Prometheus vorhanden? (auf ops) + ansible.builtin.stat: + path: "{{ item }}" + loop: ["{{ infra_k3s_token }}", "{{ infra_k3s_ca }}"] + delegate_to: localhost + register: k3s_access + +- name: k3s-Zugang für Prometheus kopieren + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ infra_state_dir }}/secrets/k3s/{{ item.dest }}" + owner: "65534" + group: "65534" + mode: "0400" + loop: + - {src: "{{ infra_k3s_token }}", dest: token} + - {src: "{{ infra_k3s_ca }}", dest: ca.crt} + loop_control: + label: "{{ item.dest }}" + when: k3s_access.results | map(attribute='stat.exists') | min + no_log: true + - name: file_sd-Ziele ansible.builtin.include_tasks: targets.yml diff --git a/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml b/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml index 449c652..d188923 100644 --- a/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml +++ b/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml @@ -1,11 +1,24 @@ # file_sd-Ziele der GPU-VMs (node, gpu, llamacpp). Auch einzeln aufrufbar: playbooks/monitoring-targets.yml # (nach refresh-vm-access.sh, wenn sich DHCP-Adressen geändert haben). Prometheus liest die Dateien selbst neu. +# k3s (Task 6.5): kubelet und kube-state-metrics aus `kubectl get nodes` auf ops. Ist der Cluster nicht +# erreichbar, bleiben diese Listen leer und der Rest läuft weiter. +- name: k3s-Knoten lesen (auf ops) + ansible.builtin.shell: mkdir -p {{ infra_targets_dir }} && kubectl get nodes -o json > {{ infra_targets_dir }}/k3s-nodes.json + args: + executable: /bin/bash + delegate_to: localhost + run_once: true + register: k3s_nodes + failed_when: false + changed_when: false + - name: file_sd-Ziele aus tofu output erzeugen (auf ops) ansible.builtin.shell: | set -euo pipefail tofu output -json gpu_vms | python3 {{ infra_repo_src }}/platform/proxmox/scripts/monitoring-targets.py \ --registry {{ infra_repo_src }}/sites/{{ knaster_site }}/registry.yaml --out {{ infra_targets_dir }} \ - --llamacpp '{{ dict(_llamacpp_hosts | zip(_llamacpp_hosts | map('extract', hostvars, 'llamacpp_profiles'))) | to_json }}' + --llamacpp '{{ dict(_llamacpp_hosts | zip(_llamacpp_hosts | map('extract', hostvars, 'llamacpp_profiles'))) | to_json }}' \ + {{ ('--k3s-nodes ' ~ infra_targets_dir ~ '/k3s-nodes.json') if k3s_nodes.rc == 0 else '' }} args: chdir: "{{ infra_tofu_dir }}" executable: /bin/bash @@ -27,4 +40,4 @@ src: "{{ infra_targets_dir }}/{{ item }}.json" dest: "{{ infra_state_dir }}/file_sd/{{ item }}.json" mode: "0644" - loop: [node, gpu, llamacpp] + loop: [node, gpu, llamacpp, kubelet, kube-state-metrics] diff --git a/platform/proxmox/scripts/monitoring-targets.py b/platform/proxmox/scripts/monitoring-targets.py index f415e0f..64d2046 100644 --- a/platform/proxmox/scripts/monitoring-targets.py +++ b/platform/proxmox/scripts/monitoring-targets.py @@ -18,6 +18,11 @@ Registry (die kennt je Slot nur eine VM, die zweite Embedding-Instanz fehlt dort `--llamacpp '{"gpu-pve44": ["embed"], ...}'`. Die Slots eines Dienstes kommen aus der Registry (engine llamacpp, gleicher Container). Labels zusätzlich docker_host (= VM) und container, damit der Healer die tatsächliche VM trifft. Diese Slots zählen auch für node/gpu der VM. + +k3s (Task 6.5): Mit `--k3s-nodes DATEI` (Ausgabe von `kubectl get nodes -o json`) entstehen zusätzlich +DIR/kubelet.json (jeder Knoten, Port 10250, für die Jobs kubelet und cadvisor) und +DIR/kube-state-metrics.json (ein Ziel: NodePort auf der Dienst-VM k3s, sonst auf dem ersten bereiten Knoten). +Ohne die Option bleiben beide Listen leer. """ from __future__ import annotations @@ -35,6 +40,9 @@ LLAMACPP_PROFILES = { "embed": [("llamacpp-embed", 8081), ("llamacpp-rerank", 8082)], "chat": [("llamacpp-chat", 8080)], } +KUBELET_PORT = 10250 +KSM_NODEPORT = 30808 # deploy/k8s/monitoring/kube-state-metrics.yaml +K3S_SERVER = "k3s" # Dienst-VM mit der Steuerung (HA), bevorzugtes Ziel für kube-state-metrics def slots_by_vmid(registry: dict) -> dict[str, list[str]]: @@ -89,11 +97,41 @@ def build_targets(gpu_vms: dict, registry: dict, return result +def _internal_ip(node: dict) -> str | None: + return next((a["address"] for a in node.get("status", {}).get("addresses", []) + if a.get("type") == "InternalIP"), None) + + +def _ready(node: dict) -> bool: + return any(c.get("type") == "Ready" and c.get("status") == "True" + for c in node.get("status", {}).get("conditions", [])) + + +def build_k3s_targets(nodes: dict) -> dict[str, list[dict]]: + """Ziele aus `kubectl get nodes -o json`. instance ist der Knotenname (bleibt bei IP-Wechsel gleich).""" + kubelet: list[dict] = [] + ready: list[tuple[str, str]] = [] + for node in sorted(nodes.get("items", []), key=lambda n: n["metadata"]["name"]): + name, ip = node["metadata"]["name"], _internal_ip(node) + if not ip: + print(f"Warnung: k3s-Knoten {name} ohne InternalIP, übersprungen", file=sys.stderr) + continue + kubelet.append({"targets": [f"{ip}:{KUBELET_PORT}"], "labels": {"instance": name, "node": name}}) + if _ready(node): + ready.append((name, ip)) + ksm: list[dict] = [] + if ready: + _, ip = next((r for r in ready if r[0] == K3S_SERVER), ready[0]) + ksm.append({"targets": [f"{ip}:{KSM_NODEPORT}"], "labels": {"instance": "kube-state-metrics"}}) + return {"kubelet": kubelet, "kube-state-metrics": ksm} + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--registry", required=True, type=Path) ap.add_argument("--out", required=True, type=Path) ap.add_argument("--llamacpp", default="{}", help='JSON {"": ["embed"|"chat", ...]} aus dem Inventar') + ap.add_argument("--k3s-nodes", type=Path, help="Ausgabe von `kubectl get nodes -o json` (Task 6.5)") args = ap.parse_args() gpu_vms = json.load(sys.stdin) @@ -103,6 +141,8 @@ def main() -> int: if unknown: ap.error(f"unbekannte llama.cpp-Profile: {sorted(unknown)}") targets = build_targets(gpu_vms, registry, llamacpp) + k3s_nodes = json.loads(args.k3s_nodes.read_text(encoding="utf-8")) if args.k3s_nodes else {} + targets.update(build_k3s_targets(k3s_nodes)) args.out.mkdir(parents=True, exist_ok=True) for job, entries in targets.items(): path = args.out / f"{job}.json"