infra: Compose-Stack mit Monitoring, ohne LiteLLM (Task 5.2)

- deploy/compose/infra: Dienst litellm entfernt (D4), Router auf LITELLM_BASE_URL
  (appsrv08, Port noch offen), alle Images gepinnt, Grafana und pve-exporter
  ohne Profile, node-exporter für den LXC, Zustand über KNASTER_STATE_DIR
- Prometheus: GPU-VMs per file_sd (node, gpu), Proxmox-Knoten per IP, infra-node
- Grafana: Datasource und Dashboards provisioniert (grafana.com 10347, 1860,
  14574 sowie knaster-Übersicht)
- Alertmanager: n8n-Receiver nur vorbereitet (auskommentiert)
- Ansible: Rollen infra-stack und gpu-exporters, Playbooks monitoring.yml und
  monitoring-targets.yml, Skript monitoring-targets.py (tofu output → file_sd)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitignore

@@ -15,3 +15,4 @@ data/
 .DS_Store
 deploy/compose/*/secrets/
 platform/proxmox/ansible/inventory.generated.yml
+platform/proxmox/ansible/monitoring-targets.generated/

core/tests/test_artifacts.py

@@ -25,12 +25,48 @@ def test_healer_labels_available_on_alerts():
     prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text())
     jobs = {j["job_name"]: j for j in prom["scrape_configs"]}
     for job in ("node", "gpu", "vllm"):
-        for sc in jobs[job]["static_configs"]:
+        for sc in jobs[job].get("static_configs", []):
             assert {"slot", "pve_node", "vmid"} <= set(sc["labels"]), (job, sc)
+    # GPU-VMs kommen per file_sd; die Labels setzt platform/proxmox/scripts/monitoring-targets.py
+    generated = _monitoring_targets()
+    for job in ("node", "gpu"):
+        files = [f for sd in jobs[job]["file_sd_configs"] for f in sd["files"]]
+        assert files == [f"/etc/knaster-sd/{job}.json"]
+        assert generated[job]
+        for entry in generated[job]:
+            assert {"slot", "pve_node", "vmid"} <= set(entry["labels"]), (job, entry)
     from knaster.canary import OPS_LABELS
     assert set(OPS_LABELS) == {"docker_host", "container", "pve_node", "vmid"}
 
 
+def _monitoring_targets() -> dict:
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location(
+        "monitoring_targets", REPO_ROOT / "platform/proxmox/scripts/monitoring-targets.py")
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    registry = yaml.safe_load((REPO_ROOT / "sites/knaster/registry.yaml").read_text(encoding="utf-8"))
+    gpu_vms = {  # Form wie `tofu output -json gpu_vms`
+        "gpu-pve44": {"node": "pve44", "vmid": 244, "ipv4": "192.168.100.186"},
+        "gpu-pve48": {"node": "pve48", "vmid": 248, "ipv4": "192.168.100.183"},
+        "gpu-pve52": {"node": "pve52", "vmid": 252, "ipv4": "192.168.100.174"},
+        "gpu-pve46": {"node": "pve46", "vmid": 246, "ipv4": None},
+    }
+    return mod.build_targets(gpu_vms, registry)
+
+
+def test_monitoring_targets_from_tofu_output():
+    t = _monitoring_targets()
+    assert [e["targets"][0] for e in t["node"]] == [
+        "192.168.100.186:9100", "192.168.100.183:9100", "192.168.100.174:9100"]   # gpu-pve46 ohne IP fehlt
+    assert t["gpu"][0]["targets"] == ["192.168.100.186:9835"]
+    pve44 = t["node"][0]["labels"]
+    assert pve44["instance"] == "gpu-pve44" and pve44["vmid"] == "244" and pve44["pve_node"] == "pve44"
+    assert pve44["slot"] == "embed-default" and pve44["slots"] == "embed-default,rerank-default"
+    assert t["node"][2]["labels"]["slot"] == "none"   # gpu-pve52 trägt noch keinen Slot
+
+
 def test_alertmanager_targets_router_endpoint():
     am = yaml.safe_load((DEPLOY / "alertmanager/alertmanager.yml").read_text())
     url = am["receivers"][0]["webhook_configs"][0]["url"]

deploy/alertmanager/alertmanager.yml

@@ -1,11 +1,20 @@
 # Alertmanager → Router-Service (Healer). Kurzes repeat_interval treibt die Eskalationsleiter voran;
-# Cooldowns pro Stufe regelt der Healer selbst.
+# Cooldowns pro Stufe regelt der Healer selbst. Der Router läuft mit KNASTER_DRY_RUN=true.
 route:
   receiver: knaster-healer
   group_by: [alertname, slot, instance]
   group_wait: 30s
   group_interval: 1m
   repeat_interval: 5m
+  # Vorbereitet, noch NICHT aktiv (Task 5.2, braucht den Nutzer: Workflow knaster-notify in n8n
+  # inaktiv importieren, Header-Auth setzen, aktivieren). Achtung: knaster-notify erwartet {text, severity, data},
+  # Alertmanager sendet sein eigenes Format – entweder den Workflow anpassen oder nur über den Router melden
+  # (Healer-Aktion notify → NOTIFY_WEBHOOK_URL). Danach einkommentieren:
+  # routes:
+  #   - receiver: knaster-healer
+  #     continue: true
+  #   - receiver: n8n-notify
+  #     matchers: ['severity=~"high|warning"']
 
 receivers:
   - name: knaster-healer
@@ -16,3 +25,12 @@ receivers:
           authorization:
             type: Bearer
             credentials_file: /etc/alertmanager/router_token
+
+  # - name: n8n-notify
+  #   webhook_configs:
+  #     - url: https://n8n.mindboost.app/webhook/knaster/notify
+  #       send_resolved: true
+  #       http_config:
+  #         authorization:
+  #           type: Bearer
+  #           credentials_file: /etc/alertmanager/n8n_token   # Datei in /opt/knaster/infra/secrets, Mount ergänzen

deploy/compose/infra/.env.example

@@ -1,13 +1,21 @@
+# Vorlage für deploy/compose/infra. Im Cluster schreibt die Ansible-Rolle infra-stack die Datei
+# /opt/knaster/infra/.env (0600) im LXC infra; Geheimnisse erzeugt sie dort mit `openssl rand`.
 KNASTER_VERSION=dev
 KNASTER_SITE=knaster
+KNASTER_STATE_DIR=.
 KNASTER_API_TOKEN=change-me
 KNASTER_DRY_RUN=true
 KNASTER_CANARY_INTERVAL=60
 POSTGRES_PASSWORD=change-me
-LITELLM_API_KEY=change-me
+GRAFANA_ADMIN_PASSWORD=change-me
-LITELLM_MASTER_KEY=change-me
+GRAFANA_KIOSK_PASSWORD=change-me
-OPENROUTER_API_KEY=
+# D4: LiteLLM mit PSE auf appsrv08, kein eigener LiteLLM im Stack.
-RERANK_URL=http://gpu-embed-1:8082
+# TODO (Nutzer, Task 6.3): Port festlegen. Der LiteLLM läuft auf appsrv08 nur im internen Netz
+# (`litellm:4000`); am 12.10. waren von ops aus 4000/4001/8000 auf 192.168.100.29 geschlossen.
+# Port auf appsrv08 für 192.168.100.0/24 freigeben (oder per Traefik) und hier eintragen.
+LITELLM_BASE_URL=http://192.168.100.29:4000
+LITELLM_API_KEY=
+RERANK_URL=
 NOTIFY_WEBHOOK_URL=
 KNASTER_JOB_WEBHOOK_URL=
 PORTAINER_HOST=
@@ -15,4 +23,4 @@ PORTAINER_KEY=
 PROXMOX_URL=
 PROXMOX_TOKEN_ID=
 PROXMOX_TOKEN_SECRET=
-KNASTER_WEBHOOK_AUTH=Bearer change-me
+KNASTER_WEBHOOK_AUTH=

deploy/compose/infra/docker-compose.yml

@@ -1,5 +1,10 @@
-# Rolle "infra" (LXC/VM auf einem i7-Node): Router, LiteLLM, Postgres, Monitoring.
+# Rolle "infra" (LXC 101 auf pve44, HA nach pve48/pve52): Router, TTS, Postgres und Monitoring in einem Stack.
-# Start: cp .env.example .env && docker compose up -d
+# Kein eigener LiteLLM (D4): Der Router nutzt den LiteLLM mit PSE auf appsrv08 (LITELLM_BASE_URL).
+#
+# Lokal:   cp .env.example .env && docker compose up -d
+# Cluster: Ansible-Rolle infra-stack (platform/proxmox/ansible). Zustand und Geheimnisse liegen dort
+#          außerhalb des Repos in KNASTER_STATE_DIR=/opt/knaster/infra (.env, secrets/, file_sd/), Aufruf:
+#          docker compose -p knaster-infra --env-file /opt/knaster/infra/.env -f <repo>/deploy/compose/infra/docker-compose.yml up -d
 name: knaster-infra
 
 x-restart: &restart
@@ -13,29 +18,15 @@ services:
       context: ../../..
       dockerfile: deploy/docker/router.Dockerfile
     image: knaster/router:${KNASTER_VERSION:-dev}
-    env_file: [{path: .env, required: false}]
+    env_file: [{path: "${KNASTER_STATE_DIR:-.}/.env", required: false}]
     environment:
       KNASTER_DB_URL: postgresql+psycopg://knaster:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}@postgres:5432/knaster
       KNASTER_SITE_DIR: /app/sites/${KNASTER_SITE:-knaster}
-      LITELLM_BASE_URL: http://litellm:4000
+      KNASTER_DRY_RUN: ${KNASTER_DRY_RUN:-true}
+      LITELLM_BASE_URL: ${LITELLM_BASE_URL:?LITELLM_BASE_URL fehlt (LiteLLM auf appsrv08, D4)}
     ports: ["8080:8080"]
     depends_on:
       postgres: {condition: service_healthy}
-      litellm: {condition: service_started}
-
-  litellm:
-    <<: *restart
-    image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable}   # vor Deployment auf feste Version pinnen
-    command: ["--config", "/app/config.yaml", "--port", "4000"]
-    env_file: [{path: .env, required: false}]
-    volumes:
-      - ../../litellm/config.yaml:/app/config.yaml:ro
-    ports: ["4000:4000"]
-    healthcheck:
-      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=4)"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
 
   tts-piper:
     <<: *restart
@@ -53,7 +44,7 @@ services:
 
   postgres:
     <<: *restart
-    image: pgvector/pgvector:pg16
+    image: pgvector/pgvector:0.8.7-pg16-trixie
     environment:
       POSTGRES_USER: knaster
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}
@@ -66,42 +57,72 @@ services:
       timeout: 5s
       retries: 5
 
+  # --- Monitoring ---------------------------------------------------------------------------------
   prometheus:
     <<: *restart
-    image: prom/prometheus:v2.53.0
+    image: prom/prometheus:v3.13.4
     command:
       - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
       - --storage.tsdb.retention.time=15d
       - --web.enable-lifecycle
     volumes:
       - ../../prometheus:/etc/prometheus:ro
+      # file_sd: Ziele der GPU-VMs (DHCP, IPs wechseln), erzeugt auf ops aus `tofu output -json gpu_vms`
+      - ${KNASTER_STATE_DIR:-.}/file_sd:/etc/knaster-sd:ro
       - prometheus:/prometheus
+    extra_hosts: ["host.docker.internal:host-gateway"]   # node-exporter des LXC läuft im Host-Netz
     ports: ["9090:9090"]
 
   alertmanager:
     <<: *restart
-    image: prom/alertmanager:v0.27.0
+    image: prom/alertmanager:v0.34.1
-    command: ["--config.file=/etc/alertmanager/alertmanager.yml"]
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+      - --storage.path=/alertmanager
     volumes:
       - ../../alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
-      - ./secrets/router_token:/etc/alertmanager/router_token:ro
+      - ${KNASTER_STATE_DIR:-.}/secrets/router_token:/etc/alertmanager/router_token:ro
+      - alertmanager:/alertmanager
     ports: ["9093:9093"]
 
   grafana:
     <<: *restart
-    image: grafana/grafana:11.1.0
+    image: grafana/grafana:12.4.12
-    profiles: ["dashboards"]
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD fehlt}   # wirkt nur beim ersten Start
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_USERS_DEFAULT_THEME: dark
+      GF_ANALYTICS_REPORTING_ENABLED: "false"
+      GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+      GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
+      GF_NEWS_NEWS_FEED_ENABLED: "false"
+      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/knaster-uebersicht.json
     volumes:
+      - ../../grafana/provisioning:/etc/grafana/provisioning:ro
+      - ../../grafana/dashboards:/var/lib/grafana/dashboards:ro
       - grafana:/var/lib/grafana
     ports: ["3000:3000"]
+    depends_on: [prometheus]
 
   pve-exporter:
     <<: *restart
-    image: prompve/prometheus-pve-exporter:${PVE_EXPORTER_TAG:-latest}   # vor Deployment pinnen
+    image: prompve/prometheus-pve-exporter:3.10.1
-    profiles: ["proxmox"]
+    # Token monitoring@pve!prom (PVEAuditor) als PVE_USER/PVE_TOKEN_NAME/PVE_TOKEN_VALUE/PVE_VERIFY_SSL
+    env_file: [{path: "${KNASTER_STATE_DIR:-.}/secrets/pve-exporter.env", required: false}]
+    expose: ["9221"]
+
+  node-exporter:
+    <<: *restart
+    image: prom/node-exporter:v1.12.1
+    command:
+      - --path.rootfs=/host
+      - --web.listen-address=:9100
+    network_mode: host
+    pid: host
     volumes:
-      - ./secrets/pve.yml:/etc/prometheus/pve.yml:ro
+      - /:/host:ro   # ohne rslave: im unprivilegierten LXC nicht erlaubt
-    ports: ["9221:9221"]
 
   autoheal:
     restart: unless-stopped
@@ -115,4 +136,5 @@ services:
 volumes:
   postgres: {}
   prometheus: {}
+  alertmanager: {}
   grafana: {}

deploy/grafana/dashboards/knaster-uebersicht.json

@@ -0,0 +1,772 @@
+{
+  "uid": "knaster-uebersicht",
+  "title": "knaster – Übersicht",
+  "tags": [
+    "knaster"
+  ],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": false,
+  "refresh": "30s",
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Proxmox-Knoten online",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "count(max by (id) (pve_up{id=~\"node/.*\"}) == 1) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Gäste laufend",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 4,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "count(max by (id) (pve_up{id=~\"(qemu|lxc)/.*\"}) == 1) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "GPUs gemeldet",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 8,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "count(count by (instance, uuid) (nvidia_smi_index)) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Ziele nicht erreichbar",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "count(up{job!~\"vllm|llamacpp\"} == 0) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 5,
+      "type": "stat",
+      "title": "Aktive Alerts",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 16,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "Slots verfügbar",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 20,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "sum(knaster_slot_available) or vector(0)",
+          "legendFormat": ""
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 7,
+      "type": "stat",
+      "title": "Knoten",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 12,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (id) (pve_up{id=~\"node/.*\"})",
+          "legendFormat": "{{id}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "aus",
+                  "color": "red"
+                },
+                "1": {
+                  "text": "an",
+                  "color": "green"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 8,
+      "type": "stat",
+      "title": "GPU-VMs und infra (node_exporter)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 12,
+        "h": 4
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "up{job=~\"node|infra-node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "aus",
+                  "color": "red"
+                },
+                "1": {
+                  "text": "an",
+                  "color": "green"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "CPU je Knoten",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 7
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (id) (pve_cpu_usage_ratio{id=~\"node/.*\"})",
+          "legendFormat": "{{id}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "RAM je Knoten",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 7
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (id) (pve_memory_usage_bytes{id=~\"node/.*\"}) / max by (id) (pve_memory_size_bytes{id=~\"node/.*\"})",
+          "legendFormat": "{{id}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "GPU-Temperatur",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 15,
+        "w": 8,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (instance, name, uuid) (nvidia_smi_temperature_gpu)",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 12,
+      "type": "timeseries",
+      "title": "GPU-Auslastung",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 8,
+        "y": 15,
+        "w": 8,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (instance, name, uuid) (nvidia_smi_utilization_gpu_ratio)",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 13,
+      "type": "timeseries",
+      "title": "GPU-Speicher belegt",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 16,
+        "y": 15,
+        "w": 8,
+        "h": 8
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "max by (instance, name, uuid) (nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes)",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 14,
+      "type": "stat",
+      "title": "Slots (Router)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 23,
+        "w": 12,
+        "h": 5
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "knaster_slot_available",
+          "legendFormat": "{{slot}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": {
+                  "text": "aus",
+                  "color": "red"
+                },
+                "1": {
+                  "text": "an",
+                  "color": "green"
+                }
+              }
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto"
+      }
+    },
+    {
+      "id": 15,
+      "type": "table",
+      "title": "Aktive Alerts",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 23,
+        "w": 12,
+        "h": 5
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "refId": "A",
+          "expr": "ALERTS{alertstate=\"firing\"}",
+          "legendFormat": "",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "alertstate": true
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  }
+}

deploy/grafana/provisioning/dashboards/knaster.yml

@@ -0,0 +1,16 @@
+# Dashboards aus deploy/grafana/dashboards (im Container /var/lib/grafana/dashboards).
+# Herkunft der Community-Dashboards (grafana.com, geladen am 12.10.2026; ${DS_PROMETHEUS} → uid prometheus, __inputs entfernt):
+#   proxmox.json             ID 10347 „Proxmox via Prometheus“ Revision 5 (für prometheus-pve-exporter)
+#   node-exporter-full.json  ID 1860  „Node Exporter Full“ Revision 45
+#   nvidia-gpu.json          ID 14574 „Nvidia GPU Metrics“ Revision 15 (für utkuozdemir/nvidia_gpu_exporter)
+#   knaster-uebersicht.json  eigenes Dashboard (Startseite, für den Kiosk in Task 5.3)
+apiVersion: 1
+providers:
+  - name: knaster
+    folder: knaster
+    type: file
+    disableDeletion: true
+    allowUiUpdates: false
+    updateIntervalSeconds: 60
+    options:
+      path: /var/lib/grafana/dashboards

deploy/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+# Feste uid "prometheus": Die Dashboards in deploy/grafana/dashboards verweisen darauf.
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false

deploy/prometheus/prometheus.yml

@@ -1,5 +1,8 @@
 # Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den
-# Alerts und damit beim Healer. Hostnamen = DNS-Namen der VMs (ANNAHME bis Deployment).
+# Alerts und damit beim Healer.
+# GPU-VMs (job node, gpu): file_sd, weil die DHCP-Adressen wechseln (D5). Die Dateien erzeugt
+# platform/proxmox/scripts/monitoring-targets.py auf ops aus `tofu output -json gpu_vms` und der Registry;
+# die Ansible-Rolle infra-stack kopiert sie nach /opt/knaster/infra/file_sd (im Container /etc/knaster-sd).
 global:
   scrape_interval: 30s
   evaluation_interval: 30s
@@ -11,10 +14,17 @@ alerting:
     - static_configs: [{targets: ["alertmanager:9093"]}]
 
 scrape_configs:
+  - job_name: prometheus
+    static_configs: [{targets: ["localhost:9090"]}]
+
+  - job_name: alertmanager
+    static_configs: [{targets: ["alertmanager:9093"]}]
+
   - job_name: knaster-router
     metrics_path: /metrics
     static_configs: [{targets: ["router:8080"]}]
 
+  # Ada-Rechner (Windows, D3): Anbindung über die Worker-Registry v2 (Task 6.4), bis dahin nicht erreichbar
   - job_name: vllm
     static_configs:
       - targets: ["gpu-main:8000"]
@@ -22,6 +32,7 @@ scrape_configs:
       - targets: ["gpu-flex:8000"]
         labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}
 
+  # llama.cpp-Slots kommen erst mit Phase 6 in die GPU-VMs; bis dahin nicht erreichbar (Platzhalter)
   - job_name: llamacpp
     static_configs:
       - targets: ["gpu-embed-1:8081"]
@@ -31,36 +42,31 @@ scrape_configs:
       - targets: ["gpu-small:8080"]
         labels: {slot: verifier-small, docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"}
 
+  # nvidia_gpu_exporter (nvidia-smi, auch für Maxwell/Pascal; DCGM kann Maxwell nicht) in jeder GPU-VM
   - job_name: gpu
-    static_configs:
+    file_sd_configs: [{files: [/etc/knaster-sd/gpu.json]}]
-      - targets: ["gpu-main:9835"]
-        labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
-      - targets: ["gpu-flex:9835"]
-        labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
-      - targets: ["gpu-embed-1:9835"]
-        labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
-      - targets: ["gpu-embed-2:9835"]
-        labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
-      - targets: ["gpu-small:9835"]
-        labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
 
-  - job_name: node   # Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
+  # node_exporter in jeder GPU-VM; Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
-    static_configs:
+  - job_name: node
-      - targets: ["gpu-main:9100"]
+    file_sd_configs: [{files: [/etc/knaster-sd/node.json]}]
-        labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
-      - targets: ["gpu-flex:9100"]
-        labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
-      - targets: ["gpu-embed-1:9100"]
-        labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
-      - targets: ["gpu-embed-2:9100"]
-        labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
-      - targets: ["gpu-small:9100"]
-        labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
 
+  # node_exporter im LXC infra selbst (Host-Netz des LXC)
+  - job_name: infra-node
+    static_configs:
+      - targets: ["host.docker.internal:9100"]
+        labels: {instance: infra}
+
+  # prometheus-pve-exporter: jeder Knoten liefert die Cluster-Sicht (cluster=1) und seine eigenen Werte (node=1).
+  # Alle fünf als Ziel, damit die Cluster-Sicht auch bei einem ausgefallenen Knoten bleibt (Dashboard wählt eine instance).
   - job_name: proxmox
     metrics_path: /pve
-    params: {module: [default]}
+    params: {module: [default], cluster: ["1"], node: ["1"]}
-    static_configs: [{targets: ["pve1", "pve2", "pve3", "pve4", "pve-ada"]}]
+    static_configs:
+      - {targets: ["192.168.100.210"], labels: {pve: pve44}}
+      - {targets: ["192.168.100.209"], labels: {pve: pve46}}
+      - {targets: ["192.168.100.212"], labels: {pve: pve48}}
+      - {targets: ["192.168.100.207"], labels: {pve: pve52}}
+      - {targets: ["192.168.100.214"], labels: {pve: pve61}}
     relabel_configs:
       - {source_labels: [__address__], target_label: __param_target}
       - {source_labels: [__param_target], target_label: instance}

platform/proxmox/ansible/playbooks/monitoring-targets.yml

@@ -0,0 +1,10 @@
+# file_sd-Ziele der GPU-VMs auffrischen, wenn sich DHCP-Adressen geändert haben. Auf ops:
+#   cd /root/cluster-knaster/platform/proxmox/tofu && . /root/.config/knaster/tofu.env && bash ../scripts/refresh-vm-access.sh
+#   cd ../ansible && ansible-playbook playbooks/monitoring-targets.yml
+- name: Infra-LXC – file_sd-Ziele
+  hosts: infra
+  tasks:
+    - name: Ziele erzeugen und kopieren
+      ansible.builtin.include_role:
+        name: infra-stack
+        tasks_from: targets

platform/proxmox/ansible/playbooks/monitoring.yml

@@ -0,0 +1,10 @@
+# Nur Monitoring (Task 5.2), ohne Treiber-/Docker-Rollen der GPU-VMs (kein Docker-Neustart, kein Reboot).
+# Aufruf auf ops (Verzeichnis platform/proxmox/ansible), vorher Adressen auffrischen (refresh-vm-access.sh):
+#   ansible-playbook playbooks/monitoring.yml
+- name: GPU-VMs – Exporter (node_exporter, nvidia_gpu_exporter)
+  hosts: gpu_vms
+  roles: [gpu-exporters]
+
+- name: Infra-LXC – Compose-Stack mit Monitoring
+  hosts: infra
+  roles: [infra-stack]

platform/proxmox/ansible/playbooks/site.yml

@@ -19,8 +19,9 @@
               eth0:
                 dhcp-identifier: mac
         mode: "0600"
-  roles: [docker-host, gpu-node]
+  roles: [docker-host, gpu-node, gpu-exporters]
 
-- name: Infra-LXC (Docker)
+# Nur Monitoring ohne Treiber/Neustart der GPU-VMs: playbooks/monitoring.yml
+- name: Infra-LXC (Docker, Compose-Stack mit Monitoring)
   hosts: infra
-  roles: [docker-host]
+  roles: [docker-host, infra-stack]

platform/proxmox/ansible/roles/gpu-exporters/defaults/main.yml

@@ -0,0 +1,10 @@
+# Monitoring-Exporter in den GPU-VMs (Task 5.2). Ziele erzeugt monitoring-targets.py als file_sd.
+# node_exporter aus Ubuntu (24.04: 1.7.x), liest die Xid-Zähler aus dem Textfile-Verzeichnis (Rolle gpu-node)
+gpu_exporters_textfile_dir: /var/lib/node_exporter/textfile
+# nvidia_gpu_exporter (nvidia-smi statt DCGM/NVML: läuft auch mit Maxwell/Pascal), Port 9835
+nvidia_gpu_exporter_version: "1.15.1"
+# sha256 aus checksums.txt des Releases v1.15.1 (am 12.10.2026 gegen die Datei geprüft)
+nvidia_gpu_exporter_sha256: 0dc8c2756c6853ec7e6a7c404b229d8f764443bef89f33e13b3c4278b309acd4
+# ufw: nur eigene allow-Regeln hinzufügen (Kommentar knaster-monitoring), ufw nie ein- oder ausschalten
+gpu_exporters_allow_from: 192.168.100.0/24
+gpu_exporters_ports: [9100, 9835]

platform/proxmox/ansible/roles/gpu-exporters/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: restart node-exporter
+  ansible.builtin.service:
+    name: prometheus-node-exporter
+    state: restarted

platform/proxmox/ansible/roles/gpu-exporters/tasks/main.yml

@@ -0,0 +1,60 @@
+# Bewusst getrennt von gpu-node: kein Treiber, kein Docker-Neustart, kein Reboot.
+- name: node_exporter installieren (ohne Zusatz-Collectors)
+  ansible.builtin.apt:
+    name: prometheus-node-exporter
+    state: present
+    install_recommends: false
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Textfile-Verzeichnis (Xid-Zähler aus gpu-xid-textfile.sh)
+  ansible.builtin.file:
+    path: "{{ gpu_exporters_textfile_dir }}"
+    state: directory
+    mode: "0755"
+
+- name: node_exporter liest das Textfile-Verzeichnis
+  ansible.builtin.lineinfile:
+    path: /etc/default/prometheus-node-exporter
+    regexp: '^ARGS='
+    line: 'ARGS="--collector.textfile.directory={{ gpu_exporters_textfile_dir }}"'
+  notify: restart node-exporter
+
+- name: node_exporter aktiv
+  ansible.builtin.service:
+    name: prometheus-node-exporter
+    state: started
+    enabled: true
+
+- name: nvidia_gpu_exporter herunterladen (Prüfsumme fest)
+  ansible.builtin.get_url:
+    url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ nvidia_gpu_exporter_version }}/nvidia-gpu-exporter_{{ nvidia_gpu_exporter_version }}_linux_amd64.deb"
+    dest: "/var/cache/apt/archives/nvidia-gpu-exporter_{{ nvidia_gpu_exporter_version }}_linux_amd64.deb"
+    checksum: "sha256:{{ nvidia_gpu_exporter_sha256 }}"
+    mode: "0644"
+
+- name: nvidia_gpu_exporter installieren
+  ansible.builtin.apt:
+    deb: "/var/cache/apt/archives/nvidia-gpu-exporter_{{ nvidia_gpu_exporter_version }}_linux_amd64.deb"
+
+- name: nvidia_gpu_exporter aktiv
+  ansible.builtin.service:
+    name: nvidia_gpu_exporter
+    state: started
+    enabled: true
+
+- name: Ist ufw vorhanden?
+  ansible.builtin.stat:
+    path: /usr/sbin/ufw
+  register: ufw_bin
+
+# Regeln wirken auch bei inaktivem ufw (werden vorgemerkt). Fremde Regeln bleiben unberührt.
+- name: ufw – Exporter-Ports aus dem LAN erlauben
+  community.general.ufw:
+    rule: allow
+    src: "{{ gpu_exporters_allow_from }}"
+    port: "{{ item | string }}"
+    proto: tcp
+    comment: knaster-monitoring
+  loop: "{{ gpu_exporters_ports }}"
+  when: ufw_bin.stat.exists

platform/proxmox/ansible/roles/infra-stack/defaults/main.yml

@@ -0,0 +1,29 @@
+# Compose-Stack im LXC infra (Task 5.2). Repo-Auszug und Zustand liegen getrennt:
+#   infra_repo_dir   Kopie der nötigen Repo-Teile von ops (wird bei jedem Lauf abgeglichen, --delete)
+#   infra_state_dir  .env (0600, Geheimnisse), secrets/, file_sd/ – nie im Repo, nie überschrieben
+infra_repo_src: "{{ playbook_dir }}/../../../.."
+infra_repo_dir: /opt/knaster/repo
+infra_state_dir: /opt/knaster/infra
+infra_compose_project: knaster-infra
+infra_compose_file: "{{ infra_repo_dir }}/deploy/compose/infra/docker-compose.yml"
+
+# Einstellungen in .env (keine Geheimnisse). Änderungen hier, nicht im Container.
+infra_settings:
+  KNASTER_VERSION: dev
+  KNASTER_SITE: "{{ knaster_site }}"
+  KNASTER_STATE_DIR: "{{ infra_state_dir }}"
+  KNASTER_DRY_RUN: "true"          # Global Constraint: erst nach einer Woche Beobachtung einzeln freischalten
+  KNASTER_CANARY_INTERVAL: "60"
+  # D4: LiteLLM mit PSE auf appsrv08. TODO (Nutzer, Task 6.3): Port ist dort noch nicht im LAN veröffentlicht.
+  LITELLM_BASE_URL: http://192.168.100.29:4000
+
+# Geheimnisse: werden im Container einmalig mit `openssl rand -hex 24` erzeugt und danach nie geändert
+# (Postgres- und Grafana-Admin-Passwort wirken nur bei der Erstinitialisierung).
+infra_secrets: [POSTGRES_PASSWORD, KNASTER_API_TOKEN, GRAFANA_ADMIN_PASSWORD, GRAFANA_KIOSK_PASSWORD]
+
+# Lese-Token monitoring@pve!prom (PVEAuditor) auf ops
+infra_pve_exporter_env: /root/.config/knaster/pve-exporter.env
+
+# file_sd: auf ops aus `tofu output -json gpu_vms` erzeugt (nicht im Repo)
+infra_tofu_dir: "{{ playbook_dir }}/../../tofu"
+infra_targets_dir: "{{ playbook_dir }}/../monitoring-targets.generated"

platform/proxmox/ansible/roles/infra-stack/tasks/main.yml

@@ -0,0 +1,134 @@
+# Compose-Stack infra: Router, TTS, Postgres, Prometheus, Alertmanager, Grafana, pve-exporter, node-exporter.
+# Idempotent: Geheimnisse nur anlegen, wenn sie fehlen; compose up ändert nur, was sich geändert hat.
+- name: Werkzeuge
+  ansible.builtin.apt:
+    name: [rsync, openssl, curl]
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Verzeichnisse
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: "{{ item.mode }}"
+  loop:
+    - {path: /opt/knaster, mode: "0755"}
+    - {path: "{{ infra_repo_dir }}", mode: "0755"}
+    - {path: "{{ infra_state_dir }}", mode: "0700"}
+    - {path: "{{ infra_state_dir }}/secrets", mode: "0700"}
+
+# Nur die Teile, die Build und Konfiguration brauchen. Nie .git, .venv, tofu-State oder .env-Dateien.
+- name: Repo-Auszug von ops abgleichen
+  ansible.posix.synchronize:
+    src: "{{ infra_repo_src }}/"
+    dest: "{{ infra_repo_dir }}/"
+    delete: true
+    rsync_opts:
+      - "--exclude=__pycache__"
+      - "--exclude=.pytest_cache"
+      - "--exclude=*.egg-info"
+      - "--exclude=.env"
+      - "--exclude=secrets/"
+      - "--include=/core/***"
+      - "--include=/config/***"
+      - "--include=/catalog/***"
+      - "--include=/skills/***"
+      - "--include=/sites/***"
+      - "--include=/deploy/***"
+      - "--exclude=*"
+
+- name: Einstellungen in .env
+  ansible.builtin.lineinfile:
+    path: "{{ infra_state_dir }}/.env"
+    regexp: "^{{ item.key }}="
+    line: "{{ item.key }}={{ item.value }}"
+    create: true
+    mode: "0600"
+  loop: "{{ infra_settings | dict2items }}"
+
+# Gibt nur Namen aus, nie Werte
+- name: Fehlende Geheimnisse im Container erzeugen
+  ansible.builtin.shell: |
+    set -eu
+    umask 077
+    f={{ infra_state_dir }}/.env
+    for k in {{ infra_secrets | join(' ') }}; do
+      grep -q "^$k=" "$f" || { echo "$k=$(openssl rand -hex 24)" >> "$f"; echo "neu: $k"; }
+    done
+    chmod 0600 "$f"
+  args:
+    executable: /bin/bash
+  register: secrets_gen
+  changed_when: "'neu:' in secrets_gen.stdout"
+
+- name: Router-Token für Alertmanager (= KNASTER_API_TOKEN, lesbar für nobody im Container)
+  ansible.builtin.shell: |
+    set -eu
+    umask 077
+    cd {{ infra_state_dir }}
+    v=$(sed -n 's/^KNASTER_API_TOKEN=//p' .env)
+    t=secrets/router_token
+    if [ ! -f "$t" ] || [ "$(cat "$t")" != "$v" ]; then printf '%s' "$v" > "$t"; echo geändert; fi
+    chown 65534:65534 "$t"
+    chmod 0400 "$t"
+  args:
+    executable: /bin/bash
+  register: router_token
+  changed_when: "'geändert' in router_token.stdout"
+  no_log: true
+
+- name: Proxmox-Lese-Token für den pve-exporter
+  ansible.builtin.copy:
+    src: "{{ infra_pve_exporter_env }}"
+    dest: "{{ infra_state_dir }}/secrets/pve-exporter.env"
+    mode: "0600"
+  no_log: true
+
+- name: file_sd-Ziele
+  ansible.builtin.include_tasks: targets.yml
+
+- name: Stack bauen und starten
+  ansible.builtin.command: >
+    docker compose -p {{ infra_compose_project }} --env-file {{ infra_state_dir }}/.env
+    -f {{ infra_compose_file }} up -d --build --remove-orphans
+  register: compose_up
+  changed_when: compose_up.stderr is search('Container \S+\s+(Created|Recreated|Started)')
+
+- name: Prometheus und Alertmanager laden geänderte Konfiguration (bind-mount, kein Neustart nötig)
+  ansible.builtin.command: "curl -fsS -m 10 -X POST http://127.0.0.1:{{ item }}/-/reload"
+  loop: [9090, 9093]
+  register: reload
+  retries: 15
+  delay: 4
+  until: reload.rc == 0
+  changed_when: false
+
+- name: Grafana-Konto kiosk (Viewer) anlegen, Passwort aus .env
+  ansible.builtin.shell: |
+    set -eu
+    cd {{ infra_state_dir }}
+    A=$(sed -n 's/^GRAFANA_ADMIN_PASSWORD=//p' .env)
+    K=$(sed -n 's/^GRAFANA_KIOSK_PASSWORD=//p' .env)
+    umask 077
+    cfg=$(mktemp)
+    trap 'rm -f "$cfg"' EXIT
+    printf 'user = "admin:%s"\n' "$A" > "$cfg"   # Passwort nicht in der Prozessliste
+    g() { curl -sS -K "$cfg" -H 'Content-Type: application/json' "$@"; }
+    for i in $(seq 60); do curl -fsS -m 3 http://127.0.0.1:3000/api/health >/dev/null 2>&1 && break; sleep 2; done
+    code=$(g -o /dev/null -w '%{http_code}' 'http://127.0.0.1:3000/api/users/lookup?loginOrEmail=kiosk')
+    if [ "$code" = 404 ]; then
+      printf '{"name":"Kiosk","login":"kiosk","password":"%s"}' "$K" \
+        | g -f -X POST --data @- http://127.0.0.1:3000/api/admin/users >/dev/null
+      echo angelegt
+    elif [ "$code" != 200 ]; then
+      echo "Grafana-API antwortet $code" >&2
+      exit 1
+    fi
+    id=$(g -f 'http://127.0.0.1:3000/api/users/lookup?loginOrEmail=kiosk' | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])')
+    g -f -X PATCH --data '{"role":"Viewer"}' "http://127.0.0.1:3000/api/org/users/$id" >/dev/null
+  args:
+    executable: /bin/bash
+  register: kiosk
+  changed_when: "'angelegt' in kiosk.stdout"
+  no_log: true

platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml

@@ -0,0 +1,26 @@
+# file_sd-Ziele der GPU-VMs. Auch einzeln aufrufbar: playbooks/monitoring-targets.yml
+# (nach refresh-vm-access.sh, wenn sich DHCP-Adressen geändert haben). Prometheus liest die Dateien selbst neu.
+- name: file_sd-Ziele aus tofu output erzeugen (auf ops)
+  ansible.builtin.shell: |
+    set -euo pipefail
+    tofu output -json gpu_vms | python3 {{ infra_repo_src }}/platform/proxmox/scripts/monitoring-targets.py \
+      --registry {{ infra_repo_src }}/sites/{{ knaster_site }}/registry.yaml --out {{ infra_targets_dir }}
+  args:
+    chdir: "{{ infra_tofu_dir }}"
+    executable: /bin/bash
+  delegate_to: localhost
+  run_once: true
+  changed_when: false
+
+- name: file_sd-Verzeichnis
+  ansible.builtin.file:
+    path: "{{ infra_state_dir }}/file_sd"
+    state: directory
+    mode: "0755"
+
+- name: file_sd-Ziele in den Container kopieren
+  ansible.builtin.copy:
+    src: "{{ infra_targets_dir }}/{{ item }}.json"
+    dest: "{{ infra_state_dir }}/file_sd/{{ item }}.json"
+    mode: "0644"
+  loop: [node, gpu]

platform/proxmox/scripts/monitoring-targets.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Prometheus-file_sd-Ziele für die GPU-VMs erzeugen (läuft auf ops).
+
+Die GPU-VMs bekommen ihre Adressen per DHCP (D5), deshalb stehen sie nicht fest in prometheus.yml.
+Eingabe ist `tofu output -json gpu_vms` (stdin), dazu die Registry für die Slot-Labels des Healers.
+
+    tofu output -json gpu_vms | python3 monitoring-targets.py --registry sites/knaster/registry.yaml --out DIR
+
+Schreibt DIR/node.json (node_exporter, Port 9100) und DIR/gpu.json (nvidia_gpu_exporter, Port 9835).
+
+Labels je Ziel: instance (VM-Name, bleibt bei IP-Wechsel gleich), vm, pve_node, vmid, slot, slots.
+`slot` ist der erste Slot der Registry mit dieser vmid (Reihenfolge der Registry), `slots` alle,
+kommagetrennt. VMs ohne Slot bekommen slot="none" (der Healer markiert dann keinen echten Slot).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import yaml
+
+PORTS = {"node": 9100, "gpu": 9835}
+
+
+def slots_by_vmid(registry: dict) -> dict[str, list[str]]:
+    out: dict[str, list[str]] = {}
+    for name, slot in (registry.get("slots") or {}).items():
+        vmid = str((slot.get("ops") or {}).get("vmid", ""))
+        if vmid:
+            out.setdefault(vmid, []).append(name)
+    return out
+
+
+def build_targets(gpu_vms: dict, registry: dict) -> dict[str, list[dict]]:
+    by_vmid = slots_by_vmid(registry)
+    result: dict[str, list[dict]] = {job: [] for job in PORTS}
+    for name in sorted(gpu_vms):
+        vm = gpu_vms[name]
+        if not vm.get("ipv4"):
+            print(f"Warnung: {name} hat keine IPv4 (Gast-Agent?), übersprungen", file=sys.stderr)
+            continue
+        vmid = str(vm["vmid"])
+        slots = by_vmid.get(vmid, [])
+        labels = {
+            "instance": name,
+            "vm": name,
+            "pve_node": vm["node"],
+            "vmid": vmid,
+            "slot": slots[0] if slots else "none",
+            "slots": ",".join(slots),
+        }
+        for job, port in PORTS.items():
+            result[job].append({"targets": [f"{vm['ipv4']}:{port}"], "labels": labels})
+    return result
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--registry", required=True, type=Path)
+    ap.add_argument("--out", required=True, type=Path)
+    args = ap.parse_args()
+
+    gpu_vms = json.load(sys.stdin)
+    registry = yaml.safe_load(args.registry.read_text(encoding="utf-8"))
+    targets = build_targets(gpu_vms, registry)
+    args.out.mkdir(parents=True, exist_ok=True)
+    for job, entries in targets.items():
+        path = args.out / f"{job}.json"
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(entries, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        tmp.replace(path)
+        print(f"{path}: {len(entries)} Ziele")
+    return 0 if targets["node"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())