deploy: Dockerfile, Compose pro Rolle, LiteLLM, Prometheus/Alertmanager, Canary-Prober, n8n-Workflows

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 00:51:31 +02:00
parent 4cbe313c62
commit cb5a413169
28 changed files with 1731 additions and 9 deletions
@@ -0,0 +1,49 @@
+# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen.
+groups:
+  - name: knaster-slots
+    rules:
+      - alert: SlotEndpointDown
+        expr: knaster_slot_endpoint_up == 0
+        for: 2m
+        labels: {severity: high}
+        annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"}
+
+      - alert: SlotCanaryFailing
+        expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1
+        for: 5m
+        labels: {severity: warning}
+        annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"}
+
+      - alert: VllmKvCacheSaturated
+        expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"}
+
+  - name: knaster-gpu
+    rules:
+      - alert: GpuXidError
+        # aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector)
+        expr: increase(knaster_gpu_xid_errors_total[10m]) > 0
+        labels: {severity: high}
+        annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"}
+
+      - alert: GpuTemperatureHigh
+        expr: nvidia_smi_temperature_gpu > 85
+        for: 5m
+        labels: {severity: warning}
+        annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"}
+
+  - name: knaster-nodes
+    rules:
+      - alert: NodeDown
+        expr: up{job="node"} == 0
+        for: 3m
+        labels: {severity: high}
+        annotations: {summary: "{{ $labels.instance }} nicht erreichbar"}
+
+      - alert: DiskAlmostFull
+        expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10
+        for: 15m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}
@@ -0,0 +1,67 @@
+# Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den
+# Alerts und damit beim Healer. Hostnamen = DNS-Namen der VMs (ANNAHME bis Deployment).
+global:
+  scrape_interval: 30s
+  evaluation_interval: 30s
+
+rule_files: [alerts.yml]
+
+alerting:
+  alertmanagers:
+    - static_configs: [{targets: ["alertmanager:9093"]}]
+
+scrape_configs:
+  - job_name: knaster-router
+    metrics_path: /metrics
+    static_configs: [{targets: ["router:8080"]}]
+
+  - job_name: vllm
+    static_configs:
+      - targets: ["gpu-main:8000"]
+        labels: {slot: tier1-general, docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"}
+      - targets: ["gpu-flex:8000"]
+        labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}
+
+  - job_name: llamacpp
+    static_configs:
+      - targets: ["gpu-embed-1:8081"]
+        labels: {slot: embed-default, docker_host: gpu-embed-1, container: llamacpp-embed, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-embed-1:8082"]
+        labels: {slot: rerank-default, docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-small:8080"]
+        labels: {slot: verifier-small, docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"}
+
+  - job_name: gpu
+    static_configs:
+      - targets: ["gpu-main:9835"]
+        labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
+      - targets: ["gpu-flex:9835"]
+        labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
+      - targets: ["gpu-embed-1:9835"]
+        labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-embed-2:9835"]
+        labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
+      - targets: ["gpu-small:9835"]
+        labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
+
+  - job_name: node   # Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
+    static_configs:
+      - targets: ["gpu-main:9100"]
+        labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
+      - targets: ["gpu-flex:9100"]
+        labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
+      - targets: ["gpu-embed-1:9100"]
+        labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-embed-2:9100"]
+        labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
+      - targets: ["gpu-small:9100"]
+        labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
+
+  - job_name: proxmox
+    metrics_path: /pve
+    params: {module: [default]}
+    static_configs: [{targets: ["pve1", "pve2", "pve3", "pve4", "pve-ada"]}]
+    relabel_configs:
+      - {source_labels: [__address__], target_label: __param_target}
+      - {source_labels: [__param_target], target_label: instance}
+      - {target_label: __address__, replacement: "pve-exporter:9221"}