deploy: Dockerfile, Compose pro Rolle, LiteLLM, Prometheus/Alertmanager, Canary-Prober, n8n-Workflows

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 00:51:31 +02:00
parent 4cbe313c62
commit cb5a413169
28 changed files with 1731 additions and 9 deletions
@@ -0,0 +1,49 @@
+# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen.
+groups:
+  - name: knaster-slots
+    rules:
+      - alert: SlotEndpointDown
+        expr: knaster_slot_endpoint_up == 0
+        for: 2m
+        labels: {severity: high}
+        annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"}
+
+      - alert: SlotCanaryFailing
+        expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1
+        for: 5m
+        labels: {severity: warning}
+        annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"}
+
+      - alert: VllmKvCacheSaturated
+        expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"}
+
+  - name: knaster-gpu
+    rules:
+      - alert: GpuXidError
+        # aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector)
+        expr: increase(knaster_gpu_xid_errors_total[10m]) > 0
+        labels: {severity: high}
+        annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"}
+
+      - alert: GpuTemperatureHigh
+        expr: nvidia_smi_temperature_gpu > 85
+        for: 5m
+        labels: {severity: warning}
+        annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"}
+
+  - name: knaster-nodes
+    rules:
+      - alert: NodeDown
+        expr: up{job="node"} == 0
+        for: 3m
+        labels: {severity: high}
+        annotations: {summary: "{{ $labels.instance }} nicht erreichbar"}
+
+      - alert: DiskAlmostFull
+        expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10
+        for: 15m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}