core: Policy, Pflege-Agent (HITL), Healer, Lifecycle-Pipeline, Graduierung + Konfiguration

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 00:43:26 +02:00
parent f79e2a8556
commit f2da893f71
28 changed files with 2179 additions and 10 deletions
@@ -0,0 +1,53 @@
+# Deterministisches Autohealing (SPEC-healer). {label} wird aus den Alert-Labels ersetzt.
+max_actions_per_hour: 10
+# Infrastruktur-Aktionen laufen nur, wenn KNASTER_DRY_RUN=false UND sie hier stehen.
+enabled_actions: []
+rules:
+  - id: slot-endpoint-down
+    match: {alertname: SlotEndpointDown}
+    ladder:
+      - {action: restart_container, params: {host: "{docker_host}", container: "{container}"}, attempts: 2, cooldown: 5m}
+      - {action: restart_vm, params: {pve_node: "{pve_node}", vmid: "{vmid}"}, attempts: 1, cooldown: 15m}
+      - {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
+    on_resolved:
+      - {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
+
+  - id: slot-canary-failing
+    match: {alertname: SlotCanaryFailing}
+    ladder:
+      - {action: restart_container, params: {host: "{docker_host}", container: "{container}"}, attempts: 1, cooldown: 10m}
+      - {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
+    on_resolved:
+      - {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
+
+  - id: gpu-xid
+    match: {alertname: GpuXidError}
+    ladder:
+      - {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
+      - {action: restart_vm, params: {pve_node: "{pve_node}", vmid: "{vmid}"}, attempts: 1, cooldown: 20m}
+      # ältere Karten ohne FLR brauchen ggf. einen Host-Neustart – nur mit Freigabe/Wartungsfenster
+      - {action: reboot_host, params: {pve_node: "{pve_node}"}, attempts: 1}
+    on_resolved:
+      - {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
+
+  - id: gpu-hot
+    match: {alertname: GpuTemperatureHigh}
+    ladder:
+      - {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
+    on_resolved:
+      - {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
+
+  - id: kv-cache-saturated
+    match: {alertname: VllmKvCacheSaturated}
+    ladder:
+      - {action: notify, params: {text: "KV-Cache von {slot} dauerhaft voll – Last verteilen oder max_model_len senken", severity: warning}, attempts: 1}
+
+  - id: node-down
+    match: {alertname: NodeDown}
+    ladder:
+      - {action: notify, params: {text: "Node {instance} nicht erreichbar – Proxmox HA prüfen", severity: high}, attempts: 1}
+
+  - id: disk-full
+    match: {alertname: DiskAlmostFull}
+    ladder:
+      - {action: notify, params: {text: "Platte auf {instance} ({mountpoint}) fast voll", severity: warning}, attempts: 1}