54 lines
2.5 KiB
YAML
54 lines
2.5 KiB
YAML
# Deterministisches Autohealing (SPEC-healer). {label} wird aus den Alert-Labels ersetzt.
|
||
max_actions_per_hour: 10
|
||
# Infrastruktur-Aktionen laufen nur, wenn KNASTER_DRY_RUN=false UND sie hier stehen.
|
||
enabled_actions: []
|
||
rules:
|
||
- id: slot-endpoint-down
|
||
match: {alertname: SlotEndpointDown}
|
||
ladder:
|
||
- {action: restart_container, params: {host: "{docker_host}", container: "{container}"}, attempts: 2, cooldown: 5m}
|
||
- {action: restart_vm, params: {pve_node: "{pve_node}", vmid: "{vmid}"}, attempts: 1, cooldown: 15m}
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: slot-canary-failing
|
||
match: {alertname: SlotCanaryFailing}
|
||
ladder:
|
||
- {action: restart_container, params: {host: "{docker_host}", container: "{container}"}, attempts: 1, cooldown: 10m}
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: gpu-xid
|
||
match: {alertname: GpuXidError}
|
||
ladder:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
- {action: restart_vm, params: {pve_node: "{pve_node}", vmid: "{vmid}"}, attempts: 1, cooldown: 20m}
|
||
# ältere Karten ohne FLR brauchen ggf. einen Host-Neustart – nur mit Freigabe/Wartungsfenster
|
||
- {action: reboot_host, params: {pve_node: "{pve_node}"}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: gpu-hot
|
||
match: {alertname: GpuTemperatureHigh}
|
||
ladder:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: kv-cache-saturated
|
||
match: {alertname: VllmKvCacheSaturated}
|
||
ladder:
|
||
- {action: notify, params: {text: "KV-Cache von {slot} dauerhaft voll – Last verteilen oder max_model_len senken", severity: warning}, attempts: 1}
|
||
|
||
- id: node-down
|
||
match: {alertname: NodeDown}
|
||
ladder:
|
||
- {action: notify, params: {text: "Node {instance} nicht erreichbar – Proxmox HA prüfen", severity: high}, attempts: 1}
|
||
|
||
- id: disk-full
|
||
match: {alertname: DiskAlmostFull}
|
||
ladder:
|
||
- {action: notify, params: {text: "Platte auf {instance} ({mountpoint}) fast voll", severity: warning}, attempts: 1}
|