50 lines
1.8 KiB
YAML
50 lines
1.8 KiB
YAML
# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen.
|
|
groups:
|
|
- name: knaster-slots
|
|
rules:
|
|
- alert: SlotEndpointDown
|
|
expr: knaster_slot_endpoint_up == 0
|
|
for: 2m
|
|
labels: {severity: high}
|
|
annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"}
|
|
|
|
- alert: SlotCanaryFailing
|
|
expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1
|
|
for: 5m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"}
|
|
|
|
- alert: VllmKvCacheSaturated
|
|
expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"}
|
|
|
|
- name: knaster-gpu
|
|
rules:
|
|
- alert: GpuXidError
|
|
# aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector)
|
|
expr: increase(knaster_gpu_xid_errors_total[10m]) > 0
|
|
labels: {severity: high}
|
|
annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"}
|
|
|
|
- alert: GpuTemperatureHigh
|
|
expr: nvidia_smi_temperature_gpu > 85
|
|
for: 5m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"}
|
|
|
|
- name: knaster-nodes
|
|
rules:
|
|
- alert: NodeDown
|
|
expr: up{job="node"} == 0
|
|
for: 3m
|
|
labels: {severity: high}
|
|
annotations: {summary: "{{ $labels.instance }} nicht erreichbar"}
|
|
|
|
- alert: DiskAlmostFull
|
|
expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10
|
|
for: 15m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}
|