Robert RappandClaude Opus 5.5
4ac251715e
ci / test (push) Failing after 23s
kube-state-metrics (NodePort 30808) und ServiceAccount prometheus-infra (nur nodes/metrics) im Cluster; Prometheus auf infra liest kubelet/cAdvisor per TLS gegen die k3s-CA. Ziele per file_sd aus kubectl get nodes, cAdvisor auf die nötigen Reihen begrenzt. Dashboard "k3s – Pods" (Vorgabe nex), Alerts K3s* nur mit notify. Playbook monitoring-k3s.yml. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
3.5 KiB
YAML
75 lines
3.5 KiB
YAML
# Deterministisches Autohealing (SPEC-healer). {label} wird aus den Alert-Labels ersetzt.
|
||
max_actions_per_hour: 10
|
||
# Infrastruktur-Aktionen laufen nur, wenn KNASTER_DRY_RUN=false UND sie hier stehen.
|
||
enabled_actions: []
|
||
rules:
|
||
- id: slot-endpoint-down
|
||
match: {alertname: SlotEndpointDown}
|
||
ladder:
|
||
- {action: restart_container, params: {host: "{docker_host}", container: "{container}"}, attempts: 2, cooldown: 5m}
|
||
- {action: restart_vm, params: {pve_node: "{pve_node}", vmid: "{vmid}"}, attempts: 1, cooldown: 15m}
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: slot-canary-failing
|
||
match: {alertname: SlotCanaryFailing}
|
||
ladder:
|
||
- {action: restart_container, params: {host: "{docker_host}", container: "{container}"}, attempts: 1, cooldown: 10m}
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: gpu-xid
|
||
match: {alertname: GpuXidError}
|
||
ladder:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
- {action: restart_vm, params: {pve_node: "{pve_node}", vmid: "{vmid}"}, attempts: 1, cooldown: 20m}
|
||
# ältere Karten ohne FLR brauchen ggf. einen Host-Neustart – nur mit Freigabe/Wartungsfenster
|
||
- {action: reboot_host, params: {pve_node: "{pve_node}"}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: gpu-hot
|
||
match: {alertname: GpuTemperatureHigh}
|
||
ladder:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: degraded}, attempts: 1}
|
||
on_resolved:
|
||
- {action: mark_slot, params: {slot: "{slot}", status: ok}, only_if_set_by_healer: true}
|
||
|
||
- id: kv-cache-saturated
|
||
match: {alertname: VllmKvCacheSaturated}
|
||
ladder:
|
||
- {action: notify, params: {text: "KV-Cache von {slot} dauerhaft voll – Last verteilen oder max_model_len senken", severity: warning}, attempts: 1}
|
||
|
||
- id: node-down
|
||
match: {alertname: NodeDown}
|
||
ladder:
|
||
- {action: notify, params: {text: "Node {instance} nicht erreichbar – Proxmox HA prüfen", severity: high}, attempts: 1}
|
||
|
||
- id: disk-full
|
||
match: {alertname: DiskAlmostFull}
|
||
ladder:
|
||
- {action: notify, params: {text: "Platte auf {instance} ({mountpoint}) fast voll", severity: warning}, attempts: 1}
|
||
|
||
# k3s (Task 6.5): nur melden; Pods startet Kubernetes selbst neu
|
||
- id: k3s-node-not-ready
|
||
match: {alertname: K3sNodeNotReady}
|
||
ladder:
|
||
- {action: notify, params: {text: "k3s-Knoten {node} nicht bereit – VM und k3s-agent prüfen", severity: high}, attempts: 1}
|
||
|
||
- id: k3s-pod-crashloop
|
||
match: {alertname: K3sPodCrashLooping}
|
||
ladder:
|
||
- {action: notify, params: {text: "Pod {namespace}/{pod} ({container}) startet immer wieder neu", severity: warning}, attempts: 1}
|
||
|
||
- id: k3s-deployment-unavailable
|
||
match: {alertname: K3sDeploymentUnavailable}
|
||
ladder:
|
||
- {action: notify, params: {text: "Deployment {namespace}/{deployment} hat zu wenige bereite Pods", severity: warning}, attempts: 1}
|
||
|
||
- id: k3s-memory-near-limit
|
||
match: {alertname: K3sContainerMemoryNearLimit}
|
||
ladder:
|
||
- {action: notify, params: {text: "{namespace}/{pod} ({container}) über 90 % des Speicherlimits", severity: warning}, attempts: 1}
|