Robert RappandClaude Opus 5.5
4ac251715e
ci / test (push) Failing after 23s
kube-state-metrics (NodePort 30808) und ServiceAccount prometheus-infra (nur nodes/metrics) im Cluster; Prometheus auf infra liest kubelet/cAdvisor per TLS gegen die k3s-CA. Ziele per file_sd aus kubectl get nodes, cAdvisor auf die nötigen Reihen begrenzt. Dashboard "k3s – Pods" (Vorgabe nex), Alerts K3s* nur mit notify. Playbook monitoring-k3s.yml. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
80 lines
3.4 KiB
YAML
80 lines
3.4 KiB
YAML
# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen.
|
|
groups:
|
|
- name: knaster-slots
|
|
rules:
|
|
- alert: SlotEndpointDown
|
|
expr: knaster_slot_endpoint_up == 0
|
|
for: 2m
|
|
labels: {severity: high}
|
|
annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"}
|
|
|
|
- alert: SlotCanaryFailing
|
|
expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1
|
|
for: 5m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"}
|
|
|
|
- alert: VllmKvCacheSaturated
|
|
expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"}
|
|
|
|
- name: knaster-gpu
|
|
rules:
|
|
- alert: GpuXidError
|
|
# aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector)
|
|
expr: increase(knaster_gpu_xid_errors_total[10m]) > 0
|
|
labels: {severity: high}
|
|
annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"}
|
|
|
|
- alert: GpuTemperatureHigh
|
|
expr: nvidia_smi_temperature_gpu > 85
|
|
for: 5m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"}
|
|
|
|
- name: knaster-nodes
|
|
rules:
|
|
- alert: NodeDown
|
|
expr: up{job="node"} == 0
|
|
for: 3m
|
|
labels: {severity: high}
|
|
annotations: {summary: "{{ $labels.instance }} nicht erreichbar"}
|
|
|
|
- alert: DiskAlmostFull
|
|
expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10
|
|
for: 15m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}
|
|
|
|
# k3s (Task 6.5): nur melden, der Healer greift nicht ein (config/healer-rules.yaml, Aktion notify)
|
|
- name: knaster-k3s
|
|
rules:
|
|
- alert: K3sNodeNotReady
|
|
expr: kube_node_status_condition{condition="Ready", status="true"} == 0
|
|
for: 5m
|
|
labels: {severity: high}
|
|
annotations: {summary: "k3s-Knoten {{ $labels.node }} nicht bereit"}
|
|
|
|
- alert: K3sPodCrashLooping
|
|
expr: max by (namespace, pod, container) (kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) == 1
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) startet immer wieder neu"}
|
|
|
|
- alert: K3sDeploymentUnavailable
|
|
expr: max by (namespace, deployment) (kube_deployment_spec_replicas) > max by (namespace, deployment) (kube_deployment_status_replicas_available)
|
|
for: 15m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} hat zu wenige bereite Pods"}
|
|
|
|
- alert: K3sContainerMemoryNearLimit
|
|
expr: >
|
|
max by (namespace, pod, container) (container_memory_working_set_bytes{job="cadvisor"})
|
|
/ on (namespace, pod, container)
|
|
max by (namespace, pod, container) (kube_pod_container_resource_limits{resource="memory"}) > 0.9
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) über 90 % des Speicherlimits"}
|