cluster-knaster/deploy/prometheus/alerts.yml

# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen.
groups:
  - name: knaster-slots
    rules:
      - alert: SlotEndpointDown
        expr: knaster_slot_endpoint_up == 0
        for: 2m
        labels: {severity: high}
        annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"}

      - alert: SlotCanaryFailing
        expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1
        for: 5m
        labels: {severity: warning}
        annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"}

      - alert: VllmKvCacheSaturated
        expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95
        for: 10m
        labels: {severity: warning}
        annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"}

  - name: knaster-gpu
    rules:
      - alert: GpuXidError
        # aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector)
        expr: increase(knaster_gpu_xid_errors_total[10m]) > 0
        labels: {severity: high}
        annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"}

      - alert: GpuTemperatureHigh
        expr: nvidia_smi_temperature_gpu > 85
        for: 5m
        labels: {severity: warning}
        annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"}

  - name: knaster-nodes
    rules:
      - alert: NodeDown
        expr: up{job="node"} == 0
        for: 3m
        labels: {severity: high}
        annotations: {summary: "{{ $labels.instance }} nicht erreichbar"}

      - alert: DiskAlmostFull
        expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10
        for: 15m
        labels: {severity: warning}
        annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}

  # k3s (Task 6.5): nur melden, der Healer greift nicht ein (config/healer-rules.yaml, Aktion notify)
  - name: knaster-k3s
    rules:
      - alert: K3sNodeNotReady
        expr: kube_node_status_condition{condition="Ready", status="true"} == 0
        for: 5m
        labels: {severity: high}
        annotations: {summary: "k3s-Knoten {{ $labels.node }} nicht bereit"}

      - alert: K3sPodCrashLooping
        # max_over_time: zwischen zwei Neustarts steht der Container kurz nicht in CrashLoopBackOff (Test 19.10.)
        expr: max by (namespace, pod, container) (max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}[5m])) >= 1
        for: 10m
        labels: {severity: warning}
        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) startet immer wieder neu"}

      - alert: K3sDeploymentUnavailable
        expr: max by (namespace, deployment) (kube_deployment_spec_replicas) > max by (namespace, deployment) (kube_deployment_status_replicas_available)
        for: 15m
        labels: {severity: warning}
        annotations: {summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} hat zu wenige bereite Pods"}

      - alert: K3sContainerMemoryNearLimit
        expr: >
          max by (namespace, pod, container) (container_memory_working_set_bytes{job="cadvisor"})
          / on (namespace, pod, container)
          max by (namespace, pod, container) (kube_pod_container_resource_limits{resource="memory"}) > 0.9
        for: 10m
        labels: {severity: warning}
        annotations: {summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) über 90 % des Speicherlimits"}