cluster-knaster/deploy/prometheus/prometheus.yml

# Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den
# Alerts und damit beim Healer.
# GPU-VMs (job node, gpu, llamacpp): file_sd, weil die DHCP-Adressen wechseln (D5). Die Dateien erzeugt
# platform/proxmox/scripts/monitoring-targets.py auf ops aus `tofu output -json gpu_vms` und der Registry;
# die Ansible-Rolle infra-stack kopiert sie nach /opt/knaster/infra/file_sd (im Container /etc/knaster-sd).
global:
  scrape_interval: 30s
  evaluation_interval: 30s

rule_files: [alerts.yml]

alerting:
  alertmanagers:
    - static_configs: [{targets: ["alertmanager:9093"]}]

scrape_configs:
  - job_name: prometheus
    static_configs: [{targets: ["localhost:9090"]}]

  - job_name: alertmanager
    static_configs: [{targets: ["alertmanager:9093"]}]

  - job_name: knaster-router
    metrics_path: /metrics
    static_configs: [{targets: ["router:8080"]}]

  # Ada-Rechner (Windows, D3): Anbindung über die Worker-Registry v2 (Task 6.4), bis dahin nicht erreichbar
  - job_name: vllm
    static_configs:
      - targets: ["gpu-main:8000"]
        labels: {slot: tier1-general, docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"}
      - targets: ["gpu-flex:8000"]
        labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}

  # llama-server (--metrics) in den GPU-VMs (Task 6.1): Embedding 8081 und Rerank 8082 in gpu-pve44/46,
  # Chat 8080 in gpu-pve48. Ziele aus monitoring-targets.py (Profile laut Inventar-Gruppe gpu_llamacpp).
  - job_name: llamacpp
    file_sd_configs: [{files: [/etc/knaster-sd/llamacpp.json]}]

  # nvidia_gpu_exporter (nvidia-smi, auch für Maxwell/Pascal; DCGM kann Maxwell nicht) in jeder GPU-VM
  - job_name: gpu
    file_sd_configs: [{files: [/etc/knaster-sd/gpu.json]}]

  # node_exporter in jeder GPU-VM; Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
  - job_name: node
    file_sd_configs: [{files: [/etc/knaster-sd/node.json]}]

  # node_exporter im LXC infra selbst (Host-Netz des LXC)
  - job_name: infra-node
    static_configs:
      - targets: ["host.docker.internal:9100"]
        labels: {instance: infra}

  # prometheus-pve-exporter: jeder Knoten liefert die Cluster-Sicht (cluster=1) und seine eigenen Werte (node=1).
  # Alle fünf als Ziel, damit die Cluster-Sicht auch bei einem ausgefallenen Knoten bleibt (Dashboard wählt eine instance).
  - job_name: proxmox
    metrics_path: /pve
    params: {module: [default], cluster: ["1"], node: ["1"]}
    static_configs:
      - {targets: ["192.168.100.210"], labels: {pve: pve44}}
      - {targets: ["192.168.100.209"], labels: {pve: pve46}}
      - {targets: ["192.168.100.212"], labels: {pve: pve48}}
      - {targets: ["192.168.100.207"], labels: {pve: pve52}}
      - {targets: ["192.168.100.214"], labels: {pve: pve61}}
    relabel_configs:
      - {source_labels: [__address__], target_label: __param_target}
      - {source_labels: [__param_target], target_label: instance}
      - {target_label: __address__, replacement: "pve-exporter:9221"}

  # k3s (Task 6.5): kubelet direkt auf jedem Knoten (10250), TLS gegen die k3s-CA, Token des ServiceAccounts
  # prometheus-infra (nur nodes/metrics). Ziele per file_sd aus `kubectl get nodes` (monitoring-targets.py auf ops).
  # infra hat 4 GB RAM: von kubelet und cAdvisor nur die Reihen, die Dashboard „k3s – Pods“ und Alerts brauchen.
  - job_name: kubelet
    scheme: https
    authorization: {credentials_file: /etc/knaster-k3s/token}
    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
    metric_relabel_configs:
      - source_labels: [__name__]
        regex: kubelet_running_pods|kubelet_running_containers|kubelet_volume_stats_(used|capacity)_bytes
        action: keep

  - job_name: cadvisor
    scheme: https
    metrics_path: /metrics/cadvisor
    authorization: {credentials_file: /etc/knaster-k3s/token}
    tls_config: {ca_file: /etc/knaster-k3s/ca.crt}
    file_sd_configs: [{files: [/etc/knaster-sd/kubelet.json]}]
    metric_relabel_configs:
      - source_labels: [__name__]
        regex: container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_oom_events_total|container_network_(receive|transmit)_bytes_total
        action: keep
      # nur Pods (keine systemd-Slices, keine Docker-Container der Compose-Stacks)
      - source_labels: [pod]
        regex: ""
        action: drop
      # CPU/Speicher je Container: Pod-Summe und Pause-Container verwerfen (Netz gibt es nur je Pod)
      - source_labels: [__name__, container]
        regex: "container_(cpu|memory|oom)[^;]*;(POD)?"
        action: drop

  - job_name: kube-state-metrics
    honor_labels: true   # namespace/pod/node kommen aus den Metriken selbst
    file_sd_configs: [{files: [/etc/knaster-sd/kube-state-metrics.json]}]