69 lines
3.0 KiB
YAML
69 lines
3.0 KiB
YAML
# Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den
|
|
# Alerts und damit beim Healer.
|
|
# GPU-VMs (job node, gpu, llamacpp): file_sd, weil die DHCP-Adressen wechseln (D5). Die Dateien erzeugt
|
|
# platform/proxmox/scripts/monitoring-targets.py auf ops aus `tofu output -json gpu_vms` und der Registry;
|
|
# die Ansible-Rolle infra-stack kopiert sie nach /opt/knaster/infra/file_sd (im Container /etc/knaster-sd).
|
|
global:
|
|
scrape_interval: 30s
|
|
evaluation_interval: 30s
|
|
|
|
rule_files: [alerts.yml]
|
|
|
|
alerting:
|
|
alertmanagers:
|
|
- static_configs: [{targets: ["alertmanager:9093"]}]
|
|
|
|
scrape_configs:
|
|
- job_name: prometheus
|
|
static_configs: [{targets: ["localhost:9090"]}]
|
|
|
|
- job_name: alertmanager
|
|
static_configs: [{targets: ["alertmanager:9093"]}]
|
|
|
|
- job_name: knaster-router
|
|
metrics_path: /metrics
|
|
static_configs: [{targets: ["router:8080"]}]
|
|
|
|
# Ada-Rechner (Windows, D3): Anbindung über die Worker-Registry v2 (Task 6.4), bis dahin nicht erreichbar
|
|
- job_name: vllm
|
|
static_configs:
|
|
- targets: ["gpu-main:8000"]
|
|
labels: {slot: tier1-general, docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"}
|
|
- targets: ["gpu-flex:8000"]
|
|
labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}
|
|
|
|
# llama-server (--metrics) in den GPU-VMs (Task 6.1): Embedding 8081 und Rerank 8082 in gpu-pve44/46,
|
|
# Chat 8080 in gpu-pve48. Ziele aus monitoring-targets.py (Profile laut Inventar-Gruppe gpu_llamacpp).
|
|
- job_name: llamacpp
|
|
file_sd_configs: [{files: [/etc/knaster-sd/llamacpp.json]}]
|
|
|
|
# nvidia_gpu_exporter (nvidia-smi, auch für Maxwell/Pascal; DCGM kann Maxwell nicht) in jeder GPU-VM
|
|
- job_name: gpu
|
|
file_sd_configs: [{files: [/etc/knaster-sd/gpu.json]}]
|
|
|
|
# node_exporter in jeder GPU-VM; Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
|
|
- job_name: node
|
|
file_sd_configs: [{files: [/etc/knaster-sd/node.json]}]
|
|
|
|
# node_exporter im LXC infra selbst (Host-Netz des LXC)
|
|
- job_name: infra-node
|
|
static_configs:
|
|
- targets: ["host.docker.internal:9100"]
|
|
labels: {instance: infra}
|
|
|
|
# prometheus-pve-exporter: jeder Knoten liefert die Cluster-Sicht (cluster=1) und seine eigenen Werte (node=1).
|
|
# Alle fünf als Ziel, damit die Cluster-Sicht auch bei einem ausgefallenen Knoten bleibt (Dashboard wählt eine instance).
|
|
- job_name: proxmox
|
|
metrics_path: /pve
|
|
params: {module: [default], cluster: ["1"], node: ["1"]}
|
|
static_configs:
|
|
- {targets: ["192.168.100.210"], labels: {pve: pve44}}
|
|
- {targets: ["192.168.100.209"], labels: {pve: pve46}}
|
|
- {targets: ["192.168.100.212"], labels: {pve: pve48}}
|
|
- {targets: ["192.168.100.207"], labels: {pve: pve52}}
|
|
- {targets: ["192.168.100.214"], labels: {pve: pve61}}
|
|
relabel_configs:
|
|
- {source_labels: [__address__], target_label: __param_target}
|
|
- {source_labels: [__param_target], target_label: instance}
|
|
- {target_label: __address__, replacement: "pve-exporter:9221"}
|