rorapp/cluster-knaster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

infra: Compose-Stack mit Monitoring, ohne LiteLLM (Task 5.2)
ci / test (push) Successful in 28s
Details

Browse Source 
- deploy/compose/infra: Dienst litellm entfernt (D4), Router auf LITELLM_BASE_URL
  (appsrv08, Port noch offen), alle Images gepinnt, Grafana und pve-exporter
  ohne Profile, node-exporter für den LXC, Zustand über KNASTER_STATE_DIR
- Prometheus: GPU-VMs per file_sd (node, gpu), Proxmox-Knoten per IP, infra-node
- Grafana: Datasource und Dashboards provisioniert (grafana.com 10347, 1860,
  14574 sowie knaster-Übersicht)
- Alertmanager: n8n-Receiver nur vorbereitet (auskommentiert)
- Ansible: Rollen infra-stack und gpu-exporters, Playbooks monitoring.yml und
  monitoring-targets.yml, Skript monitoring-targets.py (tofu output → file_sd)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Robert Rapp
2026-10-12 20:22:26 +02:00
co-authored by Claude Opus 5.5
parent fd48fd1c7f
commit bf7c137adf
22 changed files with 23101 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/alertmanager/alertmanager.yml
+19 -1
View File
@@ -1,11 +1,20 @@
# Alertmanager → Router-Service (Healer). Kurzes repeat_interval treibt die Eskalationsleiter voran;
# Cooldowns pro Stufe regelt der Healer selbst.
# Cooldowns pro Stufe regelt der Healer selbst. Der Router läuft mit KNASTER_DRY_RUN=true.
route:
receiver: knaster-healer
group_by: [alertname, slot, instance]
group_wait: 30s
group_interval: 1m
repeat_interval: 5m
# Vorbereitet, noch NICHT aktiv (Task 5.2, braucht den Nutzer: Workflow knaster-notify in n8n
# inaktiv importieren, Header-Auth setzen, aktivieren). Achtung: knaster-notify erwartet {text, severity, data},
# Alertmanager sendet sein eigenes Format – entweder den Workflow anpassen oder nur über den Router melden
# (Healer-Aktion notify → NOTIFY_WEBHOOK_URL). Danach einkommentieren:
# routes:
# - receiver: knaster-healer
# continue: true
# - receiver: n8n-notify
# matchers: ['severity=~"high|warning"']
receivers:
- name: knaster-healer
@@ -16,3 +25,12 @@ receivers:
authorization:
type: Bearer
credentials_file: /etc/alertmanager/router_token
# - name: n8n-notify
# webhook_configs:
# - url: https://n8n.mindboost.app/webhook/knaster/notify
# send_resolved: true
# http_config:
# authorization:
# type: Bearer
# credentials_file: /etc/alertmanager/n8n_token # Datei in /opt/knaster/infra/secrets, Mount ergänzen
deploy/compose/infra/.env.example
+13 -5
View File
@@ -1,13 +1,21 @@
# Vorlage für deploy/compose/infra. Im Cluster schreibt die Ansible-Rolle infra-stack die Datei
# /opt/knaster/infra/.env (0600) im LXC infra; Geheimnisse erzeugt sie dort mit `openssl rand`.
KNASTER_VERSION=dev
KNASTER_SITE=knaster
KNASTER_STATE_DIR=.
KNASTER_API_TOKEN=change-me
KNASTER_DRY_RUN=true
KNASTER_CANARY_INTERVAL=60
POSTGRES_PASSWORD=change-me
LITELLM_API_KEY=change-me
LITELLM_MASTER_KEY=change-me
OPENROUTER_API_KEY=
RERANK_URL=http://gpu-embed-1:8082
GRAFANA_ADMIN_PASSWORD=change-me
GRAFANA_KIOSK_PASSWORD=change-me
# D4: LiteLLM mit PSE auf appsrv08, kein eigener LiteLLM im Stack.
# TODO (Nutzer, Task 6.3): Port festlegen. Der LiteLLM läuft auf appsrv08 nur im internen Netz
# (`litellm:4000`); am 12.10. waren von ops aus 4000/4001/8000 auf 192.168.100.29 geschlossen.
# Port auf appsrv08 für 192.168.100.0/24 freigeben (oder per Traefik) und hier eintragen.
LITELLM_BASE_URL=http://192.168.100.29:4000
LITELLM_API_KEY=
RERANK_URL=
NOTIFY_WEBHOOK_URL=
KNASTER_JOB_WEBHOOK_URL=
PORTAINER_HOST=
@@ -15,4 +23,4 @@ PORTAINER_KEY=
PROXMOX_URL=
PROXMOX_TOKEN_ID=
PROXMOX_TOKEN_SECRET=
KNASTER_WEBHOOK_AUTH=Bearer change-me
KNASTER_WEBHOOK_AUTH=
deploy/compose/infra/docker-compose.yml
+52 -30
View File
@@ -1,5 +1,10 @@
# Rolle "infra" (LXC/VM auf einem i7-Node): Router, LiteLLM, Postgres, Monitoring.
# Start: cp .env.example .env && docker compose up -d
# Rolle "infra" (LXC 101 auf pve44, HA nach pve48/pve52): Router, TTS, Postgres und Monitoring in einem Stack.
# Kein eigener LiteLLM (D4): Der Router nutzt den LiteLLM mit PSE auf appsrv08 (LITELLM_BASE_URL).
#
# Lokal: cp .env.example .env && docker compose up -d
# Cluster: Ansible-Rolle infra-stack (platform/proxmox/ansible). Zustand und Geheimnisse liegen dort
# außerhalb des Repos in KNASTER_STATE_DIR=/opt/knaster/infra (.env, secrets/, file_sd/), Aufruf:
# docker compose -p knaster-infra --env-file /opt/knaster/infra/.env -f <repo>/deploy/compose/infra/docker-compose.yml up -d
name: knaster-infra
x-restart: &restart
@@ -13,29 +18,15 @@ services:
context: ../../..
dockerfile: deploy/docker/router.Dockerfile
image: knaster/router:${KNASTER_VERSION:-dev}
env_file: [{path: .env, required: false}]
env_file: [{path: "${KNASTER_STATE_DIR:-.}/.env", required: false}]
environment:
KNASTER_DB_URL: postgresql+psycopg://knaster:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}@postgres:5432/knaster
KNASTER_SITE_DIR: /app/sites/${KNASTER_SITE:-knaster}
LITELLM_BASE_URL: http://litellm:4000
KNASTER_DRY_RUN: ${KNASTER_DRY_RUN:-true}
LITELLM_BASE_URL: ${LITELLM_BASE_URL:?LITELLM_BASE_URL fehlt (LiteLLM auf appsrv08, D4)}
ports: ["8080:8080"]
depends_on:
postgres: {condition: service_healthy}
litellm: {condition: service_started}
litellm:
<<: *restart
image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable} # vor Deployment auf feste Version pinnen
command: ["--config", "/app/config.yaml", "--port", "4000"]
env_file: [{path: .env, required: false}]
volumes:
- ../../litellm/config.yaml:/app/config.yaml:ro
ports: ["4000:4000"]
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=4)"]
interval: 30s
timeout: 5s
retries: 3
tts-piper:
<<: *restart
@@ -53,7 +44,7 @@ services:
postgres:
<<: *restart
image: pgvector/pgvector:pg16
image: pgvector/pgvector:0.8.7-pg16-trixie
environment:
POSTGRES_USER: knaster
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}
@@ -66,42 +57,72 @@ services:
timeout: 5s
retries: 5
# --- Monitoring ---------------------------------------------------------------------------------
prometheus:
<<: *restart
image: prom/prometheus:v2.53.0
image: prom/prometheus:v3.13.4
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --web.enable-lifecycle
volumes:
- ../../prometheus:/etc/prometheus:ro
# file_sd: Ziele der GPU-VMs (DHCP, IPs wechseln), erzeugt auf ops aus `tofu output -json gpu_vms`
- ${KNASTER_STATE_DIR:-.}/file_sd:/etc/knaster-sd:ro
- prometheus:/prometheus
extra_hosts: ["host.docker.internal:host-gateway"] # node-exporter des LXC läuft im Host-Netz
ports: ["9090:9090"]
alertmanager:
<<: *restart
image: prom/alertmanager:v0.27.0
command: ["--config.file=/etc/alertmanager/alertmanager.yml"]
image: prom/alertmanager:v0.34.1
command:
- --config.file=/etc/alertmanager/alertmanager.yml
- --storage.path=/alertmanager
volumes:
- ../../alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
- ./secrets/router_token:/etc/alertmanager/router_token:ro
- ${KNASTER_STATE_DIR:-.}/secrets/router_token:/etc/alertmanager/router_token:ro
- alertmanager:/alertmanager
ports: ["9093:9093"]
grafana:
<<: *restart
image: grafana/grafana:11.1.0
profiles: ["dashboards"]
image: grafana/grafana:12.4.12
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD fehlt} # wirkt nur beim ersten Start
GF_USERS_ALLOW_SIGN_UP: "false"
GF_USERS_DEFAULT_THEME: dark
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/knaster-uebersicht.json
volumes:
- ../../grafana/provisioning:/etc/grafana/provisioning:ro
- ../../grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana:/var/lib/grafana
ports: ["3000:3000"]
depends_on: [prometheus]
pve-exporter:
<<: *restart
image: prompve/prometheus-pve-exporter:${PVE_EXPORTER_TAG:-latest} # vor Deployment pinnen
profiles: ["proxmox"]
image: prompve/prometheus-pve-exporter:3.10.1
# Token monitoring@pve!prom (PVEAuditor) als PVE_USER/PVE_TOKEN_NAME/PVE_TOKEN_VALUE/PVE_VERIFY_SSL
env_file: [{path: "${KNASTER_STATE_DIR:-.}/secrets/pve-exporter.env", required: false}]
expose: ["9221"]
node-exporter:
<<: *restart
image: prom/node-exporter:v1.12.1
command:
- --path.rootfs=/host
- --web.listen-address=:9100
network_mode: host
pid: host
volumes:
- ./secrets/pve.yml:/etc/prometheus/pve.yml:ro
ports: ["9221:9221"]
- /:/host:ro # ohne rslave: im unprivilegierten LXC nicht erlaubt
autoheal:
restart: unless-stopped
@@ -115,4 +136,5 @@ services:
volumes:
postgres: {}
prometheus: {}
alertmanager: {}
grafana: {}
deploy/grafana/dashboards/knaster-uebersicht.json
+772
View File
@@ -0,0 +1,772 @@
{
"uid": "knaster-uebersicht",
"title": "knaster – Übersicht",
"tags": [
"knaster"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"editable": false,
"refresh": "30s",
"time": {
"from": "now-6h",
"to": "now"
},
"panels": [
{
"id": 1,
"type": "stat",
"title": "Proxmox-Knoten online",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "count(max by (id) (pve_up{id=~\"node/.*\"}) == 1) or vector(0)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 2,
"type": "stat",
"title": "Gäste laufend",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 4,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "count(max by (id) (pve_up{id=~\"(qemu|lxc)/.*\"}) == 1) or vector(0)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 3,
"type": "stat",
"title": "GPUs gemeldet",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 8,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "count(count by (instance, uuid) (nvidia_smi_index)) or vector(0)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 4,
"type": "stat",
"title": "Ziele nicht erreichbar",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "count(up{job!~\"vllm|llamacpp\"} == 0) or vector(0)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 5,
"type": "stat",
"title": "Aktive Alerts",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 6,
"type": "stat",
"title": "Slots verfügbar",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 20,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum(knaster_slot_available) or vector(0)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 7,
"type": "stat",
"title": "Knoten",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 4,
"w": 12,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "max by (id) (pve_up{id=~\"node/.*\"})",
"legendFormat": "{{id}}"
}
],
"fieldConfig": {
"defaults": {
"mappings": [
{
"type": "value",
"options": {
"0": {
"text": "aus",
"color": "red"
},
"1": {
"text": "an",
"color": "green"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 8,
"type": "stat",
"title": "GPU-VMs und infra (node_exporter)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 4,
"w": 12,
"h": 4
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "up{job=~\"node|infra-node\"}",
"legendFormat": "{{instance}}"
}
],
"fieldConfig": {
"defaults": {
"mappings": [
{
"type": "value",
"options": {
"0": {
"text": "aus",
"color": "red"
},
"1": {
"text": "an",
"color": "green"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 9,
"type": "timeseries",
"title": "CPU je Knoten",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 7
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "max by (id) (pve_cpu_usage_ratio{id=~\"node/.*\"})",
"legendFormat": "{{id}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
}
},
{
"id": 10,
"type": "timeseries",
"title": "RAM je Knoten",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 7
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "max by (id) (pve_memory_usage_bytes{id=~\"node/.*\"}) / max by (id) (pve_memory_size_bytes{id=~\"node/.*\"})",
"legendFormat": "{{id}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
}
},
{
"id": 11,
"type": "timeseries",
"title": "GPU-Temperatur",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 15,
"w": 8,
"h": 8
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "max by (instance, name, uuid) (nvidia_smi_temperature_gpu)",
"legendFormat": "{{instance}} {{name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "celsius"
},
"overrides": []
}
},
{
"id": 12,
"type": "timeseries",
"title": "GPU-Auslastung",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 8,
"y": 15,
"w": 8,
"h": 8
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "max by (instance, name, uuid) (nvidia_smi_utilization_gpu_ratio)",
"legendFormat": "{{instance}} {{name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
}
},
{
"id": 13,
"type": "timeseries",
"title": "GPU-Speicher belegt",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 15,
"w": 8,
"h": 8
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "max by (instance, name, uuid) (nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes)",
"legendFormat": "{{instance}} {{name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
}
},
{
"id": 14,
"type": "stat",
"title": "Slots (Router)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 23,
"w": 12,
"h": 5
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "knaster_slot_available",
"legendFormat": "{{slot}}"
}
],
"fieldConfig": {
"defaults": {
"mappings": [
{
"type": "value",
"options": {
"0": {
"text": "aus",
"color": "red"
},
"1": {
"text": "an",
"color": "green"
}
}
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "background",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 15,
"type": "table",
"title": "Aktive Alerts",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 23,
"w": 12,
"h": 5
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "ALERTS{alertstate=\"firing\"}",
"legendFormat": "",
"instant": true,
"format": "table"
}
],
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": true,
"__name__": true,
"alertstate": true
}
}
}
]
}
],
"templating": {
"list": []
},
"annotations": {
"list": []
}
}
deploy/grafana/dashboards/node-exporter-full.json
+15536
View File
File diff suppressed because it is too large Load Diff
deploy/grafana/dashboards/nvidia-gpu.json
+4260
View File
File diff suppressed because it is too large Load Diff
deploy/grafana/dashboards/proxmox.json
+1986
View File
File diff suppressed because it is too large Load Diff
deploy/grafana/provisioning/dashboards/knaster.yml
+16
View File
@@ -0,0 +1,16 @@
# Dashboards aus deploy/grafana/dashboards (im Container /var/lib/grafana/dashboards).
# Herkunft der Community-Dashboards (grafana.com, geladen am 12.10.2026; ${DS_PROMETHEUS} → uid prometheus, __inputs entfernt):
# proxmox.json ID 10347 „Proxmox via Prometheus“ Revision 5 (für prometheus-pve-exporter)
# node-exporter-full.json ID 1860 „Node Exporter Full“ Revision 45
# nvidia-gpu.json ID 14574 „Nvidia GPU Metrics“ Revision 15 (für utkuozdemir/nvidia_gpu_exporter)
# knaster-uebersicht.json eigenes Dashboard (Startseite, für den Kiosk in Task 5.3)
apiVersion: 1
providers:
- name: knaster
folder: knaster
type: file
disableDeletion: true
allowUiUpdates: false
updateIntervalSeconds: 60
options:
path: /var/lib/grafana/dashboards
deploy/grafana/provisioning/datasources/prometheus.yml
+10
View File
@@ -0,0 +1,10 @@
# Feste uid "prometheus": Die Dashboards in deploy/grafana/dashboards verweisen darauf.
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
deploy/prometheus/prometheus.yml
+32 -26
View File
@@ -1,5 +1,8 @@
# Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den
# Alerts und damit beim Healer. Hostnamen = DNS-Namen der VMs (ANNAHME bis Deployment).
# Alerts und damit beim Healer.
# GPU-VMs (job node, gpu): file_sd, weil die DHCP-Adressen wechseln (D5). Die Dateien erzeugt
# platform/proxmox/scripts/monitoring-targets.py auf ops aus `tofu output -json gpu_vms` und der Registry;
# die Ansible-Rolle infra-stack kopiert sie nach /opt/knaster/infra/file_sd (im Container /etc/knaster-sd).
global:
scrape_interval: 30s
evaluation_interval: 30s
@@ -11,10 +14,17 @@ alerting:
- static_configs: [{targets: ["alertmanager:9093"]}]
scrape_configs:
- job_name: prometheus
static_configs: [{targets: ["localhost:9090"]}]
- job_name: alertmanager
static_configs: [{targets: ["alertmanager:9093"]}]
- job_name: knaster-router
metrics_path: /metrics
static_configs: [{targets: ["router:8080"]}]
# Ada-Rechner (Windows, D3): Anbindung über die Worker-Registry v2 (Task 6.4), bis dahin nicht erreichbar
- job_name: vllm
static_configs:
- targets: ["gpu-main:8000"]
@@ -22,6 +32,7 @@ scrape_configs:
- targets: ["gpu-flex:8000"]
labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}
# llama.cpp-Slots kommen erst mit Phase 6 in die GPU-VMs; bis dahin nicht erreichbar (Platzhalter)
- job_name: llamacpp
static_configs:
- targets: ["gpu-embed-1:8081"]
@@ -31,36 +42,31 @@ scrape_configs:
- targets: ["gpu-small:8080"]
labels: {slot: verifier-small, docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"}
# nvidia_gpu_exporter (nvidia-smi, auch für Maxwell/Pascal; DCGM kann Maxwell nicht) in jeder GPU-VM
- job_name: gpu
static_configs:
- targets: ["gpu-main:9835"]
labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
- targets: ["gpu-flex:9835"]
labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
- targets: ["gpu-embed-1:9835"]
labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
- targets: ["gpu-embed-2:9835"]
labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
- targets: ["gpu-small:9835"]
labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
file_sd_configs: [{files: [/etc/knaster-sd/gpu.json]}]
- job_name: node # Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
static_configs:
- targets: ["gpu-main:9100"]
labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
- targets: ["gpu-flex:9100"]
labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
- targets: ["gpu-embed-1:9100"]
labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
- targets: ["gpu-embed-2:9100"]
labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
- targets: ["gpu-small:9100"]
labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
# node_exporter in jeder GPU-VM; Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
- job_name: node
file_sd_configs: [{files: [/etc/knaster-sd/node.json]}]
# node_exporter im LXC infra selbst (Host-Netz des LXC)
- job_name: infra-node
static_configs:
- targets: ["host.docker.internal:9100"]
labels: {instance: infra}
# prometheus-pve-exporter: jeder Knoten liefert die Cluster-Sicht (cluster=1) und seine eigenen Werte (node=1).
# Alle fünf als Ziel, damit die Cluster-Sicht auch bei einem ausgefallenen Knoten bleibt (Dashboard wählt eine instance).
- job_name: proxmox
metrics_path: /pve
params: {module: [default]}
static_configs: [{targets: ["pve1", "pve2", "pve3", "pve4", "pve-ada"]}]
params: {module: [default], cluster: ["1"], node: ["1"]}
static_configs:
- {targets: ["192.168.100.210"], labels: {pve: pve44}}
- {targets: ["192.168.100.209"], labels: {pve: pve46}}
- {targets: ["192.168.100.212"], labels: {pve: pve48}}
- {targets: ["192.168.100.207"], labels: {pve: pve52}}
- {targets: ["192.168.100.214"], labels: {pve: pve61}}
relabel_configs:
- {source_labels: [__address__], target_label: __param_target}
- {source_labels: [__param_target], target_label: instance}