deploy: Dockerfile, Compose pro Rolle, LiteLLM, Prometheus/Alertmanager, Canary-Prober, n8n-Workflows
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,18 @@
|
||||
# Alertmanager → Router-Service (Healer). Kurzes repeat_interval treibt die Eskalationsleiter voran;
|
||||
# Cooldowns pro Stufe regelt der Healer selbst.
|
||||
route:
|
||||
receiver: knaster-healer
|
||||
group_by: [alertname, slot, instance]
|
||||
group_wait: 30s
|
||||
group_interval: 1m
|
||||
repeat_interval: 5m
|
||||
|
||||
receivers:
|
||||
- name: knaster-healer
|
||||
webhook_configs:
|
||||
- url: http://router:8080/v1/healer/alertmanager
|
||||
send_resolved: true
|
||||
http_config:
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials_file: /etc/alertmanager/router_token
|
||||
@@ -0,0 +1,8 @@
|
||||
LLAMACPP_REF=b6000
|
||||
CUDA_VERSION=12.4.0
|
||||
CUDA_DOCKER_ARCH=52;61
|
||||
MODELS_DIR=/srv/models
|
||||
EMBED_GGUF=Qwen3-Embedding-0.6B-Q8_0.gguf
|
||||
EMBED_POOLING=last
|
||||
RERANK_GGUF=bge-reranker-v2-m3-Q8_0.gguf
|
||||
CHAT_GGUF=Qwen3-4B-Q4_K_M.gguf
|
||||
@@ -0,0 +1,86 @@
|
||||
# Rollen "gpu-embed" und "gpu-small" (VMs mit Quadro P2000 / M4000): llama.cpp-Server.
|
||||
# Maxwell (sm_52) und Pascal (sm_61) brauchen CUDA 12.x – daher eigener Build mit festen Architekturen.
|
||||
# Profile: embed (Embedding + Rerank), chat (Verifier/Signatur), z. B. `docker compose --profile embed up -d`
|
||||
name: knaster-gpu-llamacpp
|
||||
|
||||
x-llamacpp: &llamacpp
|
||||
build:
|
||||
context: https://github.com/ggml-org/llama.cpp.git#${LLAMACPP_REF:-b6000}
|
||||
dockerfile: .devops/cuda.Dockerfile
|
||||
target: server
|
||||
args:
|
||||
CUDA_VERSION: ${CUDA_VERSION:-12.4.0}
|
||||
CUDA_DOCKER_ARCH: ${CUDA_DOCKER_ARCH:-52;61}
|
||||
image: knaster/llamacpp-server:${LLAMACPP_REF:-b6000}-cu12
|
||||
restart: unless-stopped
|
||||
labels: {autoheal: "true"}
|
||||
volumes:
|
||||
- ${MODELS_DIR:-/srv/models}:/models:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices: [{driver: nvidia, count: 1, capabilities: [gpu]}]
|
||||
|
||||
services:
|
||||
llamacpp-embed:
|
||||
<<: *llamacpp
|
||||
container_name: llamacpp-embed
|
||||
profiles: ["embed"]
|
||||
command: ["-m", "/models/${EMBED_GGUF:-Qwen3-Embedding-0.6B-Q8_0.gguf}", "--embedding",
|
||||
"--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-ub", "8192", "-ngl", "99",
|
||||
"--alias", "embed-default", "--metrics", "--host", "0.0.0.0", "--port", "8081"]
|
||||
ports: ["8081:8081"]
|
||||
healthcheck: &hc
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
llamacpp-rerank:
|
||||
<<: *llamacpp
|
||||
container_name: llamacpp-rerank
|
||||
profiles: ["embed"]
|
||||
command: ["-m", "/models/${RERANK_GGUF:-bge-reranker-v2-m3-Q8_0.gguf}", "--reranking", "-c", "8192",
|
||||
"-ub", "8192", "-ngl", "99", "--alias", "rerank-default", "--metrics", "--host", "0.0.0.0",
|
||||
"--port", "8082"]
|
||||
ports: ["8082:8082"]
|
||||
healthcheck:
|
||||
<<: *hc
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8082/health"]
|
||||
|
||||
llamacpp-chat:
|
||||
<<: *llamacpp
|
||||
container_name: llamacpp-chat
|
||||
profiles: ["chat"]
|
||||
command: ["-m", "/models/${CHAT_GGUF:-Qwen3-4B-Q4_K_M.gguf}", "-c", "8192", "-ngl", "99",
|
||||
"--alias", "verifier-small", "--metrics", "--host", "0.0.0.0", "--port", "8080"]
|
||||
ports: ["8080:8080"]
|
||||
healthcheck:
|
||||
<<: *hc
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health"]
|
||||
|
||||
gpu-exporter:
|
||||
image: utkuozdemir/nvidia_gpu_exporter:1.2.1
|
||||
restart: unless-stopped
|
||||
ports: ["9835:9835"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}]
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.8.1
|
||||
restart: unless-stopped
|
||||
command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"]
|
||||
pid: host
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
- /var/lib/node_exporter/textfile:/textfile:ro
|
||||
ports: ["9100:9100"]
|
||||
|
||||
autoheal:
|
||||
image: willfarrell/autoheal:1.2.0
|
||||
restart: unless-stopped
|
||||
environment: {AUTOHEAL_CONTAINER_LABEL: autoheal}
|
||||
volumes: ["/var/run/docker.sock:/var/run/docker.sock"]
|
||||
@@ -0,0 +1,9 @@
|
||||
SLOT_ALIAS=tier1-general
|
||||
VLLM_MODEL=Qwen/Qwen3-14B-AWQ
|
||||
MAX_MODEL_LEN=32768
|
||||
GPU_MEM_UTIL=0.90
|
||||
MAX_LORAS=4
|
||||
MAX_LORA_RANK=64
|
||||
MODELS_DIR=/srv/models
|
||||
ADAPTERS_DIR=/srv/adapters
|
||||
HF_TOKEN=
|
||||
@@ -0,0 +1,62 @@
|
||||
# Rolle "gpu-main" / "gpu-flex" (VM mit durchgereichter RTX 4000 Ada): vLLM mit Multi-LoRA.
|
||||
name: knaster-gpu-vllm
|
||||
|
||||
services:
|
||||
vllm:
|
||||
image: vllm/vllm-openai:${VLLM_TAG:-v0.10.0} # Engine-Updates laufen als Lifecycle-Kandidat
|
||||
container_name: vllm
|
||||
restart: unless-stopped
|
||||
labels: {autoheal: "true"}
|
||||
ipc: host
|
||||
env_file: .env
|
||||
environment:
|
||||
VLLM_ALLOW_RUNTIME_LORA_UPDATING: "True" # Adapter per /v1/load_lora_adapter nachladen
|
||||
command:
|
||||
- --model=${VLLM_MODEL:?VLLM_MODEL fehlt}
|
||||
- --served-model-name=${SLOT_ALIAS:?SLOT_ALIAS fehlt}
|
||||
- --max-model-len=${MAX_MODEL_LEN:-32768}
|
||||
- --gpu-memory-utilization=${GPU_MEM_UTIL:-0.90}
|
||||
- --enable-lora
|
||||
- --max-loras=${MAX_LORAS:-4}
|
||||
- --max-lora-rank=${MAX_LORA_RANK:-64}
|
||||
- --host=0.0.0.0
|
||||
- --port=8000
|
||||
volumes:
|
||||
- ${MODELS_DIR:-/srv/models}:/root/.cache/huggingface
|
||||
- ${ADAPTERS_DIR:-/srv/adapters}:/adapters:ro
|
||||
ports: ["8000:8000"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices: [{driver: nvidia, count: 1, capabilities: [gpu]}]
|
||||
healthcheck:
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=4)"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 300s # Laden großer Modelle
|
||||
|
||||
gpu-exporter:
|
||||
image: utkuozdemir/nvidia_gpu_exporter:1.2.1
|
||||
restart: unless-stopped
|
||||
ports: ["9835:9835"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}]
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.8.1
|
||||
restart: unless-stopped
|
||||
command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"]
|
||||
pid: host
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
- /var/lib/node_exporter/textfile:/textfile:ro
|
||||
ports: ["9100:9100"]
|
||||
|
||||
autoheal:
|
||||
image: willfarrell/autoheal:1.2.0
|
||||
restart: unless-stopped
|
||||
environment: {AUTOHEAL_CONTAINER_LABEL: autoheal, AUTOHEAL_START_PERIOD: "300"}
|
||||
volumes: ["/var/run/docker.sock:/var/run/docker.sock"]
|
||||
@@ -0,0 +1,18 @@
|
||||
KNASTER_VERSION=dev
|
||||
KNASTER_SITE=knaster
|
||||
KNASTER_API_TOKEN=change-me
|
||||
KNASTER_DRY_RUN=true
|
||||
KNASTER_CANARY_INTERVAL=60
|
||||
POSTGRES_PASSWORD=change-me
|
||||
LITELLM_API_KEY=change-me
|
||||
LITELLM_MASTER_KEY=change-me
|
||||
OPENROUTER_API_KEY=
|
||||
RERANK_URL=http://gpu-embed-1:8082
|
||||
NOTIFY_WEBHOOK_URL=
|
||||
KNASTER_JOB_WEBHOOK_URL=
|
||||
PORTAINER_HOST=
|
||||
PORTAINER_KEY=
|
||||
PROXMOX_URL=
|
||||
PROXMOX_TOKEN_ID=
|
||||
PROXMOX_TOKEN_SECRET=
|
||||
KNASTER_WEBHOOK_AUTH=Bearer change-me
|
||||
@@ -0,0 +1,104 @@
|
||||
# Rolle "infra" (LXC/VM auf einem i7-Node): Router, LiteLLM, Postgres, Monitoring.
|
||||
# Start: cp .env.example .env && docker compose up -d
|
||||
name: knaster-infra
|
||||
|
||||
x-restart: &restart
|
||||
restart: unless-stopped
|
||||
labels: {autoheal: "true"}
|
||||
|
||||
services:
|
||||
router:
|
||||
<<: *restart
|
||||
build:
|
||||
context: ../../..
|
||||
dockerfile: deploy/docker/router.Dockerfile
|
||||
image: knaster/router:${KNASTER_VERSION:-dev}
|
||||
env_file: .env
|
||||
environment:
|
||||
KNASTER_DB_URL: postgresql+psycopg://knaster:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}@postgres:5432/knaster
|
||||
KNASTER_SITE_DIR: /app/sites/${KNASTER_SITE:-knaster}
|
||||
LITELLM_BASE_URL: http://litellm:4000
|
||||
ports: ["8080:8080"]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
litellm: {condition: service_started}
|
||||
|
||||
litellm:
|
||||
<<: *restart
|
||||
image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable} # vor Deployment auf feste Version pinnen
|
||||
command: ["--config", "/app/config.yaml", "--port", "4000"]
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ../../litellm/config.yaml:/app/config.yaml:ro
|
||||
ports: ["4000:4000"]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=4)"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
postgres:
|
||||
<<: *restart
|
||||
image: pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: knaster
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}
|
||||
POSTGRES_DB: knaster
|
||||
volumes:
|
||||
- postgres:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U knaster -d knaster"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
prometheus:
|
||||
<<: *restart
|
||||
image: prom/prometheus:v2.53.0
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ../../prometheus:/etc/prometheus:ro
|
||||
- prometheus:/prometheus
|
||||
ports: ["9090:9090"]
|
||||
|
||||
alertmanager:
|
||||
<<: *restart
|
||||
image: prom/alertmanager:v0.27.0
|
||||
command: ["--config.file=/etc/alertmanager/alertmanager.yml"]
|
||||
volumes:
|
||||
- ../../alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- ./secrets/router_token:/etc/alertmanager/router_token:ro
|
||||
ports: ["9093:9093"]
|
||||
|
||||
grafana:
|
||||
<<: *restart
|
||||
image: grafana/grafana:11.1.0
|
||||
profiles: ["dashboards"]
|
||||
volumes:
|
||||
- grafana:/var/lib/grafana
|
||||
ports: ["3000:3000"]
|
||||
|
||||
pve-exporter:
|
||||
<<: *restart
|
||||
image: prompve/prometheus-pve-exporter:${PVE_EXPORTER_TAG:-latest} # vor Deployment pinnen
|
||||
profiles: ["proxmox"]
|
||||
volumes:
|
||||
- ./secrets/pve.yml:/etc/prometheus/pve.yml:ro
|
||||
ports: ["9221:9221"]
|
||||
|
||||
autoheal:
|
||||
restart: unless-stopped
|
||||
image: willfarrell/autoheal:1.2.0
|
||||
environment:
|
||||
AUTOHEAL_CONTAINER_LABEL: autoheal
|
||||
AUTOHEAL_INTERVAL: "15"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
volumes:
|
||||
postgres: {}
|
||||
prometheus: {}
|
||||
grafana: {}
|
||||
@@ -0,0 +1,27 @@
|
||||
# Router-Service (knaster). Build-Kontext: Repo-Wurzel.
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
|
||||
WORKDIR /app
|
||||
|
||||
COPY core/pyproject.toml core/pyproject.toml
|
||||
COPY core/src core/src
|
||||
RUN pip install "./core[postgres]"
|
||||
|
||||
# Portable Konfiguration; standortspezifisches (sites/) kann per Volume überschrieben werden
|
||||
COPY config config
|
||||
COPY catalog catalog
|
||||
COPY skills skills
|
||||
COPY sites sites
|
||||
|
||||
ENV KNASTER_CONFIG_DIR=/app/config \
|
||||
KNASTER_SKILLS_DIR=/app/skills \
|
||||
KNASTER_SITE_DIR=/app/sites/knaster \
|
||||
KNASTER_DRY_RUN=true
|
||||
|
||||
RUN useradd --system --uid 10001 knaster
|
||||
USER knaster
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
|
||||
CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=4).status==200 else 1)"
|
||||
CMD ["knaster", "serve", "--host", "0.0.0.0", "--port", "8080"]
|
||||
@@ -0,0 +1,49 @@
|
||||
# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen.
|
||||
groups:
|
||||
- name: knaster-slots
|
||||
rules:
|
||||
- alert: SlotEndpointDown
|
||||
expr: knaster_slot_endpoint_up == 0
|
||||
for: 2m
|
||||
labels: {severity: high}
|
||||
annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"}
|
||||
|
||||
- alert: SlotCanaryFailing
|
||||
expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"}
|
||||
|
||||
- alert: VllmKvCacheSaturated
|
||||
expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"}
|
||||
|
||||
- name: knaster-gpu
|
||||
rules:
|
||||
- alert: GpuXidError
|
||||
# aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector)
|
||||
expr: increase(knaster_gpu_xid_errors_total[10m]) > 0
|
||||
labels: {severity: high}
|
||||
annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"}
|
||||
|
||||
- alert: GpuTemperatureHigh
|
||||
expr: nvidia_smi_temperature_gpu > 85
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"}
|
||||
|
||||
- name: knaster-nodes
|
||||
rules:
|
||||
- alert: NodeDown
|
||||
expr: up{job="node"} == 0
|
||||
for: 3m
|
||||
labels: {severity: high}
|
||||
annotations: {summary: "{{ $labels.instance }} nicht erreichbar"}
|
||||
|
||||
- alert: DiskAlmostFull
|
||||
expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10
|
||||
for: 15m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}
|
||||
@@ -0,0 +1,67 @@
|
||||
# Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den
|
||||
# Alerts und damit beim Healer. Hostnamen = DNS-Namen der VMs (ANNAHME bis Deployment).
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
rule_files: [alerts.yml]
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs: [{targets: ["alertmanager:9093"]}]
|
||||
|
||||
scrape_configs:
|
||||
- job_name: knaster-router
|
||||
metrics_path: /metrics
|
||||
static_configs: [{targets: ["router:8080"]}]
|
||||
|
||||
- job_name: vllm
|
||||
static_configs:
|
||||
- targets: ["gpu-main:8000"]
|
||||
labels: {slot: tier1-general, docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"}
|
||||
- targets: ["gpu-flex:8000"]
|
||||
labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}
|
||||
|
||||
- job_name: llamacpp
|
||||
static_configs:
|
||||
- targets: ["gpu-embed-1:8081"]
|
||||
labels: {slot: embed-default, docker_host: gpu-embed-1, container: llamacpp-embed, pve_node: pve3, vmid: "221"}
|
||||
- targets: ["gpu-embed-1:8082"]
|
||||
labels: {slot: rerank-default, docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"}
|
||||
- targets: ["gpu-small:8080"]
|
||||
labels: {slot: verifier-small, docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"}
|
||||
|
||||
- job_name: gpu
|
||||
static_configs:
|
||||
- targets: ["gpu-main:9835"]
|
||||
labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
|
||||
- targets: ["gpu-flex:9835"]
|
||||
labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
|
||||
- targets: ["gpu-embed-1:9835"]
|
||||
labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
|
||||
- targets: ["gpu-embed-2:9835"]
|
||||
labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
|
||||
- targets: ["gpu-small:9835"]
|
||||
labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
|
||||
|
||||
- job_name: node # Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
|
||||
static_configs:
|
||||
- targets: ["gpu-main:9100"]
|
||||
labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
|
||||
- targets: ["gpu-flex:9100"]
|
||||
labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
|
||||
- targets: ["gpu-embed-1:9100"]
|
||||
labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
|
||||
- targets: ["gpu-embed-2:9100"]
|
||||
labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
|
||||
- targets: ["gpu-small:9100"]
|
||||
labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
|
||||
|
||||
- job_name: proxmox
|
||||
metrics_path: /pve
|
||||
params: {module: [default]}
|
||||
static_configs: [{targets: ["pve1", "pve2", "pve3", "pve4", "pve-ada"]}]
|
||||
relabel_configs:
|
||||
- {source_labels: [__address__], target_label: __param_target}
|
||||
- {source_labels: [__param_target], target_label: instance}
|
||||
- {target_label: __address__, replacement: "pve-exporter:9221"}
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# Zählt NVIDIA-Xid-Fehler aus dem Kernel-Log und schreibt sie für den node_exporter (textfile collector).
|
||||
# Aufruf per systemd-Timer jede Minute (siehe platform/proxmox/ansible/roles/gpu-node).
|
||||
set -euo pipefail
|
||||
OUT_DIR="${1:-/var/lib/node_exporter/textfile}"
|
||||
TMP="$(mktemp "${OUT_DIR}/.xid.XXXXXX")"
|
||||
{
|
||||
echo "# HELP knaster_gpu_xid_errors_total NVIDIA Xid-Fehler seit Boot"
|
||||
echo "# TYPE knaster_gpu_xid_errors_total counter"
|
||||
journalctl -k -b --no-pager -o cat 2>/dev/null \
|
||||
| grep -oE 'NVRM: Xid \([^)]*\): [0-9]+' \
|
||||
| awk '{print $NF}' | sort | uniq -c \
|
||||
| awk '{printf "knaster_gpu_xid_errors_total{xid=\"%s\"} %d\n", $2, $1}'
|
||||
} > "$TMP"
|
||||
chmod 0644 "$TMP"
|
||||
mv "$TMP" "${OUT_DIR}/gpu_xid.prom"
|
||||
Reference in New Issue
Block a user