- deploy/compose/infra: Dienst litellm entfernt (D4), Router auf LITELLM_BASE_URL (appsrv08, Port noch offen), alle Images gepinnt, Grafana und pve-exporter ohne Profile, node-exporter für den LXC, Zustand über KNASTER_STATE_DIR - Prometheus: GPU-VMs per file_sd (node, gpu), Proxmox-Knoten per IP, infra-node - Grafana: Datasource und Dashboards provisioniert (grafana.com 10347, 1860, 14574 sowie knaster-Übersicht) - Alertmanager: n8n-Receiver nur vorbereitet (auskommentiert) - Ansible: Rollen infra-stack und gpu-exporters, Playbooks monitoring.yml und monitoring-targets.yml, Skript monitoring-targets.py (tofu output → file_sd) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Robert Rapp
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
fd48fd1c7f
commit
bf7c137adf
@@ -0,0 +1,10 @@
|
||||
# file_sd-Ziele der GPU-VMs auffrischen, wenn sich DHCP-Adressen geändert haben. Auf ops:
|
||||
# cd /root/cluster-knaster/platform/proxmox/tofu && . /root/.config/knaster/tofu.env && bash ../scripts/refresh-vm-access.sh
|
||||
# cd ../ansible && ansible-playbook playbooks/monitoring-targets.yml
|
||||
- name: Infra-LXC – file_sd-Ziele
|
||||
hosts: infra
|
||||
tasks:
|
||||
- name: Ziele erzeugen und kopieren
|
||||
ansible.builtin.include_role:
|
||||
name: infra-stack
|
||||
tasks_from: targets
|
||||
@@ -0,0 +1,10 @@
|
||||
# Nur Monitoring (Task 5.2), ohne Treiber-/Docker-Rollen der GPU-VMs (kein Docker-Neustart, kein Reboot).
|
||||
# Aufruf auf ops (Verzeichnis platform/proxmox/ansible), vorher Adressen auffrischen (refresh-vm-access.sh):
|
||||
# ansible-playbook playbooks/monitoring.yml
|
||||
- name: GPU-VMs – Exporter (node_exporter, nvidia_gpu_exporter)
|
||||
hosts: gpu_vms
|
||||
roles: [gpu-exporters]
|
||||
|
||||
- name: Infra-LXC – Compose-Stack mit Monitoring
|
||||
hosts: infra
|
||||
roles: [infra-stack]
|
||||
@@ -19,8 +19,9 @@
|
||||
eth0:
|
||||
dhcp-identifier: mac
|
||||
mode: "0600"
|
||||
roles: [docker-host, gpu-node]
|
||||
roles: [docker-host, gpu-node, gpu-exporters]
|
||||
|
||||
- name: Infra-LXC (Docker)
|
||||
# Nur Monitoring ohne Treiber/Neustart der GPU-VMs: playbooks/monitoring.yml
|
||||
- name: Infra-LXC (Docker, Compose-Stack mit Monitoring)
|
||||
hosts: infra
|
||||
roles: [docker-host]
|
||||
roles: [docker-host, infra-stack]
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Monitoring-Exporter in den GPU-VMs (Task 5.2). Ziele erzeugt monitoring-targets.py als file_sd.
|
||||
# node_exporter aus Ubuntu (24.04: 1.7.x), liest die Xid-Zähler aus dem Textfile-Verzeichnis (Rolle gpu-node)
|
||||
gpu_exporters_textfile_dir: /var/lib/node_exporter/textfile
|
||||
# nvidia_gpu_exporter (nvidia-smi statt DCGM/NVML: läuft auch mit Maxwell/Pascal), Port 9835
|
||||
nvidia_gpu_exporter_version: "1.15.1"
|
||||
# sha256 aus checksums.txt des Releases v1.15.1 (am 12.10.2026 gegen die Datei geprüft)
|
||||
nvidia_gpu_exporter_sha256: 0dc8c2756c6853ec7e6a7c404b229d8f764443bef89f33e13b3c4278b309acd4
|
||||
# ufw: nur eigene allow-Regeln hinzufügen (Kommentar knaster-monitoring), ufw nie ein- oder ausschalten
|
||||
gpu_exporters_allow_from: 192.168.100.0/24
|
||||
gpu_exporters_ports: [9100, 9835]
|
||||
@@ -0,0 +1,4 @@
|
||||
- name: restart node-exporter
|
||||
ansible.builtin.service:
|
||||
name: prometheus-node-exporter
|
||||
state: restarted
|
||||
@@ -0,0 +1,60 @@
|
||||
# Bewusst getrennt von gpu-node: kein Treiber, kein Docker-Neustart, kein Reboot.
|
||||
- name: node_exporter installieren (ohne Zusatz-Collectors)
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-node-exporter
|
||||
state: present
|
||||
install_recommends: false
|
||||
update_cache: true
|
||||
cache_valid_time: 3600
|
||||
|
||||
- name: Textfile-Verzeichnis (Xid-Zähler aus gpu-xid-textfile.sh)
|
||||
ansible.builtin.file:
|
||||
path: "{{ gpu_exporters_textfile_dir }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: node_exporter liest das Textfile-Verzeichnis
|
||||
ansible.builtin.lineinfile:
|
||||
path: /etc/default/prometheus-node-exporter
|
||||
regexp: '^ARGS='
|
||||
line: 'ARGS="--collector.textfile.directory={{ gpu_exporters_textfile_dir }}"'
|
||||
notify: restart node-exporter
|
||||
|
||||
- name: node_exporter aktiv
|
||||
ansible.builtin.service:
|
||||
name: prometheus-node-exporter
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: nvidia_gpu_exporter herunterladen (Prüfsumme fest)
|
||||
ansible.builtin.get_url:
|
||||
url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ nvidia_gpu_exporter_version }}/nvidia-gpu-exporter_{{ nvidia_gpu_exporter_version }}_linux_amd64.deb"
|
||||
dest: "/var/cache/apt/archives/nvidia-gpu-exporter_{{ nvidia_gpu_exporter_version }}_linux_amd64.deb"
|
||||
checksum: "sha256:{{ nvidia_gpu_exporter_sha256 }}"
|
||||
mode: "0644"
|
||||
|
||||
- name: nvidia_gpu_exporter installieren
|
||||
ansible.builtin.apt:
|
||||
deb: "/var/cache/apt/archives/nvidia-gpu-exporter_{{ nvidia_gpu_exporter_version }}_linux_amd64.deb"
|
||||
|
||||
- name: nvidia_gpu_exporter aktiv
|
||||
ansible.builtin.service:
|
||||
name: nvidia_gpu_exporter
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: Ist ufw vorhanden?
|
||||
ansible.builtin.stat:
|
||||
path: /usr/sbin/ufw
|
||||
register: ufw_bin
|
||||
|
||||
# Regeln wirken auch bei inaktivem ufw (werden vorgemerkt). Fremde Regeln bleiben unberührt.
|
||||
- name: ufw – Exporter-Ports aus dem LAN erlauben
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
src: "{{ gpu_exporters_allow_from }}"
|
||||
port: "{{ item | string }}"
|
||||
proto: tcp
|
||||
comment: knaster-monitoring
|
||||
loop: "{{ gpu_exporters_ports }}"
|
||||
when: ufw_bin.stat.exists
|
||||
@@ -0,0 +1,29 @@
|
||||
# Compose-Stack im LXC infra (Task 5.2). Repo-Auszug und Zustand liegen getrennt:
|
||||
# infra_repo_dir Kopie der nötigen Repo-Teile von ops (wird bei jedem Lauf abgeglichen, --delete)
|
||||
# infra_state_dir .env (0600, Geheimnisse), secrets/, file_sd/ – nie im Repo, nie überschrieben
|
||||
infra_repo_src: "{{ playbook_dir }}/../../../.."
|
||||
infra_repo_dir: /opt/knaster/repo
|
||||
infra_state_dir: /opt/knaster/infra
|
||||
infra_compose_project: knaster-infra
|
||||
infra_compose_file: "{{ infra_repo_dir }}/deploy/compose/infra/docker-compose.yml"
|
||||
|
||||
# Einstellungen in .env (keine Geheimnisse). Änderungen hier, nicht im Container.
|
||||
infra_settings:
|
||||
KNASTER_VERSION: dev
|
||||
KNASTER_SITE: "{{ knaster_site }}"
|
||||
KNASTER_STATE_DIR: "{{ infra_state_dir }}"
|
||||
KNASTER_DRY_RUN: "true" # Global Constraint: erst nach einer Woche Beobachtung einzeln freischalten
|
||||
KNASTER_CANARY_INTERVAL: "60"
|
||||
# D4: LiteLLM mit PSE auf appsrv08. TODO (Nutzer, Task 6.3): Port ist dort noch nicht im LAN veröffentlicht.
|
||||
LITELLM_BASE_URL: http://192.168.100.29:4000
|
||||
|
||||
# Geheimnisse: werden im Container einmalig mit `openssl rand -hex 24` erzeugt und danach nie geändert
|
||||
# (Postgres- und Grafana-Admin-Passwort wirken nur bei der Erstinitialisierung).
|
||||
infra_secrets: [POSTGRES_PASSWORD, KNASTER_API_TOKEN, GRAFANA_ADMIN_PASSWORD, GRAFANA_KIOSK_PASSWORD]
|
||||
|
||||
# Lese-Token monitoring@pve!prom (PVEAuditor) auf ops
|
||||
infra_pve_exporter_env: /root/.config/knaster/pve-exporter.env
|
||||
|
||||
# file_sd: auf ops aus `tofu output -json gpu_vms` erzeugt (nicht im Repo)
|
||||
infra_tofu_dir: "{{ playbook_dir }}/../../tofu"
|
||||
infra_targets_dir: "{{ playbook_dir }}/../monitoring-targets.generated"
|
||||
@@ -0,0 +1,134 @@
|
||||
# Compose-Stack infra: Router, TTS, Postgres, Prometheus, Alertmanager, Grafana, pve-exporter, node-exporter.
|
||||
# Idempotent: Geheimnisse nur anlegen, wenn sie fehlen; compose up ändert nur, was sich geändert hat.
|
||||
- name: Werkzeuge
|
||||
ansible.builtin.apt:
|
||||
name: [rsync, openssl, curl]
|
||||
state: present
|
||||
update_cache: true
|
||||
cache_valid_time: 3600
|
||||
|
||||
- name: Verzeichnisse
|
||||
ansible.builtin.file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- {path: /opt/knaster, mode: "0755"}
|
||||
- {path: "{{ infra_repo_dir }}", mode: "0755"}
|
||||
- {path: "{{ infra_state_dir }}", mode: "0700"}
|
||||
- {path: "{{ infra_state_dir }}/secrets", mode: "0700"}
|
||||
|
||||
# Nur die Teile, die Build und Konfiguration brauchen. Nie .git, .venv, tofu-State oder .env-Dateien.
|
||||
- name: Repo-Auszug von ops abgleichen
|
||||
ansible.posix.synchronize:
|
||||
src: "{{ infra_repo_src }}/"
|
||||
dest: "{{ infra_repo_dir }}/"
|
||||
delete: true
|
||||
rsync_opts:
|
||||
- "--exclude=__pycache__"
|
||||
- "--exclude=.pytest_cache"
|
||||
- "--exclude=*.egg-info"
|
||||
- "--exclude=.env"
|
||||
- "--exclude=secrets/"
|
||||
- "--include=/core/***"
|
||||
- "--include=/config/***"
|
||||
- "--include=/catalog/***"
|
||||
- "--include=/skills/***"
|
||||
- "--include=/sites/***"
|
||||
- "--include=/deploy/***"
|
||||
- "--exclude=*"
|
||||
|
||||
- name: Einstellungen in .env
|
||||
ansible.builtin.lineinfile:
|
||||
path: "{{ infra_state_dir }}/.env"
|
||||
regexp: "^{{ item.key }}="
|
||||
line: "{{ item.key }}={{ item.value }}"
|
||||
create: true
|
||||
mode: "0600"
|
||||
loop: "{{ infra_settings | dict2items }}"
|
||||
|
||||
# Gibt nur Namen aus, nie Werte
|
||||
- name: Fehlende Geheimnisse im Container erzeugen
|
||||
ansible.builtin.shell: |
|
||||
set -eu
|
||||
umask 077
|
||||
f={{ infra_state_dir }}/.env
|
||||
for k in {{ infra_secrets | join(' ') }}; do
|
||||
grep -q "^$k=" "$f" || { echo "$k=$(openssl rand -hex 24)" >> "$f"; echo "neu: $k"; }
|
||||
done
|
||||
chmod 0600 "$f"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: secrets_gen
|
||||
changed_when: "'neu:' in secrets_gen.stdout"
|
||||
|
||||
- name: Router-Token für Alertmanager (= KNASTER_API_TOKEN, lesbar für nobody im Container)
|
||||
ansible.builtin.shell: |
|
||||
set -eu
|
||||
umask 077
|
||||
cd {{ infra_state_dir }}
|
||||
v=$(sed -n 's/^KNASTER_API_TOKEN=//p' .env)
|
||||
t=secrets/router_token
|
||||
if [ ! -f "$t" ] || [ "$(cat "$t")" != "$v" ]; then printf '%s' "$v" > "$t"; echo geändert; fi
|
||||
chown 65534:65534 "$t"
|
||||
chmod 0400 "$t"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: router_token
|
||||
changed_when: "'geändert' in router_token.stdout"
|
||||
no_log: true
|
||||
|
||||
- name: Proxmox-Lese-Token für den pve-exporter
|
||||
ansible.builtin.copy:
|
||||
src: "{{ infra_pve_exporter_env }}"
|
||||
dest: "{{ infra_state_dir }}/secrets/pve-exporter.env"
|
||||
mode: "0600"
|
||||
no_log: true
|
||||
|
||||
- name: file_sd-Ziele
|
||||
ansible.builtin.include_tasks: targets.yml
|
||||
|
||||
- name: Stack bauen und starten
|
||||
ansible.builtin.command: >
|
||||
docker compose -p {{ infra_compose_project }} --env-file {{ infra_state_dir }}/.env
|
||||
-f {{ infra_compose_file }} up -d --build --remove-orphans
|
||||
register: compose_up
|
||||
changed_when: compose_up.stderr is search('Container \S+\s+(Created|Recreated|Started)')
|
||||
|
||||
- name: Prometheus und Alertmanager laden geänderte Konfiguration (bind-mount, kein Neustart nötig)
|
||||
ansible.builtin.command: "curl -fsS -m 10 -X POST http://127.0.0.1:{{ item }}/-/reload"
|
||||
loop: [9090, 9093]
|
||||
register: reload
|
||||
retries: 15
|
||||
delay: 4
|
||||
until: reload.rc == 0
|
||||
changed_when: false
|
||||
|
||||
- name: Grafana-Konto kiosk (Viewer) anlegen, Passwort aus .env
|
||||
ansible.builtin.shell: |
|
||||
set -eu
|
||||
cd {{ infra_state_dir }}
|
||||
A=$(sed -n 's/^GRAFANA_ADMIN_PASSWORD=//p' .env)
|
||||
K=$(sed -n 's/^GRAFANA_KIOSK_PASSWORD=//p' .env)
|
||||
umask 077
|
||||
cfg=$(mktemp)
|
||||
trap 'rm -f "$cfg"' EXIT
|
||||
printf 'user = "admin:%s"\n' "$A" > "$cfg" # Passwort nicht in der Prozessliste
|
||||
g() { curl -sS -K "$cfg" -H 'Content-Type: application/json' "$@"; }
|
||||
for i in $(seq 60); do curl -fsS -m 3 http://127.0.0.1:3000/api/health >/dev/null 2>&1 && break; sleep 2; done
|
||||
code=$(g -o /dev/null -w '%{http_code}' 'http://127.0.0.1:3000/api/users/lookup?loginOrEmail=kiosk')
|
||||
if [ "$code" = 404 ]; then
|
||||
printf '{"name":"Kiosk","login":"kiosk","password":"%s"}' "$K" \
|
||||
| g -f -X POST --data @- http://127.0.0.1:3000/api/admin/users >/dev/null
|
||||
echo angelegt
|
||||
elif [ "$code" != 200 ]; then
|
||||
echo "Grafana-API antwortet $code" >&2
|
||||
exit 1
|
||||
fi
|
||||
id=$(g -f 'http://127.0.0.1:3000/api/users/lookup?loginOrEmail=kiosk' | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])')
|
||||
g -f -X PATCH --data '{"role":"Viewer"}' "http://127.0.0.1:3000/api/org/users/$id" >/dev/null
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: kiosk
|
||||
changed_when: "'angelegt' in kiosk.stdout"
|
||||
no_log: true
|
||||
@@ -0,0 +1,26 @@
|
||||
# file_sd-Ziele der GPU-VMs. Auch einzeln aufrufbar: playbooks/monitoring-targets.yml
|
||||
# (nach refresh-vm-access.sh, wenn sich DHCP-Adressen geändert haben). Prometheus liest die Dateien selbst neu.
|
||||
- name: file_sd-Ziele aus tofu output erzeugen (auf ops)
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
tofu output -json gpu_vms | python3 {{ infra_repo_src }}/platform/proxmox/scripts/monitoring-targets.py \
|
||||
--registry {{ infra_repo_src }}/sites/{{ knaster_site }}/registry.yaml --out {{ infra_targets_dir }}
|
||||
args:
|
||||
chdir: "{{ infra_tofu_dir }}"
|
||||
executable: /bin/bash
|
||||
delegate_to: localhost
|
||||
run_once: true
|
||||
changed_when: false
|
||||
|
||||
- name: file_sd-Verzeichnis
|
||||
ansible.builtin.file:
|
||||
path: "{{ infra_state_dir }}/file_sd"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: file_sd-Ziele in den Container kopieren
|
||||
ansible.builtin.copy:
|
||||
src: "{{ infra_targets_dir }}/{{ item }}.json"
|
||||
dest: "{{ infra_state_dir }}/file_sd/{{ item }}.json"
|
||||
mode: "0644"
|
||||
loop: [node, gpu]
|
||||
Reference in New Issue
Block a user