diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml index e7e38e4..4fbae03 100644 --- a/deploy/prometheus/prometheus.yml +++ b/deploy/prometheus/prometheus.yml @@ -1,6 +1,6 @@ # Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den # Alerts und damit beim Healer. -# GPU-VMs (job node, gpu): file_sd, weil die DHCP-Adressen wechseln (D5). Die Dateien erzeugt +# GPU-VMs (job node, gpu, llamacpp): file_sd, weil die DHCP-Adressen wechseln (D5). Die Dateien erzeugt # platform/proxmox/scripts/monitoring-targets.py auf ops aus `tofu output -json gpu_vms` und der Registry; # die Ansible-Rolle infra-stack kopiert sie nach /opt/knaster/infra/file_sd (im Container /etc/knaster-sd). global: @@ -32,15 +32,10 @@ scrape_configs: - targets: ["gpu-flex:8000"] labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"} - # llama.cpp-Slots kommen erst mit Phase 6 in die GPU-VMs; bis dahin nicht erreichbar (Platzhalter) + # llama-server (--metrics) in den GPU-VMs (Task 6.1): Embedding 8081 und Rerank 8082 in gpu-pve44/46, + # Chat 8080 in gpu-pve48. Ziele aus monitoring-targets.py (Profile laut Inventar-Gruppe gpu_llamacpp). - job_name: llamacpp - static_configs: - - targets: ["gpu-embed-1:8081"] - labels: {slot: embed-default, docker_host: gpu-embed-1, container: llamacpp-embed, pve_node: pve3, vmid: "221"} - - targets: ["gpu-embed-1:8082"] - labels: {slot: rerank-default, docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"} - - targets: ["gpu-small:8080"] - labels: {slot: verifier-small, docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"} + file_sd_configs: [{files: [/etc/knaster-sd/llamacpp.json]}] # nvidia_gpu_exporter (nvidia-smi, auch für Maxwell/Pascal; DCGM kann Maxwell nicht) in jeder GPU-VM - job_name: gpu diff --git a/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml b/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml index 57f3a53..449c652 100644 --- a/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml +++ b/platform/proxmox/ansible/roles/infra-stack/tasks/targets.yml @@ -1,15 +1,19 @@ -# file_sd-Ziele der GPU-VMs. Auch einzeln aufrufbar: playbooks/monitoring-targets.yml +# file_sd-Ziele der GPU-VMs (node, gpu, llamacpp). Auch einzeln aufrufbar: playbooks/monitoring-targets.yml # (nach refresh-vm-access.sh, wenn sich DHCP-Adressen geändert haben). Prometheus liest die Dateien selbst neu. - name: file_sd-Ziele aus tofu output erzeugen (auf ops) ansible.builtin.shell: | set -euo pipefail tofu output -json gpu_vms | python3 {{ infra_repo_src }}/platform/proxmox/scripts/monitoring-targets.py \ - --registry {{ infra_repo_src }}/sites/{{ knaster_site }}/registry.yaml --out {{ infra_targets_dir }} + --registry {{ infra_repo_src }}/sites/{{ knaster_site }}/registry.yaml --out {{ infra_targets_dir }} \ + --llamacpp '{{ dict(_llamacpp_hosts | zip(_llamacpp_hosts | map('extract', hostvars, 'llamacpp_profiles'))) | to_json }}' args: chdir: "{{ infra_tofu_dir }}" executable: /bin/bash delegate_to: localhost run_once: true + vars: + # Profile je VM aus dem Inventar (Gruppe gpu_llamacpp, Task 6.1) + _llamacpp_hosts: "{{ groups['gpu_llamacpp'] | default([]) }}" changed_when: false - name: file_sd-Verzeichnis @@ -23,4 +27,4 @@ src: "{{ infra_targets_dir }}/{{ item }}.json" dest: "{{ infra_state_dir }}/file_sd/{{ item }}.json" mode: "0644" - loop: [node, gpu] + loop: [node, gpu, llamacpp] diff --git a/platform/proxmox/ansible/roles/pve-cluster/tasks/main.yml b/platform/proxmox/ansible/roles/pve-cluster/tasks/main.yml index dd9af76..f8ea4dd 100644 --- a/platform/proxmox/ansible/roles/pve-cluster/tasks/main.yml +++ b/platform/proxmox/ansible/roles/pve-cluster/tasks/main.yml @@ -38,7 +38,7 @@ - name: Admin-Gruppe darf alles (Administrator auf /) ansible.builtin.command: pveum acl modify / --groups {{ knaster_pve.access.admin_group }} --roles Administrator - when: ['/', '@' ~ knaster_pve.access.admin_group, 'Administrator'] not in pve_access.acl + when: "['/', knaster_pve.access.admin_group, 'Administrator'] not in pve_access.acl" changed_when: true # Ohne Passwort: das setzt scripts/pve-admin-user.sh vom Laptop (zusammen mit TOTP und Recovery-Keys) @@ -57,7 +57,7 @@ - name: Healer darf GPU-VMs im Pool gpu schalten ansible.builtin.command: pveum acl modify /pool/gpu --users {{ pve_healer_user }} --roles {{ pve_healer_role }} - when: ['/pool/gpu', pve_healer_user, pve_healer_role] not in pve_access.acl + when: "['/pool/gpu', pve_healer_user, pve_healer_role] not in pve_access.acl" changed_when: true - name: Healer-Token lesen diff --git a/platform/proxmox/scripts/monitoring-targets.py b/platform/proxmox/scripts/monitoring-targets.py index 9c4352c..f415e0f 100644 --- a/platform/proxmox/scripts/monitoring-targets.py +++ b/platform/proxmox/scripts/monitoring-targets.py @@ -6,11 +6,18 @@ Eingabe ist `tofu output -json gpu_vms` (stdin), dazu die Registry für die Slot tofu output -json gpu_vms | python3 monitoring-targets.py --registry sites/knaster/registry.yaml --out DIR -Schreibt DIR/node.json (node_exporter, Port 9100) und DIR/gpu.json (nvidia_gpu_exporter, Port 9835). +Schreibt DIR/node.json (node_exporter, Port 9100), DIR/gpu.json (nvidia_gpu_exporter, Port 9835) und +DIR/llamacpp.json (llama-server /metrics je Dienst, Task 6.1). Labels je Ziel: instance (VM-Name, bleibt bei IP-Wechsel gleich), vm, pve_node, vmid, slot, slots. `slot` ist der erste Slot der Registry mit dieser vmid (Reihenfolge der Registry), `slots` alle, kommagetrennt. VMs ohne Slot bekommen slot="none" (der Healer markiert dann keinen echten Slot). + +llama.cpp: Welche VM welches Profil fährt, steht im Ansible-Inventar (Gruppe gpu_llamacpp), nicht in der +Registry (die kennt je Slot nur eine VM, die zweite Embedding-Instanz fehlt dort). Übergabe als JSON: +`--llamacpp '{"gpu-pve44": ["embed"], ...}'`. Die Slots eines Dienstes kommen aus der Registry +(engine llamacpp, gleicher Container). Labels zusätzlich docker_host (= VM) und container, damit der +Healer die tatsächliche VM trifft. Diese Slots zählen auch für node/gpu der VM. """ from __future__ import annotations @@ -23,6 +30,11 @@ from pathlib import Path import yaml PORTS = {"node": 9100, "gpu": 9835} +# Dienste je Profil wie in deploy/compose/gpu-llamacpp (feste Ports, Host-Netz) +LLAMACPP_PROFILES = { + "embed": [("llamacpp-embed", 8081), ("llamacpp-rerank", 8082)], + "chat": [("llamacpp-chat", 8080)], +} def slots_by_vmid(registry: dict) -> dict[str, list[str]]: @@ -34,16 +46,31 @@ def slots_by_vmid(registry: dict) -> dict[str, list[str]]: return out -def build_targets(gpu_vms: dict, registry: dict) -> dict[str, list[dict]]: +def llamacpp_slots_by_container(registry: dict) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + for name, slot in (registry.get("slots") or {}).items(): + container = (slot.get("ops") or {}).get("container") + if slot.get("engine") == "llamacpp" and container: + out.setdefault(container, []).append(name) + return out + + +def build_targets(gpu_vms: dict, registry: dict, + llamacpp: dict[str, list[str]] | None = None) -> dict[str, list[dict]]: by_vmid = slots_by_vmid(registry) - result: dict[str, list[dict]] = {job: [] for job in PORTS} + by_container = llamacpp_slots_by_container(registry) + llamacpp = llamacpp or {} + result: dict[str, list[dict]] = {job: [] for job in [*PORTS, "llamacpp"]} for name in sorted(gpu_vms): vm = gpu_vms[name] if not vm.get("ipv4"): print(f"Warnung: {name} hat keine IPv4 (Gast-Agent?), übersprungen", file=sys.stderr) continue vmid = str(vm["vmid"]) - slots = by_vmid.get(vmid, []) + services = [svc for profile in llamacpp.get(name, []) for svc in LLAMACPP_PROFILES[profile]] + slots = list(by_vmid.get(vmid, [])) + for container, _ in services: + slots += [s for s in by_container.get(container, []) if s not in slots] labels = { "instance": name, "vm": name, @@ -54,6 +81,11 @@ def build_targets(gpu_vms: dict, registry: dict) -> dict[str, list[dict]]: } for job, port in PORTS.items(): result[job].append({"targets": [f"{vm['ipv4']}:{port}"], "labels": labels}) + for container, port in services: + cslots = by_container.get(container, []) + result["llamacpp"].append({"targets": [f"{vm['ipv4']}:{port}"], "labels": { + **labels, "docker_host": name, "container": container, + "slot": cslots[0] if cslots else "none", "slots": ",".join(cslots)}}) return result @@ -61,11 +93,16 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--registry", required=True, type=Path) ap.add_argument("--out", required=True, type=Path) + ap.add_argument("--llamacpp", default="{}", help='JSON {"": ["embed"|"chat", ...]} aus dem Inventar') args = ap.parse_args() gpu_vms = json.load(sys.stdin) registry = yaml.safe_load(args.registry.read_text(encoding="utf-8")) - targets = build_targets(gpu_vms, registry) + llamacpp = json.loads(args.llamacpp) + unknown = {p for profiles in llamacpp.values() for p in profiles} - set(LLAMACPP_PROFILES) + if unknown: + ap.error(f"unbekannte llama.cpp-Profile: {sorted(unknown)}") + targets = build_targets(gpu_vms, registry, llamacpp) args.out.mkdir(parents=True, exist_ok=True) for job, entries in targets.items(): path = args.out / f"{job}.json"