From 5c790cc8d4fbce32d50ab8ec6fbb34f0de7193c1 Mon Sep 17 00:00:00 2001 From: Robert Rapp Date: Mon, 12 Oct 2026 21:28:33 +0200 Subject: [PATCH] gpu-llamacpp: Compose auf P2000 per device_ids, Rolle und Playbook (Task 6.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Build-Pin wie im Benchmark (b6000 = 4762ad7, CUDA 12.4, sm_52;61), Host-Netz, Log-Rotation, /slots und Web-Oberfläche aus, Qwen3 ohne Denkphase. - Exporter aus dem Compose entfernt (laufen nativ, Rolle gpu-exporters). - Rolle gpu-llamacpp: Karten-UUID über den Namen, Modelle mit Prüfsumme, ufw-Regeln nur vorgemerkt. Inventargruppe gpu_llamacpp (ohne gpu-pve52). Co-Authored-By: Claude Opus 5.5 --- deploy/compose/gpu-llamacpp/.env.example | 6 + .../compose/gpu-llamacpp/docker-compose.yml | 91 ++++++++------ platform/proxmox/ansible/inventory.yml | 6 + .../ansible/playbooks/gpu-llamacpp.yml | 8 ++ .../roles/gpu-llamacpp/defaults/main.yml | 50 ++++++++ .../ansible/roles/gpu-llamacpp/tasks/main.yml | 111 ++++++++++++++++++ 6 files changed, 233 insertions(+), 39 deletions(-) create mode 100644 platform/proxmox/ansible/playbooks/gpu-llamacpp.yml create mode 100644 platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml create mode 100644 platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml diff --git a/deploy/compose/gpu-llamacpp/.env.example b/deploy/compose/gpu-llamacpp/.env.example index f9f138d..b10a281 100644 --- a/deploy/compose/gpu-llamacpp/.env.example +++ b/deploy/compose/gpu-llamacpp/.env.example @@ -1,4 +1,6 @@ +COMPOSE_PROFILES=embed LLAMACPP_REF=b6000 +LLAMACPP_COMMIT=4762ad7316dcdec20016ab5985fb46a27902204d CUDA_VERSION=12.4.0 CUDA_DOCKER_ARCH=52;61 MODELS_DIR=/srv/models @@ -6,3 +8,7 @@ EMBED_GGUF=Qwen3-Embedding-0.6B-Q8_0.gguf EMBED_POOLING=last RERANK_GGUF=bge-reranker-v2-m3-Q8_0.gguf CHAT_GGUF=Qwen3-4B-Q4_K_M.gguf +# Karte je Dienst: Index oder UUID aus `nvidia-smi -L` (Rolle gpu-llamacpp setzt die UUID der P2000) +EMBED_GPU=1 +RERANK_GPU=1 +CHAT_GPU=1 diff --git a/deploy/compose/gpu-llamacpp/docker-compose.yml b/deploy/compose/gpu-llamacpp/docker-compose.yml index 61fbc1c..d827688 100644 --- a/deploy/compose/gpu-llamacpp/docker-compose.yml +++ b/deploy/compose/gpu-llamacpp/docker-compose.yml @@ -1,25 +1,37 @@ -# Rollen "gpu-embed" und "gpu-small" (VMs mit Quadro P2000 / M4000): llama.cpp-Server. -# Maxwell (sm_52) und Pascal (sm_61) brauchen CUDA 12.x – daher eigener Build mit festen Architekturen. -# Profile: embed (Embedding + Rerank), chat (Verifier/Signatur), z. B. `docker compose --profile embed up -d` +# Rollen "gpu-embed" und "gpu-small" (GPU-VMs mit Quadro P2000 + M4000): llama.cpp-Server (Task 6.1). +# Maxwell (sm_52) und Pascal (sm_61) brauchen CUDA 12.x – daher eigener Build mit festen Architekturen, +# derselbe Stand wie im Benchmark (ADR-0006): b6000 = Commit 4762ad7, CUDA 12.4.0, sm_52;61. +# Profile: embed (Embedding + Rerank, gpu-pve44/46), chat (verifier-small/signature-extractor, gpu-pve48). +# Ausgerollt von der Ansible-Rolle gpu-llamacpp (.env mit COMPOSE_PROFILES und Karten-UUIDs). +# +# Kartenwahl: je Dienst device_ids (UUID oder Index aus `nvidia-smi -L`). Ohne Angabe Index 1 = P2000 +# (in gpu-pve44/46/48 ist Index 0 die M4000, sie bleibt frei). `count: 1` griffe die M4000. +# Netz: Host-Netz wie nex-speech. Veröffentlichte Docker-Ports liefen über DNAT und umgingen ufw; +# so gelten spätere ufw-Regeln der VM (Task 4.4). +# Datenschutz: kein --verbose (der loggt Prompts), /slots aus (zeigt Prompt-Inhalte), keine Web-Oberfläche. name: knaster-gpu-llamacpp +x-logging: &logging + driver: local + options: {max-size: "5m", max-file: "2"} + x-llamacpp: &llamacpp build: - context: https://github.com/ggml-org/llama.cpp.git#${LLAMACPP_REF:-b6000} + context: https://github.com/ggml-org/llama.cpp.git#${LLAMACPP_COMMIT:-4762ad7316dcdec20016ab5985fb46a27902204d} dockerfile: .devops/cuda.Dockerfile target: server args: CUDA_VERSION: ${CUDA_VERSION:-12.4.0} CUDA_DOCKER_ARCH: ${CUDA_DOCKER_ARCH:-52;61} - image: knaster/llamacpp-server:${LLAMACPP_REF:-b6000}-cu12 + image: knaster/llamacpp-server:${LLAMACPP_REF:-b6000}-cu${CUDA_VERSION:-12.4.0} restart: unless-stopped + network_mode: host + read_only: true + tmpfs: [/tmp:size=64m] labels: {autoheal: "true"} volumes: - ${MODELS_DIR:-/srv/models}:/models:ro - deploy: - resources: - reservations: - devices: [{driver: nvidia, count: 1, capabilities: [gpu]}] + logging: *logging services: llamacpp-embed: @@ -27,60 +39,61 @@ services: container_name: llamacpp-embed profiles: ["embed"] command: ["-m", "/models/${EMBED_GGUF:-Qwen3-Embedding-0.6B-Q8_0.gguf}", "--embedding", - "--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-ub", "8192", "-ngl", "99", - "--alias", "embed-default", "--metrics", "--host", "0.0.0.0", "--port", "8081"] - ports: ["8081:8081"] + "--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", + "-ngl", "99", "--alias", "embed-default", "--metrics", "--no-slots", "--no-webui", + "--host", "0.0.0.0", "--port", "8081"] + deploy: + resources: + reservations: + devices: [{driver: nvidia, device_ids: ["${EMBED_GPU:-1}"], capabilities: [gpu]}] healthcheck: &hc test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"] interval: 30s timeout: 5s retries: 3 - start_period: 60s + start_period: 120s llamacpp-rerank: <<: *llamacpp container_name: llamacpp-rerank profiles: ["embed"] - command: ["-m", "/models/${RERANK_GGUF:-bge-reranker-v2-m3-Q8_0.gguf}", "--reranking", "-c", "8192", - "-ub", "8192", "-ngl", "99", "--alias", "rerank-default", "--metrics", "--host", "0.0.0.0", - "--port", "8082"] - ports: ["8082:8082"] + command: ["-m", "/models/${RERANK_GGUF:-bge-reranker-v2-m3-Q8_0.gguf}", "--reranking", + "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", "-ngl", "99", "--alias", "rerank-default", + "--metrics", "--no-slots", "--no-webui", "--host", "0.0.0.0", "--port", "8082"] + deploy: + resources: + reservations: + devices: [{driver: nvidia, device_ids: ["${RERANK_GPU:-1}"], capabilities: [gpu]}] healthcheck: <<: *hc test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8082/health"] + # Qwen3 ohne Denkphase (enable_thinking=false): Verifier und Signatur brauchen kurze, direkte Antworten, + # und der Canary-Prober fragt mit max_tokens=5 nach "OK". llamacpp-chat: <<: *llamacpp container_name: llamacpp-chat profiles: ["chat"] - command: ["-m", "/models/${CHAT_GGUF:-Qwen3-4B-Q4_K_M.gguf}", "-c", "8192", "-ngl", "99", - "--alias", "verifier-small", "--metrics", "--host", "0.0.0.0", "--port", "8080"] - ports: ["8080:8080"] + command: ["-m", "/models/${CHAT_GGUF:-Qwen3-4B-Q4_K_M.gguf}", "-c", "8192", "-np", "1", "-ngl", "99", + "--jinja", "--chat-template-kwargs", "{\"enable_thinking\": false}", + "--alias", "verifier-small", "--metrics", "--no-slots", "--no-webui", + "--host", "0.0.0.0", "--port", "8080"] + deploy: + resources: + reservations: + devices: [{driver: nvidia, device_ids: ["${CHAT_GPU:-1}"], capabilities: [gpu]}] healthcheck: <<: *hc test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health"] - gpu-exporter: - image: utkuozdemir/nvidia_gpu_exporter:1.2.1 - restart: unless-stopped - ports: ["9835:9835"] - deploy: - resources: - reservations: - devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}] - - node-exporter: - image: prom/node-exporter:v1.8.1 - restart: unless-stopped - command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"] - pid: host - volumes: - - /:/host:ro,rslave - - /var/lib/node_exporter/textfile:/textfile:ro - ports: ["9100:9100"] - + # Startet Container mit Label autoheal=true neu, wenn ihr Healthcheck dauerhaft scheitert + # (restart: unless-stopped greift nur beim Absturz). Läuft in jedem Profil mit. autoheal: image: willfarrell/autoheal:1.2.0 + container_name: llamacpp-autoheal restart: unless-stopped environment: {AUTOHEAL_CONTAINER_LABEL: autoheal} volumes: ["/var/run/docker.sock:/var/run/docker.sock"] + logging: *logging + +# Exporter (node_exporter 9100, nvidia_gpu_exporter 9835) laufen nativ in der VM (Rolle gpu-exporters). diff --git a/platform/proxmox/ansible/inventory.yml b/platform/proxmox/ansible/inventory.yml index 76278c7..17313dc 100644 --- a/platform/proxmox/ansible/inventory.yml +++ b/platform/proxmox/ansible/inventory.yml @@ -28,6 +28,12 @@ all: ansible_user: knaster ansible_become: true ansible_python_interpreter: /usr/bin/python3 + # llama.cpp-Profile je GPU-VM (Task 6.1, Rolle gpu-llamacpp). Liefert auch die Prometheus-Ziele (file_sd). + gpu_llamacpp: + hosts: + gpu-pve44: {llamacpp_profiles: [embed]} + gpu-pve46: {llamacpp_profiles: [embed]} + gpu-pve48: {llamacpp_profiles: [chat]} infra: hosts: infra: {} diff --git a/platform/proxmox/ansible/playbooks/gpu-llamacpp.yml b/platform/proxmox/ansible/playbooks/gpu-llamacpp.yml new file mode 100644 index 0000000..6ecd0c2 --- /dev/null +++ b/platform/proxmox/ansible/playbooks/gpu-llamacpp.yml @@ -0,0 +1,8 @@ +# llama.cpp-Rollen (Task 6.1): Embedding + Rerank in gpu-pve44 und gpu-pve46, Chat (verifier-small, +# signature-extractor) in gpu-pve48. Zuordnung: Inventar, Gruppe gpu_llamacpp (gpu-pve52 gehört nicht dazu). +# Aufruf auf ops (Verzeichnis platform/proxmox/ansible), vorher Zugang auffrischen (refresh-vm-access.sh): +# ansible-playbook playbooks/gpu-llamacpp.yml +# ansible-playbook playbooks/monitoring-targets.yml # Prometheus-Ziele (file_sd) nachziehen +- name: llama.cpp-Server in den GPU-VMs + hosts: gpu_llamacpp + roles: [gpu-llamacpp] diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml new file mode 100644 index 0000000..5c40a0a --- /dev/null +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml @@ -0,0 +1,50 @@ +# llama.cpp-Rollen in den GPU-VMs (Task 6.1): Compose aus deploy/compose/gpu-llamacpp. +# Profile je VM kommen aus dem Inventar (Gruppe gpu_llamacpp, Variable llamacpp_profiles). +llamacpp_dir: /opt/knaster/gpu-llamacpp +llamacpp_models_dir: /srv/models +llamacpp_profiles: [] + +# Karte je Dienst, gewählt über den Namen aus `nvidia-smi` (ADR-0006: alles auf die P2000, M4000 bleibt frei). +# Passt Rerank nicht neben Embedding auf die P2000, hier "Quadro M4000" eintragen. +llamacpp_cards: + EMBED_GPU: Quadro P2000 + RERANK_GPU: Quadro P2000 + CHAT_GPU: Quadro P2000 + +# Dienste und Ports je Profil (feste Ports im Compose, Host-Netz) +llamacpp_services: + embed: + - {name: llamacpp-embed, port: 8081} + - {name: llamacpp-rerank, port: 8082} + chat: + - {name: llamacpp-chat, port: 8080} + +# ufw: nur Regeln vormerken (Kommentar knaster-llamacpp), ufw nie ein- oder ausschalten (Firewall: Task 4.4) +llamacpp_allow_from: 192.168.100.0/24 + +# Werte für .env (keine Geheimnisse); Build-Stand wie im Benchmark (ADR-0006) +llamacpp_env: + LLAMACPP_REF: b6000 + LLAMACPP_COMMIT: 4762ad7316dcdec20016ab5985fb46a27902204d + CUDA_VERSION: 12.4.0 + CUDA_DOCKER_ARCH: "52;61" + MODELS_DIR: "{{ llamacpp_models_dir }}" + EMBED_GGUF: Qwen3-Embedding-0.6B-Q8_0.gguf + EMBED_POOLING: last + RERANK_GGUF: bge-reranker-v2-m3-Q8_0.gguf + CHAT_GGUF: Qwen3-4B-Q4_K_M.gguf + +# Modelle je Profil mit fester Revision und Prüfsumme (sha256 = LFS-OID laut Hugging Face, ADR-0006) +_hf: https://huggingface.co +llamacpp_models: + embed: + - dest: Qwen3-Embedding-0.6B-Q8_0.gguf + url: "{{ _hf }}/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/370f27d7550e0def9b39c1f16d3fbaa13aa67728/Qwen3-Embedding-0.6B-Q8_0.gguf" + checksum: sha256:06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439 + - dest: bge-reranker-v2-m3-Q8_0.gguf + url: "{{ _hf }}/gpustack/bge-reranker-v2-m3-GGUF/resolve/3093af03b1a635e67b084b1d8c03c5f5e020fd05/bge-reranker-v2-m3-Q8_0.gguf" + checksum: sha256:a43c7c9b11a4c1517e5bf95151960e1621d1b72f7a493364b01e386cf1aaa1d3 + chat: + - dest: Qwen3-4B-Q4_K_M.gguf + url: "{{ _hf }}/Qwen/Qwen3-4B-GGUF/resolve/bc640142c66e1fdd12af0bd68f40445458f3869b/Qwen3-4B-Q4_K_M.gguf" + checksum: sha256:7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5 diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml new file mode 100644 index 0000000..a92bd53 --- /dev/null +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml @@ -0,0 +1,111 @@ +# llama.cpp-Server je Profil (Task 6.1). Idempotent: Modelle per Prüfsumme, Image nur bauen, wenn es fehlt +# (Image-Tag enthält Stand und CUDA-Version), compose ändert nur, was sich geändert hat. +- name: Profile prüfen + ansible.builtin.assert: + that: + - llamacpp_profiles | length > 0 + - llamacpp_profiles | difference(llamacpp_services.keys() | list) | length == 0 + fail_msg: "llamacpp_profiles muss aus {{ llamacpp_services.keys() | list }} bestehen (Inventar, Gruppe gpu_llamacpp)" + +# Compose baut nur mit buildx (BuildKit, auch für den Git-Kontext mit Commit-Pin) +- name: docker-buildx installieren + ansible.builtin.apt: + name: docker-buildx + state: present + +# --- Kartenwahl: UUID statt Index, geprüft über den Kartennamen --- +- name: Karten in der VM lesen + ansible.builtin.command: nvidia-smi --query-gpu=index,uuid,name --format=csv,noheader + register: llamacpp_smi + changed_when: false + +- name: UUID je Dienst aus dem Kartennamen bestimmen + ansible.builtin.set_fact: + llamacpp_gpu_env: >- + {{ llamacpp_gpu_env | default({}) | combine({item.key: (llamacpp_smi.stdout_lines + | map('split', ', ') | selectattr(2, 'equalto', item.value) | map(attribute=1) | first | default(''))}) }} + loop: "{{ llamacpp_cards | dict2items }}" + +- name: Jede gewünschte Karte ist vorhanden + ansible.builtin.assert: + that: llamacpp_gpu_env[item.key] is match('GPU-') + fail_msg: "{{ item.value }} für {{ item.key }} nicht gefunden: {{ llamacpp_smi.stdout_lines }}" + loop: "{{ llamacpp_cards | dict2items }}" + +# --- Modelle --- +- name: Modelle laden (Prüfsumme gegen Hugging Face) + ansible.builtin.get_url: + url: "{{ item.url }}" + dest: "{{ llamacpp_models_dir }}/{{ item.dest }}" + checksum: "{{ item.checksum }}" + mode: "0644" + timeout: 120 + loop: "{{ llamacpp_profiles | map('extract', llamacpp_models) | flatten }}" + loop_control: + label: "{{ item.dest }}" + +# --- Compose-Projekt --- +- name: Projektverzeichnis + ansible.builtin.file: + path: "{{ llamacpp_dir }}" + state: directory + mode: "0755" + +- name: Compose-Datei kopieren + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../../../deploy/compose/gpu-llamacpp/docker-compose.yml" + dest: "{{ llamacpp_dir }}/docker-compose.yml" + mode: "0644" + +- name: .env schreiben (keine Geheimnisse) + ansible.builtin.copy: + dest: "{{ llamacpp_dir }}/.env" + mode: "0644" + content: | + # Verwaltet von Ansible (Rolle gpu-llamacpp) + COMPOSE_PROFILES={{ llamacpp_profiles | join(',') }} + {% for k, v in (llamacpp_env | combine(llamacpp_gpu_env)).items() %} + {{ k }}={{ v }} + {% endfor %} + +# ufw-Regeln wirken auch bei inaktivem ufw (werden vorgemerkt). Fremde Regeln bleiben unberührt. +- name: Ist ufw vorhanden? + ansible.builtin.stat: + path: /usr/sbin/ufw + register: llamacpp_ufw + +- name: ufw – llama.cpp-Ports aus dem LAN vormerken + community.general.ufw: + rule: allow + src: "{{ llamacpp_allow_from }}" + port: "{{ item.port | string }}" + proto: tcp + comment: knaster-llamacpp + loop: "{{ llamacpp_profiles | map('extract', llamacpp_services) | flatten }}" + loop_control: + label: "{{ item.name }}:{{ item.port }}" + when: llamacpp_ufw.stat.exists + +# Der erste Build dauert lange (llama.cpp mit CUDA, sm_52/61, alle CPU-Varianten auf 4 vCPU) +- name: Image bauen (falls es fehlt) und Dienste starten + community.docker.docker_compose_v2: + project_src: "{{ llamacpp_dir }}" + profiles: "{{ llamacpp_profiles }}" + build: policy + state: present + remove_orphans: true + wait: false + async: 7200 + poll: 30 + +- name: Warten, bis alle Dienste gesund sind + ansible.builtin.uri: + url: "http://127.0.0.1:{{ item.port }}/health" + status_code: 200 + register: llamacpp_health + until: llamacpp_health.status == 200 + retries: 60 + delay: 10 + loop: "{{ llamacpp_profiles | map('extract', llamacpp_services) | flatten }}" + loop_control: + label: "{{ item.name }}:{{ item.port }}"