From 986f1116fe5444014ea77dc11c811103aee94e9c Mon Sep 17 00:00:00 2001 From: Robert Rapp Date: Mon, 19 Oct 2026 17:27:23 +0200 Subject: [PATCH] k3s: llama.cpp-Server als Deployments (Task 6.1) Ein Pod je GPU-VM (Embed+Rerank bzw. Chat), Host-Netz mit den Ports aus dem Compose, P2000 fest per UUID (NVIDIA_VISIBLE_DEVICES). Das Device Plugin vergibt nur noch Index 0 (M4000). Rolle gpu-llamacpp: gemeinsame Vorbereitung (prepare.yml), k3s.yml mit Image-Import und Compose-Stopp; gpu-llamacpp.yml bricht ohne -e llamacpp_compose=true ab. Co-Authored-By: Claude Opus 5.5 --- deploy/k8s/llamacpp/llamacpp.yaml.j2 | 62 ++++++++++++++++ deploy/k8s/llamacpp/namespace.yaml | 6 ++ deploy/k8s/nvidia-device-plugin.yaml | 4 + .../ansible/playbooks/gpu-llamacpp-k3s.yml | 52 +++++++++++++ .../ansible/playbooks/gpu-llamacpp.yml | 8 ++ .../roles/gpu-llamacpp/defaults/main.yml | 31 ++++++++ .../ansible/roles/gpu-llamacpp/tasks/k3s.yml | 74 +++++++++++++++++++ .../ansible/roles/gpu-llamacpp/tasks/main.yml | 39 +--------- .../roles/gpu-llamacpp/tasks/prepare.yml | 39 ++++++++++ 9 files changed, 278 insertions(+), 37 deletions(-) create mode 100644 deploy/k8s/llamacpp/llamacpp.yaml.j2 create mode 100644 deploy/k8s/llamacpp/namespace.yaml create mode 100644 platform/proxmox/ansible/playbooks/gpu-llamacpp-k3s.yml create mode 100644 platform/proxmox/ansible/roles/gpu-llamacpp/tasks/k3s.yml create mode 100644 platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml diff --git a/deploy/k8s/llamacpp/llamacpp.yaml.j2 b/deploy/k8s/llamacpp/llamacpp.yaml.j2 new file mode 100644 index 0000000..909c14c --- /dev/null +++ b/deploy/k8s/llamacpp/llamacpp.yaml.j2 @@ -0,0 +1,62 @@ +# llama.cpp-Server je GPU-VM in k3s (Task 6.1, D6), gerendert von der Rolle gpu-llamacpp (tasks/k3s.yml) je Knoten. +# Ein Pod je Knoten; im Profil embed laufen Embedding und Rerank als zwei Container auf derselben P2000. +# Kartenwahl: kein nvidia.com/gpu (das Device Plugin verwaltet nur die M4000, Index 0), sondern die UUID der P2000 +# per NVIDIA_VISIBLE_DEVICES, ermittelt und geprüft über den Kartennamen (ADR-0006: M4000 bleibt frei). +# Host-Netz mit den Ports wie zuvor im Compose (8080/8081/8082): Prometheus-Ziele (file_sd) und ufw bleiben gültig. +# Image aus dem Compose-Build, in das containerd von k3s importiert (imagePullPolicy Never). +# Datenschutz wie im Compose: kein --verbose, /slots aus, keine Web-Oberfläche. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llamacpp-{{ inventory_hostname }} + namespace: knaster + labels: {app: llamacpp, knaster.node: {{ inventory_hostname }}} +spec: + replicas: 1 + strategy: {type: Recreate} # Host-Ports und VRAM der P2000 können nur einmal belegt sein + selector: + matchLabels: {app: llamacpp, knaster.node: {{ inventory_hostname }}} + template: + metadata: + labels: {app: llamacpp, knaster.node: {{ inventory_hostname }}} + spec: + runtimeClassName: nvidia + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/hostname: {{ inventory_hostname }} + containers: +{% for svc in llamacpp_k3s_containers %} + - name: {{ svc.name }} + image: {{ llamacpp_image }} + imagePullPolicy: Never + args: {{ svc.args | to_json }} + env: + - {name: NVIDIA_VISIBLE_DEVICES, value: "{{ llamacpp_gpu_env[svc.gpu_var] }}"} + - {name: NVIDIA_DRIVER_CAPABILITIES, value: "compute,utility"} + resources: + requests: {cpu: 500m, memory: 1Gi} + limits: {memory: {{ svc.memory_limit }}} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + volumeMounts: + - {name: models, mountPath: /models, readOnly: true} + - {name: tmp, mountPath: /tmp} + startupProbe: + httpGet: {path: /health, port: {{ svc.port }}, host: 127.0.0.1} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /health, port: {{ svc.port }}, host: 127.0.0.1} + periodSeconds: 30 + livenessProbe: + httpGet: {path: /health, port: {{ svc.port }}, host: 127.0.0.1} + periodSeconds: 30 + failureThreshold: 3 +{% endfor %} + volumes: + - name: models + hostPath: {path: {{ llamacpp_models_dir }}, type: Directory} + - name: tmp + emptyDir: {sizeLimit: 64Mi} diff --git a/deploy/k8s/llamacpp/namespace.yaml b/deploy/k8s/llamacpp/namespace.yaml new file mode 100644 index 0000000..294d0a9 --- /dev/null +++ b/deploy/k8s/llamacpp/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: knaster + labels: + knaster.app: llamacpp diff --git a/deploy/k8s/nvidia-device-plugin.yaml b/deploy/k8s/nvidia-device-plugin.yaml index 9f09901..69f59fc 100644 --- a/deploy/k8s/nvidia-device-plugin.yaml +++ b/deploy/k8s/nvidia-device-plugin.yaml @@ -31,6 +31,10 @@ spec: env: - name: FAIL_ON_INIT_ERROR value: "false" + # Nur Index 0 (Quadro M4000 in allen GPU-VMs, geprüft in playbooks/gpu-llamacpp-k3s.yml) wird als + # nvidia.com/gpu vergeben. Die P2000 gehört fest den llama.cpp-Pods (per UUID, Task 6.1). + - name: NVIDIA_VISIBLE_DEVICES + value: "0" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/platform/proxmox/ansible/playbooks/gpu-llamacpp-k3s.yml b/platform/proxmox/ansible/playbooks/gpu-llamacpp-k3s.yml new file mode 100644 index 0000000..2ff9bf5 --- /dev/null +++ b/platform/proxmox/ansible/playbooks/gpu-llamacpp-k3s.yml @@ -0,0 +1,52 @@ +# llama.cpp-Rollen nach k3s (Task 6.1, D6). Läuft auf ops (Verzeichnis platform/proxmox/ansible): +# ansible-playbook playbooks/gpu-llamacpp-k3s.yml # alle VMs der Gruppe gpu_llamacpp +# ansible-playbook playbooks/gpu-llamacpp-k3s.yml -e llamacpp_k3s_hosts=gpu-pve44 # nur eine VM (Pilot) +# Danach: ansible-playbook playbooks/monitoring-targets.yml (Ziele bleiben gleich, Ports wie im Compose). +# +# Kartenwahl: Das Device Plugin verwaltet nur Index 0 (M4000, in allen GPU-VMs); die P2000 bekommen die +# llama.cpp-Pods fest per UUID. Erst prüfen, dass Index 0 überall die M4000 ist, dann das Device Plugin anwenden. +- name: GPU-VMs – Index 0 ist die M4000 + hosts: gpu_vms + gather_facts: false + tasks: + - name: Karte auf Index 0 lesen + ansible.builtin.command: nvidia-smi -i 0 --query-gpu=name --format=csv,noheader + register: gpu0 + changed_when: false + + - name: Index 0 muss die M4000 sein (sonst gibt das Device Plugin die falsche Karte frei) + ansible.builtin.assert: + that: gpu0.stdout == "Quadro M4000" + fail_msg: "{{ inventory_hostname }}: Index 0 ist {{ gpu0.stdout }}" + +- name: k3s – Device Plugin nur für die M4000 + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Device Plugin anwenden + ansible.builtin.command: kubectl apply -f {{ playbook_dir }}/../../../../deploy/k8s/nvidia-device-plugin.yaml + register: dp + changed_when: "'configured' in dp.stdout" + + - name: Warten, bis das Device Plugin auf allen Knoten läuft + ansible.builtin.command: kubectl -n kube-system rollout status daemonset/nvidia-device-plugin-daemonset --timeout=180s + changed_when: false + + - name: Jede GPU-VM meldet genau eine Karte (M4000) + ansible.builtin.command: kubectl get node {{ item }} -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' + loop: "{{ groups['gpu_vms'] }}" + register: alloc + until: alloc.stdout == "1" + retries: 24 + delay: 5 + changed_when: false + +- name: llama.cpp-Server in k3s + hosts: "{{ llamacpp_k3s_hosts | default('gpu_llamacpp') }}" + serial: 1 + tasks: + - name: Rolle gpu-llamacpp (k3s) + ansible.builtin.include_role: + name: gpu-llamacpp + tasks_from: k3s diff --git a/platform/proxmox/ansible/playbooks/gpu-llamacpp.yml b/platform/proxmox/ansible/playbooks/gpu-llamacpp.yml index 6ecd0c2..1f9ee85 100644 --- a/platform/proxmox/ansible/playbooks/gpu-llamacpp.yml +++ b/platform/proxmox/ansible/playbooks/gpu-llamacpp.yml @@ -3,6 +3,14 @@ # Aufruf auf ops (Verzeichnis platform/proxmox/ansible), vorher Zugang auffrischen (refresh-vm-access.sh): # ansible-playbook playbooks/gpu-llamacpp.yml # ansible-playbook playbooks/monitoring-targets.yml # Prometheus-Ziele (file_sd) nachziehen +# +# Seit 19.10. laufen die Server in k3s (playbooks/gpu-llamacpp-k3s.yml). Dieses Playbook würde den Compose-Stack +# wieder starten und Host-Ports und VRAM doppelt belegen. Nur mit -e llamacpp_compose=true (Rückweg). - name: llama.cpp-Server in den GPU-VMs hosts: gpu_llamacpp + pre_tasks: + - name: Abbruch, wenn llama.cpp in k3s läuft + ansible.builtin.fail: + msg: "llama.cpp läuft in k3s (playbooks/gpu-llamacpp-k3s.yml). Compose nur mit -e llamacpp_compose=true." + when: not (llamacpp_compose | default(false) | bool) roles: [gpu-llamacpp] diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml index 5c40a0a..92b5696 100644 --- a/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml @@ -48,3 +48,34 @@ llamacpp_models: - dest: Qwen3-4B-Q4_K_M.gguf url: "{{ _hf }}/Qwen/Qwen3-4B-GGUF/resolve/bc640142c66e1fdd12af0bd68f40445458f3869b/Qwen3-4B-Q4_K_M.gguf" checksum: sha256:7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5 + +# --- k3s (Task 6.1, D6): tasks/k3s.yml, Playbook playbooks/gpu-llamacpp-k3s.yml --- +llamacpp_image: "knaster/llamacpp-server:{{ llamacpp_env.LLAMACPP_REF }}-cu{{ llamacpp_env.CUDA_VERSION }}" +llamacpp_k3s_template: "{{ playbook_dir }}/../../../../deploy/k8s/llamacpp/llamacpp.yaml.j2" +llamacpp_k3s_namespace: "{{ playbook_dir }}/../../../../deploy/k8s/llamacpp/namespace.yaml" +# Argumente wie im Compose (deploy/compose/gpu-llamacpp); gpu_var wählt die Karte aus llamacpp_cards +llamacpp_k3s_services: + embed: + - name: llamacpp-embed + port: 8081 + gpu_var: EMBED_GPU + memory_limit: 3Gi + args: ["-m", "/models/{{ llamacpp_env.EMBED_GGUF }}", "--embedding", "--pooling", "{{ llamacpp_env.EMBED_POOLING }}", + "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", "-ngl", "99", "--alias", "embed-default", + "--metrics", "--no-slots", "--no-webui", "--host", "0.0.0.0", "--port", "8081"] + - name: llamacpp-rerank + port: 8082 + gpu_var: RERANK_GPU + memory_limit: 3Gi + args: ["-m", "/models/{{ llamacpp_env.RERANK_GGUF }}", "--reranking", + "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", "-ngl", "99", "--alias", "rerank-default", + "--metrics", "--no-slots", "--no-webui", "--host", "0.0.0.0", "--port", "8082"] + chat: + - name: llamacpp-chat + port: 8080 + gpu_var: CHAT_GPU + memory_limit: 6Gi + args: ["-m", "/models/{{ llamacpp_env.CHAT_GGUF }}", "-c", "8192", "-np", "1", "-ngl", "99", + "--jinja", "--chat-template-kwargs", "{\"enable_thinking\": false}", + "--alias", "verifier-small", "--metrics", "--no-slots", "--no-webui", + "--host", "0.0.0.0", "--port", "8080"] diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/k3s.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/k3s.yml new file mode 100644 index 0000000..be39195 --- /dev/null +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/k3s.yml @@ -0,0 +1,74 @@ +# llama.cpp in k3s statt Compose (Task 6.1, D6). Aufruf über playbooks/gpu-llamacpp-k3s.yml (je VM nacheinander). +# Rückweg je VM: kubectl -n knaster delete deploy llamacpp-, dann in der VM +# docker compose --project-directory /opt/knaster/gpu-llamacpp start +- name: Profile, Karten, Modelle + ansible.builtin.include_tasks: prepare.yml + +# 1. Image aus Docker in das containerd von k3s übernehmen (kein Neubau, kein Registry-Umweg) +- name: Image in k3s vorhanden? + ansible.builtin.command: /usr/local/bin/k3s ctr images ls -q + register: llamacpp_k3s_images + changed_when: false + +- name: Image aus Docker importieren + ansible.builtin.shell: set -o pipefail; docker save {{ llamacpp_image }} | /usr/local/bin/k3s ctr images import - + args: + executable: /bin/bash + when: ("docker.io/" ~ llamacpp_image) not in llamacpp_k3s_images.stdout + async: 1800 + poll: 15 + +# 2. Compose-Stack anhalten (Host-Ports und VRAM werden frei), Dateien bleiben für den Rückweg +- name: Compose-Projekt vorhanden? + ansible.builtin.stat: + path: "{{ llamacpp_dir }}/docker-compose.yml" + register: llamacpp_compose_file + +- name: Compose-Stack anhalten + community.docker.docker_compose_v2: + project_src: "{{ llamacpp_dir }}" + state: stopped + when: llamacpp_compose_file.stat.exists + +# 3. Manifest je VM rendern (UUID der P2000) und anwenden +- name: Namespace knaster + ansible.builtin.command: kubectl apply -f {{ llamacpp_k3s_namespace }} + register: llamacpp_ns + changed_when: "'created' in llamacpp_ns.stdout" + delegate_to: localhost + become: false + +- name: Deployment anwenden + ansible.builtin.command: + cmd: kubectl apply -f - + stdin: "{{ lookup('template', llamacpp_k3s_template) }}" + vars: + llamacpp_k3s_containers: "{{ llamacpp_profiles | map('extract', llamacpp_k3s_services) | flatten }}" + register: llamacpp_apply + changed_when: "'configured' in llamacpp_apply.stdout or 'created' in llamacpp_apply.stdout" + delegate_to: localhost + become: false + +- name: Warten, bis der Pod bereit ist + ansible.builtin.command: kubectl -n knaster rollout status deployment/llamacpp-{{ inventory_hostname }} --timeout=600s + changed_when: false + delegate_to: localhost + become: false + +# 4. Prüfen in der VM: Dienste gesund, Prozesse nur auf der P2000 +- name: Gesundheit in der VM prüfen + ansible.builtin.uri: + url: "http://127.0.0.1:{{ item.port }}/health" + status_code: 200 + loop: "{{ llamacpp_profiles | map('extract', llamacpp_k3s_services) | flatten }}" + loop_control: + label: "{{ item.name }}:{{ item.port }}" + +- name: Belegung der Karten + ansible.builtin.command: nvidia-smi --query-gpu=index,name,memory.used,memory.total --format=csv,noheader + register: llamacpp_vram + changed_when: false + +- name: Ergebnis + ansible.builtin.debug: + msg: "{{ llamacpp_vram.stdout_lines }}" diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml index a92bd53..3203ed4 100644 --- a/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/main.yml @@ -1,11 +1,7 @@ # llama.cpp-Server je Profil (Task 6.1). Idempotent: Modelle per Prüfsumme, Image nur bauen, wenn es fehlt # (Image-Tag enthält Stand und CUDA-Version), compose ändert nur, was sich geändert hat. -- name: Profile prüfen - ansible.builtin.assert: - that: - - llamacpp_profiles | length > 0 - - llamacpp_profiles | difference(llamacpp_services.keys() | list) | length == 0 - fail_msg: "llamacpp_profiles muss aus {{ llamacpp_services.keys() | list }} bestehen (Inventar, Gruppe gpu_llamacpp)" +- name: Profile, Karten, Modelle + ansible.builtin.include_tasks: prepare.yml # Compose baut nur mit buildx (BuildKit, auch für den Git-Kontext mit Commit-Pin) - name: docker-buildx installieren @@ -13,37 +9,6 @@ name: docker-buildx state: present -# --- Kartenwahl: UUID statt Index, geprüft über den Kartennamen --- -- name: Karten in der VM lesen - ansible.builtin.command: nvidia-smi --query-gpu=index,uuid,name --format=csv,noheader - register: llamacpp_smi - changed_when: false - -- name: UUID je Dienst aus dem Kartennamen bestimmen - ansible.builtin.set_fact: - llamacpp_gpu_env: >- - {{ llamacpp_gpu_env | default({}) | combine({item.key: (llamacpp_smi.stdout_lines - | map('split', ', ') | selectattr(2, 'equalto', item.value) | map(attribute=1) | first | default(''))}) }} - loop: "{{ llamacpp_cards | dict2items }}" - -- name: Jede gewünschte Karte ist vorhanden - ansible.builtin.assert: - that: llamacpp_gpu_env[item.key] is match('GPU-') - fail_msg: "{{ item.value }} für {{ item.key }} nicht gefunden: {{ llamacpp_smi.stdout_lines }}" - loop: "{{ llamacpp_cards | dict2items }}" - -# --- Modelle --- -- name: Modelle laden (Prüfsumme gegen Hugging Face) - ansible.builtin.get_url: - url: "{{ item.url }}" - dest: "{{ llamacpp_models_dir }}/{{ item.dest }}" - checksum: "{{ item.checksum }}" - mode: "0644" - timeout: 120 - loop: "{{ llamacpp_profiles | map('extract', llamacpp_models) | flatten }}" - loop_control: - label: "{{ item.dest }}" - # --- Compose-Projekt --- - name: Projektverzeichnis ansible.builtin.file: diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml new file mode 100644 index 0000000..e815bc4 --- /dev/null +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml @@ -0,0 +1,39 @@ +# Gemeinsam für Compose (main.yml) und k3s (k3s.yml): Profile prüfen, P2000 per Name finden, Modelle laden. +- name: Profile prüfen + ansible.builtin.assert: + that: + - llamacpp_profiles | length > 0 + - llamacpp_profiles | difference(llamacpp_services.keys() | list) | length == 0 + fail_msg: "llamacpp_profiles muss aus {{ llamacpp_services.keys() | list }} bestehen (Inventar, Gruppe gpu_llamacpp)" + +# --- Kartenwahl: UUID statt Index, geprüft über den Kartennamen --- +- name: Karten in der VM lesen + ansible.builtin.command: nvidia-smi --query-gpu=index,uuid,name --format=csv,noheader + register: llamacpp_smi + changed_when: false + +- name: UUID je Dienst aus dem Kartennamen bestimmen + ansible.builtin.set_fact: + llamacpp_gpu_env: >- + {{ llamacpp_gpu_env | default({}) | combine({item.key: (llamacpp_smi.stdout_lines + | map('split', ', ') | selectattr(2, 'equalto', item.value) | map(attribute=1) | first | default(''))}) }} + loop: "{{ llamacpp_cards | dict2items }}" + +- name: Jede gewünschte Karte ist vorhanden + ansible.builtin.assert: + that: llamacpp_gpu_env[item.key] is match('GPU-') + fail_msg: "{{ item.value }} für {{ item.key }} nicht gefunden: {{ llamacpp_smi.stdout_lines }}" + loop: "{{ llamacpp_cards | dict2items }}" + +# --- Modelle --- +- name: Modelle laden (Prüfsumme gegen Hugging Face) + ansible.builtin.get_url: + url: "{{ item.url }}" + dest: "{{ llamacpp_models_dir }}/{{ item.dest }}" + checksum: "{{ item.checksum }}" + mode: "0644" + timeout: 120 + loop: "{{ llamacpp_profiles | map('extract', llamacpp_models) | flatten }}" + loop_control: + label: "{{ item.dest }}" +