diff --git a/deploy/compose/gpu-llamacpp/docker-compose.yml b/deploy/compose/gpu-llamacpp/docker-compose.yml index d827688..89c576e 100644 --- a/deploy/compose/gpu-llamacpp/docker-compose.yml +++ b/deploy/compose/gpu-llamacpp/docker-compose.yml @@ -39,7 +39,7 @@ services: container_name: llamacpp-embed profiles: ["embed"] command: ["-m", "/models/${EMBED_GGUF:-Qwen3-Embedding-0.6B-Q8_0.gguf}", "--embedding", - "--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", + "--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-b", "4096", "-ub", "4096", "-np", "1", # ub 8192 sprengt die P2000 (Task 6.1) "-ngl", "99", "--alias", "embed-default", "--metrics", "--no-slots", "--no-webui", "--host", "0.0.0.0", "--port", "8081"] deploy: diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml index 92b5696..b949732 100644 --- a/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/defaults/main.yml @@ -60,8 +60,10 @@ llamacpp_k3s_services: port: 8081 gpu_var: EMBED_GPU memory_limit: 3Gi + # -ub 4096 statt 8192: Der Rechenpuffer wächst mit ub × Vokabular (151k) und bräuchte bei 8192 allein 5 GB + # (19.10. gemessen). Mit 4096: 2,5 GB Puffer, P2000 mit Rerank 4,4–4,6 von 5 GB, 5.467 Tokens in einem Aufruf ok. args: ["-m", "/models/{{ llamacpp_env.EMBED_GGUF }}", "--embedding", "--pooling", "{{ llamacpp_env.EMBED_POOLING }}", - "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", "-ngl", "99", "--alias", "embed-default", + "-c", "8192", "-b", "4096", "-ub", "4096", "-np", "1", "-ngl", "99", "--alias", "embed-default", "--metrics", "--no-slots", "--no-webui", "--host", "0.0.0.0", "--port", "8081"] - name: llamacpp-rerank port: 8082 diff --git a/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml index e815bc4..21b0763 100644 --- a/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml +++ b/platform/proxmox/ansible/roles/gpu-llamacpp/tasks/prepare.yml @@ -36,4 +36,3 @@ loop: "{{ llamacpp_profiles | map('extract', llamacpp_models) | flatten }}" loop_control: label: "{{ item.dest }}" -