Robert RappandClaude Opus 5.5
986f1116fe
ci / test (push) Failing after 22s
Ein Pod je GPU-VM (Embed+Rerank bzw. Chat), Host-Netz mit den Ports aus dem Compose, P2000 fest per UUID (NVIDIA_VISIBLE_DEVICES). Das Device Plugin vergibt nur noch Index 0 (M4000). Rolle gpu-llamacpp: gemeinsame Vorbereitung (prepare.yml), k3s.yml mit Image-Import und Compose-Stopp; gpu-llamacpp.yml bricht ohne -e llamacpp_compose=true ab. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
82 lines
3.9 KiB
YAML
82 lines
3.9 KiB
YAML
# llama.cpp-Rollen in den GPU-VMs (Task 6.1): Compose aus deploy/compose/gpu-llamacpp.
|
|
# Profile je VM kommen aus dem Inventar (Gruppe gpu_llamacpp, Variable llamacpp_profiles).
|
|
llamacpp_dir: /opt/knaster/gpu-llamacpp
|
|
llamacpp_models_dir: /srv/models
|
|
llamacpp_profiles: []
|
|
|
|
# Karte je Dienst, gewählt über den Namen aus `nvidia-smi` (ADR-0006: alles auf die P2000, M4000 bleibt frei).
|
|
# Passt Rerank nicht neben Embedding auf die P2000, hier "Quadro M4000" eintragen.
|
|
llamacpp_cards:
|
|
EMBED_GPU: Quadro P2000
|
|
RERANK_GPU: Quadro P2000
|
|
CHAT_GPU: Quadro P2000
|
|
|
|
# Dienste und Ports je Profil (feste Ports im Compose, Host-Netz)
|
|
llamacpp_services:
|
|
embed:
|
|
- {name: llamacpp-embed, port: 8081}
|
|
- {name: llamacpp-rerank, port: 8082}
|
|
chat:
|
|
- {name: llamacpp-chat, port: 8080}
|
|
|
|
# ufw: nur Regeln vormerken (Kommentar knaster-llamacpp), ufw nie ein- oder ausschalten (Firewall: Task 4.4)
|
|
llamacpp_allow_from: 192.168.100.0/24
|
|
|
|
# Werte für .env (keine Geheimnisse); Build-Stand wie im Benchmark (ADR-0006)
|
|
llamacpp_env:
|
|
LLAMACPP_REF: b6000
|
|
LLAMACPP_COMMIT: 4762ad7316dcdec20016ab5985fb46a27902204d
|
|
CUDA_VERSION: 12.4.0
|
|
CUDA_DOCKER_ARCH: "52;61"
|
|
MODELS_DIR: "{{ llamacpp_models_dir }}"
|
|
EMBED_GGUF: Qwen3-Embedding-0.6B-Q8_0.gguf
|
|
EMBED_POOLING: last
|
|
RERANK_GGUF: bge-reranker-v2-m3-Q8_0.gguf
|
|
CHAT_GGUF: Qwen3-4B-Q4_K_M.gguf
|
|
|
|
# Modelle je Profil mit fester Revision und Prüfsumme (sha256 = LFS-OID laut Hugging Face, ADR-0006)
|
|
_hf: https://huggingface.co
|
|
llamacpp_models:
|
|
embed:
|
|
- dest: Qwen3-Embedding-0.6B-Q8_0.gguf
|
|
url: "{{ _hf }}/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/370f27d7550e0def9b39c1f16d3fbaa13aa67728/Qwen3-Embedding-0.6B-Q8_0.gguf"
|
|
checksum: sha256:06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439
|
|
- dest: bge-reranker-v2-m3-Q8_0.gguf
|
|
url: "{{ _hf }}/gpustack/bge-reranker-v2-m3-GGUF/resolve/3093af03b1a635e67b084b1d8c03c5f5e020fd05/bge-reranker-v2-m3-Q8_0.gguf"
|
|
checksum: sha256:a43c7c9b11a4c1517e5bf95151960e1621d1b72f7a493364b01e386cf1aaa1d3
|
|
chat:
|
|
- dest: Qwen3-4B-Q4_K_M.gguf
|
|
url: "{{ _hf }}/Qwen/Qwen3-4B-GGUF/resolve/bc640142c66e1fdd12af0bd68f40445458f3869b/Qwen3-4B-Q4_K_M.gguf"
|
|
checksum: sha256:7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5
|
|
|
|
# --- k3s (Task 6.1, D6): tasks/k3s.yml, Playbook playbooks/gpu-llamacpp-k3s.yml ---
|
|
llamacpp_image: "knaster/llamacpp-server:{{ llamacpp_env.LLAMACPP_REF }}-cu{{ llamacpp_env.CUDA_VERSION }}"
|
|
llamacpp_k3s_template: "{{ playbook_dir }}/../../../../deploy/k8s/llamacpp/llamacpp.yaml.j2"
|
|
llamacpp_k3s_namespace: "{{ playbook_dir }}/../../../../deploy/k8s/llamacpp/namespace.yaml"
|
|
# Argumente wie im Compose (deploy/compose/gpu-llamacpp); gpu_var wählt die Karte aus llamacpp_cards
|
|
llamacpp_k3s_services:
|
|
embed:
|
|
- name: llamacpp-embed
|
|
port: 8081
|
|
gpu_var: EMBED_GPU
|
|
memory_limit: 3Gi
|
|
args: ["-m", "/models/{{ llamacpp_env.EMBED_GGUF }}", "--embedding", "--pooling", "{{ llamacpp_env.EMBED_POOLING }}",
|
|
"-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", "-ngl", "99", "--alias", "embed-default",
|
|
"--metrics", "--no-slots", "--no-webui", "--host", "0.0.0.0", "--port", "8081"]
|
|
- name: llamacpp-rerank
|
|
port: 8082
|
|
gpu_var: RERANK_GPU
|
|
memory_limit: 3Gi
|
|
args: ["-m", "/models/{{ llamacpp_env.RERANK_GGUF }}", "--reranking",
|
|
"-c", "8192", "-b", "8192", "-ub", "8192", "-np", "1", "-ngl", "99", "--alias", "rerank-default",
|
|
"--metrics", "--no-slots", "--no-webui", "--host", "0.0.0.0", "--port", "8082"]
|
|
chat:
|
|
- name: llamacpp-chat
|
|
port: 8080
|
|
gpu_var: CHAT_GPU
|
|
memory_limit: 6Gi
|
|
args: ["-m", "/models/{{ llamacpp_env.CHAT_GGUF }}", "-c", "8192", "-np", "1", "-ngl", "99",
|
|
"--jinja", "--chat-template-kwargs", "{\"enable_thinking\": false}",
|
|
"--alias", "verifier-small", "--metrics", "--no-slots", "--no-webui",
|
|
"--host", "0.0.0.0", "--port", "8080"]
|