87 lines
3.0 KiB
YAML
87 lines
3.0 KiB
YAML
# Rollen "gpu-embed" und "gpu-small" (VMs mit Quadro P2000 / M4000): llama.cpp-Server.
|
||
# Maxwell (sm_52) und Pascal (sm_61) brauchen CUDA 12.x – daher eigener Build mit festen Architekturen.
|
||
# Profile: embed (Embedding + Rerank), chat (Verifier/Signatur), z. B. `docker compose --profile embed up -d`
|
||
name: knaster-gpu-llamacpp
|
||
|
||
x-llamacpp: &llamacpp
|
||
build:
|
||
context: https://github.com/ggml-org/llama.cpp.git#${LLAMACPP_REF:-b6000}
|
||
dockerfile: .devops/cuda.Dockerfile
|
||
target: server
|
||
args:
|
||
CUDA_VERSION: ${CUDA_VERSION:-12.4.0}
|
||
CUDA_DOCKER_ARCH: ${CUDA_DOCKER_ARCH:-52;61}
|
||
image: knaster/llamacpp-server:${LLAMACPP_REF:-b6000}-cu12
|
||
restart: unless-stopped
|
||
labels: {autoheal: "true"}
|
||
volumes:
|
||
- ${MODELS_DIR:-/srv/models}:/models:ro
|
||
deploy:
|
||
resources:
|
||
reservations:
|
||
devices: [{driver: nvidia, count: 1, capabilities: [gpu]}]
|
||
|
||
services:
|
||
llamacpp-embed:
|
||
<<: *llamacpp
|
||
container_name: llamacpp-embed
|
||
profiles: ["embed"]
|
||
command: ["-m", "/models/${EMBED_GGUF:-Qwen3-Embedding-0.6B-Q8_0.gguf}", "--embedding",
|
||
"--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-ub", "8192", "-ngl", "99",
|
||
"--alias", "embed-default", "--metrics", "--host", "0.0.0.0", "--port", "8081"]
|
||
ports: ["8081:8081"]
|
||
healthcheck: &hc
|
||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"]
|
||
interval: 30s
|
||
timeout: 5s
|
||
retries: 3
|
||
start_period: 60s
|
||
|
||
llamacpp-rerank:
|
||
<<: *llamacpp
|
||
container_name: llamacpp-rerank
|
||
profiles: ["embed"]
|
||
command: ["-m", "/models/${RERANK_GGUF:-bge-reranker-v2-m3-Q8_0.gguf}", "--reranking", "-c", "8192",
|
||
"-ub", "8192", "-ngl", "99", "--alias", "rerank-default", "--metrics", "--host", "0.0.0.0",
|
||
"--port", "8082"]
|
||
ports: ["8082:8082"]
|
||
healthcheck:
|
||
<<: *hc
|
||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8082/health"]
|
||
|
||
llamacpp-chat:
|
||
<<: *llamacpp
|
||
container_name: llamacpp-chat
|
||
profiles: ["chat"]
|
||
command: ["-m", "/models/${CHAT_GGUF:-Qwen3-4B-Q4_K_M.gguf}", "-c", "8192", "-ngl", "99",
|
||
"--alias", "verifier-small", "--metrics", "--host", "0.0.0.0", "--port", "8080"]
|
||
ports: ["8080:8080"]
|
||
healthcheck:
|
||
<<: *hc
|
||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health"]
|
||
|
||
gpu-exporter:
|
||
image: utkuozdemir/nvidia_gpu_exporter:1.2.1
|
||
restart: unless-stopped
|
||
ports: ["9835:9835"]
|
||
deploy:
|
||
resources:
|
||
reservations:
|
||
devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}]
|
||
|
||
node-exporter:
|
||
image: prom/node-exporter:v1.8.1
|
||
restart: unless-stopped
|
||
command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"]
|
||
pid: host
|
||
volumes:
|
||
- /:/host:ro,rslave
|
||
- /var/lib/node_exporter/textfile:/textfile:ro
|
||
ports: ["9100:9100"]
|
||
|
||
autoheal:
|
||
image: willfarrell/autoheal:1.2.0
|
||
restart: unless-stopped
|
||
environment: {AUTOHEAL_CONTAINER_LABEL: autoheal}
|
||
volumes: ["/var/run/docker.sock:/var/run/docker.sock"]
|