63 lines
2.1 KiB
YAML
63 lines
2.1 KiB
YAML
# Rolle "gpu-main" / "gpu-flex" (VM mit durchgereichter RTX 4000 Ada): vLLM mit Multi-LoRA.
|
|
name: knaster-gpu-vllm
|
|
|
|
services:
|
|
vllm:
|
|
image: vllm/vllm-openai:${VLLM_TAG:-v0.10.0} # Engine-Updates laufen als Lifecycle-Kandidat
|
|
container_name: vllm
|
|
restart: unless-stopped
|
|
labels: {autoheal: "true"}
|
|
ipc: host
|
|
env_file: .env
|
|
environment:
|
|
VLLM_ALLOW_RUNTIME_LORA_UPDATING: "True" # Adapter per /v1/load_lora_adapter nachladen
|
|
command:
|
|
- --model=${VLLM_MODEL:?VLLM_MODEL fehlt}
|
|
- --served-model-name=${SLOT_ALIAS:?SLOT_ALIAS fehlt}
|
|
- --max-model-len=${MAX_MODEL_LEN:-32768}
|
|
- --gpu-memory-utilization=${GPU_MEM_UTIL:-0.90}
|
|
- --enable-lora
|
|
- --max-loras=${MAX_LORAS:-4}
|
|
- --max-lora-rank=${MAX_LORA_RANK:-64}
|
|
- --host=0.0.0.0
|
|
- --port=8000
|
|
volumes:
|
|
- ${MODELS_DIR:-/srv/models}:/root/.cache/huggingface
|
|
- ${ADAPTERS_DIR:-/srv/adapters}:/adapters:ro
|
|
ports: ["8000:8000"]
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices: [{driver: nvidia, count: 1, capabilities: [gpu]}]
|
|
healthcheck:
|
|
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=4)"]
|
|
interval: 30s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 300s # Laden großer Modelle
|
|
|
|
gpu-exporter:
|
|
image: utkuozdemir/nvidia_gpu_exporter:1.2.1
|
|
restart: unless-stopped
|
|
ports: ["9835:9835"]
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}]
|
|
|
|
node-exporter:
|
|
image: prom/node-exporter:v1.8.1
|
|
restart: unless-stopped
|
|
command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"]
|
|
pid: host
|
|
volumes:
|
|
- /:/host:ro,rslave
|
|
- /var/lib/node_exporter/textfile:/textfile:ro
|
|
ports: ["9100:9100"]
|
|
|
|
autoheal:
|
|
image: willfarrell/autoheal:1.2.0
|
|
restart: unless-stopped
|
|
environment: {AUTOHEAL_CONTAINER_LABEL: autoheal, AUTOHEAL_START_PERIOD: "300"}
|
|
volumes: ["/var/run/docker.sock:/var/run/docker.sock"]
|