platform: OpenTofu/Ansible für Proxmox, CI, Makefile, Doku-Abgleich
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,24 @@
|
||||
name: ci
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: make dev
|
||||
- run: make lint
|
||||
- run: make test
|
||||
- run: make validate
|
||||
|
||||
compose:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # braucht Docker Compose v2.24+ auf dem Runner
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: make compose-check
|
||||
@@ -0,0 +1,5 @@
|
||||
extends: relaxed
|
||||
rules:
|
||||
colons: disable # bewusst ausgerichtete Tabellen-Spalten
|
||||
commas: disable
|
||||
line-length: {max: 220, level: warning}
|
||||
@@ -0,0 +1,30 @@
|
||||
VENV := .venv
|
||||
PY := $(VENV)/bin/python
|
||||
|
||||
.PHONY: dev test lint validate compose-check all
|
||||
|
||||
dev:
|
||||
python3 -m venv $(VENV)
|
||||
$(VENV)/bin/pip install -q --upgrade pip
|
||||
$(VENV)/bin/pip install -q -e "core[dev]"
|
||||
|
||||
test:
|
||||
$(VENV)/bin/pytest core
|
||||
|
||||
lint:
|
||||
$(VENV)/bin/ruff check core
|
||||
$(VENV)/bin/yamllint -s -c .yamllint.yml config catalog sites deploy/prometheus deploy/alertmanager deploy/litellm deploy/compose platform
|
||||
|
||||
validate:
|
||||
$(VENV)/bin/knaster validate --root .
|
||||
$(VENV)/bin/knaster placement --inventory sites/knaster/inventory.yaml --catalog catalog > /dev/null
|
||||
|
||||
compose-check:
|
||||
@for d in infra gpu-vllm gpu-llamacpp; do \
|
||||
(cd deploy/compose/$$d && cp .env.example .env.ci && \
|
||||
POSTGRES_PASSWORD=ci VLLM_MODEL=ci SLOT_ALIAS=ci docker compose --env-file .env.ci \
|
||||
--profile embed --profile chat --profile dashboards --profile proxmox config -q; \
|
||||
rc=$$?; rm -f .env.ci; exit $$rc) || exit 1; \
|
||||
done; echo "compose ok"
|
||||
|
||||
all: lint test validate compose-check
|
||||
@@ -153,3 +153,29 @@ def test_llm_and_embedding_clients():
|
||||
chat.chat("overflow", [])
|
||||
with pytest.raises(EndpointError):
|
||||
chat.chat("down", [])
|
||||
|
||||
|
||||
def test_proxmox_site_consistent_with_registry_and_prometheus():
|
||||
site = yaml.safe_load((REPO_ROOT / "sites/knaster/proxmox.yaml").read_text())
|
||||
reg = yaml.safe_load((REPO_ROOT / "sites/knaster/registry.yaml").read_text())
|
||||
vms = site["vms"]
|
||||
for vm in vms.values():
|
||||
assert vm["gpu"] in site["pci_mappings"]
|
||||
assert site["pci_mappings"][vm["gpu"]]["node"] == vm["node"] # Mapping liegt auf dem Host der VM
|
||||
assert vm["driver_generation"] in {"legacy", "modern"}
|
||||
gpus = [vm["gpu"] for vm in vms.values()]
|
||||
assert len(gpus) == len(set(gpus)) # keine GPU doppelt durchgereicht
|
||||
by_vmid = {str(vm["vmid"]): (name, vm) for name, vm in vms.items()}
|
||||
for slot, spec in reg["slots"].items():
|
||||
ops = spec.get("ops")
|
||||
if not ops:
|
||||
continue
|
||||
name, vm = by_vmid[ops["vmid"]]
|
||||
assert vm["node"] == ops["pve_node"], slot
|
||||
assert ops["docker_host"].startswith(name.split("-")[0]), slot
|
||||
vmids = [str(v["vmid"]) for v in vms.values()] + [str(c["vmid"]) for c in site["containers"].values()]
|
||||
assert len(vmids) == len(set(vmids))
|
||||
# vLLM nur auf modernen Karten
|
||||
for vm in vms.values():
|
||||
if vm["role"] == "gpu-vllm":
|
||||
assert vm["driver_generation"] == "modern"
|
||||
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
restart: unless-stopped
|
||||
labels: {autoheal: "true"}
|
||||
ipc: host
|
||||
env_file: .env
|
||||
env_file: [{path: .env, required: false}]
|
||||
environment:
|
||||
VLLM_ALLOW_RUNTIME_LORA_UPDATING: "True" # Adapter per /v1/load_lora_adapter nachladen
|
||||
command:
|
||||
|
||||
@@ -13,7 +13,7 @@ services:
|
||||
context: ../../..
|
||||
dockerfile: deploy/docker/router.Dockerfile
|
||||
image: knaster/router:${KNASTER_VERSION:-dev}
|
||||
env_file: .env
|
||||
env_file: [{path: .env, required: false}]
|
||||
environment:
|
||||
KNASTER_DB_URL: postgresql+psycopg://knaster:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}@postgres:5432/knaster
|
||||
KNASTER_SITE_DIR: /app/sites/${KNASTER_SITE:-knaster}
|
||||
@@ -27,7 +27,7 @@ services:
|
||||
<<: *restart
|
||||
image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable} # vor Deployment auf feste Version pinnen
|
||||
command: ["--config", "/app/config.yaml", "--port", "4000"]
|
||||
env_file: .env
|
||||
env_file: [{path: .env, required: false}]
|
||||
volumes:
|
||||
- ../../litellm/config.yaml:/app/config.yaml:ro
|
||||
ports: ["4000:4000"]
|
||||
|
||||
+21
-2
@@ -15,8 +15,8 @@ Vorgehen pro Baustein: **Spec → Implementierung → Tests → Commit**. Bis zu
|
||||
| 7 | Portabilität | SPEC-inventory | Inventar-Parser, Katalog, Placement, CLI | ✅ |
|
||||
| 8 | API | SPEC-core §8 | FastAPI-App, Auth, Metriken | ✅ |
|
||||
| 9 | Deployment-Artefakte | PLAN §1, §3 | Compose pro Rolle, LiteLLM, Prometheus-Regeln, Alertmanager, n8n-Workflows | ✅ |
|
||||
| 10 | Plattform | ADR-0001/0002 | OpenTofu (bpg/proxmox), Ansible-Rollen (Treiber-Pinning, Docker) | ✅ (ungetestet gegen echten Cluster) |
|
||||
| 11 | CI | – | Gitea Actions: Tests, Lint, Compose-Validierung | ✅ |
|
||||
| 10 | Plattform | ADR-0001/0002 | OpenTofu (bpg/proxmox), Ansible-Rollen (IOMMU/VFIO, ZFS-Replikation, Treiber-Pinning, Xid-Exporter) | ✅ geschrieben, **nicht** gegen echten Cluster/`tofu validate` geprüft |
|
||||
| 11 | CI | – | Gitea Actions: Tests, Lint, Konfig-Validierung, Compose-Prüfung | ✅ |
|
||||
|
||||
## Teststrategie
|
||||
|
||||
@@ -25,6 +25,25 @@ Vorgehen pro Baustein: **Spec → Implementierung → Tests → Commit**. Bis zu
|
||||
- **Artefakt-Tests:** Compose-Dateien, Prometheus-Regeln, LiteLLM-Konfiguration und n8n-Workflows werden strukturell geprüft (YAML/JSON gültig, referenzierte Knoten vorhanden, Slots konsistent zur Registry).
|
||||
- **Nicht getestet bis zum gemeinsamen Deployment:** echte Modelle, Proxmox-API, Portainer-Aktionen, OpenTofu-Apply.
|
||||
|
||||
## Stand der Verifikation
|
||||
|
||||
| Was | Wie geprüft |
|
||||
|---|---|
|
||||
| Kernlogik (Router, Eskalation, Verifier, Pflege-Agent, Healer, Lifecycle, Placement) | 127 Unit- und Integrationstests (`make test`) |
|
||||
| Konfiguration (Registry ↔ Skills ↔ LiteLLM ↔ Healer ↔ Policy ↔ Katalog) | `knaster validate` |
|
||||
| Compose-Dateien | `docker compose config` (`make compose-check`) |
|
||||
| Prometheus/Alertmanager/n8n-JSON | Strukturtests (`core/tests/test_artifacts.py`) |
|
||||
| n8n, Portainer | nur lesender Erreichbarkeitstest, nichts deployt |
|
||||
| OpenTofu, Ansible | nur YAML/Syntax. `tofu validate` und `ansible-lint` stehen noch aus |
|
||||
|
||||
## Offene Punkte vor dem Deployment
|
||||
|
||||
- Image-Tags pinnen: LiteLLM (`main-stable`) und pve-exporter (`latest`). vLLM-, llama.cpp- und Treiber-Versionen gegen den aktuellen Stand prüfen.
|
||||
- LiteLLM-Wildcard-Routen (`tier1-general/*`) gegen die eingesetzte Version testen.
|
||||
- Den n8n Chat-Trigger nach dem Import prüfen (Parameter der Knotenversion).
|
||||
- `sites/knaster/*`: Hosts, VM-IDs, IPs und PCI-Adressen sind Annahmen und werden durch den Inventarlauf ersetzt.
|
||||
- Classifier `ticket-kategorie.json` entsteht erst aus gelabelten Fällen (Warnung von `knaster validate`).
|
||||
|
||||
## Gemeinsames Deployment (später)
|
||||
|
||||
1. Inventarlauf auf allen Nodes → `sites/knaster/inventory.yaml` ergänzen → `knaster placement plan`
|
||||
|
||||
@@ -75,7 +75,8 @@ Ablauf:
|
||||
7. **Stufe 3/2:** Classifier bzw. Programm, ohne Slot.
|
||||
8. **Stufe 1:** Slot des Skills, falls verfügbar, sonst dessen `fallbacks`. Ist keiner verfügbar, wird mit `slot_unavailable` auf Stufe 0 geroutet.
|
||||
9. **Token-Budget** (§4): erst `full` versuchen, dann `compact`, dann die Fallback-Slots mit größerem Kontext. Passt nichts, wird mit `context_exceeded` auf Stufe 0 geroutet. Passt es auch auf Stufe 0 nicht, ist das Ergebnis ein Fehler `too_large` mit dem Signal `needs_split`.
|
||||
10. **Adapter:** Wenn `slot.supports_lora`, `skill.adapter == auto` und ein aktiver Adapter für `(slot.base_model, skill.cluster)` existiert, wird `model = adapter.name` gesetzt, sonst `model = slot.litellm_model`.
|
||||
10. **Adapter:** Wenn `slot.supports_lora`, `skill.adapter == auto` und ein aktiver Adapter für `(slot.base_model, skill.cluster)` existiert, wird `model = "<slot.litellm_model>/<adapter.name>"` gesetzt (LiteLLM-Wildcard-Route auf den vLLM des Slots), sonst `model = slot.litellm_model`.
|
||||
11. **Canary:** Hat der Slot im Laufzeitzustand ein `canary_model` (vom Lifecycle gesetzt), geht der Anteil `canary_share` der passenden Anfragen auf den Kandidaten. Bei Adapter-Kandidaten gilt das nur für den Cluster des Kandidaten. Die Entscheidung trägt dann `canary: true`.
|
||||
|
||||
`escalate(decision, reason)`:
|
||||
- Bei Infrastruktur-Fehlern (`endpoint_error`, `context_overflow`) wird auf derselben Stufe der nächste Fallback-Slot genommen, sonst wird eine Stufe höher gegangen.
|
||||
@@ -135,5 +136,11 @@ Ergebnis ist `VerifyResult{passed, checks[{type, passed, detail}]}`. Bei Format
|
||||
| GET/POST | `/v1/lifecycle/candidates` | Kandidaten (SPEC-lifecycle) |
|
||||
| POST | `/v1/lifecycle/candidates/{id}/report` | Ergebnis eines Jobs melden |
|
||||
| POST | `/v1/skills/{name}/graduate` | Graduierungslauf mit mitgelieferten Varianten |
|
||||
| POST | `/v1/skills/reload` | Skill-Dateien neu laden (nach Git-Pull) |
|
||||
| POST | `/v1/maintenance/proposals/{seq}/approve` bzw. `/reject` | Freigabe per Knopf (n8n/UI) statt per Prompt |
|
||||
|
||||
## 9. Canary-Prober (`knaster.canary`)
|
||||
|
||||
Läuft im Service-Prozess (`KNASTER_CANARY_INTERVAL`, Standard 60 s, 0 = aus). Er prüft jeden Slot außer Stufe 0 und Rerank mit einer Mini-Anfrage („Antworte nur mit OK“) bzw. einem Embedding. Daraus entstehen die Metriken `knaster_slot_endpoint_up`, `knaster_slot_canary_ok` und `knaster_slot_canary_latency_seconds`, jeweils mit den Labels `slot, node, docker_host, container, pve_node, vmid` aus `ops` der Registry. Daraus leiten sich die Alerts `SlotEndpointDown` und `SlotCanaryFailing` für den Healer ab.
|
||||
|
||||
Authentifizierung: Header `Authorization: Bearer <KNASTER_API_TOKEN>` auf allen `/v1/*`-Pfaden.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# platform/proxmox
|
||||
|
||||
Proxmox-spezifische Schicht (austauschbar, siehe PLAN §7.2). Eingabe ist immer `sites/<site>/proxmox.yaml`.
|
||||
|
||||
**Status:** geschrieben, aber noch **nicht gegen den echten Cluster getestet**. Erster Lauf beim gemeinsamen Deployment, jeweils mit `tofu plan` bzw. `ansible-playbook --check`.
|
||||
|
||||
## Ablauf
|
||||
|
||||
1. **Inventar:** auf jedem Node `knaster inventory collect` ausführen und `sites/knaster/inventory.yaml` ersetzen. Danach `knaster placement plan --inventory sites/knaster/inventory.yaml` und die Ergebnisse in `proxmox.yaml` übertragen (PCI-Pfade über `lspci -nn | grep -i nvidia`, IOMMU-Gruppen).
|
||||
2. **Hosts vorbereiten:** `ansible-playbook playbooks/site.yml --limit pve_hosts`. Setzt IOMMU und VFIO und sperrt die NVIDIA-Treiber auf dem Host. Danach die Hosts neu starten. Der ZFS-Pool `tank` wird **nicht** automatisch angelegt (Datenverlustgefahr), die Rolle gibt dafür den Befehl aus.
|
||||
3. **Template:** pro Node ein Ubuntu-24.04-Cloud-Image als Template mit VMID 9000.
|
||||
4. **VMs/LXC:** `cd tofu && tofu init && tofu plan -var proxmox_api_token=...` und danach `tofu apply`.
|
||||
5. **Gäste:** `ansible-playbook playbooks/site.yml --limit gpu_vms,infra`. Installiert die Treiber pro Generation gepinnt (legacy/modern), das Container-Toolkit, Docker und den Xid-Exporter.
|
||||
6. **Dienste:** `deploy/compose/<rolle>` auf den jeweiligen Gästen starten.
|
||||
|
||||
## Bewusste Grenzen
|
||||
|
||||
- Keine Live-Migration von GPU-VMs. HA gilt nur für den Infra-LXC (mit ZFS-Replikation alle 15 Min.).
|
||||
- Der Proxmox-API-Token für den Healer braucht nur `VM.PowerMgmt` auf den GPU-VMs und `Sys.PowerMgmt` (Host-Neustart, nur mit Freigabe).
|
||||
@@ -0,0 +1,5 @@
|
||||
[defaults]
|
||||
inventory = inventory.yml
|
||||
roles_path = roles
|
||||
host_key_checking = True
|
||||
stdout_callback = yaml
|
||||
@@ -0,0 +1,29 @@
|
||||
# ANNAHME bis Deployment – Hosts und IPs aus sites/knaster/proxmox.yaml
|
||||
all:
|
||||
vars:
|
||||
knaster_site: knaster
|
||||
children:
|
||||
pve_hosts:
|
||||
hosts:
|
||||
pve-ada: {}
|
||||
pve1: {}
|
||||
pve2: {}
|
||||
pve3: {}
|
||||
pve4: {}
|
||||
vars:
|
||||
ansible_user: root
|
||||
gpu_vms:
|
||||
hosts:
|
||||
gpu-main: {ansible_host: 10.20.0.201, driver_generation: modern, knaster_role: gpu-vllm}
|
||||
gpu-flex: {ansible_host: 10.20.0.202, driver_generation: modern, knaster_role: gpu-vllm}
|
||||
gpu-embed-1: {ansible_host: 10.20.0.221, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: embed}
|
||||
gpu-embed-2: {ansible_host: 10.20.0.222, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: embed}
|
||||
gpu-small: {ansible_host: 10.20.0.231, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: chat}
|
||||
vars:
|
||||
ansible_user: knaster
|
||||
ansible_become: true
|
||||
infra:
|
||||
hosts:
|
||||
infra: {ansible_host: 10.20.0.101}
|
||||
vars:
|
||||
ansible_user: root
|
||||
@@ -0,0 +1,12 @@
|
||||
# Reihenfolge: Hosts vorbereiten → (tofu apply) → GPU-VMs → Infra
|
||||
- name: Proxmox-Hosts (IOMMU, VFIO, ZFS-Replikation)
|
||||
hosts: pve_hosts
|
||||
roles: [pve-host]
|
||||
|
||||
- name: GPU-VMs (Treiber gepinnt, Container-Toolkit, Docker, Xid-Exporter)
|
||||
hosts: gpu_vms
|
||||
roles: [docker-host, gpu-node]
|
||||
|
||||
- name: Infra-LXC (Docker)
|
||||
hosts: infra
|
||||
roles: [docker-host]
|
||||
@@ -0,0 +1,3 @@
|
||||
collections:
|
||||
- name: ansible.posix
|
||||
- name: community.general
|
||||
@@ -0,0 +1,11 @@
|
||||
- name: Docker aus Distribution installieren
|
||||
ansible.builtin.apt:
|
||||
name: [docker.io, docker-compose-v2]
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Docker aktivieren
|
||||
ansible.builtin.service:
|
||||
name: docker
|
||||
state: started
|
||||
enabled: true
|
||||
@@ -0,0 +1,8 @@
|
||||
# Treiber pro GPU-Generation fest pinnen (ADR-0001).
|
||||
# legacy (Maxwell/Pascal): letzte Treiber-Branch mit Unterstützung; Userland CUDA 12.x
|
||||
# Paketnamen (nvidia-driver-<branch>-server) beim Deployment gegen das Gast-OS prüfen.
|
||||
# modern (Ada): eigene Branch, unabhängig aktualisierbar (Update = Snapshot + Lifecycle-Kandidat "engine")
|
||||
nvidia_driver_branch:
|
||||
legacy: "580"
|
||||
modern: "580"
|
||||
nvidia_textfile_dir: /var/lib/node_exporter/textfile
|
||||
@@ -0,0 +1,12 @@
|
||||
- name: configure nvidia runtime
|
||||
ansible.builtin.command: nvidia-ctk runtime configure --runtime=docker
|
||||
notify: restart docker
|
||||
|
||||
- name: restart docker
|
||||
ansible.builtin.service:
|
||||
name: docker
|
||||
state: restarted
|
||||
|
||||
- name: reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
@@ -0,0 +1,84 @@
|
||||
- name: Treiber-Branch bestimmen
|
||||
ansible.builtin.set_fact:
|
||||
_branch: "{{ nvidia_driver_branch[driver_generation] }}"
|
||||
|
||||
- name: NVIDIA-Treiber (Server-Variante) installieren
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- "nvidia-driver-{{ _branch }}-server"
|
||||
- "nvidia-utils-{{ _branch }}-server"
|
||||
state: present
|
||||
update_cache: true
|
||||
register: driver
|
||||
|
||||
- name: Treiber gegen ungewollte Updates sperren
|
||||
ansible.builtin.dpkg_selections:
|
||||
name: "{{ item }}"
|
||||
selection: hold
|
||||
loop:
|
||||
- "nvidia-driver-{{ _branch }}-server"
|
||||
- "nvidia-utils-{{ _branch }}-server"
|
||||
|
||||
- name: NVIDIA-Container-Toolkit-Repository
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
|
||||
curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
|
||||
| sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
|
||||
> /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||
args:
|
||||
creates: /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||
executable: /bin/bash
|
||||
|
||||
- name: Container-Toolkit installieren und für Docker konfigurieren
|
||||
ansible.builtin.apt:
|
||||
name: nvidia-container-toolkit
|
||||
state: present
|
||||
update_cache: true
|
||||
notify: configure nvidia runtime
|
||||
|
||||
- name: Modell-Disk (scsi1) formatieren, falls leer (formatiert nie eine vorhandene Partition um)
|
||||
community.general.filesystem:
|
||||
dev: /dev/sdb
|
||||
fstype: ext4
|
||||
opts: -L models
|
||||
when: ansible_devices.sdb is defined
|
||||
|
||||
- name: Modell-Disk einhängen
|
||||
ansible.posix.mount:
|
||||
path: /srv/models
|
||||
src: LABEL=models
|
||||
fstype: ext4
|
||||
state: mounted
|
||||
when: ansible_devices.sdb is defined
|
||||
|
||||
- name: Textfile-Verzeichnis für node_exporter
|
||||
ansible.builtin.file:
|
||||
path: "{{ nvidia_textfile_dir }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Xid-Exporter-Skript
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../../../../deploy/scripts/gpu-xid-textfile.sh"
|
||||
dest: /usr/local/bin/gpu-xid-textfile.sh
|
||||
mode: "0755"
|
||||
|
||||
- name: systemd-Service und -Timer für Xid-Exporter
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: "0644"
|
||||
loop: [gpu-xid-textfile.service, gpu-xid-textfile.timer]
|
||||
notify: reload systemd
|
||||
|
||||
- name: Timer aktivieren
|
||||
ansible.builtin.systemd:
|
||||
name: gpu-xid-textfile.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
|
||||
- name: Neustart nach Treiberinstallation
|
||||
ansible.builtin.reboot:
|
||||
when: driver.changed
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=NVIDIA-Xid-Fehler für node_exporter zählen
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/gpu-xid-textfile.sh {{ nvidia_textfile_dir }}
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=NVIDIA-Xid-Fehler minütlich zählen
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=1min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,4 @@
|
||||
pve_iommu_cmdline: "intel_iommu=on iommu=pt"
|
||||
pve_vfio_modules: [vfio, vfio_iommu_type1, vfio_pci]
|
||||
# ZFS-Replikation der Infra-Gäste (ADR-0002), gefüllt aus sites/<site>/proxmox.yaml
|
||||
pve_site_file: "{{ playbook_dir }}/../../../../sites/{{ knaster_site }}/proxmox.yaml"
|
||||
@@ -0,0 +1,3 @@
|
||||
- name: refresh boot
|
||||
ansible.builtin.shell: proxmox-boot-tool refresh || update-grub
|
||||
changed_when: true
|
||||
@@ -0,0 +1,70 @@
|
||||
- name: Site-Datei laden
|
||||
ansible.builtin.set_fact:
|
||||
knaster_pve: "{{ lookup('ansible.builtin.file', pve_site_file) | from_yaml }}"
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Bootloader erkennen (systemd-boot bei ZFS-Root, sonst GRUB)
|
||||
ansible.builtin.stat:
|
||||
path: /etc/kernel/cmdline
|
||||
register: kernel_cmdline
|
||||
|
||||
- name: IOMMU aktivieren (systemd-boot)
|
||||
ansible.builtin.replace:
|
||||
path: /etc/kernel/cmdline
|
||||
regexp: '^((?!.*intel_iommu=on).*)$'
|
||||
replace: '\1 {{ pve_iommu_cmdline }}'
|
||||
when: kernel_cmdline.stat.exists
|
||||
notify: refresh boot
|
||||
|
||||
- name: IOMMU aktivieren (GRUB)
|
||||
ansible.builtin.lineinfile:
|
||||
path: /etc/default/grub
|
||||
regexp: '^GRUB_CMDLINE_LINUX_DEFAULT="((?!.*intel_iommu=on).*)"$'
|
||||
line: 'GRUB_CMDLINE_LINUX_DEFAULT="\1 {{ pve_iommu_cmdline }}"'
|
||||
backrefs: true
|
||||
when: not kernel_cmdline.stat.exists
|
||||
notify: refresh boot
|
||||
|
||||
- name: VFIO-Module laden
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/modules-load.d/vfio.conf
|
||||
content: "{{ pve_vfio_modules | join('\n') }}\n"
|
||||
mode: "0644"
|
||||
|
||||
- name: NVIDIA-Treiber auf dem Host sperren (GPU gehört der VM)
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/modprobe.d/blacklist-nvidia.conf
|
||||
content: |
|
||||
blacklist nouveau
|
||||
blacklist nvidia
|
||||
blacklist nvidiafb
|
||||
mode: "0644"
|
||||
notify: refresh boot
|
||||
|
||||
- name: ZFS-Pool tank vorhanden?
|
||||
ansible.builtin.command: zpool list -H -o name tank
|
||||
register: tank
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Hinweis, falls tank fehlt (Pool wird bewusst NICHT automatisch angelegt – Datenverlustgefahr)
|
||||
ansible.builtin.debug:
|
||||
msg: "Auf {{ inventory_hostname }} fehlt der ZFS-Pool 'tank'. Anlegen z. B.: zpool create -o ashift=12 tank mirror /dev/disk/by-id/<ssd2> /dev/disk/by-id/<ssd3> && pvesm add zfspool tank -pool tank"
|
||||
when: tank.rc != 0
|
||||
|
||||
- name: Bestehende Replikationsjobs lesen
|
||||
ansible.builtin.command: pvesr list
|
||||
register: pvesr_jobs
|
||||
changed_when: false
|
||||
|
||||
- name: ZFS-Replikation für Infra-Gäste
|
||||
ansible.builtin.command: >
|
||||
pvesr create-local-job {{ item.value.vmid }}-0 {{ item.value.replicate_to }}
|
||||
--schedule '{{ knaster_pve.replication_schedule }}' --comment 'knaster {{ item.key }}'
|
||||
loop: "{{ knaster_pve.containers | dict2items }}"
|
||||
when:
|
||||
- item.value.node == inventory_hostname
|
||||
- item.value.replicate_to is defined
|
||||
- (item.value.vmid | string) + '-0' not in pvesr_jobs.stdout
|
||||
changed_when: true
|
||||
@@ -0,0 +1,153 @@
|
||||
# GPU-Mappings, GPU-VMs und Infra-LXC aus sites/<site>/proxmox.yaml.
|
||||
# Nicht gegen einen echten Cluster getestet – beim gemeinsamen Deployment zuerst `tofu plan`.
|
||||
|
||||
resource "proxmox_virtual_environment_hardware_mapping_pci" "gpu" {
|
||||
for_each = local.site.pci_mappings
|
||||
name = each.key
|
||||
map = [{
|
||||
node = each.value.node
|
||||
path = each.value.path
|
||||
id = each.value.id
|
||||
subsystem_id = each.value.subsystem_id
|
||||
iommu_group = each.value.iommu_group
|
||||
}]
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_vm" "gpu" {
|
||||
for_each = local.site.vms
|
||||
name = each.key
|
||||
node_name = each.value.node
|
||||
vm_id = each.value.vmid
|
||||
tags = ["knaster", each.value.role, each.value.driver_generation]
|
||||
|
||||
machine = "q35"
|
||||
bios = "ovmf"
|
||||
on_boot = true
|
||||
|
||||
clone {
|
||||
vm_id = local.site.cluster.template_vmid
|
||||
full = true
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = each.value.cores
|
||||
type = "host"
|
||||
}
|
||||
|
||||
# PCI-Passthrough pinnt den gesamten RAM → kein Ballooning
|
||||
memory {
|
||||
dedicated = each.value.memory_mb
|
||||
floating = 0
|
||||
}
|
||||
|
||||
efi_disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
type = "4m"
|
||||
}
|
||||
|
||||
disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
interface = "scsi0"
|
||||
size = each.value.disk_gb
|
||||
discard = "on"
|
||||
ssd = true
|
||||
}
|
||||
|
||||
# Modelle: eigenes Disk-Image, nicht repliziert, nicht gesichert (ADR-0002)
|
||||
disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
interface = "scsi1"
|
||||
size = each.value.models_disk_gb
|
||||
discard = "on"
|
||||
ssd = true
|
||||
backup = false
|
||||
replicate = false
|
||||
}
|
||||
|
||||
hostpci {
|
||||
device = "hostpci0"
|
||||
mapping = proxmox_virtual_environment_hardware_mapping_pci.gpu[each.value.gpu].name
|
||||
pcie = true
|
||||
rombar = true
|
||||
}
|
||||
|
||||
network_device {
|
||||
bridge = local.site.cluster.bridge
|
||||
vlan_id = local.site.cluster.vlan_svc
|
||||
}
|
||||
|
||||
initialization {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = each.value.ip
|
||||
gateway = local.site.cluster.gateway
|
||||
}
|
||||
}
|
||||
dns {
|
||||
servers = [local.site.cluster.dns]
|
||||
}
|
||||
user_account {
|
||||
username = "knaster"
|
||||
keys = [local.ssh_key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_container" "infra" {
|
||||
for_each = local.site.containers
|
||||
node_name = each.value.node
|
||||
vm_id = each.value.vmid
|
||||
tags = ["knaster", "infra"]
|
||||
unprivileged = true
|
||||
start_on_boot = true
|
||||
|
||||
features {
|
||||
nesting = true # Docker im LXC
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = each.value.cores
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = each.value.memory_mb
|
||||
}
|
||||
|
||||
disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
size = each.value.disk_gb
|
||||
}
|
||||
|
||||
operating_system {
|
||||
template_file_id = "${local.site.cluster.storage_iso}:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst"
|
||||
type = "debian"
|
||||
}
|
||||
|
||||
network_interface {
|
||||
name = "eth0"
|
||||
bridge = local.site.cluster.bridge
|
||||
vlan_id = local.site.cluster.vlan_svc
|
||||
}
|
||||
|
||||
initialization {
|
||||
hostname = each.key
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = each.value.ip
|
||||
gateway = local.site.cluster.gateway
|
||||
}
|
||||
}
|
||||
user_account {
|
||||
keys = [local.ssh_key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_haresource" "infra" {
|
||||
for_each = { for k, v in local.site.containers : k => v if try(v.ha, false) }
|
||||
resource_id = "ct:${each.value.vmid}"
|
||||
state = "started"
|
||||
comment = "knaster ${each.key}"
|
||||
depends_on = [proxmox_virtual_environment_container.infra]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
output "gpu_vms" {
|
||||
value = { for k, v in proxmox_virtual_environment_vm.gpu : k => { node = v.node_name, vmid = v.vm_id } }
|
||||
}
|
||||
|
||||
output "infra_containers" {
|
||||
value = { for k, v in proxmox_virtual_environment_container.infra : k => { node = v.node_name, vmid = v.vm_id } }
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
variable "site" {
|
||||
description = "Name des Standorts unter sites/"
|
||||
type = string
|
||||
default = "knaster"
|
||||
}
|
||||
|
||||
variable "proxmox_api_token" {
|
||||
description = "API-Token im Format user@realm!tokenid=secret"
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "proxmox_insecure" {
|
||||
description = "Selbstsigniertes Zertifikat akzeptieren"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
locals {
|
||||
site = yamldecode(file("${path.module}/../../../sites/${var.site}/proxmox.yaml"))
|
||||
ssh_key = trimspace(file(pathexpand(local.site.cluster.ssh_public_key_file)))
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
terraform {
|
||||
required_version = ">= 1.6"
|
||||
required_providers {
|
||||
proxmox = {
|
||||
source = "bpg/proxmox"
|
||||
version = "~> 0.66"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "proxmox" {
|
||||
endpoint = local.site.cluster.endpoint
|
||||
api_token = var.proxmox_api_token
|
||||
insecure = var.proxmox_insecure
|
||||
ssh {
|
||||
agent = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
# Proxmox-Zielbild für "knaster" (Eingabe für platform/proxmox/tofu und ansible).
|
||||
# ANNAHMEN (Hosts, VM-IDs, IPs, PCI-Adressen) bis zum Inventarlauf – siehe docs/IMPLEMENTATION.md.
|
||||
cluster:
|
||||
endpoint: https://pve1:8006/
|
||||
storage_vm: tank # ZFS-Pool auf SSD 2+3 (ADR-0002)
|
||||
storage_iso: local
|
||||
template_vmid: 9000 # Ubuntu 24.04 Cloud-Image-Template (pro Node vorhanden)
|
||||
bridge: vmbr0
|
||||
vlan_svc: 20
|
||||
gateway: 10.20.0.1
|
||||
dns: 10.20.0.1
|
||||
ssh_public_key_file: ~/.ssh/id_ed25519.pub
|
||||
|
||||
pci_mappings: # Proxmox Resource Mappings (Datacenter → Resource Mappings → PCI)
|
||||
gpu-ada-1: {node: pve-ada, path: "0000:01:00", id: "10de:27b2", subsystem_id: "", iommu_group: 0}
|
||||
gpu-ada-2: {node: pve-ada, path: "0000:02:00", id: "10de:27b2", subsystem_id: "", iommu_group: 0}
|
||||
gpu-p2000-1: {node: pve3, path: "0000:01:00", id: "10de:1c30", subsystem_id: "", iommu_group: 0}
|
||||
gpu-p2000-2: {node: pve3, path: "0000:02:00", id: "10de:1c30", subsystem_id: "", iommu_group: 0}
|
||||
gpu-p2000-3: {node: pve4, path: "0000:01:00", id: "10de:1c30", subsystem_id: "", iommu_group: 0}
|
||||
|
||||
vms: # GPU-Rollen als VM (ADR-0001)
|
||||
gpu-main: {node: pve-ada, vmid: 201, cores: 6, memory_mb: 28672, disk_gb: 40, models_disk_gb: 100, gpu: gpu-ada-1, ip: 10.20.0.201/24, role: gpu-vllm, driver_generation: modern}
|
||||
gpu-flex: {node: pve-ada, vmid: 202, cores: 6, memory_mb: 28672, disk_gb: 40, models_disk_gb: 100, gpu: gpu-ada-2, ip: 10.20.0.202/24, role: gpu-vllm, driver_generation: modern}
|
||||
gpu-embed-1: {node: pve3, vmid: 221, cores: 2, memory_mb: 6144, disk_gb: 30, models_disk_gb: 20, gpu: gpu-p2000-1, ip: 10.20.0.221/24, role: gpu-llamacpp, profile: embed, driver_generation: legacy}
|
||||
gpu-embed-2: {node: pve4, vmid: 222, cores: 2, memory_mb: 6144, disk_gb: 30, models_disk_gb: 20, gpu: gpu-p2000-3, ip: 10.20.0.222/24, role: gpu-llamacpp, profile: embed, driver_generation: legacy}
|
||||
gpu-small: {node: pve3, vmid: 231, cores: 2, memory_mb: 8192, disk_gb: 30, models_disk_gb: 20, gpu: gpu-p2000-2, ip: 10.20.0.231/24, role: gpu-llamacpp, profile: chat, driver_generation: legacy}
|
||||
|
||||
containers: # Infrastruktur als LXC (ADR-0001), HA + ZFS-Replikation (ADR-0002)
|
||||
infra: {node: pve1, vmid: 101, cores: 4, memory_mb: 12288, disk_gb: 60, ip: 10.20.0.101/24, ha: true, replicate_to: pve2}
|
||||
|
||||
replication_schedule: "*/15"
|
||||
Reference in New Issue
Block a user