platform: OpenTofu/Ansible für Proxmox, CI, Makefile, Doku-Abgleich
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,19 @@
|
||||
# platform/proxmox
|
||||
|
||||
Proxmox-spezifische Schicht (austauschbar, siehe PLAN §7.2). Eingabe ist immer `sites/<site>/proxmox.yaml`.
|
||||
|
||||
**Status:** geschrieben, aber noch **nicht gegen den echten Cluster getestet**. Erster Lauf beim gemeinsamen Deployment, jeweils mit `tofu plan` bzw. `ansible-playbook --check`.
|
||||
|
||||
## Ablauf
|
||||
|
||||
1. **Inventar:** auf jedem Node `knaster inventory collect` ausführen und `sites/knaster/inventory.yaml` ersetzen. Danach `knaster placement plan --inventory sites/knaster/inventory.yaml` und die Ergebnisse in `proxmox.yaml` übertragen (PCI-Pfade über `lspci -nn | grep -i nvidia`, IOMMU-Gruppen).
|
||||
2. **Hosts vorbereiten:** `ansible-playbook playbooks/site.yml --limit pve_hosts`. Setzt IOMMU und VFIO und sperrt die NVIDIA-Treiber auf dem Host. Danach die Hosts neu starten. Der ZFS-Pool `tank` wird **nicht** automatisch angelegt (Datenverlustgefahr), die Rolle gibt dafür den Befehl aus.
|
||||
3. **Template:** pro Node ein Ubuntu-24.04-Cloud-Image als Template mit VMID 9000.
|
||||
4. **VMs/LXC:** `cd tofu && tofu init && tofu plan -var proxmox_api_token=...` und danach `tofu apply`.
|
||||
5. **Gäste:** `ansible-playbook playbooks/site.yml --limit gpu_vms,infra`. Installiert die Treiber pro Generation gepinnt (legacy/modern), das Container-Toolkit, Docker und den Xid-Exporter.
|
||||
6. **Dienste:** `deploy/compose/<rolle>` auf den jeweiligen Gästen starten.
|
||||
|
||||
## Bewusste Grenzen
|
||||
|
||||
- Keine Live-Migration von GPU-VMs. HA gilt nur für den Infra-LXC (mit ZFS-Replikation alle 15 Min.).
|
||||
- Der Proxmox-API-Token für den Healer braucht nur `VM.PowerMgmt` auf den GPU-VMs und `Sys.PowerMgmt` (Host-Neustart, nur mit Freigabe).
|
||||
@@ -0,0 +1,5 @@
|
||||
[defaults]
|
||||
inventory = inventory.yml
|
||||
roles_path = roles
|
||||
host_key_checking = True
|
||||
stdout_callback = yaml
|
||||
@@ -0,0 +1,29 @@
|
||||
# ANNAHME bis Deployment – Hosts und IPs aus sites/knaster/proxmox.yaml
|
||||
all:
|
||||
vars:
|
||||
knaster_site: knaster
|
||||
children:
|
||||
pve_hosts:
|
||||
hosts:
|
||||
pve-ada: {}
|
||||
pve1: {}
|
||||
pve2: {}
|
||||
pve3: {}
|
||||
pve4: {}
|
||||
vars:
|
||||
ansible_user: root
|
||||
gpu_vms:
|
||||
hosts:
|
||||
gpu-main: {ansible_host: 10.20.0.201, driver_generation: modern, knaster_role: gpu-vllm}
|
||||
gpu-flex: {ansible_host: 10.20.0.202, driver_generation: modern, knaster_role: gpu-vllm}
|
||||
gpu-embed-1: {ansible_host: 10.20.0.221, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: embed}
|
||||
gpu-embed-2: {ansible_host: 10.20.0.222, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: embed}
|
||||
gpu-small: {ansible_host: 10.20.0.231, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: chat}
|
||||
vars:
|
||||
ansible_user: knaster
|
||||
ansible_become: true
|
||||
infra:
|
||||
hosts:
|
||||
infra: {ansible_host: 10.20.0.101}
|
||||
vars:
|
||||
ansible_user: root
|
||||
@@ -0,0 +1,12 @@
|
||||
# Reihenfolge: Hosts vorbereiten → (tofu apply) → GPU-VMs → Infra
|
||||
- name: Proxmox-Hosts (IOMMU, VFIO, ZFS-Replikation)
|
||||
hosts: pve_hosts
|
||||
roles: [pve-host]
|
||||
|
||||
- name: GPU-VMs (Treiber gepinnt, Container-Toolkit, Docker, Xid-Exporter)
|
||||
hosts: gpu_vms
|
||||
roles: [docker-host, gpu-node]
|
||||
|
||||
- name: Infra-LXC (Docker)
|
||||
hosts: infra
|
||||
roles: [docker-host]
|
||||
@@ -0,0 +1,3 @@
|
||||
collections:
|
||||
- name: ansible.posix
|
||||
- name: community.general
|
||||
@@ -0,0 +1,11 @@
|
||||
- name: Docker aus Distribution installieren
|
||||
ansible.builtin.apt:
|
||||
name: [docker.io, docker-compose-v2]
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Docker aktivieren
|
||||
ansible.builtin.service:
|
||||
name: docker
|
||||
state: started
|
||||
enabled: true
|
||||
@@ -0,0 +1,8 @@
|
||||
# Treiber pro GPU-Generation fest pinnen (ADR-0001).
|
||||
# legacy (Maxwell/Pascal): letzte Treiber-Branch mit Unterstützung; Userland CUDA 12.x
|
||||
# Paketnamen (nvidia-driver-<branch>-server) beim Deployment gegen das Gast-OS prüfen.
|
||||
# modern (Ada): eigene Branch, unabhängig aktualisierbar (Update = Snapshot + Lifecycle-Kandidat "engine")
|
||||
nvidia_driver_branch:
|
||||
legacy: "580"
|
||||
modern: "580"
|
||||
nvidia_textfile_dir: /var/lib/node_exporter/textfile
|
||||
@@ -0,0 +1,12 @@
|
||||
- name: configure nvidia runtime
|
||||
ansible.builtin.command: nvidia-ctk runtime configure --runtime=docker
|
||||
notify: restart docker
|
||||
|
||||
- name: restart docker
|
||||
ansible.builtin.service:
|
||||
name: docker
|
||||
state: restarted
|
||||
|
||||
- name: reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
@@ -0,0 +1,84 @@
|
||||
- name: Treiber-Branch bestimmen
|
||||
ansible.builtin.set_fact:
|
||||
_branch: "{{ nvidia_driver_branch[driver_generation] }}"
|
||||
|
||||
- name: NVIDIA-Treiber (Server-Variante) installieren
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- "nvidia-driver-{{ _branch }}-server"
|
||||
- "nvidia-utils-{{ _branch }}-server"
|
||||
state: present
|
||||
update_cache: true
|
||||
register: driver
|
||||
|
||||
- name: Treiber gegen ungewollte Updates sperren
|
||||
ansible.builtin.dpkg_selections:
|
||||
name: "{{ item }}"
|
||||
selection: hold
|
||||
loop:
|
||||
- "nvidia-driver-{{ _branch }}-server"
|
||||
- "nvidia-utils-{{ _branch }}-server"
|
||||
|
||||
- name: NVIDIA-Container-Toolkit-Repository
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
|
||||
curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
|
||||
| sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
|
||||
> /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||
args:
|
||||
creates: /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||
executable: /bin/bash
|
||||
|
||||
- name: Container-Toolkit installieren und für Docker konfigurieren
|
||||
ansible.builtin.apt:
|
||||
name: nvidia-container-toolkit
|
||||
state: present
|
||||
update_cache: true
|
||||
notify: configure nvidia runtime
|
||||
|
||||
- name: Modell-Disk (scsi1) formatieren, falls leer (formatiert nie eine vorhandene Partition um)
|
||||
community.general.filesystem:
|
||||
dev: /dev/sdb
|
||||
fstype: ext4
|
||||
opts: -L models
|
||||
when: ansible_devices.sdb is defined
|
||||
|
||||
- name: Modell-Disk einhängen
|
||||
ansible.posix.mount:
|
||||
path: /srv/models
|
||||
src: LABEL=models
|
||||
fstype: ext4
|
||||
state: mounted
|
||||
when: ansible_devices.sdb is defined
|
||||
|
||||
- name: Textfile-Verzeichnis für node_exporter
|
||||
ansible.builtin.file:
|
||||
path: "{{ nvidia_textfile_dir }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Xid-Exporter-Skript
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../../../../deploy/scripts/gpu-xid-textfile.sh"
|
||||
dest: /usr/local/bin/gpu-xid-textfile.sh
|
||||
mode: "0755"
|
||||
|
||||
- name: systemd-Service und -Timer für Xid-Exporter
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: "0644"
|
||||
loop: [gpu-xid-textfile.service, gpu-xid-textfile.timer]
|
||||
notify: reload systemd
|
||||
|
||||
- name: Timer aktivieren
|
||||
ansible.builtin.systemd:
|
||||
name: gpu-xid-textfile.timer
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
|
||||
- name: Neustart nach Treiberinstallation
|
||||
ansible.builtin.reboot:
|
||||
when: driver.changed
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=NVIDIA-Xid-Fehler für node_exporter zählen
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/gpu-xid-textfile.sh {{ nvidia_textfile_dir }}
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=NVIDIA-Xid-Fehler minütlich zählen
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=1min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,4 @@
|
||||
pve_iommu_cmdline: "intel_iommu=on iommu=pt"
|
||||
pve_vfio_modules: [vfio, vfio_iommu_type1, vfio_pci]
|
||||
# ZFS-Replikation der Infra-Gäste (ADR-0002), gefüllt aus sites/<site>/proxmox.yaml
|
||||
pve_site_file: "{{ playbook_dir }}/../../../../sites/{{ knaster_site }}/proxmox.yaml"
|
||||
@@ -0,0 +1,3 @@
|
||||
- name: refresh boot
|
||||
ansible.builtin.shell: proxmox-boot-tool refresh || update-grub
|
||||
changed_when: true
|
||||
@@ -0,0 +1,70 @@
|
||||
- name: Site-Datei laden
|
||||
ansible.builtin.set_fact:
|
||||
knaster_pve: "{{ lookup('ansible.builtin.file', pve_site_file) | from_yaml }}"
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Bootloader erkennen (systemd-boot bei ZFS-Root, sonst GRUB)
|
||||
ansible.builtin.stat:
|
||||
path: /etc/kernel/cmdline
|
||||
register: kernel_cmdline
|
||||
|
||||
- name: IOMMU aktivieren (systemd-boot)
|
||||
ansible.builtin.replace:
|
||||
path: /etc/kernel/cmdline
|
||||
regexp: '^((?!.*intel_iommu=on).*)$'
|
||||
replace: '\1 {{ pve_iommu_cmdline }}'
|
||||
when: kernel_cmdline.stat.exists
|
||||
notify: refresh boot
|
||||
|
||||
- name: IOMMU aktivieren (GRUB)
|
||||
ansible.builtin.lineinfile:
|
||||
path: /etc/default/grub
|
||||
regexp: '^GRUB_CMDLINE_LINUX_DEFAULT="((?!.*intel_iommu=on).*)"$'
|
||||
line: 'GRUB_CMDLINE_LINUX_DEFAULT="\1 {{ pve_iommu_cmdline }}"'
|
||||
backrefs: true
|
||||
when: not kernel_cmdline.stat.exists
|
||||
notify: refresh boot
|
||||
|
||||
- name: VFIO-Module laden
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/modules-load.d/vfio.conf
|
||||
content: "{{ pve_vfio_modules | join('\n') }}\n"
|
||||
mode: "0644"
|
||||
|
||||
- name: NVIDIA-Treiber auf dem Host sperren (GPU gehört der VM)
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/modprobe.d/blacklist-nvidia.conf
|
||||
content: |
|
||||
blacklist nouveau
|
||||
blacklist nvidia
|
||||
blacklist nvidiafb
|
||||
mode: "0644"
|
||||
notify: refresh boot
|
||||
|
||||
- name: ZFS-Pool tank vorhanden?
|
||||
ansible.builtin.command: zpool list -H -o name tank
|
||||
register: tank
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Hinweis, falls tank fehlt (Pool wird bewusst NICHT automatisch angelegt – Datenverlustgefahr)
|
||||
ansible.builtin.debug:
|
||||
msg: "Auf {{ inventory_hostname }} fehlt der ZFS-Pool 'tank'. Anlegen z. B.: zpool create -o ashift=12 tank mirror /dev/disk/by-id/<ssd2> /dev/disk/by-id/<ssd3> && pvesm add zfspool tank -pool tank"
|
||||
when: tank.rc != 0
|
||||
|
||||
- name: Bestehende Replikationsjobs lesen
|
||||
ansible.builtin.command: pvesr list
|
||||
register: pvesr_jobs
|
||||
changed_when: false
|
||||
|
||||
- name: ZFS-Replikation für Infra-Gäste
|
||||
ansible.builtin.command: >
|
||||
pvesr create-local-job {{ item.value.vmid }}-0 {{ item.value.replicate_to }}
|
||||
--schedule '{{ knaster_pve.replication_schedule }}' --comment 'knaster {{ item.key }}'
|
||||
loop: "{{ knaster_pve.containers | dict2items }}"
|
||||
when:
|
||||
- item.value.node == inventory_hostname
|
||||
- item.value.replicate_to is defined
|
||||
- (item.value.vmid | string) + '-0' not in pvesr_jobs.stdout
|
||||
changed_when: true
|
||||
@@ -0,0 +1,153 @@
|
||||
# GPU-Mappings, GPU-VMs und Infra-LXC aus sites/<site>/proxmox.yaml.
|
||||
# Nicht gegen einen echten Cluster getestet – beim gemeinsamen Deployment zuerst `tofu plan`.
|
||||
|
||||
resource "proxmox_virtual_environment_hardware_mapping_pci" "gpu" {
|
||||
for_each = local.site.pci_mappings
|
||||
name = each.key
|
||||
map = [{
|
||||
node = each.value.node
|
||||
path = each.value.path
|
||||
id = each.value.id
|
||||
subsystem_id = each.value.subsystem_id
|
||||
iommu_group = each.value.iommu_group
|
||||
}]
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_vm" "gpu" {
|
||||
for_each = local.site.vms
|
||||
name = each.key
|
||||
node_name = each.value.node
|
||||
vm_id = each.value.vmid
|
||||
tags = ["knaster", each.value.role, each.value.driver_generation]
|
||||
|
||||
machine = "q35"
|
||||
bios = "ovmf"
|
||||
on_boot = true
|
||||
|
||||
clone {
|
||||
vm_id = local.site.cluster.template_vmid
|
||||
full = true
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = each.value.cores
|
||||
type = "host"
|
||||
}
|
||||
|
||||
# PCI-Passthrough pinnt den gesamten RAM → kein Ballooning
|
||||
memory {
|
||||
dedicated = each.value.memory_mb
|
||||
floating = 0
|
||||
}
|
||||
|
||||
efi_disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
type = "4m"
|
||||
}
|
||||
|
||||
disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
interface = "scsi0"
|
||||
size = each.value.disk_gb
|
||||
discard = "on"
|
||||
ssd = true
|
||||
}
|
||||
|
||||
# Modelle: eigenes Disk-Image, nicht repliziert, nicht gesichert (ADR-0002)
|
||||
disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
interface = "scsi1"
|
||||
size = each.value.models_disk_gb
|
||||
discard = "on"
|
||||
ssd = true
|
||||
backup = false
|
||||
replicate = false
|
||||
}
|
||||
|
||||
hostpci {
|
||||
device = "hostpci0"
|
||||
mapping = proxmox_virtual_environment_hardware_mapping_pci.gpu[each.value.gpu].name
|
||||
pcie = true
|
||||
rombar = true
|
||||
}
|
||||
|
||||
network_device {
|
||||
bridge = local.site.cluster.bridge
|
||||
vlan_id = local.site.cluster.vlan_svc
|
||||
}
|
||||
|
||||
initialization {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = each.value.ip
|
||||
gateway = local.site.cluster.gateway
|
||||
}
|
||||
}
|
||||
dns {
|
||||
servers = [local.site.cluster.dns]
|
||||
}
|
||||
user_account {
|
||||
username = "knaster"
|
||||
keys = [local.ssh_key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_container" "infra" {
|
||||
for_each = local.site.containers
|
||||
node_name = each.value.node
|
||||
vm_id = each.value.vmid
|
||||
tags = ["knaster", "infra"]
|
||||
unprivileged = true
|
||||
start_on_boot = true
|
||||
|
||||
features {
|
||||
nesting = true # Docker im LXC
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = each.value.cores
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = each.value.memory_mb
|
||||
}
|
||||
|
||||
disk {
|
||||
datastore_id = local.site.cluster.storage_vm
|
||||
size = each.value.disk_gb
|
||||
}
|
||||
|
||||
operating_system {
|
||||
template_file_id = "${local.site.cluster.storage_iso}:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst"
|
||||
type = "debian"
|
||||
}
|
||||
|
||||
network_interface {
|
||||
name = "eth0"
|
||||
bridge = local.site.cluster.bridge
|
||||
vlan_id = local.site.cluster.vlan_svc
|
||||
}
|
||||
|
||||
initialization {
|
||||
hostname = each.key
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = each.value.ip
|
||||
gateway = local.site.cluster.gateway
|
||||
}
|
||||
}
|
||||
user_account {
|
||||
keys = [local.ssh_key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_haresource" "infra" {
|
||||
for_each = { for k, v in local.site.containers : k => v if try(v.ha, false) }
|
||||
resource_id = "ct:${each.value.vmid}"
|
||||
state = "started"
|
||||
comment = "knaster ${each.key}"
|
||||
depends_on = [proxmox_virtual_environment_container.infra]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
output "gpu_vms" {
|
||||
value = { for k, v in proxmox_virtual_environment_vm.gpu : k => { node = v.node_name, vmid = v.vm_id } }
|
||||
}
|
||||
|
||||
output "infra_containers" {
|
||||
value = { for k, v in proxmox_virtual_environment_container.infra : k => { node = v.node_name, vmid = v.vm_id } }
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
variable "site" {
|
||||
description = "Name des Standorts unter sites/"
|
||||
type = string
|
||||
default = "knaster"
|
||||
}
|
||||
|
||||
variable "proxmox_api_token" {
|
||||
description = "API-Token im Format user@realm!tokenid=secret"
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "proxmox_insecure" {
|
||||
description = "Selbstsigniertes Zertifikat akzeptieren"
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
locals {
|
||||
site = yamldecode(file("${path.module}/../../../sites/${var.site}/proxmox.yaml"))
|
||||
ssh_key = trimspace(file(pathexpand(local.site.cluster.ssh_public_key_file)))
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
terraform {
|
||||
required_version = ">= 1.6"
|
||||
required_providers {
|
||||
proxmox = {
|
||||
source = "bpg/proxmox"
|
||||
version = "~> 0.66"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "proxmox" {
|
||||
endpoint = local.site.cluster.endpoint
|
||||
api_token = var.proxmox_api_token
|
||||
insecure = var.proxmox_insecure
|
||||
ssh {
|
||||
agent = true
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user