From 908c69858968e4a7c5eaab1126bff3b18163ed21 Mon Sep 17 00:00:00 2001 From: Robert Rapp Date: Mon, 12 Oct 2026 18:58:34 +0200 Subject: [PATCH] =?UTF-8?q?proxmox:=20Site-Datei=20und=20pve-host-Rolle=20?= =?UTF-8?q?an=20die=20echten=20Knoten=20(pve44=E2=80=93pve61)=20angepasst?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit proxmox.yaml mit gelesenen PCI-IDs/IOMMU-Gruppen, eine GPU-VM pro Knoten mit allen Karten, DHCP, Templates je Knoten. pve-host nur für GPU-Knoten: vfio-pci-IDs auf dem Knoten gelesen, softdeps, sysfb aus, initramfs- Handler, Vendor-Snippet für den Gast-Agenten, Replikation erst wenn der Container existiert. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + platform/proxmox/ansible/ansible.cfg | 7 +- platform/proxmox/ansible/inventory.yml | 33 ++++---- platform/proxmox/ansible/playbooks/site.yml | 7 +- .../ansible/roles/pve-host/defaults/main.yml | 7 +- .../ansible/roles/pve-host/handlers/main.yml | 7 +- .../ansible/roles/pve-host/tasks/main.yml | 84 +++++++++++++++---- sites/knaster/proxmox.yaml | 42 +++++----- 8 files changed, 130 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index 6dff6f1..a57ae35 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ data/ *.tfstate* .DS_Store deploy/compose/*/secrets/ +platform/proxmox/ansible/inventory.generated.yml diff --git a/platform/proxmox/ansible/ansible.cfg b/platform/proxmox/ansible/ansible.cfg index 4cc6678..28fb7ce 100644 --- a/platform/proxmox/ansible/ansible.cfg +++ b/platform/proxmox/ansible/ansible.cfg @@ -1,5 +1,8 @@ [defaults] -inventory = inventory.yml +inventory = inventory.yml,inventory.generated.yml roles_path = roles host_key_checking = True -stdout_callback = yaml +callback_result_format = yaml + +[inventory] +unparsed_is_failed = False diff --git a/platform/proxmox/ansible/inventory.yml b/platform/proxmox/ansible/inventory.yml index f68999c..3088cda 100644 --- a/platform/proxmox/ansible/inventory.yml +++ b/platform/proxmox/ansible/inventory.yml @@ -1,29 +1,34 @@ -# ANNAHME bis Deployment – Hosts und IPs aus sites/knaster/proxmox.yaml +# Knoten und Gäste von knaster (Stand 12.10.2026). Läuft auf dem Steuerrechner ops. +# Gäste bekommen ihre IP per DHCP (D5 zurückgestellt): ansible_host der GPU-VMs und von infra +# kommt aus inventory.generated.yml (`tofu output -raw ansible_inventory`, nicht im Repo). all: vars: knaster_site: knaster children: - pve_hosts: - hosts: - pve-ada: {} - pve1: {} - pve2: {} - pve3: {} - pve4: {} + pve_all: + children: + pve_gpu_hosts: + hosts: + pve44: {ansible_host: 192.168.100.210} + pve46: {ansible_host: 192.168.100.209} + pve48: {ansible_host: 192.168.100.212} + pve52: {ansible_host: 192.168.100.207} + pve_kiosk: + hosts: + pve61: {ansible_host: 192.168.100.214} vars: ansible_user: root gpu_vms: hosts: - gpu-main: {ansible_host: 10.20.0.201, driver_generation: modern, knaster_role: gpu-vllm} - gpu-flex: {ansible_host: 10.20.0.202, driver_generation: modern, knaster_role: gpu-vllm} - gpu-embed-1: {ansible_host: 10.20.0.221, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: embed} - gpu-embed-2: {ansible_host: 10.20.0.222, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: embed} - gpu-small: {ansible_host: 10.20.0.231, driver_generation: legacy, knaster_role: gpu-llamacpp, llamacpp_profile: chat} + gpu-pve44: {driver_generation: legacy} + gpu-pve46: {driver_generation: legacy} + gpu-pve48: {driver_generation: legacy} + gpu-pve52: {driver_generation: legacy} vars: ansible_user: knaster ansible_become: true infra: hosts: - infra: {ansible_host: 10.20.0.101} + infra: {} vars: ansible_user: root diff --git a/platform/proxmox/ansible/playbooks/site.yml b/platform/proxmox/ansible/playbooks/site.yml index e41266e..71d3c78 100644 --- a/platform/proxmox/ansible/playbooks/site.yml +++ b/platform/proxmox/ansible/playbooks/site.yml @@ -1,6 +1,7 @@ -# Reihenfolge: Hosts vorbereiten → (tofu apply) → GPU-VMs → Infra -- name: Proxmox-Hosts (IOMMU, VFIO, ZFS-Replikation) - hosts: pve_hosts +# Reihenfolge: GPU-Knoten vorbereiten → Knoten einzeln neu starten → (tofu apply) → GPU-VMs → Infra +# pve61 (pve_kiosk) bekommt kein VFIO und keine NVIDIA-Sperre, seine iGPU bleibt für den Kiosk. +- name: GPU-Knoten (IOMMU, VFIO, NVIDIA auf dem Host gesperrt, Replikation) + hosts: pve_gpu_hosts roles: [pve-host] - name: GPU-VMs (Treiber gepinnt, Container-Toolkit, Docker, Xid-Exporter) diff --git a/platform/proxmox/ansible/roles/pve-host/defaults/main.yml b/platform/proxmox/ansible/roles/pve-host/defaults/main.yml index e75a1ed..1054178 100644 --- a/platform/proxmox/ansible/roles/pve-host/defaults/main.yml +++ b/platform/proxmox/ansible/roles/pve-host/defaults/main.yml @@ -1,4 +1,9 @@ -pve_iommu_cmdline: "intel_iommu=on iommu=pt" +# sysfb_init aus: Die M4000 ist auf allen GPU-Knoten die Boot-Grafik (keine iGPU). Sonst hält das +# Firmware-Framebuffer ihren BAR fest und die VM startet nicht. Folge: Nach dem frühen Start bleibt +# der lokale Bildschirm schwarz, Zugang nur per SSH/Weboberfläche (GRUB-Menü bleibt sichtbar). +pve_iommu_cmdline: "intel_iommu=on iommu=pt initcall_blacklist=sysfb_init" pve_vfio_modules: [vfio, vfio_iommu_type1, vfio_pci] +# cloud-init-Vendor-Daten für die GPU-VMs (qemu-guest-agent liefert tofu die DHCP-Adresse) +pve_vendor_snippet: /var/lib/vz/snippets/knaster-vendor.yaml # ZFS-Replikation der Infra-Gäste (ADR-0002), gefüllt aus sites//proxmox.yaml pve_site_file: "{{ playbook_dir }}/../../../../sites/{{ knaster_site }}/proxmox.yaml" diff --git a/platform/proxmox/ansible/roles/pve-host/handlers/main.yml b/platform/proxmox/ansible/roles/pve-host/handlers/main.yml index 94bd75f..f896cd1 100644 --- a/platform/proxmox/ansible/roles/pve-host/handlers/main.yml +++ b/platform/proxmox/ansible/roles/pve-host/handlers/main.yml @@ -1,3 +1,8 @@ +# Reihenfolge wie hier definiert: erst initramfs (vfio.conf, Sperrliste), dann Bootloader +- name: update initramfs + ansible.builtin.command: update-initramfs -u -k all + changed_when: true + - name: refresh boot - ansible.builtin.shell: proxmox-boot-tool refresh || update-grub + ansible.builtin.shell: proxmox-boot-tool refresh 2>/dev/null || update-grub changed_when: true diff --git a/platform/proxmox/ansible/roles/pve-host/tasks/main.yml b/platform/proxmox/ansible/roles/pve-host/tasks/main.yml index 70246b6..0d7f942 100644 --- a/platform/proxmox/ansible/roles/pve-host/tasks/main.yml +++ b/platform/proxmox/ansible/roles/pve-host/tasks/main.yml @@ -31,6 +31,30 @@ dest: /etc/modules-load.d/vfio.conf content: "{{ pve_vfio_modules | join('\n') }}\n" mode: "0644" + notify: update initramfs + +# IDs auf dem Knoten lesen, nicht aus dem Gedächtnis: Grafik- und Audio-Funktion jeder Karte +- name: NVIDIA-Funktionen des Knotens lesen + ansible.builtin.shell: | + set -o pipefail + lspci -n -d 10de: | awk '{print $3}' | sort -u | paste -sd, + args: + executable: /bin/bash + register: nvidia_ids + changed_when: false + failed_when: nvidia_ids.rc != 0 or nvidia_ids.stdout == '' + +- name: Karten an vfio-pci binden, bevor ein Grafik- oder Audiotreiber zugreift + ansible.builtin.copy: + dest: /etc/modprobe.d/vfio.conf + content: | + options vfio-pci ids={{ nvidia_ids.stdout }} + softdep nouveau pre: vfio-pci + softdep nvidia pre: vfio-pci + softdep nvidiafb pre: vfio-pci + softdep snd_hda_intel pre: vfio-pci + mode: "0644" + notify: update initramfs - name: NVIDIA-Treiber auf dem Host sperren (GPU gehört der VM) ansible.builtin.copy: @@ -40,31 +64,57 @@ blacklist nvidia blacklist nvidiafb mode: "0644" - notify: refresh boot + notify: update initramfs -- name: ZFS-Pool tank vorhanden? - ansible.builtin.command: zpool list -H -o name tank - register: tank +- name: Speicher local darf Snippets enthalten (clusterweite Einstellung, einmal) + ansible.builtin.shell: | + set -euo pipefail + c=$(pvesh get /storage/local --output-format json | python3 -c 'import json,sys; print(json.load(sys.stdin)["content"])') + case ",$c," in + *,snippets,*) echo unverändert ;; + *) pvesm set local --content "$c,snippets"; echo geändert ;; + esac + args: + executable: /bin/bash + register: snippets + changed_when: "'geändert' in snippets.stdout" + run_once: true + +- name: Snippet-Verzeichnis + ansible.builtin.file: + path: "{{ pve_vendor_snippet | dirname }}" + state: directory + mode: "0755" + +- name: cloud-init-Vendor-Daten für GPU-VMs (qemu-guest-agent meldet tofu die DHCP-Adresse) + ansible.builtin.copy: + dest: "{{ pve_vendor_snippet }}" + content: | + #cloud-config + packages: [qemu-guest-agent] + runcmd: + - [systemctl, enable, --now, qemu-guest-agent] + mode: "0644" + +- name: Container auf diesem Knoten lesen + ansible.builtin.command: ls /etc/pve/lxc/ + register: local_cts changed_when: false - failed_when: false - -- name: Hinweis, falls tank fehlt (Pool wird bewusst NICHT automatisch angelegt – Datenverlustgefahr) - ansible.builtin.debug: - msg: "Auf {{ inventory_hostname }} fehlt der ZFS-Pool 'tank'. Anlegen z. B.: zpool create -o ashift=12 tank mirror /dev/disk/by-id/ /dev/disk/by-id/ && pvesm add zfspool tank -pool tank" - when: tank.rc != 0 - name: Bestehende Replikationsjobs lesen ansible.builtin.command: pvesr list register: pvesr_jobs changed_when: false -- name: ZFS-Replikation für Infra-Gäste +- name: ZFS-Replikation für Infra-Gäste (erst wenn der Container auf diesem Knoten existiert) ansible.builtin.command: > - pvesr create-local-job {{ item.value.vmid }}-0 {{ item.value.replicate_to }} - --schedule '{{ knaster_pve.replication_schedule }}' --comment 'knaster {{ item.key }}' - loop: "{{ knaster_pve.containers | dict2items }}" + pvesr create-local-job {{ item.0.value.vmid }}-{{ idx }} {{ item.1 }} + --schedule '{{ knaster_pve.replication_schedule }}' --comment 'knaster {{ item.0.key }}' + loop: "{{ knaster_pve.containers | dict2items | subelements('value.replicate_to', skip_missing=True) }}" + vars: + idx: "{{ item.0.value.replicate_to.index(item.1) }}" when: - - item.value.node == inventory_hostname - - item.value.replicate_to is defined - - (item.value.vmid | string) + '-0' not in pvesr_jobs.stdout + - item.0.value.node == inventory_hostname + - (item.0.value.vmid | string) + '.conf' in local_cts.stdout_lines + - (item.0.value.vmid | string) + '-' + (idx | string) not in pvesr_jobs.stdout changed_when: true diff --git a/sites/knaster/proxmox.yaml b/sites/knaster/proxmox.yaml index bd1a56a..08ea961 100644 --- a/sites/knaster/proxmox.yaml +++ b/sites/knaster/proxmox.yaml @@ -1,31 +1,33 @@ # Proxmox-Zielbild für "knaster" (Eingabe für platform/proxmox/tofu und ansible). -# ANNAHMEN (Hosts, VM-IDs, IPs, PCI-Adressen) bis zum Inventarlauf – siehe docs/IMPLEMENTATION.md. +# Stand 12.10.2026: echte Knoten pve44–pve61. PCI-Adressen, IDs und IOMMU-Gruppen per +# `pvesh get /nodes//hardware/pci` gelesen. Gäste vorerst per DHCP (D5 zurückgestellt). cluster: - endpoint: https://pve1:8006/ - storage_vm: tank # ZFS-Pool auf SSD 2+3 (ADR-0002) + endpoint: https://192.168.100.210:8006/ storage_iso: local - template_vmid: 9000 # Ubuntu 24.04 Cloud-Image-Template (pro Node vorhanden) bridge: vmbr0 - vlan_svc: 20 - gateway: 10.20.0.1 - dns: 10.20.0.1 - ssh_public_key_file: ~/.ssh/id_ed25519.pub + gateway: 192.168.100.1 + dns: 192.168.100.120 + ssh_public_key_file: ~/.ssh/id_ed25519.pub # Schlüssel auf dem Steuerrechner ops + lxc_template: debian-13-standard_13.6-1_amd64.tar.zst + # Ubuntu-24.04-Cloud-Image-Template je Knoten (VMIDs sind clusterweit eindeutig) + templates: {pve44: 9044, pve46: 9046, pve48: 9048, pve52: 9052} pci_mappings: # Proxmox Resource Mappings (Datacenter → Resource Mappings → PCI) - gpu-ada-1: {node: pve-ada, path: "0000:01:00", id: "10de:27b2", subsystem_id: "", iommu_group: 0} - gpu-ada-2: {node: pve-ada, path: "0000:02:00", id: "10de:27b2", subsystem_id: "", iommu_group: 0} - gpu-p2000-1: {node: pve3, path: "0000:01:00", id: "10de:1c30", subsystem_id: "", iommu_group: 0} - gpu-p2000-2: {node: pve3, path: "0000:02:00", id: "10de:1c30", subsystem_id: "", iommu_group: 0} - gpu-p2000-3: {node: pve4, path: "0000:01:00", id: "10de:1c30", subsystem_id: "", iommu_group: 0} + gpu-m4000-pve44: {node: pve44, path: "0000:01:00", id: "10de:13f1", subsystem_id: "10de:1153", iommu_group: 1} + gpu-p2000-pve44: {node: pve44, path: "0000:02:00", id: "10de:1c30", subsystem_id: "10de:11b3", iommu_group: 8} + gpu-m4000-pve46: {node: pve46, path: "0000:01:00", id: "10de:13f1", subsystem_id: "10de:1153", iommu_group: 1} + gpu-p2000-pve46: {node: pve46, path: "0000:02:00", id: "10de:1c30", subsystem_id: "10de:11b3", iommu_group: 8} + gpu-m4000-pve48: {node: pve48, path: "0000:01:00", id: "10de:13f1", subsystem_id: "10de:1153", iommu_group: 1} + gpu-p2000-pve48: {node: pve48, path: "0000:02:00", id: "10de:1c30", subsystem_id: "10de:11b3", iommu_group: 8} + gpu-m4000-pve52: {node: pve52, path: "0000:01:00", id: "10de:13f1", subsystem_id: "10de:1153", iommu_group: 1} -vms: # GPU-Rollen als VM (ADR-0001) - gpu-main: {node: pve-ada, vmid: 201, cores: 6, memory_mb: 28672, disk_gb: 40, models_disk_gb: 100, gpu: gpu-ada-1, ip: 10.20.0.201/24, role: gpu-vllm, driver_generation: modern} - gpu-flex: {node: pve-ada, vmid: 202, cores: 6, memory_mb: 28672, disk_gb: 40, models_disk_gb: 100, gpu: gpu-ada-2, ip: 10.20.0.202/24, role: gpu-vllm, driver_generation: modern} - gpu-embed-1: {node: pve3, vmid: 221, cores: 2, memory_mb: 6144, disk_gb: 30, models_disk_gb: 20, gpu: gpu-p2000-1, ip: 10.20.0.221/24, role: gpu-llamacpp, profile: embed, driver_generation: legacy} - gpu-embed-2: {node: pve4, vmid: 222, cores: 2, memory_mb: 6144, disk_gb: 30, models_disk_gb: 20, gpu: gpu-p2000-3, ip: 10.20.0.222/24, role: gpu-llamacpp, profile: embed, driver_generation: legacy} - gpu-small: {node: pve3, vmid: 231, cores: 2, memory_mb: 8192, disk_gb: 30, models_disk_gb: 20, gpu: gpu-p2000-2, ip: 10.20.0.231/24, role: gpu-llamacpp, profile: chat, driver_generation: legacy} +vms: # eine GPU-VM pro Knoten mit allen Karten des Knotens (D2) + gpu-pve44: {node: pve44, vmid: 244, cores: 4, memory_mb: 12288, disk_gb: 40, models_disk_gb: 60, storage: local-lvm, gpus: [gpu-m4000-pve44, gpu-p2000-pve44], ip: dhcp, driver_generation: legacy} + gpu-pve46: {node: pve46, vmid: 246, cores: 4, memory_mb: 12288, disk_gb: 40, models_disk_gb: 60, storage: local-lvm, gpus: [gpu-m4000-pve46, gpu-p2000-pve46], ip: dhcp, driver_generation: legacy} + gpu-pve48: {node: pve48, vmid: 248, cores: 4, memory_mb: 12288, disk_gb: 40, models_disk_gb: 60, storage: local-lvm, gpus: [gpu-m4000-pve48, gpu-p2000-pve48], ip: dhcp, driver_generation: legacy} + gpu-pve52: {node: pve52, vmid: 252, cores: 4, memory_mb: 12288, disk_gb: 40, models_disk_gb: 60, storage: local-lvm, gpus: [gpu-m4000-pve52], ip: dhcp, driver_generation: legacy} containers: # Infrastruktur als LXC (ADR-0001), HA + ZFS-Replikation (ADR-0002) - infra: {node: pve1, vmid: 101, cores: 4, memory_mb: 12288, disk_gb: 60, ip: 10.20.0.101/24, ha: true, replicate_to: pve2} + infra: {node: pve44, vmid: 101, cores: 4, memory_mb: 8192, disk_gb: 60, storage: tank, ip: dhcp, ha: true, replicate_to: [pve52, pve48]} replication_schedule: "*/15"