tofu: GPU-VMs mit allen Karten des Knotens, DHCP über Gast-Agent, Provider für PVE 9

bpg/proxmox ~> 0.116.0 (nur API). Template je Knoten, Speicher je Gast,
hostpci dynamisch aus vms.*.gpus, kein Secure Boot (DKMS), Vendor-Snippet
für den qemu-guest-agent, Ausgabe ansible_inventory für die DHCP-Adressen.
scripts/make-template.sh legt das Ubuntu-24.04-Template mit Prüfsumme an.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

platform/proxmox/scripts/make-template.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Ubuntu-24.04-Cloud-Image als VM-Template auf einem Knoten anlegen (Git-Bash auf dem Laptop
+# oder auf ops). Idempotent: Ein vorhandenes Template bleibt unverändert.
+#
+#   make-template.sh <knoten> <vmid> [speicher]
+#
+#   <vmid>      laut sites/knaster/proxmox.yaml (cluster.templates), z. B. 9044 für pve44
+#   [speicher]  Ziel für Template-Disks (Standard: local-lvm)
+#
+# Das Image wird gegen SHA256SUMS von cloud-images.ubuntu.com (HTTPS) geprüft. Kein Secure Boot
+# (pre-enrolled-keys=0), weil die NVIDIA-DKMS-Module in den GPU-VMs nicht signiert sind.
+set -euo pipefail
+
+node=${1:?Aufruf: make-template.sh <knoten> <vmid> [speicher]}
+vmid=${2:?Aufruf: make-template.sh <knoten> <vmid> [speicher]}
+storage=${3:-local-lvm}
+
+ssh -T -o BatchMode=yes "$node" "VMID=$vmid STORAGE=$storage bash -s" <<'REMOTE'
+set -euo pipefail
+if qm status "$VMID" >/dev/null 2>&1; then echo "Template $VMID existiert schon auf $(hostname)"; exit 0; fi
+base=https://cloud-images.ubuntu.com/noble/current
+img=noble-server-cloudimg-amd64.img
+dir=/var/lib/vz/import
+mkdir -p "$dir" && cd "$dir"
+curl -fsSL -o SHA256SUMS "$base/SHA256SUMS"
+want=$(awk -v f="*$img" '$2 == f {print $1}' SHA256SUMS)
+[ -n "$want" ] || { echo "FEHLER: $img fehlt in SHA256SUMS" >&2; exit 1; }
+if ! echo "$want  $img" | sha256sum -c --quiet >/dev/null 2>&1; then
+  curl -fsSL -o "$img" "$base/$img"
+  echo "$want  $img" | sha256sum -c --quiet
+fi
+qm create "$VMID" --name "ubuntu-2404-tpl-$(hostname)" --ostype l26 --machine q35 --bios ovmf \
+  --cpu host --cores 2 --memory 2048 --scsihw virtio-scsi-single \
+  --efidisk0 "$STORAGE:0,efitype=4m,pre-enrolled-keys=0" \
+  --scsi0 "$STORAGE:0,import-from=$dir/$img,discard=on,ssd=1" \
+  --ide2 "$STORAGE:cloudinit" --boot order=scsi0 --serial0 socket \
+  --net0 virtio,bridge=vmbr0 --agent enabled=1 --tags "knaster;template" >/dev/null
+qm template "$VMID"
+echo "Template $VMID auf $(hostname) fertig ($(sha256sum "$img" | cut -c1-12))"
+REMOTE

platform/proxmox/tofu/main.tf

@@ -1,5 +1,5 @@
 # GPU-Mappings, GPU-VMs und Infra-LXC aus sites/<site>/proxmox.yaml.
-# Nicht gegen einen echten Cluster getestet – beim gemeinsamen Deployment zuerst `tofu plan`.
+# Gäste per DHCP (D5 zurückgestellt): Die Adresse meldet der qemu-guest-agent (siehe outputs.tf).
 
 resource "proxmox_virtual_environment_hardware_mapping_pci" "gpu" {
   for_each = local.site.pci_mappings
@@ -18,17 +18,25 @@ resource "proxmox_virtual_environment_vm" "gpu" {
   name      = each.key
   node_name = each.value.node
   vm_id     = each.value.vmid
-  tags      = ["knaster", each.value.role, each.value.driver_generation]
+  tags      = ["knaster", "gpu", each.value.driver_generation]
 
   machine       = "q35"
   bios          = "ovmf"
   on_boot       = true
+  scsi_hardware = "virtio-scsi-single"
 
+  # Template je Knoten: VMIDs sind clusterweit eindeutig, lokaler Speicher ist nicht geteilt
   clone {
-    vm_id = local.site.cluster.template_vmid
+    vm_id        = local.site.cluster.templates[each.value.node]
+    node_name    = each.value.node
+    datastore_id = each.value.storage
     full         = true
   }
 
+  agent {
+    enabled = true
+  }
+
   cpu {
     cores = each.value.cores
     type  = "host"
@@ -40,53 +48,59 @@ resource "proxmox_virtual_environment_vm" "gpu" {
     floating  = 0
   }
 
+  # Ohne vorinstallierte Schlüssel kein Secure Boot: die NVIDIA-DKMS-Module sind nicht signiert
   efi_disk {
-    datastore_id = local.site.cluster.storage_vm
+    datastore_id      = each.value.storage
     type              = "4m"
+    pre_enrolled_keys = false
   }
 
   disk {
-    datastore_id = local.site.cluster.storage_vm
+    datastore_id = each.value.storage
     interface    = "scsi0"
     size         = each.value.disk_gb
     discard      = "on"
     ssd          = true
+    iothread     = true
   }
 
   # Modelle: eigenes Disk-Image, nicht repliziert, nicht gesichert (ADR-0002)
   disk {
-    datastore_id = local.site.cluster.storage_vm
+    datastore_id = each.value.storage
     interface    = "scsi1"
+    file_format  = "raw"
     size         = each.value.models_disk_gb
     discard      = "on"
     ssd          = true
+    iothread     = true
     backup       = false
     replicate    = false
   }
 
-  hostpci {
+  # Alle Karten des Knotens (D2), jeweils mit allen Funktionen (Grafik + Audio)
-    device  = "hostpci0"
+  dynamic "hostpci" {
-    mapping = proxmox_virtual_environment_hardware_mapping_pci.gpu[each.value.gpu].name
+    for_each = each.value.gpus
+    content {
+      device  = "hostpci${hostpci.key}"
+      mapping = proxmox_virtual_environment_hardware_mapping_pci.gpu[hostpci.value].name
       pcie    = true
       rombar  = true
     }
+  }
 
   network_device {
     bridge = local.site.cluster.bridge
-    vlan_id = local.site.cluster.vlan_svc
   }
 
   initialization {
-    datastore_id = local.site.cluster.storage_vm
+    datastore_id        = each.value.storage
+    vendor_data_file_id = "local:snippets/knaster-vendor.yaml"
     ip_config {
       ipv4 {
         address = each.value.ip
-        gateway = local.site.cluster.gateway
+        gateway = each.value.ip == "dhcp" ? null : local.site.cluster.gateway
       }
     }
-    dns {
-      servers = [local.site.cluster.dns]
-    }
     user_account {
       username = "knaster"
       keys     = [local.ssh_key]
@@ -115,19 +129,18 @@ resource "proxmox_virtual_environment_container" "infra" {
   }
 
   disk {
-    datastore_id = local.site.cluster.storage_vm
+    datastore_id = each.value.storage
     size         = each.value.disk_gb
   }
 
   operating_system {
-    template_file_id = "${local.site.cluster.storage_iso}:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst"
+    template_file_id = "${local.site.cluster.storage_iso}:vztmpl/${local.site.cluster.lxc_template}"
     type             = "debian"
   }
 
   network_interface {
     name   = "eth0"
     bridge = local.site.cluster.bridge
-    vlan_id = local.site.cluster.vlan_svc
   }
 
   initialization {
@@ -135,7 +148,7 @@ resource "proxmox_virtual_environment_container" "infra" {
     ip_config {
       ipv4 {
         address = each.value.ip
-        gateway = local.site.cluster.gateway
+        gateway = each.value.ip == "dhcp" ? null : local.site.cluster.gateway
       }
     }
     user_account {

platform/proxmox/tofu/outputs.tf

@@ -1,7 +1,26 @@
+locals {
+  # Erste IPv4 der Gast-Netzkarte laut qemu-guest-agent (ohne lo, Docker- und Bridge-Interfaces)
+  gpu_vm_ipv4 = {
+    for k, v in proxmox_virtual_environment_vm.gpu : k => try(flatten([
+      for i, name in v.network_interface_names : v.ipv4_addresses[i]
+      if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth")
+    ])[0], null)
+  }
+}
+
 output "gpu_vms" {
-  value = { for k, v in proxmox_virtual_environment_vm.gpu : k => { node = v.node_name, vmid = v.vm_id } }
+  value = { for k, v in proxmox_virtual_environment_vm.gpu : k => { node = v.node_name, vmid = v.vm_id, ipv4 = local.gpu_vm_ipv4[k] } }
 }
 
 output "infra_containers" {
   value = { for k, v in proxmox_virtual_environment_container.infra : k => { node = v.node_name, vmid = v.vm_id } }
 }
+
+# tofu output -raw ansible_inventory > ../ansible/inventory.generated.yml
+output "ansible_inventory" {
+  value = yamlencode({
+    all = { children = { gpu_vms = { hosts = {
+      for k, ip in local.gpu_vm_ipv4 : k => { ansible_host = ip } if ip != null
+    } } } }
+  })
+}

platform/proxmox/tofu/versions.tf

@@ -3,16 +3,14 @@ terraform {
   required_providers {
     proxmox = {
       source  = "bpg/proxmox"
-      version = "~> 0.66"
+      version = "~> 0.116.0" # PVE 9 (HA-Regeln statt Gruppen)
     }
   }
 }
 
+# Nur API, kein SSH: Templates legt scripts/make-template.sh an, Snippets die Rolle pve-host.
 provider "proxmox" {
   endpoint  = local.site.cluster.endpoint
   api_token = var.proxmox_api_token
   insecure  = var.proxmox_insecure
-  ssh {
-    agent = true
-  }
 }