diff --git a/platform/proxmox/scripts/make-template.sh b/platform/proxmox/scripts/make-template.sh new file mode 100755 index 0000000..a6d38bc --- /dev/null +++ b/platform/proxmox/scripts/make-template.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Ubuntu-24.04-Cloud-Image als VM-Template auf einem Knoten anlegen (Git-Bash auf dem Laptop +# oder auf ops). Idempotent: Ein vorhandenes Template bleibt unverändert. +# +# make-template.sh [speicher] +# +# laut sites/knaster/proxmox.yaml (cluster.templates), z. B. 9044 für pve44 +# [speicher] Ziel für Template-Disks (Standard: local-lvm) +# +# Das Image wird gegen SHA256SUMS von cloud-images.ubuntu.com (HTTPS) geprüft. Kein Secure Boot +# (pre-enrolled-keys=0), weil die NVIDIA-DKMS-Module in den GPU-VMs nicht signiert sind. +set -euo pipefail + +node=${1:?Aufruf: make-template.sh [speicher]} +vmid=${2:?Aufruf: make-template.sh [speicher]} +storage=${3:-local-lvm} + +ssh -T -o BatchMode=yes "$node" "VMID=$vmid STORAGE=$storage bash -s" <<'REMOTE' +set -euo pipefail +if qm status "$VMID" >/dev/null 2>&1; then echo "Template $VMID existiert schon auf $(hostname)"; exit 0; fi +base=https://cloud-images.ubuntu.com/noble/current +img=noble-server-cloudimg-amd64.img +dir=/var/lib/vz/import +mkdir -p "$dir" && cd "$dir" +curl -fsSL -o SHA256SUMS "$base/SHA256SUMS" +want=$(awk -v f="*$img" '$2 == f {print $1}' SHA256SUMS) +[ -n "$want" ] || { echo "FEHLER: $img fehlt in SHA256SUMS" >&2; exit 1; } +if ! echo "$want $img" | sha256sum -c --quiet >/dev/null 2>&1; then + curl -fsSL -o "$img" "$base/$img" + echo "$want $img" | sha256sum -c --quiet +fi +qm create "$VMID" --name "ubuntu-2404-tpl-$(hostname)" --ostype l26 --machine q35 --bios ovmf \ + --cpu host --cores 2 --memory 2048 --scsihw virtio-scsi-single \ + --efidisk0 "$STORAGE:0,efitype=4m,pre-enrolled-keys=0" \ + --scsi0 "$STORAGE:0,import-from=$dir/$img,discard=on,ssd=1" \ + --ide2 "$STORAGE:cloudinit" --boot order=scsi0 --serial0 socket \ + --net0 virtio,bridge=vmbr0 --agent enabled=1 --tags "knaster;template" >/dev/null +qm template "$VMID" +echo "Template $VMID auf $(hostname) fertig ($(sha256sum "$img" | cut -c1-12))" +REMOTE diff --git a/platform/proxmox/tofu/main.tf b/platform/proxmox/tofu/main.tf index a8cc40f..907ed74 100644 --- a/platform/proxmox/tofu/main.tf +++ b/platform/proxmox/tofu/main.tf @@ -1,5 +1,5 @@ # GPU-Mappings, GPU-VMs und Infra-LXC aus sites//proxmox.yaml. -# Nicht gegen einen echten Cluster getestet – beim gemeinsamen Deployment zuerst `tofu plan`. +# Gäste per DHCP (D5 zurückgestellt): Die Adresse meldet der qemu-guest-agent (siehe outputs.tf). resource "proxmox_virtual_environment_hardware_mapping_pci" "gpu" { for_each = local.site.pci_mappings @@ -18,15 +18,23 @@ resource "proxmox_virtual_environment_vm" "gpu" { name = each.key node_name = each.value.node vm_id = each.value.vmid - tags = ["knaster", each.value.role, each.value.driver_generation] + tags = ["knaster", "gpu", each.value.driver_generation] - machine = "q35" - bios = "ovmf" - on_boot = true + machine = "q35" + bios = "ovmf" + on_boot = true + scsi_hardware = "virtio-scsi-single" + # Template je Knoten: VMIDs sind clusterweit eindeutig, lokaler Speicher ist nicht geteilt clone { - vm_id = local.site.cluster.template_vmid - full = true + vm_id = local.site.cluster.templates[each.value.node] + node_name = each.value.node + datastore_id = each.value.storage + full = true + } + + agent { + enabled = true } cpu { @@ -40,53 +48,59 @@ resource "proxmox_virtual_environment_vm" "gpu" { floating = 0 } + # Ohne vorinstallierte Schlüssel kein Secure Boot: die NVIDIA-DKMS-Module sind nicht signiert efi_disk { - datastore_id = local.site.cluster.storage_vm - type = "4m" + datastore_id = each.value.storage + type = "4m" + pre_enrolled_keys = false } disk { - datastore_id = local.site.cluster.storage_vm + datastore_id = each.value.storage interface = "scsi0" size = each.value.disk_gb discard = "on" ssd = true + iothread = true } # Modelle: eigenes Disk-Image, nicht repliziert, nicht gesichert (ADR-0002) disk { - datastore_id = local.site.cluster.storage_vm + datastore_id = each.value.storage interface = "scsi1" + file_format = "raw" size = each.value.models_disk_gb discard = "on" ssd = true + iothread = true backup = false replicate = false } - hostpci { - device = "hostpci0" - mapping = proxmox_virtual_environment_hardware_mapping_pci.gpu[each.value.gpu].name - pcie = true - rombar = true + # Alle Karten des Knotens (D2), jeweils mit allen Funktionen (Grafik + Audio) + dynamic "hostpci" { + for_each = each.value.gpus + content { + device = "hostpci${hostpci.key}" + mapping = proxmox_virtual_environment_hardware_mapping_pci.gpu[hostpci.value].name + pcie = true + rombar = true + } } network_device { - bridge = local.site.cluster.bridge - vlan_id = local.site.cluster.vlan_svc + bridge = local.site.cluster.bridge } initialization { - datastore_id = local.site.cluster.storage_vm + datastore_id = each.value.storage + vendor_data_file_id = "local:snippets/knaster-vendor.yaml" ip_config { ipv4 { address = each.value.ip - gateway = local.site.cluster.gateway + gateway = each.value.ip == "dhcp" ? null : local.site.cluster.gateway } } - dns { - servers = [local.site.cluster.dns] - } user_account { username = "knaster" keys = [local.ssh_key] @@ -95,11 +109,11 @@ resource "proxmox_virtual_environment_vm" "gpu" { } resource "proxmox_virtual_environment_container" "infra" { - for_each = local.site.containers - node_name = each.value.node - vm_id = each.value.vmid - tags = ["knaster", "infra"] - unprivileged = true + for_each = local.site.containers + node_name = each.value.node + vm_id = each.value.vmid + tags = ["knaster", "infra"] + unprivileged = true start_on_boot = true features { @@ -115,19 +129,18 @@ resource "proxmox_virtual_environment_container" "infra" { } disk { - datastore_id = local.site.cluster.storage_vm + datastore_id = each.value.storage size = each.value.disk_gb } operating_system { - template_file_id = "${local.site.cluster.storage_iso}:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst" + template_file_id = "${local.site.cluster.storage_iso}:vztmpl/${local.site.cluster.lxc_template}" type = "debian" } network_interface { - name = "eth0" - bridge = local.site.cluster.bridge - vlan_id = local.site.cluster.vlan_svc + name = "eth0" + bridge = local.site.cluster.bridge } initialization { @@ -135,7 +148,7 @@ resource "proxmox_virtual_environment_container" "infra" { ip_config { ipv4 { address = each.value.ip - gateway = local.site.cluster.gateway + gateway = each.value.ip == "dhcp" ? null : local.site.cluster.gateway } } user_account { diff --git a/platform/proxmox/tofu/outputs.tf b/platform/proxmox/tofu/outputs.tf index cc906b2..324c5b5 100644 --- a/platform/proxmox/tofu/outputs.tf +++ b/platform/proxmox/tofu/outputs.tf @@ -1,7 +1,26 @@ +locals { + # Erste IPv4 der Gast-Netzkarte laut qemu-guest-agent (ohne lo, Docker- und Bridge-Interfaces) + gpu_vm_ipv4 = { + for k, v in proxmox_virtual_environment_vm.gpu : k => try(flatten([ + for i, name in v.network_interface_names : v.ipv4_addresses[i] + if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth") + ])[0], null) + } +} + output "gpu_vms" { - value = { for k, v in proxmox_virtual_environment_vm.gpu : k => { node = v.node_name, vmid = v.vm_id } } + value = { for k, v in proxmox_virtual_environment_vm.gpu : k => { node = v.node_name, vmid = v.vm_id, ipv4 = local.gpu_vm_ipv4[k] } } } output "infra_containers" { value = { for k, v in proxmox_virtual_environment_container.infra : k => { node = v.node_name, vmid = v.vm_id } } } + +# tofu output -raw ansible_inventory > ../ansible/inventory.generated.yml +output "ansible_inventory" { + value = yamlencode({ + all = { children = { gpu_vms = { hosts = { + for k, ip in local.gpu_vm_ipv4 : k => { ansible_host = ip } if ip != null + } } } } + }) +} diff --git a/platform/proxmox/tofu/versions.tf b/platform/proxmox/tofu/versions.tf index b419b65..f275555 100644 --- a/platform/proxmox/tofu/versions.tf +++ b/platform/proxmox/tofu/versions.tf @@ -3,16 +3,14 @@ terraform { required_providers { proxmox = { source = "bpg/proxmox" - version = "~> 0.66" + version = "~> 0.116.0" # PVE 9 (HA-Regeln statt Gruppen) } } } +# Nur API, kein SSH: Templates legt scripts/make-template.sh an, Snippets die Rolle pve-host. provider "proxmox" { endpoint = local.site.cluster.endpoint api_token = var.proxmox_api_token insecure = var.proxmox_insecure - ssh { - agent = true - } }