k3s (D6): Dienst-VM k3s per tofu, Rollen k3s-server/k3s-agent/k3s-apps, Manifeste, NEX-Sprachdienste als k3s-Umzugstest

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

platform/proxmox/ansible/roles/k3s-agent/defaults/main.yml

@@ -0,0 +1,10 @@
+# k3s-Agent in den GPU-VMs (D6). Der Server heißt k3s.knaster (/etc/hosts), Token aus der Datei auf ops.
+k3s_version: v1.36.5+k3s1
+k3s_server_name: k3s.knaster
+k3s_server_host: "{{ hostvars['k3s'].ansible_host }}"
+k3s_env_file: /root/.config/knaster/k3s.env
+k3s_token: "{{ lookup('ini', 'K3S_TOKEN', type='properties', file=k3s_env_file) }}"
+k3s_install_script: /usr/local/src/k3s-install.sh
+k3s_node_labels: []
+k3s_lan: 192.168.100.0/24
+k3s_pod_cidr: 10.42.0.0/16

platform/proxmox/ansible/roles/k3s-agent/tasks/main.yml

@@ -0,0 +1,92 @@
+# k3s-Agent neben Docker: k3s bringt sein eigenes containerd mit und erkennt nvidia-container-runtime
+# von selbst (Runtime "nvidia"). Docker bleibt für die bestehenden Compose-Stacks, bis sie umgezogen sind.
+- name: Server-Name in /etc/hosts (Adresse per DHCP, D5)
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    regexp: '\s{{ k3s_server_name | regex_escape }}$'
+    line: "{{ k3s_server_host }} {{ k3s_server_name }}"
+
+- name: curl vorhanden
+  ansible.builtin.apt:
+    name: curl
+    state: present
+
+- name: Installationsskript laden
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: "{{ k3s_install_script }}"
+    mode: "0755"
+
+- name: Konfiguration /etc/rancher/k3s/config.yaml
+  ansible.builtin.copy:
+    dest: /etc/rancher/k3s/config.yaml
+    mode: "0600"
+    content: |
+      # Verwaltet von Ansible (Rolle k3s-agent)
+      server: https://{{ k3s_server_name }}:6443
+      token: {{ k3s_token }}
+      node-label: {{ k3s_node_labels | to_json }}
+  register: k3s_agent_config
+  no_log: true
+
+# ufw (nur in VMs mit aktiver Firewall): Kubelet und NodePorts aus dem LAN, Flannel-VXLAN zwischen den
+# Knoten, alles aus dem Pod-Netz (CoreDNS, Device-Plugin, Portainer-Agent sprechen mit dem Knoten).
+- name: Ist ufw vorhanden?
+  ansible.builtin.stat:
+    path: /usr/sbin/ufw
+  register: k3s_ufw
+
+- name: ufw – k3s-Ports
+  community.general.ufw:
+    rule: allow
+    from_ip: "{{ item.from }}"
+    port: "{{ item.port | default(omit) }}"
+    proto: "{{ item.proto | default(omit) }}"
+    comment: "{{ item.comment }}"
+  loop:
+    - {from: "{{ k3s_lan }}", port: "10250", proto: tcp, comment: k3s-kubelet}
+    - {from: "{{ k3s_lan }}", port: "8472", proto: udp, comment: k3s-flannel}
+    - {from: "{{ k3s_lan }}", port: "30000:32767", proto: tcp, comment: k3s-nodeport}
+    - {from: "{{ k3s_pod_cidr }}", comment: k3s-pods}
+  loop_control:
+    label: "{{ item.comment }}"
+  when: k3s_ufw.stat.exists
+
+- name: Installierte Version lesen
+  ansible.builtin.command: /usr/local/bin/k3s --version
+  register: k3s_installed
+  failed_when: false
+  changed_when: false
+
+- name: k3s-Agent installieren oder auf die gepinnte Version bringen
+  ansible.builtin.command: "{{ k3s_install_script }} agent"
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version }}"
+  when: k3s_version not in (k3s_installed.stdout | default(''))
+
+- name: Agent neu starten, wenn sich die Konfiguration geändert hat
+  ansible.builtin.service:
+    name: k3s-agent
+    state: restarted
+  when: k3s_agent_config.changed and k3s_version in (k3s_installed.stdout | default(''))
+
+- name: Warten, bis containerd von k3s läuft
+  ansible.builtin.wait_for:
+    path: /var/lib/rancher/k3s/agent/etc/containerd/config.toml
+    timeout: 120
+
+- name: NVIDIA-Runtime von k3s erkannt?
+  ansible.builtin.command: grep -c nvidia /var/lib/rancher/k3s/agent/etc/containerd/config.toml
+  register: k3s_nvidia
+  changed_when: false
+  failed_when: k3s_nvidia.stdout | int == 0
+
+- name: Warten, bis der Knoten im Cluster bereit ist (von ops aus)
+  ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  register: k3s_node_ready
+  until: k3s_node_ready.stdout == "True"
+  retries: 36
+  delay: 5
+  changed_when: false
+  delegate_to: localhost
+  become: false

platform/proxmox/ansible/roles/k3s-apps/defaults/main.yml

@@ -0,0 +1,6 @@
+# Grundausstattung im k3s-Cluster (von ops per kubectl): NVIDIA-Runtime und Device-Plugin, Portainer-Agent.
+k3s_manifest_dir: "{{ playbook_dir }}/../../../../deploy/k8s"
+k3s_base_manifests:
+  - runtimeclass-nvidia.yaml
+  - nvidia-device-plugin.yaml
+  - portainer-agent.yaml

platform/proxmox/ansible/roles/k3s-apps/tasks/main.yml

@@ -0,0 +1,28 @@
+- name: Grund-Manifeste anwenden
+  ansible.builtin.command: kubectl apply -f {{ k3s_manifest_dir }}/{{ item }}
+  loop: "{{ k3s_base_manifests }}"
+  register: k3s_apply
+  changed_when: "'configured' in k3s_apply.stdout or 'created' in k3s_apply.stdout"
+
+- name: Warten, bis jede GPU-VM ihre GPUs meldet (nvidia.com/gpu)
+  ansible.builtin.command: >
+    kubectl get node {{ item }} -o jsonpath='{.status.allocatable.nvidia\.com/gpu}'
+  loop: "{{ groups['k3s_agents'] }}"
+  register: k3s_gpus
+  until: (k3s_gpus.stdout | default('0') | int) > 0
+  retries: 24
+  delay: 5
+  changed_when: false
+
+- name: Warten, bis der Portainer-Agent läuft
+  ansible.builtin.command: kubectl -n portainer rollout status deployment/portainer-agent --timeout=180s
+  changed_when: false
+
+- name: Übersicht
+  ansible.builtin.command: kubectl get nodes -o custom-columns=NAME:.metadata.name,STATUS:.status.conditions[-1].type,GPU:.status.allocatable.nvidia\.com/gpu,VERSION:.status.nodeInfo.kubeletVersion
+  register: k3s_nodes
+  changed_when: false
+
+- name: Knoten
+  ansible.builtin.debug:
+    msg: "{{ k3s_nodes.stdout_lines }}"

platform/proxmox/ansible/roles/k3s-server/defaults/main.yml

@@ -0,0 +1,8 @@
+# k3s-Steuerung (D6) auf der Dienst-VM "k3s". Version gepinnt (Kanal stable, 15.10.2026).
+k3s_version: v1.36.5+k3s1
+k3s_server_name: k3s.knaster            # Name, unter dem die Agents den Server finden (/etc/hosts, Rolle k3s-agent)
+k3s_install_script: /usr/local/src/k3s-install.sh
+# Geheimnisse und Zugang landen auf ops, nicht im Repo
+k3s_env_file: /root/.config/knaster/k3s.env   # K3S_URL und K3S_TOKEN für die Agents
+k3s_kubeconfig: /root/.kube/config            # kubectl auf ops
+k3s_kubectl_version: "{{ k3s_version | regex_replace('\+k3s\d+$', '') }}"

platform/proxmox/ansible/roles/k3s-server/tasks/main.yml

@@ -0,0 +1,98 @@
+# k3s-Server (Steuerung, Datenbank, API) auf der Dienst-VM. Ohne traefik (kein Ingress nötig),
+# servicelb bleibt für Dienste vom Typ LoadBalancer. Die Agents (GPU-VMs) kommen über die Rolle k3s-agent.
+- name: curl vorhanden
+  ansible.builtin.apt:
+    name: curl
+    state: present
+    update_cache: true
+
+- name: Installationsskript laden
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: "{{ k3s_install_script }}"
+    mode: "0755"
+
+- name: Konfiguration /etc/rancher/k3s/config.yaml
+  ansible.builtin.copy:
+    dest: /etc/rancher/k3s/config.yaml
+    mode: "0600"
+    content: |
+      # Verwaltet von Ansible (Rolle k3s-server)
+      disable: [traefik]
+      write-kubeconfig-mode: "0600"
+      tls-san: [{{ k3s_server_name }}, {{ ansible_host }}]
+      node-label: [knaster.role=control]
+  register: k3s_server_config
+  vars:
+    ansible_become: true
+
+- name: Installierte Version lesen
+  ansible.builtin.command: /usr/local/bin/k3s --version
+  register: k3s_installed
+  failed_when: false
+  changed_when: false
+
+- name: k3s-Server installieren oder auf die gepinnte Version bringen
+  ansible.builtin.command: "{{ k3s_install_script }} server"
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version }}"
+  when: k3s_version not in (k3s_installed.stdout | default(''))
+
+- name: k3s neu starten, wenn sich die Konfiguration geändert hat
+  ansible.builtin.service:
+    name: k3s
+    state: restarted
+  when: k3s_server_config.changed and k3s_version in (k3s_installed.stdout | default(''))
+
+- name: Warten, bis der Knoten bereit ist
+  ansible.builtin.command: /usr/local/bin/k3s kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  register: k3s_ready
+  until: k3s_ready.stdout == "True"
+  retries: 30
+  delay: 5
+  changed_when: false
+
+- name: Token und kubeconfig lesen
+  ansible.builtin.slurp:
+    src: "{{ item }}"
+  loop: [/var/lib/rancher/k3s/server/node-token, /etc/rancher/k3s/k3s.yaml]
+  register: k3s_secrets
+  no_log: true
+
+- name: Zugang nach ops schreiben
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0700"
+  loop: ["{{ k3s_env_file | dirname }}", "{{ k3s_kubeconfig | dirname }}"]
+  delegate_to: localhost
+  become: false
+
+- name: K3S_URL und K3S_TOKEN auf ops ({{ k3s_env_file }})
+  ansible.builtin.copy:
+    dest: "{{ k3s_env_file }}"
+    mode: "0600"
+    content: |
+      K3S_URL=https://{{ k3s_server_name }}:6443
+      K3S_TOKEN={{ k3s_secrets.results[0].content | b64decode | trim }}
+  delegate_to: localhost
+  become: false
+  no_log: true
+
+- name: kubeconfig auf ops ({{ k3s_kubeconfig }}, Server = VM-Adresse)
+  ansible.builtin.copy:
+    dest: "{{ k3s_kubeconfig }}"
+    mode: "0600"
+    content: "{{ k3s_secrets.results[1].content | b64decode | replace('https://127.0.0.1:6443', 'https://' ~ ansible_host ~ ':6443') }}"
+  delegate_to: localhost
+  become: false
+  no_log: true
+
+- name: kubectl auf ops (Version passend zum Server)
+  ansible.builtin.get_url:
+    url: "https://dl.k8s.io/release/{{ k3s_kubectl_version }}/bin/linux/amd64/kubectl"
+    checksum: "sha256:https://dl.k8s.io/release/{{ k3s_kubectl_version }}/bin/linux/amd64/kubectl.sha256"
+    dest: /usr/local/bin/kubectl
+    mode: "0755"
+  delegate_to: localhost
+  become: false

platform/proxmox/ansible/roles/nex-speech-k3s/defaults/main.yml

@@ -0,0 +1,6 @@
+# Umzug der NEX-Sprachdienste von Docker Compose nach k3s (gleiche VM, gleiche Ports, gleiche Images).
+nex_speech_dir: /opt/nex-speech
+nex_speech_images:
+  - knaster/nex-whisper:v1.9.4-cu12.6
+  - knaster/nex-tts-piper:1.8.0
+nex_speech_k3s_manifests: "{{ playbook_dir }}/../../../../deploy/k8s/nex-speech"

platform/proxmox/ansible/roles/nex-speech-k3s/tasks/main.yml

@@ -0,0 +1,40 @@
+# 1. Images aus Docker in das containerd von k3s übernehmen (kein Neubau, kein Registry-Umweg)
+- name: Images in k3s vorhanden?
+  ansible.builtin.command: /usr/local/bin/k3s ctr images ls -q
+  register: k3s_images
+  changed_when: false
+
+- name: Images aus Docker importieren
+  ansible.builtin.shell: docker save {{ item }} | /usr/local/bin/k3s ctr images import -
+  loop: "{{ nex_speech_images }}"
+  when: ("docker.io/" ~ item) not in k3s_images.stdout
+  async: 1800
+  poll: 15
+
+# 2. Compose-Stack anhalten (Host-Ports 8178/8179 werden frei), Dateien bleiben für den Rückweg
+- name: Compose-Stack nex-speech anhalten
+  community.docker.docker_compose_v2:
+    project_src: "{{ nex_speech_dir }}"
+    state: stopped
+  when: nex_speech_compose_stop | default(true)
+
+# 3. Manifeste anwenden und auf Bereitschaft warten (Probes prüfen /health in der VM)
+- name: Manifeste anwenden
+  ansible.builtin.command: kubectl apply -f {{ nex_speech_k3s_manifests }}/
+  register: nex_apply
+  changed_when: "'configured' in nex_apply.stdout or 'created' in nex_apply.stdout"
+  delegate_to: localhost
+  become: false
+
+- name: Warten, bis beide Deployments bereit sind
+  ansible.builtin.command: kubectl -n nex rollout status deployment/{{ item }} --timeout=600s
+  loop: [nex-stt, nex-tts]
+  changed_when: false
+  delegate_to: localhost
+  become: false
+
+- name: Gesundheit in der VM prüfen
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ item }}/health"
+    status_code: 200
+  loop: [8178, 8179]

platform/proxmox/ansible/roles/pve-host/tasks/main.yml

@@ -103,6 +103,20 @@
   check_mode: false
   changed_when: false
 
+- name: VMs auf diesem Knoten lesen
+  ansible.builtin.command: ls /etc/pve/qemu-server/
+  register: local_vms
+  check_mode: false
+  changed_when: false
+
+# Infra-Gäste = Container (ct:) plus Dienst-VMs ohne GPU (vm:, D6); beide liegen auf tank mit HA
+- name: Infra-Gäste zusammenstellen
+  ansible.builtin.set_fact:
+    pve_infra_guests: >-
+      {{ (knaster_pve.containers | dict2items | map('combine', {'kind': 'ct'}) | list)
+         + (knaster_pve.service_vms | default({}) | dict2items | map('combine', {'kind': 'vm'}) | list) }}
+    pve_local_guest_confs: "{{ local_cts.stdout_lines + local_vms.stdout_lines }}"
+
 - name: Bestehende Replikationsjobs lesen
   ansible.builtin.command: pvesr list
   register: pvesr_jobs
@@ -113,12 +127,12 @@
   ansible.builtin.command: >
     pvesr create-local-job {{ item.0.value.vmid }}-{{ idx }} {{ item.1 }}
     --schedule '{{ knaster_pve.replication_schedule }}' --comment 'knaster {{ item.0.key }}'
-  loop: "{{ knaster_pve.containers | dict2items | subelements('value.replicate_to', skip_missing=True) }}"
+  loop: "{{ pve_infra_guests | subelements('value.replicate_to', skip_missing=True) }}"
   vars:
     idx: "{{ item.0.value.replicate_to.index(item.1) }}"
   when:
     - item.0.value.node == inventory_hostname
-    - (item.0.value.vmid | string) + '.conf' in local_cts.stdout_lines
+    - (item.0.value.vmid | string) + '.conf' in pve_local_guest_confs
     - (item.0.value.vmid | string) + '-' + (idx | string) not in pvesr_jobs.stdout
   changed_when: true
 
@@ -130,14 +144,14 @@
 
 - name: Infra-Gäste unter HA stellen (erst wenn der Container auf diesem Knoten existiert)
   ansible.builtin.command: >
-    ha-manager add ct:{{ item.value.vmid }} --state started --max_restart 1 --max_relocate 1
+    ha-manager add {{ item.kind }}:{{ item.value.vmid }} --state started --max_restart 1 --max_relocate 1
     --comment 'knaster {{ item.key }}'
-  loop: "{{ knaster_pve.containers | dict2items }}"
+  loop: "{{ pve_infra_guests }}"
   when:
     - item.value.ha | default(false)
     - item.value.node == inventory_hostname
-    - (item.value.vmid | string) + '.conf' in local_cts.stdout_lines
-    - ('ct:' + (item.value.vmid | string)) not in ha_config.stdout
+    - (item.value.vmid | string) + '.conf' in pve_local_guest_confs
+    - (item.kind + ':' + (item.value.vmid | string)) not in ha_config.stdout
   changed_when: true
 
 - name: HA-Regeln lesen
@@ -149,13 +163,13 @@
 # Strikt: Die Gäste liegen auf tank, den gibt es nur auf dem Heimatknoten und den Replikationszielen
 - name: Node-Affinity für Infra-Gäste (Heimatknoten bevorzugt, sonst Replikationsziele)
   ansible.builtin.command: >
-    ha-manager rules add node-affinity knaster-{{ item.key }} --resources ct:{{ item.value.vmid }}
+    ha-manager rules add node-affinity knaster-{{ item.key }} --resources {{ item.kind }}:{{ item.value.vmid }}
     --nodes {{ ([item.value.node ~ ':2'] + (item.value.replicate_to | map('regex_replace', '$', ':1') | list)) | join(',') }}
     --strict 1 --comment 'knaster {{ item.key }}'
-  loop: "{{ knaster_pve.containers | dict2items }}"
+  loop: "{{ pve_infra_guests }}"
   when:
     - item.value.ha | default(false)
     - item.value.node == inventory_hostname
-    - (item.value.vmid | string) + '.conf' in local_cts.stdout_lines
+    - (item.value.vmid | string) + '.conf' in pve_local_guest_confs
     - ('knaster-' + item.key) not in ha_rules.stdout
   changed_when: true