k3s (D6): Dienst-VM k3s per tofu, Rollen k3s-server/k3s-agent/k3s-apps, Manifeste, NEX-Sprachdienste als k3s-Umzugstest

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-15 11:55:18 +02:00
parent 6cccc01d14
commit 2c1d43e9c7
23 changed files with 701 additions and 15 deletions
@@ -0,0 +1,12 @@
+# Kubernetes-Manifeste (k3s, D6)
+
+Ausgerollt von ops mit `kubectl` (Rolle `k3s-apps`, Playbook `playbooks/k3s.yml`):
+
+| Datei | Zweck |
+|---|---|
+| `runtimeclass-nvidia.yaml` | RuntimeClass `nvidia` (Runtime legt k3s in den GPU-VMs selbst an) |
+| `nvidia-device-plugin.yaml` | NVIDIA Device Plugin v0.20.1, nur auf Knoten mit Label `knaster.gpu=true`; liefert `nvidia.com/gpu` |
+| `portainer-agent.yaml` | Portainer Agent 2.45.2 (NodePort 30778), Quelle downloads.portainer.io/ce2-45 |
+| `nex-speech/` | NEX-Sprachdienste auf gpu-pve52 (Playbook `playbooks/nex-speech-k3s.yml`) |
+
+Portainer: Umgebung „Kubernetes → Agent“, Adresse `<IP eines Knotens>:30778`.
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nex
+  labels:
+    knaster.app: nex-speech
@@ -0,0 +1,56 @@
+# NEX-Spracherkennung (whisper.cpp, M4000) auf gpu-pve52, umgezogen von Docker Compose (Task 6.2, D6).
+# Host-Netz wie zuvor: Port 8178 auf der VM-Adresse, die ufw-Regel der VM (nur appsrv08) gilt weiter.
+# Image stammt aus dem Compose-Build und wurde in das containerd von k3s importiert (Rolle nex-speech-k3s).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nex-stt
+  namespace: nex
+  labels: {app: nex-stt}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}   # Host-Port 8178 kann nur einmal belegt sein
+  selector:
+    matchLabels: {app: nex-stt}
+  template:
+    metadata:
+      labels: {app: nex-stt}
+    spec:
+      runtimeClassName: nvidia
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      nodeSelector:
+        kubernetes.io/hostname: gpu-pve52
+      containers:
+        - name: whisper
+          image: knaster/nex-whisper:v1.9.4-cu12.6
+          imagePullPolicy: Never
+          env:
+            - {name: WHISPER_PORT, value: "8178"}
+            - {name: WHISPER_MODEL, value: /models/whisper/ggml-large-v3-turbo.bin}
+            - {name: WHISPER_LANGUAGE, value: de}
+            - {name: WHISPER_BEAM_SIZE, value: "5"}
+            - {name: WHISPER_EXTRA_ARGS, value: ""}
+          resources:
+            limits: {nvidia.com/gpu: 1}
+          securityContext:
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - {name: models, mountPath: /models/whisper, readOnly: true}
+            - {name: run, mountPath: /run/nex-whisper}
+          startupProbe:
+            httpGet: {path: /health, port: 8178, host: 127.0.0.1}
+            periodSeconds: 5
+            failureThreshold: 60
+          readinessProbe:
+            httpGet: {path: /health, port: 8178, host: 127.0.0.1}
+            periodSeconds: 30
+          livenessProbe:
+            httpGet: {path: /health, port: 8178, host: 127.0.0.1}
+            periodSeconds: 30
+            failureThreshold: 3
+      volumes:
+        - name: models
+          hostPath: {path: /srv/models/whisper, type: Directory}
+        - name: run
+          emptyDir: {medium: Memory, sizeLimit: 512Mi}
@@ -0,0 +1,56 @@
+# NEX-Sprachausgabe (Piper, CPU) auf gpu-pve52, Host-Netz Port 8179 (ufw: nur appsrv08).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nex-tts
+  namespace: nex
+  labels: {app: nex-tts}
+spec:
+  replicas: 1
+  strategy: {type: Recreate}
+  selector:
+    matchLabels: {app: nex-tts}
+  template:
+    metadata:
+      labels: {app: nex-tts}
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      nodeSelector:
+        kubernetes.io/hostname: gpu-pve52
+      containers:
+        - name: piper
+          image: knaster/nex-tts-piper:1.8.0
+          imagePullPolicy: Never
+          env:
+            - {name: TTS_PORT, value: "8179"}
+            - {name: PIPER_VOICES_DIR, value: /voices}
+            - {name: PIPER_VOICES, value: "de_DE-thorsten-high,de_DE-kerstin-low"}
+            - {name: PIPER_DEFAULT_VOICE, value: de_DE-thorsten-high}
+            - {name: PCM_SAMPLE_RATE, value: "24000"}
+            - {name: TTS_MAX_CHARS, value: "5000"}
+            - {name: TTS_PARALLEL, value: "2"}
+          resources:
+            limits: {cpu: "3", memory: 2Gi}   # ein Kern bleibt für whisper-server und das System
+            requests: {cpu: "500m", memory: 512Mi}
+          securityContext:
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - {name: voices, mountPath: /voices, readOnly: true}
+            - {name: tmp, mountPath: /tmp}
+          startupProbe:
+            httpGet: {path: /health, port: 8179, host: 127.0.0.1}
+            periodSeconds: 5
+            failureThreshold: 24
+          readinessProbe:
+            httpGet: {path: /health, port: 8179, host: 127.0.0.1}
+            periodSeconds: 30
+          livenessProbe:
+            httpGet: {path: /health, port: 8179, host: 127.0.0.1}
+            periodSeconds: 30
+            failureThreshold: 3
+      volumes:
+        - name: voices
+          hostPath: {path: /srv/models/piper, type: Directory}
+        - name: tmp
+          emptyDir: {sizeLimit: 64Mi}
@@ -0,0 +1,44 @@
+# NVIDIA Device Plugin v0.20.1 (Quelle: deployments/static/nvidia-device-plugin.yml des Projekts),
+# ergänzt um runtimeClassName nvidia und nodeSelector knaster.gpu=true (nur die GPU-VMs).
+# Meldet je Knoten die Ressource nvidia.com/gpu, die Pods anfordern können.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-device-plugin-daemonset
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: nvidia-device-plugin-ds
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: nvidia-device-plugin-ds
+    spec:
+      runtimeClassName: nvidia
+      nodeSelector:
+        knaster.gpu: "true"
+      tolerations:
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+      priorityClassName: system-node-critical
+      containers:
+        - image: nvcr.io/nvidia/k8s-device-plugin:v0.20.1
+          name: nvidia-device-plugin-ctr
+          env:
+            - name: FAIL_ON_INIT_ERROR
+              value: "false"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: device-plugin
+              mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+        - name: device-plugin
+          hostPath:
+            path: /var/lib/kubelet/device-plugins
@@ -0,0 +1,81 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: portainer
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: portainer-sa-clusteradmin
+  namespace: portainer
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: portainer-crb-clusteradmin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: portainer-sa-clusteradmin
+  namespace: portainer
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: portainer-agent
+  namespace: portainer
+spec:
+  type: NodePort
+  selector:
+    app: portainer-agent
+  ports:
+    - name: http
+      protocol: TCP
+      port: 9001
+      targetPort: 9001
+      nodePort: 30778
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: portainer-agent-headless
+  namespace: portainer
+spec:
+  clusterIP: None
+  selector:
+    app: portainer-agent
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: portainer-agent
+  namespace: portainer
+spec:
+  selector:
+    matchLabels:
+      app: portainer-agent
+  template:
+    metadata:
+      labels:
+        app: portainer-agent
+    spec:
+      serviceAccountName: portainer-sa-clusteradmin
+      containers:
+      - name: portainer-agent
+        image: portainer/agent:2.45.2
+        imagePullPolicy: Always
+        env:
+        - name: LOG_LEVEL
+          value: DEBUG
+        - name: AGENT_CLUSTER_ADDR
+          value: "portainer-agent-headless"
+        - name: KUBERNETES_POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        ports:
+        - containerPort: 9001
+          protocol: TCP
@@ -0,0 +1,7 @@
+# Runtime "nvidia" legt k3s in seinem containerd an, sobald nvidia-container-runtime in der VM liegt.
+# Pods mit runtimeClassName: nvidia bekommen die GPU-Bibliotheken eingehängt.
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: nvidia
+handler: nvidia
@@ -19,11 +19,11 @@ all:
      vars:
        ansible_user: root
    gpu_vms:
-      hosts:
-        gpu-pve44: {driver_generation: legacy}
-        gpu-pve46: {driver_generation: legacy}
-        gpu-pve48: {driver_generation: legacy}
-        gpu-pve52: {driver_generation: legacy}
+      hosts:   # k3s_node_labels: Knoten-Labels für die Platzierung von GPU-Pods (D6)
+        gpu-pve44: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1, knaster.gpu.p2000=1]}
+        gpu-pve46: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1, knaster.gpu.p2000=1]}
+        gpu-pve48: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1, knaster.gpu.p2000=1]}
+        gpu-pve52: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1]}
      vars:
        ansible_user: knaster
        ansible_become: true
@@ -39,3 +39,17 @@ all:
        infra: {}
      vars:
        ansible_user: root
+    # Dienst-VMs ohne GPU (D6): k3s-Steuerung. Adresse aus inventory.generated.yml (Gruppe service_vms).
+    service_vms:
+      hosts:
+        k3s: {}
+      vars:
+        ansible_user: knaster
+        ansible_become: true
+        ansible_python_interpreter: /usr/bin/python3
+    k3s_server:
+      hosts:
+        k3s: {}
+    k3s_agents:
+      children:
+        gpu_vms: {}
@@ -0,0 +1,16 @@
+# k3s-Cluster (D6): Server auf der Dienst-VM "k3s", Agents in den GPU-VMs, danach Runtime, Device-Plugin,
+# Portainer-Agent. Aufruf auf ops (vorher bash ../scripts/refresh-vm-access.sh, Adressen per DHCP):
+#   ansible-playbook playbooks/k3s.yml
+- name: k3s-Server
+  hosts: k3s_server
+  roles: [k3s-server]
+
+- name: k3s-Agents (GPU-VMs)
+  hosts: k3s_agents
+  roles: [k3s-agent]
+
+- name: Grundausstattung im Cluster (von ops)
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  roles: [k3s-apps]
@@ -0,0 +1,5 @@
+# NEX-Sprachdienste von Docker Compose nach k3s umziehen (Test für D6). Rückweg: Deployments löschen
+# (kubectl -n nex delete deploy nex-stt nex-tts) und auf gpu-pve52 `docker compose -f /opt/nex-speech start`.
+- name: NEX-Sprachdienste nach k3s
+  hosts: "{{ nex_speech_hosts | default('gpu-pve52') }}"
+  roles: [nex-speech-k3s]
@@ -0,0 +1,10 @@
+# k3s-Agent in den GPU-VMs (D6). Der Server heißt k3s.knaster (/etc/hosts), Token aus der Datei auf ops.
+k3s_version: v1.36.5+k3s1
+k3s_server_name: k3s.knaster
+k3s_server_host: "{{ hostvars['k3s'].ansible_host }}"
+k3s_env_file: /root/.config/knaster/k3s.env
+k3s_token: "{{ lookup('ini', 'K3S_TOKEN', type='properties', file=k3s_env_file) }}"
+k3s_install_script: /usr/local/src/k3s-install.sh
+k3s_node_labels: []
+k3s_lan: 192.168.100.0/24
+k3s_pod_cidr: 10.42.0.0/16
@@ -0,0 +1,92 @@
+# k3s-Agent neben Docker: k3s bringt sein eigenes containerd mit und erkennt nvidia-container-runtime
+# von selbst (Runtime "nvidia"). Docker bleibt für die bestehenden Compose-Stacks, bis sie umgezogen sind.
+- name: Server-Name in /etc/hosts (Adresse per DHCP, D5)
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    regexp: '\s{{ k3s_server_name | regex_escape }}$'
+    line: "{{ k3s_server_host }} {{ k3s_server_name }}"
+
+- name: curl vorhanden
+  ansible.builtin.apt:
+    name: curl
+    state: present
+
+- name: Installationsskript laden
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: "{{ k3s_install_script }}"
+    mode: "0755"
+
+- name: Konfiguration /etc/rancher/k3s/config.yaml
+  ansible.builtin.copy:
+    dest: /etc/rancher/k3s/config.yaml
+    mode: "0600"
+    content: |
+      # Verwaltet von Ansible (Rolle k3s-agent)
+      server: https://{{ k3s_server_name }}:6443
+      token: {{ k3s_token }}
+      node-label: {{ k3s_node_labels | to_json }}
+  register: k3s_agent_config
+  no_log: true
+
+# ufw (nur in VMs mit aktiver Firewall): Kubelet und NodePorts aus dem LAN, Flannel-VXLAN zwischen den
+# Knoten, alles aus dem Pod-Netz (CoreDNS, Device-Plugin, Portainer-Agent sprechen mit dem Knoten).
+- name: Ist ufw vorhanden?
+  ansible.builtin.stat:
+    path: /usr/sbin/ufw
+  register: k3s_ufw
+
+- name: ufw – k3s-Ports
+  community.general.ufw:
+    rule: allow
+    from_ip: "{{ item.from }}"
+    port: "{{ item.port | default(omit) }}"
+    proto: "{{ item.proto | default(omit) }}"
+    comment: "{{ item.comment }}"
+  loop:
+    - {from: "{{ k3s_lan }}", port: "10250", proto: tcp, comment: k3s-kubelet}
+    - {from: "{{ k3s_lan }}", port: "8472", proto: udp, comment: k3s-flannel}
+    - {from: "{{ k3s_lan }}", port: "30000:32767", proto: tcp, comment: k3s-nodeport}
+    - {from: "{{ k3s_pod_cidr }}", comment: k3s-pods}
+  loop_control:
+    label: "{{ item.comment }}"
+  when: k3s_ufw.stat.exists
+
+- name: Installierte Version lesen
+  ansible.builtin.command: /usr/local/bin/k3s --version
+  register: k3s_installed
+  failed_when: false
+  changed_when: false
+
+- name: k3s-Agent installieren oder auf die gepinnte Version bringen
+  ansible.builtin.command: "{{ k3s_install_script }} agent"
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version }}"
+  when: k3s_version not in (k3s_installed.stdout | default(''))
+
+- name: Agent neu starten, wenn sich die Konfiguration geändert hat
+  ansible.builtin.service:
+    name: k3s-agent
+    state: restarted
+  when: k3s_agent_config.changed and k3s_version in (k3s_installed.stdout | default(''))
+
+- name: Warten, bis containerd von k3s läuft
+  ansible.builtin.wait_for:
+    path: /var/lib/rancher/k3s/agent/etc/containerd/config.toml
+    timeout: 120
+
+- name: NVIDIA-Runtime von k3s erkannt?
+  ansible.builtin.command: grep -c nvidia /var/lib/rancher/k3s/agent/etc/containerd/config.toml
+  register: k3s_nvidia
+  changed_when: false
+  failed_when: k3s_nvidia.stdout | int == 0
+
+- name: Warten, bis der Knoten im Cluster bereit ist (von ops aus)
+  ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  register: k3s_node_ready
+  until: k3s_node_ready.stdout == "True"
+  retries: 36
+  delay: 5
+  changed_when: false
+  delegate_to: localhost
+  become: false
@@ -0,0 +1,6 @@
+# Grundausstattung im k3s-Cluster (von ops per kubectl): NVIDIA-Runtime und Device-Plugin, Portainer-Agent.
+k3s_manifest_dir: "{{ playbook_dir }}/../../../../deploy/k8s"
+k3s_base_manifests:
+  - runtimeclass-nvidia.yaml
+  - nvidia-device-plugin.yaml
+  - portainer-agent.yaml
@@ -0,0 +1,28 @@
+- name: Grund-Manifeste anwenden
+  ansible.builtin.command: kubectl apply -f {{ k3s_manifest_dir }}/{{ item }}
+  loop: "{{ k3s_base_manifests }}"
+  register: k3s_apply
+  changed_when: "'configured' in k3s_apply.stdout or 'created' in k3s_apply.stdout"
+
+- name: Warten, bis jede GPU-VM ihre GPUs meldet (nvidia.com/gpu)
+  ansible.builtin.command: >
+    kubectl get node {{ item }} -o jsonpath='{.status.allocatable.nvidia\.com/gpu}'
+  loop: "{{ groups['k3s_agents'] }}"
+  register: k3s_gpus
+  until: (k3s_gpus.stdout | default('0') | int) > 0
+  retries: 24
+  delay: 5
+  changed_when: false
+
+- name: Warten, bis der Portainer-Agent läuft
+  ansible.builtin.command: kubectl -n portainer rollout status deployment/portainer-agent --timeout=180s
+  changed_when: false
+
+- name: Übersicht
+  ansible.builtin.command: kubectl get nodes -o custom-columns=NAME:.metadata.name,STATUS:.status.conditions[-1].type,GPU:.status.allocatable.nvidia\.com/gpu,VERSION:.status.nodeInfo.kubeletVersion
+  register: k3s_nodes
+  changed_when: false
+
+- name: Knoten
+  ansible.builtin.debug:
+    msg: "{{ k3s_nodes.stdout_lines }}"
@@ -0,0 +1,8 @@
+# k3s-Steuerung (D6) auf der Dienst-VM "k3s". Version gepinnt (Kanal stable, 15.10.2026).
+k3s_version: v1.36.5+k3s1
+k3s_server_name: k3s.knaster            # Name, unter dem die Agents den Server finden (/etc/hosts, Rolle k3s-agent)
+k3s_install_script: /usr/local/src/k3s-install.sh
+# Geheimnisse und Zugang landen auf ops, nicht im Repo
+k3s_env_file: /root/.config/knaster/k3s.env   # K3S_URL und K3S_TOKEN für die Agents
+k3s_kubeconfig: /root/.kube/config            # kubectl auf ops
+k3s_kubectl_version: "{{ k3s_version | regex_replace('\+k3s\d+$', '') }}"
@@ -0,0 +1,98 @@
+# k3s-Server (Steuerung, Datenbank, API) auf der Dienst-VM. Ohne traefik (kein Ingress nötig),
+# servicelb bleibt für Dienste vom Typ LoadBalancer. Die Agents (GPU-VMs) kommen über die Rolle k3s-agent.
+- name: curl vorhanden
+  ansible.builtin.apt:
+    name: curl
+    state: present
+    update_cache: true
+
+- name: Installationsskript laden
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: "{{ k3s_install_script }}"
+    mode: "0755"
+
+- name: Konfiguration /etc/rancher/k3s/config.yaml
+  ansible.builtin.copy:
+    dest: /etc/rancher/k3s/config.yaml
+    mode: "0600"
+    content: |
+      # Verwaltet von Ansible (Rolle k3s-server)
+      disable: [traefik]
+      write-kubeconfig-mode: "0600"
+      tls-san: [{{ k3s_server_name }}, {{ ansible_host }}]
+      node-label: [knaster.role=control]
+  register: k3s_server_config
+  vars:
+    ansible_become: true
+
+- name: Installierte Version lesen
+  ansible.builtin.command: /usr/local/bin/k3s --version
+  register: k3s_installed
+  failed_when: false
+  changed_when: false
+
+- name: k3s-Server installieren oder auf die gepinnte Version bringen
+  ansible.builtin.command: "{{ k3s_install_script }} server"
+  environment:
+    INSTALL_K3S_VERSION: "{{ k3s_version }}"
+  when: k3s_version not in (k3s_installed.stdout | default(''))
+
+- name: k3s neu starten, wenn sich die Konfiguration geändert hat
+  ansible.builtin.service:
+    name: k3s
+    state: restarted
+  when: k3s_server_config.changed and k3s_version in (k3s_installed.stdout | default(''))
+
+- name: Warten, bis der Knoten bereit ist
+  ansible.builtin.command: /usr/local/bin/k3s kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  register: k3s_ready
+  until: k3s_ready.stdout == "True"
+  retries: 30
+  delay: 5
+  changed_when: false
+
+- name: Token und kubeconfig lesen
+  ansible.builtin.slurp:
+    src: "{{ item }}"
+  loop: [/var/lib/rancher/k3s/server/node-token, /etc/rancher/k3s/k3s.yaml]
+  register: k3s_secrets
+  no_log: true
+
+- name: Zugang nach ops schreiben
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0700"
+  loop: ["{{ k3s_env_file | dirname }}", "{{ k3s_kubeconfig | dirname }}"]
+  delegate_to: localhost
+  become: false
+
+- name: K3S_URL und K3S_TOKEN auf ops ({{ k3s_env_file }})
+  ansible.builtin.copy:
+    dest: "{{ k3s_env_file }}"
+    mode: "0600"
+    content: |
+      K3S_URL=https://{{ k3s_server_name }}:6443
+      K3S_TOKEN={{ k3s_secrets.results[0].content | b64decode | trim }}
+  delegate_to: localhost
+  become: false
+  no_log: true
+
+- name: kubeconfig auf ops ({{ k3s_kubeconfig }}, Server = VM-Adresse)
+  ansible.builtin.copy:
+    dest: "{{ k3s_kubeconfig }}"
+    mode: "0600"
+    content: "{{ k3s_secrets.results[1].content | b64decode | replace('https://127.0.0.1:6443', 'https://' ~ ansible_host ~ ':6443') }}"
+  delegate_to: localhost
+  become: false
+  no_log: true
+
+- name: kubectl auf ops (Version passend zum Server)
+  ansible.builtin.get_url:
+    url: "https://dl.k8s.io/release/{{ k3s_kubectl_version }}/bin/linux/amd64/kubectl"
+    checksum: "sha256:https://dl.k8s.io/release/{{ k3s_kubectl_version }}/bin/linux/amd64/kubectl.sha256"
+    dest: /usr/local/bin/kubectl
+    mode: "0755"
+  delegate_to: localhost
+  become: false
@@ -0,0 +1,6 @@
+# Umzug der NEX-Sprachdienste von Docker Compose nach k3s (gleiche VM, gleiche Ports, gleiche Images).
+nex_speech_dir: /opt/nex-speech
+nex_speech_images:
+  - knaster/nex-whisper:v1.9.4-cu12.6
+  - knaster/nex-tts-piper:1.8.0
+nex_speech_k3s_manifests: "{{ playbook_dir }}/../../../../deploy/k8s/nex-speech"
@@ -0,0 +1,40 @@
+# 1. Images aus Docker in das containerd von k3s übernehmen (kein Neubau, kein Registry-Umweg)
+- name: Images in k3s vorhanden?
+  ansible.builtin.command: /usr/local/bin/k3s ctr images ls -q
+  register: k3s_images
+  changed_when: false
+
+- name: Images aus Docker importieren
+  ansible.builtin.shell: docker save {{ item }} | /usr/local/bin/k3s ctr images import -
+  loop: "{{ nex_speech_images }}"
+  when: ("docker.io/" ~ item) not in k3s_images.stdout
+  async: 1800
+  poll: 15
+
+# 2. Compose-Stack anhalten (Host-Ports 8178/8179 werden frei), Dateien bleiben für den Rückweg
+- name: Compose-Stack nex-speech anhalten
+  community.docker.docker_compose_v2:
+    project_src: "{{ nex_speech_dir }}"
+    state: stopped
+  when: nex_speech_compose_stop | default(true)
+
+# 3. Manifeste anwenden und auf Bereitschaft warten (Probes prüfen /health in der VM)
+- name: Manifeste anwenden
+  ansible.builtin.command: kubectl apply -f {{ nex_speech_k3s_manifests }}/
+  register: nex_apply
+  changed_when: "'configured' in nex_apply.stdout or 'created' in nex_apply.stdout"
+  delegate_to: localhost
+  become: false
+
+- name: Warten, bis beide Deployments bereit sind
+  ansible.builtin.command: kubectl -n nex rollout status deployment/{{ item }} --timeout=600s
+  loop: [nex-stt, nex-tts]
+  changed_when: false
+  delegate_to: localhost
+  become: false
+
+- name: Gesundheit in der VM prüfen
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ item }}/health"
+    status_code: 200
+  loop: [8178, 8179]
@@ -103,6 +103,20 @@
  check_mode: false
  changed_when: false

+- name: VMs auf diesem Knoten lesen
+  ansible.builtin.command: ls /etc/pve/qemu-server/
+  register: local_vms
+  check_mode: false
+  changed_when: false
+
+# Infra-Gäste = Container (ct:) plus Dienst-VMs ohne GPU (vm:, D6); beide liegen auf tank mit HA
+- name: Infra-Gäste zusammenstellen
+  ansible.builtin.set_fact:
+    pve_infra_guests: >-
+      {{ (knaster_pve.containers | dict2items | map('combine', {'kind': 'ct'}) | list)
+         + (knaster_pve.service_vms | default({}) | dict2items | map('combine', {'kind': 'vm'}) | list) }}
+    pve_local_guest_confs: "{{ local_cts.stdout_lines + local_vms.stdout_lines }}"
+
 - name: Bestehende Replikationsjobs lesen
  ansible.builtin.command: pvesr list
  register: pvesr_jobs
@@ -113,12 +127,12 @@
  ansible.builtin.command: >
    pvesr create-local-job {{ item.0.value.vmid }}-{{ idx }} {{ item.1 }}
    --schedule '{{ knaster_pve.replication_schedule }}' --comment 'knaster {{ item.0.key }}'
-  loop: "{{ knaster_pve.containers | dict2items | subelements('value.replicate_to', skip_missing=True) }}"
+  loop: "{{ pve_infra_guests | subelements('value.replicate_to', skip_missing=True) }}"
  vars:
    idx: "{{ item.0.value.replicate_to.index(item.1) }}"
  when:
    - item.0.value.node == inventory_hostname
-    - (item.0.value.vmid | string) + '.conf' in local_cts.stdout_lines
+    - (item.0.value.vmid | string) + '.conf' in pve_local_guest_confs
    - (item.0.value.vmid | string) + '-' + (idx | string) not in pvesr_jobs.stdout
  changed_when: true

@@ -130,14 +144,14 @@

 - name: Infra-Gäste unter HA stellen (erst wenn der Container auf diesem Knoten existiert)
  ansible.builtin.command: >
-    ha-manager add ct:{{ item.value.vmid }} --state started --max_restart 1 --max_relocate 1
+    ha-manager add {{ item.kind }}:{{ item.value.vmid }} --state started --max_restart 1 --max_relocate 1
    --comment 'knaster {{ item.key }}'
-  loop: "{{ knaster_pve.containers | dict2items }}"
+  loop: "{{ pve_infra_guests }}"
  when:
    - item.value.ha | default(false)
    - item.value.node == inventory_hostname
-    - (item.value.vmid | string) + '.conf' in local_cts.stdout_lines
-    - ('ct:' + (item.value.vmid | string)) not in ha_config.stdout
+    - (item.value.vmid | string) + '.conf' in pve_local_guest_confs
+    - (item.kind + ':' + (item.value.vmid | string)) not in ha_config.stdout
  changed_when: true

 - name: HA-Regeln lesen
@@ -149,13 +163,13 @@
 # Strikt: Die Gäste liegen auf tank, den gibt es nur auf dem Heimatknoten und den Replikationszielen
 - name: Node-Affinity für Infra-Gäste (Heimatknoten bevorzugt, sonst Replikationsziele)
  ansible.builtin.command: >
-    ha-manager rules add node-affinity knaster-{{ item.key }} --resources ct:{{ item.value.vmid }}
+    ha-manager rules add node-affinity knaster-{{ item.key }} --resources {{ item.kind }}:{{ item.value.vmid }}
    --nodes {{ ([item.value.node ~ ':2'] + (item.value.replicate_to | map('regex_replace', '$', ':1') | list)) | join(',') }}
    --strict 1 --comment 'knaster {{ item.key }}'
-  loop: "{{ knaster_pve.containers | dict2items }}"
+  loop: "{{ pve_infra_guests }}"
  when:
    - item.value.ha | default(false)
    - item.value.node == inventory_hostname
-    - (item.value.vmid | string) + '.conf' in local_cts.stdout_lines
+    - (item.value.vmid | string) + '.conf' in pve_local_guest_confs
    - ('knaster-' + item.key) not in ha_rules.stdout
  changed_when: true
@@ -169,3 +169,74 @@ resource "proxmox_virtual_environment_container" "infra" {

 # HA (Ressource + Node-Affinity-Regel) und Replikation setzt die Ansible-Rolle pve-host per root-SSH.
 # So braucht der tofu-Token kein Sys.Console auf /.
+
+# Dienst-VMs ohne GPU (D6, k3s-Steuerung): Klon vom Template des Knotens auf tank, HA + Replikation
+# setzt die Rolle pve-host (wie bei den Containern).
+resource "proxmox_virtual_environment_vm" "svc" {
+  for_each  = try(local.site.service_vms, {})
+  name      = each.key
+  node_name = each.value.node
+  vm_id     = each.value.vmid
+  tags      = ["knaster", "infra"]
+  pool_id   = try(each.value.pool, "infra")
+
+  machine       = "q35"
+  bios          = "ovmf"
+  on_boot       = true
+  scsi_hardware = "virtio-scsi-single"
+
+  clone {
+    vm_id        = local.site.cluster.templates[each.value.node]
+    node_name    = each.value.node
+    datastore_id = each.value.storage
+    full         = true
+  }
+
+  agent {
+    enabled = true
+  }
+
+  cpu {
+    cores = each.value.cores
+    type  = "host"
+  }
+
+  memory {
+    dedicated = each.value.memory_mb
+    floating  = each.value.memory_mb # Ballooning erlaubt (kein Passthrough)
+  }
+
+  efi_disk {
+    datastore_id      = each.value.storage
+    type              = "4m"
+    pre_enrolled_keys = false
+  }
+
+  disk {
+    datastore_id = each.value.storage
+    interface    = "scsi0"
+    size         = each.value.disk_gb
+    discard      = "on"
+    ssd          = true
+    iothread     = true
+  }
+
+  network_device {
+    bridge = local.site.cluster.bridge
+  }
+
+  initialization {
+    datastore_id        = each.value.storage
+    vendor_data_file_id = "local:snippets/knaster-vendor.yaml"
+    ip_config {
+      ipv4 {
+        address = each.value.ip
+        gateway = each.value.ip == "dhcp" ? null : local.site.cluster.gateway
+      }
+    }
+    user_account {
+      username = "knaster"
+      keys     = [local.ssh_key]
+    }
+  }
+}
@@ -3,9 +3,19 @@ locals {
  gpu_vm_ipv4 = {
    for k, v in proxmox_virtual_environment_vm.gpu : k => try(flatten([
      for i, name in v.network_interface_names : v.ipv4_addresses[i]
-      if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth")
+      if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth") && !startswith(name, "cni") && !startswith(name, "flannel")
    ])[0], null)
  }
+  svc_vm_ipv4 = {
+    for k, v in proxmox_virtual_environment_vm.svc : k => try(flatten([
+      for i, name in v.network_interface_names : v.ipv4_addresses[i]
+      if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth") && !startswith(name, "cni") && !startswith(name, "flannel")
+    ])[0], null)
+  }
+}
+
+output "service_vms" {
+  value = { for k, v in proxmox_virtual_environment_vm.svc : k => { node = v.node_name, vmid = v.vm_id, ipv4 = local.svc_vm_ipv4[k] } }
 }

 output "gpu_vms" {
@@ -21,6 +31,7 @@ output "ansible_inventory" {
  value = yamlencode({
    all = { children = {
      gpu_vms = { hosts = { for k, ip in local.gpu_vm_ipv4 : k => { ansible_host = ip } if ip != null } }
+      service_vms = { hosts = { for k, ip in local.svc_vm_ipv4 : k => { ansible_host = ip } if ip != null } }
      infra = { hosts = {
        for k, v in proxmox_virtual_environment_container.infra : k => { ansible_host = v.ipv4["eth0"] } if try(v.ipv4["eth0"], null) != null
      } }
@@ -30,6 +30,11 @@ vms:                          # eine GPU-VM pro Knoten mit allen Karten des Knot
 containers:                   # Infrastruktur als LXC (ADR-0001), HA + ZFS-Replikation (ADR-0002)
  infra: {node: pve44, vmid: 101, cores: 4, memory_mb: 8192, disk_gb: 60, storage: tank, ip: 192.168.100.188/24, ha: true, replicate_to: [pve52, pve48]}

+# Dienst-VMs ohne GPU (D6): k3s-Steuerung, geklont vom Template des Knotens, auf tank mit HA.
+# Adresse per DHCP wie die GPU-VMs (D5); die Agents finden den Server über /etc/hosts (Rolle k3s-agent).
+service_vms:
+  k3s: {node: pve48, vmid: 110, cores: 2, memory_mb: 3072, disk_gb: 20, storage: tank, ip: dhcp, pool: infra, ha: true, replicate_to: [pve52, pve44]}
+
 replication_schedule: "*/15"

 # Cluster-Grundkonfiguration (Phase 4, Rolle pve-cluster, Playbook playbooks/cluster.yml)