diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md new file mode 100644 index 0000000..b9614b0 --- /dev/null +++ b/deploy/k8s/README.md @@ -0,0 +1,12 @@ +# Kubernetes-Manifeste (k3s, D6) + +Ausgerollt von ops mit `kubectl` (Rolle `k3s-apps`, Playbook `playbooks/k3s.yml`): + +| Datei | Zweck | +|---|---| +| `runtimeclass-nvidia.yaml` | RuntimeClass `nvidia` (Runtime legt k3s in den GPU-VMs selbst an) | +| `nvidia-device-plugin.yaml` | NVIDIA Device Plugin v0.20.1, nur auf Knoten mit Label `knaster.gpu=true`; liefert `nvidia.com/gpu` | +| `portainer-agent.yaml` | Portainer Agent 2.45.2 (NodePort 30778), Quelle downloads.portainer.io/ce2-45 | +| `nex-speech/` | NEX-Sprachdienste auf gpu-pve52 (Playbook `playbooks/nex-speech-k3s.yml`) | + +Portainer: Umgebung „Kubernetes → Agent“, Adresse `:30778`. diff --git a/deploy/k8s/nex-speech/namespace.yaml b/deploy/k8s/nex-speech/namespace.yaml new file mode 100644 index 0000000..9847d59 --- /dev/null +++ b/deploy/k8s/nex-speech/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nex + labels: + knaster.app: nex-speech diff --git a/deploy/k8s/nex-speech/nex-stt.yaml b/deploy/k8s/nex-speech/nex-stt.yaml new file mode 100644 index 0000000..6db7c98 --- /dev/null +++ b/deploy/k8s/nex-speech/nex-stt.yaml @@ -0,0 +1,56 @@ +# NEX-Spracherkennung (whisper.cpp, M4000) auf gpu-pve52, umgezogen von Docker Compose (Task 6.2, D6). +# Host-Netz wie zuvor: Port 8178 auf der VM-Adresse, die ufw-Regel der VM (nur appsrv08) gilt weiter. +# Image stammt aus dem Compose-Build und wurde in das containerd von k3s importiert (Rolle nex-speech-k3s). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nex-stt + namespace: nex + labels: {app: nex-stt} +spec: + replicas: 1 + strategy: {type: Recreate} # Host-Port 8178 kann nur einmal belegt sein + selector: + matchLabels: {app: nex-stt} + template: + metadata: + labels: {app: nex-stt} + spec: + runtimeClassName: nvidia + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/hostname: gpu-pve52 + containers: + - name: whisper + image: knaster/nex-whisper:v1.9.4-cu12.6 + imagePullPolicy: Never + env: + - {name: WHISPER_PORT, value: "8178"} + - {name: WHISPER_MODEL, value: /models/whisper/ggml-large-v3-turbo.bin} + - {name: WHISPER_LANGUAGE, value: de} + - {name: WHISPER_BEAM_SIZE, value: "5"} + - {name: WHISPER_EXTRA_ARGS, value: ""} + resources: + limits: {nvidia.com/gpu: 1} + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - {name: models, mountPath: /models/whisper, readOnly: true} + - {name: run, mountPath: /run/nex-whisper} + startupProbe: + httpGet: {path: /health, port: 8178, host: 127.0.0.1} + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: {path: /health, port: 8178, host: 127.0.0.1} + periodSeconds: 30 + livenessProbe: + httpGet: {path: /health, port: 8178, host: 127.0.0.1} + periodSeconds: 30 + failureThreshold: 3 + volumes: + - name: models + hostPath: {path: /srv/models/whisper, type: Directory} + - name: run + emptyDir: {medium: Memory, sizeLimit: 512Mi} diff --git a/deploy/k8s/nex-speech/nex-tts.yaml b/deploy/k8s/nex-speech/nex-tts.yaml new file mode 100644 index 0000000..799a86b --- /dev/null +++ b/deploy/k8s/nex-speech/nex-tts.yaml @@ -0,0 +1,56 @@ +# NEX-Sprachausgabe (Piper, CPU) auf gpu-pve52, Host-Netz Port 8179 (ufw: nur appsrv08). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nex-tts + namespace: nex + labels: {app: nex-tts} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: nex-tts} + template: + metadata: + labels: {app: nex-tts} + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/hostname: gpu-pve52 + containers: + - name: piper + image: knaster/nex-tts-piper:1.8.0 + imagePullPolicy: Never + env: + - {name: TTS_PORT, value: "8179"} + - {name: PIPER_VOICES_DIR, value: /voices} + - {name: PIPER_VOICES, value: "de_DE-thorsten-high,de_DE-kerstin-low"} + - {name: PIPER_DEFAULT_VOICE, value: de_DE-thorsten-high} + - {name: PCM_SAMPLE_RATE, value: "24000"} + - {name: TTS_MAX_CHARS, value: "5000"} + - {name: TTS_PARALLEL, value: "2"} + resources: + limits: {cpu: "3", memory: 2Gi} # ein Kern bleibt für whisper-server und das System + requests: {cpu: "500m", memory: 512Mi} + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - {name: voices, mountPath: /voices, readOnly: true} + - {name: tmp, mountPath: /tmp} + startupProbe: + httpGet: {path: /health, port: 8179, host: 127.0.0.1} + periodSeconds: 5 + failureThreshold: 24 + readinessProbe: + httpGet: {path: /health, port: 8179, host: 127.0.0.1} + periodSeconds: 30 + livenessProbe: + httpGet: {path: /health, port: 8179, host: 127.0.0.1} + periodSeconds: 30 + failureThreshold: 3 + volumes: + - name: voices + hostPath: {path: /srv/models/piper, type: Directory} + - name: tmp + emptyDir: {sizeLimit: 64Mi} diff --git a/deploy/k8s/nvidia-device-plugin.yaml b/deploy/k8s/nvidia-device-plugin.yaml new file mode 100644 index 0000000..9f09901 --- /dev/null +++ b/deploy/k8s/nvidia-device-plugin.yaml @@ -0,0 +1,44 @@ +# NVIDIA Device Plugin v0.20.1 (Quelle: deployments/static/nvidia-device-plugin.yml des Projekts), +# ergänzt um runtimeClassName nvidia und nodeSelector knaster.gpu=true (nur die GPU-VMs). +# Meldet je Knoten die Ressource nvidia.com/gpu, die Pods anfordern können. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: nvidia-device-plugin-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: nvidia-device-plugin-ds + spec: + runtimeClassName: nvidia + nodeSelector: + knaster.gpu: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + priorityClassName: system-node-critical + containers: + - image: nvcr.io/nvidia/k8s-device-plugin:v0.20.1 + name: nvidia-device-plugin-ctr + env: + - name: FAIL_ON_INIT_ERROR + value: "false" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/deploy/k8s/portainer-agent.yaml b/deploy/k8s/portainer-agent.yaml new file mode 100644 index 0000000..308438d --- /dev/null +++ b/deploy/k8s/portainer-agent.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: portainer +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: portainer-sa-clusteradmin + namespace: portainer +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: portainer-crb-clusteradmin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: portainer-sa-clusteradmin + namespace: portainer +--- +apiVersion: v1 +kind: Service +metadata: + name: portainer-agent + namespace: portainer +spec: + type: NodePort + selector: + app: portainer-agent + ports: + - name: http + protocol: TCP + port: 9001 + targetPort: 9001 + nodePort: 30778 +--- +apiVersion: v1 +kind: Service +metadata: + name: portainer-agent-headless + namespace: portainer +spec: + clusterIP: None + selector: + app: portainer-agent +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: portainer-agent + namespace: portainer +spec: + selector: + matchLabels: + app: portainer-agent + template: + metadata: + labels: + app: portainer-agent + spec: + serviceAccountName: portainer-sa-clusteradmin + containers: + - name: portainer-agent + image: portainer/agent:2.45.2 + imagePullPolicy: Always + env: + - name: LOG_LEVEL + value: DEBUG + - name: AGENT_CLUSTER_ADDR + value: "portainer-agent-headless" + - name: KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: 9001 + protocol: TCP diff --git a/deploy/k8s/runtimeclass-nvidia.yaml b/deploy/k8s/runtimeclass-nvidia.yaml new file mode 100644 index 0000000..c98d858 --- /dev/null +++ b/deploy/k8s/runtimeclass-nvidia.yaml @@ -0,0 +1,7 @@ +# Runtime "nvidia" legt k3s in seinem containerd an, sobald nvidia-container-runtime in der VM liegt. +# Pods mit runtimeClassName: nvidia bekommen die GPU-Bibliotheken eingehängt. +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia diff --git a/platform/proxmox/ansible/inventory.yml b/platform/proxmox/ansible/inventory.yml index 17313dc..456d80e 100644 --- a/platform/proxmox/ansible/inventory.yml +++ b/platform/proxmox/ansible/inventory.yml @@ -19,11 +19,11 @@ all: vars: ansible_user: root gpu_vms: - hosts: - gpu-pve44: {driver_generation: legacy} - gpu-pve46: {driver_generation: legacy} - gpu-pve48: {driver_generation: legacy} - gpu-pve52: {driver_generation: legacy} + hosts: # k3s_node_labels: Knoten-Labels für die Platzierung von GPU-Pods (D6) + gpu-pve44: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1, knaster.gpu.p2000=1]} + gpu-pve46: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1, knaster.gpu.p2000=1]} + gpu-pve48: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1, knaster.gpu.p2000=1]} + gpu-pve52: {driver_generation: legacy, k3s_node_labels: [knaster.gpu=true, knaster.gpu.m4000=1]} vars: ansible_user: knaster ansible_become: true @@ -39,3 +39,17 @@ all: infra: {} vars: ansible_user: root + # Dienst-VMs ohne GPU (D6): k3s-Steuerung. Adresse aus inventory.generated.yml (Gruppe service_vms). + service_vms: + hosts: + k3s: {} + vars: + ansible_user: knaster + ansible_become: true + ansible_python_interpreter: /usr/bin/python3 + k3s_server: + hosts: + k3s: {} + k3s_agents: + children: + gpu_vms: {} diff --git a/platform/proxmox/ansible/playbooks/k3s.yml b/platform/proxmox/ansible/playbooks/k3s.yml new file mode 100644 index 0000000..eae8353 --- /dev/null +++ b/platform/proxmox/ansible/playbooks/k3s.yml @@ -0,0 +1,16 @@ +# k3s-Cluster (D6): Server auf der Dienst-VM "k3s", Agents in den GPU-VMs, danach Runtime, Device-Plugin, +# Portainer-Agent. Aufruf auf ops (vorher bash ../scripts/refresh-vm-access.sh, Adressen per DHCP): +# ansible-playbook playbooks/k3s.yml +- name: k3s-Server + hosts: k3s_server + roles: [k3s-server] + +- name: k3s-Agents (GPU-VMs) + hosts: k3s_agents + roles: [k3s-agent] + +- name: Grundausstattung im Cluster (von ops) + hosts: localhost + connection: local + gather_facts: false + roles: [k3s-apps] diff --git a/platform/proxmox/ansible/playbooks/nex-speech-k3s.yml b/platform/proxmox/ansible/playbooks/nex-speech-k3s.yml new file mode 100644 index 0000000..6ef8949 --- /dev/null +++ b/platform/proxmox/ansible/playbooks/nex-speech-k3s.yml @@ -0,0 +1,5 @@ +# NEX-Sprachdienste von Docker Compose nach k3s umziehen (Test für D6). Rückweg: Deployments löschen +# (kubectl -n nex delete deploy nex-stt nex-tts) und auf gpu-pve52 `docker compose -f /opt/nex-speech start`. +- name: NEX-Sprachdienste nach k3s + hosts: "{{ nex_speech_hosts | default('gpu-pve52') }}" + roles: [nex-speech-k3s] diff --git a/platform/proxmox/ansible/roles/k3s-agent/defaults/main.yml b/platform/proxmox/ansible/roles/k3s-agent/defaults/main.yml new file mode 100644 index 0000000..fd74485 --- /dev/null +++ b/platform/proxmox/ansible/roles/k3s-agent/defaults/main.yml @@ -0,0 +1,10 @@ +# k3s-Agent in den GPU-VMs (D6). Der Server heißt k3s.knaster (/etc/hosts), Token aus der Datei auf ops. +k3s_version: v1.36.5+k3s1 +k3s_server_name: k3s.knaster +k3s_server_host: "{{ hostvars['k3s'].ansible_host }}" +k3s_env_file: /root/.config/knaster/k3s.env +k3s_token: "{{ lookup('ini', 'K3S_TOKEN', type='properties', file=k3s_env_file) }}" +k3s_install_script: /usr/local/src/k3s-install.sh +k3s_node_labels: [] +k3s_lan: 192.168.100.0/24 +k3s_pod_cidr: 10.42.0.0/16 diff --git a/platform/proxmox/ansible/roles/k3s-agent/tasks/main.yml b/platform/proxmox/ansible/roles/k3s-agent/tasks/main.yml new file mode 100644 index 0000000..3de0410 --- /dev/null +++ b/platform/proxmox/ansible/roles/k3s-agent/tasks/main.yml @@ -0,0 +1,92 @@ +# k3s-Agent neben Docker: k3s bringt sein eigenes containerd mit und erkennt nvidia-container-runtime +# von selbst (Runtime "nvidia"). Docker bleibt für die bestehenden Compose-Stacks, bis sie umgezogen sind. +- name: Server-Name in /etc/hosts (Adresse per DHCP, D5) + ansible.builtin.lineinfile: + path: /etc/hosts + regexp: '\s{{ k3s_server_name | regex_escape }}$' + line: "{{ k3s_server_host }} {{ k3s_server_name }}" + +- name: curl vorhanden + ansible.builtin.apt: + name: curl + state: present + +- name: Installationsskript laden + ansible.builtin.get_url: + url: https://get.k3s.io + dest: "{{ k3s_install_script }}" + mode: "0755" + +- name: Konfiguration /etc/rancher/k3s/config.yaml + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml + mode: "0600" + content: | + # Verwaltet von Ansible (Rolle k3s-agent) + server: https://{{ k3s_server_name }}:6443 + token: {{ k3s_token }} + node-label: {{ k3s_node_labels | to_json }} + register: k3s_agent_config + no_log: true + +# ufw (nur in VMs mit aktiver Firewall): Kubelet und NodePorts aus dem LAN, Flannel-VXLAN zwischen den +# Knoten, alles aus dem Pod-Netz (CoreDNS, Device-Plugin, Portainer-Agent sprechen mit dem Knoten). +- name: Ist ufw vorhanden? + ansible.builtin.stat: + path: /usr/sbin/ufw + register: k3s_ufw + +- name: ufw – k3s-Ports + community.general.ufw: + rule: allow + from_ip: "{{ item.from }}" + port: "{{ item.port | default(omit) }}" + proto: "{{ item.proto | default(omit) }}" + comment: "{{ item.comment }}" + loop: + - {from: "{{ k3s_lan }}", port: "10250", proto: tcp, comment: k3s-kubelet} + - {from: "{{ k3s_lan }}", port: "8472", proto: udp, comment: k3s-flannel} + - {from: "{{ k3s_lan }}", port: "30000:32767", proto: tcp, comment: k3s-nodeport} + - {from: "{{ k3s_pod_cidr }}", comment: k3s-pods} + loop_control: + label: "{{ item.comment }}" + when: k3s_ufw.stat.exists + +- name: Installierte Version lesen + ansible.builtin.command: /usr/local/bin/k3s --version + register: k3s_installed + failed_when: false + changed_when: false + +- name: k3s-Agent installieren oder auf die gepinnte Version bringen + ansible.builtin.command: "{{ k3s_install_script }} agent" + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + when: k3s_version not in (k3s_installed.stdout | default('')) + +- name: Agent neu starten, wenn sich die Konfiguration geändert hat + ansible.builtin.service: + name: k3s-agent + state: restarted + when: k3s_agent_config.changed and k3s_version in (k3s_installed.stdout | default('')) + +- name: Warten, bis containerd von k3s läuft + ansible.builtin.wait_for: + path: /var/lib/rancher/k3s/agent/etc/containerd/config.toml + timeout: 120 + +- name: NVIDIA-Runtime von k3s erkannt? + ansible.builtin.command: grep -c nvidia /var/lib/rancher/k3s/agent/etc/containerd/config.toml + register: k3s_nvidia + changed_when: false + failed_when: k3s_nvidia.stdout | int == 0 + +- name: Warten, bis der Knoten im Cluster bereit ist (von ops aus) + ansible.builtin.command: kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' + register: k3s_node_ready + until: k3s_node_ready.stdout == "True" + retries: 36 + delay: 5 + changed_when: false + delegate_to: localhost + become: false diff --git a/platform/proxmox/ansible/roles/k3s-apps/defaults/main.yml b/platform/proxmox/ansible/roles/k3s-apps/defaults/main.yml new file mode 100644 index 0000000..33cb82f --- /dev/null +++ b/platform/proxmox/ansible/roles/k3s-apps/defaults/main.yml @@ -0,0 +1,6 @@ +# Grundausstattung im k3s-Cluster (von ops per kubectl): NVIDIA-Runtime und Device-Plugin, Portainer-Agent. +k3s_manifest_dir: "{{ playbook_dir }}/../../../../deploy/k8s" +k3s_base_manifests: + - runtimeclass-nvidia.yaml + - nvidia-device-plugin.yaml + - portainer-agent.yaml diff --git a/platform/proxmox/ansible/roles/k3s-apps/tasks/main.yml b/platform/proxmox/ansible/roles/k3s-apps/tasks/main.yml new file mode 100644 index 0000000..919ad50 --- /dev/null +++ b/platform/proxmox/ansible/roles/k3s-apps/tasks/main.yml @@ -0,0 +1,28 @@ +- name: Grund-Manifeste anwenden + ansible.builtin.command: kubectl apply -f {{ k3s_manifest_dir }}/{{ item }} + loop: "{{ k3s_base_manifests }}" + register: k3s_apply + changed_when: "'configured' in k3s_apply.stdout or 'created' in k3s_apply.stdout" + +- name: Warten, bis jede GPU-VM ihre GPUs meldet (nvidia.com/gpu) + ansible.builtin.command: > + kubectl get node {{ item }} -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' + loop: "{{ groups['k3s_agents'] }}" + register: k3s_gpus + until: (k3s_gpus.stdout | default('0') | int) > 0 + retries: 24 + delay: 5 + changed_when: false + +- name: Warten, bis der Portainer-Agent läuft + ansible.builtin.command: kubectl -n portainer rollout status deployment/portainer-agent --timeout=180s + changed_when: false + +- name: Übersicht + ansible.builtin.command: kubectl get nodes -o custom-columns=NAME:.metadata.name,STATUS:.status.conditions[-1].type,GPU:.status.allocatable.nvidia\.com/gpu,VERSION:.status.nodeInfo.kubeletVersion + register: k3s_nodes + changed_when: false + +- name: Knoten + ansible.builtin.debug: + msg: "{{ k3s_nodes.stdout_lines }}" diff --git a/platform/proxmox/ansible/roles/k3s-server/defaults/main.yml b/platform/proxmox/ansible/roles/k3s-server/defaults/main.yml new file mode 100644 index 0000000..947fc15 --- /dev/null +++ b/platform/proxmox/ansible/roles/k3s-server/defaults/main.yml @@ -0,0 +1,8 @@ +# k3s-Steuerung (D6) auf der Dienst-VM "k3s". Version gepinnt (Kanal stable, 15.10.2026). +k3s_version: v1.36.5+k3s1 +k3s_server_name: k3s.knaster # Name, unter dem die Agents den Server finden (/etc/hosts, Rolle k3s-agent) +k3s_install_script: /usr/local/src/k3s-install.sh +# Geheimnisse und Zugang landen auf ops, nicht im Repo +k3s_env_file: /root/.config/knaster/k3s.env # K3S_URL und K3S_TOKEN für die Agents +k3s_kubeconfig: /root/.kube/config # kubectl auf ops +k3s_kubectl_version: "{{ k3s_version | regex_replace('\+k3s\d+$', '') }}" diff --git a/platform/proxmox/ansible/roles/k3s-server/tasks/main.yml b/platform/proxmox/ansible/roles/k3s-server/tasks/main.yml new file mode 100644 index 0000000..fdfc8de --- /dev/null +++ b/platform/proxmox/ansible/roles/k3s-server/tasks/main.yml @@ -0,0 +1,98 @@ +# k3s-Server (Steuerung, Datenbank, API) auf der Dienst-VM. Ohne traefik (kein Ingress nötig), +# servicelb bleibt für Dienste vom Typ LoadBalancer. Die Agents (GPU-VMs) kommen über die Rolle k3s-agent. +- name: curl vorhanden + ansible.builtin.apt: + name: curl + state: present + update_cache: true + +- name: Installationsskript laden + ansible.builtin.get_url: + url: https://get.k3s.io + dest: "{{ k3s_install_script }}" + mode: "0755" + +- name: Konfiguration /etc/rancher/k3s/config.yaml + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml + mode: "0600" + content: | + # Verwaltet von Ansible (Rolle k3s-server) + disable: [traefik] + write-kubeconfig-mode: "0600" + tls-san: [{{ k3s_server_name }}, {{ ansible_host }}] + node-label: [knaster.role=control] + register: k3s_server_config + vars: + ansible_become: true + +- name: Installierte Version lesen + ansible.builtin.command: /usr/local/bin/k3s --version + register: k3s_installed + failed_when: false + changed_when: false + +- name: k3s-Server installieren oder auf die gepinnte Version bringen + ansible.builtin.command: "{{ k3s_install_script }} server" + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + when: k3s_version not in (k3s_installed.stdout | default('')) + +- name: k3s neu starten, wenn sich die Konfiguration geändert hat + ansible.builtin.service: + name: k3s + state: restarted + when: k3s_server_config.changed and k3s_version in (k3s_installed.stdout | default('')) + +- name: Warten, bis der Knoten bereit ist + ansible.builtin.command: /usr/local/bin/k3s kubectl get node {{ inventory_hostname }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' + register: k3s_ready + until: k3s_ready.stdout == "True" + retries: 30 + delay: 5 + changed_when: false + +- name: Token und kubeconfig lesen + ansible.builtin.slurp: + src: "{{ item }}" + loop: [/var/lib/rancher/k3s/server/node-token, /etc/rancher/k3s/k3s.yaml] + register: k3s_secrets + no_log: true + +- name: Zugang nach ops schreiben + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0700" + loop: ["{{ k3s_env_file | dirname }}", "{{ k3s_kubeconfig | dirname }}"] + delegate_to: localhost + become: false + +- name: K3S_URL und K3S_TOKEN auf ops ({{ k3s_env_file }}) + ansible.builtin.copy: + dest: "{{ k3s_env_file }}" + mode: "0600" + content: | + K3S_URL=https://{{ k3s_server_name }}:6443 + K3S_TOKEN={{ k3s_secrets.results[0].content | b64decode | trim }} + delegate_to: localhost + become: false + no_log: true + +- name: kubeconfig auf ops ({{ k3s_kubeconfig }}, Server = VM-Adresse) + ansible.builtin.copy: + dest: "{{ k3s_kubeconfig }}" + mode: "0600" + content: "{{ k3s_secrets.results[1].content | b64decode | replace('https://127.0.0.1:6443', 'https://' ~ ansible_host ~ ':6443') }}" + delegate_to: localhost + become: false + no_log: true + +- name: kubectl auf ops (Version passend zum Server) + ansible.builtin.get_url: + url: "https://dl.k8s.io/release/{{ k3s_kubectl_version }}/bin/linux/amd64/kubectl" + checksum: "sha256:https://dl.k8s.io/release/{{ k3s_kubectl_version }}/bin/linux/amd64/kubectl.sha256" + dest: /usr/local/bin/kubectl + mode: "0755" + delegate_to: localhost + become: false diff --git a/platform/proxmox/ansible/roles/nex-speech-k3s/defaults/main.yml b/platform/proxmox/ansible/roles/nex-speech-k3s/defaults/main.yml new file mode 100644 index 0000000..6e1f8eb --- /dev/null +++ b/platform/proxmox/ansible/roles/nex-speech-k3s/defaults/main.yml @@ -0,0 +1,6 @@ +# Umzug der NEX-Sprachdienste von Docker Compose nach k3s (gleiche VM, gleiche Ports, gleiche Images). +nex_speech_dir: /opt/nex-speech +nex_speech_images: + - knaster/nex-whisper:v1.9.4-cu12.6 + - knaster/nex-tts-piper:1.8.0 +nex_speech_k3s_manifests: "{{ playbook_dir }}/../../../../deploy/k8s/nex-speech" diff --git a/platform/proxmox/ansible/roles/nex-speech-k3s/tasks/main.yml b/platform/proxmox/ansible/roles/nex-speech-k3s/tasks/main.yml new file mode 100644 index 0000000..2c92aeb --- /dev/null +++ b/platform/proxmox/ansible/roles/nex-speech-k3s/tasks/main.yml @@ -0,0 +1,40 @@ +# 1. Images aus Docker in das containerd von k3s übernehmen (kein Neubau, kein Registry-Umweg) +- name: Images in k3s vorhanden? + ansible.builtin.command: /usr/local/bin/k3s ctr images ls -q + register: k3s_images + changed_when: false + +- name: Images aus Docker importieren + ansible.builtin.shell: docker save {{ item }} | /usr/local/bin/k3s ctr images import - + loop: "{{ nex_speech_images }}" + when: ("docker.io/" ~ item) not in k3s_images.stdout + async: 1800 + poll: 15 + +# 2. Compose-Stack anhalten (Host-Ports 8178/8179 werden frei), Dateien bleiben für den Rückweg +- name: Compose-Stack nex-speech anhalten + community.docker.docker_compose_v2: + project_src: "{{ nex_speech_dir }}" + state: stopped + when: nex_speech_compose_stop | default(true) + +# 3. Manifeste anwenden und auf Bereitschaft warten (Probes prüfen /health in der VM) +- name: Manifeste anwenden + ansible.builtin.command: kubectl apply -f {{ nex_speech_k3s_manifests }}/ + register: nex_apply + changed_when: "'configured' in nex_apply.stdout or 'created' in nex_apply.stdout" + delegate_to: localhost + become: false + +- name: Warten, bis beide Deployments bereit sind + ansible.builtin.command: kubectl -n nex rollout status deployment/{{ item }} --timeout=600s + loop: [nex-stt, nex-tts] + changed_when: false + delegate_to: localhost + become: false + +- name: Gesundheit in der VM prüfen + ansible.builtin.uri: + url: "http://127.0.0.1:{{ item }}/health" + status_code: 200 + loop: [8178, 8179] diff --git a/platform/proxmox/ansible/roles/pve-host/tasks/main.yml b/platform/proxmox/ansible/roles/pve-host/tasks/main.yml index bf721ff..dfba659 100644 --- a/platform/proxmox/ansible/roles/pve-host/tasks/main.yml +++ b/platform/proxmox/ansible/roles/pve-host/tasks/main.yml @@ -103,6 +103,20 @@ check_mode: false changed_when: false +- name: VMs auf diesem Knoten lesen + ansible.builtin.command: ls /etc/pve/qemu-server/ + register: local_vms + check_mode: false + changed_when: false + +# Infra-Gäste = Container (ct:) plus Dienst-VMs ohne GPU (vm:, D6); beide liegen auf tank mit HA +- name: Infra-Gäste zusammenstellen + ansible.builtin.set_fact: + pve_infra_guests: >- + {{ (knaster_pve.containers | dict2items | map('combine', {'kind': 'ct'}) | list) + + (knaster_pve.service_vms | default({}) | dict2items | map('combine', {'kind': 'vm'}) | list) }} + pve_local_guest_confs: "{{ local_cts.stdout_lines + local_vms.stdout_lines }}" + - name: Bestehende Replikationsjobs lesen ansible.builtin.command: pvesr list register: pvesr_jobs @@ -113,12 +127,12 @@ ansible.builtin.command: > pvesr create-local-job {{ item.0.value.vmid }}-{{ idx }} {{ item.1 }} --schedule '{{ knaster_pve.replication_schedule }}' --comment 'knaster {{ item.0.key }}' - loop: "{{ knaster_pve.containers | dict2items | subelements('value.replicate_to', skip_missing=True) }}" + loop: "{{ pve_infra_guests | subelements('value.replicate_to', skip_missing=True) }}" vars: idx: "{{ item.0.value.replicate_to.index(item.1) }}" when: - item.0.value.node == inventory_hostname - - (item.0.value.vmid | string) + '.conf' in local_cts.stdout_lines + - (item.0.value.vmid | string) + '.conf' in pve_local_guest_confs - (item.0.value.vmid | string) + '-' + (idx | string) not in pvesr_jobs.stdout changed_when: true @@ -130,14 +144,14 @@ - name: Infra-Gäste unter HA stellen (erst wenn der Container auf diesem Knoten existiert) ansible.builtin.command: > - ha-manager add ct:{{ item.value.vmid }} --state started --max_restart 1 --max_relocate 1 + ha-manager add {{ item.kind }}:{{ item.value.vmid }} --state started --max_restart 1 --max_relocate 1 --comment 'knaster {{ item.key }}' - loop: "{{ knaster_pve.containers | dict2items }}" + loop: "{{ pve_infra_guests }}" when: - item.value.ha | default(false) - item.value.node == inventory_hostname - - (item.value.vmid | string) + '.conf' in local_cts.stdout_lines - - ('ct:' + (item.value.vmid | string)) not in ha_config.stdout + - (item.value.vmid | string) + '.conf' in pve_local_guest_confs + - (item.kind + ':' + (item.value.vmid | string)) not in ha_config.stdout changed_when: true - name: HA-Regeln lesen @@ -149,13 +163,13 @@ # Strikt: Die Gäste liegen auf tank, den gibt es nur auf dem Heimatknoten und den Replikationszielen - name: Node-Affinity für Infra-Gäste (Heimatknoten bevorzugt, sonst Replikationsziele) ansible.builtin.command: > - ha-manager rules add node-affinity knaster-{{ item.key }} --resources ct:{{ item.value.vmid }} + ha-manager rules add node-affinity knaster-{{ item.key }} --resources {{ item.kind }}:{{ item.value.vmid }} --nodes {{ ([item.value.node ~ ':2'] + (item.value.replicate_to | map('regex_replace', '$', ':1') | list)) | join(',') }} --strict 1 --comment 'knaster {{ item.key }}' - loop: "{{ knaster_pve.containers | dict2items }}" + loop: "{{ pve_infra_guests }}" when: - item.value.ha | default(false) - item.value.node == inventory_hostname - - (item.value.vmid | string) + '.conf' in local_cts.stdout_lines + - (item.value.vmid | string) + '.conf' in pve_local_guest_confs - ('knaster-' + item.key) not in ha_rules.stdout changed_when: true diff --git a/platform/proxmox/scripts/refresh-vm-access.sh b/platform/proxmox/scripts/refresh-vm-access.sh index 0af580e..86ff9e5 100755 Binary files a/platform/proxmox/scripts/refresh-vm-access.sh and b/platform/proxmox/scripts/refresh-vm-access.sh differ diff --git a/platform/proxmox/tofu/main.tf b/platform/proxmox/tofu/main.tf index 5661ed8..9c36b03 100644 --- a/platform/proxmox/tofu/main.tf +++ b/platform/proxmox/tofu/main.tf @@ -169,3 +169,74 @@ resource "proxmox_virtual_environment_container" "infra" { # HA (Ressource + Node-Affinity-Regel) und Replikation setzt die Ansible-Rolle pve-host per root-SSH. # So braucht der tofu-Token kein Sys.Console auf /. + +# Dienst-VMs ohne GPU (D6, k3s-Steuerung): Klon vom Template des Knotens auf tank, HA + Replikation +# setzt die Rolle pve-host (wie bei den Containern). +resource "proxmox_virtual_environment_vm" "svc" { + for_each = try(local.site.service_vms, {}) + name = each.key + node_name = each.value.node + vm_id = each.value.vmid + tags = ["knaster", "infra"] + pool_id = try(each.value.pool, "infra") + + machine = "q35" + bios = "ovmf" + on_boot = true + scsi_hardware = "virtio-scsi-single" + + clone { + vm_id = local.site.cluster.templates[each.value.node] + node_name = each.value.node + datastore_id = each.value.storage + full = true + } + + agent { + enabled = true + } + + cpu { + cores = each.value.cores + type = "host" + } + + memory { + dedicated = each.value.memory_mb + floating = each.value.memory_mb # Ballooning erlaubt (kein Passthrough) + } + + efi_disk { + datastore_id = each.value.storage + type = "4m" + pre_enrolled_keys = false + } + + disk { + datastore_id = each.value.storage + interface = "scsi0" + size = each.value.disk_gb + discard = "on" + ssd = true + iothread = true + } + + network_device { + bridge = local.site.cluster.bridge + } + + initialization { + datastore_id = each.value.storage + vendor_data_file_id = "local:snippets/knaster-vendor.yaml" + ip_config { + ipv4 { + address = each.value.ip + gateway = each.value.ip == "dhcp" ? null : local.site.cluster.gateway + } + } + user_account { + username = "knaster" + keys = [local.ssh_key] + } + } +} diff --git a/platform/proxmox/tofu/outputs.tf b/platform/proxmox/tofu/outputs.tf index 15bdff2..5a6b84a 100644 --- a/platform/proxmox/tofu/outputs.tf +++ b/platform/proxmox/tofu/outputs.tf @@ -3,9 +3,19 @@ locals { gpu_vm_ipv4 = { for k, v in proxmox_virtual_environment_vm.gpu : k => try(flatten([ for i, name in v.network_interface_names : v.ipv4_addresses[i] - if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth") + if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth") && !startswith(name, "cni") && !startswith(name, "flannel") ])[0], null) } + svc_vm_ipv4 = { + for k, v in proxmox_virtual_environment_vm.svc : k => try(flatten([ + for i, name in v.network_interface_names : v.ipv4_addresses[i] + if !contains(["lo"], name) && !startswith(name, "docker") && !startswith(name, "br-") && !startswith(name, "veth") && !startswith(name, "cni") && !startswith(name, "flannel") + ])[0], null) + } +} + +output "service_vms" { + value = { for k, v in proxmox_virtual_environment_vm.svc : k => { node = v.node_name, vmid = v.vm_id, ipv4 = local.svc_vm_ipv4[k] } } } output "gpu_vms" { @@ -21,6 +31,7 @@ output "ansible_inventory" { value = yamlencode({ all = { children = { gpu_vms = { hosts = { for k, ip in local.gpu_vm_ipv4 : k => { ansible_host = ip } if ip != null } } + service_vms = { hosts = { for k, ip in local.svc_vm_ipv4 : k => { ansible_host = ip } if ip != null } } infra = { hosts = { for k, v in proxmox_virtual_environment_container.infra : k => { ansible_host = v.ipv4["eth0"] } if try(v.ipv4["eth0"], null) != null } } diff --git a/sites/knaster/proxmox.yaml b/sites/knaster/proxmox.yaml index 104ce65..94bb1ff 100644 --- a/sites/knaster/proxmox.yaml +++ b/sites/knaster/proxmox.yaml @@ -30,6 +30,11 @@ vms: # eine GPU-VM pro Knoten mit allen Karten des Knot containers: # Infrastruktur als LXC (ADR-0001), HA + ZFS-Replikation (ADR-0002) infra: {node: pve44, vmid: 101, cores: 4, memory_mb: 8192, disk_gb: 60, storage: tank, ip: 192.168.100.188/24, ha: true, replicate_to: [pve52, pve48]} +# Dienst-VMs ohne GPU (D6): k3s-Steuerung, geklont vom Template des Knotens, auf tank mit HA. +# Adresse per DHCP wie die GPU-VMs (D5); die Agents finden den Server über /etc/hosts (Rolle k3s-agent). +service_vms: + k3s: {node: pve48, vmid: 110, cores: 2, memory_mb: 3072, disk_gb: 20, storage: tank, ip: dhcp, pool: infra, ha: true, replicate_to: [pve52, pve44]} + replication_schedule: "*/15" # Cluster-Grundkonfiguration (Phase 4, Rolle pve-cluster, Playbook playbooks/cluster.yml)