k3s (D6): Dienst-VM k3s per tofu, Rollen k3s-server/k3s-agent/k3s-apps, Manifeste, NEX-Sprachdienste als k3s-Umzugstest
ci / test (push) Failing after 22s
ci / test (push) Failing after 22s
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Robert Rapp
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
6cccc01d14
commit
2c1d43e9c7
@@ -0,0 +1,12 @@
|
||||
# Kubernetes-Manifeste (k3s, D6)
|
||||
|
||||
Ausgerollt von ops mit `kubectl` (Rolle `k3s-apps`, Playbook `playbooks/k3s.yml`):
|
||||
|
||||
| Datei | Zweck |
|
||||
|---|---|
|
||||
| `runtimeclass-nvidia.yaml` | RuntimeClass `nvidia` (Runtime legt k3s in den GPU-VMs selbst an) |
|
||||
| `nvidia-device-plugin.yaml` | NVIDIA Device Plugin v0.20.1, nur auf Knoten mit Label `knaster.gpu=true`; liefert `nvidia.com/gpu` |
|
||||
| `portainer-agent.yaml` | Portainer Agent 2.45.2 (NodePort 30778), Quelle downloads.portainer.io/ce2-45 |
|
||||
| `nex-speech/` | NEX-Sprachdienste auf gpu-pve52 (Playbook `playbooks/nex-speech-k3s.yml`) |
|
||||
|
||||
Portainer: Umgebung „Kubernetes → Agent“, Adresse `<IP eines Knotens>:30778`.
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: nex
|
||||
labels:
|
||||
knaster.app: nex-speech
|
||||
@@ -0,0 +1,56 @@
|
||||
# NEX-Spracherkennung (whisper.cpp, M4000) auf gpu-pve52, umgezogen von Docker Compose (Task 6.2, D6).
|
||||
# Host-Netz wie zuvor: Port 8178 auf der VM-Adresse, die ufw-Regel der VM (nur appsrv08) gilt weiter.
|
||||
# Image stammt aus dem Compose-Build und wurde in das containerd von k3s importiert (Rolle nex-speech-k3s).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: nex-stt
|
||||
namespace: nex
|
||||
labels: {app: nex-stt}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate} # Host-Port 8178 kann nur einmal belegt sein
|
||||
selector:
|
||||
matchLabels: {app: nex-stt}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: nex-stt}
|
||||
spec:
|
||||
runtimeClassName: nvidia
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: gpu-pve52
|
||||
containers:
|
||||
- name: whisper
|
||||
image: knaster/nex-whisper:v1.9.4-cu12.6
|
||||
imagePullPolicy: Never
|
||||
env:
|
||||
- {name: WHISPER_PORT, value: "8178"}
|
||||
- {name: WHISPER_MODEL, value: /models/whisper/ggml-large-v3-turbo.bin}
|
||||
- {name: WHISPER_LANGUAGE, value: de}
|
||||
- {name: WHISPER_BEAM_SIZE, value: "5"}
|
||||
- {name: WHISPER_EXTRA_ARGS, value: ""}
|
||||
resources:
|
||||
limits: {nvidia.com/gpu: 1}
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
volumeMounts:
|
||||
- {name: models, mountPath: /models/whisper, readOnly: true}
|
||||
- {name: run, mountPath: /run/nex-whisper}
|
||||
startupProbe:
|
||||
httpGet: {path: /health, port: 8178, host: 127.0.0.1}
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet: {path: /health, port: 8178, host: 127.0.0.1}
|
||||
periodSeconds: 30
|
||||
livenessProbe:
|
||||
httpGet: {path: /health, port: 8178, host: 127.0.0.1}
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
- name: models
|
||||
hostPath: {path: /srv/models/whisper, type: Directory}
|
||||
- name: run
|
||||
emptyDir: {medium: Memory, sizeLimit: 512Mi}
|
||||
@@ -0,0 +1,56 @@
|
||||
# NEX-Sprachausgabe (Piper, CPU) auf gpu-pve52, Host-Netz Port 8179 (ufw: nur appsrv08).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: nex-tts
|
||||
namespace: nex
|
||||
labels: {app: nex-tts}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate}
|
||||
selector:
|
||||
matchLabels: {app: nex-tts}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: nex-tts}
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: gpu-pve52
|
||||
containers:
|
||||
- name: piper
|
||||
image: knaster/nex-tts-piper:1.8.0
|
||||
imagePullPolicy: Never
|
||||
env:
|
||||
- {name: TTS_PORT, value: "8179"}
|
||||
- {name: PIPER_VOICES_DIR, value: /voices}
|
||||
- {name: PIPER_VOICES, value: "de_DE-thorsten-high,de_DE-kerstin-low"}
|
||||
- {name: PIPER_DEFAULT_VOICE, value: de_DE-thorsten-high}
|
||||
- {name: PCM_SAMPLE_RATE, value: "24000"}
|
||||
- {name: TTS_MAX_CHARS, value: "5000"}
|
||||
- {name: TTS_PARALLEL, value: "2"}
|
||||
resources:
|
||||
limits: {cpu: "3", memory: 2Gi} # ein Kern bleibt für whisper-server und das System
|
||||
requests: {cpu: "500m", memory: 512Mi}
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
volumeMounts:
|
||||
- {name: voices, mountPath: /voices, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
startupProbe:
|
||||
httpGet: {path: /health, port: 8179, host: 127.0.0.1}
|
||||
periodSeconds: 5
|
||||
failureThreshold: 24
|
||||
readinessProbe:
|
||||
httpGet: {path: /health, port: 8179, host: 127.0.0.1}
|
||||
periodSeconds: 30
|
||||
livenessProbe:
|
||||
httpGet: {path: /health, port: 8179, host: 127.0.0.1}
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
- name: voices
|
||||
hostPath: {path: /srv/models/piper, type: Directory}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
@@ -0,0 +1,44 @@
|
||||
# NVIDIA Device Plugin v0.20.1 (Quelle: deployments/static/nvidia-device-plugin.yml des Projekts),
|
||||
# ergänzt um runtimeClassName nvidia und nodeSelector knaster.gpu=true (nur die GPU-VMs).
|
||||
# Meldet je Knoten die Ressource nvidia.com/gpu, die Pods anfordern können.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: nvidia-device-plugin-daemonset
|
||||
namespace: kube-system
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
name: nvidia-device-plugin-ds
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
name: nvidia-device-plugin-ds
|
||||
spec:
|
||||
runtimeClassName: nvidia
|
||||
nodeSelector:
|
||||
knaster.gpu: "true"
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
priorityClassName: system-node-critical
|
||||
containers:
|
||||
- image: nvcr.io/nvidia/k8s-device-plugin:v0.20.1
|
||||
name: nvidia-device-plugin-ctr
|
||||
env:
|
||||
- name: FAIL_ON_INIT_ERROR
|
||||
value: "false"
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: device-plugin
|
||||
mountPath: /var/lib/kubelet/device-plugins
|
||||
volumes:
|
||||
- name: device-plugin
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/device-plugins
|
||||
@@ -0,0 +1,81 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: portainer
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: portainer-sa-clusteradmin
|
||||
namespace: portainer
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: portainer-crb-clusteradmin
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: cluster-admin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: portainer-sa-clusteradmin
|
||||
namespace: portainer
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: portainer-agent
|
||||
namespace: portainer
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
app: portainer-agent
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 9001
|
||||
targetPort: 9001
|
||||
nodePort: 30778
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: portainer-agent-headless
|
||||
namespace: portainer
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app: portainer-agent
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: portainer-agent
|
||||
namespace: portainer
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: portainer-agent
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: portainer-agent
|
||||
spec:
|
||||
serviceAccountName: portainer-sa-clusteradmin
|
||||
containers:
|
||||
- name: portainer-agent
|
||||
image: portainer/agent:2.45.2
|
||||
imagePullPolicy: Always
|
||||
env:
|
||||
- name: LOG_LEVEL
|
||||
value: DEBUG
|
||||
- name: AGENT_CLUSTER_ADDR
|
||||
value: "portainer-agent-headless"
|
||||
- name: KUBERNETES_POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
ports:
|
||||
- containerPort: 9001
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,7 @@
|
||||
# Runtime "nvidia" legt k3s in seinem containerd an, sobald nvidia-container-runtime in der VM liegt.
|
||||
# Pods mit runtimeClassName: nvidia bekommen die GPU-Bibliotheken eingehängt.
|
||||
apiVersion: node.k8s.io/v1
|
||||
kind: RuntimeClass
|
||||
metadata:
|
||||
name: nvidia
|
||||
handler: nvidia
|
||||
Reference in New Issue
Block a user