TTS-Slots (Piper CPU, Qwen3-TTS auf Ada), M4000 außer Betrieb, Engine-Grenzen im Placement (ADR-0006)
ci / test (push) Successful in 24s
ci / test (push) Successful in 24s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,12 @@
|
|||||||
# Modell-Katalog (SPEC-inventory §2). vram_gb = Gewichte + Mindest-KV-Cache (Richtwerte, per Kalibrierung prüfen).
|
# Modell-Katalog (SPEC-inventory §2). vram_gb = Gewichte + Mindest-KV-Cache (Richtwerte, per Kalibrierung prüfen).
|
||||||
# Ist ein Modell nicht mehr aktuell, wird ein neuer Eintrag ergänzt. Der Wechsel läuft über die Lifecycle-Pipeline.
|
# Ist ein Modell nicht mehr aktuell, wird ein neuer Eintrag ergänzt. Der Wechsel läuft über die Lifecycle-Pipeline.
|
||||||
|
# Engine-Grenzen gelten für alle Modelle der Engine (ADR-0006).
|
||||||
|
engines:
|
||||||
|
llamacpp: {min_cc: 5.0, note: "CUDA-12-Build mit sm_52/61 – einzige Engine für Maxwell/Pascal"}
|
||||||
|
vllm: {min_cc: 7.0}
|
||||||
|
pytorch: {min_cc: 7.5, note: "Maxwell/Pascal nur bis torch 2.14 cu126, bf16/SDPA fehlen → sehr langsam (Qwen3-TTS auf P40: RTF ~7)"}
|
||||||
|
onnx: {min_cc: 0.0, note: "läuft auch auf CPU"}
|
||||||
|
|
||||||
models:
|
models:
|
||||||
qwen3-14b-awq:
|
qwen3-14b-awq:
|
||||||
kind: llm
|
kind: llm
|
||||||
@@ -49,3 +56,22 @@ models:
|
|||||||
engines: [llamacpp]
|
engines: [llamacpp]
|
||||||
context: 8192
|
context: 8192
|
||||||
cpu_ok: true
|
cpu_ok: true
|
||||||
|
# --- TTS (ADR-0006) ---
|
||||||
|
qwen3-tts-1.7b:
|
||||||
|
kind: tts
|
||||||
|
vram_gb: 8 # bf16; mit faster-qwen3-tts (CUDA Graphs)
|
||||||
|
min_cc: 8.0 # bf16
|
||||||
|
engines: [pytorch]
|
||||||
|
note: "Apache 2.0, Deutsch, Voice Cloning/Design. Auf Ada geschätzt RTF 0,3–0,5 (messen)"
|
||||||
|
qwen3-tts-0.6b:
|
||||||
|
kind: tts
|
||||||
|
vram_gb: 4
|
||||||
|
min_cc: 8.0
|
||||||
|
engines: [pytorch]
|
||||||
|
note: "Apache 2.0, Deutsch"
|
||||||
|
piper-de-thorsten-medium:
|
||||||
|
kind: tts
|
||||||
|
vram_gb: 0.2
|
||||||
|
engines: [onnx]
|
||||||
|
cpu_ok: true
|
||||||
|
note: "schnell auf CPU (≪ Echtzeit), solide Qualität; piper-tts 1.3+ ist GPL-3.0 (eigener Dienst)"
|
||||||
|
|||||||
@@ -32,3 +32,16 @@ roles:
|
|||||||
priority: 5
|
priority: 5
|
||||||
needs: {gpu: false, min_ram_gb: 16}
|
needs: {gpu: false, min_ram_gb: 16}
|
||||||
slots: []
|
slots: []
|
||||||
|
tts-hq:
|
||||||
|
priority: 6
|
||||||
|
share_with: gpu-flex # Qwen3-TTS auf der zweiten Ada, außerhalb der Trainingsfenster
|
||||||
|
needs: {gpu: true, engine: pytorch, min_cc: 8.0}
|
||||||
|
kinds: [tts]
|
||||||
|
optional: true
|
||||||
|
slots: [tts-de-hq]
|
||||||
|
tts-cpu:
|
||||||
|
priority: 7
|
||||||
|
share_with: infra # Piper läuft im Infra-LXC mit
|
||||||
|
needs: {gpu: false, min_ram_gb: 8}
|
||||||
|
kinds: [tts]
|
||||||
|
slots: [tts-de]
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
"""Hilfen für Audio (WAV-Dauer, Real-Time-Factor)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import wave
|
||||||
|
|
||||||
|
|
||||||
|
def wav_duration(data: bytes) -> float:
|
||||||
|
"""Dauer einer WAV-Datei in Sekunden; 0.0 wenn keine gültige WAV."""
|
||||||
|
try:
|
||||||
|
with wave.open(io.BytesIO(data)) as w:
|
||||||
|
rate = w.getframerate()
|
||||||
|
return w.getnframes() / rate if rate else 0.0
|
||||||
|
except (wave.Error, EOFError):
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def rtf(seconds_compute: float, seconds_audio: float) -> float | None:
|
||||||
|
"""Real-Time-Factor: Rechenzeit ÷ Audiolänge (< 1 = schneller als Echtzeit)."""
|
||||||
|
return seconds_compute / seconds_audio if seconds_audio > 0 else None
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""Benchmarks, deren Ergebnisse in Katalog/Registry zurückfließen (Kalibrierung am Deployment-Tag)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from statistics import median
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .audio import rtf, wav_duration
|
||||||
|
from .llm import OpenAIChatClient
|
||||||
|
|
||||||
|
DEFAULT_TEXT = (
|
||||||
|
"Guten Tag. Dies ist ein Test der Sprachausgabe im Cluster. Wir messen, wie lange die Berechnung "
|
||||||
|
"im Verhältnis zur Länge des erzeugten Audios dauert. Zahlen wie 1.250,75 Euro und Daten wie der "
|
||||||
|
"3. Oktober 2026 sollen korrekt gesprochen werden."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def bench_tts(url: str, model: str, *, api_key: str = "", runs: int = 3, text: str | None = None,
|
||||||
|
client: OpenAIChatClient | None = None) -> dict[str, Any]:
|
||||||
|
client = client or OpenAIChatClient(url, api_key, timeout=600)
|
||||||
|
text = text or DEFAULT_TEXT
|
||||||
|
results = []
|
||||||
|
errors = []
|
||||||
|
for _ in range(runs):
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
audio = client.speech(model, text) # type: ignore[attr-defined]
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
errors.append(str(exc))
|
||||||
|
continue
|
||||||
|
elapsed = time.monotonic() - start
|
||||||
|
duration = wav_duration(audio)
|
||||||
|
results.append({"seconds": round(elapsed, 2), "audio_seconds": round(duration, 2),
|
||||||
|
"rtf": round(rtf(elapsed, duration), 3) if duration else None})
|
||||||
|
rtfs = [r["rtf"] for r in results if r["rtf"] is not None]
|
||||||
|
return {"model": model, "chars": len(text), "runs": results, "errors": errors,
|
||||||
|
"rtf_median": round(median(rtfs), 3) if rtfs else None}
|
||||||
@@ -8,6 +8,7 @@ from dataclasses import dataclass
|
|||||||
|
|
||||||
from prometheus_client import Gauge
|
from prometheus_client import Gauge
|
||||||
|
|
||||||
|
from .audio import wav_duration
|
||||||
from .embeddings import Embedder
|
from .embeddings import Embedder
|
||||||
from .llm import ChatClient, LLMError
|
from .llm import ChatClient, LLMError
|
||||||
from .registry import Registry, Slot
|
from .registry import Registry, Slot
|
||||||
@@ -51,7 +52,12 @@ class CanaryProber:
|
|||||||
def probe(self, slot: Slot) -> ProbeResult:
|
def probe(self, slot: Slot) -> ProbeResult:
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
try:
|
try:
|
||||||
if slot.spec.kind == "embedding":
|
if slot.spec.kind == "tts":
|
||||||
|
speech = getattr(self.chat, "speech", None)
|
||||||
|
if speech is None:
|
||||||
|
return ProbeResult(slot.name, True, True, 0.0, "kein Speech-Client")
|
||||||
|
ok = wav_duration(speech(slot.spec.litellm_model, "Test.")) >= 0.2
|
||||||
|
elif slot.spec.kind == "embedding":
|
||||||
embedder = self.embedder_for.get(slot.name)
|
embedder = self.embedder_for.get(slot.name)
|
||||||
if embedder is None:
|
if embedder is None:
|
||||||
return ProbeResult(slot.name, True, True, 0.0, "kein Embedder konfiguriert")
|
return ProbeResult(slot.name, True, True, 0.0, "kein Embedder konfiguriert")
|
||||||
|
|||||||
@@ -30,6 +30,19 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
p_plan.add_argument("--inventory", required=True)
|
p_plan.add_argument("--inventory", required=True)
|
||||||
p_plan.add_argument("--catalog", default="catalog")
|
p_plan.add_argument("--catalog", default="catalog")
|
||||||
|
|
||||||
|
p_tts = sub.add_parser("tts-serve", help="OpenAI-kompatiblen TTS-Dienst (Piper o. ä.) starten")
|
||||||
|
p_tts.add_argument("--host", default="0.0.0.0")
|
||||||
|
p_tts.add_argument("--port", type=int, default=8090)
|
||||||
|
|
||||||
|
p_bench = sub.add_parser("bench", help="Benchmarks für den Deployment-Tag")
|
||||||
|
bench_sub = p_bench.add_subparsers(dest="bench_cmd", required=True)
|
||||||
|
p_bt = bench_sub.add_parser("tts", help="Real-Time-Factor eines TTS-Endpunkts messen")
|
||||||
|
p_bt.add_argument("--url", required=True, help="Basis-URL, z. B. http://litellm:4000 oder http://tts:8090")
|
||||||
|
p_bt.add_argument("--model", default="tts-de")
|
||||||
|
p_bt.add_argument("--api-key", default="")
|
||||||
|
p_bt.add_argument("--runs", type=int, default=3)
|
||||||
|
p_bt.add_argument("--text", default=None)
|
||||||
|
|
||||||
p_reg = sub.add_parser("registry", help="Registry")
|
p_reg = sub.add_parser("registry", help="Registry")
|
||||||
reg_sub = p_reg.add_subparsers(dest="reg_cmd", required=True)
|
reg_sub = p_reg.add_subparsers(dest="reg_cmd", required=True)
|
||||||
reg_sub.add_parser("export", help="effektiven Stand (inkl. Laufzeitzustand) als YAML ausgeben")
|
reg_sub.add_parser("export", help="effektiven Stand (inkl. Laufzeitzustand) als YAML ausgeben")
|
||||||
@@ -42,6 +55,19 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
uvicorn.run("knaster.api:create_app", factory=True, host=args.host, port=args.port)
|
uvicorn.run("knaster.api:create_app", factory=True, host=args.host, port=args.port)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
if args.cmd == "tts-serve":
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
uvicorn.run("knaster.tts_server:create_tts_app", factory=True, host=args.host, port=args.port)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if args.cmd == "bench":
|
||||||
|
from .bench import bench_tts
|
||||||
|
|
||||||
|
result = bench_tts(args.url, args.model, api_key=args.api_key, runs=args.runs, text=args.text)
|
||||||
|
print(yaml.safe_dump(result, sort_keys=False, allow_unicode=True))
|
||||||
|
return 0 if result["runs"] else 1
|
||||||
|
|
||||||
if args.cmd == "validate":
|
if args.cmd == "validate":
|
||||||
from .validate import validate_repo
|
from .validate import validate_repo
|
||||||
|
|
||||||
|
|||||||
@@ -11,9 +11,17 @@ from ..util import load_yaml
|
|||||||
from .profile import GPU, NodeProfile
|
from .profile import GPU, NodeProfile
|
||||||
|
|
||||||
|
|
||||||
|
class EngineSpec(BaseModel):
|
||||||
|
"""Laufzeit-Anforderungen, die für alle Modelle einer Engine gelten."""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
min_cc: float = 0.0
|
||||||
|
note: str = ""
|
||||||
|
|
||||||
|
|
||||||
class ModelSpec(BaseModel):
|
class ModelSpec(BaseModel):
|
||||||
name: str
|
name: str
|
||||||
kind: Literal["llm", "embedding", "reranker"]
|
kind: Literal["llm", "embedding", "reranker", "tts"]
|
||||||
vram_gb: float
|
vram_gb: float
|
||||||
min_cc: float = 5.0
|
min_cc: float = 5.0
|
||||||
engines: list[str]
|
engines: list[str]
|
||||||
@@ -41,6 +49,7 @@ class RoleSpec(BaseModel):
|
|||||||
kinds: list[str] = Field(default_factory=list)
|
kinds: list[str] = Field(default_factory=list)
|
||||||
cpu_fallback: bool = False
|
cpu_fallback: bool = False
|
||||||
optional: bool = False
|
optional: bool = False
|
||||||
|
share_with: str | None = None # teilt sich die GPU einer bereits platzierten Rolle (z. B. TTS auf gpu-flex)
|
||||||
slots: list[str] = Field(default_factory=list)
|
slots: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
@@ -50,7 +59,7 @@ class Assignment(BaseModel):
|
|||||||
gpu_index: int | None = None
|
gpu_index: int | None = None
|
||||||
gpu_name: str | None = None
|
gpu_name: str | None = None
|
||||||
vram_gb: float | None = None
|
vram_gb: float | None = None
|
||||||
mode: Literal["gpu", "cpu", "cpu_fallback"]
|
mode: Literal["gpu", "shared", "cpu", "cpu_fallback"]
|
||||||
models_fitting: list[str] = Field(default_factory=list)
|
models_fitting: list[str] = Field(default_factory=list)
|
||||||
slots: list[str] = Field(default_factory=list)
|
slots: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
@@ -65,15 +74,25 @@ class PlacementPlan(BaseModel):
|
|||||||
assignments: list[Assignment] = Field(default_factory=list)
|
assignments: list[Assignment] = Field(default_factory=list)
|
||||||
unplaced: list[Unplaced] = Field(default_factory=list)
|
unplaced: list[Unplaced] = Field(default_factory=list)
|
||||||
free_gpus: list[str] = Field(default_factory=list)
|
free_gpus: list[str] = Field(default_factory=list)
|
||||||
|
disabled_gpus: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
def roles_on(self, node: str) -> list[str]:
|
def roles_on(self, node: str) -> list[str]:
|
||||||
return [a.role for a in self.assignments if a.node == node]
|
return [a.role for a in self.assignments if a.node == node]
|
||||||
|
|
||||||
|
|
||||||
def load_catalog(directory: str | Path) -> tuple[list[RoleSpec], list[ModelSpec]]:
|
def load_catalog(directory: str | Path) -> tuple[list[RoleSpec], list[ModelSpec]]:
|
||||||
|
"""Lädt Rollen und Modelle. Engine-Grenzen (z. B. PyTorch nicht auf Maxwell/Pascal) fließen in min_cc ein."""
|
||||||
d = Path(directory)
|
d = Path(directory)
|
||||||
|
data = load_yaml(d / "models.yaml")
|
||||||
|
engines = {k: EngineSpec(name=k, **v) for k, v in (data.get("engines") or {}).items()}
|
||||||
|
models = []
|
||||||
|
for name, spec in (data.get("models") or {}).items():
|
||||||
|
model = ModelSpec(name=name, **spec)
|
||||||
|
floors = [engines[e].min_cc for e in model.engines if e in engines]
|
||||||
|
if floors:
|
||||||
|
model.min_cc = max(model.min_cc, min(floors))
|
||||||
|
models.append(model)
|
||||||
roles = [RoleSpec(name=k, **v) for k, v in (load_yaml(d / "roles.yaml").get("roles") or {}).items()]
|
roles = [RoleSpec(name=k, **v) for k, v in (load_yaml(d / "roles.yaml").get("roles") or {}).items()]
|
||||||
models = [ModelSpec(name=k, **v) for k, v in (load_yaml(d / "models.yaml").get("models") or {}).items()]
|
|
||||||
return roles, models
|
return roles, models
|
||||||
|
|
||||||
|
|
||||||
@@ -105,11 +124,39 @@ def _fitting_models(role: RoleSpec, models: list[ModelSpec], gpu: GPU | None) ->
|
|||||||
|
|
||||||
def plan(nodes: list[NodeProfile], roles: list[RoleSpec], models: list[ModelSpec]) -> PlacementPlan:
|
def plan(nodes: list[NodeProfile], roles: list[RoleSpec], models: list[ModelSpec]) -> PlacementPlan:
|
||||||
result = PlacementPlan()
|
result = PlacementPlan()
|
||||||
free: list[tuple[NodeProfile, GPU]] = [(n, g) for n in nodes for g in n.gpus]
|
free: list[tuple[NodeProfile, GPU]] = [(n, g) for n in nodes for g in n.gpus if not g.disabled]
|
||||||
|
result.disabled_gpus = [f"{n.name}/gpu{g.index} {g.name}" + (f" – {g.note}" if g.note else "")
|
||||||
|
for n in nodes for g in n.gpus if g.disabled]
|
||||||
cpu_load: dict[str, int] = {n.name: 0 for n in nodes}
|
cpu_load: dict[str, int] = {n.name: 0 for n in nodes}
|
||||||
|
|
||||||
for role in sorted(roles, key=lambda r: (r.priority, r.name)):
|
for role in sorted(roles, key=lambda r: (r.priority, r.name)):
|
||||||
placed = 0
|
placed = 0
|
||||||
|
if role.share_with and not role.needs.gpu:
|
||||||
|
host = next((a for a in result.assignments if a.role == role.share_with), None)
|
||||||
|
if host is None:
|
||||||
|
result.unplaced.append(Unplaced(role=role.name, reason=f"Rolle {role.share_with} nicht platziert",
|
||||||
|
optional=role.optional))
|
||||||
|
else:
|
||||||
|
cpu_load[host.node] += 1
|
||||||
|
result.assignments.append(Assignment(role=role.name, node=host.node, mode="cpu",
|
||||||
|
models_fitting=_fitting_models(role, models, None),
|
||||||
|
slots=role.slots))
|
||||||
|
continue
|
||||||
|
if role.share_with:
|
||||||
|
host = next((a for a in result.assignments if a.role == role.share_with and a.mode == "gpu"), None)
|
||||||
|
gpu = None
|
||||||
|
if host is not None:
|
||||||
|
node = next(n for n in nodes if n.name == host.node)
|
||||||
|
gpu = next(g for g in node.gpus if g.index == host.gpu_index)
|
||||||
|
fitting = _fitting_models(role, models, gpu) if gpu is not None else []
|
||||||
|
if gpu is not None and fitting:
|
||||||
|
result.assignments.append(Assignment(
|
||||||
|
role=role.name, node=host.node, gpu_index=gpu.index, gpu_name=gpu.name, vram_gb=gpu.memory_gb,
|
||||||
|
mode="shared", models_fitting=fitting, slots=role.slots))
|
||||||
|
else:
|
||||||
|
reason = f"Rolle {role.share_with} nicht platziert" if gpu is None else "kein passendes Modell"
|
||||||
|
result.unplaced.append(Unplaced(role=role.name, reason=reason, optional=role.optional))
|
||||||
|
continue
|
||||||
if role.needs.gpu:
|
if role.needs.gpu:
|
||||||
used_nodes: set[str] = set()
|
used_nodes: set[str] = set()
|
||||||
for _ in range(role.count):
|
for _ in range(role.count):
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ class GPU(BaseModel):
|
|||||||
memory_gb: float
|
memory_gb: float
|
||||||
compute_cap: float
|
compute_cap: float
|
||||||
driver: str | None = None
|
driver: str | None = None
|
||||||
|
disabled: bool = False # bewusst außer Betrieb (z. B. Strom sparen) – Placement ignoriert die Karte
|
||||||
|
note: str = ""
|
||||||
|
|
||||||
|
|
||||||
class Disk(BaseModel):
|
class Disk(BaseModel):
|
||||||
|
|||||||
@@ -69,3 +69,19 @@ class OpenAIChatClient:
|
|||||||
usage=data.get("usage") or {},
|
usage=data.get("usage") or {},
|
||||||
model=data.get("model"),
|
model=data.get("model"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def speech(self, model: str, text: str, *, voice: str = "default", response_format: str = "wav") -> bytes:
|
||||||
|
"""OpenAI-kompatibles TTS (`/v1/audio/speech`), z. B. über LiteLLM."""
|
||||||
|
body = {"model": model, "input": text, "voice": voice, "response_format": response_format}
|
||||||
|
try:
|
||||||
|
resp = self.client.post(f"{self.base_url}/v1/audio/speech", json=body, headers=self.headers)
|
||||||
|
except httpx.HTTPError as exc:
|
||||||
|
raise EndpointError(f"{type(exc).__name__}: {exc}") from exc
|
||||||
|
if resp.status_code >= 400:
|
||||||
|
raise EndpointError(f"HTTP {resp.status_code}: {resp.text[:300]}")
|
||||||
|
return resp.content
|
||||||
|
|
||||||
|
|
||||||
|
class SpeechClient(Protocol):
|
||||||
|
def speech(self, model: str, text: str, *, voice: str = "default", response_format: str = "wav") -> bytes: ...
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ class SlotSpec(BaseModel):
|
|||||||
effective_context: int | None = None
|
effective_context: int | None = None
|
||||||
supports_lora: bool = False
|
supports_lora: bool = False
|
||||||
fallbacks: list[str] = Field(default_factory=list)
|
fallbacks: list[str] = Field(default_factory=list)
|
||||||
kind: Literal["chat", "embedding", "rerank"] = "chat"
|
kind: Literal["chat", "embedding", "rerank", "tts"] = "chat"
|
||||||
canary: bool = True # vom Prober geprüft (für tier 0 standardmäßig aus, siehe canary.py)
|
canary: bool = True # vom Prober geprüft (für tier 0 standardmäßig aus, siehe canary.py)
|
||||||
ops: dict[str, str] = Field(default_factory=dict) # docker_host, container, pve_node, vmid → Healer-Labels
|
ops: dict[str, str] = Field(default_factory=dict) # docker_host, container, pve_node, vmid → Healer-Labels
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""Schlanker OpenAI-kompatibler TTS-Dienst (`/v1/audio/speech`) für CPU-Engines wie Piper.
|
||||||
|
|
||||||
|
Der Befehl ist als Vorlage konfigurierbar, weil sich die Piper-CLI zwischen Versionen unterscheidet:
|
||||||
|
KNASTER_TTS_CMD Standard: "piper -m {model} -f {out}" (Text kommt über stdin)
|
||||||
|
KNASTER_TTS_VOICES "thorsten=/voices/de_DE-thorsten-medium.onnx,kerstin=/voices/de_DE-kerstin-low.onnx"
|
||||||
|
KNASTER_TTS_DEFAULT_VOICE Name der Standardstimme (sonst die erste)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from fastapi import FastAPI, Header, HTTPException, Response
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
DEFAULT_CMD = "piper -m {model} -f {out}"
|
||||||
|
MAX_CHARS = 5000
|
||||||
|
|
||||||
|
|
||||||
|
class SpeechIn(BaseModel):
|
||||||
|
model: str = "tts-de"
|
||||||
|
input: str = Field(min_length=1, max_length=MAX_CHARS)
|
||||||
|
voice: str = "default"
|
||||||
|
response_format: str = "wav"
|
||||||
|
|
||||||
|
|
||||||
|
class CommandBackend:
|
||||||
|
def __init__(self, voices: dict[str, str], default_voice: str | None = None, cmd: str = DEFAULT_CMD,
|
||||||
|
timeout: float = 120.0, max_parallel: int = 2):
|
||||||
|
if not voices:
|
||||||
|
raise ValueError("Mindestens eine Stimme konfigurieren")
|
||||||
|
self.voices = voices
|
||||||
|
self.default_voice = default_voice or next(iter(voices))
|
||||||
|
self.cmd = cmd
|
||||||
|
self.timeout = timeout
|
||||||
|
self._slots = threading.Semaphore(max_parallel) # CPU nicht überbuchen
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls) -> CommandBackend:
|
||||||
|
raw = os.environ.get("KNASTER_TTS_VOICES", "")
|
||||||
|
voices = dict(item.split("=", 1) for item in raw.split(",") if "=" in item)
|
||||||
|
return cls(voices, os.environ.get("KNASTER_TTS_DEFAULT_VOICE") or None,
|
||||||
|
os.environ.get("KNASTER_TTS_CMD", DEFAULT_CMD),
|
||||||
|
max_parallel=int(os.environ.get("KNASTER_TTS_PARALLEL", "2")))
|
||||||
|
|
||||||
|
def synthesize(self, text: str, voice: str = "default") -> bytes:
|
||||||
|
name = self.default_voice if voice in ("default", "", None) else voice
|
||||||
|
if name not in self.voices:
|
||||||
|
raise KeyError(f"Stimme {name} unbekannt (verfügbar: {', '.join(self.voices)})")
|
||||||
|
with self._slots, tempfile.TemporaryDirectory() as tmp:
|
||||||
|
out = Path(tmp) / "out.wav"
|
||||||
|
args = [part.format(model=self.voices[name], out=str(out)) for part in shlex.split(self.cmd)]
|
||||||
|
proc = subprocess.run(args, input=text.encode("utf-8"), capture_output=True, timeout=self.timeout)
|
||||||
|
if proc.returncode != 0 or not out.exists():
|
||||||
|
raise RuntimeError(f"TTS-Befehl fehlgeschlagen ({proc.returncode}): "
|
||||||
|
f"{proc.stderr.decode('utf-8', 'replace')[-300:]}")
|
||||||
|
return out.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def create_tts_app(backend: CommandBackend | None = None, token: str | None = None) -> FastAPI:
|
||||||
|
backend = backend or CommandBackend.from_env()
|
||||||
|
token = os.environ.get("KNASTER_TTS_TOKEN", "") if token is None else token
|
||||||
|
app = FastAPI(title="knaster-tts")
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
def health() -> dict:
|
||||||
|
return {"status": "ok", "voices": list(backend.voices), "default": backend.default_voice}
|
||||||
|
|
||||||
|
@app.post("/v1/audio/speech")
|
||||||
|
def speech(body: SpeechIn, authorization: str | None = Header(default=None)) -> Response:
|
||||||
|
if token and authorization != f"Bearer {token}":
|
||||||
|
raise HTTPException(401, "unauthorized")
|
||||||
|
if body.response_format != "wav":
|
||||||
|
raise HTTPException(400, "nur response_format=wav")
|
||||||
|
try:
|
||||||
|
audio = backend.synthesize(body.input, body.voice)
|
||||||
|
except KeyError as exc:
|
||||||
|
raise HTTPException(400, str(exc)) from exc
|
||||||
|
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
||||||
|
raise HTTPException(502, str(exc)) from exc
|
||||||
|
return Response(audio, media_type="audio/wav")
|
||||||
|
|
||||||
|
return app
|
||||||
@@ -165,7 +165,7 @@ def test_proxmox_site_consistent_with_registry_and_prometheus():
|
|||||||
assert vm["driver_generation"] in {"legacy", "modern"}
|
assert vm["driver_generation"] in {"legacy", "modern"}
|
||||||
gpus = [vm["gpu"] for vm in vms.values()]
|
gpus = [vm["gpu"] for vm in vms.values()]
|
||||||
assert len(gpus) == len(set(gpus)) # keine GPU doppelt durchgereicht
|
assert len(gpus) == len(set(gpus)) # keine GPU doppelt durchgereicht
|
||||||
by_vmid = {str(vm["vmid"]): (name, vm) for name, vm in vms.items()}
|
by_vmid = {str(vm["vmid"]): (name, vm) for name, vm in {**vms, **site["containers"]}.items()}
|
||||||
for slot, spec in reg["slots"].items():
|
for slot, spec in reg["slots"].items():
|
||||||
ops = spec.get("ops")
|
ops = spec.get("ops")
|
||||||
if not ops:
|
if not ops:
|
||||||
|
|||||||
@@ -41,7 +41,14 @@ def test_placement_knaster():
|
|||||||
assert all("P2000" in a.gpu_name for a in by_role["gpu-embed"] + by_role["gpu-small"])
|
assert all("P2000" in a.gpu_name for a in by_role["gpu-embed"] + by_role["gpu-small"])
|
||||||
assert by_role["infra"][0].mode == "cpu"
|
assert by_role["infra"][0].mode == "cpu"
|
||||||
assert not p.unplaced
|
assert not p.unplaced
|
||||||
assert len(p.free_gpus) == 4 and all("M4000" in g for g in p.free_gpus)
|
# M4000 bewusst außer Betrieb (ADR-0006): weder frei noch belegt
|
||||||
|
assert p.free_gpus == []
|
||||||
|
assert len(p.disabled_gpus) == 4 and all("M4000" in g for g in p.disabled_gpus)
|
||||||
|
tts = {a.role: a for a in p.assignments if a.role.startswith("tts")}
|
||||||
|
assert tts["tts-hq"].mode == "shared" and tts["tts-hq"].node == "pve-ada"
|
||||||
|
assert "qwen3-tts-1.7b" in tts["tts-hq"].models_fitting
|
||||||
|
assert tts["tts-cpu"].node == by_role["infra"][0].node
|
||||||
|
assert tts["tts-cpu"].models_fitting == ["piper-de-thorsten-medium"]
|
||||||
|
|
||||||
|
|
||||||
def test_placement_other_cluster():
|
def test_placement_other_cluster():
|
||||||
@@ -49,7 +56,30 @@ def test_placement_other_cluster():
|
|||||||
p = plan(load_nodes(REPO_ROOT / "sites/example-single-gpu/inventory.yaml"), roles, models)
|
p = plan(load_nodes(REPO_ROOT / "sites/example-single-gpu/inventory.yaml"), roles, models)
|
||||||
modes = {(a.role, a.mode) for a in p.assignments}
|
modes = {(a.role, a.mode) for a in p.assignments}
|
||||||
assert ("gpu-main", "gpu") in modes and ("gpu-embed", "cpu_fallback") in modes
|
assert ("gpu-main", "gpu") in modes and ("gpu-embed", "cpu_fallback") in modes
|
||||||
assert [u.role for u in p.unplaced] == ["gpu-flex"] and p.unplaced[0].optional
|
assert [u.role for u in p.unplaced] == ["gpu-flex", "tts-hq"] and all(u.optional for u in p.unplaced)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pytorch_never_on_legacy_gpus():
|
||||||
|
"""Engine-Grenze: PyTorch-Modelle (TTS) landen nie auf Maxwell/Pascal, llama.cpp schon (ADR-0006)."""
|
||||||
|
from knaster.inventory.placement import RoleSpec
|
||||||
|
|
||||||
|
_, models = load_catalog(REPO_ROOT / "catalog")
|
||||||
|
assert {m.name: m.min_cc for m in models}["qwen3-tts-0.6b"] >= 7.5
|
||||||
|
role = RoleSpec(name="tts-gpu", needs={"gpu": True, "engine": "pytorch"}, kinds=["tts"], optional=True)
|
||||||
|
legacy = [NodeProfile(name="old", memory_gb=32, gpus=[
|
||||||
|
{"index": 0, "name": "Quadro M4000", "memory_gb": 8, "compute_cap": 5.2},
|
||||||
|
{"index": 1, "name": "Quadro P2000", "memory_gb": 5, "compute_cap": 6.1}])]
|
||||||
|
p = plan(legacy, [role], models)
|
||||||
|
assert [u.role for u in p.unplaced] == ["tts-gpu"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_gpus_are_ignored():
|
||||||
|
roles, models = load_catalog(REPO_ROOT / "catalog")
|
||||||
|
nodes = [NodeProfile(name="n", memory_gb=32, gpus=[
|
||||||
|
{"index": 0, "name": "Quadro P2000", "memory_gb": 5, "compute_cap": 6.1, "disabled": True}])]
|
||||||
|
p = plan(nodes, roles, models)
|
||||||
|
assert all(a.mode != "gpu" for a in p.assignments)
|
||||||
|
assert p.disabled_gpus == ["n/gpu0 Quadro P2000"]
|
||||||
|
|
||||||
|
|
||||||
def test_placement_maxwell_never_gets_vllm():
|
def test_placement_maxwell_never_gets_vllm():
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import io
|
||||||
|
import json
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
import wave
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from knaster.audio import rtf, wav_duration
|
||||||
|
from knaster.bench import bench_tts
|
||||||
|
from knaster.canary import CanaryProber
|
||||||
|
from knaster.llm import OpenAIChatClient
|
||||||
|
from knaster.tts_server import CommandBackend, create_tts_app
|
||||||
|
|
||||||
|
|
||||||
|
def make_wav(seconds: float, rate: int = 22050) -> bytes:
|
||||||
|
buf = io.BytesIO()
|
||||||
|
with wave.open(buf, "wb") as w:
|
||||||
|
w.setnchannels(1)
|
||||||
|
w.setsampwidth(2)
|
||||||
|
w.setframerate(rate)
|
||||||
|
w.writeframes(struct.pack("<h", 0) * int(seconds * rate))
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
FAKE_PIPER = """
|
||||||
|
import sys, wave, struct
|
||||||
|
args = sys.argv[1:]
|
||||||
|
out = args[args.index("-f") + 1]
|
||||||
|
model = args[args.index("-m") + 1]
|
||||||
|
text = sys.stdin.read()
|
||||||
|
if "FEHLER" in text:
|
||||||
|
print("kaputt", file=sys.stderr); sys.exit(3)
|
||||||
|
with wave.open(out, "wb") as w:
|
||||||
|
w.setnchannels(1); w.setsampwidth(2); w.setframerate(16000)
|
||||||
|
w.writeframes(struct.pack("<h", 0) * (160 * len(text))) # 10 ms pro Zeichen
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def backend(tmp_path):
|
||||||
|
script = tmp_path / "fake_piper.py"
|
||||||
|
script.write_text(FAKE_PIPER)
|
||||||
|
return CommandBackend({"thorsten": "/voices/t.onnx", "kerstin": "/voices/k.onnx"},
|
||||||
|
cmd=f"{sys.executable} {script} -m {{model}} -f {{out}}")
|
||||||
|
|
||||||
|
|
||||||
|
def test_wav_helpers():
|
||||||
|
assert wav_duration(make_wav(1.5)) == pytest.approx(1.5)
|
||||||
|
assert wav_duration(b"kein wav") == 0.0
|
||||||
|
assert rtf(3.0, 1.5) == 2.0 and rtf(1.0, 0) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_tts_server(backend):
|
||||||
|
client = TestClient(create_tts_app(backend, token="t"))
|
||||||
|
assert client.get("/health").json()["default"] == "thorsten"
|
||||||
|
H = {"Authorization": "Bearer t"}
|
||||||
|
r = client.post("/v1/audio/speech", json={"input": "Hallo Welt"}, headers=H)
|
||||||
|
assert r.status_code == 200 and r.headers["content-type"] == "audio/wav"
|
||||||
|
assert wav_duration(r.content) == pytest.approx(0.1)
|
||||||
|
assert client.post("/v1/audio/speech", json={"input": "x", "voice": "kerstin"}, headers=H).status_code == 200
|
||||||
|
assert client.post("/v1/audio/speech", json={"input": "x", "voice": "nope"}, headers=H).status_code == 400
|
||||||
|
assert client.post("/v1/audio/speech", json={"input": "FEHLER"}, headers=H).status_code == 502
|
||||||
|
assert client.post("/v1/audio/speech", json={"input": "x", "response_format": "mp3"},
|
||||||
|
headers=H).status_code == 400
|
||||||
|
assert client.post("/v1/audio/speech", json={"input": "x"}).status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_backend_requires_voice():
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
CommandBackend({})
|
||||||
|
|
||||||
|
|
||||||
|
def tts_transport(seconds: float = 2.0, fail: bool = False):
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
assert request.url.path == "/v1/audio/speech"
|
||||||
|
body = json.loads(request.content)
|
||||||
|
if fail or body["model"] == "kaputt":
|
||||||
|
return httpx.Response(503, text="down")
|
||||||
|
return httpx.Response(200, content=make_wav(seconds), headers={"content-type": "audio/wav"})
|
||||||
|
|
||||||
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||||
|
|
||||||
|
|
||||||
|
def test_bench_tts():
|
||||||
|
client = OpenAIChatClient("http://x", client=tts_transport(2.0))
|
||||||
|
result = bench_tts("http://x", "tts-de", runs=2, client=client)
|
||||||
|
assert len(result["runs"]) == 2 and result["errors"] == []
|
||||||
|
assert result["runs"][0]["audio_seconds"] == 2.0 and result["rtf_median"] is not None
|
||||||
|
bad = bench_tts("http://x", "kaputt", runs=1, client=client)
|
||||||
|
assert bad["runs"] == [] and bad["errors"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_canary_probes_tts_slots(registry):
|
||||||
|
from prometheus_client import CollectorRegistry
|
||||||
|
|
||||||
|
from knaster.registry import Registry
|
||||||
|
from knaster.store import Store
|
||||||
|
|
||||||
|
from .conftest import REGISTRY
|
||||||
|
|
||||||
|
data = {"slots": {**REGISTRY["slots"], "tts-de": {
|
||||||
|
"tier": 1, "device": "cpu", "engine": "piper", "litellm_model": "tts-de", "base_model": "piper",
|
||||||
|
"max_context": 5000, "kind": "tts"}}}
|
||||||
|
reg = Registry.from_dict(data, Store("sqlite://"))
|
||||||
|
chat = OpenAIChatClient("http://x", client=tts_transport(0.8))
|
||||||
|
prober = CanaryProber(reg, chat, metrics_registry=CollectorRegistry())
|
||||||
|
tts = next(prober.probe(s) for s in reg.slots() if s.name == "tts-de")
|
||||||
|
assert tts.up and tts.ok
|
||||||
|
silent = OpenAIChatClient("http://x", client=tts_transport(0.05))
|
||||||
|
assert not CanaryProber(reg, silent, metrics_registry=CollectorRegistry()).probe(reg.slot("tts-de")).ok
|
||||||
|
down = OpenAIChatClient("http://x", client=tts_transport(fail=True))
|
||||||
|
assert not CanaryProber(reg, down, metrics_registry=CollectorRegistry()).probe(reg.slot("tts-de")).up
|
||||||
|
|
||||||
@@ -7,3 +7,6 @@ MAX_LORA_RANK=64
|
|||||||
MODELS_DIR=/srv/models
|
MODELS_DIR=/srv/models
|
||||||
ADAPTERS_DIR=/srv/adapters
|
ADAPTERS_DIR=/srv/adapters
|
||||||
HF_TOKEN=
|
HF_TOKEN=
|
||||||
|
# nur gpu-flex mit Profil "tts": GPU_MEM_UTIL für vLLM dann auf ~0.55 senken, damit Qwen3-TTS Platz hat
|
||||||
|
QWEN_TTS_IMAGE=
|
||||||
|
QWEN_TTS_MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice
|
||||||
|
|||||||
@@ -36,6 +36,24 @@ services:
|
|||||||
retries: 3
|
retries: 3
|
||||||
start_period: 300s # Laden großer Modelle
|
start_period: 300s # Laden großer Modelle
|
||||||
|
|
||||||
|
# Qwen3-TTS (ADR-0006) – nur auf gpu-flex, Profil "tts". Server muss OpenAI-kompatibles
|
||||||
|
# /v1/audio/speech anbieten (z. B. ein Wrapper um faster-qwen3-tts); Image-Wahl beim Deployment.
|
||||||
|
qwen-tts:
|
||||||
|
image: ${QWEN_TTS_IMAGE:-knaster/qwen-tts:unset}
|
||||||
|
container_name: qwen-tts
|
||||||
|
profiles: ["tts"]
|
||||||
|
restart: unless-stopped
|
||||||
|
labels: {autoheal: "true"}
|
||||||
|
environment:
|
||||||
|
TTS_MODEL: ${QWEN_TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}
|
||||||
|
volumes:
|
||||||
|
- ${MODELS_DIR:-/srv/models}:/root/.cache/huggingface
|
||||||
|
ports: ["8091:8091"]
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices: [{driver: nvidia, count: 1, capabilities: [gpu]}]
|
||||||
|
|
||||||
gpu-exporter:
|
gpu-exporter:
|
||||||
image: utkuozdemir/nvidia_gpu_exporter:1.2.1
|
image: utkuozdemir/nvidia_gpu_exporter:1.2.1
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|||||||
@@ -37,6 +37,20 @@ services:
|
|||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 3
|
retries: 3
|
||||||
|
|
||||||
|
tts-piper:
|
||||||
|
<<: *restart
|
||||||
|
build:
|
||||||
|
context: ../../..
|
||||||
|
dockerfile: deploy/docker/tts-piper.Dockerfile
|
||||||
|
image: knaster/tts-piper:${KNASTER_VERSION:-dev}
|
||||||
|
container_name: tts-piper
|
||||||
|
environment:
|
||||||
|
KNASTER_TTS_PARALLEL: ${TTS_PARALLEL:-2} # i7 mit 4 Kernen: max. 2 parallele Synthesen
|
||||||
|
ports: ["8090:8090"]
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "2", memory: 1G}
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
<<: *restart
|
<<: *restart
|
||||||
image: pgvector/pgvector:pg16
|
image: pgvector/pgvector:pg16
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# Piper-TTS hinter dem OpenAI-kompatiblen knaster-TTS-Dienst. Build-Kontext: Repo-Wurzel.
|
||||||
|
# Hinweis: piper-tts ab 1.3 ist GPL-3.0 – läuft hier als eigenständiger Dienst.
|
||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
ARG PIPER_VERSION=1.3.0
|
||||||
|
ARG VOICE_BASE=https://huggingface.co/rhasspy/piper-voices/resolve/main/de/de_DE
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||||
|
RUN pip install "piper-tts==${PIPER_VERSION}"
|
||||||
|
|
||||||
|
COPY core/pyproject.toml core/pyproject.toml
|
||||||
|
COPY core/src core/src
|
||||||
|
RUN pip install ./core
|
||||||
|
|
||||||
|
RUN mkdir -p /voices && cd /voices \
|
||||||
|
&& curl -fsSL -o de_DE-thorsten-medium.onnx "${VOICE_BASE}/thorsten/medium/de_DE-thorsten-medium.onnx" \
|
||||||
|
&& curl -fsSL -o de_DE-thorsten-medium.onnx.json "${VOICE_BASE}/thorsten/medium/de_DE-thorsten-medium.onnx.json"
|
||||||
|
|
||||||
|
ENV KNASTER_TTS_VOICES=thorsten=/voices/de_DE-thorsten-medium.onnx \
|
||||||
|
KNASTER_TTS_CMD="piper -m {model} -f {out}" \
|
||||||
|
KNASTER_TTS_PARALLEL=2
|
||||||
|
|
||||||
|
RUN useradd --system --uid 10002 tts
|
||||||
|
USER tts
|
||||||
|
EXPOSE 8090
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
|
||||||
|
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/health', timeout=4)"
|
||||||
|
CMD ["knaster", "tts-serve", "--host", "0.0.0.0", "--port", "8090"]
|
||||||
@@ -58,10 +58,25 @@ model_list:
|
|||||||
api_key: none
|
api_key: none
|
||||||
model_info: {mode: rerank}
|
model_info: {mode: rerank}
|
||||||
|
|
||||||
|
# --- Sprachausgabe (OpenAI-kompatibles /v1/audio/speech) ---
|
||||||
|
- model_name: tts-de # Piper im Infra-LXC (knaster tts-serve)
|
||||||
|
litellm_params:
|
||||||
|
model: openai/tts-de
|
||||||
|
api_base: http://tts-piper:8090/v1
|
||||||
|
api_key: none
|
||||||
|
model_info: {mode: audio_speech}
|
||||||
|
- model_name: tts-de-hq # Qwen3-TTS auf gpu-flex (Server-Wahl beim Deployment)
|
||||||
|
litellm_params:
|
||||||
|
model: openai/tts-de-hq
|
||||||
|
api_base: http://gpu-flex:8091/v1
|
||||||
|
api_key: none
|
||||||
|
model_info: {mode: audio_speech}
|
||||||
|
|
||||||
router_settings:
|
router_settings:
|
||||||
# Infrastruktur-Fallbacks innerhalb einer Stufe; Qualitäts-Eskalation macht der Router-Service
|
# Infrastruktur-Fallbacks innerhalb einer Stufe; Qualitäts-Eskalation macht der Router-Service
|
||||||
fallbacks:
|
fallbacks:
|
||||||
- tier1-general: [tier1-flex]
|
- tier1-general: [tier1-flex]
|
||||||
|
- tts-de-hq: [tts-de]
|
||||||
allowed_fails: 2
|
allowed_fails: 2
|
||||||
cooldown_time: 60
|
cooldown_time: 60
|
||||||
num_retries: 1
|
num_retries: 1
|
||||||
|
|||||||
@@ -16,7 +16,8 @@ Vorgehen pro Baustein: **Spec → Implementierung → Tests → Commit**. Bis zu
|
|||||||
| 8 | API | SPEC-core §8 | FastAPI-App, Auth, Metriken | ✅ |
|
| 8 | API | SPEC-core §8 | FastAPI-App, Auth, Metriken | ✅ |
|
||||||
| 9 | Deployment-Artefakte | PLAN §1, §3 | Compose pro Rolle, LiteLLM, Prometheus-Regeln, Alertmanager, n8n-Workflows | ✅ |
|
| 9 | Deployment-Artefakte | PLAN §1, §3 | Compose pro Rolle, LiteLLM, Prometheus-Regeln, Alertmanager, n8n-Workflows | ✅ |
|
||||||
| 10 | Plattform | ADR-0001/0002 | OpenTofu (bpg/proxmox), Ansible-Rollen (IOMMU/VFIO, ZFS-Replikation, Treiber-Pinning, Xid-Exporter) | ✅ geschrieben, **nicht** gegen echten Cluster/`tofu validate` geprüft |
|
| 10 | Plattform | ADR-0001/0002 | OpenTofu (bpg/proxmox), Ansible-Rollen (IOMMU/VFIO, ZFS-Replikation, Treiber-Pinning, Xid-Exporter) | ✅ geschrieben, **nicht** gegen echten Cluster/`tofu validate` geprüft |
|
||||||
| 11 | CI | – | Gitea Actions: Tests, Lint, Konfig-Validierung, Compose-Prüfung | ✅ |
|
| 11 | CI | – | Gitea Actions: Tests, Lint, Konfig-Validierung (Runner `gitrunner-knaster`) | ✅ |
|
||||||
|
| 12 | TTS + Legacy-GPUs | ADR-0006 | Engine-Grenzen im Katalog, deaktivierte GPUs, geteilte GPU-Rollen, TTS-Slots, Piper-Dienst, TTS-Canary, `knaster bench tts` | ✅ |
|
||||||
|
|
||||||
## Teststrategie
|
## Teststrategie
|
||||||
|
|
||||||
@@ -29,7 +30,7 @@ Vorgehen pro Baustein: **Spec → Implementierung → Tests → Commit**. Bis zu
|
|||||||
|
|
||||||
| Was | Wie geprüft |
|
| Was | Wie geprüft |
|
||||||
|---|---|
|
|---|---|
|
||||||
| Kernlogik (Router, Eskalation, Verifier, Pflege-Agent, Healer, Lifecycle, Placement) | 127 Unit- und Integrationstests (`make test`) |
|
| Kernlogik (Router, Eskalation, Verifier, Pflege-Agent, Healer, Lifecycle, Placement) | 136 Unit- und Integrationstests (`make test`) |
|
||||||
| Konfiguration (Registry ↔ Skills ↔ LiteLLM ↔ Healer ↔ Policy ↔ Katalog) | `knaster validate` |
|
| Konfiguration (Registry ↔ Skills ↔ LiteLLM ↔ Healer ↔ Policy ↔ Katalog) | `knaster validate` |
|
||||||
| Compose-Dateien | `docker compose config` (`make compose-check`) |
|
| Compose-Dateien | `docker compose config` (`make compose-check`) |
|
||||||
| Prometheus/Alertmanager/n8n-JSON | Strukturtests (`core/tests/test_artifacts.py`) |
|
| Prometheus/Alertmanager/n8n-JSON | Strukturtests (`core/tests/test_artifacts.py`) |
|
||||||
@@ -43,6 +44,8 @@ Vorgehen pro Baustein: **Spec → Implementierung → Tests → Commit**. Bis zu
|
|||||||
- Den n8n Chat-Trigger nach dem Import prüfen (Parameter der Knotenversion).
|
- Den n8n Chat-Trigger nach dem Import prüfen (Parameter der Knotenversion).
|
||||||
- `sites/knaster/*`: Hosts, VM-IDs, IPs und PCI-Adressen sind Annahmen und werden durch den Inventarlauf ersetzt.
|
- `sites/knaster/*`: Hosts, VM-IDs, IPs und PCI-Adressen sind Annahmen und werden durch den Inventarlauf ersetzt.
|
||||||
- Classifier `ticket-kategorie.json` entsteht erst aus gelabelten Fällen (Warnung von `knaster validate`).
|
- Classifier `ticket-kategorie.json` entsteht erst aus gelabelten Fällen (Warnung von `knaster validate`).
|
||||||
|
- Qwen3-TTS-Server für `tts-de-hq` auswählen (OpenAI-kompatibler Wrapper um faster-qwen3-tts) und Image setzen.
|
||||||
|
- Benchmark-Tag: `llama-bench` auf P2000 (und M4000), `knaster bench tts` für beide TTS-Slots, dazu Strom messen. Ergebnisse in Katalog und Registry übernehmen.
|
||||||
|
|
||||||
## Gemeinsames Deployment (später)
|
## Gemeinsames Deployment (später)
|
||||||
|
|
||||||
|
|||||||
+8
-4
@@ -51,12 +51,15 @@ Ziel: Ein starkes LLM (OpenRouter) erarbeitet neue Aufgaben einmalig und bricht
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `gpu-main` | RTX 4000 Ada #1 (20 GB) | vLLM, Multi-LoRA | Slot `tier1-general` plus Skill-Adapter |
|
| `gpu-main` | RTX 4000 Ada #1 (20 GB) | vLLM, Multi-LoRA | Slot `tier1-general` plus Skill-Adapter |
|
||||||
| `gpu-flex` | RTX 4000 Ada #2 (20 GB) | vLLM / Unsloth | tagsüber zweiter Slot oder Reserve, nachts LoRA-Training |
|
| `gpu-flex` | RTX 4000 Ada #2 (20 GB) | vLLM / Unsloth | tagsüber zweiter Slot oder Reserve, nachts LoRA-Training |
|
||||||
| `gpu-embed` | 3× Quadro P2000 (5 GB) | llama.cpp (Embedding/Rerank) | `embed-default`, `rerank-default`, redundant |
|
| `gpu-embed` | 2× Quadro P2000 (5 GB) | llama.cpp (Embedding/Rerank) | `embed-default`, `rerank-default`, redundant auf zwei Hosts |
|
||||||
| `gpu-small` | 4× Quadro M4000 (8 GB) | llama.cpp | `verifier-small`, `signature-extractor` (3–4B) – **auf Nutzen prüfen** |
|
| `gpu-small` | Quadro P2000 (5 GB) | llama.cpp | `verifier-small`, `signature-extractor` (3–4B) |
|
||||||
|
| – | 4× Quadro M4000 (8 GB) | – | **vorerst außer Betrieb** (ADR-0006), nur nach Benchmark für llama.cpp-Rollen |
|
||||||
|
| `tts-hq` | teilt RTX 4000 Ada #2 | PyTorch | `tts-de-hq`: Qwen3-TTS 1.7B (ADR-0006) |
|
||||||
|
| `tts-cpu` | Infra-LXC (CPU) | Piper (ONNX) | `tts-de`: schnelle Sprachausgabe, Fallback |
|
||||||
| `infra` | i7 6./7. Gen, 32 GB | Docker | n8n, LiteLLM, Postgres+pgvector, Redis, Router, Healer, Langfuse, Prometheus/Grafana |
|
| `infra` | i7 6./7. Gen, 32 GB | Docker | n8n, LiteLLM, Postgres+pgvector, Redis, Router, Healer, Langfuse, Prometheus/Grafana |
|
||||||
| `strong` | extern | OpenRouter | `tier0-strong`, Skill-Autor (Hermes) |
|
| `strong` | extern | OpenRouter | `tier0-strong`, Skill-Autor (Hermes) |
|
||||||
|
|
||||||
Wichtig: Auf Maxwell und Pascal läuft kein vLLM (das braucht Compute Capability ≥ 7.0), und CUDA 13 unterstützt sie nicht mehr. Daher auf diesen Nodes Treiber und Images pinnen.
|
Wichtig: Auf Maxwell und Pascal läuft kein vLLM (das braucht Compute Capability ≥ 7.0), und CUDA 13 unterstützt sie nicht mehr. Daher auf diesen Nodes Treiber und Images pinnen. Auch PyTorch hat dort keine Zukunft mehr (2.14 ist die letzte Version mit CUDA-12-Wheels). Deshalb laufen auf alten Karten nur llama.cpp-Workloads (ADR-0006).
|
||||||
|
|
||||||
### 1.4 Netzwerk und Storage (entschieden, siehe ADR-0002)
|
### 1.4 Netzwerk und Storage (entschieden, siehe ADR-0002)
|
||||||
|
|
||||||
@@ -296,10 +299,11 @@ Die Portabilität beruht also auf den **Daten und der Pipeline**, nicht auf Gewi
|
|||||||
- Slots statt Modellnamen, eine Lifecycle-Pipeline für alle Wechsel (ADR-0003)
|
- Slots statt Modellnamen, eine Lifecycle-Pipeline für alle Wechsel (ADR-0003)
|
||||||
- Halbautomatische Pflege über einen Pflege-Agenten mit Freigabe per Prompt (ADR-0004)
|
- Halbautomatische Pflege über einen Pflege-Agenten mit Freigabe per Prompt (ADR-0004)
|
||||||
- Infrastruktur heilt deterministisch, ohne LLM-Entscheidungen (ADR-0005)
|
- Infrastruktur heilt deterministisch, ohne LLM-Entscheidungen (ADR-0005)
|
||||||
|
- Alte GPUs nur llama.cpp, M4000 vorerst aus, TTS mit Piper (CPU) und Qwen3-TTS (Ada) (ADR-0006)
|
||||||
|
|
||||||
### Offen
|
### Offen
|
||||||
1. Erstes Basismodell für `tier1-general` (8–14B, gut quantisierbar, LoRA-freundlich, gutes Deutsch). Entscheidung per Kalibrierungslauf.
|
1. Erstes Basismodell für `tier1-general` (8–14B, gut quantisierbar, LoRA-freundlich, gutes Deutsch). Entscheidung per Kalibrierungslauf.
|
||||||
2. Bleiben die M4000 im Einsatz? Benchmark gegen P2000 bzw. CPU und Stromverbrauch
|
2. Kommen die M4000 zurück? `llama-bench` plus Strommessung am Deployment-Tag (bis dahin außer Betrieb, ADR-0006)
|
||||||
3. Verifier-Strategie pro Aufgabentyp: Code-Checks, Schema, kleines Judge-Modell, Stichproben durch das starke Modell
|
3. Verifier-Strategie pro Aufgabentyp: Code-Checks, Schema, kleines Judge-Modell, Stichproben durch das starke Modell
|
||||||
4. Schwellwerte: Graduierung (Start 90 %), Fallzahl fürs LoRA-Training (Start 300), Canary-Anteil (Start 10 %)
|
4. Schwellwerte: Graduierung (Start 90 %), Fallzahl fürs LoRA-Training (Start 300), Canary-Anteil (Start 10 %)
|
||||||
5. Kanal für den Pflege-Agenten: n8n Chat, Telegram oder Matrix
|
5. Kanal für den Pflege-Agenten: n8n Chat, Telegram oder Matrix
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# ADR-0006: Alte GPUs nur für llama.cpp, M4000 vorerst außer Betrieb, TTS auf Ada und CPU
|
||||||
|
|
||||||
|
- Status: angenommen
|
||||||
|
- Datum: 2026-10-12
|
||||||
|
|
||||||
|
## Kontext
|
||||||
|
- PyTorch unterstützt Maxwell und Pascal seit 2.8 nur noch in den cu126-Builds. **PyTorch 2.14 ist die letzte Version mit CUDA-12.x-Wheels** (Ankündigung vom 04.09.2026), danach geht es nur noch über einen eigenen Build.
|
||||||
|
- Pascal hat weder bf16 noch schnelles FP16, und FlashAttention fehlt. Qwen3-TTS 1.7B erreicht auf einer Tesla P40 nur **RTF ~7,3** (Referenz-Implementierung, gemessen von tinycomputers.io). P2000 und M4000 sind schwächer.
|
||||||
|
- llama.cpp unterstützt Maxwell und Pascal mit CUDA 12 weiterhin. FlashAttention läuft dort auch auf Maxwell.
|
||||||
|
- Für die M4000 gibt es keine direkten Messungen. Geschätzt liegt sie bei 7B Q4 auf 12–25 Token/s, bei 120 W pro Karte. Die P2000 decken Embedding, Rerank und Verifier bereits ab.
|
||||||
|
|
||||||
|
## Entscheidung
|
||||||
|
1. **Engine-Grenzen im Katalog** (`catalog/models.yaml → engines`): `pytorch` ab Compute Capability 7.5, `vllm` ab 7.0, `llamacpp` ab 5.0. Das Placement setzt PyTorch-Workloads damit nie auf Maxwell/Pascal.
|
||||||
|
2. **M4000 vorerst außer Betrieb** (`disabled: true` im Inventar). Wieder aufgenommen werden sie nur nach `llama-bench` plus Strommessung am Deployment-Tag, und dann nur für llama.cpp-Rollen.
|
||||||
|
3. **TTS:**
|
||||||
|
- `tts-de`: Piper auf der CPU im Infra-LXC. Schnell, robust, ohne GPU, zugleich Fallback.
|
||||||
|
- `tts-de-hq`: Qwen3-TTS 1.7B (Apache 2.0, Deutsch, Stimmenklonen) in bf16 auf `gpu-flex`, im Wechsel mit Training und `tier1-flex`. Wenn TTS aktiv ist, sinkt der vLLM-Anteil am VRAM auf ~55 %.
|
||||||
|
- Kein TTS auf P2000 oder M4000.
|
||||||
|
|
||||||
|
## Folgen
|
||||||
|
- Bei `gpu-flex` muss sich der Lifecycle die Karte mit TTS teilen. Während des Trainings gehen sowohl `tier1-flex` als auch `tts-de-hq` auf `unavailable`, und LiteLLM fällt auf `tts-de` zurück.
|
||||||
|
- Die tatsächlichen RTF-Werte misst `knaster bench tts` beim Deployment. Sie ersetzen die Schätzungen im Katalog.
|
||||||
@@ -17,7 +17,8 @@ Datenquellen: `nvidia-smi --query-gpu=index,name,memory.total,compute_cap,driver
|
|||||||
|
|
||||||
## 2. Katalog (`catalog/models.yaml`, `catalog/roles.yaml`)
|
## 2. Katalog (`catalog/models.yaml`, `catalog/roles.yaml`)
|
||||||
|
|
||||||
Ein Modell hat `kind` (llm, embedding, reranker), `vram_gb` (Gewichte plus Mindest-KV-Cache), `min_cc`, `engines`, `context`, `lora`.
|
Engines (`models.yaml → engines`) setzen Untergrenzen für die Compute Capability, die für alle ihre Modelle gelten (z. B. `pytorch ≥ 7.5`). GPUs mit `disabled: true` im Inventar ignoriert das Placement und listet sie unter `disabled_gpus`.
|
||||||
|
Ein Modell hat `kind` (llm, embedding, reranker, tts), `vram_gb` (Gewichte plus Mindest-KV-Cache), `min_cc`, `engines`, `context`, `lora`.
|
||||||
Eine Rolle hat `needs` (`engine`, `min_vram_gb`, `min_cc`, `max_cc?`, `gpu: true|false`), `prefer` (z. B. `smallest_fitting`), `count` und `slots`.
|
Eine Rolle hat `needs` (`engine`, `min_vram_gb`, `min_cc`, `max_cc?`, `gpu: true|false`), `prefer` (z. B. `smallest_fitting`), `count` und `slots`.
|
||||||
|
|
||||||
## 3. Placement (`plan(nodes, roles, models) -> PlacementPlan`)
|
## 3. Placement (`plan(nodes, roles, models) -> PlacementPlan`)
|
||||||
@@ -27,8 +28,9 @@ Deterministisch:
|
|||||||
2. Rollen nach `priority` (aufsteigend) abarbeiten. Für jede Rolle werden `count` GPUs gesucht, die `needs` erfüllen und noch frei sind.
|
2. Rollen nach `priority` (aufsteigend) abarbeiten. Für jede Rolle werden `count` GPUs gesucht, die `needs` erfüllen und noch frei sind.
|
||||||
- `prefer: largest` wählt die GPU mit dem meisten VRAM (für LLM-Rollen).
|
- `prefer: largest` wählt die GPU mit dem meisten VRAM (für LLM-Rollen).
|
||||||
- `prefer: smallest_fitting` wählt die kleinste passende GPU (für Embedding und Rerank). So bleiben große Karten frei.
|
- `prefer: smallest_fitting` wählt die kleinste passende GPU (für Embedding und Rerank). So bleiben große Karten frei.
|
||||||
3. Rollen mit `gpu: false` bzw. `cpu_fallback: true` werden ohne passende GPU den CPU-Nodes zugewiesen, sortiert nach RAM.
|
3. Rollen mit `share_with: <rolle>` teilen sich die GPU (bzw. bei `gpu: false` den Node) einer bereits platzierten Rolle. Ein Beispiel ist TTS auf `gpu-flex`.
|
||||||
4. Pro zugewiesener Rolle werden die Katalogmodelle gelistet, die passen (Engine, VRAM, CC).
|
4. Rollen mit `gpu: false` bzw. `cpu_fallback: true` werden ohne passende GPU den CPU-Nodes zugewiesen, sortiert nach RAM.
|
||||||
5. Ergebnis: `assignments[{role, node, gpu_index?, models_fitting[]}]` und `unplaced[{role, reason}]`.
|
5. Pro zugewiesener Rolle werden die Katalogmodelle gelistet, die passen (Engine, VRAM, CC).
|
||||||
|
6. Ergebnis: `assignments[{role, node, gpu_index?, models_fitting[]}]` und `unplaced[{role, reason}]`.
|
||||||
|
|
||||||
Der Plan ist der Input für `platform/proxmox` (welche VM-Rolle auf welchem Host mit welchem Resource Mapping) und für die Registry (welche Slots existieren).
|
Der Plan ist der Input für `platform/proxmox` (welche VM-Rolle auf welchem Host mit welchem Resource Mapping) und für die Registry (welche Slots existieren).
|
||||||
|
|||||||
@@ -14,15 +14,15 @@ nodes:
|
|||||||
memory_gb: 32
|
memory_gb: 32
|
||||||
disks: [{name: sda, size_gb: 128}, {name: sdb, size_gb: 256}]
|
disks: [{name: sda, size_gb: 128}, {name: sdb, size_gb: 256}]
|
||||||
gpus:
|
gpus:
|
||||||
- {index: 0, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2}
|
- {index: 0, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2, disabled: true, note: "außer Betrieb bis llama-bench + Strommessung (ADR-0006)"}
|
||||||
- {index: 1, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2}
|
- {index: 1, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2, disabled: true, note: "außer Betrieb bis llama-bench + Strommessung (ADR-0006)"}
|
||||||
- name: pve2
|
- name: pve2
|
||||||
cpu: {model: "Intel Core i7 (7. Gen) 3.6 GHz", cores: 4, threads: 8}
|
cpu: {model: "Intel Core i7 (7. Gen) 3.6 GHz", cores: 4, threads: 8}
|
||||||
memory_gb: 32
|
memory_gb: 32
|
||||||
disks: [{name: sda, size_gb: 128}, {name: sdb, size_gb: 256}, {name: sdc, size_gb: 256}]
|
disks: [{name: sda, size_gb: 128}, {name: sdb, size_gb: 256}, {name: sdc, size_gb: 256}]
|
||||||
gpus:
|
gpus:
|
||||||
- {index: 0, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2}
|
- {index: 0, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2, disabled: true, note: "außer Betrieb bis llama-bench + Strommessung (ADR-0006)"}
|
||||||
- {index: 1, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2}
|
- {index: 1, name: "Quadro M4000", memory_gb: 8, compute_cap: 5.2, disabled: true, note: "außer Betrieb bis llama-bench + Strommessung (ADR-0006)"}
|
||||||
- name: pve3
|
- name: pve3
|
||||||
cpu: {model: "Intel Core i7 (7. Gen) 3.6 GHz", cores: 4, threads: 8}
|
cpu: {model: "Intel Core i7 (7. Gen) 3.6 GHz", cores: 4, threads: 8}
|
||||||
memory_gb: 32
|
memory_gb: 32
|
||||||
|
|||||||
@@ -80,4 +80,27 @@ slots:
|
|||||||
kind: rerank
|
kind: rerank
|
||||||
ops: {docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"} # VM-IDs/Hosts: ANNAHME bis Deployment
|
ops: {docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"} # VM-IDs/Hosts: ANNAHME bis Deployment
|
||||||
|
|
||||||
|
# --- Sprachausgabe (ADR-0006) ---
|
||||||
|
tts-de:
|
||||||
|
tier: 1
|
||||||
|
device: cpu
|
||||||
|
engine: piper
|
||||||
|
node: infra
|
||||||
|
litellm_model: tts-de
|
||||||
|
base_model: piper-de-thorsten-medium
|
||||||
|
max_context: 5000 # Zeichen pro Anfrage (TTS-Dienst-Limit)
|
||||||
|
kind: tts
|
||||||
|
ops: {docker_host: infra, container: tts-piper, pve_node: pve1, vmid: "101"} # ANNAHME bis Deployment
|
||||||
|
|
||||||
|
tts-de-hq:
|
||||||
|
tier: 1
|
||||||
|
device: gpu
|
||||||
|
engine: pytorch
|
||||||
|
node: gpu-flex # teilt sich die Ada mit tier1-flex; während Training unavailable
|
||||||
|
litellm_model: tts-de-hq
|
||||||
|
base_model: qwen3-tts-1.7b
|
||||||
|
max_context: 5000
|
||||||
|
kind: tts
|
||||||
|
ops: {docker_host: gpu-flex, container: qwen-tts, pve_node: pve-ada, vmid: "202"} # ANNAHME bis Deployment
|
||||||
|
|
||||||
adapters: []
|
adapters: []
|
||||||
|
|||||||
Reference in New Issue
Block a user