deploy: Dockerfile, Compose pro Rolle, LiteLLM, Prometheus/Alertmanager, Canary-Prober, n8n-Workflows

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitignore

@@ -13,3 +13,4 @@ data/
 .terraform/
 *.tfstate*
 .DS_Store
+deploy/compose/*/secrets/

core/src/knaster/api.py

@@ -53,10 +53,14 @@ class GraduateIn(BaseModel):
     min_variants: int = 10
 
 
-def create_app(system: System | None = None) -> FastAPI:
+def create_app(system: System | None = None, start_prober: bool | None = None) -> FastAPI:
+    if start_prober is None:
+        start_prober = system is None  # nur im echten Betrieb, nicht in Tests mit injiziertem System
     system = system or build_system(Settings.from_env())
     app = FastAPI(title="knaster", version="0.1.0")
     app.state.system = system
+    if start_prober and system.settings.canary_interval > 0:
+        system.prober.start(system.settings.canary_interval)
 
     def auth(authorization: str | None = Header(default=None)) -> None:
         token = system.settings.api_token

core/src/knaster/canary.py

@@ -0,0 +1,89 @@
+"""Canary-Prober: prüft Slots regelmäßig mit einer Mini-Anfrage und liefert Metriken für die Healer-Alerts."""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+
+from prometheus_client import Gauge
+
+from .embeddings import Embedder
+from .llm import ChatClient, LLMError
+from .registry import Registry, Slot
+
+OPS_LABELS = ("docker_host", "container", "pve_node", "vmid")
+PROMPT = [{"role": "system", "content": "Antworte nur mit dem Wort OK."}, {"role": "user", "content": "Test"}]
+
+
+@dataclass
+class ProbeResult:
+    slot: str
+    up: bool       # Endpunkt antwortet
+    ok: bool       # Antwort inhaltlich korrekt
+    latency: float
+    error: str = ""
+
+
+class CanaryProber:
+    def __init__(self, registry: Registry, chat: ChatClient, embedder_for: dict[str, Embedder] | None = None,
+                 metrics_registry=None, include_tier0: bool = False):  # noqa: ANN001
+        self.registry = registry
+        self.chat = chat
+        self.embedder_for = embedder_for or {}
+        self.include_tier0 = include_tier0
+        labels = ["slot", "node", *OPS_LABELS]
+        kw = {"registry": metrics_registry} if metrics_registry is not None else {}
+        self.g_up = Gauge("knaster_slot_endpoint_up", "1 = Endpunkt erreichbar", labels, **kw)
+        self.g_ok = Gauge("knaster_slot_canary_ok", "1 = Canary-Antwort korrekt", labels, **kw)
+        self.g_latency = Gauge("knaster_slot_canary_latency_seconds", "Latenz der Canary-Anfrage", labels, **kw)
+
+    def _targets(self) -> list[Slot]:
+        out = []
+        for slot in self.registry.slots():
+            if not slot.spec.canary or slot.spec.kind == "rerank":
+                continue
+            if slot.spec.tier == 0 and not self.include_tier0:
+                continue  # kostet Geld; Erreichbarkeit überwacht LiteLLM
+            out.append(slot)
+        return out
+
+    def probe(self, slot: Slot) -> ProbeResult:
+        start = time.monotonic()
+        try:
+            if slot.spec.kind == "embedding":
+                embedder = self.embedder_for.get(slot.name)
+                if embedder is None:
+                    return ProbeResult(slot.name, True, True, 0.0, "kein Embedder konfiguriert")
+                vec = embedder.embed(["Test"])[0]
+                ok = len(vec) > 0
+            else:
+                res = self.chat.chat(slot.spec.litellm_model, PROMPT, max_tokens=5, temperature=0)
+                ok = "ok" in res.content.lower()
+            return ProbeResult(slot.name, True, ok, time.monotonic() - start)
+        except LLMError as exc:
+            return ProbeResult(slot.name, False, False, time.monotonic() - start, str(exc))
+        except Exception as exc:  # noqa: BLE001 – Netzwerkfehler des Embedders etc.
+            return ProbeResult(slot.name, False, False, time.monotonic() - start, f"{type(exc).__name__}: {exc}")
+
+    def run_once(self) -> list[ProbeResult]:
+        results = []
+        for slot in self._targets():
+            r = self.probe(slot)
+            labels = {"slot": slot.name, "node": slot.spec.node or "",
+                      **{k: slot.spec.ops.get(k, "") for k in OPS_LABELS}}
+            self.g_up.labels(**labels).set(1 if r.up else 0)
+            self.g_ok.labels(**labels).set(1 if r.ok else 0)
+            self.g_latency.labels(**labels).set(r.latency)
+            results.append(r)
+        return results
+
+    def start(self, interval: float) -> threading.Thread:
+        def loop() -> None:
+            while True:
+                self.run_once()
+                time.sleep(interval)
+
+        t = threading.Thread(target=loop, name="canary-prober", daemon=True)
+        t.start()
+        return t

core/src/knaster/healer/drivers.py

@@ -91,12 +91,14 @@ class ProxmoxDriver:
 
 
 class WebhookNotifier:
-    def __init__(self, url: str, client: httpx.Client | None = None):
+    def __init__(self, url: str, auth: str = "", client: httpx.Client | None = None):
         self.url = url
+        self.headers = {"Authorization": auth} if auth else {}
         self.client = client or httpx.Client(timeout=15.0)
 
     def notify(self, text: str, severity: str = "info", data: dict[str, Any] | None = None) -> str:
-        resp = self.client.post(self.url, json={"text": text, "severity": severity, "data": data or {}})
+        resp = self.client.post(self.url, json={"text": text, "severity": severity, "data": data or {}},
+                                headers=self.headers)
         resp.raise_for_status()
         return "benachrichtigt"
 

core/src/knaster/lifecycle.py

@@ -52,14 +52,15 @@ class LoggingJobRunner:
 
 
 class WebhookJobRunner:
-    def __init__(self, url: str, client: Any | None = None):
+    def __init__(self, url: str, auth: str = "", client: Any | None = None):
         import httpx
 
         self.url = url
+        self.headers = {"Authorization": auth} if auth else {}
         self.client = client or httpx.Client(timeout=15.0)
 
     def submit(self, job: str, candidate: dict[str, Any]) -> str:
-        resp = self.client.post(self.url, json={"job": job, "candidate": candidate})
+        resp = self.client.post(self.url, json={"job": job, "candidate": candidate}, headers=self.headers)
         resp.raise_for_status()
         return f"Job {job} übergeben"
 

core/src/knaster/registry.py

@@ -26,6 +26,8 @@ class SlotSpec(BaseModel):
     supports_lora: bool = False
     fallbacks: list[str] = Field(default_factory=list)
     kind: Literal["chat", "embedding", "rerank"] = "chat"
+    canary: bool = True            # vom Prober geprüft (für tier 0 standardmäßig aus, siehe canary.py)
+    ops: dict[str, str] = Field(default_factory=dict)  # docker_host, container, pve_node, vmid → Healer-Labels
 
 
 class AdapterSpec(BaseModel):

core/src/knaster/settings.py

@@ -35,7 +35,9 @@ class Settings:
     proxmox_token_secret: str = ""
     notify_webhook_url: str = ""
     job_webhook_url: str = ""
+    webhook_auth: str = ""  # Wert des Authorization-Headers für ausgehende Webhooks (n8n Header Auth)
     request_timeout: float = 120.0
+    canary_interval: float = 60.0  # 0 = Prober aus
 
     @classmethod
     def from_env(cls) -> Settings:
@@ -60,5 +62,7 @@ class Settings:
             proxmox_token_secret=env.get("PROXMOX_TOKEN_SECRET", ""),
             notify_webhook_url=env.get("NOTIFY_WEBHOOK_URL", ""),
             job_webhook_url=env.get("KNASTER_JOB_WEBHOOK_URL", ""),
+            webhook_auth=env.get("KNASTER_WEBHOOK_AUTH", ""),
             request_timeout=float(env.get("KNASTER_REQUEST_TIMEOUT", base.request_timeout)),
+            canary_interval=float(env.get("KNASTER_CANARY_INTERVAL", base.canary_interval)),
         )

core/src/knaster/system.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 
+from .canary import CanaryProber
 from .classifier import EmbeddingClassifier
 from .embeddings import Embedder, LlamaCppReranker, OpenAIEmbedder, Reranker
 from .executor import Executor
@@ -46,6 +47,7 @@ class System:
     maintenance: MaintenanceAgent
     healer: Healer
     metrics: Metrics
+    prober: CanaryProber
 
 
 def _load_classifiers(directory: Path) -> dict[str, EmbeddingClassifier]:
@@ -95,8 +97,8 @@ def build_system(
     runner = Runner(router, executor, verifier, library, store, metrics)
 
     if jobs is None:
-        jobs = WebhookJobRunner(settings.job_webhook_url) if settings.job_webhook_url and not settings.dry_run \
-            else LoggingJobRunner()
+        use_webhook = settings.job_webhook_url and not settings.dry_run
+        jobs = WebhookJobRunner(settings.job_webhook_url, settings.webhook_auth) if use_webhook else LoggingJobRunner()
     lifecycle = Lifecycle(store, registry, jobs, life_cfg)
 
     if infra is None and not settings.dry_run:
@@ -106,7 +108,7 @@ def build_system(
             if settings.proxmox_url and settings.proxmox_token_id else None
         infra = CompositeInfra(portainer, proxmox)
     if notifier is None and settings.notify_webhook_url:
-        notifier = WebhookNotifier(settings.notify_webhook_url)
+        notifier = WebhookNotifier(settings.notify_webhook_url, settings.webhook_auth)
 
     actions = ActionExecutor(store=store, registry=registry, library=library, lifecycle=lifecycle, infra=infra,
                              notifier=notifier, dry_run=settings.dry_run,
@@ -117,5 +119,7 @@ def build_system(
                                    policy=policy, actions=actions, lifecycle=lifecycle, config=maint_cfg,
                                    intent_mapper=intent_mapper, clock=clock, wallclock=wallclock)
     healer = Healer(healer_cfg, store, actions, policy, propose=maintenance.propose, clock=clock)
+    prober = CanaryProber(registry, chat, embedder_for={settings.embed_model: embedder},
+                          metrics_registry=metrics.registry)
     return System(settings, store, registry, library, embedder, chat, router, executor, verifier, runner, policy,
-                  lifecycle, actions, maintenance, healer, metrics)
+                  lifecycle, actions, maintenance, healer, metrics, prober)

core/tests/test_artifacts.py

@@ -0,0 +1,155 @@
+"""Strukturprüfungen der Deployment-Artefakte (ohne laufende Infrastruktur)."""
+
+import json
+
+import httpx
+import pytest
+import yaml
+
+from knaster.healer.engine import HealerConfig
+from knaster.settings import REPO_ROOT
+
+DEPLOY = REPO_ROOT / "deploy"
+
+
+def test_alert_names_match_healer_rules():
+    alerts = yaml.safe_load((DEPLOY / "prometheus/alerts.yml").read_text())
+    names = {r["alert"] for g in alerts["groups"] for r in g["rules"]}
+    rules = HealerConfig.from_file(REPO_ROOT / "config/healer-rules.yaml").rules
+    matched = {r.match["alertname"] for r in rules}
+    assert names == matched
+
+
+def test_healer_labels_available_on_alerts():
+    """Labels, die Healer-Regeln brauchen, müssen von Metrik oder Scrape-Target kommen."""
+    prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text())
+    jobs = {j["job_name"]: j for j in prom["scrape_configs"]}
+    for job in ("node", "gpu", "vllm"):
+        for sc in jobs[job]["static_configs"]:
+            assert {"slot", "pve_node", "vmid"} <= set(sc["labels"]), (job, sc)
+    from knaster.canary import OPS_LABELS
+    assert set(OPS_LABELS) == {"docker_host", "container", "pve_node", "vmid"}
+
+
+def test_alertmanager_targets_router_endpoint():
+    am = yaml.safe_load((DEPLOY / "alertmanager/alertmanager.yml").read_text())
+    url = am["receivers"][0]["webhook_configs"][0]["url"]
+    assert url.endswith("/v1/healer/alertmanager")
+    assert am["receivers"][0]["webhook_configs"][0]["send_resolved"] is True
+
+
+@pytest.mark.parametrize("role", ["infra", "gpu-vllm", "gpu-llamacpp"])
+def test_compose_files_parse(role):
+    data = yaml.safe_load((DEPLOY / f"compose/{role}/docker-compose.yml").read_text())
+    assert data["services"]
+    assert (DEPLOY / f"compose/{role}/.env.example").exists()
+
+
+def test_ops_labels_consistent_between_registry_and_prometheus():
+    reg = yaml.safe_load((REPO_ROOT / "sites/knaster/registry.yaml").read_text())
+    prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text())
+    vllm = {sc["labels"]["slot"]: sc["labels"] for sc in
+            next(j for j in prom["scrape_configs"] if j["job_name"] == "vllm")["static_configs"]}
+    for slot, labels in vllm.items():
+        ops = reg["slots"][slot]["ops"]
+        assert ops["vmid"] == labels["vmid"] and ops["container"] == labels["container"]
+
+
+WORKFLOWS = sorted((REPO_ROOT / "n8n/workflows").glob("*.json"))
+
+
+@pytest.mark.parametrize("path", WORKFLOWS, ids=[p.name for p in WORKFLOWS])
+def test_n8n_workflow_structure(path):
+    wf = json.loads(path.read_text())
+    assert wf["active"] is False
+    names = [n["name"] for n in wf["nodes"]]
+    assert len(names) == len(set(names))
+    ids = [n["id"] for n in wf["nodes"]]
+    assert len(ids) == len(set(ids))
+    for src, outs in wf["connections"].items():
+        assert src in names
+        for branch in outs["main"]:
+            for target in branch:
+                assert target["node"] in names
+    # jeder Knoten außer dem Trigger ist erreichbar
+    targets = {t["node"] for o in wf["connections"].values() for b in o["main"] for t in b}
+    triggers = [n for n in wf["nodes"] if "trigger" in n["type"].lower() or n["type"].endswith(".webhook")]
+    assert len(triggers) == 1
+    assert set(names) - targets == {triggers[0]["name"]}
+
+
+def test_n8n_router_paths_exist(make_system):
+    from knaster.api import create_app
+
+    paths = {r.path for r in create_app(make_system()).routes}
+    for path in WORKFLOWS:
+        for n in json.loads(path.read_text())["nodes"]:
+            url = n["parameters"].get("url", "")
+            if "router_url" in url:
+                assert url.split("}}")[-1] in paths
+
+
+def test_webhook_notifier_sends_auth_header():
+    from knaster.healer.drivers import WebhookNotifier
+
+    seen = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen["auth"] = request.headers.get("authorization")
+        seen["body"] = json.loads(request.content)
+        return httpx.Response(200)
+
+    n = WebhookNotifier("http://n8n/webhook/knaster/notify", "Bearer t",
+                        client=httpx.Client(transport=httpx.MockTransport(handler)))
+    n.notify("hallo", "high")
+    assert seen == {"auth": "Bearer t", "body": {"text": "hallo", "severity": "high", "data": {}}}
+
+
+def test_drivers_build_correct_requests():
+    from knaster.healer.drivers import PortainerDriver, ProxmoxDriver
+
+    calls = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append((request.method, request.url.path, dict(request.headers)))
+        if request.url.path == "/api/endpoints":
+            return httpx.Response(200, json=[{"Id": 3, "Name": "gpu-main"}])
+        return httpx.Response(200, json={"data": "UPID:1"})
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    PortainerDriver("https://portainer", "k", client=client).restart_container("gpu-main", "vllm")
+    assert calls[-1][:2] == ("POST", "/api/endpoints/3/docker/containers/vllm/restart")
+    assert calls[-1][2]["x-api-key"] == "k"
+    ProxmoxDriver("https://pve:8006", "knaster@pve!healer", "s", client=client).restart_vm("pve-ada", "201")
+    assert calls[-1][:2] == ("POST", "/api2/json/nodes/pve-ada/qemu/201/status/reboot")
+    assert calls[-1][2]["authorization"] == "PVEAPIToken=knaster@pve!healer=s"
+
+
+def test_llm_and_embedding_clients():
+    from knaster.embeddings import LlamaCppReranker, OpenAIEmbedder
+    from knaster.llm import ContextOverflowError, EndpointError, OpenAIChatClient
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        body = json.loads(request.content)
+        if request.url.path == "/v1/embeddings":
+            return httpx.Response(200, json={"data": [{"index": 1, "embedding": [0.0, 1.0]},
+                                                      {"index": 0, "embedding": [1.0, 0.0]}]})
+        if request.url.path == "/v1/rerank":
+            return httpx.Response(200, json={"results": [{"index": 1, "relevance_score": 0.9},
+                                                         {"index": 0, "relevance_score": 0.1}]})
+        if body["model"] == "overflow":
+            return httpx.Response(400, text="This model's maximum context length is 8192 tokens")
+        if body["model"] == "down":
+            return httpx.Response(503, text="unavailable")
+        return httpx.Response(200, json={"model": body["model"], "usage": {"total_tokens": 3},
+                                         "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}]})
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    assert OpenAIEmbedder("http://x", "e", client=client).embed(["a", "b"]) == [[1.0, 0.0], [0.0, 1.0]]
+    assert LlamaCppReranker("http://x", client=client).rerank("q", ["a", "b"]) == [0.1, 0.9]
+    chat = OpenAIChatClient("http://x", "k", client=client)
+    assert chat.chat("m", [{"role": "user", "content": "x"}]).content == "hi"
+    with pytest.raises(ContextOverflowError):
+        chat.chat("overflow", [])
+    with pytest.raises(EndpointError):
+        chat.chat("down", [])

core/tests/test_inventory_validate.py

@@ -88,3 +88,26 @@ def test_cli(capsys):
                  "--catalog", str(REPO_ROOT / "catalog")]) == 0
     out = capsys.readouterr().out
     assert "gpu-main" in out
+
+
+def test_canary_prober(registry, fake_chat):
+    from prometheus_client import CollectorRegistry, generate_latest
+
+    from knaster.canary import CanaryProber
+    from knaster.embeddings import HashingEmbedder
+    from knaster.llm import EndpointError
+
+    def down(m):
+        raise EndpointError("weg")
+
+    fake_chat.handlers.update({"tier1-general": lambda m: "OK", "tier1-flex": down,
+                               "verifier-small": lambda m: "Hallo"})
+    reg = CollectorRegistry()
+    prober = CanaryProber(registry, fake_chat, {"embed-default": HashingEmbedder()}, metrics_registry=reg)
+    results = {r.slot: r for r in prober.run_once()}
+    assert "tier0-strong" not in results
+    assert results["tier1-general"].ok and not results["tier1-flex"].up
+    assert results["verifier-small"].up and not results["verifier-small"].ok
+    assert results["embed-default"].ok
+    text = generate_latest(reg).decode()
+    assert 'knaster_slot_endpoint_up{container="",docker_host="",node="gpu-flex"' in text

deploy/alertmanager/alertmanager.yml

@@ -0,0 +1,18 @@
+# Alertmanager → Router-Service (Healer). Kurzes repeat_interval treibt die Eskalationsleiter voran;
+# Cooldowns pro Stufe regelt der Healer selbst.
+route:
+  receiver: knaster-healer
+  group_by: [alertname, slot, instance]
+  group_wait: 30s
+  group_interval: 1m
+  repeat_interval: 5m
+
+receivers:
+  - name: knaster-healer
+    webhook_configs:
+      - url: http://router:8080/v1/healer/alertmanager
+        send_resolved: true
+        http_config:
+          authorization:
+            type: Bearer
+            credentials_file: /etc/alertmanager/router_token

deploy/compose/gpu-llamacpp/.env.example

@@ -0,0 +1,8 @@
+LLAMACPP_REF=b6000
+CUDA_VERSION=12.4.0
+CUDA_DOCKER_ARCH=52;61
+MODELS_DIR=/srv/models
+EMBED_GGUF=Qwen3-Embedding-0.6B-Q8_0.gguf
+EMBED_POOLING=last
+RERANK_GGUF=bge-reranker-v2-m3-Q8_0.gguf
+CHAT_GGUF=Qwen3-4B-Q4_K_M.gguf

deploy/compose/gpu-llamacpp/docker-compose.yml

@@ -0,0 +1,86 @@
+# Rollen "gpu-embed" und "gpu-small" (VMs mit Quadro P2000 / M4000): llama.cpp-Server.
+# Maxwell (sm_52) und Pascal (sm_61) brauchen CUDA 12.x – daher eigener Build mit festen Architekturen.
+# Profile: embed (Embedding + Rerank), chat (Verifier/Signatur), z. B. `docker compose --profile embed up -d`
+name: knaster-gpu-llamacpp
+
+x-llamacpp: &llamacpp
+  build:
+    context: https://github.com/ggml-org/llama.cpp.git#${LLAMACPP_REF:-b6000}
+    dockerfile: .devops/cuda.Dockerfile
+    target: server
+    args:
+      CUDA_VERSION: ${CUDA_VERSION:-12.4.0}
+      CUDA_DOCKER_ARCH: ${CUDA_DOCKER_ARCH:-52;61}
+  image: knaster/llamacpp-server:${LLAMACPP_REF:-b6000}-cu12
+  restart: unless-stopped
+  labels: {autoheal: "true"}
+  volumes:
+    - ${MODELS_DIR:-/srv/models}:/models:ro
+  deploy:
+    resources:
+      reservations:
+        devices: [{driver: nvidia, count: 1, capabilities: [gpu]}]
+
+services:
+  llamacpp-embed:
+    <<: *llamacpp
+    container_name: llamacpp-embed
+    profiles: ["embed"]
+    command: ["-m", "/models/${EMBED_GGUF:-Qwen3-Embedding-0.6B-Q8_0.gguf}", "--embedding",
+              "--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-ub", "8192", "-ngl", "99",
+              "--alias", "embed-default", "--metrics", "--host", "0.0.0.0", "--port", "8081"]
+    ports: ["8081:8081"]
+    healthcheck: &hc
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 60s
+
+  llamacpp-rerank:
+    <<: *llamacpp
+    container_name: llamacpp-rerank
+    profiles: ["embed"]
+    command: ["-m", "/models/${RERANK_GGUF:-bge-reranker-v2-m3-Q8_0.gguf}", "--reranking", "-c", "8192",
+              "-ub", "8192", "-ngl", "99", "--alias", "rerank-default", "--metrics", "--host", "0.0.0.0",
+              "--port", "8082"]
+    ports: ["8082:8082"]
+    healthcheck:
+      <<: *hc
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8082/health"]
+
+  llamacpp-chat:
+    <<: *llamacpp
+    container_name: llamacpp-chat
+    profiles: ["chat"]
+    command: ["-m", "/models/${CHAT_GGUF:-Qwen3-4B-Q4_K_M.gguf}", "-c", "8192", "-ngl", "99",
+              "--alias", "verifier-small", "--metrics", "--host", "0.0.0.0", "--port", "8080"]
+    ports: ["8080:8080"]
+    healthcheck:
+      <<: *hc
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health"]
+
+  gpu-exporter:
+    image: utkuozdemir/nvidia_gpu_exporter:1.2.1
+    restart: unless-stopped
+    ports: ["9835:9835"]
+    deploy:
+      resources:
+        reservations:
+          devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}]
+
+  node-exporter:
+    image: prom/node-exporter:v1.8.1
+    restart: unless-stopped
+    command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"]
+    pid: host
+    volumes:
+      - /:/host:ro,rslave
+      - /var/lib/node_exporter/textfile:/textfile:ro
+    ports: ["9100:9100"]
+
+  autoheal:
+    image: willfarrell/autoheal:1.2.0
+    restart: unless-stopped
+    environment: {AUTOHEAL_CONTAINER_LABEL: autoheal}
+    volumes: ["/var/run/docker.sock:/var/run/docker.sock"]

deploy/compose/gpu-vllm/.env.example

@@ -0,0 +1,9 @@
+SLOT_ALIAS=tier1-general
+VLLM_MODEL=Qwen/Qwen3-14B-AWQ
+MAX_MODEL_LEN=32768
+GPU_MEM_UTIL=0.90
+MAX_LORAS=4
+MAX_LORA_RANK=64
+MODELS_DIR=/srv/models
+ADAPTERS_DIR=/srv/adapters
+HF_TOKEN=

deploy/compose/gpu-vllm/docker-compose.yml

@@ -0,0 +1,62 @@
+# Rolle "gpu-main" / "gpu-flex" (VM mit durchgereichter RTX 4000 Ada): vLLM mit Multi-LoRA.
+name: knaster-gpu-vllm
+
+services:
+  vllm:
+    image: vllm/vllm-openai:${VLLM_TAG:-v0.10.0}   # Engine-Updates laufen als Lifecycle-Kandidat
+    container_name: vllm
+    restart: unless-stopped
+    labels: {autoheal: "true"}
+    ipc: host
+    env_file: .env
+    environment:
+      VLLM_ALLOW_RUNTIME_LORA_UPDATING: "True"   # Adapter per /v1/load_lora_adapter nachladen
+    command:
+      - --model=${VLLM_MODEL:?VLLM_MODEL fehlt}
+      - --served-model-name=${SLOT_ALIAS:?SLOT_ALIAS fehlt}
+      - --max-model-len=${MAX_MODEL_LEN:-32768}
+      - --gpu-memory-utilization=${GPU_MEM_UTIL:-0.90}
+      - --enable-lora
+      - --max-loras=${MAX_LORAS:-4}
+      - --max-lora-rank=${MAX_LORA_RANK:-64}
+      - --host=0.0.0.0
+      - --port=8000
+    volumes:
+      - ${MODELS_DIR:-/srv/models}:/root/.cache/huggingface
+      - ${ADAPTERS_DIR:-/srv/adapters}:/adapters:ro
+    ports: ["8000:8000"]
+    deploy:
+      resources:
+        reservations:
+          devices: [{driver: nvidia, count: 1, capabilities: [gpu]}]
+    healthcheck:
+      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=4)"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 300s   # Laden großer Modelle
+
+  gpu-exporter:
+    image: utkuozdemir/nvidia_gpu_exporter:1.2.1
+    restart: unless-stopped
+    ports: ["9835:9835"]
+    deploy:
+      resources:
+        reservations:
+          devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}]
+
+  node-exporter:
+    image: prom/node-exporter:v1.8.1
+    restart: unless-stopped
+    command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"]
+    pid: host
+    volumes:
+      - /:/host:ro,rslave
+      - /var/lib/node_exporter/textfile:/textfile:ro
+    ports: ["9100:9100"]
+
+  autoheal:
+    image: willfarrell/autoheal:1.2.0
+    restart: unless-stopped
+    environment: {AUTOHEAL_CONTAINER_LABEL: autoheal, AUTOHEAL_START_PERIOD: "300"}
+    volumes: ["/var/run/docker.sock:/var/run/docker.sock"]

deploy/compose/infra/.env.example

@@ -0,0 +1,18 @@
+KNASTER_VERSION=dev
+KNASTER_SITE=knaster
+KNASTER_API_TOKEN=change-me
+KNASTER_DRY_RUN=true
+KNASTER_CANARY_INTERVAL=60
+POSTGRES_PASSWORD=change-me
+LITELLM_API_KEY=change-me
+LITELLM_MASTER_KEY=change-me
+OPENROUTER_API_KEY=
+RERANK_URL=http://gpu-embed-1:8082
+NOTIFY_WEBHOOK_URL=
+KNASTER_JOB_WEBHOOK_URL=
+PORTAINER_HOST=
+PORTAINER_KEY=
+PROXMOX_URL=
+PROXMOX_TOKEN_ID=
+PROXMOX_TOKEN_SECRET=
+KNASTER_WEBHOOK_AUTH=Bearer change-me

deploy/compose/infra/docker-compose.yml

@@ -0,0 +1,104 @@
+# Rolle "infra" (LXC/VM auf einem i7-Node): Router, LiteLLM, Postgres, Monitoring.
+# Start: cp .env.example .env && docker compose up -d
+name: knaster-infra
+
+x-restart: &restart
+  restart: unless-stopped
+  labels: {autoheal: "true"}
+
+services:
+  router:
+    <<: *restart
+    build:
+      context: ../../..
+      dockerfile: deploy/docker/router.Dockerfile
+    image: knaster/router:${KNASTER_VERSION:-dev}
+    env_file: .env
+    environment:
+      KNASTER_DB_URL: postgresql+psycopg://knaster:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}@postgres:5432/knaster
+      KNASTER_SITE_DIR: /app/sites/${KNASTER_SITE:-knaster}
+      LITELLM_BASE_URL: http://litellm:4000
+    ports: ["8080:8080"]
+    depends_on:
+      postgres: {condition: service_healthy}
+      litellm: {condition: service_started}
+
+  litellm:
+    <<: *restart
+    image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable}   # vor Deployment auf feste Version pinnen
+    command: ["--config", "/app/config.yaml", "--port", "4000"]
+    env_file: .env
+    volumes:
+      - ../../litellm/config.yaml:/app/config.yaml:ro
+    ports: ["4000:4000"]
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=4)"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+
+  postgres:
+    <<: *restart
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_USER: knaster
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}
+      POSTGRES_DB: knaster
+    volumes:
+      - postgres:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U knaster -d knaster"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  prometheus:
+    <<: *restart
+    image: prom/prometheus:v2.53.0
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.retention.time=15d
+      - --web.enable-lifecycle
+    volumes:
+      - ../../prometheus:/etc/prometheus:ro
+      - prometheus:/prometheus
+    ports: ["9090:9090"]
+
+  alertmanager:
+    <<: *restart
+    image: prom/alertmanager:v0.27.0
+    command: ["--config.file=/etc/alertmanager/alertmanager.yml"]
+    volumes:
+      - ../../alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - ./secrets/router_token:/etc/alertmanager/router_token:ro
+    ports: ["9093:9093"]
+
+  grafana:
+    <<: *restart
+    image: grafana/grafana:11.1.0
+    profiles: ["dashboards"]
+    volumes:
+      - grafana:/var/lib/grafana
+    ports: ["3000:3000"]
+
+  pve-exporter:
+    <<: *restart
+    image: prompve/prometheus-pve-exporter:${PVE_EXPORTER_TAG:-latest}   # vor Deployment pinnen
+    profiles: ["proxmox"]
+    volumes:
+      - ./secrets/pve.yml:/etc/prometheus/pve.yml:ro
+    ports: ["9221:9221"]
+
+  autoheal:
+    restart: unless-stopped
+    image: willfarrell/autoheal:1.2.0
+    environment:
+      AUTOHEAL_CONTAINER_LABEL: autoheal
+      AUTOHEAL_INTERVAL: "15"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  postgres: {}
+  prometheus: {}
+  grafana: {}

deploy/docker/router.Dockerfile

@@ -0,0 +1,27 @@
+# Router-Service (knaster). Build-Kontext: Repo-Wurzel.
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
+WORKDIR /app
+
+COPY core/pyproject.toml core/pyproject.toml
+COPY core/src core/src
+RUN pip install "./core[postgres]"
+
+# Portable Konfiguration; standortspezifisches (sites/) kann per Volume überschrieben werden
+COPY config config
+COPY catalog catalog
+COPY skills skills
+COPY sites sites
+
+ENV KNASTER_CONFIG_DIR=/app/config \
+    KNASTER_SKILLS_DIR=/app/skills \
+    KNASTER_SITE_DIR=/app/sites/knaster \
+    KNASTER_DRY_RUN=true
+
+RUN useradd --system --uid 10001 knaster
+USER knaster
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
+  CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=4).status==200 else 1)"
+CMD ["knaster", "serve", "--host", "0.0.0.0", "--port", "8080"]

deploy/prometheus/alerts.yml

@@ -0,0 +1,49 @@
+# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen.
+groups:
+  - name: knaster-slots
+    rules:
+      - alert: SlotEndpointDown
+        expr: knaster_slot_endpoint_up == 0
+        for: 2m
+        labels: {severity: high}
+        annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"}
+
+      - alert: SlotCanaryFailing
+        expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1
+        for: 5m
+        labels: {severity: warning}
+        annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"}
+
+      - alert: VllmKvCacheSaturated
+        expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95
+        for: 10m
+        labels: {severity: warning}
+        annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"}
+
+  - name: knaster-gpu
+    rules:
+      - alert: GpuXidError
+        # aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector)
+        expr: increase(knaster_gpu_xid_errors_total[10m]) > 0
+        labels: {severity: high}
+        annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"}
+
+      - alert: GpuTemperatureHigh
+        expr: nvidia_smi_temperature_gpu > 85
+        for: 5m
+        labels: {severity: warning}
+        annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"}
+
+  - name: knaster-nodes
+    rules:
+      - alert: NodeDown
+        expr: up{job="node"} == 0
+        for: 3m
+        labels: {severity: high}
+        annotations: {summary: "{{ $labels.instance }} nicht erreichbar"}
+
+      - alert: DiskAlmostFull
+        expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10
+        for: 15m
+        labels: {severity: warning}
+        annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"}

deploy/prometheus/prometheus.yml

@@ -0,0 +1,67 @@
+# Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den
+# Alerts und damit beim Healer. Hostnamen = DNS-Namen der VMs (ANNAHME bis Deployment).
+global:
+  scrape_interval: 30s
+  evaluation_interval: 30s
+
+rule_files: [alerts.yml]
+
+alerting:
+  alertmanagers:
+    - static_configs: [{targets: ["alertmanager:9093"]}]
+
+scrape_configs:
+  - job_name: knaster-router
+    metrics_path: /metrics
+    static_configs: [{targets: ["router:8080"]}]
+
+  - job_name: vllm
+    static_configs:
+      - targets: ["gpu-main:8000"]
+        labels: {slot: tier1-general, docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"}
+      - targets: ["gpu-flex:8000"]
+        labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}
+
+  - job_name: llamacpp
+    static_configs:
+      - targets: ["gpu-embed-1:8081"]
+        labels: {slot: embed-default, docker_host: gpu-embed-1, container: llamacpp-embed, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-embed-1:8082"]
+        labels: {slot: rerank-default, docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-small:8080"]
+        labels: {slot: verifier-small, docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"}
+
+  - job_name: gpu
+    static_configs:
+      - targets: ["gpu-main:9835"]
+        labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
+      - targets: ["gpu-flex:9835"]
+        labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
+      - targets: ["gpu-embed-1:9835"]
+        labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-embed-2:9835"]
+        labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
+      - targets: ["gpu-small:9835"]
+        labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
+
+  - job_name: node   # Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht
+    static_configs:
+      - targets: ["gpu-main:9100"]
+        labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"}
+      - targets: ["gpu-flex:9100"]
+        labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"}
+      - targets: ["gpu-embed-1:9100"]
+        labels: {slot: embed-default, pve_node: pve3, vmid: "221"}
+      - targets: ["gpu-embed-2:9100"]
+        labels: {slot: embed-default, pve_node: pve4, vmid: "222"}
+      - targets: ["gpu-small:9100"]
+        labels: {slot: verifier-small, pve_node: pve3, vmid: "231"}
+
+  - job_name: proxmox
+    metrics_path: /pve
+    params: {module: [default]}
+    static_configs: [{targets: ["pve1", "pve2", "pve3", "pve4", "pve-ada"]}]
+    relabel_configs:
+      - {source_labels: [__address__], target_label: __param_target}
+      - {source_labels: [__param_target], target_label: instance}
+      - {target_label: __address__, replacement: "pve-exporter:9221"}

deploy/scripts/gpu-xid-textfile.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Zählt NVIDIA-Xid-Fehler aus dem Kernel-Log und schreibt sie für den node_exporter (textfile collector).
+# Aufruf per systemd-Timer jede Minute (siehe platform/proxmox/ansible/roles/gpu-node).
+set -euo pipefail
+OUT_DIR="${1:-/var/lib/node_exporter/textfile}"
+TMP="$(mktemp "${OUT_DIR}/.xid.XXXXXX")"
+{
+  echo "# HELP knaster_gpu_xid_errors_total NVIDIA Xid-Fehler seit Boot"
+  echo "# TYPE knaster_gpu_xid_errors_total counter"
+  journalctl -k -b --no-pager -o cat 2>/dev/null \
+    | grep -oE 'NVRM: Xid \([^)]*\): [0-9]+' \
+    | awk '{print $NF}' | sort | uniq -c \
+    | awk '{printf "knaster_gpu_xid_errors_total{xid=\"%s\"} %d\n", $2, $1}'
+} > "$TMP"
+chmod 0644 "$TMP"
+mv "$TMP" "${OUT_DIR}/gpu_xid.prom"

n8n/README.md

@@ -0,0 +1,18 @@
+# n8n-Workflows
+
+Importierbar über *Workflows → Import from File*. Alle Workflows werden **inaktiv** importiert.
+
+| Datei | Zweck | Trigger |
+|---|---|---|
+| `knaster-request.json` | Anfrage an den Router (`/v1/run`) und Antwort | Webhook `POST /webhook/knaster/run` `{input, skill?}` |
+| `knaster-maintenance-chat.json` | Pflege-Agent per Prompt (HITL) | n8n Chat |
+| `knaster-service-routine.json` | wöchentliche Service-Routine, Bericht an Kanal | Zeitplan Mo 08:00 |
+| `knaster-notify.json` | Meldungen von Healer/Policy an einen Kanal | Webhook `POST /webhook/knaster/notify` (= `NOTIFY_WEBHOOK_URL`) |
+| `knaster-jobs.json` | Lifecycle-/Skill-Jobs an Trainings-Runner bzw. Hermes | Webhook `POST /webhook/knaster/jobs` (= `KNASTER_JOB_WEBHOOK_URL`) |
+
+Nach dem Import:
+1. Credential **knaster-router** (Header Auth): Name `Authorization`, Wert `Bearer <KNASTER_API_TOKEN>`.
+2. Credential **knaster-webhook** (Header Auth) für eingehende Webhooks. Das gleiche Header-Paar muss im Aufrufer gesetzt werden.
+3. Im Knoten **Konfiguration** `router_url` und ggf. `channel_url`, `trainer_url`, `hermes_url` anpassen.
+
+Die Entscheidungslogik liegt bewusst im Router-Service (ADR-0005). n8n kümmert sich um Trigger, Kanäle und Weiterleitung.

n8n/workflows/knaster-jobs.json

@@ -0,0 +1,249 @@
+{
+  "name": "knaster: Jobs an Runner/Hermes",
+  "nodes": [
+    {
+      "parameters": {
+        "httpMethod": "POST",
+        "path": "knaster/jobs",
+        "responseMode": "responseNode",
+        "options": {},
+        "authentication": "headerAuth"
+      },
+      "id": "24f3f3a3-47ae-5b87-8902-bcc3465b593a",
+      "name": "Job",
+      "type": "n8n-nodes-base.webhook",
+      "typeVersion": 2,
+      "position": [
+        0,
+        300
+      ],
+      "webhookId": "03e1babe-7daa-5ab1-ba6e-8cc4de50268c",
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-webhook"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "b3b14ae1-2d7f-5ef9-a96f-1d3693c703f6",
+              "name": "router_url",
+              "value": "http://router:8080",
+              "type": "string"
+            },
+            {
+              "id": "a3123e42-afa4-53f7-a266-066cce1ff763",
+              "name": "trainer_url",
+              "value": "http://gpu-flex:9000/jobs",
+              "type": "string"
+            },
+            {
+              "id": "c672b90a-49d5-5e50-9b23-3766353f0487",
+              "name": "hermes_url",
+              "value": "http://hermes:8000/jobs",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "acb2e1cb-0776-564b-b909-b5b3f613dc88",
+      "name": "Konfiguration",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.4,
+      "position": [
+        220,
+        300
+      ],
+      "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer <KNASTER_API_TOKEN>"
+    },
+    {
+      "parameters": {
+        "conditions": {
+          "options": {
+            "caseSensitive": true,
+            "leftValue": "",
+            "typeValidation": "loose"
+          },
+          "conditions": [
+            {
+              "id": "9dec25ae-5535-5f01-aebb-b9d7e7397686",
+              "leftValue": "={{ ['graduate_skill','merge_skills'].includes($('Job').item.json.body.job) }}",
+              "rightValue": true,
+              "operator": {
+                "type": "boolean",
+                "operation": "true"
+              }
+            }
+          ],
+          "combinator": "and"
+        },
+        "options": {}
+      },
+      "id": "a2881a83-cbb5-5cdb-96bb-d6a1833ac388",
+      "name": "Skill-Job?",
+      "type": "n8n-nodes-base.if",
+      "typeVersion": 2,
+      "position": [
+        440,
+        300
+      ]
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "={{ $('Konfiguration').item.json.hermes_url }}",
+        "authentication": "genericCredentialType",
+        "genericAuthType": "httpHeaderAuth",
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={{ JSON.stringify($('Job').item.json.body) }}",
+        "options": {
+          "timeout": 30000
+        }
+      },
+      "id": "ba36e0ed-7cca-5c04-af60-9644177cac6f",
+      "name": "An Hermes (Skill-Autor)",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        660,
+        200
+      ],
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-router"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "={{ $('Konfiguration').item.json.trainer_url }}",
+        "authentication": "genericCredentialType",
+        "genericAuthType": "httpHeaderAuth",
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={{ JSON.stringify($('Job').item.json.body) }}",
+        "options": {
+          "timeout": 30000
+        }
+      },
+      "id": "98abbd0d-1384-52bb-b095-cb978f2bcc37",
+      "name": "An Trainings-Runner",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        660,
+        420
+      ],
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-router"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "respondWith": "json",
+        "responseBody": "={{ JSON.stringify({ accepted: true }) }}",
+        "options": {
+          "responseCode": 202
+        }
+      },
+      "id": "6678356f-6c70-58fe-a9e0-be3d7cf23851",
+      "name": "Angenommen",
+      "type": "n8n-nodes-base.respondToWebhook",
+      "typeVersion": 1.1,
+      "position": [
+        880,
+        300
+      ]
+    }
+  ],
+  "connections": {
+    "Job": {
+      "main": [
+        [
+          {
+            "node": "Konfiguration",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Konfiguration": {
+      "main": [
+        [
+          {
+            "node": "Skill-Job?",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Skill-Job?": {
+      "main": [
+        [
+          {
+            "node": "An Hermes (Skill-Autor)",
+            "type": "main",
+            "index": 0
+          }
+        ],
+        [
+          {
+            "node": "An Trainings-Runner",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "An Hermes (Skill-Autor)": {
+      "main": [
+        [
+          {
+            "node": "Angenommen",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "An Trainings-Runner": {
+      "main": [
+        [
+          {
+            "node": "Angenommen",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    }
+  },
+  "active": false,
+  "settings": {
+    "executionOrder": "v1"
+  },
+  "tags": [
+    {
+      "name": "knaster"
+    },
+    {
+      "name": "lifecycle"
+    }
+  ],
+  "meta": {
+    "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen."
+  }
+}

n8n/workflows/knaster-maintenance-chat.json

@@ -0,0 +1,147 @@
+{
+  "name": "knaster: Pflege-Agent (Chat)",
+  "nodes": [
+    {
+      "parameters": {
+        "public": false,
+        "options": {
+          "responseMode": "lastNode"
+        }
+      },
+      "id": "ef50a82c-c64a-5a2e-b380-f40ae2a7f738",
+      "name": "Chat",
+      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
+      "typeVersion": 1.1,
+      "position": [
+        0,
+        300
+      ],
+      "webhookId": "f819ed24-d095-5120-b2f2-e996135967e1"
+    },
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "fecd0058-75b7-569f-8970-e4647b315631",
+              "name": "router_url",
+              "value": "http://router:8080",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "a08a0680-3d4b-5845-8693-86627a2aaeb4",
+      "name": "Konfiguration",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.4,
+      "position": [
+        220,
+        300
+      ],
+      "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer <KNASTER_API_TOKEN>"
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "={{ $('Konfiguration').item.json.router_url }}/v1/maintenance/command",
+        "authentication": "genericCredentialType",
+        "genericAuthType": "httpHeaderAuth",
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={{ JSON.stringify({ text: $('Chat').item.json.chatInput, session: $('Chat').item.json.sessionId, user: 'n8n-chat' }) }}",
+        "options": {
+          "timeout": 60000
+        }
+      },
+      "id": "4fc64c5d-b22d-5168-a4bb-d56a3c4e1001",
+      "name": "Befehl an Pflege-Agent",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        440,
+        300
+      ],
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-router"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "4d75dea4-a1de-51e1-b324-06282f454d47",
+              "name": "output",
+              "value": "={{ $json.text }}",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "ca5fef45-e097-52a6-af88-8d9a6c8d1461",
+      "name": "Antwort",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.4,
+      "position": [
+        660,
+        300
+      ]
+    }
+  ],
+  "connections": {
+    "Chat": {
+      "main": [
+        [
+          {
+            "node": "Konfiguration",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Konfiguration": {
+      "main": [
+        [
+          {
+            "node": "Befehl an Pflege-Agent",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Befehl an Pflege-Agent": {
+      "main": [
+        [
+          {
+            "node": "Antwort",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    }
+  },
+  "active": false,
+  "settings": {
+    "executionOrder": "v1"
+  },
+  "tags": [
+    {
+      "name": "knaster"
+    },
+    {
+      "name": "hitl"
+    }
+  ],
+  "meta": {
+    "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen."
+  }
+}

n8n/workflows/knaster-notify.json

@@ -0,0 +1,141 @@
+{
+  "name": "knaster: Benachrichtigungen",
+  "nodes": [
+    {
+      "parameters": {
+        "httpMethod": "POST",
+        "path": "knaster/notify",
+        "responseMode": "responseNode",
+        "options": {},
+        "authentication": "headerAuth"
+      },
+      "id": "e324dc52-1df2-54f4-916f-540c3033562a",
+      "name": "Healer-Meldung",
+      "type": "n8n-nodes-base.webhook",
+      "typeVersion": 2,
+      "position": [
+        0,
+        300
+      ],
+      "webhookId": "978b850a-ca87-5dca-a16e-b737df5acad2",
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-webhook"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "904e09c8-286a-5272-8b3f-4c6c49899e9d",
+              "name": "router_url",
+              "value": "http://router:8080",
+              "type": "string"
+            },
+            {
+              "id": "fe09313b-633c-58b6-8e80-ef4db981118f",
+              "name": "channel_url",
+              "value": "https://example.invalid/kanal-webhook",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "7868766e-ff24-5694-9338-f7b0c3aa6603",
+      "name": "Konfiguration",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.4,
+      "position": [
+        220,
+        300
+      ],
+      "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer <KNASTER_API_TOKEN>"
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "={{ $('Konfiguration').item.json.channel_url }}",
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={{ JSON.stringify({ text: '[' + $('Healer-Meldung').item.json.body.severity + '] ' + $('Healer-Meldung').item.json.body.text }) }}",
+        "options": {}
+      },
+      "id": "28c5f6c3-74a0-5b70-8225-2196f1b9164d",
+      "name": "An Kanal senden",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        440,
+        300
+      ]
+    },
+    {
+      "parameters": {
+        "respondWith": "json",
+        "responseBody": "={{ JSON.stringify({ ok: true }) }}",
+        "options": {
+          "responseCode": 200
+        }
+      },
+      "id": "3518decf-c8b3-5d29-b88b-b59a0980d173",
+      "name": "OK",
+      "type": "n8n-nodes-base.respondToWebhook",
+      "typeVersion": 1.1,
+      "position": [
+        660,
+        300
+      ]
+    }
+  ],
+  "connections": {
+    "Healer-Meldung": {
+      "main": [
+        [
+          {
+            "node": "Konfiguration",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Konfiguration": {
+      "main": [
+        [
+          {
+            "node": "An Kanal senden",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "An Kanal senden": {
+      "main": [
+        [
+          {
+            "node": "OK",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    }
+  },
+  "active": false,
+  "settings": {
+    "executionOrder": "v1"
+  },
+  "tags": [
+    {
+      "name": "knaster"
+    }
+  ],
+  "meta": {
+    "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen."
+  }
+}

n8n/workflows/knaster-request.json

@@ -0,0 +1,212 @@
+{
+  "name": "knaster: Anfrage ausführen",
+  "nodes": [
+    {
+      "parameters": {
+        "httpMethod": "POST",
+        "path": "knaster/run",
+        "responseMode": "responseNode",
+        "options": {},
+        "authentication": "headerAuth"
+      },
+      "id": "9463206f-d1cb-52fc-98ba-01370b80290b",
+      "name": "Anfrage",
+      "type": "n8n-nodes-base.webhook",
+      "typeVersion": 2,
+      "position": [
+        0,
+        300
+      ],
+      "webhookId": "bba18b52-dcd1-5231-a132-0434e5f27505",
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-webhook"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "b65c35b7-cf8b-5067-8c3c-d90533ad04dd",
+              "name": "router_url",
+              "value": "http://router:8080",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "4ccf1d69-7b78-5ad7-9d1a-40ae77e7b3d1",
+      "name": "Konfiguration",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.4,
+      "position": [
+        220,
+        300
+      ],
+      "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer <KNASTER_API_TOKEN>"
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "={{ $('Konfiguration').item.json.router_url }}/v1/run",
+        "authentication": "genericCredentialType",
+        "genericAuthType": "httpHeaderAuth",
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={{ JSON.stringify({ input: $('Anfrage').item.json.body.input, skill: $('Anfrage').item.json.body.skill || null }) }}",
+        "options": {
+          "timeout": 180000
+        }
+      },
+      "id": "d6f5c10d-2a30-598a-af5e-3f8040098cb4",
+      "name": "Router /v1/run",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        440,
+        300
+      ],
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-router"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "conditions": {
+          "options": {
+            "caseSensitive": true,
+            "leftValue": "",
+            "typeValidation": "loose"
+          },
+          "conditions": [
+            {
+              "id": "3fc7e0c8-fd00-5368-819c-dff9572baf12",
+              "leftValue": "={{ $json.status }}",
+              "rightValue": "ok",
+              "operator": {
+                "type": "string",
+                "operation": "equals"
+              }
+            }
+          ],
+          "combinator": "and"
+        },
+        "options": {}
+      },
+      "id": "01605cd8-f4e0-53c6-acfe-32c389d6cbe6",
+      "name": "Erfolgreich?",
+      "type": "n8n-nodes-base.if",
+      "typeVersion": 2,
+      "position": [
+        660,
+        300
+      ]
+    },
+    {
+      "parameters": {
+        "respondWith": "json",
+        "responseBody": "={{ JSON.stringify({ output: $json.output, stage: $json.stage, skill: $json.skill, escalations: $json.escalations, run_id: $json.run_id }) }}",
+        "options": {
+          "responseCode": 200
+        }
+      },
+      "id": "9ca6279a-dfb6-525e-aa18-8385bed2f448",
+      "name": "Antwort",
+      "type": "n8n-nodes-base.respondToWebhook",
+      "typeVersion": 1.1,
+      "position": [
+        880,
+        200
+      ]
+    },
+    {
+      "parameters": {
+        "respondWith": "json",
+        "responseBody": "={{ JSON.stringify({ status: $json.status, run_id: $json.run_id, reasons: $json.attempts[$json.attempts.length-1].decision.reasons }) }}",
+        "options": {
+          "responseCode": 422
+        }
+      },
+      "id": "8728502e-e051-5259-a2bd-57646e3b1c5e",
+      "name": "Fehler",
+      "type": "n8n-nodes-base.respondToWebhook",
+      "typeVersion": 1.1,
+      "position": [
+        880,
+        420
+      ]
+    }
+  ],
+  "connections": {
+    "Anfrage": {
+      "main": [
+        [
+          {
+            "node": "Konfiguration",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Konfiguration": {
+      "main": [
+        [
+          {
+            "node": "Router /v1/run",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Router /v1/run": {
+      "main": [
+        [
+          {
+            "node": "Erfolgreich?",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Erfolgreich?": {
+      "main": [
+        [
+          {
+            "node": "Antwort",
+            "type": "main",
+            "index": 0
+          }
+        ],
+        [
+          {
+            "node": "Fehler",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    }
+  },
+  "active": false,
+  "settings": {
+    "executionOrder": "v1"
+  },
+  "tags": [
+    {
+      "name": "knaster"
+    }
+  ],
+  "meta": {
+    "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen."
+  }
+}

n8n/workflows/knaster-service-routine.json

@@ -0,0 +1,199 @@
+{
+  "name": "knaster: Service-Routine (wöchentlich)",
+  "nodes": [
+    {
+      "parameters": {
+        "rule": {
+          "interval": [
+            {
+              "field": "weeks",
+              "weeksInterval": 1,
+              "triggerAtDay": [
+                1
+              ],
+              "triggerAtHour": 8
+            }
+          ]
+        }
+      },
+      "id": "c58239bb-f1c9-557c-952c-21645fcb8a57",
+      "name": "Zeitplan",
+      "type": "n8n-nodes-base.scheduleTrigger",
+      "typeVersion": 1.2,
+      "position": [
+        0,
+        300
+      ]
+    },
+    {
+      "parameters": {
+        "assignments": {
+          "assignments": [
+            {
+              "id": "df17a04d-3369-5ef6-83b2-150c90db08fa",
+              "name": "router_url",
+              "value": "http://router:8080",
+              "type": "string"
+            },
+            {
+              "id": "fb597b99-b6c3-5306-b5fa-68ffc02938d3",
+              "name": "channel_url",
+              "value": "https://example.invalid/kanal-webhook",
+              "type": "string"
+            }
+          ]
+        },
+        "options": {}
+      },
+      "id": "dc94a1c9-74b6-5403-942a-59a84e634a59",
+      "name": "Konfiguration",
+      "type": "n8n-nodes-base.set",
+      "typeVersion": 3.4,
+      "position": [
+        220,
+        300
+      ],
+      "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer <KNASTER_API_TOKEN>"
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "={{ $('Konfiguration').item.json.router_url }}/v1/maintenance/routine",
+        "authentication": "genericCredentialType",
+        "genericAuthType": "httpHeaderAuth",
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={{ JSON.stringify({}) }}",
+        "options": {
+          "timeout": 180000
+        }
+      },
+      "id": "ba3983aa-c877-5686-8297-60b9be29bdb3",
+      "name": "Routine starten",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        440,
+        300
+      ],
+      "credentials": {
+        "httpHeaderAuth": {
+          "id": "",
+          "name": "knaster-router"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "conditions": {
+          "options": {
+            "caseSensitive": true,
+            "leftValue": "",
+            "typeValidation": "loose"
+          },
+          "conditions": [
+            {
+              "id": "f74160f5-984f-5078-9fc3-5557cdc71e5f",
+              "leftValue": "={{ $json.findings.length + $json.proposals.length + $json.auto_executed.length }}",
+              "rightValue": 0,
+              "operator": {
+                "type": "number",
+                "operation": "gt"
+              }
+            }
+          ],
+          "combinator": "and"
+        },
+        "options": {}
+      },
+      "id": "07062ef5-09ab-5043-83d6-7d4806fe945a",
+      "name": "Etwas zu tun?",
+      "type": "n8n-nodes-base.if",
+      "typeVersion": 2,
+      "position": [
+        660,
+        300
+      ]
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "={{ $('Konfiguration').item.json.channel_url }}",
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={{ JSON.stringify({ text: $json.text }) }}",
+        "options": {}
+      },
+      "id": "f6ec4fd3-03a4-5226-beec-ac4f11c0fb30",
+      "name": "An Kanal senden",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        880,
+        200
+      ],
+      "notes": "Kanal (Telegram/Matrix/Mail) hier anbinden. Antworten laufen über den Pflege-Chat."
+    }
+  ],
+  "connections": {
+    "Zeitplan": {
+      "main": [
+        [
+          {
+            "node": "Konfiguration",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Konfiguration": {
+      "main": [
+        [
+          {
+            "node": "Routine starten",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Routine starten": {
+      "main": [
+        [
+          {
+            "node": "Etwas zu tun?",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Etwas zu tun?": {
+      "main": [
+        [
+          {
+            "node": "An Kanal senden",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    }
+  },
+  "active": false,
+  "settings": {
+    "executionOrder": "v1"
+  },
+  "tags": [
+    {
+      "name": "knaster"
+    },
+    {
+      "name": "hitl"
+    }
+  ],
+  "meta": {
+    "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen."
+  }
+}

sites/knaster/registry.yaml

@@ -1,5 +1,6 @@
 # Registry für den Cluster "knaster" (SPEC-core §1).
 # Clients sprechen nur Slot-Namen an; litellm_model ist der Alias in deploy/litellm/config.yaml.
+# ops: Labels für den Healer (Portainer-Endpoint, Container, Proxmox-Node, VM-ID).
 slots:
   tier0-strong:
     tier: 0
@@ -21,6 +22,7 @@ slots:
     effective_context: 16384
     supports_lora: true
     fallbacks: [tier1-flex]
+    ops: {docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"}   # VM-IDs/Hosts: ANNAHME bis Deployment
 
   tier1-flex:
     tier: 1
@@ -32,6 +34,7 @@ slots:
     max_context: 32768
     effective_context: 16384
     supports_lora: true
+    ops: {docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"}   # VM-IDs/Hosts: ANNAHME bis Deployment
 
   verifier-small:
     tier: 1
@@ -42,6 +45,7 @@ slots:
     base_model: qwen3-4b-q4km
     max_context: 8192
     effective_context: 6144
+    ops: {docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"}   # VM-IDs/Hosts: ANNAHME bis Deployment
 
   signature-extractor:
     tier: 1
@@ -52,6 +56,7 @@ slots:
     base_model: qwen3-4b-q4km
     max_context: 8192
     effective_context: 6144
+    ops: {docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"}   # VM-IDs/Hosts: ANNAHME bis Deployment
 
   embed-default:
     tier: 1
@@ -62,6 +67,7 @@ slots:
     base_model: qwen3-embedding-0.6b-q8
     max_context: 8192
     kind: embedding
+    ops: {docker_host: gpu-embed-1, container: llamacpp-embed, pve_node: pve3, vmid: "221"}   # VM-IDs/Hosts: ANNAHME bis Deployment
 
   rerank-default:
     tier: 1
@@ -72,5 +78,6 @@ slots:
     base_model: bge-reranker-v2-m3-q8
     max_context: 8192
     kind: rerank
+    ops: {docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"}   # VM-IDs/Hosts: ANNAHME bis Deployment
 
 adapters: []