From cb5a41316904033f69ed8608ec2f5925f9181bac Mon Sep 17 00:00:00 2001 From: Robert Rapp Date: Mon, 12 Oct 2026 00:51:31 +0200 Subject: [PATCH] deploy: Dockerfile, Compose pro Rolle, LiteLLM, Prometheus/Alertmanager, Canary-Prober, n8n-Workflows Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + core/src/knaster/api.py | 6 +- core/src/knaster/canary.py | 89 +++++++ core/src/knaster/healer/drivers.py | 6 +- core/src/knaster/lifecycle.py | 5 +- core/src/knaster/registry.py | 2 + core/src/knaster/settings.py | 4 + core/src/knaster/system.py | 12 +- core/tests/test_artifacts.py | 155 +++++++++++ core/tests/test_inventory_validate.py | 23 ++ deploy/alertmanager/alertmanager.yml | 18 ++ deploy/compose/gpu-llamacpp/.env.example | 8 + .../compose/gpu-llamacpp/docker-compose.yml | 86 ++++++ deploy/compose/gpu-vllm/.env.example | 9 + deploy/compose/gpu-vllm/docker-compose.yml | 62 +++++ deploy/compose/infra/.env.example | 18 ++ deploy/compose/infra/docker-compose.yml | 104 ++++++++ deploy/docker/router.Dockerfile | 27 ++ deploy/prometheus/alerts.yml | 49 ++++ deploy/prometheus/prometheus.yml | 67 +++++ deploy/scripts/gpu-xid-textfile.sh | 16 ++ n8n/README.md | 18 ++ n8n/workflows/knaster-jobs.json | 249 ++++++++++++++++++ n8n/workflows/knaster-maintenance-chat.json | 147 +++++++++++ n8n/workflows/knaster-notify.json | 141 ++++++++++ n8n/workflows/knaster-request.json | 212 +++++++++++++++ n8n/workflows/knaster-service-routine.json | 199 ++++++++++++++ sites/knaster/registry.yaml | 7 + 28 files changed, 1731 insertions(+), 9 deletions(-) create mode 100644 core/src/knaster/canary.py create mode 100644 core/tests/test_artifacts.py create mode 100644 deploy/alertmanager/alertmanager.yml create mode 100644 deploy/compose/gpu-llamacpp/.env.example create mode 100644 deploy/compose/gpu-llamacpp/docker-compose.yml create mode 100644 deploy/compose/gpu-vllm/.env.example create mode 100644 deploy/compose/gpu-vllm/docker-compose.yml create mode 100644 deploy/compose/infra/.env.example create mode 100644 deploy/compose/infra/docker-compose.yml create mode 100644 deploy/docker/router.Dockerfile create mode 100644 deploy/prometheus/alerts.yml create mode 100644 deploy/prometheus/prometheus.yml create mode 100755 deploy/scripts/gpu-xid-textfile.sh create mode 100644 n8n/README.md create mode 100644 n8n/workflows/knaster-jobs.json create mode 100644 n8n/workflows/knaster-maintenance-chat.json create mode 100644 n8n/workflows/knaster-notify.json create mode 100644 n8n/workflows/knaster-request.json create mode 100644 n8n/workflows/knaster-service-routine.json diff --git a/.gitignore b/.gitignore index 19cc47a..6dff6f1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ data/ .terraform/ *.tfstate* .DS_Store +deploy/compose/*/secrets/ diff --git a/core/src/knaster/api.py b/core/src/knaster/api.py index 189812e..cadc285 100644 --- a/core/src/knaster/api.py +++ b/core/src/knaster/api.py @@ -53,10 +53,14 @@ class GraduateIn(BaseModel): min_variants: int = 10 -def create_app(system: System | None = None) -> FastAPI: +def create_app(system: System | None = None, start_prober: bool | None = None) -> FastAPI: + if start_prober is None: + start_prober = system is None # nur im echten Betrieb, nicht in Tests mit injiziertem System system = system or build_system(Settings.from_env()) app = FastAPI(title="knaster", version="0.1.0") app.state.system = system + if start_prober and system.settings.canary_interval > 0: + system.prober.start(system.settings.canary_interval) def auth(authorization: str | None = Header(default=None)) -> None: token = system.settings.api_token diff --git a/core/src/knaster/canary.py b/core/src/knaster/canary.py new file mode 100644 index 0000000..d100baa --- /dev/null +++ b/core/src/knaster/canary.py @@ -0,0 +1,89 @@ +"""Canary-Prober: prüft Slots regelmäßig mit einer Mini-Anfrage und liefert Metriken für die Healer-Alerts.""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass + +from prometheus_client import Gauge + +from .embeddings import Embedder +from .llm import ChatClient, LLMError +from .registry import Registry, Slot + +OPS_LABELS = ("docker_host", "container", "pve_node", "vmid") +PROMPT = [{"role": "system", "content": "Antworte nur mit dem Wort OK."}, {"role": "user", "content": "Test"}] + + +@dataclass +class ProbeResult: + slot: str + up: bool # Endpunkt antwortet + ok: bool # Antwort inhaltlich korrekt + latency: float + error: str = "" + + +class CanaryProber: + def __init__(self, registry: Registry, chat: ChatClient, embedder_for: dict[str, Embedder] | None = None, + metrics_registry=None, include_tier0: bool = False): # noqa: ANN001 + self.registry = registry + self.chat = chat + self.embedder_for = embedder_for or {} + self.include_tier0 = include_tier0 + labels = ["slot", "node", *OPS_LABELS] + kw = {"registry": metrics_registry} if metrics_registry is not None else {} + self.g_up = Gauge("knaster_slot_endpoint_up", "1 = Endpunkt erreichbar", labels, **kw) + self.g_ok = Gauge("knaster_slot_canary_ok", "1 = Canary-Antwort korrekt", labels, **kw) + self.g_latency = Gauge("knaster_slot_canary_latency_seconds", "Latenz der Canary-Anfrage", labels, **kw) + + def _targets(self) -> list[Slot]: + out = [] + for slot in self.registry.slots(): + if not slot.spec.canary or slot.spec.kind == "rerank": + continue + if slot.spec.tier == 0 and not self.include_tier0: + continue # kostet Geld; Erreichbarkeit überwacht LiteLLM + out.append(slot) + return out + + def probe(self, slot: Slot) -> ProbeResult: + start = time.monotonic() + try: + if slot.spec.kind == "embedding": + embedder = self.embedder_for.get(slot.name) + if embedder is None: + return ProbeResult(slot.name, True, True, 0.0, "kein Embedder konfiguriert") + vec = embedder.embed(["Test"])[0] + ok = len(vec) > 0 + else: + res = self.chat.chat(slot.spec.litellm_model, PROMPT, max_tokens=5, temperature=0) + ok = "ok" in res.content.lower() + return ProbeResult(slot.name, True, ok, time.monotonic() - start) + except LLMError as exc: + return ProbeResult(slot.name, False, False, time.monotonic() - start, str(exc)) + except Exception as exc: # noqa: BLE001 – Netzwerkfehler des Embedders etc. + return ProbeResult(slot.name, False, False, time.monotonic() - start, f"{type(exc).__name__}: {exc}") + + def run_once(self) -> list[ProbeResult]: + results = [] + for slot in self._targets(): + r = self.probe(slot) + labels = {"slot": slot.name, "node": slot.spec.node or "", + **{k: slot.spec.ops.get(k, "") for k in OPS_LABELS}} + self.g_up.labels(**labels).set(1 if r.up else 0) + self.g_ok.labels(**labels).set(1 if r.ok else 0) + self.g_latency.labels(**labels).set(r.latency) + results.append(r) + return results + + def start(self, interval: float) -> threading.Thread: + def loop() -> None: + while True: + self.run_once() + time.sleep(interval) + + t = threading.Thread(target=loop, name="canary-prober", daemon=True) + t.start() + return t diff --git a/core/src/knaster/healer/drivers.py b/core/src/knaster/healer/drivers.py index 284d822..37b0c09 100644 --- a/core/src/knaster/healer/drivers.py +++ b/core/src/knaster/healer/drivers.py @@ -91,12 +91,14 @@ class ProxmoxDriver: class WebhookNotifier: - def __init__(self, url: str, client: httpx.Client | None = None): + def __init__(self, url: str, auth: str = "", client: httpx.Client | None = None): self.url = url + self.headers = {"Authorization": auth} if auth else {} self.client = client or httpx.Client(timeout=15.0) def notify(self, text: str, severity: str = "info", data: dict[str, Any] | None = None) -> str: - resp = self.client.post(self.url, json={"text": text, "severity": severity, "data": data or {}}) + resp = self.client.post(self.url, json={"text": text, "severity": severity, "data": data or {}}, + headers=self.headers) resp.raise_for_status() return "benachrichtigt" diff --git a/core/src/knaster/lifecycle.py b/core/src/knaster/lifecycle.py index c86c3f2..54b9667 100644 --- a/core/src/knaster/lifecycle.py +++ b/core/src/knaster/lifecycle.py @@ -52,14 +52,15 @@ class LoggingJobRunner: class WebhookJobRunner: - def __init__(self, url: str, client: Any | None = None): + def __init__(self, url: str, auth: str = "", client: Any | None = None): import httpx self.url = url + self.headers = {"Authorization": auth} if auth else {} self.client = client or httpx.Client(timeout=15.0) def submit(self, job: str, candidate: dict[str, Any]) -> str: - resp = self.client.post(self.url, json={"job": job, "candidate": candidate}) + resp = self.client.post(self.url, json={"job": job, "candidate": candidate}, headers=self.headers) resp.raise_for_status() return f"Job {job} übergeben" diff --git a/core/src/knaster/registry.py b/core/src/knaster/registry.py index d83e8db..b33270e 100644 --- a/core/src/knaster/registry.py +++ b/core/src/knaster/registry.py @@ -26,6 +26,8 @@ class SlotSpec(BaseModel): supports_lora: bool = False fallbacks: list[str] = Field(default_factory=list) kind: Literal["chat", "embedding", "rerank"] = "chat" + canary: bool = True # vom Prober geprüft (für tier 0 standardmäßig aus, siehe canary.py) + ops: dict[str, str] = Field(default_factory=dict) # docker_host, container, pve_node, vmid → Healer-Labels class AdapterSpec(BaseModel): diff --git a/core/src/knaster/settings.py b/core/src/knaster/settings.py index 4cebd55..02cbc35 100644 --- a/core/src/knaster/settings.py +++ b/core/src/knaster/settings.py @@ -35,7 +35,9 @@ class Settings: proxmox_token_secret: str = "" notify_webhook_url: str = "" job_webhook_url: str = "" + webhook_auth: str = "" # Wert des Authorization-Headers für ausgehende Webhooks (n8n Header Auth) request_timeout: float = 120.0 + canary_interval: float = 60.0 # 0 = Prober aus @classmethod def from_env(cls) -> Settings: @@ -60,5 +62,7 @@ class Settings: proxmox_token_secret=env.get("PROXMOX_TOKEN_SECRET", ""), notify_webhook_url=env.get("NOTIFY_WEBHOOK_URL", ""), job_webhook_url=env.get("KNASTER_JOB_WEBHOOK_URL", ""), + webhook_auth=env.get("KNASTER_WEBHOOK_AUTH", ""), request_timeout=float(env.get("KNASTER_REQUEST_TIMEOUT", base.request_timeout)), + canary_interval=float(env.get("KNASTER_CANARY_INTERVAL", base.canary_interval)), ) diff --git a/core/src/knaster/system.py b/core/src/knaster/system.py index 728e5dd..14178b7 100644 --- a/core/src/knaster/system.py +++ b/core/src/knaster/system.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path +from .canary import CanaryProber from .classifier import EmbeddingClassifier from .embeddings import Embedder, LlamaCppReranker, OpenAIEmbedder, Reranker from .executor import Executor @@ -46,6 +47,7 @@ class System: maintenance: MaintenanceAgent healer: Healer metrics: Metrics + prober: CanaryProber def _load_classifiers(directory: Path) -> dict[str, EmbeddingClassifier]: @@ -95,8 +97,8 @@ def build_system( runner = Runner(router, executor, verifier, library, store, metrics) if jobs is None: - jobs = WebhookJobRunner(settings.job_webhook_url) if settings.job_webhook_url and not settings.dry_run \ - else LoggingJobRunner() + use_webhook = settings.job_webhook_url and not settings.dry_run + jobs = WebhookJobRunner(settings.job_webhook_url, settings.webhook_auth) if use_webhook else LoggingJobRunner() lifecycle = Lifecycle(store, registry, jobs, life_cfg) if infra is None and not settings.dry_run: @@ -106,7 +108,7 @@ def build_system( if settings.proxmox_url and settings.proxmox_token_id else None infra = CompositeInfra(portainer, proxmox) if notifier is None and settings.notify_webhook_url: - notifier = WebhookNotifier(settings.notify_webhook_url) + notifier = WebhookNotifier(settings.notify_webhook_url, settings.webhook_auth) actions = ActionExecutor(store=store, registry=registry, library=library, lifecycle=lifecycle, infra=infra, notifier=notifier, dry_run=settings.dry_run, @@ -117,5 +119,7 @@ def build_system( policy=policy, actions=actions, lifecycle=lifecycle, config=maint_cfg, intent_mapper=intent_mapper, clock=clock, wallclock=wallclock) healer = Healer(healer_cfg, store, actions, policy, propose=maintenance.propose, clock=clock) + prober = CanaryProber(registry, chat, embedder_for={settings.embed_model: embedder}, + metrics_registry=metrics.registry) return System(settings, store, registry, library, embedder, chat, router, executor, verifier, runner, policy, - lifecycle, actions, maintenance, healer, metrics) + lifecycle, actions, maintenance, healer, metrics, prober) diff --git a/core/tests/test_artifacts.py b/core/tests/test_artifacts.py new file mode 100644 index 0000000..80af685 --- /dev/null +++ b/core/tests/test_artifacts.py @@ -0,0 +1,155 @@ +"""Strukturprüfungen der Deployment-Artefakte (ohne laufende Infrastruktur).""" + +import json + +import httpx +import pytest +import yaml + +from knaster.healer.engine import HealerConfig +from knaster.settings import REPO_ROOT + +DEPLOY = REPO_ROOT / "deploy" + + +def test_alert_names_match_healer_rules(): + alerts = yaml.safe_load((DEPLOY / "prometheus/alerts.yml").read_text()) + names = {r["alert"] for g in alerts["groups"] for r in g["rules"]} + rules = HealerConfig.from_file(REPO_ROOT / "config/healer-rules.yaml").rules + matched = {r.match["alertname"] for r in rules} + assert names == matched + + +def test_healer_labels_available_on_alerts(): + """Labels, die Healer-Regeln brauchen, müssen von Metrik oder Scrape-Target kommen.""" + prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text()) + jobs = {j["job_name"]: j for j in prom["scrape_configs"]} + for job in ("node", "gpu", "vllm"): + for sc in jobs[job]["static_configs"]: + assert {"slot", "pve_node", "vmid"} <= set(sc["labels"]), (job, sc) + from knaster.canary import OPS_LABELS + assert set(OPS_LABELS) == {"docker_host", "container", "pve_node", "vmid"} + + +def test_alertmanager_targets_router_endpoint(): + am = yaml.safe_load((DEPLOY / "alertmanager/alertmanager.yml").read_text()) + url = am["receivers"][0]["webhook_configs"][0]["url"] + assert url.endswith("/v1/healer/alertmanager") + assert am["receivers"][0]["webhook_configs"][0]["send_resolved"] is True + + +@pytest.mark.parametrize("role", ["infra", "gpu-vllm", "gpu-llamacpp"]) +def test_compose_files_parse(role): + data = yaml.safe_load((DEPLOY / f"compose/{role}/docker-compose.yml").read_text()) + assert data["services"] + assert (DEPLOY / f"compose/{role}/.env.example").exists() + + +def test_ops_labels_consistent_between_registry_and_prometheus(): + reg = yaml.safe_load((REPO_ROOT / "sites/knaster/registry.yaml").read_text()) + prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text()) + vllm = {sc["labels"]["slot"]: sc["labels"] for sc in + next(j for j in prom["scrape_configs"] if j["job_name"] == "vllm")["static_configs"]} + for slot, labels in vllm.items(): + ops = reg["slots"][slot]["ops"] + assert ops["vmid"] == labels["vmid"] and ops["container"] == labels["container"] + + +WORKFLOWS = sorted((REPO_ROOT / "n8n/workflows").glob("*.json")) + + +@pytest.mark.parametrize("path", WORKFLOWS, ids=[p.name for p in WORKFLOWS]) +def test_n8n_workflow_structure(path): + wf = json.loads(path.read_text()) + assert wf["active"] is False + names = [n["name"] for n in wf["nodes"]] + assert len(names) == len(set(names)) + ids = [n["id"] for n in wf["nodes"]] + assert len(ids) == len(set(ids)) + for src, outs in wf["connections"].items(): + assert src in names + for branch in outs["main"]: + for target in branch: + assert target["node"] in names + # jeder Knoten außer dem Trigger ist erreichbar + targets = {t["node"] for o in wf["connections"].values() for b in o["main"] for t in b} + triggers = [n for n in wf["nodes"] if "trigger" in n["type"].lower() or n["type"].endswith(".webhook")] + assert len(triggers) == 1 + assert set(names) - targets == {triggers[0]["name"]} + + +def test_n8n_router_paths_exist(make_system): + from knaster.api import create_app + + paths = {r.path for r in create_app(make_system()).routes} + for path in WORKFLOWS: + for n in json.loads(path.read_text())["nodes"]: + url = n["parameters"].get("url", "") + if "router_url" in url: + assert url.split("}}")[-1] in paths + + +def test_webhook_notifier_sends_auth_header(): + from knaster.healer.drivers import WebhookNotifier + + seen = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["auth"] = request.headers.get("authorization") + seen["body"] = json.loads(request.content) + return httpx.Response(200) + + n = WebhookNotifier("http://n8n/webhook/knaster/notify", "Bearer t", + client=httpx.Client(transport=httpx.MockTransport(handler))) + n.notify("hallo", "high") + assert seen == {"auth": "Bearer t", "body": {"text": "hallo", "severity": "high", "data": {}}} + + +def test_drivers_build_correct_requests(): + from knaster.healer.drivers import PortainerDriver, ProxmoxDriver + + calls = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append((request.method, request.url.path, dict(request.headers))) + if request.url.path == "/api/endpoints": + return httpx.Response(200, json=[{"Id": 3, "Name": "gpu-main"}]) + return httpx.Response(200, json={"data": "UPID:1"}) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + PortainerDriver("https://portainer", "k", client=client).restart_container("gpu-main", "vllm") + assert calls[-1][:2] == ("POST", "/api/endpoints/3/docker/containers/vllm/restart") + assert calls[-1][2]["x-api-key"] == "k" + ProxmoxDriver("https://pve:8006", "knaster@pve!healer", "s", client=client).restart_vm("pve-ada", "201") + assert calls[-1][:2] == ("POST", "/api2/json/nodes/pve-ada/qemu/201/status/reboot") + assert calls[-1][2]["authorization"] == "PVEAPIToken=knaster@pve!healer=s" + + +def test_llm_and_embedding_clients(): + from knaster.embeddings import LlamaCppReranker, OpenAIEmbedder + from knaster.llm import ContextOverflowError, EndpointError, OpenAIChatClient + + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + if request.url.path == "/v1/embeddings": + return httpx.Response(200, json={"data": [{"index": 1, "embedding": [0.0, 1.0]}, + {"index": 0, "embedding": [1.0, 0.0]}]}) + if request.url.path == "/v1/rerank": + return httpx.Response(200, json={"results": [{"index": 1, "relevance_score": 0.9}, + {"index": 0, "relevance_score": 0.1}]}) + if body["model"] == "overflow": + return httpx.Response(400, text="This model's maximum context length is 8192 tokens") + if body["model"] == "down": + return httpx.Response(503, text="unavailable") + return httpx.Response(200, json={"model": body["model"], "usage": {"total_tokens": 3}, + "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}]}) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + assert OpenAIEmbedder("http://x", "e", client=client).embed(["a", "b"]) == [[1.0, 0.0], [0.0, 1.0]] + assert LlamaCppReranker("http://x", client=client).rerank("q", ["a", "b"]) == [0.1, 0.9] + chat = OpenAIChatClient("http://x", "k", client=client) + assert chat.chat("m", [{"role": "user", "content": "x"}]).content == "hi" + with pytest.raises(ContextOverflowError): + chat.chat("overflow", []) + with pytest.raises(EndpointError): + chat.chat("down", []) diff --git a/core/tests/test_inventory_validate.py b/core/tests/test_inventory_validate.py index 195765c..f93fffc 100644 --- a/core/tests/test_inventory_validate.py +++ b/core/tests/test_inventory_validate.py @@ -88,3 +88,26 @@ def test_cli(capsys): "--catalog", str(REPO_ROOT / "catalog")]) == 0 out = capsys.readouterr().out assert "gpu-main" in out + + +def test_canary_prober(registry, fake_chat): + from prometheus_client import CollectorRegistry, generate_latest + + from knaster.canary import CanaryProber + from knaster.embeddings import HashingEmbedder + from knaster.llm import EndpointError + + def down(m): + raise EndpointError("weg") + + fake_chat.handlers.update({"tier1-general": lambda m: "OK", "tier1-flex": down, + "verifier-small": lambda m: "Hallo"}) + reg = CollectorRegistry() + prober = CanaryProber(registry, fake_chat, {"embed-default": HashingEmbedder()}, metrics_registry=reg) + results = {r.slot: r for r in prober.run_once()} + assert "tier0-strong" not in results + assert results["tier1-general"].ok and not results["tier1-flex"].up + assert results["verifier-small"].up and not results["verifier-small"].ok + assert results["embed-default"].ok + text = generate_latest(reg).decode() + assert 'knaster_slot_endpoint_up{container="",docker_host="",node="gpu-flex"' in text diff --git a/deploy/alertmanager/alertmanager.yml b/deploy/alertmanager/alertmanager.yml new file mode 100644 index 0000000..b790d27 --- /dev/null +++ b/deploy/alertmanager/alertmanager.yml @@ -0,0 +1,18 @@ +# Alertmanager → Router-Service (Healer). Kurzes repeat_interval treibt die Eskalationsleiter voran; +# Cooldowns pro Stufe regelt der Healer selbst. +route: + receiver: knaster-healer + group_by: [alertname, slot, instance] + group_wait: 30s + group_interval: 1m + repeat_interval: 5m + +receivers: + - name: knaster-healer + webhook_configs: + - url: http://router:8080/v1/healer/alertmanager + send_resolved: true + http_config: + authorization: + type: Bearer + credentials_file: /etc/alertmanager/router_token diff --git a/deploy/compose/gpu-llamacpp/.env.example b/deploy/compose/gpu-llamacpp/.env.example new file mode 100644 index 0000000..f9f138d --- /dev/null +++ b/deploy/compose/gpu-llamacpp/.env.example @@ -0,0 +1,8 @@ +LLAMACPP_REF=b6000 +CUDA_VERSION=12.4.0 +CUDA_DOCKER_ARCH=52;61 +MODELS_DIR=/srv/models +EMBED_GGUF=Qwen3-Embedding-0.6B-Q8_0.gguf +EMBED_POOLING=last +RERANK_GGUF=bge-reranker-v2-m3-Q8_0.gguf +CHAT_GGUF=Qwen3-4B-Q4_K_M.gguf diff --git a/deploy/compose/gpu-llamacpp/docker-compose.yml b/deploy/compose/gpu-llamacpp/docker-compose.yml new file mode 100644 index 0000000..61fbc1c --- /dev/null +++ b/deploy/compose/gpu-llamacpp/docker-compose.yml @@ -0,0 +1,86 @@ +# Rollen "gpu-embed" und "gpu-small" (VMs mit Quadro P2000 / M4000): llama.cpp-Server. +# Maxwell (sm_52) und Pascal (sm_61) brauchen CUDA 12.x – daher eigener Build mit festen Architekturen. +# Profile: embed (Embedding + Rerank), chat (Verifier/Signatur), z. B. `docker compose --profile embed up -d` +name: knaster-gpu-llamacpp + +x-llamacpp: &llamacpp + build: + context: https://github.com/ggml-org/llama.cpp.git#${LLAMACPP_REF:-b6000} + dockerfile: .devops/cuda.Dockerfile + target: server + args: + CUDA_VERSION: ${CUDA_VERSION:-12.4.0} + CUDA_DOCKER_ARCH: ${CUDA_DOCKER_ARCH:-52;61} + image: knaster/llamacpp-server:${LLAMACPP_REF:-b6000}-cu12 + restart: unless-stopped + labels: {autoheal: "true"} + volumes: + - ${MODELS_DIR:-/srv/models}:/models:ro + deploy: + resources: + reservations: + devices: [{driver: nvidia, count: 1, capabilities: [gpu]}] + +services: + llamacpp-embed: + <<: *llamacpp + container_name: llamacpp-embed + profiles: ["embed"] + command: ["-m", "/models/${EMBED_GGUF:-Qwen3-Embedding-0.6B-Q8_0.gguf}", "--embedding", + "--pooling", "${EMBED_POOLING:-last}", "-c", "8192", "-ub", "8192", "-ngl", "99", + "--alias", "embed-default", "--metrics", "--host", "0.0.0.0", "--port", "8081"] + ports: ["8081:8081"] + healthcheck: &hc + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 60s + + llamacpp-rerank: + <<: *llamacpp + container_name: llamacpp-rerank + profiles: ["embed"] + command: ["-m", "/models/${RERANK_GGUF:-bge-reranker-v2-m3-Q8_0.gguf}", "--reranking", "-c", "8192", + "-ub", "8192", "-ngl", "99", "--alias", "rerank-default", "--metrics", "--host", "0.0.0.0", + "--port", "8082"] + ports: ["8082:8082"] + healthcheck: + <<: *hc + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8082/health"] + + llamacpp-chat: + <<: *llamacpp + container_name: llamacpp-chat + profiles: ["chat"] + command: ["-m", "/models/${CHAT_GGUF:-Qwen3-4B-Q4_K_M.gguf}", "-c", "8192", "-ngl", "99", + "--alias", "verifier-small", "--metrics", "--host", "0.0.0.0", "--port", "8080"] + ports: ["8080:8080"] + healthcheck: + <<: *hc + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health"] + + gpu-exporter: + image: utkuozdemir/nvidia_gpu_exporter:1.2.1 + restart: unless-stopped + ports: ["9835:9835"] + deploy: + resources: + reservations: + devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}] + + node-exporter: + image: prom/node-exporter:v1.8.1 + restart: unless-stopped + command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"] + pid: host + volumes: + - /:/host:ro,rslave + - /var/lib/node_exporter/textfile:/textfile:ro + ports: ["9100:9100"] + + autoheal: + image: willfarrell/autoheal:1.2.0 + restart: unless-stopped + environment: {AUTOHEAL_CONTAINER_LABEL: autoheal} + volumes: ["/var/run/docker.sock:/var/run/docker.sock"] diff --git a/deploy/compose/gpu-vllm/.env.example b/deploy/compose/gpu-vllm/.env.example new file mode 100644 index 0000000..258648d --- /dev/null +++ b/deploy/compose/gpu-vllm/.env.example @@ -0,0 +1,9 @@ +SLOT_ALIAS=tier1-general +VLLM_MODEL=Qwen/Qwen3-14B-AWQ +MAX_MODEL_LEN=32768 +GPU_MEM_UTIL=0.90 +MAX_LORAS=4 +MAX_LORA_RANK=64 +MODELS_DIR=/srv/models +ADAPTERS_DIR=/srv/adapters +HF_TOKEN= diff --git a/deploy/compose/gpu-vllm/docker-compose.yml b/deploy/compose/gpu-vllm/docker-compose.yml new file mode 100644 index 0000000..0267e86 --- /dev/null +++ b/deploy/compose/gpu-vllm/docker-compose.yml @@ -0,0 +1,62 @@ +# Rolle "gpu-main" / "gpu-flex" (VM mit durchgereichter RTX 4000 Ada): vLLM mit Multi-LoRA. +name: knaster-gpu-vllm + +services: + vllm: + image: vllm/vllm-openai:${VLLM_TAG:-v0.10.0} # Engine-Updates laufen als Lifecycle-Kandidat + container_name: vllm + restart: unless-stopped + labels: {autoheal: "true"} + ipc: host + env_file: .env + environment: + VLLM_ALLOW_RUNTIME_LORA_UPDATING: "True" # Adapter per /v1/load_lora_adapter nachladen + command: + - --model=${VLLM_MODEL:?VLLM_MODEL fehlt} + - --served-model-name=${SLOT_ALIAS:?SLOT_ALIAS fehlt} + - --max-model-len=${MAX_MODEL_LEN:-32768} + - --gpu-memory-utilization=${GPU_MEM_UTIL:-0.90} + - --enable-lora + - --max-loras=${MAX_LORAS:-4} + - --max-lora-rank=${MAX_LORA_RANK:-64} + - --host=0.0.0.0 + - --port=8000 + volumes: + - ${MODELS_DIR:-/srv/models}:/root/.cache/huggingface + - ${ADAPTERS_DIR:-/srv/adapters}:/adapters:ro + ports: ["8000:8000"] + deploy: + resources: + reservations: + devices: [{driver: nvidia, count: 1, capabilities: [gpu]}] + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=4)"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 300s # Laden großer Modelle + + gpu-exporter: + image: utkuozdemir/nvidia_gpu_exporter:1.2.1 + restart: unless-stopped + ports: ["9835:9835"] + deploy: + resources: + reservations: + devices: [{driver: nvidia, count: all, capabilities: [gpu, utility]}] + + node-exporter: + image: prom/node-exporter:v1.8.1 + restart: unless-stopped + command: ["--path.rootfs=/host", "--collector.textfile.directory=/textfile"] + pid: host + volumes: + - /:/host:ro,rslave + - /var/lib/node_exporter/textfile:/textfile:ro + ports: ["9100:9100"] + + autoheal: + image: willfarrell/autoheal:1.2.0 + restart: unless-stopped + environment: {AUTOHEAL_CONTAINER_LABEL: autoheal, AUTOHEAL_START_PERIOD: "300"} + volumes: ["/var/run/docker.sock:/var/run/docker.sock"] diff --git a/deploy/compose/infra/.env.example b/deploy/compose/infra/.env.example new file mode 100644 index 0000000..561d113 --- /dev/null +++ b/deploy/compose/infra/.env.example @@ -0,0 +1,18 @@ +KNASTER_VERSION=dev +KNASTER_SITE=knaster +KNASTER_API_TOKEN=change-me +KNASTER_DRY_RUN=true +KNASTER_CANARY_INTERVAL=60 +POSTGRES_PASSWORD=change-me +LITELLM_API_KEY=change-me +LITELLM_MASTER_KEY=change-me +OPENROUTER_API_KEY= +RERANK_URL=http://gpu-embed-1:8082 +NOTIFY_WEBHOOK_URL= +KNASTER_JOB_WEBHOOK_URL= +PORTAINER_HOST= +PORTAINER_KEY= +PROXMOX_URL= +PROXMOX_TOKEN_ID= +PROXMOX_TOKEN_SECRET= +KNASTER_WEBHOOK_AUTH=Bearer change-me diff --git a/deploy/compose/infra/docker-compose.yml b/deploy/compose/infra/docker-compose.yml new file mode 100644 index 0000000..e6eb73b --- /dev/null +++ b/deploy/compose/infra/docker-compose.yml @@ -0,0 +1,104 @@ +# Rolle "infra" (LXC/VM auf einem i7-Node): Router, LiteLLM, Postgres, Monitoring. +# Start: cp .env.example .env && docker compose up -d +name: knaster-infra + +x-restart: &restart + restart: unless-stopped + labels: {autoheal: "true"} + +services: + router: + <<: *restart + build: + context: ../../.. + dockerfile: deploy/docker/router.Dockerfile + image: knaster/router:${KNASTER_VERSION:-dev} + env_file: .env + environment: + KNASTER_DB_URL: postgresql+psycopg://knaster:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt}@postgres:5432/knaster + KNASTER_SITE_DIR: /app/sites/${KNASTER_SITE:-knaster} + LITELLM_BASE_URL: http://litellm:4000 + ports: ["8080:8080"] + depends_on: + postgres: {condition: service_healthy} + litellm: {condition: service_started} + + litellm: + <<: *restart + image: ghcr.io/berriai/litellm:${LITELLM_TAG:-main-stable} # vor Deployment auf feste Version pinnen + command: ["--config", "/app/config.yaml", "--port", "4000"] + env_file: .env + volumes: + - ../../litellm/config.yaml:/app/config.yaml:ro + ports: ["4000:4000"] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=4)"] + interval: 30s + timeout: 5s + retries: 3 + + postgres: + <<: *restart + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: knaster + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD fehlt} + POSTGRES_DB: knaster + volumes: + - postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U knaster -d knaster"] + interval: 10s + timeout: 5s + retries: 5 + + prometheus: + <<: *restart + image: prom/prometheus:v2.53.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=15d + - --web.enable-lifecycle + volumes: + - ../../prometheus:/etc/prometheus:ro + - prometheus:/prometheus + ports: ["9090:9090"] + + alertmanager: + <<: *restart + image: prom/alertmanager:v0.27.0 + command: ["--config.file=/etc/alertmanager/alertmanager.yml"] + volumes: + - ../../alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - ./secrets/router_token:/etc/alertmanager/router_token:ro + ports: ["9093:9093"] + + grafana: + <<: *restart + image: grafana/grafana:11.1.0 + profiles: ["dashboards"] + volumes: + - grafana:/var/lib/grafana + ports: ["3000:3000"] + + pve-exporter: + <<: *restart + image: prompve/prometheus-pve-exporter:${PVE_EXPORTER_TAG:-latest} # vor Deployment pinnen + profiles: ["proxmox"] + volumes: + - ./secrets/pve.yml:/etc/prometheus/pve.yml:ro + ports: ["9221:9221"] + + autoheal: + restart: unless-stopped + image: willfarrell/autoheal:1.2.0 + environment: + AUTOHEAL_CONTAINER_LABEL: autoheal + AUTOHEAL_INTERVAL: "15" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + +volumes: + postgres: {} + prometheus: {} + grafana: {} diff --git a/deploy/docker/router.Dockerfile b/deploy/docker/router.Dockerfile new file mode 100644 index 0000000..42b3799 --- /dev/null +++ b/deploy/docker/router.Dockerfile @@ -0,0 +1,27 @@ +# Router-Service (knaster). Build-Kontext: Repo-Wurzel. +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 +WORKDIR /app + +COPY core/pyproject.toml core/pyproject.toml +COPY core/src core/src +RUN pip install "./core[postgres]" + +# Portable Konfiguration; standortspezifisches (sites/) kann per Volume überschrieben werden +COPY config config +COPY catalog catalog +COPY skills skills +COPY sites sites + +ENV KNASTER_CONFIG_DIR=/app/config \ + KNASTER_SKILLS_DIR=/app/skills \ + KNASTER_SITE_DIR=/app/sites/knaster \ + KNASTER_DRY_RUN=true + +RUN useradd --system --uid 10001 knaster +USER knaster +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=4).status==200 else 1)" +CMD ["knaster", "serve", "--host", "0.0.0.0", "--port", "8080"] diff --git a/deploy/prometheus/alerts.yml b/deploy/prometheus/alerts.yml new file mode 100644 index 0000000..6d3b0ed --- /dev/null +++ b/deploy/prometheus/alerts.yml @@ -0,0 +1,49 @@ +# Alerts für den Healer (config/healer-rules.yaml). Namen müssen zu den Regeln passen. +groups: + - name: knaster-slots + rules: + - alert: SlotEndpointDown + expr: knaster_slot_endpoint_up == 0 + for: 2m + labels: {severity: high} + annotations: {summary: "Slot {{ $labels.slot }} antwortet nicht"} + + - alert: SlotCanaryFailing + expr: knaster_slot_canary_ok == 0 and knaster_slot_endpoint_up == 1 + for: 5m + labels: {severity: warning} + annotations: {summary: "Slot {{ $labels.slot }} antwortet, aber falsch"} + + - alert: VllmKvCacheSaturated + expr: max by (slot) (vllm:gpu_cache_usage_perc or vllm:kv_cache_usage_perc) > 0.95 + for: 10m + labels: {severity: warning} + annotations: {summary: "KV-Cache von {{ $labels.slot }} dauerhaft voll"} + + - name: knaster-gpu + rules: + - alert: GpuXidError + # aus deploy/scripts/gpu-xid-textfile.sh (node_exporter textfile collector) + expr: increase(knaster_gpu_xid_errors_total[10m]) > 0 + labels: {severity: high} + annotations: {summary: "NVIDIA-Xid-Fehler auf {{ $labels.instance }} (Xid {{ $labels.xid }})"} + + - alert: GpuTemperatureHigh + expr: nvidia_smi_temperature_gpu > 85 + for: 5m + labels: {severity: warning} + annotations: {summary: "GPU {{ $labels.slot }} über 85 °C"} + + - name: knaster-nodes + rules: + - alert: NodeDown + expr: up{job="node"} == 0 + for: 3m + labels: {severity: high} + annotations: {summary: "{{ $labels.instance }} nicht erreichbar"} + + - alert: DiskAlmostFull + expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes < 0.10 + for: 15m + labels: {severity: warning} + annotations: {summary: "{{ $labels.instance }} {{ $labels.mountpoint }} < 10 % frei"} diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml new file mode 100644 index 0000000..31ba743 --- /dev/null +++ b/deploy/prometheus/prometheus.yml @@ -0,0 +1,67 @@ +# Scrape-Ziele. Die Labels an den GPU-Zielen (slot, docker_host, container, pve_node, vmid) landen in den +# Alerts und damit beim Healer. Hostnamen = DNS-Namen der VMs (ANNAHME bis Deployment). +global: + scrape_interval: 30s + evaluation_interval: 30s + +rule_files: [alerts.yml] + +alerting: + alertmanagers: + - static_configs: [{targets: ["alertmanager:9093"]}] + +scrape_configs: + - job_name: knaster-router + metrics_path: /metrics + static_configs: [{targets: ["router:8080"]}] + + - job_name: vllm + static_configs: + - targets: ["gpu-main:8000"] + labels: {slot: tier1-general, docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"} + - targets: ["gpu-flex:8000"] + labels: {slot: tier1-flex, docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"} + + - job_name: llamacpp + static_configs: + - targets: ["gpu-embed-1:8081"] + labels: {slot: embed-default, docker_host: gpu-embed-1, container: llamacpp-embed, pve_node: pve3, vmid: "221"} + - targets: ["gpu-embed-1:8082"] + labels: {slot: rerank-default, docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"} + - targets: ["gpu-small:8080"] + labels: {slot: verifier-small, docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"} + + - job_name: gpu + static_configs: + - targets: ["gpu-main:9835"] + labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"} + - targets: ["gpu-flex:9835"] + labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"} + - targets: ["gpu-embed-1:9835"] + labels: {slot: embed-default, pve_node: pve3, vmid: "221"} + - targets: ["gpu-embed-2:9835"] + labels: {slot: embed-default, pve_node: pve4, vmid: "222"} + - targets: ["gpu-small:9835"] + labels: {slot: verifier-small, pve_node: pve3, vmid: "231"} + + - job_name: node # Labels werden für GpuXidError → Healer (VM-Neustart) gebraucht + static_configs: + - targets: ["gpu-main:9100"] + labels: {slot: tier1-general, pve_node: pve-ada, vmid: "201"} + - targets: ["gpu-flex:9100"] + labels: {slot: tier1-flex, pve_node: pve-ada, vmid: "202"} + - targets: ["gpu-embed-1:9100"] + labels: {slot: embed-default, pve_node: pve3, vmid: "221"} + - targets: ["gpu-embed-2:9100"] + labels: {slot: embed-default, pve_node: pve4, vmid: "222"} + - targets: ["gpu-small:9100"] + labels: {slot: verifier-small, pve_node: pve3, vmid: "231"} + + - job_name: proxmox + metrics_path: /pve + params: {module: [default]} + static_configs: [{targets: ["pve1", "pve2", "pve3", "pve4", "pve-ada"]}] + relabel_configs: + - {source_labels: [__address__], target_label: __param_target} + - {source_labels: [__param_target], target_label: instance} + - {target_label: __address__, replacement: "pve-exporter:9221"} diff --git a/deploy/scripts/gpu-xid-textfile.sh b/deploy/scripts/gpu-xid-textfile.sh new file mode 100755 index 0000000..bdf8f36 --- /dev/null +++ b/deploy/scripts/gpu-xid-textfile.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Zählt NVIDIA-Xid-Fehler aus dem Kernel-Log und schreibt sie für den node_exporter (textfile collector). +# Aufruf per systemd-Timer jede Minute (siehe platform/proxmox/ansible/roles/gpu-node). +set -euo pipefail +OUT_DIR="${1:-/var/lib/node_exporter/textfile}" +TMP="$(mktemp "${OUT_DIR}/.xid.XXXXXX")" +{ + echo "# HELP knaster_gpu_xid_errors_total NVIDIA Xid-Fehler seit Boot" + echo "# TYPE knaster_gpu_xid_errors_total counter" + journalctl -k -b --no-pager -o cat 2>/dev/null \ + | grep -oE 'NVRM: Xid \([^)]*\): [0-9]+' \ + | awk '{print $NF}' | sort | uniq -c \ + | awk '{printf "knaster_gpu_xid_errors_total{xid=\"%s\"} %d\n", $2, $1}' +} > "$TMP" +chmod 0644 "$TMP" +mv "$TMP" "${OUT_DIR}/gpu_xid.prom" diff --git a/n8n/README.md b/n8n/README.md new file mode 100644 index 0000000..8118cc9 --- /dev/null +++ b/n8n/README.md @@ -0,0 +1,18 @@ +# n8n-Workflows + +Importierbar über *Workflows → Import from File*. Alle Workflows werden **inaktiv** importiert. + +| Datei | Zweck | Trigger | +|---|---|---| +| `knaster-request.json` | Anfrage an den Router (`/v1/run`) und Antwort | Webhook `POST /webhook/knaster/run` `{input, skill?}` | +| `knaster-maintenance-chat.json` | Pflege-Agent per Prompt (HITL) | n8n Chat | +| `knaster-service-routine.json` | wöchentliche Service-Routine, Bericht an Kanal | Zeitplan Mo 08:00 | +| `knaster-notify.json` | Meldungen von Healer/Policy an einen Kanal | Webhook `POST /webhook/knaster/notify` (= `NOTIFY_WEBHOOK_URL`) | +| `knaster-jobs.json` | Lifecycle-/Skill-Jobs an Trainings-Runner bzw. Hermes | Webhook `POST /webhook/knaster/jobs` (= `KNASTER_JOB_WEBHOOK_URL`) | + +Nach dem Import: +1. Credential **knaster-router** (Header Auth): Name `Authorization`, Wert `Bearer `. +2. Credential **knaster-webhook** (Header Auth) für eingehende Webhooks. Das gleiche Header-Paar muss im Aufrufer gesetzt werden. +3. Im Knoten **Konfiguration** `router_url` und ggf. `channel_url`, `trainer_url`, `hermes_url` anpassen. + +Die Entscheidungslogik liegt bewusst im Router-Service (ADR-0005). n8n kümmert sich um Trigger, Kanäle und Weiterleitung. diff --git a/n8n/workflows/knaster-jobs.json b/n8n/workflows/knaster-jobs.json new file mode 100644 index 0000000..7b0cb0e --- /dev/null +++ b/n8n/workflows/knaster-jobs.json @@ -0,0 +1,249 @@ +{ + "name": "knaster: Jobs an Runner/Hermes", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "knaster/jobs", + "responseMode": "responseNode", + "options": {}, + "authentication": "headerAuth" + }, + "id": "24f3f3a3-47ae-5b87-8902-bcc3465b593a", + "name": "Job", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 300 + ], + "webhookId": "03e1babe-7daa-5ab1-ba6e-8cc4de50268c", + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-webhook" + } + } + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "b3b14ae1-2d7f-5ef9-a96f-1d3693c703f6", + "name": "router_url", + "value": "http://router:8080", + "type": "string" + }, + { + "id": "a3123e42-afa4-53f7-a266-066cce1ff763", + "name": "trainer_url", + "value": "http://gpu-flex:9000/jobs", + "type": "string" + }, + { + "id": "c672b90a-49d5-5e50-9b23-3766353f0487", + "name": "hermes_url", + "value": "http://hermes:8000/jobs", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "acb2e1cb-0776-564b-b909-b5b3f613dc88", + "name": "Konfiguration", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 220, + 300 + ], + "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer " + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "loose" + }, + "conditions": [ + { + "id": "9dec25ae-5535-5f01-aebb-b9d7e7397686", + "leftValue": "={{ ['graduate_skill','merge_skills'].includes($('Job').item.json.body.job) }}", + "rightValue": true, + "operator": { + "type": "boolean", + "operation": "true" + } + } + ], + "combinator": "and" + }, + "options": {} + }, + "id": "a2881a83-cbb5-5cdb-96bb-d6a1833ac388", + "name": "Skill-Job?", + "type": "n8n-nodes-base.if", + "typeVersion": 2, + "position": [ + 440, + 300 + ] + }, + { + "parameters": { + "method": "POST", + "url": "={{ $('Konfiguration').item.json.hermes_url }}", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify($('Job').item.json.body) }}", + "options": { + "timeout": 30000 + } + }, + "id": "ba36e0ed-7cca-5c04-af60-9644177cac6f", + "name": "An Hermes (Skill-Autor)", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 660, + 200 + ], + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-router" + } + } + }, + { + "parameters": { + "method": "POST", + "url": "={{ $('Konfiguration').item.json.trainer_url }}", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify($('Job').item.json.body) }}", + "options": { + "timeout": 30000 + } + }, + "id": "98abbd0d-1384-52bb-b095-cb978f2bcc37", + "name": "An Trainings-Runner", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 660, + 420 + ], + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-router" + } + } + }, + { + "parameters": { + "respondWith": "json", + "responseBody": "={{ JSON.stringify({ accepted: true }) }}", + "options": { + "responseCode": 202 + } + }, + "id": "6678356f-6c70-58fe-a9e0-be3d7cf23851", + "name": "Angenommen", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1.1, + "position": [ + 880, + 300 + ] + } + ], + "connections": { + "Job": { + "main": [ + [ + { + "node": "Konfiguration", + "type": "main", + "index": 0 + } + ] + ] + }, + "Konfiguration": { + "main": [ + [ + { + "node": "Skill-Job?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Skill-Job?": { + "main": [ + [ + { + "node": "An Hermes (Skill-Autor)", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "An Trainings-Runner", + "type": "main", + "index": 0 + } + ] + ] + }, + "An Hermes (Skill-Autor)": { + "main": [ + [ + { + "node": "Angenommen", + "type": "main", + "index": 0 + } + ] + ] + }, + "An Trainings-Runner": { + "main": [ + [ + { + "node": "Angenommen", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "tags": [ + { + "name": "knaster" + }, + { + "name": "lifecycle" + } + ], + "meta": { + "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen." + } +} diff --git a/n8n/workflows/knaster-maintenance-chat.json b/n8n/workflows/knaster-maintenance-chat.json new file mode 100644 index 0000000..7c9746c --- /dev/null +++ b/n8n/workflows/knaster-maintenance-chat.json @@ -0,0 +1,147 @@ +{ + "name": "knaster: Pflege-Agent (Chat)", + "nodes": [ + { + "parameters": { + "public": false, + "options": { + "responseMode": "lastNode" + } + }, + "id": "ef50a82c-c64a-5a2e-b380-f40ae2a7f738", + "name": "Chat", + "type": "@n8n/n8n-nodes-langchain.chatTrigger", + "typeVersion": 1.1, + "position": [ + 0, + 300 + ], + "webhookId": "f819ed24-d095-5120-b2f2-e996135967e1" + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "fecd0058-75b7-569f-8970-e4647b315631", + "name": "router_url", + "value": "http://router:8080", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "a08a0680-3d4b-5845-8693-86627a2aaeb4", + "name": "Konfiguration", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 220, + 300 + ], + "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer " + }, + { + "parameters": { + "method": "POST", + "url": "={{ $('Konfiguration').item.json.router_url }}/v1/maintenance/command", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ text: $('Chat').item.json.chatInput, session: $('Chat').item.json.sessionId, user: 'n8n-chat' }) }}", + "options": { + "timeout": 60000 + } + }, + "id": "4fc64c5d-b22d-5168-a4bb-d56a3c4e1001", + "name": "Befehl an Pflege-Agent", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 440, + 300 + ], + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-router" + } + } + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "4d75dea4-a1de-51e1-b324-06282f454d47", + "name": "output", + "value": "={{ $json.text }}", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "ca5fef45-e097-52a6-af88-8d9a6c8d1461", + "name": "Antwort", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 660, + 300 + ] + } + ], + "connections": { + "Chat": { + "main": [ + [ + { + "node": "Konfiguration", + "type": "main", + "index": 0 + } + ] + ] + }, + "Konfiguration": { + "main": [ + [ + { + "node": "Befehl an Pflege-Agent", + "type": "main", + "index": 0 + } + ] + ] + }, + "Befehl an Pflege-Agent": { + "main": [ + [ + { + "node": "Antwort", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "tags": [ + { + "name": "knaster" + }, + { + "name": "hitl" + } + ], + "meta": { + "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen." + } +} diff --git a/n8n/workflows/knaster-notify.json b/n8n/workflows/knaster-notify.json new file mode 100644 index 0000000..d3cd73c --- /dev/null +++ b/n8n/workflows/knaster-notify.json @@ -0,0 +1,141 @@ +{ + "name": "knaster: Benachrichtigungen", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "knaster/notify", + "responseMode": "responseNode", + "options": {}, + "authentication": "headerAuth" + }, + "id": "e324dc52-1df2-54f4-916f-540c3033562a", + "name": "Healer-Meldung", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 300 + ], + "webhookId": "978b850a-ca87-5dca-a16e-b737df5acad2", + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-webhook" + } + } + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "904e09c8-286a-5272-8b3f-4c6c49899e9d", + "name": "router_url", + "value": "http://router:8080", + "type": "string" + }, + { + "id": "fe09313b-633c-58b6-8e80-ef4db981118f", + "name": "channel_url", + "value": "https://example.invalid/kanal-webhook", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "7868766e-ff24-5694-9338-f7b0c3aa6603", + "name": "Konfiguration", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 220, + 300 + ], + "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer " + }, + { + "parameters": { + "method": "POST", + "url": "={{ $('Konfiguration').item.json.channel_url }}", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ text: '[' + $('Healer-Meldung').item.json.body.severity + '] ' + $('Healer-Meldung').item.json.body.text }) }}", + "options": {} + }, + "id": "28c5f6c3-74a0-5b70-8225-2196f1b9164d", + "name": "An Kanal senden", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 440, + 300 + ] + }, + { + "parameters": { + "respondWith": "json", + "responseBody": "={{ JSON.stringify({ ok: true }) }}", + "options": { + "responseCode": 200 + } + }, + "id": "3518decf-c8b3-5d29-b88b-b59a0980d173", + "name": "OK", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1.1, + "position": [ + 660, + 300 + ] + } + ], + "connections": { + "Healer-Meldung": { + "main": [ + [ + { + "node": "Konfiguration", + "type": "main", + "index": 0 + } + ] + ] + }, + "Konfiguration": { + "main": [ + [ + { + "node": "An Kanal senden", + "type": "main", + "index": 0 + } + ] + ] + }, + "An Kanal senden": { + "main": [ + [ + { + "node": "OK", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "tags": [ + { + "name": "knaster" + } + ], + "meta": { + "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen." + } +} diff --git a/n8n/workflows/knaster-request.json b/n8n/workflows/knaster-request.json new file mode 100644 index 0000000..bb8bc6e --- /dev/null +++ b/n8n/workflows/knaster-request.json @@ -0,0 +1,212 @@ +{ + "name": "knaster: Anfrage ausführen", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "knaster/run", + "responseMode": "responseNode", + "options": {}, + "authentication": "headerAuth" + }, + "id": "9463206f-d1cb-52fc-98ba-01370b80290b", + "name": "Anfrage", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 300 + ], + "webhookId": "bba18b52-dcd1-5231-a132-0434e5f27505", + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-webhook" + } + } + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "b65c35b7-cf8b-5067-8c3c-d90533ad04dd", + "name": "router_url", + "value": "http://router:8080", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "4ccf1d69-7b78-5ad7-9d1a-40ae77e7b3d1", + "name": "Konfiguration", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 220, + 300 + ], + "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer " + }, + { + "parameters": { + "method": "POST", + "url": "={{ $('Konfiguration').item.json.router_url }}/v1/run", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ input: $('Anfrage').item.json.body.input, skill: $('Anfrage').item.json.body.skill || null }) }}", + "options": { + "timeout": 180000 + } + }, + "id": "d6f5c10d-2a30-598a-af5e-3f8040098cb4", + "name": "Router /v1/run", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 440, + 300 + ], + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-router" + } + } + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "loose" + }, + "conditions": [ + { + "id": "3fc7e0c8-fd00-5368-819c-dff9572baf12", + "leftValue": "={{ $json.status }}", + "rightValue": "ok", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + }, + "options": {} + }, + "id": "01605cd8-f4e0-53c6-acfe-32c389d6cbe6", + "name": "Erfolgreich?", + "type": "n8n-nodes-base.if", + "typeVersion": 2, + "position": [ + 660, + 300 + ] + }, + { + "parameters": { + "respondWith": "json", + "responseBody": "={{ JSON.stringify({ output: $json.output, stage: $json.stage, skill: $json.skill, escalations: $json.escalations, run_id: $json.run_id }) }}", + "options": { + "responseCode": 200 + } + }, + "id": "9ca6279a-dfb6-525e-aa18-8385bed2f448", + "name": "Antwort", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1.1, + "position": [ + 880, + 200 + ] + }, + { + "parameters": { + "respondWith": "json", + "responseBody": "={{ JSON.stringify({ status: $json.status, run_id: $json.run_id, reasons: $json.attempts[$json.attempts.length-1].decision.reasons }) }}", + "options": { + "responseCode": 422 + } + }, + "id": "8728502e-e051-5259-a2bd-57646e3b1c5e", + "name": "Fehler", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1.1, + "position": [ + 880, + 420 + ] + } + ], + "connections": { + "Anfrage": { + "main": [ + [ + { + "node": "Konfiguration", + "type": "main", + "index": 0 + } + ] + ] + }, + "Konfiguration": { + "main": [ + [ + { + "node": "Router /v1/run", + "type": "main", + "index": 0 + } + ] + ] + }, + "Router /v1/run": { + "main": [ + [ + { + "node": "Erfolgreich?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Erfolgreich?": { + "main": [ + [ + { + "node": "Antwort", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Fehler", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "tags": [ + { + "name": "knaster" + } + ], + "meta": { + "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen." + } +} diff --git a/n8n/workflows/knaster-service-routine.json b/n8n/workflows/knaster-service-routine.json new file mode 100644 index 0000000..e16994d --- /dev/null +++ b/n8n/workflows/knaster-service-routine.json @@ -0,0 +1,199 @@ +{ + "name": "knaster: Service-Routine (wöchentlich)", + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "weeks", + "weeksInterval": 1, + "triggerAtDay": [ + 1 + ], + "triggerAtHour": 8 + } + ] + } + }, + "id": "c58239bb-f1c9-557c-952c-21645fcb8a57", + "name": "Zeitplan", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 300 + ] + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "df17a04d-3369-5ef6-83b2-150c90db08fa", + "name": "router_url", + "value": "http://router:8080", + "type": "string" + }, + { + "id": "fb597b99-b6c3-5306-b5fa-68ffc02938d3", + "name": "channel_url", + "value": "https://example.invalid/kanal-webhook", + "type": "string" + } + ] + }, + "options": {} + }, + "id": "dc94a1c9-74b6-5403-942a-59a84e634a59", + "name": "Konfiguration", + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 220, + 300 + ], + "notes": "Nach dem Import anpassen. Credential 'knaster-router' = Header Auth: Authorization / Bearer " + }, + { + "parameters": { + "method": "POST", + "url": "={{ $('Konfiguration').item.json.router_url }}/v1/maintenance/routine", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({}) }}", + "options": { + "timeout": 180000 + } + }, + "id": "ba3983aa-c877-5686-8297-60b9be29bdb3", + "name": "Routine starten", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 440, + 300 + ], + "credentials": { + "httpHeaderAuth": { + "id": "", + "name": "knaster-router" + } + } + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "loose" + }, + "conditions": [ + { + "id": "f74160f5-984f-5078-9fc3-5557cdc71e5f", + "leftValue": "={{ $json.findings.length + $json.proposals.length + $json.auto_executed.length }}", + "rightValue": 0, + "operator": { + "type": "number", + "operation": "gt" + } + } + ], + "combinator": "and" + }, + "options": {} + }, + "id": "07062ef5-09ab-5043-83d6-7d4806fe945a", + "name": "Etwas zu tun?", + "type": "n8n-nodes-base.if", + "typeVersion": 2, + "position": [ + 660, + 300 + ] + }, + { + "parameters": { + "method": "POST", + "url": "={{ $('Konfiguration').item.json.channel_url }}", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ text: $json.text }) }}", + "options": {} + }, + "id": "f6ec4fd3-03a4-5226-beec-ac4f11c0fb30", + "name": "An Kanal senden", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 880, + 200 + ], + "notes": "Kanal (Telegram/Matrix/Mail) hier anbinden. Antworten laufen über den Pflege-Chat." + } + ], + "connections": { + "Zeitplan": { + "main": [ + [ + { + "node": "Konfiguration", + "type": "main", + "index": 0 + } + ] + ] + }, + "Konfiguration": { + "main": [ + [ + { + "node": "Routine starten", + "type": "main", + "index": 0 + } + ] + ] + }, + "Routine starten": { + "main": [ + [ + { + "node": "Etwas zu tun?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Etwas zu tun?": { + "main": [ + [ + { + "node": "An Kanal senden", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "tags": [ + { + "name": "knaster" + }, + { + "name": "hitl" + } + ], + "meta": { + "description": "cluster-knaster – siehe docs/spec. Import inaktiv, Credentials setzen." + } +} diff --git a/sites/knaster/registry.yaml b/sites/knaster/registry.yaml index 0bd0613..36b4313 100644 --- a/sites/knaster/registry.yaml +++ b/sites/knaster/registry.yaml @@ -1,5 +1,6 @@ # Registry für den Cluster "knaster" (SPEC-core §1). # Clients sprechen nur Slot-Namen an; litellm_model ist der Alias in deploy/litellm/config.yaml. +# ops: Labels für den Healer (Portainer-Endpoint, Container, Proxmox-Node, VM-ID). slots: tier0-strong: tier: 0 @@ -21,6 +22,7 @@ slots: effective_context: 16384 supports_lora: true fallbacks: [tier1-flex] + ops: {docker_host: gpu-main, container: vllm, pve_node: pve-ada, vmid: "201"} # VM-IDs/Hosts: ANNAHME bis Deployment tier1-flex: tier: 1 @@ -32,6 +34,7 @@ slots: max_context: 32768 effective_context: 16384 supports_lora: true + ops: {docker_host: gpu-flex, container: vllm, pve_node: pve-ada, vmid: "202"} # VM-IDs/Hosts: ANNAHME bis Deployment verifier-small: tier: 1 @@ -42,6 +45,7 @@ slots: base_model: qwen3-4b-q4km max_context: 8192 effective_context: 6144 + ops: {docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"} # VM-IDs/Hosts: ANNAHME bis Deployment signature-extractor: tier: 1 @@ -52,6 +56,7 @@ slots: base_model: qwen3-4b-q4km max_context: 8192 effective_context: 6144 + ops: {docker_host: gpu-small, container: llamacpp-chat, pve_node: pve3, vmid: "231"} # VM-IDs/Hosts: ANNAHME bis Deployment embed-default: tier: 1 @@ -62,6 +67,7 @@ slots: base_model: qwen3-embedding-0.6b-q8 max_context: 8192 kind: embedding + ops: {docker_host: gpu-embed-1, container: llamacpp-embed, pve_node: pve3, vmid: "221"} # VM-IDs/Hosts: ANNAHME bis Deployment rerank-default: tier: 1 @@ -72,5 +78,6 @@ slots: base_model: bge-reranker-v2-m3-q8 max_context: 8192 kind: rerank + ops: {docker_host: gpu-embed-1, container: llamacpp-rerank, pve_node: pve3, vmid: "221"} # VM-IDs/Hosts: ANNAHME bis Deployment adapters: []