rorapp/cluster-knaster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy: Dockerfile, Compose pro Rolle, LiteLLM, Prometheus/Alertmanager, Canary-Prober, n8n-Workflows

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rorapp
2026-10-12 00:51:31 +02:00
co-authored by Claude Opus 5.5
parent 4cbe313c62
commit cb5a413169
28 changed files with 1731 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/knaster/api.py
+5 -1
View File
@@ -53,10 +53,14 @@ class GraduateIn(BaseModel):
min_variants: int = 10
def create_app(system: System | None = None) -> FastAPI:
def create_app(system: System | None = None, start_prober: bool | None = None) -> FastAPI:
if start_prober is None:
start_prober = system is None # nur im echten Betrieb, nicht in Tests mit injiziertem System
system = system or build_system(Settings.from_env())
app = FastAPI(title="knaster", version="0.1.0")
app.state.system = system
if start_prober and system.settings.canary_interval > 0:
system.prober.start(system.settings.canary_interval)
def auth(authorization: str | None = Header(default=None)) -> None:
token = system.settings.api_token
core/src/knaster/canary.py
+89
View File
@@ -0,0 +1,89 @@
"""Canary-Prober: prüft Slots regelmäßig mit einer Mini-Anfrage und liefert Metriken für die Healer-Alerts."""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass
from prometheus_client import Gauge
from .embeddings import Embedder
from .llm import ChatClient, LLMError
from .registry import Registry, Slot
OPS_LABELS = ("docker_host", "container", "pve_node", "vmid")
PROMPT = [{"role": "system", "content": "Antworte nur mit dem Wort OK."}, {"role": "user", "content": "Test"}]
@dataclass
class ProbeResult:
slot: str
up: bool # Endpunkt antwortet
ok: bool # Antwort inhaltlich korrekt
latency: float
error: str = ""
class CanaryProber:
def __init__(self, registry: Registry, chat: ChatClient, embedder_for: dict[str, Embedder] | None = None,
metrics_registry=None, include_tier0: bool = False): # noqa: ANN001
self.registry = registry
self.chat = chat
self.embedder_for = embedder_for or {}
self.include_tier0 = include_tier0
labels = ["slot", "node", *OPS_LABELS]
kw = {"registry": metrics_registry} if metrics_registry is not None else {}
self.g_up = Gauge("knaster_slot_endpoint_up", "1 = Endpunkt erreichbar", labels, **kw)
self.g_ok = Gauge("knaster_slot_canary_ok", "1 = Canary-Antwort korrekt", labels, **kw)
self.g_latency = Gauge("knaster_slot_canary_latency_seconds", "Latenz der Canary-Anfrage", labels, **kw)
def _targets(self) -> list[Slot]:
out = []
for slot in self.registry.slots():
if not slot.spec.canary or slot.spec.kind == "rerank":
continue
if slot.spec.tier == 0 and not self.include_tier0:
continue # kostet Geld; Erreichbarkeit überwacht LiteLLM
out.append(slot)
return out
def probe(self, slot: Slot) -> ProbeResult:
start = time.monotonic()
try:
if slot.spec.kind == "embedding":
embedder = self.embedder_for.get(slot.name)
if embedder is None:
return ProbeResult(slot.name, True, True, 0.0, "kein Embedder konfiguriert")
vec = embedder.embed(["Test"])[0]
ok = len(vec) > 0
else:
res = self.chat.chat(slot.spec.litellm_model, PROMPT, max_tokens=5, temperature=0)
ok = "ok" in res.content.lower()
return ProbeResult(slot.name, True, ok, time.monotonic() - start)
except LLMError as exc:
return ProbeResult(slot.name, False, False, time.monotonic() - start, str(exc))
except Exception as exc: # noqa: BLE001 – Netzwerkfehler des Embedders etc.
return ProbeResult(slot.name, False, False, time.monotonic() - start, f"{type(exc).__name__}: {exc}")
def run_once(self) -> list[ProbeResult]:
results = []
for slot in self._targets():
r = self.probe(slot)
labels = {"slot": slot.name, "node": slot.spec.node or "",
**{k: slot.spec.ops.get(k, "") for k in OPS_LABELS}}
self.g_up.labels(**labels).set(1 if r.up else 0)
self.g_ok.labels(**labels).set(1 if r.ok else 0)
self.g_latency.labels(**labels).set(r.latency)
results.append(r)
return results
def start(self, interval: float) -> threading.Thread:
def loop() -> None:
while True:
self.run_once()
time.sleep(interval)
t = threading.Thread(target=loop, name="canary-prober", daemon=True)
t.start()
return t
core/src/knaster/healer/drivers.py
+4 -2
View File
@@ -91,12 +91,14 @@ class ProxmoxDriver:
class WebhookNotifier:
def __init__(self, url: str, client: httpx.Client | None = None):
def __init__(self, url: str, auth: str = "", client: httpx.Client | None = None):
self.url = url
self.headers = {"Authorization": auth} if auth else {}
self.client = client or httpx.Client(timeout=15.0)
def notify(self, text: str, severity: str = "info", data: dict[str, Any] | None = None) -> str:
resp = self.client.post(self.url, json={"text": text, "severity": severity, "data": data or {}})
resp = self.client.post(self.url, json={"text": text, "severity": severity, "data": data or {}},
headers=self.headers)
resp.raise_for_status()
return "benachrichtigt"
core/src/knaster/lifecycle.py
+3 -2
View File
@@ -52,14 +52,15 @@ class LoggingJobRunner:
class WebhookJobRunner:
def __init__(self, url: str, client: Any | None = None):
def __init__(self, url: str, auth: str = "", client: Any | None = None):
import httpx
self.url = url
self.headers = {"Authorization": auth} if auth else {}
self.client = client or httpx.Client(timeout=15.0)
def submit(self, job: str, candidate: dict[str, Any]) -> str:
resp = self.client.post(self.url, json={"job": job, "candidate": candidate})
resp = self.client.post(self.url, json={"job": job, "candidate": candidate}, headers=self.headers)
resp.raise_for_status()
return f"Job {job} übergeben"
core/src/knaster/registry.py
+2
View File
@@ -26,6 +26,8 @@ class SlotSpec(BaseModel):
supports_lora: bool = False
fallbacks: list[str] = Field(default_factory=list)
kind: Literal["chat", "embedding", "rerank"] = "chat"
canary: bool = True # vom Prober geprüft (für tier 0 standardmäßig aus, siehe canary.py)
ops: dict[str, str] = Field(default_factory=dict) # docker_host, container, pve_node, vmid → Healer-Labels
class AdapterSpec(BaseModel):
core/src/knaster/settings.py
+4
View File
@@ -35,7 +35,9 @@ class Settings:
proxmox_token_secret: str = ""
notify_webhook_url: str = ""
job_webhook_url: str = ""
webhook_auth: str = "" # Wert des Authorization-Headers für ausgehende Webhooks (n8n Header Auth)
request_timeout: float = 120.0
canary_interval: float = 60.0 # 0 = Prober aus
@classmethod
def from_env(cls) -> Settings:
@@ -60,5 +62,7 @@ class Settings:
proxmox_token_secret=env.get("PROXMOX_TOKEN_SECRET", ""),
notify_webhook_url=env.get("NOTIFY_WEBHOOK_URL", ""),
job_webhook_url=env.get("KNASTER_JOB_WEBHOOK_URL", ""),
webhook_auth=env.get("KNASTER_WEBHOOK_AUTH", ""),
request_timeout=float(env.get("KNASTER_REQUEST_TIMEOUT", base.request_timeout)),
canary_interval=float(env.get("KNASTER_CANARY_INTERVAL", base.canary_interval)),
)
core/src/knaster/system.py
+8 -4
View File
@@ -7,6 +7,7 @@ from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from .canary import CanaryProber
from .classifier import EmbeddingClassifier
from .embeddings import Embedder, LlamaCppReranker, OpenAIEmbedder, Reranker
from .executor import Executor
@@ -46,6 +47,7 @@ class System:
maintenance: MaintenanceAgent
healer: Healer
metrics: Metrics
prober: CanaryProber
def _load_classifiers(directory: Path) -> dict[str, EmbeddingClassifier]:
@@ -95,8 +97,8 @@ def build_system(
runner = Runner(router, executor, verifier, library, store, metrics)
if jobs is None:
jobs = WebhookJobRunner(settings.job_webhook_url) if settings.job_webhook_url and not settings.dry_run \
else LoggingJobRunner()
use_webhook = settings.job_webhook_url and not settings.dry_run
jobs = WebhookJobRunner(settings.job_webhook_url, settings.webhook_auth) if use_webhook else LoggingJobRunner()
lifecycle = Lifecycle(store, registry, jobs, life_cfg)
if infra is None and not settings.dry_run:
@@ -106,7 +108,7 @@ def build_system(
if settings.proxmox_url and settings.proxmox_token_id else None
infra = CompositeInfra(portainer, proxmox)
if notifier is None and settings.notify_webhook_url:
notifier = WebhookNotifier(settings.notify_webhook_url)
notifier = WebhookNotifier(settings.notify_webhook_url, settings.webhook_auth)
actions = ActionExecutor(store=store, registry=registry, library=library, lifecycle=lifecycle, infra=infra,
notifier=notifier, dry_run=settings.dry_run,
@@ -117,5 +119,7 @@ def build_system(
policy=policy, actions=actions, lifecycle=lifecycle, config=maint_cfg,
intent_mapper=intent_mapper, clock=clock, wallclock=wallclock)
healer = Healer(healer_cfg, store, actions, policy, propose=maintenance.propose, clock=clock)
prober = CanaryProber(registry, chat, embedder_for={settings.embed_model: embedder},
metrics_registry=metrics.registry)
return System(settings, store, registry, library, embedder, chat, router, executor, verifier, runner, policy,
lifecycle, actions, maintenance, healer, metrics)
lifecycle, actions, maintenance, healer, metrics, prober)