core: Policy, Pflege-Agent (HITL), Healer, Lifecycle-Pipeline, Graduierung + Konfiguration

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 00:43:26 +02:00
parent f79e2a8556
commit f2da893f71
28 changed files with 2179 additions and 10 deletions
@@ -0,0 +1 @@
+"""Deterministisches Autohealing."""
@@ -0,0 +1,124 @@
+"""Treiber für Infrastruktur-Aktionen (Portainer, Proxmox, Webhook) mit Dry-Run."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+import httpx
+
+
+class InfraDriver(Protocol):
+    def restart_container(self, host: str, container: str) -> str: ...
+    def restart_vm(self, pve_node: str, vmid: str) -> str: ...
+    def reboot_host(self, pve_node: str) -> str: ...
+
+
+class Notifier(Protocol):
+    def notify(self, text: str, severity: str = "info", data: dict[str, Any] | None = None) -> str: ...
+
+
+@dataclass
+class DryRunDriver:
+    """Protokolliert nur. Standard bis zur bewussten Freischaltung."""
+
+    calls: list[tuple[str, dict[str, Any]]] = field(default_factory=list)
+
+    def _log(self, action: str, **kw: Any) -> str:
+        self.calls.append((action, kw))
+        args = ", ".join(f"{k}={v}" for k, v in kw.items())
+        return f"[dry-run] {action}({args})"
+
+    def restart_container(self, host: str, container: str) -> str:
+        return self._log("restart_container", host=host, container=container)
+
+    def restart_vm(self, pve_node: str, vmid: str) -> str:
+        return self._log("restart_vm", pve_node=pve_node, vmid=vmid)
+
+    def reboot_host(self, pve_node: str) -> str:
+        return self._log("reboot_host", pve_node=pve_node)
+
+    def notify(self, text: str, severity: str = "info", data: dict[str, Any] | None = None) -> str:
+        return self._log("notify", severity=severity, text=text)
+
+
+class PortainerDriver:
+    """Container-Neustart über die Portainer-API. `host` = Endpoint-ID oder Endpoint-Name."""
+
+    def __init__(self, base_url: str, api_key: str, client: httpx.Client | None = None):
+        self.base_url = base_url.rstrip("/")
+        self.client = client or httpx.Client(timeout=30.0, follow_redirects=True)
+        self.headers = {"X-API-Key": api_key}
+
+    def _endpoint_id(self, host: str) -> str:
+        if host.isdigit():
+            return host
+        resp = self.client.get(f"{self.base_url}/api/endpoints", headers=self.headers)
+        resp.raise_for_status()
+        for ep in resp.json():
+            if ep.get("Name") == host:
+                return str(ep["Id"])
+        raise LookupError(f"Portainer-Endpoint {host} nicht gefunden")
+
+    def restart_container(self, host: str, container: str) -> str:
+        ep = self._endpoint_id(host)
+        resp = self.client.post(
+            f"{self.base_url}/api/endpoints/{ep}/docker/containers/{container}/restart", headers=self.headers
+        )
+        resp.raise_for_status()
+        return f"Container {container} auf {host} neu gestartet"
+
+
+class ProxmoxDriver:
+    def __init__(self, base_url: str, token_id: str, token_secret: str, verify_tls: bool = True,
+                 client: httpx.Client | None = None):
+        self.base_url = base_url.rstrip("/")
+        self.client = client or httpx.Client(timeout=30.0, verify=verify_tls)
+        self.headers = {"Authorization": f"PVEAPIToken={token_id}={token_secret}"}
+
+    def _post(self, path: str, data: dict[str, Any] | None = None) -> Any:
+        resp = self.client.post(f"{self.base_url}/api2/json{path}", headers=self.headers, data=data)
+        resp.raise_for_status()
+        return resp.json().get("data")
+
+    def restart_vm(self, pve_node: str, vmid: str) -> str:
+        task = self._post(f"/nodes/{pve_node}/qemu/{vmid}/status/reboot")
+        return f"VM {vmid} auf {pve_node} Neustart angestoßen ({task})"
+
+    def reboot_host(self, pve_node: str) -> str:
+        self._post(f"/nodes/{pve_node}/status", {"command": "reboot"})
+        return f"Host {pve_node} Neustart angestoßen"
+
+
+class WebhookNotifier:
+    def __init__(self, url: str, client: httpx.Client | None = None):
+        self.url = url
+        self.client = client or httpx.Client(timeout=15.0)
+
+    def notify(self, text: str, severity: str = "info", data: dict[str, Any] | None = None) -> str:
+        resp = self.client.post(self.url, json={"text": text, "severity": severity, "data": data or {}})
+        resp.raise_for_status()
+        return "benachrichtigt"
+
+
+class CompositeInfra:
+    """Verteilt auf Portainer (Container) und Proxmox (VM/Host)."""
+
+    def __init__(self, portainer: PortainerDriver | None, proxmox: ProxmoxDriver | None):
+        self.portainer = portainer
+        self.proxmox = proxmox
+
+    def restart_container(self, host: str, container: str) -> str:
+        if not self.portainer:
+            raise RuntimeError("Portainer nicht konfiguriert")
+        return self.portainer.restart_container(host, container)
+
+    def restart_vm(self, pve_node: str, vmid: str) -> str:
+        if not self.proxmox:
+            raise RuntimeError("Proxmox nicht konfiguriert")
+        return self.proxmox.restart_vm(pve_node, vmid)
+
+    def reboot_host(self, pve_node: str) -> str:
+        if not self.proxmox:
+            raise RuntimeError("Proxmox nicht konfiguriert")
+        return self.proxmox.reboot_host(pve_node)
@@ -0,0 +1,165 @@
+"""Regel-Engine: Alert → Eskalationsleiter von Aktionen mit Versuchsbudget, Cooldown und globaler Bremse."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from ..maintenance.actions import ActionError, ActionExecutor
+from ..policy import Policy
+from ..store import Store
+from ..util import MissingLabel, fill_template, load_yaml, now, parse_duration
+
+
+class Step(BaseModel):
+    action: str
+    params: dict[str, Any] = Field(default_factory=dict)
+    attempts: int = 1
+    cooldown: str | float = 0
+    only_if_set_by_healer: bool = False
+
+
+class Rule(BaseModel):
+    id: str
+    match: dict[str, str]
+    ladder: list[Step]
+    on_exhausted: str = "notify"
+    on_resolved: list[Step] = Field(default_factory=list)
+
+    def matches(self, labels: dict[str, str]) -> bool:
+        return all(labels.get(k) == v for k, v in self.match.items())
+
+
+class HealerConfig(BaseModel):
+    rules: list[Rule] = Field(default_factory=list)
+    max_actions_per_hour: int = 10
+    enabled_actions: list[str] = Field(default_factory=list)
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> HealerConfig:
+        return cls(**load_yaml(path))
+
+
+class Outcome(BaseModel):
+    fingerprint: str
+    rule: str | None
+    status: str  # executed | proposed | cooldown | exhausted | unmatched | throttled | resolved | skipped | failed
+    action: str | None = None
+    message: str = ""
+
+
+class Healer:
+    def __init__(self, config: HealerConfig, store: Store, actions: ActionExecutor, policy: Policy,
+                 propose: Callable[..., Any] | None = None, clock: Callable[[], float] = now):
+        self.config = config
+        self.store = store
+        self.actions = actions
+        self.policy = policy
+        self.propose = propose
+        self.clock = clock
+
+    def handle_alertmanager(self, payload: dict[str, Any]) -> list[Outcome]:
+        return [self.handle_alert(a) for a in payload.get("alerts", [])]
+
+    def handle_alert(self, alert: dict[str, Any]) -> Outcome:
+        labels = {k: str(v) for k, v in (alert.get("labels") or {}).items()}
+        fp = alert.get("fingerprint") or "|".join(f"{k}={v}" for k, v in sorted(labels.items()))
+        rule = next((r for r in self.config.rules if r.matches(labels)), None)
+        if alert.get("status") == "resolved":
+            return self._resolved(rule, fp, labels)
+        if rule is None:
+            msg = self._notify(f"Unbekannter Alert {labels.get('alertname')}: {labels}", "warning")
+            return Outcome(fingerprint=fp, rule=None, status="unmatched", message=msg)
+        return self._firing(rule, fp, labels)
+
+    # --- intern -----------------------------------------------------------------------------
+    def _key(self, rule: Rule, fp: str) -> str:
+        return f"{rule.id}|{fp}"
+
+    def _throttled(self) -> bool:
+        return self.store.count_events("healer_action", since=self.clock() - 3600) >= self.config.max_actions_per_hour
+
+    def _notify(self, text: str, severity: str) -> str:
+        try:
+            return self.actions.execute("notify", {"text": text, "severity": severity})["message"]
+        except ActionError as exc:
+            return f"Benachrichtigung fehlgeschlagen: {exc}"
+
+    def _firing(self, rule: Rule, fp: str, labels: dict[str, str]) -> Outcome:
+        key = self._key(rule, fp)
+        state = self.store.get_healer_state(key) or {"used": [0] * len(rule.ladder), "last": 0.0,
+                                                      "cooldown": 0.0, "notified": False, "set_slot": None}
+        if self.clock() - state["last"] < state["cooldown"]:
+            return Outcome(fingerprint=fp, rule=rule.id, status="cooldown")
+
+        for i, step in enumerate(rule.ladder):
+            if state["used"][i] >= step.attempts:
+                continue
+            state["used"][i] += 1
+            state["last"] = self.clock()
+            state["cooldown"] = parse_duration(step.cooldown)
+            try:
+                params = fill_template(step.params, labels)
+            except MissingLabel as exc:
+                self.store.set_healer_state(key, state)
+                return Outcome(fingerprint=fp, rule=rule.id, status="skipped", action=step.action,
+                               message=f"Label {exc} fehlt")
+            mode = self.policy.decide(step.action)
+            if mode == "deny":
+                self.store.set_healer_state(key, state)
+                return Outcome(fingerprint=fp, rule=rule.id, status="skipped", action=step.action,
+                               message="durch Richtlinie verboten")
+            if mode == "approve" and self.propose is not None:
+                p = self.propose(step.action, params, f"Healer: {step.action} für {labels.get('alertname')}",
+                                 reason=f"Regel {rule.id}", source="healer", force_approval=True)
+                self.store.set_healer_state(key, state)
+                return Outcome(fingerprint=fp, rule=rule.id, status="proposed", action=step.action,
+                               message=f"Vorschlag {p.id}")
+            if self._throttled():
+                self.store.set_healer_state(key, state)
+                msg = self._notify(f"Healer gebremst (>{self.config.max_actions_per_hour}/h): {rule.id}", "high")
+                return Outcome(fingerprint=fp, rule=rule.id, status="throttled", action=step.action, message=msg)
+            if step.action == "mark_slot":
+                params = {**params, "set_by": "healer", "reason": f"Healer-Regel {rule.id}"}
+                state["set_slot"] = params.get("slot")
+            try:
+                result = self.actions.execute(step.action, params)
+                status, message = "executed", result["message"]
+            except ActionError as exc:
+                status, message = "failed", str(exc)
+            self.store.add_event("healer_action", subject=rule.id, data={"action": step.action, "status": status},
+                                 ts=self.clock())
+            self.store.set_healer_state(key, state)
+            return Outcome(fingerprint=fp, rule=rule.id, status=status, action=step.action, message=message)
+
+        if not state["notified"]:
+            state["notified"] = True
+            self.store.set_healer_state(key, state)
+            msg = self._notify(f"Healer-Regel {rule.id} erschöpft – bitte eingreifen. Labels: {labels}", "high")
+            return Outcome(fingerprint=fp, rule=rule.id, status="exhausted", message=msg)
+        return Outcome(fingerprint=fp, rule=rule.id, status="exhausted", message="bereits gemeldet")
+
+    def _resolved(self, rule: Rule | None, fp: str, labels: dict[str, str]) -> Outcome:
+        if rule is None:
+            return Outcome(fingerprint=fp, rule=None, status="resolved")
+        key = self._key(rule, fp)
+        state = self.store.get_healer_state(key) or {}
+        messages = []
+        for step in rule.on_resolved:
+            try:
+                params = fill_template(step.params, labels)
+            except MissingLabel:
+                continue
+            if step.only_if_set_by_healer and step.action == "mark_slot":
+                if state.get("set_slot") != params.get("slot"):
+                    continue
+                params = {**params, "set_by": "healer", "reason": "Alert behoben"}
+            try:
+                messages.append(self.actions.execute(step.action, params)["message"])
+            except ActionError as exc:
+                messages.append(str(exc))
+        self.store.delete_healer_state(key)
+        return Outcome(fingerprint=fp, rule=rule.id, status="resolved", message="; ".join(messages))