core: Policy, Pflege-Agent (HITL), Healer, Lifecycle-Pipeline, Graduierung + Konfiguration

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/tests/conftest.py

@@ -176,3 +176,46 @@ class FakeChat:
 @pytest.fixture
 def fake_chat() -> FakeChat:
     return FakeChat()
+
+
+class Clock:
+    def __init__(self, t: float | None = None):
+        import time
+
+        self.t = time.time() if t is None else t
+
+    def __call__(self) -> float:
+        return self.t
+
+    def advance(self, seconds: float) -> None:
+        self.t += seconds
+
+
+@pytest.fixture
+def clock() -> Clock:
+    return Clock()
+
+
+@pytest.fixture
+def make_system(tmp_path, skills_dir, fake_chat, clock):
+    import yaml
+
+    from knaster.embeddings import HashingEmbedder
+    from knaster.healer.drivers import DryRunDriver
+    from knaster.lifecycle import LoggingJobRunner
+    from knaster.settings import REPO_ROOT, Settings
+    from knaster.system import build_system
+
+    def make(**overrides):
+        site = tmp_path / "site"
+        site.mkdir(exist_ok=True)
+        (site / "registry.yaml").write_text(yaml.safe_dump(REGISTRY))
+        settings = Settings(db_url="sqlite://", site_dir=site, skills_dir=skills_dir,
+                            config_dir=overrides.pop("config_dir", REPO_ROOT / "config"),
+                            api_token=overrides.pop("api_token", "secret"))
+        kw = dict(embedder=HashingEmbedder(), chat=fake_chat, jobs=LoggingJobRunner(), notifier=DryRunDriver(),
+                  clock=clock)
+        kw.update(overrides)
+        return build_system(settings, **kw)
+
+    return make

core/tests/test_lifecycle.py

@@ -0,0 +1,119 @@
+import pytest
+
+from knaster.lifecycle import Lifecycle, LifecycleConfig, LifecycleError, LoggingJobRunner
+from knaster.registry import AdapterSpec
+
+
+@pytest.fixture
+def life(store, registry):
+    return Lifecycle(store, registry, LoggingJobRunner(), LifecycleConfig(min_n=10))
+
+
+def run_to_awaiting(life, cid, model="cand-model"):
+    life.start(cid)
+    life.report(cid, "build", {"ok": True, "model": model})
+    life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
+    life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
+    return life.report(cid, "canary", {"pass_rate": 0.93, "n": 12})
+
+
+def test_full_adapter_flow_and_rollback(life, registry):
+    cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
+    cand = run_to_awaiting(life, cid, model="rechnungen-v2")
+    assert cand["state"] == "awaiting_promotion"
+    assert [j for j, _ in life.jobs.jobs] == ["build", "eval", "shadow"]
+    assert registry.slot("tier1-general").canary_model is None  # Canary beendet
+
+    life.promote(cid)
+    active = registry.active_adapter("base-14b", "rechnungen")
+    assert active.name == "rechnungen-v2" and active.version == 2
+    assert {a.name: a.status for a in registry.adapters()}["rechnungen-v1"] == "retired"
+
+    life.rollback(cid)
+    assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
+    assert life.get(cid)["state"] == "rolled_back"
+
+
+def test_canary_is_set_during_canary_phase(life, registry):
+    cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
+    life.start(cid)
+    life.report(cid, "build", {"model": "rechnungen-v2"})
+    life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
+    life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
+    slot = registry.slot("tier1-general")
+    assert (slot.canary_model, slot.canary_cluster, slot.canary_share) == ("rechnungen-v2", "rechnungen", 0.1)
+    life.report(cid, "canary", {"pass_rate": 0.5, "n": 12})
+    assert life.get(cid)["state"] == "rejected"
+    assert registry.slot("tier1-general").canary_model is None
+
+
+def test_base_model_switch_retires_adapters_and_spawns_retraining(life, registry, store):
+    cid = life.create("base_model", "tier1-general", {"base_model": "base-32b"})
+    run_to_awaiting(life, cid, model="tier1-general-candidate")
+    life.promote(cid)
+    assert registry.slot("tier1-general").base_model == "base-32b"
+    assert registry.active_adapter("base-14b", "rechnungen") is None
+    follow = [c for c in store.list_candidates() if c["kind"] == "adapter"]
+    assert len(follow) == 1 and follow[0]["target"] == {"cluster": "rechnungen", "base_model": "base-32b"}
+
+    life.rollback(cid)
+    assert registry.slot("tier1-general").base_model == "base-14b"
+    assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
+    assert life.get(follow[0]["id"])["state"] == "rejected"
+
+
+def test_gates_reject(life):
+    cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"}, baseline={"pass_rate": 0.9})
+    life.start(cid)
+    life.report(cid, "build", {"model": "m"})
+    assert life.report(cid, "eval", {"pass_rate": 0.85, "n": 40})["state"] == "rejected"
+
+    cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
+    life.start(cid)
+    life.report(cid, "build", {"model": "m"})
+    life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
+    assert life.report(cid, "shadow", {"agreement": 0.95, "n": 3})["state"] == "rejected"  # zu wenig Daten
+
+
+def test_error_report_and_invalid_transitions(life):
+    cid = life.create("engine", "tier1-general", {"image": "vllm:v2"})
+    with pytest.raises(LifecycleError):
+        life.report(cid, "build", {"ok": True})  # noch nicht gestartet
+    life.start(cid)
+    with pytest.raises(LifecycleError):
+        life.promote(cid)
+    assert life.report(cid, "build", {"error": "OOM"})["state"] == "failed"
+    with pytest.raises(LifecycleError):
+        life.create("adapter", "tier1-general", {})
+    with pytest.raises(LifecycleError):
+        life.create("base_model", "nope", {"base_model": "x"})
+
+
+def test_canary_metrics_from_cases(life, store):
+    cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
+    life.start(cid)
+    life.report(cid, "build", {"model": "rv2"})
+    life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
+    life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
+    for i in range(4):
+        store.add_case(input=f"x{i}", stage=1, slot="tier1-general", model="rv2", passed=i < 3, split="train",
+                       origin="route", ts=life.get(cid)["updated_at"] + 1)
+    store.add_case(input="y", stage=1, slot="tier1-general", model="other", passed=False, split="train",
+                   origin="route")
+    assert life.canary_metrics(cid) == {"pass_rate": 0.75, "n": 4}
+
+
+def test_router_uses_canary_model(registry, library, embedder):
+    import random
+
+    from knaster.router import Router
+
+    from .test_router import RECHNUNG
+
+    registry.set_canary("tier1-general", "rechnungen-v2", 1.0, "rechnungen")
+    r = Router(registry, library, embedder, rng=random.Random(1))
+    d = r.route(RECHNUNG)
+    assert d.canary and d.model == d.adapter == "rechnungen-v2"
+    registry.set_canary("tier1-general", "rechnungen-v2", 0.0, "rechnungen")
+    assert not r.route(RECHNUNG).canary
+    registry.set_adapter(AdapterSpec(name="x", base_model="base-14b", cluster="andere"))

core/tests/test_maintenance.py

@@ -0,0 +1,210 @@
+from datetime import datetime
+
+import pytest
+
+from knaster.maintenance.agent import parse_command
+
+
+@pytest.mark.parametrize("text,name,args", [
+    ("Status", "status", {}),
+    ("starte die Service-Routine", "routine", {}),
+    ("zeige vorschläge", "list", {}),
+    ("genehmige P-3", "approve", {"ids": "P-3"}),
+    ("genehmige P-3, P-5 und p7", "approve", {"ids": "P-3, P-5 und p7"}),
+    ("approve p-2", "approve", {"ids": "p-2"}),
+    ("genehmige alle mit Risiko niedrig", "approve_low", {}),
+    ("lehne P-4 ab: zu teuer", "reject", {"id": "P-4", "reason": "zu teuer"}),
+    ("lehne P-4 ab", "reject", {"id": "P-4"}),
+    ("wechsle tier1-general auf Qwen/Qwen3-32B-AWQ", "switch_base",
+     {"slot": "tier1-general", "base": "Qwen/Qwen3-32B-AWQ"}),
+    ("trainiere einen Adapter für den Cluster rechnungen", "train_adapter", {"cluster": "rechnungen"}),
+    ("rollback tier1-general", "rollback", {"slot": "tier1-general"}),
+    ("stufe rechnung-extrahieren zurück", "demote", {"skill": "rechnung-extrahieren"}),
+    ("markiere tier1-flex als unavailable", "mark", {"slot": "tier1-flex", "status": "unavailable"}),
+    ("ja", "confirm", {}),
+])
+def test_parse_command(text, name, args):
+    parsed = parse_command(text)
+    assert parsed is not None and parsed[0] == name
+    for k, v in args.items():
+        assert parsed[1][k] == v
+
+
+def test_parse_unknown():
+    assert parse_command("mach mal was schönes") is None
+
+
+def test_command_creates_proposal_and_needs_approval(make_system):
+    sysm = make_system()
+    m = sysm.maintenance
+    reply = m.handle_command("stufe rechnung-extrahieren zurück", user="robert")
+    assert "Bestätige" in reply.text
+    p = reply.proposals[0]
+    assert p.state == "open"  # obwohl Richtlinie auto sagt: Prompt-Aufträge brauchen Bestätigung
+    assert sysm.library.get("rechnung-extrahieren").meta.status == "active"
+    reply = m.handle_command(f"genehmige {p.id}", user="robert")
+    assert "executed" in reply.text
+    assert sysm.library.get("rechnung-extrahieren").meta.status == "demoted"
+    assert m._load(p.seq).decided_by == "robert"
+
+
+def test_reject_and_errors(make_system):
+    m = make_system().maintenance
+    p = m.handle_command("markiere tier1-flex als degraded").proposals[0]
+    assert "rejected" in m.handle_command(f"lehne {p.id} ab: nicht nötig").text
+    assert "Fehler" in m.handle_command(f"genehmige {p.id}").text
+    assert "Fehler" in m.handle_command("markiere gibtsnicht als ok").text
+    assert "Fehler" in m.handle_command("genehmige P-999").text
+
+
+def test_switch_base_runs_pipeline_after_approval(make_system):
+    sysm = make_system()
+    m = sysm.maintenance
+    p = m.handle_command("wechsle tier1-general auf base-32b").proposals[0]
+    m.handle_command(f"genehmige {p.id}")
+    cands = sysm.store.list_candidates()
+    assert cands[0]["kind"] == "base_model" and cands[0]["state"] == "building"
+    assert sysm.lifecycle.jobs.jobs[0][0] == "build"
+
+
+def test_intent_mapper_requires_confirmation(make_system):
+    class Mapper:
+        def map(self, text):
+            return "stufe vertrag-pruefen zurück" if "vertrag" in text else None
+
+    sysm = make_system(intent_mapper=Mapper())
+    m = sysm.maintenance
+    reply = m.handle_command("der vertrag skill macht nur mist", session="s1")
+    assert "Meinst du" in reply.text
+    assert sysm.store.list_proposals() == []
+    reply = m.handle_command("ja", session="s1")
+    assert reply.proposals and reply.proposals[0].state == "open"
+    assert "nicht verstanden" in m.handle_command("blabla").text
+    assert "nichts zu bestätigen" in m.handle_command("ja", session="s1").text
+
+
+def test_llm_intent_mapper(fake_chat):
+    from knaster.maintenance.agent import LLMIntentMapper
+
+    fake_chat.handlers["tier1-general"] = lambda msgs: "rollback tier1-general"
+    assert LLMIntentMapper(fake_chat).map("mach das letzte update rückgängig") == "rollback tier1-general"
+    fake_chat.handlers["tier1-general"] = lambda msgs: "NONE"
+    assert LLMIntentMapper(fake_chat).map("hallo") is None
+
+
+def test_deferred_until_window(make_system):
+    sysm = make_system(wallclock=lambda: datetime(2026, 1, 1, 12, 0))
+    m = sysm.maintenance
+    p = m.propose("reboot_host", {"pve_node": "pve1"}, "Host neu starten", source="healer")
+    p = m.approve(p.seq)
+    assert p.state == "approved" and p.result["deferred"]
+    assert m.run_deferred() == []
+    m.wallclock = lambda: datetime(2026, 1, 2, 3, 0)
+    done = m.run_deferred()
+    assert done[0].state == "executed" and "[dry-run] reboot_host" in done[0].result["message"]
+
+
+def test_dedup_of_open_proposals(make_system):
+    m = make_system().maintenance
+    a = m.propose("merge_skills", {"a": "x", "b": "y"}, "merge")
+    b = m.propose("merge_skills", {"a": "x", "b": "y"}, "merge")
+    assert a.seq == b.seq
+
+
+def _cases(store, n, *, skill, stage, passed_ratio, slot="tier1-general", cluster=None, split="train",
+           tokens=None):
+    for i in range(n):
+        store.add_case(input=f"{skill}-{stage}-{i}", skill=skill, cluster=cluster or skill, stage=stage, slot=slot,
+                       passed=i < n * passed_ratio, split=split, origin="route", prompt_tokens=tokens)
+
+
+def test_routine_drift_auto_demotes(make_system):
+    sysm = make_system()
+    _cases(sysm.store, 30, skill="rechnung-extrahieren", cluster="rechnungen", stage=1, passed_ratio=0.5)
+    report = sysm.maintenance.run_routine()
+    assert any("Drift" in f for f in report.findings)
+    assert any("rechnung-extrahieren" in a for a in report.auto_executed)
+    assert sysm.library.get("rechnung-extrahieren").meta.status == "demoted"
+    # Router sieht die Änderung sofort
+    from .test_router import RECHNUNG
+    assert sysm.router.route(RECHNUNG).stage == 0
+
+
+def test_routine_graduation_candidate(make_system):
+    sysm = make_system()
+    _cases(sysm.store, 25, skill="vertrag-pruefen", stage=0, passed_ratio=1.0, slot="tier0-strong")
+    report = sysm.maintenance.run_routine()
+    kinds = {p.kind for p in report.proposals}
+    assert "request_graduation" in kinds
+    p = next(p for p in report.proposals if p.kind == "request_graduation")
+    sysm.maintenance.approve(p.seq)
+    assert sysm.lifecycle.jobs.jobs[-1][0] == "graduate_skill"
+
+
+def test_routine_degraded_slot_and_context(make_system, clock):
+    sysm = make_system()
+    sysm.registry.set_status("tier1-flex", "unavailable", reason="Training")
+    for _ in range(12):
+        sysm.store.add_event("compact_prompt", subject="tier1-general", ts=clock())
+    clock.advance(2 * 3600)
+    _cases(sysm.store, 30, skill="rechnung-extrahieren", stage=1, passed_ratio=1.0, cluster="rechnungen",
+           tokens=1000)
+    report = sysm.maintenance.run_routine()
+    assert any("tier1-flex" in f for f in report.findings)
+    assert any(p.kind == "mark_slot" for p in report.proposals)
+    assert any("effektiver Kontext = 2048" in a for a in report.auto_executed)
+    assert sysm.registry.slot("tier1-general").calibrated_context == 2048
+    assert "Offene Vorschläge" in report.text
+
+
+def test_routine_adapter_need(make_system):
+    sysm = make_system()
+    sysm.maintenance.config.adapter_min_cases = 20
+    _cases(sysm.store, 40, skill="rechnung-extrahieren", stage=0, passed_ratio=1.0, cluster="rechnungen",
+           slot="tier0-strong")
+    _cases(sysm.store, 10, skill="rechnung-extrahieren", stage=1, passed_ratio=0.7, cluster="rechnungen")
+    report = sysm.maintenance.run_routine()
+    p = next(p for p in report.proposals if p.kind == "train_adapter")
+    assert p.params == {"cluster": "rechnungen", "slot": "tier1-general"}
+    sysm.maintenance.approve(p.seq)
+    cand = sysm.store.list_candidates()[0]
+    assert cand["kind"] == "adapter" and cand["target"]["base_model"] == "base-14b"
+    # zweite Routine schlägt nicht erneut vor
+    assert not any(p.kind == "train_adapter" for p in sysm.maintenance.run_routine().proposals)
+
+
+def test_routine_merge_similar_skills(make_system):
+    sysm = make_system()
+    sysm.maintenance.config.merge_similarity = 0.0
+    report = sysm.maintenance.run_routine()
+    assert any(p.kind == "merge_skills" for p in report.proposals)
+
+
+def test_routine_evaluates_canary_and_proposes_promotion(make_system):
+    sysm = make_system()
+    life = sysm.lifecycle
+    life.config.min_n = 5
+    cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
+    life.start(cid)
+    life.report(cid, "build", {"model": "rv2"})
+    life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
+    life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
+    for i in range(6):
+        sysm.store.add_case(input=f"c{i}", stage=1, slot="tier1-general", model="rv2", passed=True,
+                            split="train", origin="route")
+    report = sysm.maintenance.run_routine()
+    assert any("Canary" in a for a in report.auto_executed)
+    p = next(p for p in report.proposals if p.kind == "promote_candidate")
+    assert p.risk == "high"
+    sysm.maintenance.approve(p.seq)
+    assert sysm.registry.active_adapter("base-14b", "rechnungen").name == "rv2"
+    # rollback per Prompt
+    m = sysm.maintenance
+    rp = m.handle_command("rollback tier1-general").proposals[0]
+    m.approve(rp.seq)
+    assert sysm.registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
+
+
+def test_status_command(make_system):
+    text = make_system().maintenance.handle_command("status").text
+    assert "tier1-general: ok" in text and "Skills:" in text

core/tests/test_policy_healer.py

@@ -0,0 +1,124 @@
+from datetime import datetime
+
+from knaster.healer.engine import HealerConfig
+from knaster.policy import Policy
+from knaster.util import fill_template, parse_duration
+
+
+def test_policy_modes_and_windows():
+    p = Policy(**{"default": "approve", "actions": {
+        "a": {"mode": "auto"}, "night": {"mode": "approve", "window": "22:00-04:00"},
+        "day": {"mode": "auto", "window": "08:00-17:00"}}})
+    assert p.decide("a") == "auto" and p.decide("unknown") == "approve"
+    assert p.in_window("night", datetime(2026, 1, 1, 23, 0))
+    assert p.in_window("night", datetime(2026, 1, 1, 3, 0))
+    assert not p.in_window("night", datetime(2026, 1, 1, 12, 0))
+    assert p.in_window("day", datetime(2026, 1, 1, 9, 0))
+    assert p.in_window("a")
+
+
+def test_repo_configs_load():
+    from knaster.settings import REPO_ROOT
+
+    cfg = HealerConfig.from_file(REPO_ROOT / "config" / "healer-rules.yaml")
+    assert {r.id for r in cfg.rules} >= {"slot-endpoint-down", "gpu-xid"}
+    assert cfg.enabled_actions == []  # sicherer Default
+    pol = Policy.from_file(REPO_ROOT / "config" / "policy.yaml")
+    assert pol.decide("promote_candidate") == "approve" and pol.decide("demote_skill") == "auto"
+
+
+def test_util():
+    assert parse_duration("5m") == 300 and parse_duration(10) == 10 and parse_duration("2h") == 7200
+    assert fill_template({"a": "{x}-{y}", "b": ["{x}"]}, {"x": "1", "y": "2"}) == {"a": "1-2", "b": ["1"]}
+
+
+def alert(status="firing", **labels):
+    base = {"alertname": "SlotEndpointDown", "slot": "tier1-general", "docker_host": "gpu-main",
+            "container": "vllm", "pve_node": "pve1", "vmid": "201"}
+    base.update(labels)
+    return {"status": status, "labels": base, "fingerprint": "fp1"}
+
+
+def test_healer_ladder_cooldown_and_resolve(make_system, clock):
+    sysm = make_system()
+    h = sysm.healer
+    dry = sysm.actions.dry
+    out = h.handle_alert(alert())
+    assert (out.status, out.action) == ("executed", "restart_container")
+    assert out.message.startswith("[dry-run]")
+    assert h.handle_alert(alert()).status == "cooldown"
+    clock.advance(301)
+    assert h.handle_alert(alert()).action == "restart_container"
+    clock.advance(301)
+    assert h.handle_alert(alert()).action == "restart_vm"
+    clock.advance(901)
+    out = h.handle_alert(alert())
+    assert out.action == "mark_slot"
+    slot = sysm.registry.slot("tier1-general")
+    assert slot.status == "degraded" and slot.set_by == "healer"
+    out = h.handle_alert(alert())
+    assert out.status == "exhausted"
+    assert h.handle_alert(alert()).message == "bereits gemeldet"
+    assert [c[0] for c in dry.calls].count("restart_container") == 2
+
+    out = h.handle_alert(alert(status="resolved"))
+    assert out.status == "resolved"
+    assert sysm.registry.slot("tier1-general").status == "ok"
+
+
+def test_healer_does_not_clear_human_degrade(make_system):
+    sysm = make_system()
+    sysm.registry.set_status("tier1-general", "degraded", set_by="human")
+    sysm.healer.handle_alert(alert())
+    sysm.healer.handle_alert(alert(status="resolved"))
+    assert sysm.registry.slot("tier1-general").status == "degraded"
+
+
+def test_healer_approval_step_creates_proposal(make_system, clock):
+    sysm = make_system()
+    a = alert(alertname="GpuXidError")
+    assert sysm.healer.handle_alert(a).action == "mark_slot"
+    assert sysm.healer.handle_alert(a).action == "restart_vm"
+    clock.advance(1201)
+    out = sysm.healer.handle_alert(a)
+    assert out.status == "proposed" and out.action == "reboot_host"
+    props = sysm.maintenance.proposals()
+    assert props[0].kind == "reboot_host" and props[0].source == "healer"
+
+
+def test_healer_missing_label_and_unmatched(make_system):
+    sysm = make_system()
+    a = alert()
+    del a["labels"]["container"]
+    out = sysm.healer.handle_alert(a)
+    assert out.status == "skipped" and "container" in out.message
+    assert sysm.healer.handle_alert({"status": "firing", "labels": {"alertname": "Neu"}}).status == "unmatched"
+
+
+def test_healer_throttle(make_system, clock):
+    sysm = make_system()
+    sysm.healer.config.max_actions_per_hour = 2
+    for i in range(2):
+        sysm.healer.handle_alert({**alert(), "fingerprint": f"fp{i}"})
+    out = sysm.healer.handle_alert({**alert(), "fingerprint": "fp9"})
+    assert out.status == "throttled"
+
+
+def test_infra_actions_need_explicit_enable(make_system):
+    from knaster.healer.drivers import DryRunDriver
+
+    real = DryRunDriver()
+    sysm = make_system(infra=real)
+    sysm.actions.dry_run = False
+    sysm.healer.handle_alert(alert())
+    assert real.calls == []  # nicht in enabled_actions
+    sysm.actions.enabled_infra_actions = {"restart_container"}
+    sysm.healer.handle_alert({**alert(), "fingerprint": "other"})
+    assert real.calls[0][0] == "restart_container"
+
+
+def test_alertmanager_payload(make_system):
+    sysm = make_system()
+    outs = sysm.healer.handle_alertmanager({"version": "4", "alerts": [alert(), alert(alertname="NodeDown",
+                                                                                    instance="pve2")]})
+    assert [o.status for o in outs] == ["executed", "executed"]