211 lines
9.2 KiB
Python
211 lines
9.2 KiB
Python
from datetime import datetime
|
|
|
|
import pytest
|
|
|
|
from knaster.maintenance.agent import parse_command
|
|
|
|
|
|
@pytest.mark.parametrize("text,name,args", [
|
|
("Status", "status", {}),
|
|
("starte die Service-Routine", "routine", {}),
|
|
("zeige vorschläge", "list", {}),
|
|
("genehmige P-3", "approve", {"ids": "P-3"}),
|
|
("genehmige P-3, P-5 und p7", "approve", {"ids": "P-3, P-5 und p7"}),
|
|
("approve p-2", "approve", {"ids": "p-2"}),
|
|
("genehmige alle mit Risiko niedrig", "approve_low", {}),
|
|
("lehne P-4 ab: zu teuer", "reject", {"id": "P-4", "reason": "zu teuer"}),
|
|
("lehne P-4 ab", "reject", {"id": "P-4"}),
|
|
("wechsle tier1-general auf Qwen/Qwen3-32B-AWQ", "switch_base",
|
|
{"slot": "tier1-general", "base": "Qwen/Qwen3-32B-AWQ"}),
|
|
("trainiere einen Adapter für den Cluster rechnungen", "train_adapter", {"cluster": "rechnungen"}),
|
|
("rollback tier1-general", "rollback", {"slot": "tier1-general"}),
|
|
("stufe rechnung-extrahieren zurück", "demote", {"skill": "rechnung-extrahieren"}),
|
|
("markiere tier1-flex als unavailable", "mark", {"slot": "tier1-flex", "status": "unavailable"}),
|
|
("ja", "confirm", {}),
|
|
])
|
|
def test_parse_command(text, name, args):
|
|
parsed = parse_command(text)
|
|
assert parsed is not None and parsed[0] == name
|
|
for k, v in args.items():
|
|
assert parsed[1][k] == v
|
|
|
|
|
|
def test_parse_unknown():
|
|
assert parse_command("mach mal was schönes") is None
|
|
|
|
|
|
def test_command_creates_proposal_and_needs_approval(make_system):
|
|
sysm = make_system()
|
|
m = sysm.maintenance
|
|
reply = m.handle_command("stufe rechnung-extrahieren zurück", user="robert")
|
|
assert "Bestätige" in reply.text
|
|
p = reply.proposals[0]
|
|
assert p.state == "open" # obwohl Richtlinie auto sagt: Prompt-Aufträge brauchen Bestätigung
|
|
assert sysm.library.get("rechnung-extrahieren").meta.status == "active"
|
|
reply = m.handle_command(f"genehmige {p.id}", user="robert")
|
|
assert "executed" in reply.text
|
|
assert sysm.library.get("rechnung-extrahieren").meta.status == "demoted"
|
|
assert m._load(p.seq).decided_by == "robert"
|
|
|
|
|
|
def test_reject_and_errors(make_system):
|
|
m = make_system().maintenance
|
|
p = m.handle_command("markiere tier1-flex als degraded").proposals[0]
|
|
assert "rejected" in m.handle_command(f"lehne {p.id} ab: nicht nötig").text
|
|
assert "Fehler" in m.handle_command(f"genehmige {p.id}").text
|
|
assert "Fehler" in m.handle_command("markiere gibtsnicht als ok").text
|
|
assert "Fehler" in m.handle_command("genehmige P-999").text
|
|
|
|
|
|
def test_switch_base_runs_pipeline_after_approval(make_system):
|
|
sysm = make_system()
|
|
m = sysm.maintenance
|
|
p = m.handle_command("wechsle tier1-general auf base-32b").proposals[0]
|
|
m.handle_command(f"genehmige {p.id}")
|
|
cands = sysm.store.list_candidates()
|
|
assert cands[0]["kind"] == "base_model" and cands[0]["state"] == "building"
|
|
assert sysm.lifecycle.jobs.jobs[0][0] == "build"
|
|
|
|
|
|
def test_intent_mapper_requires_confirmation(make_system):
|
|
class Mapper:
|
|
def map(self, text):
|
|
return "stufe vertrag-pruefen zurück" if "vertrag" in text else None
|
|
|
|
sysm = make_system(intent_mapper=Mapper())
|
|
m = sysm.maintenance
|
|
reply = m.handle_command("der vertrag skill macht nur mist", session="s1")
|
|
assert "Meinst du" in reply.text
|
|
assert sysm.store.list_proposals() == []
|
|
reply = m.handle_command("ja", session="s1")
|
|
assert reply.proposals and reply.proposals[0].state == "open"
|
|
assert "nicht verstanden" in m.handle_command("blabla").text
|
|
assert "nichts zu bestätigen" in m.handle_command("ja", session="s1").text
|
|
|
|
|
|
def test_llm_intent_mapper(fake_chat):
|
|
from knaster.maintenance.agent import LLMIntentMapper
|
|
|
|
fake_chat.handlers["tier1-general"] = lambda msgs: "rollback tier1-general"
|
|
assert LLMIntentMapper(fake_chat).map("mach das letzte update rückgängig") == "rollback tier1-general"
|
|
fake_chat.handlers["tier1-general"] = lambda msgs: "NONE"
|
|
assert LLMIntentMapper(fake_chat).map("hallo") is None
|
|
|
|
|
|
def test_deferred_until_window(make_system):
|
|
sysm = make_system(wallclock=lambda: datetime(2026, 1, 1, 12, 0))
|
|
m = sysm.maintenance
|
|
p = m.propose("reboot_host", {"pve_node": "pve1"}, "Host neu starten", source="healer")
|
|
p = m.approve(p.seq)
|
|
assert p.state == "approved" and p.result["deferred"]
|
|
assert m.run_deferred() == []
|
|
m.wallclock = lambda: datetime(2026, 1, 2, 3, 0)
|
|
done = m.run_deferred()
|
|
assert done[0].state == "executed" and "[dry-run] reboot_host" in done[0].result["message"]
|
|
|
|
|
|
def test_dedup_of_open_proposals(make_system):
|
|
m = make_system().maintenance
|
|
a = m.propose("merge_skills", {"a": "x", "b": "y"}, "merge")
|
|
b = m.propose("merge_skills", {"a": "x", "b": "y"}, "merge")
|
|
assert a.seq == b.seq
|
|
|
|
|
|
def _cases(store, n, *, skill, stage, passed_ratio, slot="tier1-general", cluster=None, split="train",
|
|
tokens=None):
|
|
for i in range(n):
|
|
store.add_case(input=f"{skill}-{stage}-{i}", skill=skill, cluster=cluster or skill, stage=stage, slot=slot,
|
|
passed=i < n * passed_ratio, split=split, origin="route", prompt_tokens=tokens)
|
|
|
|
|
|
def test_routine_drift_auto_demotes(make_system):
|
|
sysm = make_system()
|
|
_cases(sysm.store, 30, skill="rechnung-extrahieren", cluster="rechnungen", stage=1, passed_ratio=0.5)
|
|
report = sysm.maintenance.run_routine()
|
|
assert any("Drift" in f for f in report.findings)
|
|
assert any("rechnung-extrahieren" in a for a in report.auto_executed)
|
|
assert sysm.library.get("rechnung-extrahieren").meta.status == "demoted"
|
|
# Router sieht die Änderung sofort
|
|
from .test_router import RECHNUNG
|
|
assert sysm.router.route(RECHNUNG).stage == 0
|
|
|
|
|
|
def test_routine_graduation_candidate(make_system):
|
|
sysm = make_system()
|
|
_cases(sysm.store, 25, skill="vertrag-pruefen", stage=0, passed_ratio=1.0, slot="tier0-strong")
|
|
report = sysm.maintenance.run_routine()
|
|
kinds = {p.kind for p in report.proposals}
|
|
assert "request_graduation" in kinds
|
|
p = next(p for p in report.proposals if p.kind == "request_graduation")
|
|
sysm.maintenance.approve(p.seq)
|
|
assert sysm.lifecycle.jobs.jobs[-1][0] == "graduate_skill"
|
|
|
|
|
|
def test_routine_degraded_slot_and_context(make_system, clock):
|
|
sysm = make_system()
|
|
sysm.registry.set_status("tier1-flex", "unavailable", reason="Training")
|
|
for _ in range(12):
|
|
sysm.store.add_event("compact_prompt", subject="tier1-general", ts=clock())
|
|
clock.advance(2 * 3600)
|
|
_cases(sysm.store, 30, skill="rechnung-extrahieren", stage=1, passed_ratio=1.0, cluster="rechnungen",
|
|
tokens=1000)
|
|
report = sysm.maintenance.run_routine()
|
|
assert any("tier1-flex" in f for f in report.findings)
|
|
assert any(p.kind == "mark_slot" for p in report.proposals)
|
|
assert any("effektiver Kontext = 2048" in a for a in report.auto_executed)
|
|
assert sysm.registry.slot("tier1-general").calibrated_context == 2048
|
|
assert "Offene Vorschläge" in report.text
|
|
|
|
|
|
def test_routine_adapter_need(make_system):
|
|
sysm = make_system()
|
|
sysm.maintenance.config.adapter_min_cases = 20
|
|
_cases(sysm.store, 40, skill="rechnung-extrahieren", stage=0, passed_ratio=1.0, cluster="rechnungen",
|
|
slot="tier0-strong")
|
|
_cases(sysm.store, 10, skill="rechnung-extrahieren", stage=1, passed_ratio=0.7, cluster="rechnungen")
|
|
report = sysm.maintenance.run_routine()
|
|
p = next(p for p in report.proposals if p.kind == "train_adapter")
|
|
assert p.params == {"cluster": "rechnungen", "slot": "tier1-general"}
|
|
sysm.maintenance.approve(p.seq)
|
|
cand = sysm.store.list_candidates()[0]
|
|
assert cand["kind"] == "adapter" and cand["target"]["base_model"] == "base-14b"
|
|
# zweite Routine schlägt nicht erneut vor
|
|
assert not any(p.kind == "train_adapter" for p in sysm.maintenance.run_routine().proposals)
|
|
|
|
|
|
def test_routine_merge_similar_skills(make_system):
|
|
sysm = make_system()
|
|
sysm.maintenance.config.merge_similarity = 0.0
|
|
report = sysm.maintenance.run_routine()
|
|
assert any(p.kind == "merge_skills" for p in report.proposals)
|
|
|
|
|
|
def test_routine_evaluates_canary_and_proposes_promotion(make_system):
|
|
sysm = make_system()
|
|
life = sysm.lifecycle
|
|
life.config.min_n = 5
|
|
cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
|
|
life.start(cid)
|
|
life.report(cid, "build", {"model": "rv2"})
|
|
life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
|
|
life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
|
|
for i in range(6):
|
|
sysm.store.add_case(input=f"c{i}", stage=1, slot="tier1-general", model="rv2", passed=True,
|
|
split="train", origin="route")
|
|
report = sysm.maintenance.run_routine()
|
|
assert any("Canary" in a for a in report.auto_executed)
|
|
p = next(p for p in report.proposals if p.kind == "promote_candidate")
|
|
assert p.risk == "high"
|
|
sysm.maintenance.approve(p.seq)
|
|
assert sysm.registry.active_adapter("base-14b", "rechnungen").name == "rv2"
|
|
# rollback per Prompt
|
|
m = sysm.maintenance
|
|
rp = m.handle_command("rollback tier1-general").proposals[0]
|
|
m.approve(rp.seq)
|
|
assert sysm.registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
|
|
|
|
|
|
def test_status_command(make_system):
|
|
text = make_system().maintenance.handle_command("status").text
|
|
assert "tier1-general: ok" in text and "Skills:" in text
|