import json import pytest from knaster.calibration import effective_context from knaster.cases import export_preferences, export_sft, split_for from knaster.classifier import EmbeddingClassifier from knaster.executor import Executor from knaster.llm import ChatResult, ContextOverflowError, EndpointError from knaster.problems import Signal, extract_json, is_degenerate, repetition_ratio from knaster.router import Router from knaster.runner import Runner from knaster.verifier import Verifier from .test_router import RECHNUNG GOOD = json.dumps({"nummer": "4711", "betrag": 99}) @pytest.fixture def classifiers(embedder): examples = [] for label, texts in { "drucker": ["Drucker druckt nicht", "Drucker Papierstau", "Drucker offline"], "netzwerk": ["WLAN geht nicht", "Netzwerk langsam", "kein Internet"], }.items(): examples += [(v, label) for v in embedder.embed(texts)] return {"ticket-kat": EmbeddingClassifier.train(examples)} @pytest.fixture def make_runner(registry, library, embedder, store, fake_chat, classifiers): def make(**handlers): fake_chat.handlers.update(handlers) router = Router(registry, library, embedder) executor = Executor(library, fake_chat, embedder, classifiers) verifier = Verifier(registry, fake_chat) return Runner(router, executor, verifier, library, store) return make def test_weak_success_records_case(make_runner, store): runner = make_runner(**{"rechnungen-v1": lambda m: GOOD}) res = runner.run(RECHNUNG) assert res.status == "ok" and res.stage == 1 and res.escalations == 0 cases = store.list_cases() assert len(cases) == 1 and cases[0]["passed"] is True and cases[0]["cluster"] == "rechnungen" assert cases[0]["messages"][0]["role"] == "system" def test_schema_failure_escalates_and_creates_pair(make_runner, store): runner = make_runner(**{ "rechnungen-v1": lambda m: json.dumps({"nummer": "4711"}), # betrag fehlt "tier0-strong": lambda m: GOOD, }) res = runner.run(RECHNUNG) assert res.status == "ok" and res.stage == 0 and res.escalations == 1 pairs = store.list_preference_pairs() assert len(pairs) == 1 assert pairs[0]["chosen"] == GOOD and "betrag" not in pairs[0]["rejected"] assert {c["stage"]: c["passed"] for c in store.list_cases()} == {1: False, 0: True} def test_invalid_json_signal(make_runner, store): runner = make_runner(**{"rechnungen-v1": lambda m: "kein json", "tier0-strong": lambda m: GOOD}) res = runner.run(RECHNUNG) assert Signal.INVALID_FORMAT in res.attempts[0].execution.signals assert res.stage == 0 assert store.count_events("invalid_format") == 1 def test_endpoint_error_uses_fallback_slot_without_case(make_runner, store): def boom(m): raise EndpointError("down") runner = make_runner(**{"rechnungen-v1": boom}) # Fallback-Slot tier1-flex hat dieselbe Basis → gleicher Adapter-Name; daher Fehler nur einmal calls = {"n": 0} def flaky(m): calls["n"] += 1 if calls["n"] == 1: raise EndpointError("down") return GOOD runner.executor.chat.handlers["rechnungen-v1"] = flaky res = runner.run(RECHNUNG) assert res.status == "ok" assert [a.decision.slot for a in res.attempts] == ["tier1-general", "tier1-flex"] assert len(store.list_cases()) == 1 # Infrastrukturfehler erzeugt keinen Fall assert store.list_preference_pairs() == [] def test_context_overflow_detected(make_runner): def overflow(m): raise ContextOverflowError("maximum context length") runner = make_runner(**{"rechnungen-v1": overflow, "tier0-strong": lambda m: GOOD}) res = runner.run(RECHNUNG) assert res.attempts[0].execution.signals == [Signal.CONTEXT_OVERFLOW] assert res.status == "ok" def test_truncated_and_degenerate(make_runner): loop = " ".join(["immer wieder das gleiche"] * 30) runner = make_runner(**{ "rechnungen-v1": lambda m: ChatResult(content=loop, finish_reason="length"), "tier0-strong": lambda m: GOOD, }) res = runner.run(RECHNUNG) sig = res.attempts[0].execution.signals assert Signal.OUTPUT_TRUNCATED in sig and Signal.DEGENERATE_OUTPUT in sig def test_total_failure(make_runner): runner = make_runner(**{"rechnungen-v1": lambda m: "x", "tier0-strong": lambda m: "y"}) res = runner.run(RECHNUNG) assert res.status == "failed" def test_classifier_stage(make_runner): runner = make_runner() res = runner.run("Ticket: Drucker druckt nicht Papierstau") assert res.status == "ok" and res.stage == 3 and res.output == "drucker" def test_classifier_low_confidence_escalates(make_runner): runner = make_runner(**{"tier1-general": lambda m: "drucker"}) res = runner.run("Ticket einer Kategorie zuordnen: Monitor flackert") assert res.attempts[0].execution.signals == [Signal.CLASSIFIER_LOW_CONFIDENCE] assert res.attempts[1].decision.stage == 1 def test_program_stage(make_runner): res = make_runner().run("Text in Großbuchstaben: hallo welt") assert res.output == "TEXT IN GROSSBUCHSTABEN: HALLO WELT" def test_judge_check(registry, library, fake_chat): skill = library.get("vertrag-pruefen").model_copy(deep=True) skill.meta.verifier = [{"type": "judge", "rubric": "muss Frist nennen"}] v = Verifier(registry, fake_chat) fake_chat.handlers["verifier-small"] = lambda m: "FAIL: Frist fehlt" r = v.verify(skill, "Vertrag", "Antwort") assert not r.passed and r.failed_checks[0].detail.startswith("FAIL") fake_chat.handlers["verifier-small"] = lambda m: "PASS" assert v.verify(skill, "Vertrag", "Antwort").passed def test_judge_unavailable_is_inconclusive(registry, library, fake_chat): skill = library.get("vertrag-pruefen").model_copy(deep=True) skill.meta.verifier = [{"type": "judge"}, {"type": "contains", "value": "Frist"}] registry.set_status("verifier-small", "unavailable") r = Verifier(registry, fake_chat).verify(skill, "Vertrag", "Frist 3 Monate") assert r.passed and r.inconclusive def test_inconclusive_case_not_used_for_training(make_runner, registry, library, store): library.set_state("vertrag-pruefen", status="active") registry.set_status("verifier-small", "unavailable") skill_file = library._skills["vertrag-pruefen"] skill_file.meta.verifier = [{"type": "judge"}] runner = make_runner(**{"tier1-general": lambda m: "Kündigungsfrist: 3 Monate"}) res = runner.run("Vertrag prüfen auf Kündigungsfrist") assert res.status == "ok" assert store.list_cases()[0]["passed"] is None def test_verifier_rules(): from knaster.skills import KnasterMeta, Skill skill = Skill(name="s", description="d", body="b", meta=KnasterMeta(verifier=[ {"type": "regex", "pattern": r"^\d+$"}, {"type": "not_contains", "value": "Fehler"}, {"type": "max_chars", "value": 5}, {"type": "unbekannt"}])) r = Verifier().verify(skill, "t", "123") assert [c.passed for c in r.checks] == [True, True, True, False] def test_problem_helpers(): assert extract_json('```json\n{"a": 1}\n```') == {"a": 1} with pytest.raises(ValueError): extract_json("{a:") assert repetition_ratio("a b c d e f") == 0 assert is_degenerate(" ".join(["la le lu"] * 20)) assert not is_degenerate("kurz kurz kurz kurz") def test_split_is_deterministic(): assert split_for("abc") == split_for("abc") splits = {split_for(f"text {i}") for i in range(100)} assert splits == {"train", "eval"} def test_exports(make_runner, store): runner = make_runner(**{"rechnungen-v1": lambda m: '{"nummer": "1"}', "tier0-strong": lambda m: GOOD}) for i in range(30): runner.run(f"{RECHNUNG} Nr {i}") train = list(export_sft(store, cluster="rechnungen", split="train")) evals = list(export_sft(store, cluster="rechnungen", split="eval")) assert len(train) + len(evals) == 30 row = json.loads(train[0]) assert row["messages"][-1] == {"role": "assistant", "content": GOOD} assert row["meta"]["stage"] == 0 prefs = list(export_preferences(store, cluster="rechnungen")) assert prefs and json.loads(prefs[0])["chosen"] == GOOD def test_effective_context_calibration(): obs = [(1000, True)] * 30 + [(3000, True)] * 28 + [(3000, False)] * 2 + [(6000, True)] * 15 + \ [(6000, False)] * 10 + [(12000, True)] * 5 assert effective_context(obs, target=0.9, min_n=20) == 4096 assert effective_context([(1000, True)] * 5) is None # Buckets ohne genug Daten unterbrechen nicht assert effective_context([(1000, True)] * 25 + [(20000, True)] * 25, min_n=20) == 32768 def test_classifier_persistence(tmp_path, classifiers, embedder): clf = classifiers["ticket-kat"] clf.save(tmp_path / "c.json") loaded = EmbeddingClassifier.load(tmp_path / "c.json") assert loaded.predict(embedder.embed(["Drucker"])[0])[0] == "drucker"