cluster-knaster/core/tests/test_execution.py

import json

import pytest

from knaster.calibration import effective_context
from knaster.cases import export_preferences, export_sft, split_for
from knaster.classifier import EmbeddingClassifier
from knaster.executor import Executor
from knaster.llm import ChatResult, ContextOverflowError, EndpointError
from knaster.problems import Signal, extract_json, is_degenerate, repetition_ratio
from knaster.router import Router
from knaster.runner import Runner
from knaster.verifier import Verifier

from .test_router import RECHNUNG

GOOD = json.dumps({"nummer": "4711", "betrag": 99})


@pytest.fixture
def classifiers(embedder):
    examples = []
    for label, texts in {
        "drucker": ["Drucker druckt nicht", "Drucker Papierstau", "Drucker offline"],
        "netzwerk": ["WLAN geht nicht", "Netzwerk langsam", "kein Internet"],
    }.items():
        examples += [(v, label) for v in embedder.embed(texts)]
    return {"ticket-kat": EmbeddingClassifier.train(examples)}


@pytest.fixture
def make_runner(registry, library, embedder, store, fake_chat, classifiers):
    def make(**handlers):
        fake_chat.handlers.update(handlers)
        router = Router(registry, library, embedder)
        executor = Executor(library, fake_chat, embedder, classifiers)
        verifier = Verifier(registry, fake_chat)
        return Runner(router, executor, verifier, library, store)

    return make


def test_weak_success_records_case(make_runner, store):
    runner = make_runner(**{"tier1-general/rechnungen-v1": lambda m: GOOD})
    res = runner.run(RECHNUNG)
    assert res.status == "ok" and res.stage == 1 and res.escalations == 0
    cases = store.list_cases()
    assert len(cases) == 1 and cases[0]["passed"] is True and cases[0]["cluster"] == "rechnungen"
    assert cases[0]["messages"][0]["role"] == "system"


def test_schema_failure_escalates_and_creates_pair(make_runner, store):
    runner = make_runner(**{
        "tier1-general/rechnungen-v1": lambda m: json.dumps({"nummer": "4711"}),  # betrag fehlt
        "tier0-strong": lambda m: GOOD,
    })
    res = runner.run(RECHNUNG)
    assert res.status == "ok" and res.stage == 0 and res.escalations == 1
    pairs = store.list_preference_pairs()
    assert len(pairs) == 1
    assert pairs[0]["chosen"] == GOOD and "betrag" not in pairs[0]["rejected"]
    assert {c["stage"]: c["passed"] for c in store.list_cases()} == {1: False, 0: True}


def test_invalid_json_signal(make_runner, store):
    runner = make_runner(**{"tier1-general/rechnungen-v1": lambda m: "kein json", "tier0-strong": lambda m: GOOD})
    res = runner.run(RECHNUNG)
    assert Signal.INVALID_FORMAT in res.attempts[0].execution.signals
    assert res.stage == 0
    assert store.count_events("invalid_format") == 1


def test_endpoint_error_uses_fallback_slot_without_case(make_runner, store):
    def boom(m):
        raise EndpointError("down")

    runner = make_runner(**{"tier1-general/rechnungen-v1": boom})
    # Fallback-Slot tier1-flex hat dieselbe Basis → gleicher Adapter-Name; daher Fehler nur einmal
    calls = {"n": 0}

    def flaky(m):
        calls["n"] += 1
        if calls["n"] == 1:
            raise EndpointError("down")
        return GOOD

    runner.executor.chat.handlers["tier1-general/rechnungen-v1"] = flaky
    runner.executor.chat.handlers["tier1-flex/rechnungen-v1"] = lambda m: GOOD
    res = runner.run(RECHNUNG)
    assert res.status == "ok"
    assert [a.decision.slot for a in res.attempts] == ["tier1-general", "tier1-flex"]
    assert len(store.list_cases()) == 1  # Infrastrukturfehler erzeugt keinen Fall
    assert store.list_preference_pairs() == []


def test_context_overflow_detected(make_runner):
    def overflow(m):
        raise ContextOverflowError("maximum context length")

    runner = make_runner(**{"tier1-general/rechnungen-v1": overflow, "tier0-strong": lambda m: GOOD})
    res = runner.run(RECHNUNG)
    assert res.attempts[0].execution.signals == [Signal.CONTEXT_OVERFLOW]
    assert res.status == "ok"


def test_truncated_and_degenerate(make_runner):
    loop = " ".join(["immer wieder das gleiche"] * 30)
    runner = make_runner(**{
        "tier1-general/rechnungen-v1": lambda m: ChatResult(content=loop, finish_reason="length"),
        "tier0-strong": lambda m: GOOD,
    })
    res = runner.run(RECHNUNG)
    sig = res.attempts[0].execution.signals
    assert Signal.OUTPUT_TRUNCATED in sig and Signal.DEGENERATE_OUTPUT in sig


def test_total_failure(make_runner):
    runner = make_runner(**{"tier1-general/rechnungen-v1": lambda m: "x", "tier0-strong": lambda m: "y"})
    res = runner.run(RECHNUNG)
    assert res.status == "failed"


def test_classifier_stage(make_runner):
    runner = make_runner()
    res = runner.run("Ticket: Drucker druckt nicht Papierstau")
    assert res.status == "ok" and res.stage == 3 and res.output == "drucker"


def test_classifier_low_confidence_escalates(make_runner):
    runner = make_runner(**{"tier1-general": lambda m: "drucker"})
    res = runner.run("Ticket einer Kategorie zuordnen: Monitor flackert")
    assert res.attempts[0].execution.signals == [Signal.CLASSIFIER_LOW_CONFIDENCE]
    assert res.attempts[1].decision.stage == 1


def test_program_stage(make_runner):
    res = make_runner().run("Text in Großbuchstaben: hallo welt")
    assert res.output == "TEXT IN GROSSBUCHSTABEN: HALLO WELT"


def test_judge_check(registry, library, fake_chat):
    skill = library.get("vertrag-pruefen").model_copy(deep=True)
    skill.meta.verifier = [{"type": "judge", "rubric": "muss Frist nennen"}]
    v = Verifier(registry, fake_chat)
    fake_chat.handlers["verifier-small"] = lambda m: "FAIL: Frist fehlt"
    r = v.verify(skill, "Vertrag", "Antwort")
    assert not r.passed and r.failed_checks[0].detail.startswith("FAIL")
    fake_chat.handlers["verifier-small"] = lambda m: "PASS"
    assert v.verify(skill, "Vertrag", "Antwort").passed


def test_judge_unavailable_is_inconclusive(registry, library, fake_chat):
    skill = library.get("vertrag-pruefen").model_copy(deep=True)
    skill.meta.verifier = [{"type": "judge"}, {"type": "contains", "value": "Frist"}]
    registry.set_status("verifier-small", "unavailable")
    r = Verifier(registry, fake_chat).verify(skill, "Vertrag", "Frist 3 Monate")
    assert r.passed and r.inconclusive


def test_inconclusive_case_not_used_for_training(make_runner, registry, library, store):
    library.set_state("vertrag-pruefen", status="active")
    registry.set_status("verifier-small", "unavailable")
    skill_file = library._skills["vertrag-pruefen"]
    skill_file.meta.verifier = [{"type": "judge"}]
    runner = make_runner(**{"tier1-general": lambda m: "Kündigungsfrist: 3 Monate"})
    res = runner.run("Vertrag prüfen auf Kündigungsfrist")
    assert res.status == "ok"
    assert store.list_cases()[0]["passed"] is None


def test_verifier_rules():
    from knaster.skills import KnasterMeta, Skill

    skill = Skill(name="s", description="d", body="b", meta=KnasterMeta(verifier=[
        {"type": "regex", "pattern": r"^\d+$"}, {"type": "not_contains", "value": "Fehler"},
        {"type": "max_chars", "value": 5}, {"type": "unbekannt"}]))
    r = Verifier().verify(skill, "t", "123")
    assert [c.passed for c in r.checks] == [True, True, True, False]


def test_problem_helpers():
    assert extract_json('```json\n{"a": 1}\n```') == {"a": 1}
    with pytest.raises(ValueError):
        extract_json("{a:")
    assert repetition_ratio("a b c d e f") == 0
    assert is_degenerate(" ".join(["la le lu"] * 20))
    assert not is_degenerate("kurz kurz kurz kurz")


def test_split_is_deterministic():
    assert split_for("abc") == split_for("abc")
    splits = {split_for(f"text {i}") for i in range(100)}
    assert splits == {"train", "eval"}


def test_exports(make_runner, store):
    runner = make_runner(**{"tier1-general/rechnungen-v1": lambda m: '{"nummer": "1"}', "tier0-strong": lambda m: GOOD})
    for i in range(30):
        runner.run(f"{RECHNUNG} Nr {i}")
    train = list(export_sft(store, cluster="rechnungen", split="train"))
    evals = list(export_sft(store, cluster="rechnungen", split="eval"))
    assert len(train) + len(evals) == 30
    row = json.loads(train[0])
    assert row["messages"][-1] == {"role": "assistant", "content": GOOD}
    assert row["meta"]["stage"] == 0
    prefs = list(export_preferences(store, cluster="rechnungen"))
    assert prefs and json.loads(prefs[0])["chosen"] == GOOD


def test_effective_context_calibration():
    obs = [(1000, True)] * 30 + [(3000, True)] * 28 + [(3000, False)] * 2 + [(6000, True)] * 15 + \
        [(6000, False)] * 10 + [(12000, True)] * 5
    assert effective_context(obs, target=0.9, min_n=20) == 4096
    assert effective_context([(1000, True)] * 5) is None
    # Buckets ohne genug Daten unterbrechen nicht
    assert effective_context([(1000, True)] * 25 + [(20000, True)] * 25, min_n=20) == 32768


def test_classifier_persistence(tmp_path, classifiers, embedder):
    clf = classifiers["ticket-kat"]
    clf.save(tmp_path / "c.json")
    loaded = EmbeddingClassifier.load(tmp_path / "c.json")
    assert loaded.predict(embedder.embed(["Drucker"])[0])[0] == "drucker"