156 lines
5.5 KiB
Python
156 lines
5.5 KiB
Python
import pytest
|
|
|
|
from knaster.problems import Signal
|
|
from knaster.router import Router, RouterConfig
|
|
|
|
RECHNUNG = "Rechnung auslesen: Rechnungsnummer und Betrag extrahieren. Rechnung 4711 über 99 EUR"
|
|
|
|
|
|
@pytest.fixture
|
|
def router(registry, library, embedder):
|
|
return Router(registry, library, embedder)
|
|
|
|
|
|
def test_routes_to_weak_slot_with_adapter(router):
|
|
d = router.route(RECHNUNG)
|
|
assert d.status == "routed"
|
|
assert (d.stage, d.slot, d.skill) == (1, "tier1-general", "rechnung-extrahieren")
|
|
assert d.model == "tier1-general/rechnungen-v1" and d.adapter == "rechnungen-v1"
|
|
assert d.prompt_variant == "full"
|
|
|
|
|
|
def test_unknown_request_goes_to_strong(router):
|
|
d = router.route("Wie ist das Wetter morgen in Berlin?")
|
|
assert (d.stage, d.slot, d.skill) == (0, "tier0-strong", None)
|
|
assert Signal.NO_SKILL in d.signals
|
|
|
|
|
|
def test_draft_skill_not_routed(router):
|
|
d = router.route("Gedicht über Rechnung schreiben")
|
|
assert d.skill != "entwurf"
|
|
|
|
|
|
def test_demoted_skill_runs_on_strong_with_hint(router):
|
|
d = router.route("Vertrag prüfen auf Kündigungsfrist")
|
|
assert (d.stage, d.skill, d.prompt_variant) == (0, "vertrag-pruefen", "full")
|
|
|
|
|
|
def test_classifier_and_program_stages(router):
|
|
assert router.route("Ticket: Drucker druckt nicht").stage == 3
|
|
d = router.route("Text in Großbuchstaben: hallo welt")
|
|
assert (d.stage, d.slot) == (2, None)
|
|
|
|
|
|
def test_unavailable_slot_uses_fallback(router, registry):
|
|
registry.set_status("tier1-general", "degraded")
|
|
d = router.route(RECHNUNG)
|
|
assert d.slot == "tier1-flex"
|
|
assert Signal.SLOT_UNAVAILABLE in d.signals
|
|
|
|
|
|
def test_all_weak_slots_unavailable_goes_strong(router, registry):
|
|
registry.set_status("tier1-general", "unavailable")
|
|
registry.set_status("tier1-flex", "unavailable")
|
|
d = router.route(RECHNUNG)
|
|
assert d.stage == 0 and d.skill == "rechnung-extrahieren"
|
|
|
|
|
|
def test_long_input_uses_compact_then_larger_slot(router):
|
|
# Eingabe ~1585 Tokens: mit Beispielen (64) zu groß für 2000, kompakt (38) passt es
|
|
filler = "x" * (3 * 1585 - len(RECHNUNG) - 1)
|
|
d = router.route(RECHNUNG + " " + filler)
|
|
assert d.slot == "tier1-general" and d.prompt_variant == "compact"
|
|
assert Signal.COMPACT_PROMPT in d.signals
|
|
|
|
d = router.route(RECHNUNG + " " + "x" * (3 * 5000))
|
|
assert d.slot == "tier1-flex"
|
|
assert Signal.LARGER_SLOT in d.signals
|
|
|
|
|
|
def test_too_large_for_everything(router):
|
|
d = router.route(RECHNUNG + " " + "x" * (3 * 120000))
|
|
assert d.status == "too_large"
|
|
assert Signal.NEEDS_SPLIT in d.signals
|
|
|
|
|
|
def test_context_exceeded_goes_to_strong(router):
|
|
d = router.route(RECHNUNG + " " + "x" * (3 * 30000))
|
|
assert d.stage == 0 and Signal.CONTEXT_EXCEEDED in d.signals
|
|
|
|
|
|
def test_hint_skill(router):
|
|
d = router.route("irgendwas", hint_skill="rechnung-extrahieren")
|
|
assert d.skill == "rechnung-extrahieren" and d.similarity is None
|
|
with pytest.raises(KeyError):
|
|
router.route("x", hint_skill="entwurf")
|
|
|
|
|
|
def test_escalation_quality_goes_to_strong(router):
|
|
d = router.route(RECHNUNG)
|
|
e = router.escalate(d, Signal.VERIFIER_FAILED, RECHNUNG)
|
|
assert (e.stage, e.slot, e.attempt) == (0, "tier0-strong", 1)
|
|
f = router.escalate(e, Signal.VERIFIER_FAILED, RECHNUNG)
|
|
assert f.status == "failed"
|
|
|
|
|
|
def test_escalation_infra_uses_fallback_slot(router):
|
|
d = router.route(RECHNUNG)
|
|
e = router.escalate(d, Signal.ENDPOINT_ERROR, RECHNUNG)
|
|
assert (e.stage, e.slot) == (1, "tier1-flex")
|
|
f = router.escalate(e, Signal.ENDPOINT_ERROR, RECHNUNG)
|
|
assert f.stage == 0
|
|
|
|
|
|
def test_escalation_from_program_to_weak(router):
|
|
d = router.route("Text in Großbuchstaben: hallo welt")
|
|
e = router.escalate(d, Signal.PROGRAM_ERROR, "Text in Großbuchstaben: hallo welt")
|
|
assert e.stage == 1
|
|
|
|
|
|
def test_max_escalations(registry, library, embedder):
|
|
r = Router(registry, library, embedder, config=RouterConfig(max_escalations=1))
|
|
d = r.route(RECHNUNG)
|
|
e = r.escalate(d, Signal.ENDPOINT_ERROR, RECHNUNG)
|
|
assert r.escalate(e, Signal.ENDPOINT_ERROR, RECHNUNG).status == "failed"
|
|
|
|
|
|
class PreferRechnung:
|
|
def rerank(self, query, documents):
|
|
return [1.0 if "Rechnung" in d else 0.0 for d in documents]
|
|
|
|
|
|
def test_conflict_without_reranker_goes_strong(registry, library, embedder):
|
|
r = Router(registry, library, embedder, config=RouterConfig(conflict_margin=1.0))
|
|
d = r.route(RECHNUNG)
|
|
assert d.stage == 0 and Signal.SKILL_CONFLICT in d.signals
|
|
|
|
|
|
def test_conflict_with_reranker_trusts_reranker(registry, library, embedder):
|
|
r = Router(registry, library, embedder, reranker=PreferRechnung(), config=RouterConfig(conflict_margin=1.0))
|
|
d = r.route("Vertrag prüfen Kündigungsfrist. Rechnung auslesen Rechnungsnummer Betrag")
|
|
assert Signal.SKILL_CONFLICT in d.signals
|
|
assert "Reranking aktiv" in d.reasons
|
|
assert d.skill == "rechnung-extrahieren" and d.stage == 1
|
|
|
|
|
|
class FlakyEmbedder:
|
|
def __init__(self, inner):
|
|
self.inner = inner
|
|
self.down = True
|
|
|
|
def embed(self, texts):
|
|
if self.down:
|
|
raise ConnectionError("embed-default weg")
|
|
return self.inner.embed(texts)
|
|
|
|
|
|
def test_embedding_outage_degrades_to_strong_and_recovers(registry, library, embedder):
|
|
flaky = FlakyEmbedder(embedder)
|
|
r = Router(registry, library, flaky) # Start ohne Embedding darf nicht abstürzen
|
|
assert r.index_error
|
|
d = r.route(RECHNUNG)
|
|
assert d.stage == 0 and any("Embedding nicht verfügbar" in x for x in d.reasons)
|
|
flaky.down = False
|
|
d = r.route(RECHNUNG)
|
|
assert d.stage == 1 and r.index_error is None
|