core: Registry, Skills, Router, Executor, Verifier, Runner mit Tests
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,178 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import textwrap
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from knaster.embeddings import HashingEmbedder
|
||||
from knaster.llm import ChatResult
|
||||
from knaster.registry import Registry
|
||||
from knaster.skills import SkillLibrary
|
||||
from knaster.store import Store
|
||||
|
||||
REGISTRY = {
|
||||
"slots": {
|
||||
"tier0-strong": {
|
||||
"tier": 0, "device": "external", "engine": "openrouter", "litellm_model": "tier0-strong",
|
||||
"base_model": "strong-x", "max_context": 128000, "effective_context": 100000,
|
||||
},
|
||||
"tier1-general": {
|
||||
"tier": 1, "engine": "vllm", "node": "gpu-main", "litellm_model": "tier1-general",
|
||||
"base_model": "base-14b", "max_context": 8192, "effective_context": 2000,
|
||||
"supports_lora": True, "fallbacks": ["tier1-flex"],
|
||||
},
|
||||
"tier1-flex": {
|
||||
"tier": 1, "engine": "vllm", "node": "gpu-flex", "litellm_model": "tier1-flex",
|
||||
"base_model": "base-14b", "max_context": 32768, "effective_context": 16000,
|
||||
"supports_lora": True,
|
||||
},
|
||||
"verifier-small": {
|
||||
"tier": 1, "engine": "llamacpp", "node": "gpu-small", "litellm_model": "verifier-small",
|
||||
"base_model": "small-4b", "max_context": 8192,
|
||||
},
|
||||
"embed-default": {
|
||||
"tier": 1, "engine": "llamacpp", "node": "gpu-embed", "litellm_model": "embed-default",
|
||||
"base_model": "embed-0.6b", "max_context": 8192, "kind": "embedding",
|
||||
},
|
||||
},
|
||||
"adapters": [
|
||||
{"name": "rechnungen-v1", "base_model": "base-14b", "cluster": "rechnungen", "version": 1},
|
||||
],
|
||||
}
|
||||
|
||||
SKILLS = {
|
||||
"rechnung-extrahieren": """\
|
||||
---
|
||||
name: rechnung-extrahieren
|
||||
description: Rechnung auslesen und Rechnungsnummer Betrag Datum als JSON extrahieren
|
||||
metadata:
|
||||
knaster:
|
||||
status: active
|
||||
stage: 1
|
||||
cluster: rechnungen
|
||||
threshold: 0.3
|
||||
triggers: ["Rechnungsdaten aus einer Rechnung extrahieren"]
|
||||
max_output_tokens: 300
|
||||
output:
|
||||
format: json
|
||||
schema:
|
||||
type: object
|
||||
required: [nummer, betrag]
|
||||
properties:
|
||||
nummer: {type: string}
|
||||
betrag: {type: number}
|
||||
---
|
||||
# Rechnung extrahieren
|
||||
Lies Nummer und Betrag aus.
|
||||
|
||||
## Beispiele
|
||||
Eingabe: Rechnung 7 über 10 EUR -> {"nummer": "7", "betrag": 10}
|
||||
""",
|
||||
"ticket-kategorisieren": """\
|
||||
---
|
||||
name: ticket-kategorisieren
|
||||
description: Support Ticket einer Kategorie zuordnen Drucker Netzwerk Passwort
|
||||
metadata:
|
||||
knaster:
|
||||
status: active
|
||||
stage: 3
|
||||
execution: classifier
|
||||
classifier: ticket-kat
|
||||
threshold: 0.3
|
||||
min_confidence: 0.6
|
||||
---
|
||||
Ordne das Ticket einer Kategorie zu: drucker, netzwerk, passwort.
|
||||
""",
|
||||
"text-gross": """\
|
||||
---
|
||||
name: text-gross
|
||||
description: Text in Großbuchstaben umwandeln uppercase
|
||||
metadata:
|
||||
knaster:
|
||||
status: active
|
||||
stage: 2
|
||||
execution: program
|
||||
program: echo-upper
|
||||
threshold: 0.3
|
||||
---
|
||||
Wandle den Text in Großbuchstaben um.
|
||||
""",
|
||||
"entwurf": """\
|
||||
---
|
||||
name: entwurf
|
||||
description: Gedicht über Rechnung schreiben
|
||||
metadata:
|
||||
knaster:
|
||||
status: draft
|
||||
---
|
||||
Entwurf.
|
||||
""",
|
||||
"vertrag-pruefen": """\
|
||||
---
|
||||
name: vertrag-pruefen
|
||||
description: Vertrag prüfen Kündigungsfrist Laufzeit Klauseln
|
||||
metadata:
|
||||
knaster:
|
||||
status: demoted
|
||||
stage: 1
|
||||
threshold: 0.3
|
||||
verifier:
|
||||
- {type: contains, value: "Kündigungsfrist"}
|
||||
---
|
||||
Prüfe den Vertrag.
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store() -> Store:
|
||||
return Store("sqlite://")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(store: Store) -> Registry:
|
||||
return Registry.from_dict(REGISTRY, store)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def skills_dir(tmp_path: Path) -> Path:
|
||||
for name, text in SKILLS.items():
|
||||
d = tmp_path / "skills" / name
|
||||
d.mkdir(parents=True)
|
||||
(d / "SKILL.md").write_text(textwrap.dedent(text), encoding="utf-8")
|
||||
return tmp_path / "skills"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def library(skills_dir: Path, store: Store) -> SkillLibrary:
|
||||
return SkillLibrary.from_dir(skills_dir, store)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def embedder() -> HashingEmbedder:
|
||||
return HashingEmbedder()
|
||||
|
||||
|
||||
class FakeChat:
|
||||
"""Antworten pro Modellname; ein Handler darf ChatResult liefern oder eine Exception werfen."""
|
||||
|
||||
def __init__(self, handlers: dict[str, Callable[[list[dict[str, str]]], ChatResult | str]] | None = None):
|
||||
self.handlers = handlers or {}
|
||||
self.calls: list[tuple[str, list[dict[str, str]]]] = []
|
||||
|
||||
def chat(self, model, messages, *, max_tokens=None, temperature=None):
|
||||
self.calls.append((model, messages))
|
||||
handler = self.handlers.get(model)
|
||||
if handler is None:
|
||||
return ChatResult(content="ok", finish_reason="stop")
|
||||
result = handler(messages)
|
||||
if isinstance(result, str):
|
||||
return ChatResult(content=result, finish_reason="stop")
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_chat() -> FakeChat:
|
||||
return FakeChat()
|
||||
@@ -0,0 +1,222 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from knaster.calibration import effective_context
|
||||
from knaster.cases import export_preferences, export_sft, split_for
|
||||
from knaster.classifier import EmbeddingClassifier
|
||||
from knaster.executor import Executor
|
||||
from knaster.llm import ChatResult, ContextOverflowError, EndpointError
|
||||
from knaster.problems import Signal, extract_json, is_degenerate, repetition_ratio
|
||||
from knaster.router import Router
|
||||
from knaster.runner import Runner
|
||||
from knaster.verifier import Verifier
|
||||
|
||||
from .test_router import RECHNUNG
|
||||
|
||||
GOOD = json.dumps({"nummer": "4711", "betrag": 99})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def classifiers(embedder):
|
||||
examples = []
|
||||
for label, texts in {
|
||||
"drucker": ["Drucker druckt nicht", "Drucker Papierstau", "Drucker offline"],
|
||||
"netzwerk": ["WLAN geht nicht", "Netzwerk langsam", "kein Internet"],
|
||||
}.items():
|
||||
examples += [(v, label) for v in embedder.embed(texts)]
|
||||
return {"ticket-kat": EmbeddingClassifier.train(examples)}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_runner(registry, library, embedder, store, fake_chat, classifiers):
|
||||
def make(**handlers):
|
||||
fake_chat.handlers.update(handlers)
|
||||
router = Router(registry, library, embedder)
|
||||
executor = Executor(library, fake_chat, embedder, classifiers)
|
||||
verifier = Verifier(registry, fake_chat)
|
||||
return Runner(router, executor, verifier, library, store)
|
||||
|
||||
return make
|
||||
|
||||
|
||||
def test_weak_success_records_case(make_runner, store):
|
||||
runner = make_runner(**{"rechnungen-v1": lambda m: GOOD})
|
||||
res = runner.run(RECHNUNG)
|
||||
assert res.status == "ok" and res.stage == 1 and res.escalations == 0
|
||||
cases = store.list_cases()
|
||||
assert len(cases) == 1 and cases[0]["passed"] is True and cases[0]["cluster"] == "rechnungen"
|
||||
assert cases[0]["messages"][0]["role"] == "system"
|
||||
|
||||
|
||||
def test_schema_failure_escalates_and_creates_pair(make_runner, store):
|
||||
runner = make_runner(**{
|
||||
"rechnungen-v1": lambda m: json.dumps({"nummer": "4711"}), # betrag fehlt
|
||||
"tier0-strong": lambda m: GOOD,
|
||||
})
|
||||
res = runner.run(RECHNUNG)
|
||||
assert res.status == "ok" and res.stage == 0 and res.escalations == 1
|
||||
pairs = store.list_preference_pairs()
|
||||
assert len(pairs) == 1
|
||||
assert pairs[0]["chosen"] == GOOD and "betrag" not in pairs[0]["rejected"]
|
||||
assert {c["stage"]: c["passed"] for c in store.list_cases()} == {1: False, 0: True}
|
||||
|
||||
|
||||
def test_invalid_json_signal(make_runner, store):
|
||||
runner = make_runner(**{"rechnungen-v1": lambda m: "kein json", "tier0-strong": lambda m: GOOD})
|
||||
res = runner.run(RECHNUNG)
|
||||
assert Signal.INVALID_FORMAT in res.attempts[0].execution.signals
|
||||
assert res.stage == 0
|
||||
assert store.count_events("invalid_format") == 1
|
||||
|
||||
|
||||
def test_endpoint_error_uses_fallback_slot_without_case(make_runner, store):
|
||||
def boom(m):
|
||||
raise EndpointError("down")
|
||||
|
||||
runner = make_runner(**{"rechnungen-v1": boom})
|
||||
# Fallback-Slot tier1-flex hat dieselbe Basis → gleicher Adapter-Name; daher Fehler nur einmal
|
||||
calls = {"n": 0}
|
||||
|
||||
def flaky(m):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise EndpointError("down")
|
||||
return GOOD
|
||||
|
||||
runner.executor.chat.handlers["rechnungen-v1"] = flaky
|
||||
res = runner.run(RECHNUNG)
|
||||
assert res.status == "ok"
|
||||
assert [a.decision.slot for a in res.attempts] == ["tier1-general", "tier1-flex"]
|
||||
assert len(store.list_cases()) == 1 # Infrastrukturfehler erzeugt keinen Fall
|
||||
assert store.list_preference_pairs() == []
|
||||
|
||||
|
||||
def test_context_overflow_detected(make_runner):
|
||||
def overflow(m):
|
||||
raise ContextOverflowError("maximum context length")
|
||||
|
||||
runner = make_runner(**{"rechnungen-v1": overflow, "tier0-strong": lambda m: GOOD})
|
||||
res = runner.run(RECHNUNG)
|
||||
assert res.attempts[0].execution.signals == [Signal.CONTEXT_OVERFLOW]
|
||||
assert res.status == "ok"
|
||||
|
||||
|
||||
def test_truncated_and_degenerate(make_runner):
|
||||
loop = " ".join(["immer wieder das gleiche"] * 30)
|
||||
runner = make_runner(**{
|
||||
"rechnungen-v1": lambda m: ChatResult(content=loop, finish_reason="length"),
|
||||
"tier0-strong": lambda m: GOOD,
|
||||
})
|
||||
res = runner.run(RECHNUNG)
|
||||
sig = res.attempts[0].execution.signals
|
||||
assert Signal.OUTPUT_TRUNCATED in sig and Signal.DEGENERATE_OUTPUT in sig
|
||||
|
||||
|
||||
def test_total_failure(make_runner):
|
||||
runner = make_runner(**{"rechnungen-v1": lambda m: "x", "tier0-strong": lambda m: "y"})
|
||||
res = runner.run(RECHNUNG)
|
||||
assert res.status == "failed"
|
||||
|
||||
|
||||
def test_classifier_stage(make_runner):
|
||||
runner = make_runner()
|
||||
res = runner.run("Ticket: Drucker druckt nicht Papierstau")
|
||||
assert res.status == "ok" and res.stage == 3 and res.output == "drucker"
|
||||
|
||||
|
||||
def test_classifier_low_confidence_escalates(make_runner):
|
||||
runner = make_runner(**{"tier1-general": lambda m: "drucker"})
|
||||
res = runner.run("Ticket einer Kategorie zuordnen: Monitor flackert")
|
||||
assert res.attempts[0].execution.signals == [Signal.CLASSIFIER_LOW_CONFIDENCE]
|
||||
assert res.attempts[1].decision.stage == 1
|
||||
|
||||
|
||||
def test_program_stage(make_runner):
|
||||
res = make_runner().run("Text in Großbuchstaben: hallo welt")
|
||||
assert res.output == "TEXT IN GROSSBUCHSTABEN: HALLO WELT"
|
||||
|
||||
|
||||
def test_judge_check(registry, library, fake_chat):
|
||||
skill = library.get("vertrag-pruefen").model_copy(deep=True)
|
||||
skill.meta.verifier = [{"type": "judge", "rubric": "muss Frist nennen"}]
|
||||
v = Verifier(registry, fake_chat)
|
||||
fake_chat.handlers["verifier-small"] = lambda m: "FAIL: Frist fehlt"
|
||||
r = v.verify(skill, "Vertrag", "Antwort")
|
||||
assert not r.passed and r.failed_checks[0].detail.startswith("FAIL")
|
||||
fake_chat.handlers["verifier-small"] = lambda m: "PASS"
|
||||
assert v.verify(skill, "Vertrag", "Antwort").passed
|
||||
|
||||
|
||||
def test_judge_unavailable_is_inconclusive(registry, library, fake_chat):
|
||||
skill = library.get("vertrag-pruefen").model_copy(deep=True)
|
||||
skill.meta.verifier = [{"type": "judge"}, {"type": "contains", "value": "Frist"}]
|
||||
registry.set_status("verifier-small", "unavailable")
|
||||
r = Verifier(registry, fake_chat).verify(skill, "Vertrag", "Frist 3 Monate")
|
||||
assert r.passed and r.inconclusive
|
||||
|
||||
|
||||
def test_inconclusive_case_not_used_for_training(make_runner, registry, library, store):
|
||||
library.set_state("vertrag-pruefen", status="active")
|
||||
registry.set_status("verifier-small", "unavailable")
|
||||
skill_file = library._skills["vertrag-pruefen"]
|
||||
skill_file.meta.verifier = [{"type": "judge"}]
|
||||
runner = make_runner(**{"tier1-general": lambda m: "Kündigungsfrist: 3 Monate"})
|
||||
res = runner.run("Vertrag prüfen auf Kündigungsfrist")
|
||||
assert res.status == "ok"
|
||||
assert store.list_cases()[0]["passed"] is None
|
||||
|
||||
|
||||
def test_verifier_rules():
|
||||
from knaster.skills import KnasterMeta, Skill
|
||||
|
||||
skill = Skill(name="s", description="d", body="b", meta=KnasterMeta(verifier=[
|
||||
{"type": "regex", "pattern": r"^\d+$"}, {"type": "not_contains", "value": "Fehler"},
|
||||
{"type": "max_chars", "value": 5}, {"type": "unbekannt"}]))
|
||||
r = Verifier().verify(skill, "t", "123")
|
||||
assert [c.passed for c in r.checks] == [True, True, True, False]
|
||||
|
||||
|
||||
def test_problem_helpers():
|
||||
assert extract_json('```json\n{"a": 1}\n```') == {"a": 1}
|
||||
with pytest.raises(ValueError):
|
||||
extract_json("{a:")
|
||||
assert repetition_ratio("a b c d e f") == 0
|
||||
assert is_degenerate(" ".join(["la le lu"] * 20))
|
||||
assert not is_degenerate("kurz kurz kurz kurz")
|
||||
|
||||
|
||||
def test_split_is_deterministic():
|
||||
assert split_for("abc") == split_for("abc")
|
||||
splits = {split_for(f"text {i}") for i in range(100)}
|
||||
assert splits == {"train", "eval"}
|
||||
|
||||
|
||||
def test_exports(make_runner, store):
|
||||
runner = make_runner(**{"rechnungen-v1": lambda m: '{"nummer": "1"}', "tier0-strong": lambda m: GOOD})
|
||||
for i in range(30):
|
||||
runner.run(f"{RECHNUNG} Nr {i}")
|
||||
train = list(export_sft(store, cluster="rechnungen", split="train"))
|
||||
evals = list(export_sft(store, cluster="rechnungen", split="eval"))
|
||||
assert len(train) + len(evals) == 30
|
||||
row = json.loads(train[0])
|
||||
assert row["messages"][-1] == {"role": "assistant", "content": GOOD}
|
||||
assert row["meta"]["stage"] == 0
|
||||
prefs = list(export_preferences(store, cluster="rechnungen"))
|
||||
assert prefs and json.loads(prefs[0])["chosen"] == GOOD
|
||||
|
||||
|
||||
def test_effective_context_calibration():
|
||||
obs = [(1000, True)] * 30 + [(3000, True)] * 28 + [(3000, False)] * 2 + [(6000, True)] * 15 + \
|
||||
[(6000, False)] * 10 + [(12000, True)] * 5
|
||||
assert effective_context(obs, target=0.9, min_n=20) == 4096
|
||||
assert effective_context([(1000, True)] * 5) is None
|
||||
# Buckets ohne genug Daten unterbrechen nicht
|
||||
assert effective_context([(1000, True)] * 25 + [(20000, True)] * 25, min_n=20) == 32768
|
||||
|
||||
|
||||
def test_classifier_persistence(tmp_path, classifiers, embedder):
|
||||
clf = classifiers["ticket-kat"]
|
||||
clf.save(tmp_path / "c.json")
|
||||
loaded = EmbeddingClassifier.load(tmp_path / "c.json")
|
||||
assert loaded.predict(embedder.embed(["Drucker"])[0])[0] == "drucker"
|
||||
@@ -0,0 +1,78 @@
|
||||
from knaster.registry import AdapterSpec, Registry
|
||||
from knaster.skills import compact_body, parse_skill
|
||||
|
||||
|
||||
def test_slot_effective_context_prefers_calibration(registry):
|
||||
slot = registry.slot("tier1-general")
|
||||
assert slot.effective_context == 2000
|
||||
registry.set_calibrated_context("tier1-general", 1500)
|
||||
assert registry.slot("tier1-general").effective_context == 1500
|
||||
|
||||
|
||||
def test_effective_context_never_exceeds_max(registry):
|
||||
registry.set_calibrated_context("tier1-general", 999999)
|
||||
assert registry.slot("tier1-general").effective_context == 8192
|
||||
|
||||
|
||||
def test_status_and_strong_slot(registry):
|
||||
registry.set_status("tier1-general", "degraded", reason="test", set_by="healer")
|
||||
slot = registry.slot("tier1-general")
|
||||
assert not slot.available and slot.set_by == "healer"
|
||||
assert registry.strong_slot().name == "tier0-strong"
|
||||
|
||||
|
||||
def test_unknown_fallback_rejected(store):
|
||||
import pytest
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
Registry.from_dict({"slots": {"a": {"tier": 1, "litellm_model": "a", "base_model": "b",
|
||||
"max_context": 10, "fallbacks": ["x"]}}}, store)
|
||||
|
||||
|
||||
def test_adapter_overlay_and_active(registry):
|
||||
assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
|
||||
registry.set_adapter(AdapterSpec(name="rechnungen-v2", base_model="base-14b", cluster="rechnungen",
|
||||
version=2, status="active"))
|
||||
assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v2"
|
||||
registry.set_adapter(AdapterSpec(name="rechnungen-v2", base_model="base-14b", cluster="rechnungen",
|
||||
version=2, status="retired"))
|
||||
assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
|
||||
assert registry.active_adapter("other-base", "rechnungen") is None
|
||||
|
||||
|
||||
def test_base_model_override_and_export(registry):
|
||||
registry.set_base_model("tier1-general", "new-base")
|
||||
assert registry.slot("tier1-general").base_model == "new-base"
|
||||
exported = registry.export()
|
||||
assert exported["slots"]["tier1-general"]["base_model"] == "new-base"
|
||||
|
||||
|
||||
def test_parse_skill_and_compact(library):
|
||||
skill = library.get("rechnung-extrahieren")
|
||||
assert skill.cluster == "rechnungen"
|
||||
assert skill.meta.output.format == "json"
|
||||
assert "Beispiele" in skill.prompt("full")
|
||||
assert "Beispiele" not in skill.prompt("compact")
|
||||
assert "Lies Nummer" in skill.prompt("compact")
|
||||
|
||||
|
||||
def test_compact_body_keeps_following_sections():
|
||||
body = "# T\nA\n## Beispiele\nx\ny\n## Regeln\nR\n"
|
||||
assert compact_body(body) == "# T\nA\n## Regeln\nR"
|
||||
|
||||
|
||||
def test_parse_skill_requires_fields():
|
||||
import pytest
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
parse_skill("---\nname: x\n---\nbody")
|
||||
with pytest.raises(ValueError):
|
||||
parse_skill("kein frontmatter")
|
||||
|
||||
|
||||
def test_routable_and_state_overlay(library):
|
||||
names = {s.name for s in library.routable()}
|
||||
assert "entwurf" not in names and "vertrag-pruefen" in names
|
||||
library.set_state("rechnung-extrahieren", status="demoted", reason="drift")
|
||||
assert library.get("rechnung-extrahieren").meta.status == "demoted"
|
||||
assert library.get("rechnung-extrahieren").meta.stage == 1
|
||||
@@ -0,0 +1,133 @@
|
||||
import pytest
|
||||
|
||||
from knaster.problems import Signal
|
||||
from knaster.router import Router, RouterConfig
|
||||
|
||||
RECHNUNG = "Rechnung auslesen: Rechnungsnummer und Betrag extrahieren. Rechnung 4711 über 99 EUR"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def router(registry, library, embedder):
|
||||
return Router(registry, library, embedder)
|
||||
|
||||
|
||||
def test_routes_to_weak_slot_with_adapter(router):
|
||||
d = router.route(RECHNUNG)
|
||||
assert d.status == "routed"
|
||||
assert (d.stage, d.slot, d.skill) == (1, "tier1-general", "rechnung-extrahieren")
|
||||
assert d.model == d.adapter == "rechnungen-v1"
|
||||
assert d.prompt_variant == "full"
|
||||
|
||||
|
||||
def test_unknown_request_goes_to_strong(router):
|
||||
d = router.route("Wie ist das Wetter morgen in Berlin?")
|
||||
assert (d.stage, d.slot, d.skill) == (0, "tier0-strong", None)
|
||||
assert Signal.NO_SKILL in d.signals
|
||||
|
||||
|
||||
def test_draft_skill_not_routed(router):
|
||||
d = router.route("Gedicht über Rechnung schreiben")
|
||||
assert d.skill != "entwurf"
|
||||
|
||||
|
||||
def test_demoted_skill_runs_on_strong_with_hint(router):
|
||||
d = router.route("Vertrag prüfen auf Kündigungsfrist")
|
||||
assert (d.stage, d.skill, d.prompt_variant) == (0, "vertrag-pruefen", "full")
|
||||
|
||||
|
||||
def test_classifier_and_program_stages(router):
|
||||
assert router.route("Ticket: Drucker druckt nicht").stage == 3
|
||||
d = router.route("Text in Großbuchstaben: hallo welt")
|
||||
assert (d.stage, d.slot) == (2, None)
|
||||
|
||||
|
||||
def test_unavailable_slot_uses_fallback(router, registry):
|
||||
registry.set_status("tier1-general", "degraded")
|
||||
d = router.route(RECHNUNG)
|
||||
assert d.slot == "tier1-flex"
|
||||
assert Signal.SLOT_UNAVAILABLE in d.signals
|
||||
|
||||
|
||||
def test_all_weak_slots_unavailable_goes_strong(router, registry):
|
||||
registry.set_status("tier1-general", "unavailable")
|
||||
registry.set_status("tier1-flex", "unavailable")
|
||||
d = router.route(RECHNUNG)
|
||||
assert d.stage == 0 and d.skill == "rechnung-extrahieren"
|
||||
|
||||
|
||||
def test_long_input_uses_compact_then_larger_slot(router):
|
||||
# Eingabe ~1585 Tokens: mit Beispielen (64) zu groß für 2000, kompakt (38) passt es
|
||||
filler = "x" * (3 * 1585 - len(RECHNUNG) - 1)
|
||||
d = router.route(RECHNUNG + " " + filler)
|
||||
assert d.slot == "tier1-general" and d.prompt_variant == "compact"
|
||||
assert Signal.COMPACT_PROMPT in d.signals
|
||||
|
||||
d = router.route(RECHNUNG + " " + "x" * (3 * 5000))
|
||||
assert d.slot == "tier1-flex"
|
||||
assert Signal.LARGER_SLOT in d.signals
|
||||
|
||||
|
||||
def test_too_large_for_everything(router):
|
||||
d = router.route(RECHNUNG + " " + "x" * (3 * 120000))
|
||||
assert d.status == "too_large"
|
||||
assert Signal.NEEDS_SPLIT in d.signals
|
||||
|
||||
|
||||
def test_context_exceeded_goes_to_strong(router):
|
||||
d = router.route(RECHNUNG + " " + "x" * (3 * 30000))
|
||||
assert d.stage == 0 and Signal.CONTEXT_EXCEEDED in d.signals
|
||||
|
||||
|
||||
def test_hint_skill(router):
|
||||
d = router.route("irgendwas", hint_skill="rechnung-extrahieren")
|
||||
assert d.skill == "rechnung-extrahieren" and d.similarity is None
|
||||
with pytest.raises(KeyError):
|
||||
router.route("x", hint_skill="entwurf")
|
||||
|
||||
|
||||
def test_escalation_quality_goes_to_strong(router):
|
||||
d = router.route(RECHNUNG)
|
||||
e = router.escalate(d, Signal.VERIFIER_FAILED, RECHNUNG)
|
||||
assert (e.stage, e.slot, e.attempt) == (0, "tier0-strong", 1)
|
||||
f = router.escalate(e, Signal.VERIFIER_FAILED, RECHNUNG)
|
||||
assert f.status == "failed"
|
||||
|
||||
|
||||
def test_escalation_infra_uses_fallback_slot(router):
|
||||
d = router.route(RECHNUNG)
|
||||
e = router.escalate(d, Signal.ENDPOINT_ERROR, RECHNUNG)
|
||||
assert (e.stage, e.slot) == (1, "tier1-flex")
|
||||
f = router.escalate(e, Signal.ENDPOINT_ERROR, RECHNUNG)
|
||||
assert f.stage == 0
|
||||
|
||||
|
||||
def test_escalation_from_program_to_weak(router):
|
||||
d = router.route("Text in Großbuchstaben: hallo welt")
|
||||
e = router.escalate(d, Signal.PROGRAM_ERROR, "Text in Großbuchstaben: hallo welt")
|
||||
assert e.stage == 1
|
||||
|
||||
|
||||
def test_max_escalations(registry, library, embedder):
|
||||
r = Router(registry, library, embedder, config=RouterConfig(max_escalations=1))
|
||||
d = r.route(RECHNUNG)
|
||||
e = r.escalate(d, Signal.ENDPOINT_ERROR, RECHNUNG)
|
||||
assert r.escalate(e, Signal.ENDPOINT_ERROR, RECHNUNG).status == "failed"
|
||||
|
||||
|
||||
class PreferRechnung:
|
||||
def rerank(self, query, documents):
|
||||
return [1.0 if "Rechnung" in d else 0.0 for d in documents]
|
||||
|
||||
|
||||
def test_conflict_without_reranker_goes_strong(registry, library, embedder):
|
||||
r = Router(registry, library, embedder, config=RouterConfig(conflict_margin=1.0))
|
||||
d = r.route(RECHNUNG)
|
||||
assert d.stage == 0 and Signal.SKILL_CONFLICT in d.signals
|
||||
|
||||
|
||||
def test_conflict_with_reranker_trusts_reranker(registry, library, embedder):
|
||||
r = Router(registry, library, embedder, reranker=PreferRechnung(), config=RouterConfig(conflict_margin=1.0))
|
||||
d = r.route("Vertrag prüfen Kündigungsfrist. Rechnung auslesen Rechnungsnummer Betrag")
|
||||
assert Signal.SKILL_CONFLICT in d.signals
|
||||
assert "Reranking aktiv" in d.reasons
|
||||
assert d.skill == "rechnung-extrahieren" and d.stage == 1
|
||||
Reference in New Issue
Block a user