120 lines
5.3 KiB
Python
120 lines
5.3 KiB
Python
import pytest
|
|
|
|
from knaster.lifecycle import Lifecycle, LifecycleConfig, LifecycleError, LoggingJobRunner
|
|
from knaster.registry import AdapterSpec
|
|
|
|
|
|
@pytest.fixture
|
|
def life(store, registry):
|
|
return Lifecycle(store, registry, LoggingJobRunner(), LifecycleConfig(min_n=10))
|
|
|
|
|
|
def run_to_awaiting(life, cid, model="cand-model"):
|
|
life.start(cid)
|
|
life.report(cid, "build", {"ok": True, "model": model})
|
|
life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
|
|
life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
|
|
return life.report(cid, "canary", {"pass_rate": 0.93, "n": 12})
|
|
|
|
|
|
def test_full_adapter_flow_and_rollback(life, registry):
|
|
cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
|
|
cand = run_to_awaiting(life, cid, model="rechnungen-v2")
|
|
assert cand["state"] == "awaiting_promotion"
|
|
assert [j for j, _ in life.jobs.jobs] == ["build", "eval", "shadow"]
|
|
assert registry.slot("tier1-general").canary_model is None # Canary beendet
|
|
|
|
life.promote(cid)
|
|
active = registry.active_adapter("base-14b", "rechnungen")
|
|
assert active.name == "rechnungen-v2" and active.version == 2
|
|
assert {a.name: a.status for a in registry.adapters()}["rechnungen-v1"] == "retired"
|
|
|
|
life.rollback(cid)
|
|
assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
|
|
assert life.get(cid)["state"] == "rolled_back"
|
|
|
|
|
|
def test_canary_is_set_during_canary_phase(life, registry):
|
|
cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
|
|
life.start(cid)
|
|
life.report(cid, "build", {"model": "rechnungen-v2"})
|
|
life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
|
|
life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
|
|
slot = registry.slot("tier1-general")
|
|
assert (slot.canary_model, slot.canary_cluster, slot.canary_share) == ("rechnungen-v2", "rechnungen", 0.1)
|
|
life.report(cid, "canary", {"pass_rate": 0.5, "n": 12})
|
|
assert life.get(cid)["state"] == "rejected"
|
|
assert registry.slot("tier1-general").canary_model is None
|
|
|
|
|
|
def test_base_model_switch_retires_adapters_and_spawns_retraining(life, registry, store):
|
|
cid = life.create("base_model", "tier1-general", {"base_model": "base-32b"})
|
|
run_to_awaiting(life, cid, model="tier1-general-candidate")
|
|
life.promote(cid)
|
|
assert registry.slot("tier1-general").base_model == "base-32b"
|
|
assert registry.active_adapter("base-14b", "rechnungen") is None
|
|
follow = [c for c in store.list_candidates() if c["kind"] == "adapter"]
|
|
assert len(follow) == 1 and follow[0]["target"] == {"cluster": "rechnungen", "base_model": "base-32b"}
|
|
|
|
life.rollback(cid)
|
|
assert registry.slot("tier1-general").base_model == "base-14b"
|
|
assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1"
|
|
assert life.get(follow[0]["id"])["state"] == "rejected"
|
|
|
|
|
|
def test_gates_reject(life):
|
|
cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"}, baseline={"pass_rate": 0.9})
|
|
life.start(cid)
|
|
life.report(cid, "build", {"model": "m"})
|
|
assert life.report(cid, "eval", {"pass_rate": 0.85, "n": 40})["state"] == "rejected"
|
|
|
|
cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
|
|
life.start(cid)
|
|
life.report(cid, "build", {"model": "m"})
|
|
life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
|
|
assert life.report(cid, "shadow", {"agreement": 0.95, "n": 3})["state"] == "rejected" # zu wenig Daten
|
|
|
|
|
|
def test_error_report_and_invalid_transitions(life):
|
|
cid = life.create("engine", "tier1-general", {"image": "vllm:v2"})
|
|
with pytest.raises(LifecycleError):
|
|
life.report(cid, "build", {"ok": True}) # noch nicht gestartet
|
|
life.start(cid)
|
|
with pytest.raises(LifecycleError):
|
|
life.promote(cid)
|
|
assert life.report(cid, "build", {"error": "OOM"})["state"] == "failed"
|
|
with pytest.raises(LifecycleError):
|
|
life.create("adapter", "tier1-general", {})
|
|
with pytest.raises(LifecycleError):
|
|
life.create("base_model", "nope", {"base_model": "x"})
|
|
|
|
|
|
def test_canary_metrics_from_cases(life, store):
|
|
cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"})
|
|
life.start(cid)
|
|
life.report(cid, "build", {"model": "rv2"})
|
|
life.report(cid, "eval", {"pass_rate": 0.95, "n": 40})
|
|
life.report(cid, "shadow", {"agreement": 0.9, "n": 20})
|
|
for i in range(4):
|
|
store.add_case(input=f"x{i}", stage=1, slot="tier1-general", model="rv2", passed=i < 3, split="train",
|
|
origin="route", ts=life.get(cid)["updated_at"] + 1)
|
|
store.add_case(input="y", stage=1, slot="tier1-general", model="other", passed=False, split="train",
|
|
origin="route")
|
|
assert life.canary_metrics(cid) == {"pass_rate": 0.75, "n": 4}
|
|
|
|
|
|
def test_router_uses_canary_model(registry, library, embedder):
|
|
import random
|
|
|
|
from knaster.router import Router
|
|
|
|
from .test_router import RECHNUNG
|
|
|
|
registry.set_canary("tier1-general", "rechnungen-v2", 1.0, "rechnungen")
|
|
r = Router(registry, library, embedder, rng=random.Random(1))
|
|
d = r.route(RECHNUNG)
|
|
assert d.canary and d.model == d.adapter == "rechnungen-v2"
|
|
registry.set_canary("tier1-general", "rechnungen-v2", 0.0, "rechnungen")
|
|
assert not r.route(RECHNUNG).canary
|
|
registry.set_adapter(AdapterSpec(name="x", base_model="base-14b", cluster="andere"))
|