import pytest from knaster.lifecycle import Lifecycle, LifecycleConfig, LifecycleError, LoggingJobRunner from knaster.registry import AdapterSpec @pytest.fixture def life(store, registry): return Lifecycle(store, registry, LoggingJobRunner(), LifecycleConfig(min_n=10)) def run_to_awaiting(life, cid, model="cand-model"): life.start(cid) life.report(cid, "build", {"ok": True, "model": model}) life.report(cid, "eval", {"pass_rate": 0.95, "n": 40}) life.report(cid, "shadow", {"agreement": 0.9, "n": 20}) return life.report(cid, "canary", {"pass_rate": 0.93, "n": 12}) def test_full_adapter_flow_and_rollback(life, registry): cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"}) cand = run_to_awaiting(life, cid, model="rechnungen-v2") assert cand["state"] == "awaiting_promotion" assert [j for j, _ in life.jobs.jobs] == ["build", "eval", "shadow"] assert registry.slot("tier1-general").canary_model is None # Canary beendet life.promote(cid) active = registry.active_adapter("base-14b", "rechnungen") assert active.name == "rechnungen-v2" and active.version == 2 assert {a.name: a.status for a in registry.adapters()}["rechnungen-v1"] == "retired" life.rollback(cid) assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1" assert life.get(cid)["state"] == "rolled_back" def test_canary_is_set_during_canary_phase(life, registry): cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"}) life.start(cid) life.report(cid, "build", {"model": "rechnungen-v2"}) life.report(cid, "eval", {"pass_rate": 0.95, "n": 40}) life.report(cid, "shadow", {"agreement": 0.9, "n": 20}) slot = registry.slot("tier1-general") assert (slot.canary_model, slot.canary_cluster, slot.canary_share) == ("rechnungen-v2", "rechnungen", 0.1) life.report(cid, "canary", {"pass_rate": 0.5, "n": 12}) assert life.get(cid)["state"] == "rejected" assert registry.slot("tier1-general").canary_model is None def test_base_model_switch_retires_adapters_and_spawns_retraining(life, registry, store): cid = life.create("base_model", "tier1-general", {"base_model": "base-32b"}) run_to_awaiting(life, cid, model="tier1-general-candidate") life.promote(cid) assert registry.slot("tier1-general").base_model == "base-32b" assert registry.active_adapter("base-14b", "rechnungen") is None follow = [c for c in store.list_candidates() if c["kind"] == "adapter"] assert len(follow) == 1 and follow[0]["target"] == {"cluster": "rechnungen", "base_model": "base-32b"} life.rollback(cid) assert registry.slot("tier1-general").base_model == "base-14b" assert registry.active_adapter("base-14b", "rechnungen").name == "rechnungen-v1" assert life.get(follow[0]["id"])["state"] == "rejected" def test_gates_reject(life): cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"}, baseline={"pass_rate": 0.9}) life.start(cid) life.report(cid, "build", {"model": "m"}) assert life.report(cid, "eval", {"pass_rate": 0.85, "n": 40})["state"] == "rejected" cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"}) life.start(cid) life.report(cid, "build", {"model": "m"}) life.report(cid, "eval", {"pass_rate": 0.95, "n": 40}) assert life.report(cid, "shadow", {"agreement": 0.95, "n": 3})["state"] == "rejected" # zu wenig Daten def test_error_report_and_invalid_transitions(life): cid = life.create("engine", "tier1-general", {"image": "vllm:v2"}) with pytest.raises(LifecycleError): life.report(cid, "build", {"ok": True}) # noch nicht gestartet life.start(cid) with pytest.raises(LifecycleError): life.promote(cid) assert life.report(cid, "build", {"error": "OOM"})["state"] == "failed" with pytest.raises(LifecycleError): life.create("adapter", "tier1-general", {}) with pytest.raises(LifecycleError): life.create("base_model", "nope", {"base_model": "x"}) def test_canary_metrics_from_cases(life, store): cid = life.create("adapter", "tier1-general", {"cluster": "rechnungen"}) life.start(cid) life.report(cid, "build", {"model": "rv2"}) life.report(cid, "eval", {"pass_rate": 0.95, "n": 40}) life.report(cid, "shadow", {"agreement": 0.9, "n": 20}) for i in range(4): store.add_case(input=f"x{i}", stage=1, slot="tier1-general", model="rv2", passed=i < 3, split="train", origin="route", ts=life.get(cid)["updated_at"] + 1) store.add_case(input="y", stage=1, slot="tier1-general", model="other", passed=False, split="train", origin="route") assert life.canary_metrics(cid) == {"pass_rate": 0.75, "n": 4} def test_router_uses_canary_model(registry, library, embedder): import random from knaster.router import Router from .test_router import RECHNUNG registry.set_canary("tier1-general", "rechnungen-v2", 1.0, "rechnungen") r = Router(registry, library, embedder, rng=random.Random(1)) d = r.route(RECHNUNG) assert d.canary and d.model == d.adapter == "rechnungen-v2" registry.set_canary("tier1-general", "rechnungen-v2", 0.0, "rechnungen") assert not r.route(RECHNUNG).canary registry.set_adapter(AdapterSpec(name="x", base_model="base-14b", cluster="andere"))