core: Registry, Skills, Router, Executor, Verifier, Runner mit Tests
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
"""Fälle und Präferenzpaare: modellneutral speichern, deterministisch splitten, exportieren."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from .store import Store
|
||||
|
||||
EVAL_SHARE_PERCENT = 20
|
||||
|
||||
|
||||
def split_for(text: str) -> str:
|
||||
"""Fester Split pro Eingabe: gleiche Eingabe landet immer im selben Split."""
|
||||
bucket = int(hashlib.sha256(text.encode("utf-8")).hexdigest(), 16) % 100
|
||||
return "eval" if bucket < EVAL_SHARE_PERCENT else "train"
|
||||
|
||||
|
||||
def export_sft(store: Store, *, cluster: str, split: str = "train") -> Iterator[str]:
|
||||
"""JSONL für Supervised Finetuning: nur unabhängig verifizierte Fälle."""
|
||||
for case in reversed(store.list_cases(cluster=cluster, split=split, passed=True)):
|
||||
messages = list(case["messages"] or [])
|
||||
messages.append({"role": "assistant", "content": case["output"]})
|
||||
meta = {k: case[k] for k in ("id", "skill", "skill_version", "stage", "model", "origin")}
|
||||
yield json.dumps({"messages": messages, "meta": meta}, ensure_ascii=False)
|
||||
|
||||
|
||||
def export_preferences(store: Store, *, cluster: str, split: str = "train") -> Iterator[str]:
|
||||
for pair in store.list_preference_pairs(cluster=cluster):
|
||||
if pair["split"] != split:
|
||||
continue
|
||||
yield json.dumps(
|
||||
{"prompt": pair["prompt"], "chosen": pair["chosen"], "rejected": pair["rejected"]},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
def pass_rate(rows: list[dict[str, Any]]) -> float | None:
|
||||
decided = [r for r in rows if r.get("passed") is not None]
|
||||
if not decided:
|
||||
return None
|
||||
return sum(1 for r in decided if r["passed"]) / len(decided)
|
||||
Reference in New Issue
Block a user