45 lines
1.7 KiB
Python
45 lines
1.7 KiB
Python
"""Fälle und Präferenzpaare: modellneutral speichern, deterministisch splitten, exportieren."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Iterator
|
|
from typing import Any
|
|
|
|
from .store import Store
|
|
|
|
EVAL_SHARE_PERCENT = 20
|
|
|
|
|
|
def split_for(text: str) -> str:
|
|
"""Fester Split pro Eingabe: gleiche Eingabe landet immer im selben Split."""
|
|
bucket = int(hashlib.sha256(text.encode("utf-8")).hexdigest(), 16) % 100
|
|
return "eval" if bucket < EVAL_SHARE_PERCENT else "train"
|
|
|
|
|
|
def export_sft(store: Store, *, cluster: str, split: str = "train") -> Iterator[str]:
|
|
"""JSONL für Supervised Finetuning: nur unabhängig verifizierte Fälle."""
|
|
for case in reversed(store.list_cases(cluster=cluster, split=split, passed=True)):
|
|
messages = list(case["messages"] or [])
|
|
messages.append({"role": "assistant", "content": case["output"]})
|
|
meta = {k: case[k] for k in ("id", "skill", "skill_version", "stage", "model", "origin")}
|
|
yield json.dumps({"messages": messages, "meta": meta}, ensure_ascii=False)
|
|
|
|
|
|
def export_preferences(store: Store, *, cluster: str, split: str = "train") -> Iterator[str]:
|
|
for pair in store.list_preference_pairs(cluster=cluster):
|
|
if pair["split"] != split:
|
|
continue
|
|
yield json.dumps(
|
|
{"prompt": pair["prompt"], "chosen": pair["chosen"], "rejected": pair["rejected"]},
|
|
ensure_ascii=False,
|
|
)
|
|
|
|
|
|
def pass_rate(rows: list[dict[str, Any]]) -> float | None:
|
|
decided = [r for r in rows if r.get("passed") is not None]
|
|
if not decided:
|
|
return None
|
|
return sum(1 for r in decided if r["passed"]) / len(decided)
|