ontology: Ontologie als Quelle im Repo, Abgleich mit n8n/KNOW in der Service-Routine
ci / test (push) Successful in 27s
ci / test (push) Successful in 27s
- ontology/know.yaml: 49 Klassen, 66 Beziehungen, 12 Vokabulare (72 Konzepte); erzeugt: know.ttl (OWL + SKOS), Ansicht aus viewer.html.tmpl - knaster.ontology: Modell + Prüfung, Export, Momentaufnahme der n8n-Workflows (nur GET; KNOW-Spezifikation aus dem Workflow-Code, Aufrufe inkl. Laufzeit- Operationen, direktes SQL), Abgleich, Übernahme von Entscheidungen - Service-Routine Schritt 8: Verstöße als Befunde, Ontologie-Änderungen als Vorschläge ontology_change (immer Freigabe); „ordne P-x <konzept> zu“; Entscheidungen in ontology_decisions, `knaster ontology apply` schreibt sie in die YAML (Version +1) - CLI: knaster ontology validate|export|snapshot|check|apply; knaster validate prüft Modell und Aktualität der Turtle-Datei - Gegen echten n8n-Export geprüft: neue Befunde B31–B33 (verified/confirmed, KAN ruft unbekannte Operation, MPA ordnet Projektereignisse nie ein) - SPEC-ontology, BETRIEB §6, Router-Image enthält die Ontologie Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+107
-1
@@ -1,4 +1,4 @@
|
||||
"""Kommandozeile: serve, validate, inventory, placement, registry export."""
|
||||
"""Kommandozeile: serve, validate, inventory, placement, registry export, ontology."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -47,8 +47,28 @@ def main(argv: list[str] | None = None) -> int:
|
||||
reg_sub = p_reg.add_subparsers(dest="reg_cmd", required=True)
|
||||
reg_sub.add_parser("export", help="effektiven Stand (inkl. Laufzeitzustand) als YAML ausgeben")
|
||||
|
||||
p_onto = sub.add_parser("ontology", help="Ontologie der Datenhaltung (SPEC-ontology)")
|
||||
p_onto.add_argument("--path", default=None, help="Ontologie-Datei (Standard: ontology/know.yaml)")
|
||||
onto_sub = p_onto.add_subparsers(dest="onto_cmd", required=True)
|
||||
onto_sub.add_parser("validate", help="Modell prüfen")
|
||||
p_exp = onto_sub.add_parser("export", help="Turtle, Graph-JSON und/oder HTML-Ansicht erzeugen")
|
||||
p_exp.add_argument("--ttl")
|
||||
p_exp.add_argument("--graph")
|
||||
p_exp.add_argument("--html")
|
||||
p_exp.add_argument("--template", default=None, help="Standard: viewer.html.tmpl neben der Ontologie")
|
||||
p_snap = onto_sub.add_parser("snapshot", help="n8n-Workflows lesen und als JSON speichern (nur GET)")
|
||||
p_snap.add_argument("--out", required=True)
|
||||
p_chk = onto_sub.add_parser("check", help="Ontologie mit der Datenhaltung in n8n abgleichen")
|
||||
p_chk.add_argument("--snapshot", help="Workflows aus Datei statt aus der n8n-API")
|
||||
p_chk.add_argument("--json", action="store_true")
|
||||
p_app = onto_sub.add_parser("apply", help="freigegebene Entscheidungen in die Ontologie übernehmen")
|
||||
p_app.add_argument("--dry-run", action="store_true")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.cmd == "ontology":
|
||||
return _ontology(args)
|
||||
|
||||
if args.cmd == "serve":
|
||||
import uvicorn
|
||||
|
||||
@@ -104,5 +124,91 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 2
|
||||
|
||||
|
||||
def _ontology(args: argparse.Namespace) -> int:
|
||||
import json
|
||||
|
||||
from .ontology import (
|
||||
Ontology,
|
||||
apply_decisions,
|
||||
build_snapshot,
|
||||
check,
|
||||
fetch_workflows,
|
||||
to_graph,
|
||||
to_html,
|
||||
to_turtle,
|
||||
)
|
||||
from .settings import Settings
|
||||
|
||||
settings = Settings.from_env()
|
||||
path = Path(args.path) if args.path else settings.ontology_path
|
||||
onto = Ontology.from_file(path)
|
||||
|
||||
if args.onto_cmd == "validate":
|
||||
errors = onto.validate_model()
|
||||
for e in errors:
|
||||
print(f"ERROR {e}")
|
||||
print(f"{len(onto.classes)} Klassen, {len(onto.relations)} Beziehungen, {len(onto.vocabularies)} Vokabulare, "
|
||||
f"{len(errors)} Fehler")
|
||||
return 1 if errors else 0
|
||||
|
||||
if args.onto_cmd == "export":
|
||||
if args.ttl:
|
||||
Path(args.ttl).write_text(to_turtle(onto), encoding="utf-8")
|
||||
if args.graph:
|
||||
Path(args.graph).write_text(json.dumps(to_graph(onto), ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
if args.html:
|
||||
template = Path(args.template) if args.template else path.parent / "viewer.html.tmpl"
|
||||
Path(args.html).write_text(to_html(onto, template), encoding="utf-8")
|
||||
return 0
|
||||
|
||||
if args.onto_cmd in {"snapshot", "check"}:
|
||||
if getattr(args, "snapshot", None):
|
||||
data = json.loads(Path(args.snapshot).read_text(encoding="utf-8"))
|
||||
workflows = data.get("data", data) if isinstance(data, dict) else data
|
||||
else:
|
||||
if not (settings.n8n_host and settings.n8n_api_key):
|
||||
print("N8N_HOST und N8N_API_KEY fehlen (oder --snapshot angeben)", file=sys.stderr)
|
||||
return 2
|
||||
workflows = fetch_workflows(settings.n8n_host, settings.n8n_api_key)
|
||||
if args.onto_cmd == "snapshot":
|
||||
Path(args.out).write_text(json.dumps({"data": workflows}, ensure_ascii=False), encoding="utf-8")
|
||||
print(f"{len(workflows)} Workflows gespeichert in {args.out} (enthält Workflow-Code, nicht committen)")
|
||||
return 0
|
||||
from .store import Store
|
||||
|
||||
decided = {d["key"] for d in Store(settings.db_url).list_ontology_decisions()}
|
||||
findings = check(onto, build_snapshot(workflows), decided=decided)
|
||||
if args.json:
|
||||
print(json.dumps([f.model_dump() for f in findings], ensure_ascii=False, indent=1))
|
||||
else:
|
||||
for f in findings:
|
||||
mark = "→ Vorschlag" if f.change else ""
|
||||
print(f"{f.severity.upper():7} {f.summary} {mark}".rstrip())
|
||||
print(f"{len(findings)} Befunde, davon {sum(1 for f in findings if f.change)} Änderungsvorschläge")
|
||||
return 1 if any(f.severity == "high" for f in findings) else 0
|
||||
|
||||
if args.onto_cmd == "apply":
|
||||
from .store import Store
|
||||
|
||||
store = Store(settings.db_url)
|
||||
pending = store.list_ontology_decisions(applied=False)
|
||||
applied, messages = apply_decisions(onto, pending)
|
||||
for m in messages:
|
||||
print(m)
|
||||
errors = onto.validate_model()
|
||||
if errors:
|
||||
print("Ontologie wäre ungültig, nichts gespeichert:\n" + "\n".join(errors), file=sys.stderr)
|
||||
return 1
|
||||
if applied and not args.dry_run:
|
||||
header = "".join(line + "\n" for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.startswith("#"))
|
||||
onto.save(path, header)
|
||||
store.mark_ontology_applied(applied)
|
||||
print(f"{len(applied)} Entscheidungen übernommen, Version {onto.version}. "
|
||||
"Danach `knaster ontology export` und committen.")
|
||||
return 0
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
@@ -101,6 +101,17 @@ class ActionExecutor:
|
||||
self.registry.set_calibrated_context(slot, value)
|
||||
return f"Slot {slot}: effektiver Kontext = {value} Tokens"
|
||||
|
||||
# --- Ontologie --------------------------------------------------------------------------
|
||||
def _do_ontology_change(self, key: str, type: str, decision: str = "accept", decided_by: str = "human",
|
||||
**params: Any) -> str:
|
||||
if decision not in {"accept", "map"}:
|
||||
raise ValueError(f"Entscheidung {decision} unbekannt")
|
||||
if decision == "map" and not params.get("concept"):
|
||||
raise ValueError("Zuordnung braucht ein Konzept")
|
||||
self.store.set_ontology_decision(key, decision, {"type": type, **params}, decided_by)
|
||||
target = f" → {params['concept']}" if decision == "map" else ""
|
||||
return f"Entscheidung gespeichert ({decision}{target}); in die Ontologie mit `knaster ontology apply`"
|
||||
|
||||
# --- Lifecycle --------------------------------------------------------------------------
|
||||
def _do_create_candidate(self, kind: str, slot: str, target: dict[str, Any], **_: Any) -> str:
|
||||
cid = self.lifecycle.create(kind, slot, target) # type: ignore[arg-type]
|
||||
|
||||
@@ -35,6 +35,7 @@ class MaintenanceConfig(BaseModel):
|
||||
adapter_min_cases: int = 300
|
||||
adapter_target_rate: float = 0.90
|
||||
lookback_days: float = 7.0
|
||||
ontology_max_findings: int = 15
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: str | Path) -> MaintenanceConfig:
|
||||
@@ -83,7 +84,8 @@ HELP = """Befehle:
|
||||
- status · hilfe · routine · vorschläge
|
||||
- genehmige P-3 [P-5 …] · genehmige alle mit risiko niedrig · lehne P-3 ab: <Grund>
|
||||
- wechsle <slot> auf <basismodell> · trainiere adapter für <cluster> · rollback <slot>
|
||||
- stufe <skill> zurück · markiere <slot> als ok|degraded|unavailable"""
|
||||
- stufe <skill> zurück · markiere <slot> als ok|degraded|unavailable
|
||||
- ordne P-7 <konzept> zu (Vokabel-Vorschlag einem vorhandenen Konzept zuordnen)"""
|
||||
|
||||
|
||||
class LLMIntentMapper:
|
||||
@@ -120,6 +122,7 @@ COMMANDS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("rollback", r"^rollback\s+(?P<slot>[\w.-]+)$"),
|
||||
("demote", r"^(stufe\s+(?P<skill>[\w.-]+)\s+zurück|degradiere\s+(?P<skill2>[\w.-]+))$"),
|
||||
("mark", r"^markiere\s+(?P<slot>[\w.-]+)\s+(als\s+)?(?P<status>ok|degraded|unavailable)$"),
|
||||
("map_concept", rf"^ordne\s+(?P<id>{_ID})\s+(dem\s+konzept\s+)?(?P<concept>[\w.-]+)\s+zu$"),
|
||||
("confirm", r"^(ja|yes|j|ok)$"),
|
||||
("cancel", r"^(nein|no|n|abbrechen)$"),
|
||||
]
|
||||
@@ -143,6 +146,7 @@ class MaintenanceAgent:
|
||||
def __init__(self, *, store: Store, registry: Registry, library: SkillLibrary, index: SkillIndex,
|
||||
policy: Policy, actions: ActionExecutor, lifecycle: Lifecycle,
|
||||
config: MaintenanceConfig | None = None, intent_mapper: IntentMapper | None = None,
|
||||
ontology_check: Callable[[], list[Any]] | None = None,
|
||||
clock: Callable[[], float] = now, wallclock: Callable[[], datetime] = datetime.now):
|
||||
self.store = store
|
||||
self.registry = registry
|
||||
@@ -153,6 +157,7 @@ class MaintenanceAgent:
|
||||
self.lifecycle = lifecycle
|
||||
self.config = config or MaintenanceConfig()
|
||||
self.intent_mapper = intent_mapper
|
||||
self.ontology_check = ontology_check
|
||||
self.clock = clock
|
||||
self.wallclock = wallclock
|
||||
self._pending: dict[str, str] = {} # Sitzung → zu bestätigender Befehl
|
||||
@@ -197,8 +202,9 @@ class MaintenanceAgent:
|
||||
|
||||
def _execute(self, seq: int) -> Proposal:
|
||||
p = self._load(seq)
|
||||
params = {**p.params, "decided_by": p.decided_by} if p.kind == "ontology_change" else p.params
|
||||
try:
|
||||
result = self.actions.execute(p.kind, p.params)
|
||||
result = self.actions.execute(p.kind, params)
|
||||
self.store.update_proposal(seq, state="executed", result=result)
|
||||
except ActionError as exc:
|
||||
self.store.update_proposal(seq, state="failed", result={"ok": False, "message": str(exc)})
|
||||
@@ -212,6 +218,9 @@ class MaintenanceAgent:
|
||||
result={"message": reason or "abgelehnt"})
|
||||
if p.kind == "promote_candidate":
|
||||
self.lifecycle.reject(p.params["candidate"], f"abgelehnt: {reason}")
|
||||
if p.kind == "ontology_change":
|
||||
params = {k: v for k, v in p.params.items() if k != "key"}
|
||||
self.store.set_ontology_decision(p.params["key"], "reject", params, by)
|
||||
return self._load(seq)
|
||||
|
||||
def run_deferred(self) -> list[Proposal]:
|
||||
@@ -318,6 +327,24 @@ class MaintenanceAgent:
|
||||
f"{cand['kind']} für {cand['slot']} übernehmen ({cand['target']})",
|
||||
reason="alle Prüfstufen bestanden", source="lifecycle"))
|
||||
|
||||
# 8. Ontologie ↔ Datenhaltung (SPEC-ontology §3)
|
||||
if self.ontology_check is not None:
|
||||
try:
|
||||
ontology_findings = self.ontology_check()
|
||||
except Exception as exc: # noqa: BLE001 – externe Quelle (n8n-API) darf die Routine nicht abbrechen
|
||||
report.findings.append(f"Ontologie-Abgleich nicht möglich: {type(exc).__name__}: {exc}")
|
||||
ontology_findings = []
|
||||
violations = [f for f in ontology_findings if not f.change]
|
||||
for f in violations[:c.ontology_max_findings]:
|
||||
report.findings.append(f"Ontologie [{f.severity}]: {f.summary}")
|
||||
if len(violations) > c.ontology_max_findings:
|
||||
report.findings.append(f"Ontologie: {len(violations) - c.ontology_max_findings} weitere Befunde "
|
||||
"(knaster ontology check)")
|
||||
for f in ontology_findings:
|
||||
if f.change:
|
||||
add(self.propose("ontology_change", {"key": f.key, **f.change}, f.summary, reason=f.detail,
|
||||
dedup_key=f"ontology:{f.key}"))
|
||||
|
||||
for p in self.run_deferred():
|
||||
report.auto_executed.append(f"{p.id} im Wartungsfenster ausgeführt: {p.summary}")
|
||||
|
||||
@@ -446,3 +473,13 @@ class MaintenanceAgent:
|
||||
return self._new("mark_slot", {"slot": args["slot"], "status": args["status"].lower(),
|
||||
"reason": f"Auftrag von {user}", "set_by": user},
|
||||
f"Slot {args['slot']} als {args['status'].lower()} markieren", user)
|
||||
|
||||
def _cmd_map_concept(self, args: dict[str, str], user: str) -> CommandReply:
|
||||
p = self._load(_seq(args["id"]))
|
||||
if p.kind != "ontology_change" or p.params.get("type") != "vocab_value":
|
||||
raise ValueError(f"{p.id} ist kein Vokabel-Vorschlag")
|
||||
if p.state != "open":
|
||||
raise ValueError(f"{p.id} ist {p.state}")
|
||||
self.store.update_proposal(p.seq, params={**p.params, "decision": "map", "concept": args["concept"]})
|
||||
done = self.approve(p.seq, by=user)
|
||||
return CommandReply(text=self._decided_text(done), proposals=[done])
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Ontologie der Datenhaltung: Modell, Export, Abgleich mit n8n, Übernahme freigegebener Entscheidungen."""
|
||||
|
||||
from .apply import apply_decisions
|
||||
from .check import OntologyFinding, check
|
||||
from .export import to_graph, to_html, to_turtle
|
||||
from .model import Ontology
|
||||
from .n8n import Snapshot, build_snapshot, fetch_workflows
|
||||
|
||||
__all__ = ["Ontology", "OntologyFinding", "Snapshot", "apply_decisions", "build_snapshot", "check",
|
||||
"fetch_workflows", "to_graph", "to_html", "to_turtle"]
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Freigegebene Entscheidungen in die Ontologie übernehmen (`knaster ontology apply`)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from .model import Concept, Ontology, OntologyClass
|
||||
|
||||
NEW_MODULE = "neu"
|
||||
|
||||
|
||||
def _slug(value: str) -> str:
|
||||
text = value.strip().lower()
|
||||
for a, b in (("ä", "ae"), ("ö", "oe"), ("ü", "ue"), ("ß", "ss"), ("+", "_"), ("&", "_")):
|
||||
text = text.replace(a, b)
|
||||
return re.sub(r"_+", "_", re.sub(r"[^a-z0-9]+", "_", text)).strip("_") or "wert"
|
||||
|
||||
|
||||
def _class_id(table: str) -> str:
|
||||
return "".join(part.capitalize() for part in table.split("_"))
|
||||
|
||||
|
||||
def apply_decision(onto: Ontology, key: str, decision: str, params: dict[str, Any]) -> str:
|
||||
"""Eine Entscheidung anwenden. decision ∈ {accept, map, reject}. Gibt eine Beschreibung zurück."""
|
||||
change = params.get("type")
|
||||
if decision == "reject":
|
||||
if change == "vocab_value":
|
||||
v = onto.vocabularies[params["vocabulary"]]
|
||||
if not v.is_rejected(params["value"]):
|
||||
v.rejected.append(params["value"])
|
||||
return f"{v.label}: „{params['value']}“ verworfen"
|
||||
if key not in onto.ignore:
|
||||
onto.ignore.append(key)
|
||||
return f"{key} wird künftig ignoriert"
|
||||
|
||||
if change == "vocab_value":
|
||||
v = onto.vocabularies[params["vocabulary"]]
|
||||
value = params["value"]
|
||||
if decision == "map":
|
||||
target = v.resolve(params["concept"])
|
||||
if target is None:
|
||||
raise KeyError(f"Konzept {params['concept']} gibt es in {v.label} nicht")
|
||||
if value.lower() not in (a.lower() for a in v.concepts[target].alt):
|
||||
v.concepts[target].alt.append(value)
|
||||
return f"{v.label}: „{value}“ → {target}"
|
||||
cid = _slug(value)
|
||||
if cid in v.concepts:
|
||||
raise ValueError(f"{v.label}: Konzept {cid} existiert bereits")
|
||||
v.concepts[cid] = Concept(label=value if value != cid else "")
|
||||
return f"{v.label}: neues Konzept {cid}"
|
||||
if change == "add_attribute":
|
||||
onto.classes[params["class"]].attributes[params["column"]] = params["sql_type"]
|
||||
return f"{params['class']}: Attribut {params['column']} ({params['sql_type']})"
|
||||
if change == "add_class":
|
||||
cid = _class_id(params["table"])
|
||||
if cid in onto.classes:
|
||||
raise ValueError(f"Klasse {cid} existiert bereits")
|
||||
onto.modules.setdefault(NEW_MODULE, "Noch nicht zugeordnet")
|
||||
onto.classes[cid] = OntologyClass(label=params["table"], module=NEW_MODULE, table=params["table"],
|
||||
definition="Automatisch aufgenommen; Definition und Beziehungen ergänzen.",
|
||||
attributes=dict(params.get("attributes") or {}))
|
||||
return f"neue Klasse {cid} für Tabelle {params['table']}"
|
||||
raise ValueError(f"Unbekannte Änderung {change}")
|
||||
|
||||
|
||||
def apply_decisions(onto: Ontology, decisions: list[dict[str, Any]]) -> tuple[list[str], list[str]]:
|
||||
"""→ (übernommene Schlüssel, Meldungen). Fehlerhafte Entscheidungen bleiben offen und erscheinen als Meldung."""
|
||||
applied, messages = [], []
|
||||
for d in decisions:
|
||||
try:
|
||||
messages.append(apply_decision(onto, d["key"], d["decision"], d.get("params") or {}))
|
||||
applied.append(d["key"])
|
||||
except (KeyError, ValueError) as exc:
|
||||
messages.append(f"{d['key']}: nicht übernommen ({exc})")
|
||||
if applied:
|
||||
onto.version += 1
|
||||
return applied, messages
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Abgleich Ontologie ↔ tatsächliche Datenhaltung in n8n (SPEC-ontology §3).
|
||||
|
||||
Zwei Arten von Ergebnissen:
|
||||
- Befunde mit `change`: die Ontologie sollte wachsen (neuer Vokabelwert, neue Spalte, neue Tabelle).
|
||||
Daraus werden Vorschläge zur Freigabe.
|
||||
- Befunde ohne `change`: die Datenhaltung verstößt gegen die Ontologie bzw. gegen KNOW
|
||||
(unbekannte Operation, SQL an KNOW vorbei, Konzept ohne Erzeuger, Tabelle ohne Schreiber).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .model import Ontology
|
||||
from .n8n import Snapshot, is_closed, literal_values, value_segments
|
||||
|
||||
Severity = Literal["info", "low", "medium", "high"]
|
||||
SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2, "info": 3}
|
||||
|
||||
|
||||
class OntologyFinding(BaseModel):
|
||||
key: str
|
||||
kind: str
|
||||
severity: Severity
|
||||
summary: str
|
||||
detail: str = ""
|
||||
change: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class _Obs:
|
||||
"""Beobachtete Werte eines Vokabulars: Wert → Quellen, dazu welche Quellen nur feste Werte liefern."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.values: dict[str, set[str]] = {}
|
||||
self.closed: set[str] = set() # Quellen mit geschlossener Wertemenge (aktiv)
|
||||
self.open: set[str] = set() # Quellen mit freien Werten (aktiv)
|
||||
self.produced: set[str] = set() # Werte aus aktiven Quellen
|
||||
|
||||
def add(self, value: str, source: str, active: bool) -> None:
|
||||
self.values.setdefault(value, set()).add(source)
|
||||
if active:
|
||||
self.produced.add(value)
|
||||
|
||||
|
||||
def _vocab_observations(onto: Ontology, snap: Snapshot) -> dict[str, _Obs]:
|
||||
assert snap.know is not None
|
||||
by_column = onto.vocab_columns()
|
||||
obs: dict[str, _Obs] = {}
|
||||
for call in snap.calls:
|
||||
if not call.operation:
|
||||
continue
|
||||
for table, cols in snap.know.writes.get(call.operation, {}).items():
|
||||
for col in cols:
|
||||
ref = f"{table}.{col}"
|
||||
for vid in by_column.get(ref, []):
|
||||
o = obs.setdefault(vid, _Obs())
|
||||
label = f"{call.workflow}{'' if call.active else ' (inaktiv)'}"
|
||||
for value in snap.know.assigned.get(call.operation, {}).get(ref, set()):
|
||||
o.add(value, f"KNOW {call.operation} (aufgerufen von {label})", call.active)
|
||||
if call.inferred or not call.payload:
|
||||
continue
|
||||
values = literal_values(call.payload, col)
|
||||
for value in values:
|
||||
o.add(value, f"{label} → {call.operation}.{col}", call.active)
|
||||
if call.active and value_segments(call.payload, col):
|
||||
(o.closed if is_closed(call.payload, col) else o.open).add(call.workflow)
|
||||
for ref, values in snap.know.seeds.items():
|
||||
for vid in by_column.get(ref, []):
|
||||
for value in values:
|
||||
obs.setdefault(vid, _Obs()).add(value, f"KNOW-Seed {ref}", False)
|
||||
return obs
|
||||
|
||||
|
||||
def check(onto: Ontology, snap: Snapshot, decided: Iterable[str] = ()) -> list[OntologyFinding]:
|
||||
skip = set(decided) | set(onto.ignore)
|
||||
out: list[OntologyFinding] = []
|
||||
know = snap.know
|
||||
if know is None:
|
||||
return [OntologyFinding(key="know:missing", kind="know_missing", severity="high",
|
||||
summary="KNOW-Workflow nicht gefunden (Code mit `const allowed` und Schema)")]
|
||||
|
||||
# 1. Schema ↔ Ontologie
|
||||
mapped = onto.tables()
|
||||
for table, cols in sorted(know.schema.items()):
|
||||
cid = mapped.get(table)
|
||||
if cid is None:
|
||||
out.append(OntologyFinding(
|
||||
key=f"table:{table}", kind="unmapped_table", severity="medium",
|
||||
summary=f"Tabelle {table} ({len(cols)} Spalten) fehlt in der Ontologie",
|
||||
detail="Als neue Klasse aufnehmen; Modul, Definition und Beziehungen danach im Repo ergänzen.",
|
||||
change={"type": "add_class", "table": table, "attributes": dict(cols)}))
|
||||
continue
|
||||
attrs = onto.classes[cid].attributes
|
||||
for col, typ in cols.items():
|
||||
if col not in attrs:
|
||||
out.append(OntologyFinding(
|
||||
key=f"column:{table}.{col}", kind="unmapped_column", severity="low",
|
||||
summary=f"Neue Spalte {table}.{col} ({typ}) – als Attribut von {cid} aufnehmen?",
|
||||
change={"type": "add_attribute", "class": cid, "column": col, "sql_type": typ}))
|
||||
for col in attrs:
|
||||
if col not in cols:
|
||||
out.append(OntologyFinding(key=f"gone:{table}.{col}", kind="missing_column", severity="low",
|
||||
summary=f"Attribut {cid}.{col} gibt es in KNOW nicht (mehr)"))
|
||||
for cid, c in onto.classes.items():
|
||||
if c.table and c.table not in know.schema:
|
||||
out.append(OntologyFinding(key=f"gone:{c.table}", kind="missing_table", severity="medium",
|
||||
summary=f"Klasse {cid} verweist auf Tabelle {c.table}, die KNOW nicht anlegt"))
|
||||
|
||||
# 2. Aufrufe unbekannter Operationen (pro Workflow und Operation zusammengefasst)
|
||||
unknown: dict[tuple[str, str], list[str]] = {}
|
||||
active_of: dict[str, bool] = {}
|
||||
for call in snap.calls:
|
||||
if call.operation and not call.inferred and call.operation not in know.allowed:
|
||||
unknown.setdefault((call.workflow, call.operation), []).append(call.node)
|
||||
active_of[call.workflow] = call.active
|
||||
for (wf, op), nodes in sorted(unknown.items()):
|
||||
out.append(OntologyFinding(
|
||||
key=f"op:{wf}:{op}", kind="unknown_operation", severity="high" if active_of[wf] else "low",
|
||||
summary=f"{wf}{'' if active_of[wf] else ' (inaktiv)'} ruft `{op}` auf, die Operation gibt es in KNOW nicht",
|
||||
detail=f"Knoten: {', '.join(sorted(set(nodes)))}. KNOW lehnt mit „Nicht erlaubte KNOW operation“ ab."))
|
||||
|
||||
# 3. SQL an KNOW vorbei
|
||||
for d in snap.direct_sql:
|
||||
out.append(OntologyFinding(
|
||||
key=f"sql:{d.workflow}:{d.node}", kind="direct_sql",
|
||||
severity=("medium" if d.writes else "low") if d.active else "info",
|
||||
summary=f"{d.workflow} {'schreibt' if d.writes else 'liest'} per eigenem SQL in "
|
||||
f"{', '.join(sorted(d.tables))} (an KNOW vorbei)",
|
||||
detail=f"Knoten „{d.node}“"))
|
||||
|
||||
# 4. Vokabulare: neue Werte und Konzepte, die keine feste Zuordnung erzeugt
|
||||
obs = _vocab_observations(onto, snap)
|
||||
for vid, o in sorted(obs.items()):
|
||||
v = onto.vocabularies[vid]
|
||||
for value, sources in sorted(o.values.items()):
|
||||
if v.resolve(value) or v.is_rejected(value):
|
||||
continue
|
||||
only_inactive = all("(inaktiv)" in s_ for s_ in sources)
|
||||
out.append(OntologyFinding(
|
||||
key=f"vocab:{vid}:{value.lower()}", kind="vocab_value", severity="low" if only_inactive else "medium",
|
||||
summary=f"{v.label}: neuer Wert „{value}“",
|
||||
detail="Quelle: " + "; ".join(sorted(sources)) + ". Als Konzept aufnehmen, einem Konzept "
|
||||
"zuordnen („ordne P-x <konzept> zu“) oder verwerfen.",
|
||||
change={"type": "vocab_value", "vocabulary": vid, "value": value, "sources": sorted(sources)}))
|
||||
if o.closed:
|
||||
produced = {v.resolve(x) for x in o.produced}
|
||||
missing = [cid for cid, c in v.concepts.items() if cid not in produced and not c.technical]
|
||||
if missing:
|
||||
extra = f"; freie Werte kommen zusätzlich aus {', '.join(sorted(o.open))}" if o.open else ""
|
||||
out.append(OntologyFinding(
|
||||
key=f"unused:{vid}", kind="concept_without_producer", severity="info",
|
||||
summary=f"{v.label}: {', '.join(missing)} erzeugt keine feste Zuordnung "
|
||||
f"(geprüft: {', '.join(sorted(o.closed))}{extra})"))
|
||||
|
||||
# 5. Tabellenklassen ohne aktiven Schreibweg
|
||||
active_ops = {c.operation for c in snap.calls if c.active and c.operation} # inkl. abgeleiteter
|
||||
for cid, c in onto.classes.items():
|
||||
if not c.table or c.table not in know.schema:
|
||||
continue
|
||||
writers = {op for op, tables in know.writes.items() if c.table in tables and op != "setup_schema"}
|
||||
seeded = any(ref.startswith(f"{c.table}.") for ref in know.seeds)
|
||||
if not writers and seeded:
|
||||
continue # Konfigurationstabelle, die setup_schema befüllt
|
||||
if not writers:
|
||||
out.append(OntologyFinding(key=f"nowriter:{c.table}", kind="no_write_operation", severity="info",
|
||||
summary=f"{cid} ({c.table}): KNOW hat keine Schreib-Operation"))
|
||||
elif not writers & active_ops:
|
||||
out.append(OntologyFinding(key=f"nowriter:{c.table}", kind="no_active_writer", severity="info",
|
||||
summary=f"{cid} ({c.table}): kein aktiver Workflow ruft "
|
||||
f"{', '.join(sorted(writers))} auf"))
|
||||
|
||||
out = [f for f in out if f.key not in skip]
|
||||
out.sort(key=lambda f: (SEVERITY_ORDER[f.severity], f.kind, f.key))
|
||||
return out
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Erzeugte Artefakte aus der Ontologie: OWL/Turtle, Graph-JSON für die Ansicht, eigenständige HTML-Seite."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .model import Ontology
|
||||
|
||||
XSD = {"TEXT": "xsd:string", "NUMERIC": "xsd:decimal", "INTEGER": "xsd:integer", "BIGINT": "xsd:integer",
|
||||
"SMALLINT": "xsd:integer", "BIGSERIAL": "xsd:integer", "BOOLEAN": "xsd:boolean", "DATE": "xsd:date",
|
||||
"TIMESTAMPTZ": "xsd:dateTime", "JSONB": "rdf:JSON", "DOUBLE PRECISION": "xsd:double"}
|
||||
BOOKKEEPING = {"id", "created_at", "updated_at"}
|
||||
|
||||
|
||||
def _lit(text: str) -> str:
|
||||
return '"' + text.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ") + '"@de'
|
||||
|
||||
|
||||
def _local(text: str) -> str:
|
||||
text = (text.replace("+", "plus").replace("ä", "ae").replace("ö", "oe").replace("ü", "ue")
|
||||
.replace("Ä", "Ae").replace("Ö", "Oe").replace("Ü", "Ue").replace("ß", "ss"))
|
||||
return re.sub(r"[^A-Za-z0-9_]", "_", text)
|
||||
|
||||
|
||||
def _union(ids: list[str], prefix: str) -> str:
|
||||
if len(ids) == 1:
|
||||
return f"{prefix}:{ids[0]}"
|
||||
return "[ a owl:Class ; owl:unionOf ( " + " ".join(f"{prefix}:{i}" for i in ids) + " ) ]"
|
||||
|
||||
|
||||
def relation_columns(onto: Ontology, cid: str) -> set[str]:
|
||||
"""Spalten einer Tabellenklasse, über die eine Beziehung läuft (keine Attribute im engeren Sinn)."""
|
||||
c = onto.classes[cid]
|
||||
cols: set[str] = set()
|
||||
for r in onto.relations:
|
||||
if r.source == cid and c.table:
|
||||
cols |= set(re.findall(r"\b([a-z_]+)\b", r.via.split("=")[0])) & set(c.attributes)
|
||||
return cols
|
||||
|
||||
|
||||
def to_turtle(onto: Ontology) -> str:
|
||||
p = onto.id
|
||||
out = [f"@prefix {p}: <{onto.namespace}> .", "@prefix owl: <http://www.w3.org/2002/07/owl#> .",
|
||||
"@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .",
|
||||
"@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .",
|
||||
"@prefix skos: <http://www.w3.org/2004/02/skos/core#> .",
|
||||
"@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .", "",
|
||||
f"<{onto.namespace.rstrip('#/')}> a owl:Ontology ;",
|
||||
f" rdfs:label {_lit(onto.label or onto.id)} ;",
|
||||
f' owl:versionInfo "{onto.version}" .', "",
|
||||
f"{p}:quelleTabelle a owl:AnnotationProperty .", f"{p}:viaSpalte a owl:AnnotationProperty .", ""]
|
||||
for cid, c in onto.classes.items():
|
||||
lines = [f"{p}:{cid} a owl:Class", f" rdfs:label {_lit(c.label)}"]
|
||||
if c.definition:
|
||||
lines.append(f" rdfs:comment {_lit(c.definition)}")
|
||||
if c.table:
|
||||
lines.append(f' {p}:quelleTabelle "{c.table}"')
|
||||
if c.subclass_of:
|
||||
lines.append(f" rdfs:subClassOf {p}:{c.subclass_of}")
|
||||
out.append(" ;\n".join(lines) + " .\n")
|
||||
props: dict[str, dict[str, list[str]]] = {}
|
||||
for r in onto.relations:
|
||||
e = props.setdefault(r.name, {"dom": [], "rng": [], "via": []})
|
||||
for k, v in (("dom", r.source), ("rng", r.target)):
|
||||
if v not in e[k]:
|
||||
e[k].append(v)
|
||||
e["via"].append(f"{r.source}→{r.target}: {r.via}" if r.via else f"{r.source}→{r.target}")
|
||||
for name, e in props.items():
|
||||
out.append(f"{p}:{name} a owl:ObjectProperty ;\n rdfs:domain {_union(e['dom'], p)} ;\n"
|
||||
f" rdfs:range {_union(e['rng'], p)} ;\n"
|
||||
+ " ;\n".join(f" {p}:viaSpalte {_lit(v)}" for v in e["via"]) + " .\n")
|
||||
dprops: dict[str, dict[str, Any]] = {}
|
||||
for cid, c in onto.classes.items():
|
||||
skip = relation_columns(onto, cid) | BOOKKEEPING
|
||||
for col, typ in c.attributes.items():
|
||||
if col in skip:
|
||||
continue
|
||||
d = dprops.setdefault(col, {"dom": [], "rng": XSD.get(typ.split("(")[0].strip(), "xsd:string")})
|
||||
d["dom"].append(cid)
|
||||
for col, d in sorted(dprops.items()):
|
||||
out.append(f"{p}:{_local(col)} a owl:DatatypeProperty ;\n rdfs:domain {_union(d['dom'], p)} ;\n"
|
||||
f" rdfs:range {d['rng']} .\n")
|
||||
for vid, v in onto.vocabularies.items():
|
||||
scheme = f"{p}:{_local(vid)}Schema"
|
||||
out.append(f"{scheme} a skos:ConceptScheme ;\n rdfs:label {_lit(v.label)} .\n")
|
||||
for cid, c in v.concepts.items():
|
||||
types = f"skos:Concept, {p}:{v.class_}" if v.class_ else "skos:Concept"
|
||||
lines = [f"{p}:{_local(vid)}_{_local(cid)} a {types}", f" skos:inScheme {scheme}",
|
||||
f" skos:notation \"{cid}\"", f" skos:prefLabel {_lit(c.label or cid)}"]
|
||||
lines += [f" skos:altLabel {_lit(a)}" for a in c.alt]
|
||||
if c.note:
|
||||
lines.append(f" skos:scopeNote {_lit(c.note)}")
|
||||
out.append(" ;\n".join(lines) + " .\n")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def to_graph(onto: Ontology, generated: str | None = None) -> dict[str, Any]:
|
||||
"""Datenformat der Ansicht (ontology/viewer.html.tmpl)."""
|
||||
enums: dict[str, dict[str, list[str]]] = {}
|
||||
individuals: dict[str, list[str]] = {}
|
||||
for v in onto.vocabularies.values():
|
||||
values = [c.label or cid for cid, c in v.concepts.items()]
|
||||
if v.class_:
|
||||
individuals.setdefault(v.class_, []).extend(x for x in values if x not in individuals.get(v.class_, []))
|
||||
else:
|
||||
for ref in v.columns:
|
||||
table, _, col = ref.partition(".")
|
||||
owner = onto.class_for_table(table)
|
||||
if owner:
|
||||
enums.setdefault(owner, {})[col] = list(v.concepts)
|
||||
classes = []
|
||||
for cid, c in onto.classes.items():
|
||||
skip = relation_columns(onto, cid) | BOOKKEEPING
|
||||
classes.append({"id": cid, "label": c.label, "module": c.module, "table": c.table, "def": c.definition,
|
||||
"key": c.key or "–", "attrs": [[k, t] for k, t in c.attributes.items() if k not in skip],
|
||||
"individuals": individuals.get(cid, []), "enums": enums.get(cid, {}), "sens": c.sensitive,
|
||||
"abstract": c.abstract})
|
||||
relations = [r.model_dump() for r in onto.relations]
|
||||
relations += [{"source": cid, "name": "subClassOf", "target": c.subclass_of, "via": "", "kind": "subclass"}
|
||||
for cid, c in onto.classes.items() if c.subclass_of]
|
||||
return {"classes": classes, "relations": relations, "version": onto.version,
|
||||
"generated": generated or date.today().isoformat()}
|
||||
|
||||
|
||||
def to_html(onto: Ontology, template: str | Path, generated: str | None = None) -> str:
|
||||
tmpl = Path(template).read_text(encoding="utf-8")
|
||||
data = json.dumps(to_graph(onto, generated), ensure_ascii=False).replace("</", "<\\/")
|
||||
ttl = json.dumps(to_turtle(onto), ensure_ascii=False).replace("</", "<\\/")
|
||||
return tmpl.replace("__DATA__", data).replace("__TTL__", ttl)
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Ontologie als Quelle der Wahrheit: Kern (Klassen, Beziehungen), Vokabulare, Abbildung auf Tabellen.
|
||||
|
||||
Drei Ebenen mit unterschiedlicher Pflege (SPEC-ontology §1):
|
||||
- Kern: bewusst gepflegt, Änderungen per Commit.
|
||||
- Vokabulare: wachsen über Vorschläge aus der Service-Routine (beobachtet → vorgeschlagen → bestätigt).
|
||||
- Instanzen: leben in den Daten, nicht in dieser Datei.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..util import load_yaml
|
||||
|
||||
RelationKind = Literal["key", "soft", "external", "subclass"]
|
||||
|
||||
|
||||
class OntologyClass(BaseModel):
|
||||
label: str
|
||||
module: str
|
||||
definition: str = ""
|
||||
table: str | None = None # None = implizit (nur als Wert in Spalten) oder externes System
|
||||
key: str = ""
|
||||
sensitive: str = ""
|
||||
abstract: bool = False
|
||||
subclass_of: str | None = None
|
||||
attributes: dict[str, str] = Field(default_factory=dict) # Spalte → SQL-Typ (Tabellenklassen)
|
||||
|
||||
|
||||
class Relation(BaseModel):
|
||||
source: str
|
||||
name: str
|
||||
target: str
|
||||
via: str = ""
|
||||
kind: RelationKind = "key"
|
||||
|
||||
|
||||
class Concept(BaseModel):
|
||||
label: str = ""
|
||||
alt: list[str] = Field(default_factory=list) # weitere Schreibweisen, die auf dieses Konzept zeigen
|
||||
note: str = ""
|
||||
technical: bool = False # kein fachlicher Wert (z. B. Rückfall „default“); zählt nicht als „fehlt“
|
||||
|
||||
|
||||
class Vocabulary(BaseModel):
|
||||
label: str
|
||||
class_: str | None = Field(default=None, alias="class")
|
||||
columns: list[str] = Field(default_factory=list) # "tabelle.spalte", in denen Werte dieses Vokabulars stehen
|
||||
concepts: dict[str, Concept] = Field(default_factory=dict)
|
||||
rejected: list[str] = Field(default_factory=list) # bewusst verworfene Werte (kein Konzept)
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolve(self, value: str) -> str | None:
|
||||
"""Wert → Konzept-ID (über ID, Label oder alternative Schreibweise), sonst None."""
|
||||
v = value.strip().lower()
|
||||
for cid, c in self.concepts.items():
|
||||
if v == cid.lower() or v == c.label.lower() or v in (a.lower() for a in c.alt):
|
||||
return cid
|
||||
return None
|
||||
|
||||
def is_rejected(self, value: str) -> bool:
|
||||
return value.strip().lower() in (r.lower() for r in self.rejected)
|
||||
|
||||
|
||||
class Ontology(BaseModel):
|
||||
id: str = "know"
|
||||
label: str = ""
|
||||
namespace: str = "urn:example:onto#"
|
||||
version: int = 1
|
||||
modules: dict[str, str] = Field(default_factory=dict)
|
||||
classes: dict[str, OntologyClass] = Field(default_factory=dict)
|
||||
relations: list[Relation] = Field(default_factory=list)
|
||||
vocabularies: dict[str, Vocabulary] = Field(default_factory=dict)
|
||||
ignore: list[str] = Field(default_factory=list) # bewusst hingenommene Befunde (Schlüssel aus `check`)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: str | Path) -> Ontology:
|
||||
return cls(**load_yaml(path))
|
||||
|
||||
def save(self, path: str | Path, header: str = "") -> None:
|
||||
data = {"id": self.id, "version": self.version, **self.model_dump(by_alias=True, exclude_defaults=True)}
|
||||
text = yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=120)
|
||||
Path(path).write_text(header + text, encoding="utf-8")
|
||||
|
||||
# --- Abfragen ---------------------------------------------------------------------------
|
||||
def class_for_table(self, table: str) -> str | None:
|
||||
return next((cid for cid, c in self.classes.items() if c.table == table), None)
|
||||
|
||||
def tables(self) -> dict[str, str]:
|
||||
return {c.table: cid for cid, c in self.classes.items() if c.table}
|
||||
|
||||
def vocabularies_for(self, table: str, column: str) -> list[str]:
|
||||
ref = f"{table}.{column}"
|
||||
return [vid for vid, v in self.vocabularies.items() if ref in v.columns]
|
||||
|
||||
def vocab_columns(self) -> dict[str, list[str]]:
|
||||
out: dict[str, list[str]] = {}
|
||||
for vid, v in self.vocabularies.items():
|
||||
for ref in v.columns:
|
||||
out.setdefault(ref, []).append(vid)
|
||||
return out
|
||||
|
||||
# --- Prüfung ----------------------------------------------------------------------------
|
||||
def validate_model(self) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for cid, c in self.classes.items():
|
||||
if c.module not in self.modules:
|
||||
errors.append(f"Klasse {cid}: Modul {c.module} unbekannt")
|
||||
if c.subclass_of and c.subclass_of not in self.classes:
|
||||
errors.append(f"Klasse {cid}: Oberklasse {c.subclass_of} unbekannt")
|
||||
if c.attributes and not c.table:
|
||||
errors.append(f"Klasse {cid}: Attribute ohne Tabelle")
|
||||
seen_tables: dict[str, str] = {}
|
||||
for cid, c in self.classes.items():
|
||||
if c.table:
|
||||
if c.table in seen_tables:
|
||||
errors.append(f"Tabelle {c.table} ist {seen_tables[c.table]} und {cid} zugeordnet")
|
||||
seen_tables[c.table] = cid
|
||||
for r in self.relations:
|
||||
for end in (r.source, r.target):
|
||||
if end not in self.classes:
|
||||
errors.append(f"Beziehung {r.source} {r.name} {r.target}: Klasse {end} unbekannt")
|
||||
for vid, v in self.vocabularies.items():
|
||||
if v.class_ and v.class_ not in self.classes:
|
||||
errors.append(f"Vokabular {vid}: Klasse {v.class_} unbekannt")
|
||||
for ref in v.columns:
|
||||
table, _, column = ref.partition(".")
|
||||
owner = self.class_for_table(table)
|
||||
if owner is None:
|
||||
errors.append(f"Vokabular {vid}: Tabelle {table} keiner Klasse zugeordnet")
|
||||
elif column not in self.classes[owner].attributes:
|
||||
errors.append(f"Vokabular {vid}: Spalte {ref} fehlt in Klasse {owner}")
|
||||
labels: dict[str, str] = {}
|
||||
for cid, c in v.concepts.items():
|
||||
for name in [cid, c.label, *c.alt]:
|
||||
key = name.strip().lower()
|
||||
if key and key in labels and labels[key] != cid:
|
||||
errors.append(f"Vokabular {vid}: „{name}“ zeigt auf {labels[key]} und {cid}")
|
||||
labels.setdefault(key, cid)
|
||||
for r in v.rejected:
|
||||
if v.resolve(r):
|
||||
errors.append(f"Vokabular {vid}: „{r}“ ist zugleich verworfen und Konzept {v.resolve(r)}")
|
||||
return errors
|
||||
@@ -0,0 +1,325 @@
|
||||
"""Momentaufnahme der n8n-Datenhaltung: KNOW-Spezifikation, Aufrufe, direktes SQL (nur lesend).
|
||||
|
||||
KNOW ist ein n8n-Sub-Workflow, der feste Operationen auf SQL-Vorlagen abbildet. Seine Spezifikation
|
||||
(erlaubte Operationen, Schema, geschriebene Spalten) wird aus dem Code des Workflows gelesen, nicht aus der DB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
_ALLOWED = re.compile(r"const allowed\s*=\s*\[([^\]]+)\]")
|
||||
_OP = re.compile(r"""if\s*\(\s*operation\s*===\s*['"]([a-z_0-9]+)['"]""")
|
||||
_INSERT = re.compile(r"INSERT INTO \$\{t\.(\w+)\}(?:\s+AS\s+\w+)?\s*\(")
|
||||
_UPDATE = re.compile(r"UPDATE \$\{t\.(\w+)\}(?:\s+AS)?(?:\s+\w+)?\s+SET\s")
|
||||
_DELETE = re.compile(r"DELETE FROM \$\{t\.(\w+)\}")
|
||||
_WRITE_SQL = re.compile(r"\b(INSERT\s+INTO|UPDATE|DELETE\s+FROM)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KnowSpec:
|
||||
workflow_id: str
|
||||
workflow_name: str
|
||||
allowed: set[str] = field(default_factory=set)
|
||||
schema: dict[str, dict[str, str]] = field(default_factory=dict) # Tabelle → Spalte → Typ
|
||||
writes: dict[str, dict[str, set[str]]] = field(default_factory=dict) # Operation → Tabelle → Spalten
|
||||
seeds: dict[str, set[str]] = field(default_factory=dict) # "tabelle.spalte" → Literalwerte aus Seed-INSERTs
|
||||
assigned: dict[str, dict[str, set[str]]] = field(default_factory=dict) # Operation → "tab.spalte" → Literale
|
||||
|
||||
|
||||
@dataclass
|
||||
class KnowCall:
|
||||
workflow: str
|
||||
active: bool
|
||||
node: str
|
||||
operation: str | None # None = erst zur Laufzeit bestimmt
|
||||
payload: str = ""
|
||||
inferred: bool = False # Operation aus Literalen im Workflow abgeleitet (Laufzeit-Operation)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DirectSql:
|
||||
workflow: str
|
||||
active: bool
|
||||
node: str
|
||||
query: str
|
||||
tables: set[str]
|
||||
writes: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class Snapshot:
|
||||
know: KnowSpec | None
|
||||
calls: list[KnowCall]
|
||||
direct_sql: list[DirectSql]
|
||||
workflow_count: int
|
||||
taken_at: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
# --- Hilfsfunktionen für JS/SQL-Text ------------------------------------------------------------
|
||||
def _balanced(text: str, start: int) -> tuple[str, int]:
|
||||
"""text[start] ist eine öffnende Klammer → (Inhalt, Index der schließenden Klammer)."""
|
||||
depth = 0
|
||||
for i in range(start, len(text)):
|
||||
if text[i] in "([{":
|
||||
depth += 1
|
||||
elif text[i] in ")]}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start + 1:i], i
|
||||
return text[start + 1:], len(text)
|
||||
|
||||
|
||||
def _split_top(text: str) -> list[str]:
|
||||
parts, depth, cur, quote = [], 0, "", ""
|
||||
for ch in text:
|
||||
if quote:
|
||||
cur += ch
|
||||
if ch == quote:
|
||||
quote = ""
|
||||
continue
|
||||
if ch in "'\"`":
|
||||
quote = ch
|
||||
elif ch in "([{":
|
||||
depth += 1
|
||||
elif ch in ")]}":
|
||||
depth -= 1
|
||||
if ch == "," and depth == 0:
|
||||
parts.append(cur)
|
||||
cur = ""
|
||||
else:
|
||||
cur += ch
|
||||
if cur.strip():
|
||||
parts.append(cur)
|
||||
return parts
|
||||
|
||||
|
||||
def _column_type(definition: str) -> str:
|
||||
words = definition.split()
|
||||
if len(words) < 2:
|
||||
return "TEXT"
|
||||
return "DOUBLE PRECISION" if words[1].upper() == "DOUBLE" else words[1].upper()
|
||||
|
||||
|
||||
def _take_sets(writes: dict[str, set[str]], assigned: dict[str, set[str]], table: str, setp: str) -> None:
|
||||
"""SET-Klausel auswerten: geschriebene Spalten und dabei gesetzte Literale merken."""
|
||||
for part in _split_top(setp):
|
||||
if "=" not in part:
|
||||
continue
|
||||
col, _, expr = part.partition("=")
|
||||
col = re.sub(r"^\w+\.", "", col.strip())
|
||||
writes.setdefault(table, set()).add(col)
|
||||
lits = {x for x in re.findall(r"'([^'$]*)'", expr) if x and "{" not in x}
|
||||
if lits:
|
||||
assigned.setdefault(f"{table}.{col}", set()).update(lits)
|
||||
|
||||
|
||||
def parse_know_code(code: str, workflow_id: str = "", workflow_name: str = "") -> KnowSpec:
|
||||
spec = KnowSpec(workflow_id=workflow_id, workflow_name=workflow_name)
|
||||
m = _ALLOWED.search(code)
|
||||
if m:
|
||||
spec.allowed = set(re.findall(r"['\"]([a-z_0-9]+)['\"]", m.group(1)))
|
||||
|
||||
for m in re.finditer(r"CREATE TABLE IF NOT EXISTS \$\{t\.(\w+)\}\s*\(", code):
|
||||
body, _ = _balanced(code, m.end() - 1)
|
||||
cols = spec.schema.setdefault(m.group(1), {})
|
||||
for part in _split_top(body):
|
||||
part = " ".join(part.split())
|
||||
if not part or re.match(r"^(UNIQUE|PRIMARY KEY|CONSTRAINT|CHECK)\b", part, re.IGNORECASE):
|
||||
continue
|
||||
cols.setdefault(part.split()[0], _column_type(part))
|
||||
for m in re.finditer(r"ALTER TABLE \$\{t\.(\w+)\}([^;`]*)", code):
|
||||
for a in re.finditer(r"ADD COLUMN IF NOT EXISTS\s+(\w+)\s+([^,]+)", m.group(2)):
|
||||
spec.schema.setdefault(m.group(1), {}).setdefault(a.group(1), _column_type("x " + a.group(2)))
|
||||
|
||||
starts = [(m.start(), m.group(1)) for m in _OP.finditer(code)]
|
||||
for i, (pos, op) in enumerate(starts):
|
||||
block = code[pos:starts[i + 1][0] if i + 1 < len(starts) else len(code)]
|
||||
writes = spec.writes.setdefault(op, {})
|
||||
assigned = spec.assigned.setdefault(op, {})
|
||||
for m in _INSERT.finditer(block):
|
||||
inner, end = _balanced(block, m.end() - 1)
|
||||
vm = re.match(r"\s*VALUES\s*\(", block[end + 1:end + 40])
|
||||
if vm and "${" not in _balanced(block, end + 1 + vm.end() - 1)[0]:
|
||||
continue # reines Literal-INSERT = Seed aus setup_schema, keine Schreiboperation von `op`
|
||||
writes.setdefault(m.group(1), set()).update(c.strip() for c in inner.split(",") if c.strip())
|
||||
rest = block[end:end + 6000]
|
||||
conflict = re.search(r"ON CONFLICT\s*\([^)]*\)\s*DO\s+UPDATE SET\s", rest)
|
||||
if conflict:
|
||||
tail = rest[conflict.end():]
|
||||
stop = re.search(r"\sRETURNING\s|\sWHERE\s|;|`", tail)
|
||||
_take_sets(writes, assigned, m.group(1), tail[:stop.start()] if stop else tail)
|
||||
for m in _UPDATE.finditer(block):
|
||||
tail = block[m.end():m.end() + 6000]
|
||||
stop = re.search(r"\sWHERE\s|\sRETURNING\s|\sFROM\s|;|`", tail)
|
||||
_take_sets(writes, assigned, m.group(1), tail[:stop.start()] if stop else tail)
|
||||
for m in _DELETE.finditer(block):
|
||||
writes.setdefault(m.group(1), set())
|
||||
|
||||
# Literalwerte in INSERTs (Seeds wie die Hinweis-Konfiguration, feste Werte wie alias_type 'learned')
|
||||
for m in _INSERT.finditer(code):
|
||||
inner, end = _balanced(code, m.end() - 1)
|
||||
cols = [c.strip() for c in inner.split(",")]
|
||||
vm = re.match(r"\s*VALUES\s*\(", code[end + 1:end + 40])
|
||||
if not vm:
|
||||
continue
|
||||
values, _ = _balanced(code, end + 1 + vm.end() - 1)
|
||||
for col, val in zip(cols, _split_top(values), strict=False):
|
||||
lit = re.fullmatch(r"\s*'([^'$]*)'\s*", val)
|
||||
if lit and lit.group(1):
|
||||
spec.seeds.setdefault(f"{m.group(1)}.{col}", set()).add(lit.group(1))
|
||||
return spec
|
||||
|
||||
|
||||
def value_segments(expression: str, key: str) -> list[str]:
|
||||
"""Ausdrücke, die `key` auf der obersten Ebene des Payload-Objekts zugewiesen werden.
|
||||
|
||||
Verschachtelte Schlüssel (z. B. `raw_signals: { category: … }`) gehören zu einer JSON-Spalte, nicht zu `key`.
|
||||
"""
|
||||
anchor = expression.find("JSON.stringify(")
|
||||
brace = expression.find("{", anchor if anchor >= 0 else 0)
|
||||
while brace >= 0 and expression[brace + 1:brace + 2] == "{": # n8n-Ausdruck „={{ … }}“ überspringen
|
||||
brace = expression.find("{", brace + 2)
|
||||
if brace < 0:
|
||||
return []
|
||||
body, _ = _balanced(expression, brace)
|
||||
out = []
|
||||
for part in _split_top(body):
|
||||
m = re.match(rf"\s*['\"]?{re.escape(key)}['\"]?\s*:", part)
|
||||
if m:
|
||||
out.append(part[m.end():])
|
||||
return out
|
||||
|
||||
|
||||
def literal_values(expression: str, key: str) -> set[str]:
|
||||
"""Literale, die ein Payload-Ausdruck für `key` liefern kann.
|
||||
|
||||
Erkennt `key: 'x'`, `key: a || 'x'` und Zuordnungstabellen `key: ({'Anzeige': 'wert', …}[expr] || 'rest')`.
|
||||
Literale in Knotenverweisen wie `$('Knoten')` zählen nicht.
|
||||
"""
|
||||
found: set[str] = set()
|
||||
for seg in value_segments(expression, key):
|
||||
seg = re.sub(r"\$\(\s*(['\"]).*?\1\s*\)", "", seg) # $('Knoten') entfernen
|
||||
pairs = re.findall(r"(['\"])([^'\"]*)\1\s*:\s*(['\"])([^'\"]*)\3", seg)
|
||||
found |= {p[3] for p in pairs}
|
||||
seg = re.sub(r"(['\"])[^'\"]*\1\s*:\s*(['\"])[^'\"]*\2", "", seg)
|
||||
for lit in re.findall(r"(['\"])([^'\"]*)\1(?!\s*:)", seg):
|
||||
if lit[1] and not re.search(r"[{}$]", lit[1]):
|
||||
found.add(lit[1])
|
||||
return found
|
||||
|
||||
|
||||
def is_closed(expression: str, key: str) -> bool:
|
||||
"""True, wenn `key` nur feste Werte annehmen kann (Literal oder Zuordnungstabelle mit Literal-Rückfall)."""
|
||||
segments = value_segments(expression, key)
|
||||
if not segments:
|
||||
return False
|
||||
for seg in segments:
|
||||
seg = re.sub(r"(['\"])(?:(?!\1).)*\1", "L", seg) # Literale
|
||||
while True: # Zuordnungstabelle {L:L, …}[…] und Klammern entfernen
|
||||
new = re.sub(r"\{[L:,\s]*\}\s*\[[^\[\]]*\]", "L", seg)
|
||||
new = re.sub(r"\(\s*([L|\s]*)\s*\)", r"\1", new)
|
||||
if new == seg:
|
||||
break
|
||||
seg = new
|
||||
if not (re.fullmatch(r"[L|\s]+", seg) and "L" in seg):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# --- Workflows auswerten -----------------------------------------------------------------------
|
||||
def find_know(workflows: list[dict[str, Any]]) -> KnowSpec | None:
|
||||
for wf in workflows:
|
||||
for node in wf.get("nodes", []):
|
||||
code = (node.get("parameters") or {}).get("jsCode") or ""
|
||||
if "const allowed" in code and "CREATE TABLE IF NOT EXISTS" in code:
|
||||
return parse_know_code(code, wf.get("id", ""), wf.get("name", ""))
|
||||
return None
|
||||
|
||||
|
||||
def _inputs(node: dict[str, Any]) -> dict[str, Any]:
|
||||
return ((node.get("parameters") or {}).get("workflowInputs") or {}).get("value") or {}
|
||||
|
||||
|
||||
def extract_calls(workflows: list[dict[str, Any]], know_id: str, allowed: set[str] | None = None) -> list[KnowCall]:
|
||||
calls = []
|
||||
for wf in workflows:
|
||||
if wf.get("id") == know_id:
|
||||
continue
|
||||
for node in wf.get("nodes", []):
|
||||
if not node.get("type", "").endswith("executeWorkflow"):
|
||||
continue
|
||||
if know_id not in json.dumps(node.get("parameters") or {}):
|
||||
continue
|
||||
values = _inputs(node)
|
||||
op = str(values.get("operation", "")).strip()
|
||||
literal = op if re.fullmatch(r"[a-z_0-9]+", op) else None
|
||||
if literal is None and allowed:
|
||||
# Operation wird zur Laufzeit gesetzt: Kandidaten sind bekannte Operationen, die im Workflow als
|
||||
# Literal vorkommen (z. B. in einem Code-Knoten davor)
|
||||
text = json.dumps(wf.get("nodes", []))
|
||||
for cand in sorted(o for o in allowed if re.search(rf"""['"\\]{o}['"\\]""", text)):
|
||||
calls.append(KnowCall(wf.get("name", ""), bool(wf.get("active")), node.get("name", ""), cand,
|
||||
"", inferred=True))
|
||||
continue
|
||||
calls.append(KnowCall(wf.get("name", ""), bool(wf.get("active")), node.get("name", ""), literal,
|
||||
str(values.get("payload_json", ""))))
|
||||
return calls
|
||||
|
||||
|
||||
def extract_direct_sql(workflows: list[dict[str, Any]], know_id: str,
|
||||
schema_name: str = "knowledge") -> list[DirectSql]:
|
||||
out = []
|
||||
ref = re.compile(rf"""["']?{re.escape(schema_name)}["']?\s*\.\s*["']?(\w+)""", re.IGNORECASE)
|
||||
for wf in workflows:
|
||||
if wf.get("id") == know_id:
|
||||
continue
|
||||
for node in wf.get("nodes", []):
|
||||
if not node.get("type", "").endswith(".postgres"):
|
||||
continue
|
||||
query = str((node.get("parameters") or {}).get("query", ""))
|
||||
tables = set(ref.findall(query))
|
||||
if not tables:
|
||||
continue
|
||||
out.append(DirectSql(wf.get("name", ""), bool(wf.get("active")), node.get("name", ""), query, tables,
|
||||
bool(_WRITE_SQL.search(query))))
|
||||
return out
|
||||
|
||||
|
||||
def build_snapshot(workflows: list[dict[str, Any]]) -> Snapshot:
|
||||
know = find_know(workflows)
|
||||
know_id = know.workflow_id if know else ""
|
||||
return Snapshot(know=know, calls=extract_calls(workflows, know_id, know.allowed) if know else [],
|
||||
direct_sql=extract_direct_sql(workflows, know_id), workflow_count=len(workflows))
|
||||
|
||||
|
||||
def fetch_workflows(host: str, api_key: str, *, client: httpx.Client | None = None,
|
||||
timeout: float = 30.0) -> list[dict[str, Any]]:
|
||||
"""Alle Workflows über die n8n-API lesen (nur GET)."""
|
||||
base = host if host.startswith("http") else f"https://{host}"
|
||||
base = base.rstrip("/")
|
||||
own = client is None
|
||||
client = client or httpx.Client(timeout=timeout)
|
||||
try:
|
||||
out: list[dict[str, Any]] = []
|
||||
cursor = None
|
||||
while True:
|
||||
params: dict[str, Any] = {"limit": 250}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
r = client.get(f"{base}/api/v1/workflows", params=params, headers={"X-N8N-API-KEY": api_key})
|
||||
r.raise_for_status()
|
||||
body = r.json()
|
||||
out.extend(body.get("data", []))
|
||||
cursor = body.get("nextCursor")
|
||||
if not cursor:
|
||||
return out
|
||||
finally:
|
||||
if own:
|
||||
client.close()
|
||||
@@ -38,6 +38,10 @@ class Settings:
|
||||
webhook_auth: str = "" # Wert des Authorization-Headers für ausgehende Webhooks (n8n Header Auth)
|
||||
request_timeout: float = 120.0
|
||||
canary_interval: float = 60.0 # 0 = Prober aus
|
||||
ontology_path: Path = field(default_factory=lambda: REPO_ROOT / "ontology" / "know.yaml")
|
||||
ontology_snapshot: str = "" # Datei mit n8n-Workflows (JSON) statt Live-Abruf, z. B. für Tests/offline
|
||||
n8n_host: str = "" # nur lesend: GET /api/v1/workflows für den Ontologie-Abgleich
|
||||
n8n_api_key: str = ""
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> Settings:
|
||||
@@ -65,4 +69,8 @@ class Settings:
|
||||
webhook_auth=env.get("KNASTER_WEBHOOK_AUTH", ""),
|
||||
request_timeout=float(env.get("KNASTER_REQUEST_TIMEOUT", base.request_timeout)),
|
||||
canary_interval=float(env.get("KNASTER_CANARY_INTERVAL", base.canary_interval)),
|
||||
ontology_path=Path(env.get("KNASTER_ONTOLOGY", base.ontology_path)),
|
||||
ontology_snapshot=env.get("KNASTER_ONTOLOGY_SNAPSHOT", ""),
|
||||
n8n_host=env.get("N8N_HOST", ""),
|
||||
n8n_api_key=env.get("N8N_API_KEY", ""),
|
||||
)
|
||||
|
||||
@@ -144,6 +144,18 @@ healer_state = Table(
|
||||
Column("updated_at", Float),
|
||||
)
|
||||
|
||||
ontology_decisions = Table(
|
||||
"ontology_decisions",
|
||||
metadata,
|
||||
Column("key", String(300), primary_key=True),
|
||||
Column("decision", String(20), nullable=False), # accept | map | reject
|
||||
Column("params", JSON),
|
||||
Column("decided_by", String(100)),
|
||||
Column("decided_at", Float),
|
||||
Column("applied_at", Float),
|
||||
Column("updated_at", Float),
|
||||
)
|
||||
|
||||
events = Table(
|
||||
"events",
|
||||
metadata,
|
||||
@@ -339,6 +351,26 @@ class Store:
|
||||
with self.engine.connect() as conn:
|
||||
return [dict(r._mapping) for r in conn.execute(q)]
|
||||
|
||||
# --- Ontologie-Entscheidungen ----------------------------------------------------------
|
||||
def set_ontology_decision(self, key: str, decision: str, params: dict[str, Any], decided_by: str) -> None:
|
||||
self._upsert(ontology_decisions, "key", key, {"decision": decision, "params": params,
|
||||
"decided_by": decided_by, "decided_at": now(),
|
||||
"applied_at": None})
|
||||
|
||||
def list_ontology_decisions(self, *, applied: bool | None = None) -> list[dict[str, Any]]:
|
||||
q = select(ontology_decisions).order_by(ontology_decisions.c.decided_at)
|
||||
if applied is True:
|
||||
q = q.where(ontology_decisions.c.applied_at.is_not(None))
|
||||
elif applied is False:
|
||||
q = q.where(ontology_decisions.c.applied_at.is_(None))
|
||||
with self.engine.connect() as conn:
|
||||
return [dict(r._mapping) for r in conn.execute(q)]
|
||||
|
||||
def mark_ontology_applied(self, keys: list[str]) -> None:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(update(ontology_decisions).where(ontology_decisions.c.key.in_(keys))
|
||||
.values(applied_at=now()))
|
||||
|
||||
# --- Healer-Zustand --------------------------------------------------------------------
|
||||
def get_healer_state(self, key: str) -> dict[str, Any] | None:
|
||||
row = self._get(healer_state, "key", key)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
@@ -18,6 +19,7 @@ from .llm import ChatClient, OpenAIChatClient
|
||||
from .maintenance.actions import ActionExecutor
|
||||
from .maintenance.agent import IntentMapper, LLMIntentMapper, MaintenanceAgent, MaintenanceConfig
|
||||
from .metrics import Metrics
|
||||
from .ontology import Ontology, OntologyFinding, build_snapshot, check, fetch_workflows
|
||||
from .policy import Policy
|
||||
from .registry import Registry
|
||||
from .router import Router
|
||||
@@ -60,6 +62,35 @@ def _optional(path: Path, loader: Callable, default): # type: ignore[no-untyped
|
||||
return loader(path) if path.exists() else default
|
||||
|
||||
|
||||
def ontology_source_for(settings: Settings) -> Callable[[], list[dict]] | None:
|
||||
"""Woher die n8n-Workflows für den Ontologie-Abgleich kommen: Datei (offline) oder n8n-API (nur lesend)."""
|
||||
if settings.ontology_snapshot:
|
||||
path = Path(settings.ontology_snapshot)
|
||||
|
||||
def from_file() -> list[dict]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return data.get("data", data) if isinstance(data, dict) else data
|
||||
|
||||
return from_file
|
||||
if settings.n8n_host and settings.n8n_api_key:
|
||||
return lambda: fetch_workflows(settings.n8n_host, settings.n8n_api_key, timeout=settings.request_timeout)
|
||||
return None
|
||||
|
||||
|
||||
def ontology_checker(settings: Settings, store: Store,
|
||||
source: Callable[[], list[dict]] | None) -> Callable[[], list[OntologyFinding]] | None:
|
||||
source = source or ontology_source_for(settings)
|
||||
if source is None or not settings.ontology_path.exists():
|
||||
return None
|
||||
|
||||
def run() -> list[OntologyFinding]:
|
||||
onto = Ontology.from_file(settings.ontology_path) # bei jedem Lauf neu: Änderungen im Repo zählen sofort
|
||||
decided = {d["key"] for d in store.list_ontology_decisions()}
|
||||
return check(onto, build_snapshot(source()), decided=decided)
|
||||
|
||||
return run
|
||||
|
||||
|
||||
def build_system(
|
||||
settings: Settings,
|
||||
*,
|
||||
@@ -70,6 +101,7 @@ def build_system(
|
||||
notifier: Notifier | None = None,
|
||||
jobs: JobRunner | None = None,
|
||||
intent_mapper: IntentMapper | None = None,
|
||||
ontology_source: Callable[[], list[dict]] | None = None,
|
||||
clock: Callable[[], float] = now,
|
||||
wallclock: Callable[[], datetime] = datetime.now,
|
||||
) -> System:
|
||||
@@ -117,7 +149,9 @@ def build_system(
|
||||
intent_mapper = LLMIntentMapper(chat, registry.slot("tier1-general").spec.litellm_model)
|
||||
maintenance = MaintenanceAgent(store=store, registry=registry, library=library, index=router.index,
|
||||
policy=policy, actions=actions, lifecycle=lifecycle, config=maint_cfg,
|
||||
intent_mapper=intent_mapper, clock=clock, wallclock=wallclock)
|
||||
intent_mapper=intent_mapper,
|
||||
ontology_check=ontology_checker(settings, store, ontology_source),
|
||||
clock=clock, wallclock=wallclock)
|
||||
healer = Healer(healer_cfg, store, actions, policy, propose=maintenance.propose, clock=clock)
|
||||
prober = CanaryProber(registry, chat, embedder_for={settings.embed_model: embedder},
|
||||
metrics_registry=metrics.registry)
|
||||
|
||||
@@ -87,6 +87,21 @@ def validate_repo(root: str | Path, site: str = "knaster") -> list[tuple[str, st
|
||||
if kind not in known:
|
||||
err(f"Policy: unbekannte Aktion {kind}")
|
||||
|
||||
onto_path = root / "ontology" / "know.yaml"
|
||||
if onto_path.exists():
|
||||
from .ontology import Ontology, to_turtle
|
||||
|
||||
try:
|
||||
onto = Ontology.from_file(onto_path)
|
||||
except (ValidationError, ValueError) as exc:
|
||||
err(f"Ontologie: {exc}")
|
||||
else:
|
||||
for e in onto.validate_model():
|
||||
err(f"Ontologie: {e}")
|
||||
ttl = onto_path.with_suffix(".ttl")
|
||||
if not ttl.exists() or ttl.read_text(encoding="utf-8") != to_turtle(onto):
|
||||
warn(f"Ontologie: {ttl.name} ist nicht aktuell (knaster ontology export --ttl {ttl})")
|
||||
|
||||
if (root / "catalog").exists():
|
||||
roles, _ = load_catalog(root / "catalog")
|
||||
for role in roles:
|
||||
|
||||
@@ -212,7 +212,8 @@ def make_system(tmp_path, skills_dir, fake_chat, clock):
|
||||
(site / "registry.yaml").write_text(yaml.safe_dump(REGISTRY))
|
||||
settings = Settings(db_url="sqlite://", site_dir=site, skills_dir=skills_dir,
|
||||
config_dir=overrides.pop("config_dir", REPO_ROOT / "config"),
|
||||
api_token=overrides.pop("api_token", "secret"))
|
||||
api_token=overrides.pop("api_token", "secret"),
|
||||
ontology_path=overrides.pop("ontology_path", tmp_path / "keine-ontologie.yaml"))
|
||||
kw = dict(embedder=HashingEmbedder(), chat=fake_chat, jobs=LoggingJobRunner(), notifier=DryRunDriver(),
|
||||
clock=clock)
|
||||
kw.update(overrides)
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Ontologie: Modell, Export, Abgleich mit n8n (synthetische Workflows), Übernahme, Service-Routine."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from knaster.ontology import Ontology, apply_decisions, build_snapshot, check, to_graph, to_turtle
|
||||
from knaster.ontology.n8n import is_closed, literal_values, parse_know_code
|
||||
from knaster.settings import REPO_ROOT
|
||||
|
||||
KNOW_CODE = r"""
|
||||
const allowed=['setup_schema','store_mail_event','upsert_hint','promote_hint','find_mail'];
|
||||
if(operation==='setup_schema'){ query=`CREATE TABLE IF NOT EXISTS ${t.mail_events} (id BIGSERIAL PRIMARY KEY,
|
||||
message_id TEXT UNIQUE NOT NULL, category TEXT, sender TEXT);
|
||||
CREATE TABLE IF NOT EXISTS ${t.hints} (hint_key TEXT UNIQUE NOT NULL, hint_type TEXT NOT NULL,
|
||||
quality_status TEXT NOT NULL DEFAULT 'observed');
|
||||
CREATE TABLE IF NOT EXISTS ${t.config} (config_key TEXT UNIQUE NOT NULL, weight NUMERIC);
|
||||
CREATE TABLE IF NOT EXISTS ${t.orphans} (id BIGSERIAL PRIMARY KEY, x TEXT);
|
||||
ALTER TABLE ${t.mail_events} ADD COLUMN IF NOT EXISTS received_at TIMESTAMPTZ, ADD COLUMN IF NOT EXISTS mood TEXT;
|
||||
INSERT INTO ${t.config} (config_key, weight) VALUES ('mahnung', 3) ON CONFLICT (config_key) DO NOTHING;
|
||||
INSERT INTO ${t.config} (config_key, weight) VALUES ('default', 0) ON CONFLICT (config_key) DO NOTHING;`;}
|
||||
if(operation==='store_mail_event'){ query=`INSERT INTO ${t.mail_events} (message_id,category,sender)
|
||||
VALUES (${sqlString(payload.message_id)},${sqlString(payload.category)},${sqlString(payload.sender)})
|
||||
ON CONFLICT (message_id) DO UPDATE SET category=EXCLUDED.category RETURNING *`;}
|
||||
if(operation==='upsert_hint'){ query=`INSERT INTO ${t.hints} (hint_key,hint_type) VALUES (${k},${ht})
|
||||
ON CONFLICT (hint_key) DO UPDATE SET quality_status=CASE WHEN x THEN 'proposed' ELSE ${t.hints}.quality_status END`;}
|
||||
if(operation==='promote_hint'){ query=`UPDATE ${t.hints} SET quality_status='confirmed' WHERE hint_key=${k}`;}
|
||||
if(operation==='find_mail'){ query=`SELECT * FROM ${t.mail_events}`;}
|
||||
"""
|
||||
|
||||
ONTO = {
|
||||
"id": "t", "namespace": "urn:test#", "modules": {"m": "Mail"},
|
||||
"classes": {
|
||||
"Mail": {"label": "Mail", "module": "m", "table": "mail_events", "key": "message_id",
|
||||
"attributes": {"id": "BIGSERIAL", "message_id": "TEXT", "category": "TEXT", "sender": "TEXT",
|
||||
"received_at": "TIMESTAMPTZ"}},
|
||||
"Hinweis": {"label": "Hinweis", "module": "m", "table": "hints",
|
||||
"attributes": {"hint_key": "TEXT", "hint_type": "TEXT", "quality_status": "TEXT"}},
|
||||
"Kategorie": {"label": "Kategorie", "module": "m", "table": "config",
|
||||
"attributes": {"config_key": "TEXT", "weight": "NUMERIC"}},
|
||||
"Alt": {"label": "Alt", "module": "m", "table": "removed_table", "attributes": {"a": "TEXT"}},
|
||||
},
|
||||
"relations": [{"source": "Hinweis", "name": "hatKategorie", "target": "Kategorie", "via": "hint_type"}],
|
||||
"vocabularies": {
|
||||
"Kat": {"label": "Kategorie", "class": "Kategorie",
|
||||
"columns": ["mail_events.category", "hints.hint_type", "config.config_key"],
|
||||
"concepts": {"mahnung": {"label": "Mahnung"}, "rechnung": {"label": "Rechnung", "alt": ["Buchhaltung"]},
|
||||
"sonstiges": {}, "default": {"technical": True}}},
|
||||
"QS": {"label": "Qualität", "columns": ["hints.quality_status"],
|
||||
"concepts": {"observed": {}, "proposed": {}, "confirmed": {}, "rejected": {}}},
|
||||
},
|
||||
}
|
||||
|
||||
CAT_PAYLOAD = ("={{ JSON.stringify({ message_id: $json.id, sender: ($('Vorfilter').item.json.from||'').toLowerCase(), "
|
||||
"category: ({'Buchhaltung':'rechnung','Bauvorhaben':'bauvorhaben','Sonstiges':'sonstiges'}"
|
||||
"[$json.output?.category]||'sonstiges'), raw_signals: { category: 'x' } }) }}")
|
||||
|
||||
|
||||
def _call(name, op, payload="", node="KNOW"):
|
||||
return {"name": node, "type": "n8n-nodes-base.executeWorkflow",
|
||||
"parameters": {"workflowId": {"value": "know1"},
|
||||
"workflowInputs": {"value": {"operation": op, "payload_json": payload}}}}
|
||||
|
||||
|
||||
WORKFLOWS = [
|
||||
{"id": "know1", "name": "KNOW", "active": True,
|
||||
"nodes": [{"name": "Normalize", "type": "n8n-nodes-base.code", "parameters": {"jsCode": KNOW_CODE}}]},
|
||||
{"id": "w1", "name": "CAT", "active": True, "nodes": [
|
||||
_call("CAT", "store_mail_event", CAT_PAYLOAD),
|
||||
{"name": "Mark", "type": "n8n-nodes-base.postgres",
|
||||
"parameters": {"query": "UPDATE knowledge.hints SET x=1 WHERE hint_key='{{ $json.k }}'"}}]},
|
||||
{"id": "w2", "name": "MPA", "active": True, "nodes": [
|
||||
_call("MPA", "store_mail_event", '={{ JSON.stringify({ message_id: $json.id, category: "MPA" }) }}'),
|
||||
_call("MPA", "log_decision", "{}", node="Log1"), _call("MPA", "log_decision", "{}", node="Log2")]},
|
||||
{"id": "w3", "name": "Dyn", "active": True, "nodes": [
|
||||
{"name": "Build", "type": "n8n-nodes-base.code", "parameters": {"jsCode": "return {operation: 'upsert_hint'}"}},
|
||||
_call("Dyn", "={{ $json.operation }}")]},
|
||||
{"id": "w4", "name": "Old", "active": False, "nodes": [
|
||||
_call("Old", "store_mail_event", "={{ JSON.stringify({ category: 'legacy_kat' }) }}")]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def onto():
|
||||
return Ontology(**json.loads(json.dumps(ONTO)))
|
||||
|
||||
|
||||
def test_repo_ontology_is_valid_and_turtle_current():
|
||||
path = REPO_ROOT / "ontology" / "know.yaml"
|
||||
o = Ontology.from_file(path)
|
||||
assert o.validate_model() == []
|
||||
assert len(o.classes) >= 49 and "MailKategorie" in o.vocabularies
|
||||
assert (REPO_ROOT / "ontology" / "know.ttl").read_text(encoding="utf-8") == to_turtle(o)
|
||||
|
||||
|
||||
def test_model_validation_finds_errors(onto):
|
||||
onto.relations.append(onto.relations[0].model_copy(update={"target": "Gibtsnicht"}))
|
||||
onto.vocabularies["Kat"].concepts["sonstiges"].alt = ["Mahnung"]
|
||||
onto.vocabularies["Kat"].columns.append("mail_events.fehlt")
|
||||
onto.vocabularies["Kat"].rejected = ["rechnung"]
|
||||
errors = "\n".join(onto.validate_model())
|
||||
assert "Gibtsnicht" in errors and "„Mahnung“" in errors and "mail_events.fehlt" in errors
|
||||
assert "zugleich verworfen" in errors
|
||||
|
||||
|
||||
def test_vocabulary_resolve(onto):
|
||||
v = onto.vocabularies["Kat"]
|
||||
assert v.resolve("Buchhaltung") == "rechnung" and v.resolve("MAHNUNG") == "mahnung" and v.resolve("x") is None
|
||||
|
||||
|
||||
def test_exports(onto):
|
||||
ttl = to_turtle(onto)
|
||||
assert "t:Mail a owl:Class" in ttl and "skos:altLabel \"Buchhaltung\"@de" in ttl
|
||||
assert "t:hatKategorie a owl:ObjectProperty" in ttl
|
||||
g = to_graph(onto, generated="2026-10-13")
|
||||
mail = next(c for c in g["classes"] if c["id"] == "Mail")
|
||||
assert "id" not in [a[0] for a in mail["attrs"]]
|
||||
hint = next(c for c in g["classes"] if c["id"] == "Hinweis")
|
||||
assert hint["enums"]["quality_status"] == ["observed", "proposed", "confirmed", "rejected"]
|
||||
assert "Mahnung" in next(c for c in g["classes"] if c["id"] == "Kategorie")["individuals"]
|
||||
|
||||
|
||||
def test_parse_know_code():
|
||||
spec = parse_know_code(KNOW_CODE)
|
||||
assert spec.allowed == {"setup_schema", "store_mail_event", "upsert_hint", "promote_hint", "find_mail"}
|
||||
assert set(spec.schema["mail_events"]) == {"id", "message_id", "category", "sender", "received_at", "mood"}
|
||||
assert spec.writes["store_mail_event"]["mail_events"] == {"message_id", "category", "sender"}
|
||||
assert spec.writes["promote_hint"]["hints"] == {"quality_status"}
|
||||
assert spec.seeds["config.config_key"] == {"mahnung", "default"}
|
||||
assert "config" not in spec.writes.get("setup_schema", {}) # Seeds gehören zu keiner Operation
|
||||
assert spec.assigned["upsert_hint"]["hints.quality_status"] == {"proposed"}
|
||||
assert spec.assigned["promote_hint"]["hints.quality_status"] == {"confirmed"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expr,key,values,closed", [
|
||||
(CAT_PAYLOAD, "category", {"rechnung", "bauvorhaben", "sonstiges"}, True),
|
||||
(CAT_PAYLOAD, "sender", set(), False),
|
||||
('{ category: "MPA", x: 1 }', "category", {"MPA"}, True),
|
||||
("{ category: $json.cat || 'info' }", "category", {"info"}, False),
|
||||
("={{ JSON.stringify({ x: { category: 'tief' } }) }}", "category", set(), False),
|
||||
])
|
||||
def test_literal_values_and_closed(expr, key, values, closed):
|
||||
assert literal_values(expr, key) == values
|
||||
assert is_closed(expr, key) is closed
|
||||
|
||||
|
||||
def test_check_against_synthetic_n8n(onto):
|
||||
snap = build_snapshot(WORKFLOWS)
|
||||
assert snap.know and snap.know.workflow_id == "know1"
|
||||
dyn = [c for c in snap.calls if c.workflow == "Dyn"]
|
||||
assert [(c.operation, c.inferred) for c in dyn] == [("upsert_hint", True)]
|
||||
findings = check(onto, snap)
|
||||
by_kind = {}
|
||||
for f in findings:
|
||||
by_kind.setdefault(f.kind, []).append(f)
|
||||
# unbekannte Operation: zwei Knoten → ein Befund
|
||||
[unknown] = by_kind["unknown_operation"]
|
||||
assert unknown.severity == "high" and "Log1, Log2" in unknown.detail
|
||||
[sql] = by_kind["direct_sql"]
|
||||
assert sql.severity == "medium" and "hints" in sql.summary
|
||||
vocab = {f.change["value"]: f for f in by_kind["vocab_value"]}
|
||||
assert set(vocab) == {"bauvorhaben", "MPA", "legacy_kat"}
|
||||
assert vocab["legacy_kat"].severity == "low" and vocab["MPA"].severity == "medium"
|
||||
# mahnung wird von keiner festen Zuordnung erzeugt, default ist technisch
|
||||
[unused] = [f for f in by_kind["concept_without_producer"] if f.key == "unused:Kat"]
|
||||
assert "mahnung" in unused.summary and "default" not in unused.summary
|
||||
# Qualitätsstatus: proposed/confirmed setzt KNOW selbst (upsert_hint ist aktiv abgeleitet, promote_hint nicht)
|
||||
qs = [f for f in by_kind.get("concept_without_producer", []) if f.key == "unused:QS"]
|
||||
assert not qs # keine feste Zuordnung aus Workflows → keine Aussage
|
||||
assert {f.change["column"] for f in by_kind["unmapped_column"]} == {"mood"}
|
||||
assert by_kind["unmapped_table"][0].change["table"] == "orphans"
|
||||
assert by_kind["missing_table"][0].key == "gone:removed_table"
|
||||
assert "no_write_operation" not in by_kind # orphans ist keiner Klasse zugeordnet, config ist Seed-Tabelle
|
||||
assert "nowriter:config" not in {f.key for f in findings} # Seed-Tabelle
|
||||
# entschiedene bzw. ignorierte Schlüssel verschwinden
|
||||
onto.ignore.append("table:orphans")
|
||||
keys = {f.key for f in check(onto, snap, decided={"vocab:Kat:mpa"})}
|
||||
assert "vocab:Kat:mpa" not in keys and "table:orphans" not in keys
|
||||
|
||||
|
||||
def test_apply_decisions(onto):
|
||||
decisions = [
|
||||
{"key": "vocab:Kat:bauvorhaben", "decision": "accept",
|
||||
"params": {"type": "vocab_value", "vocabulary": "Kat", "value": "Bauvorhaben"}},
|
||||
{"key": "vocab:Kat:buchhaltg", "decision": "map",
|
||||
"params": {"type": "vocab_value", "vocabulary": "Kat", "value": "Buchhaltg", "concept": "rechnung"}},
|
||||
{"key": "vocab:Kat:mpa", "decision": "reject", "params": {"type": "vocab_value", "vocabulary": "Kat",
|
||||
"value": "MPA"}},
|
||||
{"key": "column:mail_events.mood", "decision": "accept",
|
||||
"params": {"type": "add_attribute", "class": "Mail", "column": "mood", "sql_type": "TEXT"}},
|
||||
{"key": "table:orphans", "decision": "accept",
|
||||
"params": {"type": "add_class", "table": "orphans", "attributes": {"x": "TEXT"}}},
|
||||
{"key": "sql:CAT:Mark", "decision": "reject", "params": {"type": "x"}},
|
||||
{"key": "vocab:Kat:kaputt", "decision": "map",
|
||||
"params": {"type": "vocab_value", "vocabulary": "Kat", "value": "k", "concept": "gibtsnicht"}},
|
||||
]
|
||||
applied, messages = apply_decisions(onto, decisions)
|
||||
v = onto.vocabularies["Kat"]
|
||||
assert v.resolve("Bauvorhaben") == "bauvorhaben" and v.resolve("Buchhaltg") == "rechnung"
|
||||
assert v.is_rejected("mpa") and "mood" in onto.classes["Mail"].attributes
|
||||
assert onto.classes["Orphans"].module == "neu" and "sql:CAT:Mark" in onto.ignore
|
||||
assert "vocab:Kat:kaputt" not in applied and any("nicht übernommen" in m for m in messages)
|
||||
assert onto.version == 2 and onto.validate_model() == []
|
||||
|
||||
|
||||
def test_routine_proposes_and_remembers_decisions(make_system, tmp_path, onto):
|
||||
path = tmp_path / "know.yaml"
|
||||
onto.save(path)
|
||||
sysm = make_system(ontology_path=path, ontology_source=lambda: WORKFLOWS)
|
||||
m = sysm.maintenance
|
||||
report = m.run_routine()
|
||||
assert any("log_decision" in f for f in report.findings)
|
||||
vocab = {p.params["value"]: p for p in report.proposals if p.kind == "ontology_change"
|
||||
and p.params["type"] == "vocab_value"}
|
||||
assert {"bauvorhaben", "MPA", "legacy_kat"} <= set(vocab)
|
||||
assert all(p.state == "open" for p in vocab.values()) # Ontologie-Änderungen nie automatisch
|
||||
|
||||
# zuordnen, ablehnen, aufnehmen
|
||||
assert "executed" in m.handle_command(f"ordne {vocab['legacy_kat'].id} sonstiges zu", user="robert").text
|
||||
assert "rejected" in m.handle_command(f"lehne {vocab['MPA'].id} ab: Workflow-Name, keine Kategorie").text
|
||||
assert "executed" in m.handle_command(f"genehmige {vocab['bauvorhaben'].id}").text
|
||||
assert "Fehler" in m.handle_command("ordne P-999 sonstiges zu").text
|
||||
|
||||
decisions = {d["key"]: d for d in sysm.store.list_ontology_decisions()}
|
||||
assert decisions["vocab:Kat:legacy_kat"]["decision"] == "map"
|
||||
assert decisions["vocab:Kat:legacy_kat"]["decided_by"] == "robert"
|
||||
assert decisions["vocab:Kat:mpa"]["decision"] == "reject"
|
||||
|
||||
# zweiter Lauf: entschiedene Werte kommen nicht wieder
|
||||
again = m.run_routine()
|
||||
values = {p.params.get("value") for p in again.proposals if p.kind == "ontology_change"}
|
||||
assert not values & {"bauvorhaben", "MPA", "legacy_kat"}
|
||||
|
||||
# Übernahme in die Datei
|
||||
pending = sysm.store.list_ontology_decisions(applied=False)
|
||||
applied, _ = apply_decisions(onto, pending)
|
||||
assert set(applied) == {"vocab:Kat:legacy_kat", "vocab:Kat:mpa", "vocab:Kat:bauvorhaben"}
|
||||
assert onto.vocabularies["Kat"].resolve("legacy_kat") == "sonstiges"
|
||||
|
||||
|
||||
def test_routine_survives_unreachable_n8n(make_system, tmp_path, onto):
|
||||
path = tmp_path / "know.yaml"
|
||||
onto.save(path)
|
||||
|
||||
def broken():
|
||||
raise ConnectionError("n8n nicht erreichbar")
|
||||
|
||||
report = make_system(ontology_path=path, ontology_source=broken).maintenance.run_routine()
|
||||
assert any("Ontologie-Abgleich nicht möglich" in f for f in report.findings)
|
||||
|
||||
|
||||
def test_cli_check_with_snapshot(tmp_path, onto, monkeypatch, capsys):
|
||||
from knaster.cli import main
|
||||
|
||||
path = tmp_path / "know.yaml"
|
||||
onto.save(path)
|
||||
snap = tmp_path / "wf.json"
|
||||
snap.write_text(json.dumps({"data": WORKFLOWS}))
|
||||
monkeypatch.setenv("KNASTER_DB_URL", "sqlite://")
|
||||
assert main(["ontology", "--path", str(path), "check", "--snapshot", str(snap)]) == 1 # high-Befund
|
||||
out = capsys.readouterr().out
|
||||
assert "log_decision" in out and "Änderungsvorschläge" in out
|
||||
assert main(["ontology", "--path", str(path), "validate"]) == 0
|
||||
html = tmp_path / "o.html"
|
||||
assert main(["ontology", "--path", str(path), "export", "--html", str(html),
|
||||
"--template", str(REPO_ROOT / "ontology" / "viewer.html.tmpl")]) == 0
|
||||
assert "__DATA__" not in html.read_text() and "owl:Class" in html.read_text()
|
||||
Reference in New Issue
Block a user