- deploy/compose/infra: Dienst litellm entfernt (D4), Router auf LITELLM_BASE_URL (appsrv08, Port noch offen), alle Images gepinnt, Grafana und pve-exporter ohne Profile, node-exporter für den LXC, Zustand über KNASTER_STATE_DIR - Prometheus: GPU-VMs per file_sd (node, gpu), Proxmox-Knoten per IP, infra-node - Grafana: Datasource und Dashboards provisioniert (grafana.com 10347, 1860, 14574 sowie knaster-Übersicht) - Alertmanager: n8n-Receiver nur vorbereitet (auskommentiert) - Ansible: Rollen infra-stack und gpu-exporters, Playbooks monitoring.yml und monitoring-targets.yml, Skript monitoring-targets.py (tofu output → file_sd) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fd48fd1c7f
commit
bf7c137adf
@@ -25,12 +25,48 @@ def test_healer_labels_available_on_alerts():
|
||||
prom = yaml.safe_load((DEPLOY / "prometheus/prometheus.yml").read_text())
|
||||
jobs = {j["job_name"]: j for j in prom["scrape_configs"]}
|
||||
for job in ("node", "gpu", "vllm"):
|
||||
for sc in jobs[job]["static_configs"]:
|
||||
for sc in jobs[job].get("static_configs", []):
|
||||
assert {"slot", "pve_node", "vmid"} <= set(sc["labels"]), (job, sc)
|
||||
# GPU-VMs kommen per file_sd; die Labels setzt platform/proxmox/scripts/monitoring-targets.py
|
||||
generated = _monitoring_targets()
|
||||
for job in ("node", "gpu"):
|
||||
files = [f for sd in jobs[job]["file_sd_configs"] for f in sd["files"]]
|
||||
assert files == [f"/etc/knaster-sd/{job}.json"]
|
||||
assert generated[job]
|
||||
for entry in generated[job]:
|
||||
assert {"slot", "pve_node", "vmid"} <= set(entry["labels"]), (job, entry)
|
||||
from knaster.canary import OPS_LABELS
|
||||
assert set(OPS_LABELS) == {"docker_host", "container", "pve_node", "vmid"}
|
||||
|
||||
|
||||
def _monitoring_targets() -> dict:
|
||||
import importlib.util
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"monitoring_targets", REPO_ROOT / "platform/proxmox/scripts/monitoring-targets.py")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
registry = yaml.safe_load((REPO_ROOT / "sites/knaster/registry.yaml").read_text(encoding="utf-8"))
|
||||
gpu_vms = { # Form wie `tofu output -json gpu_vms`
|
||||
"gpu-pve44": {"node": "pve44", "vmid": 244, "ipv4": "192.168.100.186"},
|
||||
"gpu-pve48": {"node": "pve48", "vmid": 248, "ipv4": "192.168.100.183"},
|
||||
"gpu-pve52": {"node": "pve52", "vmid": 252, "ipv4": "192.168.100.174"},
|
||||
"gpu-pve46": {"node": "pve46", "vmid": 246, "ipv4": None},
|
||||
}
|
||||
return mod.build_targets(gpu_vms, registry)
|
||||
|
||||
|
||||
def test_monitoring_targets_from_tofu_output():
|
||||
t = _monitoring_targets()
|
||||
assert [e["targets"][0] for e in t["node"]] == [
|
||||
"192.168.100.186:9100", "192.168.100.183:9100", "192.168.100.174:9100"] # gpu-pve46 ohne IP fehlt
|
||||
assert t["gpu"][0]["targets"] == ["192.168.100.186:9835"]
|
||||
pve44 = t["node"][0]["labels"]
|
||||
assert pve44["instance"] == "gpu-pve44" and pve44["vmid"] == "244" and pve44["pve_node"] == "pve44"
|
||||
assert pve44["slot"] == "embed-default" and pve44["slots"] == "embed-default,rerank-default"
|
||||
assert t["node"][2]["labels"]["slot"] == "none" # gpu-pve52 trägt noch keinen Slot
|
||||
|
||||
|
||||
def test_alertmanager_targets_router_endpoint():
|
||||
am = yaml.safe_load((DEPLOY / "alertmanager/alertmanager.yml").read_text())
|
||||
url = am["receivers"][0]["webhook_configs"][0]["url"]
|
||||
|
||||
Reference in New Issue
Block a user