diff --git a/nautobot/tp37-dienste-seed.py b/nautobot/tp37-dienste-seed.py new file mode 100644 index 0000000..84b42be --- /dev/null +++ b/nautobot/tp37-dienste-seed.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +TP37 Punkt 2 (Dienste) — exponierte Ports als native ipam.Service-Objekte. + +Idempotenter Seeder (get-or-create) fuer ipam.Service: + je VM-Name + Port + Protokoll ein Service-Eintrag. + +Vorbedingung: TP37 P1 (tp37-substrat-seed.py --commit) muss vorher gelaufen sein, +da die VirtualMachine-Objekte existieren muessen. + +Datenmodell-Entscheid (User 2026-09-06): native Modelle, KEINE eigene App. +Muster: nautobot/tp37-substrat-seed.py. + +Umfang (User-Default 2026-09-07): + Caddy (:80/:443), Caddy-Backends (haos:8123, vaultwarden:80, gitea:3000, n8n:5678), + Nautobot:8080, AdGuard:53 (bf1+bf2, tcp+udp), platform-pg:5432, mariadb:3306, + influxdb2:8086, mqtt5:1883, portainer:9443, + MCP-Dienste (pve-mcp:3000, shelly-mcp:3030, n8n-mcp:3001, gitea-mcp:3000). + +Token-Quelle: docs/nas-vlan20/stacks/210-nautobot.yml (gitignored), +Schluessel NAUTOBOT_SUPERUSER_API_TOKEN. + +Aufruf: + python3 nautobot/tp37-dienste-seed.py # Dry-Run (default) + python3 nautobot/tp37-dienste-seed.py --commit # legt an +""" +import json +import os +import re +import sys +import urllib.request +import urllib.parse +import urllib.error +from pathlib import Path + +NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080") +REPO = Path(__file__).resolve().parent.parent +YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml" +COMMIT = "--commit" in sys.argv + + +def load_token() -> str: + for line in YML.read_text().splitlines(): + m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line) + if m: + return m.group(1) + sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden") + + +TOKEN = load_token() +HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json", + "Content-Type": "application/json"} + + +def req(method: str, path: str, body=None): + url = f"{NB}{path}" + data = json.dumps(body).encode() if body is not None else None + r = urllib.request.Request(url, data=data, headers=HDRS, method=method) + try: + with urllib.request.urlopen(r, timeout=20) as resp: + return resp.status, json.loads(resp.read().decode() or "{}") + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read().decode() or "{}") + + +def find(endpoint: str, params: dict, match_key: str, match_val: str): + q = urllib.parse.urlencode(dict(params, depth=0)) + st, d = req("GET", f"/api/{endpoint}/?{q}") + for row in d.get("results", []): + if row.get(match_key) == match_val: + return row + return None + + +def get_or_create(endpoint: str, find_params: dict, match_key: str, + match_val: str, payload: dict, label: str): + existing = find(endpoint, find_params, match_key, match_val) + if existing: + print(f" = {label}: '{match_val}' existiert ({existing['id']})") + return existing + if not COMMIT: + print(f" + {label}: '{match_val}' [DRY-RUN]") + return {"id": f""} + st, d = req("POST", f"/api/{endpoint}/", payload) + if st in (200, 201): + print(f" + {label}: '{match_val}' angelegt ({d.get('id')})") + return d + print(f" ! {label}: '{match_val}' FEHLER {st}: {json.dumps(d)[:300]}") + return None + + +# ---------------------------------------------------------------- Inventar +# (vm_name, svc_name, ports, protocol, description) +SERVICES = [ + # Caddy Reverse-Proxy (TP34 critical) + ("caddy-bf1", "caddy-http", [80], "tcp", "Caddy HTTP"), + ("caddy-bf1", "caddy-https", [443], "tcp", "Caddy HTTPS Reverse Proxy"), + # Caddy-Backends via bf-public Tunnel + ("haos", "home-assistant", [8123], "tcp", "Home Assistant Web"), + ("vaultwarden", "vaultwarden-http", [80], "tcp", "Vaultwarden (HTTP, TLS via Caddy)"), + ("gitea", "gitea-http", [3000], "tcp", "Gitea Web"), + ("n8n", "n8n-web", [5678], "tcp", "n8n Workflow Engine"), + # Nautobot + ("nautobot", "nautobot-web", [8080], "tcp", "Nautobot API/Web"), + # DNS (TCP + UDP je Resolver) + ("agh-bf1", "adguard-dns-tcp", [53], "tcp", "AdGuard Home DNS/TCP"), + ("agh-bf1", "adguard-dns-udp", [53], "udp", "AdGuard Home DNS/UDP"), + ("agh-bf2", "adguard-dns-tcp", [53], "tcp", "AdGuard Home DNS/TCP"), + ("agh-bf2", "adguard-dns-udp", [53], "udp", "AdGuard Home DNS/UDP"), + # Infrastruktur + ("platform-pg", "postgresql", [5432], "tcp", "PostgreSQL 16 (Nautobot-DB)"), + ("mariadb", "mariadb", [3306], "tcp", "MariaDB (vw/hass/gitea)"), + ("influxdb2", "influxdb", [8086], "tcp", "InfluxDB 2"), + ("mqtt5", "mqtt", [1883], "tcp", "MQTT Broker (Mosquitto)"), + ("docker", "portainer", [9443], "tcp", "Portainer CE"), + # MCP-Dienste + ("pve-mcp", "pve-mcp", [3000], "tcp", "Proxmox MCP (StreamableHTTP)"), + ("shelly-mcp", "shelly-mcp", [3030], "tcp", "Shelly MCP"), + ("n8n-mcp", "n8n-mcp", [3001], "tcp", "n8n MCP"), + ("gitea-mcp", "gitea-mcp", [3000], "tcp", "Gitea MCP"), +] + + +def main(): + print(f"== TP37 Dienste-Seed ({'COMMIT' if COMMIT else 'DRY-RUN'}) @ {NB} ==") + + # VM-IDs vorab holen (unique Namen, Batch) + vm_names = sorted({vm for vm, *_ in SERVICES}) + print(f"\n[0] VM-IDs holen ({len(vm_names)} unique VMs)") + vm_ids = {} + for name in vm_names: + row = find("virtualization/virtual-machines", {"name": name}, "name", name) + if row: + vm_ids[name] = row["id"] + print(f" = {name}: {row['id']}") + else: + vm_ids[name] = None + print(f" ! {name}: NICHT gefunden — Service-Eintraege werden uebersprungen") + + print(f"\n[1] ipam.Services ({len(SERVICES)} Eintraege)") + n_ok = n_skip = 0 + for vm_name, svc_name, ports, protocol, desc in SERVICES: + vm_id = vm_ids.get(vm_name) + if not vm_id and COMMIT: + print(f" ~ {vm_name}/{svc_name}: VM fehlt, uebersprungen") + n_skip += 1 + continue + svc = get_or_create( + "ipam/services", + {"name": svc_name, "virtual_machine": vm_name}, + "name", svc_name, + {"virtual_machine": vm_id, "name": svc_name, + "ports": ports, "protocol": protocol, "description": desc}, + f"{vm_name}/{protocol}/{ports[0]}", + ) + if svc: + n_ok += 1 + else: + n_skip += 1 + + print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN — nichts geschrieben'}) " + f"| Services={n_ok} uebersprungen={n_skip} ==") + + +if __name__ == "__main__": + main() diff --git a/nautobot/tp37-substrat-seed.py b/nautobot/tp37-substrat-seed.py new file mode 100644 index 0000000..57a9d24 --- /dev/null +++ b/nautobot/tp37-substrat-seed.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +""" +TP37 Punkt 1 (Substrat) — Hosts/Container als native Nautobot-Objekte. + +Idempotenter Seeder (get-or-create) fuer die native virtualization-Modellwelt: + ClusterType (3x: Proxmox VE / Synology Docker / Docker Desktop) + -> Cluster (pve / nas-bf / macmini) + -> VirtualMachine je Workload (Status Active) + + VMInterface "eth0" + ipam.IPAddress (Namespace Global) + primary_ip4 + +Datenmodell-Entscheid (User 2026-09-06): native Modelle, KEINE eigene App +(konsistent zu TP44). Muster: nautobot/tp44-cf-estate-seed.py. + +Umfang Etappe 1 (User-Default 2026-09-07): + - komplettes Substrat: alle 15 pve-Gaeste (inkl. Smart-Home) + alle running + Container auf nas-bf (Portainer ep14) und macmini (ep20). + - macmini-Container teilen sich die Host-IP 10.40.20.167 (Bridge/Port-Mapping) + -> KEIN primary_ip4; Port kommt in TP37 P2 (ipam.Service). + - portainer_agent (Bridge 172.17.0.2, kein Workload) weggelassen. + - gestoppte (exited) NAS-Container weggelassen (nur running). + +Datenstand: Live-Inventur 2026-09-07 (pve-mcp pct config, Portainer ep14, +ssh macmini docker ps, ARP fuer die 2 QEMU-IPs). + +Token-Quelle: docs/nas-vlan20/stacks/210-nautobot.yml (gitignored), +Schluessel NAUTOBOT_SUPERUSER_API_TOKEN. + +Aufruf: + python3 nautobot/tp37-substrat-seed.py # Dry-Run (default) + python3 nautobot/tp37-substrat-seed.py --commit # legt an +""" +import json +import os +import re +import sys +import urllib.request +import urllib.parse +import urllib.error +from pathlib import Path + +NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080") +REPO = Path(__file__).resolve().parent.parent +YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml" +COMMIT = "--commit" in sys.argv + + +def load_token() -> str: + for line in YML.read_text().splitlines(): + m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line) + if m: + return m.group(1) + sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden") + + +TOKEN = load_token() +HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json", + "Content-Type": "application/json"} + + +def req(method: str, path: str, body=None): + url = f"{NB}{path}" + data = json.dumps(body).encode() if body is not None else None + r = urllib.request.Request(url, data=data, headers=HDRS, method=method) + try: + with urllib.request.urlopen(r, timeout=20) as resp: + return resp.status, json.loads(resp.read().decode() or "{}") + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read().decode() or "{}") + + +def is_dry(v) -> bool: + return isinstance(v, str) and v.startswith(""} + st, d = req("POST", f"/api/{endpoint}/", payload) + if st in (200, 201): + print(f" + {label}: '{match_val}' angelegt ({d.get('id')})") + return d + print(f" ! {label}: '{match_val}' FEHLER {st}: {json.dumps(d)[:300]}") + return None + + +def status_id(name: str): + return find("extras/statuses", {"name": name}, "name", name)["id"] + + +def namespace_id(name: str): + return find("ipam/namespaces", {"name": name}, "name", name)["id"] + + +# ---------------------------------------------------------------- Inventar +CLUSTER_TYPES = ["Proxmox VE", "Synology Docker", "Docker Desktop"] + +# cluster -> clustertype +CLUSTERS = [("pve", "Proxmox VE"), + ("nas-bf", "Synology Docker"), + ("macmini", "Docker Desktop")] + +# cluster -> [ (vm_name, primary_ip_with_mask_or_None, comment) ] +VMS = { + "pve": [ + ("haos", "10.40.20.105/24", "Home Assistant OS (QEMU 100)"), + ("iob-master", "10.40.20.107/24", "ioBroker Master (LXC 101)"), + ("mqtt5", "10.40.20.108/24", "MQTT-Broker Mosquitto (LXC 102)"), + ("platform-pg", "10.40.20.20/24", "PostgreSQL 16 - Nautobot-DB (LXC 105, TP20)"), + ("influxdb2", "10.40.20.8/24", "InfluxDB 2 (LXC 106)"), + ("redis", "10.40.20.111/24", "Redis (LXC 107)"), + ("mcast-relay", "10.40.10.21/24", "SSDP-Relay VLAN10<->40 (LXC 108); 2. Bein 10.40.40.21"), + ("agh-bf1", "10.40.20.36/24", "AdGuard Home Resolver (LXC 110, TP23)"), + ("agh-bf2", "10.40.20.37/24", "AdGuard Home Resolver (LXC 111, TP23)"), + ("z2m25", "10.40.20.106/24", "Zigbee2MQTT (LXC 112)"), + ("caddy-bf1", "10.40.20.40/24", "Caddy Reverse-Proxy (LXC 130, TP31)"), + ("iob-exposed", "10.40.20.19/24", "ioBroker exposed (LXC 150)"), + ("mariadb", "10.40.20.23/24", "MariaDB - vw/hass/gitea (LXC 302)"), + ("RaspberryMatic","10.40.20.232/24", "Homematic CCU (QEMU 330)"), + ("docker", "10.40.20.126/24", "Docker-Host nested (LXC 400)"), + ], + "nas-bf": [ + ("nautobot", "10.40.20.30/24", "Nautobot Web (DS923 ep14)"), + ("nautobot-worker", "10.40.20.31/24", "Nautobot Golden-Config Worker"), + ("nautobot-scheduler", "10.40.20.32/24", "Nautobot Scheduler"), + ("nautobot-redis", "10.40.20.34/24", "Nautobot Redis"), + ("gitea", "10.40.20.157/24", "Gitea (git-SSoT)"), + ("vaultwarden", "10.40.20.127/24", "Vaultwarden (DB=mariadb LXC302)"), + ("shelly-mcp", "10.40.20.180/24", "Shelly-MCP"), + ("pve-mcp", "10.40.20.173/24", "Proxmox-MCP (mcp-pve)"), + ("iob_slave", "10.40.20.33/24", "ioBroker Slave"), + ], + "macmini": [ + # Bridge/Port-Mapping auf Host 10.40.20.167 -> kein eigenes primary_ip4 + ("n8n", None, "n8n (Docker Desktop, Host 10.40.20.167:5678)"), + ("n8n-redis", None, "n8n Redis (intern, kein Host-Port)"), + ("n8n-mcp", None, "n8n-MCP (Host 10.40.20.167:3001)"), + ("gitea-mcp", None, "Gitea-MCP (Host 10.40.20.167:3000)"), + ], +} + + +def main(): + print(f"== TP37 Substrat-Seed ({'COMMIT' if COMMIT else 'DRY-RUN'}) @ {NB} ==") + + st_active = status_id("Active") + ns_global = namespace_id("Global") + print(f" (Status Active={st_active} Namespace Global={ns_global})") + + print("\n[1] ClusterTypes") + for ct in CLUSTER_TYPES: + get_or_create("virtualization/cluster-types", {"name": ct}, "name", ct, + {"name": ct}, "ClusterType") + + print("\n[2] Clusters") + ct_lookup = {ct: find("virtualization/cluster-types", {"name": ct}, "name", ct) + for ct in CLUSTER_TYPES} + cluster_ids = {} + for cname, ctname in CLUSTERS: + ctrow = ct_lookup.get(ctname) + ct_id = ctrow["id"] if ctrow else None + cl = get_or_create("virtualization/clusters", {"name": cname}, "name", cname, + {"name": cname, "cluster_type": ct_id}, "Cluster") + cluster_ids[cname] = cl["id"] if cl else None + + print("\n[3] VirtualMachines (+ eth0 + IPAddress + primary_ip4)") + n_vm = n_ip = 0 + for cname, _ in CLUSTERS: + cl_id = cluster_ids.get(cname) + print(f" -- Cluster {cname} --") + for name, ip, comment in VMS[cname]: + vm = get_or_create( + "virtualization/virtual-machines", {"name": name}, "name", name, + {"name": name, "cluster": cl_id, "status": st_active, + "comments": comment}, "VM") + if not vm: + continue + n_vm += 1 + if not ip: + print(f" (ohne primary_ip4 - Host-geteilt)") + continue + # VMInterface eth0 + vmif = None + if is_dry(vm["id"]): + print(f" + eth0 @ {name} [DRY-RUN]") + vmif = {"id": f""} + else: + vmif = find("virtualization/interfaces", + {"virtual_machine": name, "name": "eth0"}, "name", "eth0") + if vmif: + print(f" = eth0 @ {name} ({vmif['id']})") + else: + s, d = req("POST", "/api/virtualization/interfaces/", + {"virtual_machine": vm["id"], "name": "eth0", + "status": st_active}) + if s in (200, 201): + vmif = d + print(f" + eth0 @ {name} ({d.get('id')})") + else: + print(f" ! eth0 @ {name} FEHLER {s}: {json.dumps(d)[:200]}") + continue + # IPAddress + ipobj = get_or_create("ipam/ip-addresses", {"address": ip}, "address", ip, + {"address": ip, "status": st_active, + "namespace": ns_global, "dns_name": ""}, + "IP") + if not ipobj: + continue + n_ip += 1 + if is_dry(vm["id"]) or is_dry(ipobj["id"]) or is_dry(vmif["id"]): + print(f" + assign {ip} -> {name}/eth0 + primary_ip4 [DRY-RUN]") + continue + # assign IP <-> interface (idempotent) + q = urllib.parse.urlencode({"vm_interface": vmif["id"], "depth": 0}) + s, d = req("GET", f"/api/ipam/ip-address-to-interface/?{q}") + have = any((r.get("ip_address") or {}).get("id") == ipobj["id"] + for r in d.get("results", [])) + if have: + print(f" = assign {ip} -> {name}/eth0") + else: + s, d = req("POST", "/api/ipam/ip-address-to-interface/", + {"ip_address": ipobj["id"], "vm_interface": vmif["id"]}) + print(f" {'+' if s in (200,201) else '!'} assign {ip} -> {name}/eth0" + + ("" if s in (200, 201) else f" FEHLER {s}: {json.dumps(d)[:150]}")) + # primary_ip4 (idempotent) + s, vmnow = req("GET", f"/api/virtualization/virtual-machines/{vm['id']}/?depth=0") + cur = vmnow.get("primary_ip4") + cur = cur.get("id") if isinstance(cur, dict) else cur + if cur == ipobj["id"]: + print(f" = primary_ip4 {ip} @ {name}") + else: + s, d = req("PATCH", f"/api/virtualization/virtual-machines/{vm['id']}/", + {"primary_ip4": ipobj["id"]}) + print(f" {'+' if s == 200 else '!'} primary_ip4 {ip} @ {name}" + + ("" if s == 200 else f" FEHLER {s}: {json.dumps(d)[:150]}")) + + print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN - nichts geschrieben'}) " + f"| VMs={n_vm} IPs={n_ip} ==") + + +if __name__ == "__main__": + main()