TP37 P1+P2: VirtualMachines + ipam.Service seeden

P1 (tp37-substrat-seed.py): idempotenter Seeder fuer 3 ClusterTypes,
3 Clusters, 28 VirtualMachines mit eth0/IPAddress/primary_ip4.
Schon aus dem Worktree ausgefuehrt -- Skript damit in main nachgezogen.

P2 (tp37-dienste-seed.py): 20 ipam.Service-Eintraege angelegt:
Caddy :80/:443, Caddy-Backends (haos/vaultwarden/gitea/n8n),
Nautobot :8080, AdGuard :53 TCP+UDP (bf1+bf2), platform-pg :5432,
mariadb :3306, influxdb2 :8086, mqtt5 :1883, portainer :9443,
pve-mcp/shelly-mcp/n8n-mcp/gitea-mcp.
Compliance-Gegenprobe: 140/140 unveraendert.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-09-07 17:19:59 +02:00
parent 674e5fff0d
commit 83fd69ef8d
2 changed files with 422 additions and 0 deletions

View File

@@ -0,0 +1,166 @@
#!/usr/bin/env python3
"""
TP37 Punkt 2 (Dienste) — exponierte Ports als native ipam.Service-Objekte.
Idempotenter Seeder (get-or-create) fuer ipam.Service:
je VM-Name + Port + Protokoll ein Service-Eintrag.
Vorbedingung: TP37 P1 (tp37-substrat-seed.py --commit) muss vorher gelaufen sein,
da die VirtualMachine-Objekte existieren muessen.
Datenmodell-Entscheid (User 2026-09-06): native Modelle, KEINE eigene App.
Muster: nautobot/tp37-substrat-seed.py.
Umfang (User-Default 2026-09-07):
Caddy (:80/:443), Caddy-Backends (haos:8123, vaultwarden:80, gitea:3000, n8n:5678),
Nautobot:8080, AdGuard:53 (bf1+bf2, tcp+udp), platform-pg:5432, mariadb:3306,
influxdb2:8086, mqtt5:1883, portainer:9443,
MCP-Dienste (pve-mcp:3000, shelly-mcp:3030, n8n-mcp:3001, gitea-mcp:3000).
Token-Quelle: docs/nas-vlan20/stacks/210-nautobot.yml (gitignored),
Schluessel NAUTOBOT_SUPERUSER_API_TOKEN.
Aufruf:
python3 nautobot/tp37-dienste-seed.py # Dry-Run (default)
python3 nautobot/tp37-dienste-seed.py --commit # legt an
"""
import json
import os
import re
import sys
import urllib.request
import urllib.parse
import urllib.error
from pathlib import Path
NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080")
REPO = Path(__file__).resolve().parent.parent
YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml"
COMMIT = "--commit" in sys.argv
def load_token() -> str:
for line in YML.read_text().splitlines():
m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line)
if m:
return m.group(1)
sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden")
TOKEN = load_token()
HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json",
"Content-Type": "application/json"}
def req(method: str, path: str, body=None):
url = f"{NB}{path}"
data = json.dumps(body).encode() if body is not None else None
r = urllib.request.Request(url, data=data, headers=HDRS, method=method)
try:
with urllib.request.urlopen(r, timeout=20) as resp:
return resp.status, json.loads(resp.read().decode() or "{}")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read().decode() or "{}")
def find(endpoint: str, params: dict, match_key: str, match_val: str):
q = urllib.parse.urlencode(dict(params, depth=0))
st, d = req("GET", f"/api/{endpoint}/?{q}")
for row in d.get("results", []):
if row.get(match_key) == match_val:
return row
return None
def get_or_create(endpoint: str, find_params: dict, match_key: str,
match_val: str, payload: dict, label: str):
existing = find(endpoint, find_params, match_key, match_val)
if existing:
print(f" = {label}: '{match_val}' existiert ({existing['id']})")
return existing
if not COMMIT:
print(f" + {label}: '{match_val}' [DRY-RUN]")
return {"id": f"<dry:{match_val}>"}
st, d = req("POST", f"/api/{endpoint}/", payload)
if st in (200, 201):
print(f" + {label}: '{match_val}' angelegt ({d.get('id')})")
return d
print(f" ! {label}: '{match_val}' FEHLER {st}: {json.dumps(d)[:300]}")
return None
# ---------------------------------------------------------------- Inventar
# (vm_name, svc_name, ports, protocol, description)
SERVICES = [
# Caddy Reverse-Proxy (TP34 critical)
("caddy-bf1", "caddy-http", [80], "tcp", "Caddy HTTP"),
("caddy-bf1", "caddy-https", [443], "tcp", "Caddy HTTPS Reverse Proxy"),
# Caddy-Backends via bf-public Tunnel
("haos", "home-assistant", [8123], "tcp", "Home Assistant Web"),
("vaultwarden", "vaultwarden-http", [80], "tcp", "Vaultwarden (HTTP, TLS via Caddy)"),
("gitea", "gitea-http", [3000], "tcp", "Gitea Web"),
("n8n", "n8n-web", [5678], "tcp", "n8n Workflow Engine"),
# Nautobot
("nautobot", "nautobot-web", [8080], "tcp", "Nautobot API/Web"),
# DNS (TCP + UDP je Resolver)
("agh-bf1", "adguard-dns-tcp", [53], "tcp", "AdGuard Home DNS/TCP"),
("agh-bf1", "adguard-dns-udp", [53], "udp", "AdGuard Home DNS/UDP"),
("agh-bf2", "adguard-dns-tcp", [53], "tcp", "AdGuard Home DNS/TCP"),
("agh-bf2", "adguard-dns-udp", [53], "udp", "AdGuard Home DNS/UDP"),
# Infrastruktur
("platform-pg", "postgresql", [5432], "tcp", "PostgreSQL 16 (Nautobot-DB)"),
("mariadb", "mariadb", [3306], "tcp", "MariaDB (vw/hass/gitea)"),
("influxdb2", "influxdb", [8086], "tcp", "InfluxDB 2"),
("mqtt5", "mqtt", [1883], "tcp", "MQTT Broker (Mosquitto)"),
("docker", "portainer", [9443], "tcp", "Portainer CE"),
# MCP-Dienste
("pve-mcp", "pve-mcp", [3000], "tcp", "Proxmox MCP (StreamableHTTP)"),
("shelly-mcp", "shelly-mcp", [3030], "tcp", "Shelly MCP"),
("n8n-mcp", "n8n-mcp", [3001], "tcp", "n8n MCP"),
("gitea-mcp", "gitea-mcp", [3000], "tcp", "Gitea MCP"),
]
def main():
print(f"== TP37 Dienste-Seed ({'COMMIT' if COMMIT else 'DRY-RUN'}) @ {NB} ==")
# VM-IDs vorab holen (unique Namen, Batch)
vm_names = sorted({vm for vm, *_ in SERVICES})
print(f"\n[0] VM-IDs holen ({len(vm_names)} unique VMs)")
vm_ids = {}
for name in vm_names:
row = find("virtualization/virtual-machines", {"name": name}, "name", name)
if row:
vm_ids[name] = row["id"]
print(f" = {name}: {row['id']}")
else:
vm_ids[name] = None
print(f" ! {name}: NICHT gefunden — Service-Eintraege werden uebersprungen")
print(f"\n[1] ipam.Services ({len(SERVICES)} Eintraege)")
n_ok = n_skip = 0
for vm_name, svc_name, ports, protocol, desc in SERVICES:
vm_id = vm_ids.get(vm_name)
if not vm_id and COMMIT:
print(f" ~ {vm_name}/{svc_name}: VM fehlt, uebersprungen")
n_skip += 1
continue
svc = get_or_create(
"ipam/services",
{"name": svc_name, "virtual_machine": vm_name},
"name", svc_name,
{"virtual_machine": vm_id, "name": svc_name,
"ports": ports, "protocol": protocol, "description": desc},
f"{vm_name}/{protocol}/{ports[0]}",
)
if svc:
n_ok += 1
else:
n_skip += 1
print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN — nichts geschrieben'}) "
f"| Services={n_ok} uebersprungen={n_skip} ==")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,256 @@
#!/usr/bin/env python3
"""
TP37 Punkt 1 (Substrat) — Hosts/Container als native Nautobot-Objekte.
Idempotenter Seeder (get-or-create) fuer die native virtualization-Modellwelt:
ClusterType (3x: Proxmox VE / Synology Docker / Docker Desktop)
-> Cluster (pve / nas-bf / macmini)
-> VirtualMachine je Workload (Status Active)
+ VMInterface "eth0" + ipam.IPAddress (Namespace Global) + primary_ip4
Datenmodell-Entscheid (User 2026-09-06): native Modelle, KEINE eigene App
(konsistent zu TP44). Muster: nautobot/tp44-cf-estate-seed.py.
Umfang Etappe 1 (User-Default 2026-09-07):
- komplettes Substrat: alle 15 pve-Gaeste (inkl. Smart-Home) + alle running
Container auf nas-bf (Portainer ep14) und macmini (ep20).
- macmini-Container teilen sich die Host-IP 10.40.20.167 (Bridge/Port-Mapping)
-> KEIN primary_ip4; Port kommt in TP37 P2 (ipam.Service).
- portainer_agent (Bridge 172.17.0.2, kein Workload) weggelassen.
- gestoppte (exited) NAS-Container weggelassen (nur running).
Datenstand: Live-Inventur 2026-09-07 (pve-mcp pct config, Portainer ep14,
ssh macmini docker ps, ARP fuer die 2 QEMU-IPs).
Token-Quelle: docs/nas-vlan20/stacks/210-nautobot.yml (gitignored),
Schluessel NAUTOBOT_SUPERUSER_API_TOKEN.
Aufruf:
python3 nautobot/tp37-substrat-seed.py # Dry-Run (default)
python3 nautobot/tp37-substrat-seed.py --commit # legt an
"""
import json
import os
import re
import sys
import urllib.request
import urllib.parse
import urllib.error
from pathlib import Path
NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080")
REPO = Path(__file__).resolve().parent.parent
YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml"
COMMIT = "--commit" in sys.argv
def load_token() -> str:
for line in YML.read_text().splitlines():
m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line)
if m:
return m.group(1)
sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden")
TOKEN = load_token()
HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json",
"Content-Type": "application/json"}
def req(method: str, path: str, body=None):
url = f"{NB}{path}"
data = json.dumps(body).encode() if body is not None else None
r = urllib.request.Request(url, data=data, headers=HDRS, method=method)
try:
with urllib.request.urlopen(r, timeout=20) as resp:
return resp.status, json.loads(resp.read().decode() or "{}")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read().decode() or "{}")
def is_dry(v) -> bool:
return isinstance(v, str) and v.startswith("<dry:")
def find(endpoint: str, params: dict, match_key: str, match_val: str):
q = urllib.parse.urlencode(dict(params, depth=0))
st, d = req("GET", f"/api/{endpoint}/?{q}")
for row in d.get("results", []):
if row.get(match_key) == match_val:
return row
return None
def get_or_create(endpoint: str, find_params: dict, match_key: str,
match_val: str, payload: dict, label: str):
existing = find(endpoint, find_params, match_key, match_val)
if existing:
print(f" = {label}: '{match_val}' existiert ({existing['id']})")
return existing
if not COMMIT:
print(f" + {label}: '{match_val}' [DRY-RUN]")
return {"id": f"<dry:{match_val}>"}
st, d = req("POST", f"/api/{endpoint}/", payload)
if st in (200, 201):
print(f" + {label}: '{match_val}' angelegt ({d.get('id')})")
return d
print(f" ! {label}: '{match_val}' FEHLER {st}: {json.dumps(d)[:300]}")
return None
def status_id(name: str):
return find("extras/statuses", {"name": name}, "name", name)["id"]
def namespace_id(name: str):
return find("ipam/namespaces", {"name": name}, "name", name)["id"]
# ---------------------------------------------------------------- Inventar
CLUSTER_TYPES = ["Proxmox VE", "Synology Docker", "Docker Desktop"]
# cluster -> clustertype
CLUSTERS = [("pve", "Proxmox VE"),
("nas-bf", "Synology Docker"),
("macmini", "Docker Desktop")]
# cluster -> [ (vm_name, primary_ip_with_mask_or_None, comment) ]
VMS = {
"pve": [
("haos", "10.40.20.105/24", "Home Assistant OS (QEMU 100)"),
("iob-master", "10.40.20.107/24", "ioBroker Master (LXC 101)"),
("mqtt5", "10.40.20.108/24", "MQTT-Broker Mosquitto (LXC 102)"),
("platform-pg", "10.40.20.20/24", "PostgreSQL 16 - Nautobot-DB (LXC 105, TP20)"),
("influxdb2", "10.40.20.8/24", "InfluxDB 2 (LXC 106)"),
("redis", "10.40.20.111/24", "Redis (LXC 107)"),
("mcast-relay", "10.40.10.21/24", "SSDP-Relay VLAN10<->40 (LXC 108); 2. Bein 10.40.40.21"),
("agh-bf1", "10.40.20.36/24", "AdGuard Home Resolver (LXC 110, TP23)"),
("agh-bf2", "10.40.20.37/24", "AdGuard Home Resolver (LXC 111, TP23)"),
("z2m25", "10.40.20.106/24", "Zigbee2MQTT (LXC 112)"),
("caddy-bf1", "10.40.20.40/24", "Caddy Reverse-Proxy (LXC 130, TP31)"),
("iob-exposed", "10.40.20.19/24", "ioBroker exposed (LXC 150)"),
("mariadb", "10.40.20.23/24", "MariaDB - vw/hass/gitea (LXC 302)"),
("RaspberryMatic","10.40.20.232/24", "Homematic CCU (QEMU 330)"),
("docker", "10.40.20.126/24", "Docker-Host nested (LXC 400)"),
],
"nas-bf": [
("nautobot", "10.40.20.30/24", "Nautobot Web (DS923 ep14)"),
("nautobot-worker", "10.40.20.31/24", "Nautobot Golden-Config Worker"),
("nautobot-scheduler", "10.40.20.32/24", "Nautobot Scheduler"),
("nautobot-redis", "10.40.20.34/24", "Nautobot Redis"),
("gitea", "10.40.20.157/24", "Gitea (git-SSoT)"),
("vaultwarden", "10.40.20.127/24", "Vaultwarden (DB=mariadb LXC302)"),
("shelly-mcp", "10.40.20.180/24", "Shelly-MCP"),
("pve-mcp", "10.40.20.173/24", "Proxmox-MCP (mcp-pve)"),
("iob_slave", "10.40.20.33/24", "ioBroker Slave"),
],
"macmini": [
# Bridge/Port-Mapping auf Host 10.40.20.167 -> kein eigenes primary_ip4
("n8n", None, "n8n (Docker Desktop, Host 10.40.20.167:5678)"),
("n8n-redis", None, "n8n Redis (intern, kein Host-Port)"),
("n8n-mcp", None, "n8n-MCP (Host 10.40.20.167:3001)"),
("gitea-mcp", None, "Gitea-MCP (Host 10.40.20.167:3000)"),
],
}
def main():
print(f"== TP37 Substrat-Seed ({'COMMIT' if COMMIT else 'DRY-RUN'}) @ {NB} ==")
st_active = status_id("Active")
ns_global = namespace_id("Global")
print(f" (Status Active={st_active} Namespace Global={ns_global})")
print("\n[1] ClusterTypes")
for ct in CLUSTER_TYPES:
get_or_create("virtualization/cluster-types", {"name": ct}, "name", ct,
{"name": ct}, "ClusterType")
print("\n[2] Clusters")
ct_lookup = {ct: find("virtualization/cluster-types", {"name": ct}, "name", ct)
for ct in CLUSTER_TYPES}
cluster_ids = {}
for cname, ctname in CLUSTERS:
ctrow = ct_lookup.get(ctname)
ct_id = ctrow["id"] if ctrow else None
cl = get_or_create("virtualization/clusters", {"name": cname}, "name", cname,
{"name": cname, "cluster_type": ct_id}, "Cluster")
cluster_ids[cname] = cl["id"] if cl else None
print("\n[3] VirtualMachines (+ eth0 + IPAddress + primary_ip4)")
n_vm = n_ip = 0
for cname, _ in CLUSTERS:
cl_id = cluster_ids.get(cname)
print(f" -- Cluster {cname} --")
for name, ip, comment in VMS[cname]:
vm = get_or_create(
"virtualization/virtual-machines", {"name": name}, "name", name,
{"name": name, "cluster": cl_id, "status": st_active,
"comments": comment}, "VM")
if not vm:
continue
n_vm += 1
if not ip:
print(f" (ohne primary_ip4 - Host-geteilt)")
continue
# VMInterface eth0
vmif = None
if is_dry(vm["id"]):
print(f" + eth0 @ {name} [DRY-RUN]")
vmif = {"id": f"<dry:{name}/eth0>"}
else:
vmif = find("virtualization/interfaces",
{"virtual_machine": name, "name": "eth0"}, "name", "eth0")
if vmif:
print(f" = eth0 @ {name} ({vmif['id']})")
else:
s, d = req("POST", "/api/virtualization/interfaces/",
{"virtual_machine": vm["id"], "name": "eth0",
"status": st_active})
if s in (200, 201):
vmif = d
print(f" + eth0 @ {name} ({d.get('id')})")
else:
print(f" ! eth0 @ {name} FEHLER {s}: {json.dumps(d)[:200]}")
continue
# IPAddress
ipobj = get_or_create("ipam/ip-addresses", {"address": ip}, "address", ip,
{"address": ip, "status": st_active,
"namespace": ns_global, "dns_name": ""},
"IP")
if not ipobj:
continue
n_ip += 1
if is_dry(vm["id"]) or is_dry(ipobj["id"]) or is_dry(vmif["id"]):
print(f" + assign {ip} -> {name}/eth0 + primary_ip4 [DRY-RUN]")
continue
# assign IP <-> interface (idempotent)
q = urllib.parse.urlencode({"vm_interface": vmif["id"], "depth": 0})
s, d = req("GET", f"/api/ipam/ip-address-to-interface/?{q}")
have = any((r.get("ip_address") or {}).get("id") == ipobj["id"]
for r in d.get("results", []))
if have:
print(f" = assign {ip} -> {name}/eth0")
else:
s, d = req("POST", "/api/ipam/ip-address-to-interface/",
{"ip_address": ipobj["id"], "vm_interface": vmif["id"]})
print(f" {'+' if s in (200,201) else '!'} assign {ip} -> {name}/eth0"
+ ("" if s in (200, 201) else f" FEHLER {s}: {json.dumps(d)[:150]}"))
# primary_ip4 (idempotent)
s, vmnow = req("GET", f"/api/virtualization/virtual-machines/{vm['id']}/?depth=0")
cur = vmnow.get("primary_ip4")
cur = cur.get("id") if isinstance(cur, dict) else cur
if cur == ipobj["id"]:
print(f" = primary_ip4 {ip} @ {name}")
else:
s, d = req("PATCH", f"/api/virtualization/virtual-machines/{vm['id']}/",
{"primary_ip4": ipobj["id"]})
print(f" {'+' if s == 200 else '!'} primary_ip4 {ip} @ {name}"
+ ("" if s == 200 else f" FEHLER {s}: {json.dumps(d)[:150]}"))
print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN - nichts geschrieben'}) "
f"| VMs={n_vm} IPs={n_ip} ==")
if __name__ == "__main__":
main()