bf-infra/nautobot/tp37-substrat-seed.py

#!/usr/bin/env python3
"""
TP37 Punkt 1 (Substrat) — Hosts/Container als native Nautobot-Objekte.

Idempotenter Seeder (get-or-create) fuer die native virtualization-Modellwelt:
  ClusterType (3x: Proxmox VE / Synology Docker / Docker Desktop)
    -> Cluster (pve / nas-bf / macmini)
      -> VirtualMachine je Workload (Status Active)
         + VMInterface "eth0" + ipam.IPAddress (Namespace Global) + primary_ip4

Datenmodell-Entscheid (User 2026-09-06): native Modelle, KEINE eigene App
(konsistent zu TP44). Muster: nautobot/tp44-cf-estate-seed.py.

Umfang Etappe 1 (User-Default 2026-09-07):
  - komplettes Substrat: alle 15 pve-Gaeste (inkl. Smart-Home) + alle running
    Container auf nas-bf (Portainer ep14) und macmini (ep20).
  - macmini-Container teilen sich die Host-IP 10.40.20.167 (Bridge/Port-Mapping)
    -> KEIN primary_ip4; Port kommt in TP37 P2 (ipam.Service).
  - portainer_agent (Bridge 172.17.0.2, kein Workload) weggelassen.
  - gestoppte (exited) NAS-Container weggelassen (nur running).

Datenstand: Live-Inventur 2026-09-07 (pve-mcp pct config, Portainer ep14,
ssh macmini docker ps, ARP fuer die 2 QEMU-IPs).

Token-Quelle: docs/nas-vlan20/stacks/210-nautobot.yml (gitignored),
Schluessel NAUTOBOT_SUPERUSER_API_TOKEN.

Aufruf:
  python3 nautobot/tp37-substrat-seed.py            # Dry-Run (default)
  python3 nautobot/tp37-substrat-seed.py --commit   # legt an
"""
import json
import os
import re
import sys
import urllib.request
import urllib.parse
import urllib.error
from pathlib import Path

NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.154:8080")
REPO = Path(__file__).resolve().parent.parent
YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml"
COMMIT = "--commit" in sys.argv


def load_token() -> str:
    for line in YML.read_text().splitlines():
        m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line)
        if m:
            return m.group(1)
    sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden")


TOKEN = load_token()
HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json",
        "Content-Type": "application/json"}


def req(method: str, path: str, body=None):
    url = f"{NB}{path}"
    data = json.dumps(body).encode() if body is not None else None
    r = urllib.request.Request(url, data=data, headers=HDRS, method=method)
    try:
        with urllib.request.urlopen(r, timeout=20) as resp:
            return resp.status, json.loads(resp.read().decode() or "{}")
    except urllib.error.HTTPError as e:
        return e.code, json.loads(e.read().decode() or "{}")


def is_dry(v) -> bool:
    return isinstance(v, str) and v.startswith("<dry:")


def find(endpoint: str, params: dict, match_key: str, match_val: str):
    q = urllib.parse.urlencode(dict(params, depth=0))
    st, d = req("GET", f"/api/{endpoint}/?{q}")
    for row in d.get("results", []):
        if row.get(match_key) == match_val:
            return row
    return None


def get_or_create(endpoint: str, find_params: dict, match_key: str,
                  match_val: str, payload: dict, label: str):
    existing = find(endpoint, find_params, match_key, match_val)
    if existing:
        print(f"  = {label}: '{match_val}' existiert ({existing['id']})")
        return existing
    if not COMMIT:
        print(f"  + {label}: '{match_val}'  [DRY-RUN]")
        return {"id": f"<dry:{match_val}>"}
    st, d = req("POST", f"/api/{endpoint}/", payload)
    if st in (200, 201):
        print(f"  + {label}: '{match_val}' angelegt ({d.get('id')})")
        return d
    print(f"  ! {label}: '{match_val}' FEHLER {st}: {json.dumps(d)[:300]}")
    return None


def status_id(name: str):
    return find("extras/statuses", {"name": name}, "name", name)["id"]


def namespace_id(name: str):
    return find("ipam/namespaces", {"name": name}, "name", name)["id"]


# ---------------------------------------------------------------- Inventar
CLUSTER_TYPES = ["Proxmox VE", "Synology Docker", "Docker Desktop"]

# cluster -> clustertype
CLUSTERS = [("pve", "Proxmox VE"),
            ("nas-bf", "Synology Docker"),
            ("macmini", "Docker Desktop")]

# cluster -> [ (vm_name, primary_ip_with_mask_or_None, comment) ]
VMS = {
    "pve": [
        ("haos",          "10.40.20.105/24", "Home Assistant OS (QEMU 100)"),
        ("iob-master",    "10.40.20.107/24", "ioBroker Master (LXC 101)"),
        ("mqtt5",         "10.40.20.108/24", "MQTT-Broker Mosquitto (LXC 102)"),
        ("platform-pg",   "10.40.20.20/24",  "PostgreSQL 16 - Nautobot-DB (LXC 105, TP20)"),
        ("influxdb2",     "10.40.20.8/24",   "InfluxDB 2 (LXC 106)"),
        ("redis",         "10.40.20.111/24", "Redis (LXC 107)"),
        ("mcast-relay",   "10.40.10.21/24",  "SSDP-Relay VLAN10<->40 (LXC 108); 2. Bein 10.40.40.21"),
        ("agh-bf1",       "10.40.20.36/24",  "AdGuard Home Resolver (LXC 110, TP23)"),
        ("agh-bf2",       "10.40.20.37/24",  "AdGuard Home Resolver (LXC 111, TP23)"),
        ("z2m25",         "10.40.20.106/24", "Zigbee2MQTT (LXC 112)"),
        ("caddy-bf1",     "10.40.20.40/24",  "Caddy Reverse-Proxy (LXC 130, TP31)"),
        ("iob-exposed",   "10.40.20.19/24",  "ioBroker exposed (LXC 150)"),
        ("mariadb",       "10.40.20.23/24",  "MariaDB - vw/hass/gitea (LXC 302)"),
        ("RaspberryMatic","10.40.20.232/24", "Homematic CCU (QEMU 330)"),
        ("docker",        "10.40.20.126/24", "Docker-Host nested (LXC 400)"),
    ],
    "nas-bf": [
        ("nautobot",           "10.40.20.154/24",  "Nautobot Web (DS923 ep14)"),
        ("nautobot-worker",    "10.40.20.155/24",  "Nautobot Golden-Config Worker"),
        ("nautobot-scheduler", "10.40.20.156/24",  "Nautobot Scheduler"),
        ("nautobot-redis",     "10.40.20.158/24",  "Nautobot Redis"),
        ("gitea",              "10.40.20.157/24", "Gitea (git-SSoT)"),
        ("vaultwarden",        "10.40.20.127/24", "Vaultwarden (DB=mariadb LXC302)"),
        ("shelly-mcp",         "10.40.20.180/24", "Shelly-MCP"),
        ("pve-mcp",            "10.40.20.173/24", "Proxmox-MCP (mcp-pve)"),
        ("iob_slave",          "10.40.20.33/24",  "ioBroker Slave"),
    ],
    "macmini": [
        # Bridge/Port-Mapping auf Host 10.40.20.167 -> kein eigenes primary_ip4
        ("n8n",       None, "n8n (Docker Desktop, Host 10.40.20.167:5678)"),
        ("n8n-redis", None, "n8n Redis (intern, kein Host-Port)"),
        ("n8n-mcp",   None, "n8n-MCP (Host 10.40.20.167:3001)"),
        ("gitea-mcp", None, "Gitea-MCP (Host 10.40.20.167:3000)"),
    ],
}


def main():
    print(f"== TP37 Substrat-Seed  ({'COMMIT' if COMMIT else 'DRY-RUN'})  @ {NB} ==")

    st_active = status_id("Active")
    ns_global = namespace_id("Global")
    print(f"  (Status Active={st_active}  Namespace Global={ns_global})")

    print("\n[1] ClusterTypes")
    for ct in CLUSTER_TYPES:
        get_or_create("virtualization/cluster-types", {"name": ct}, "name", ct,
                      {"name": ct}, "ClusterType")

    print("\n[2] Clusters")
    ct_lookup = {ct: find("virtualization/cluster-types", {"name": ct}, "name", ct)
                 for ct in CLUSTER_TYPES}
    cluster_ids = {}
    for cname, ctname in CLUSTERS:
        ctrow = ct_lookup.get(ctname)
        ct_id = ctrow["id"] if ctrow else None
        cl = get_or_create("virtualization/clusters", {"name": cname}, "name", cname,
                           {"name": cname, "cluster_type": ct_id}, "Cluster")
        cluster_ids[cname] = cl["id"] if cl else None

    print("\n[3] VirtualMachines (+ eth0 + IPAddress + primary_ip4)")
    n_vm = n_ip = 0
    for cname, _ in CLUSTERS:
        cl_id = cluster_ids.get(cname)
        print(f"  -- Cluster {cname} --")
        for name, ip, comment in VMS[cname]:
            vm = get_or_create(
                "virtualization/virtual-machines", {"name": name}, "name", name,
                {"name": name, "cluster": cl_id, "status": st_active,
                 "comments": comment}, "VM")
            if not vm:
                continue
            n_vm += 1
            if not ip:
                print(f"      (ohne primary_ip4 - Host-geteilt)")
                continue
            # VMInterface eth0
            vmif = None
            if is_dry(vm["id"]):
                print(f"      + eth0 @ {name}  [DRY-RUN]")
                vmif = {"id": f"<dry:{name}/eth0>"}
            else:
                vmif = find("virtualization/interfaces",
                            {"virtual_machine": name, "name": "eth0"}, "name", "eth0")
                if vmif:
                    print(f"      = eth0 @ {name} ({vmif['id']})")
                else:
                    s, d = req("POST", "/api/virtualization/interfaces/",
                               {"virtual_machine": vm["id"], "name": "eth0",
                                "status": st_active})
                    if s in (200, 201):
                        vmif = d
                        print(f"      + eth0 @ {name} ({d.get('id')})")
                    else:
                        print(f"      ! eth0 @ {name} FEHLER {s}: {json.dumps(d)[:200]}")
                        continue
            # IPAddress
            ipobj = get_or_create("ipam/ip-addresses", {"address": ip}, "address", ip,
                                  {"address": ip, "status": st_active,
                                   "namespace": ns_global, "dns_name": ""},
                                  "IP")
            if not ipobj:
                continue
            n_ip += 1
            if is_dry(vm["id"]) or is_dry(ipobj["id"]) or is_dry(vmif["id"]):
                print(f"      + assign {ip} -> {name}/eth0 + primary_ip4  [DRY-RUN]")
                continue
            # assign IP <-> interface (idempotent)
            q = urllib.parse.urlencode({"vm_interface": vmif["id"], "depth": 0})
            s, d = req("GET", f"/api/ipam/ip-address-to-interface/?{q}")
            have = any((r.get("ip_address") or {}).get("id") == ipobj["id"]
                       for r in d.get("results", []))
            if have:
                print(f"      = assign {ip} -> {name}/eth0")
            else:
                s, d = req("POST", "/api/ipam/ip-address-to-interface/",
                           {"ip_address": ipobj["id"], "vm_interface": vmif["id"]})
                print(f"      {'+' if s in (200,201) else '!'} assign {ip} -> {name}/eth0"
                      + ("" if s in (200, 201) else f" FEHLER {s}: {json.dumps(d)[:150]}"))
            # primary_ip4 (idempotent)
            s, vmnow = req("GET", f"/api/virtualization/virtual-machines/{vm['id']}/?depth=0")
            cur = vmnow.get("primary_ip4")
            cur = cur.get("id") if isinstance(cur, dict) else cur
            if cur == ipobj["id"]:
                print(f"      = primary_ip4 {ip} @ {name}")
            else:
                s, d = req("PATCH", f"/api/virtualization/virtual-machines/{vm['id']}/",
                           {"primary_ip4": ipobj["id"]})
                print(f"      {'+' if s == 200 else '!'} primary_ip4 {ip} @ {name}"
                      + ("" if s == 200 else f" FEHLER {s}: {json.dumps(d)[:150]}"))

    print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN - nichts geschrieben'}) "
          f"| VMs={n_vm} IPs={n_ip} ==")


if __name__ == "__main__":
    main()