TP34: terpene Quick-Win + Caddy-CF-Seeder + Renderer

- caddy/Caddyfile: terpene.teilzeitladen.xyz → 10.40.20.167:8890 (Quick-Win Schritt 0)
- nautobot/tp34-caddy-cf-seed.py: legt CustomFields caddy_fqdn/caddy_group auf
  ipam.Service an und setzt Werte fuer die 9 Routen mit vorhandenem Service-Objekt
- nautobot/tp34-caddy-render.py: rendert Caddyfile aus Nautobot, Diff gegen
  caddy/Caddyfile, optionaler Deploy via pct push + caddy reload

Deploy-Auflage fuer den terpene-Block:
  pct push 130 caddy/Caddyfile /etc/caddy/Caddyfile
  ssh root@10.40.20.40 "caddy reload --config /etc/caddy/Caddyfile"
(Plus CF-Tunnel-Route fuer terpene.teilzeitladen.xyz pruefen)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

caddy/Caddyfile

@@ -39,6 +39,10 @@ n8n.emsenhuber.net {
 	reverse_proxy http://10.40.20.167:5678
 }
 
+terpene.teilzeitladen.xyz {
+	reverse_proxy http://10.40.20.167:8890
+}
+
 gitea.emsenhuber.net {
 	reverse_proxy http://10.40.20.157:3000
 }

nautobot/tp34-caddy-cf-seed.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""
+TP34 — Caddy-CustomFields + Werte auf ipam.Service-Objekte setzen.
+
+Idempotenter Seeder (get-or-create) fuer zwei CustomFields auf ipam.Service:
+  caddy_fqdn  (text)   — Caddy-Hostname (z.B. vw.emsenhuber.net)
+  caddy_group (choice) — Kategorie: public / internal / mcp / https-backend
+
+Danach werden die Werte aus dem aktuellen Caddyfile (Ist-Stand 2026-09-07)
+auf die zugehoerigen ipam.Service-Objekte gesetzt.
+
+Vorbedingung: TP37 P2 muss gelaufen sein (ipam.Service-Objekte existieren).
+
+Token-Quelle: docs/nas-vlan20/stacks/210-nautobot.yml (gitignored),
+Schluessel NAUTOBOT_SUPERUSER_API_TOKEN.
+
+Aufruf:
+  python3 nautobot/tp34-caddy-cf-seed.py            # Dry-Run (default)
+  python3 nautobot/tp34-caddy-cf-seed.py --commit   # schreibt nach Nautobot
+"""
+import json
+import os
+import re
+import sys
+import urllib.request
+import urllib.parse
+import urllib.error
+from pathlib import Path
+
+NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080")
+REPO = Path(__file__).resolve().parent.parent
+YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml"
+COMMIT = "--commit" in sys.argv
+
+
+def load_token() -> str:
+    for line in YML.read_text().splitlines():
+        m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line)
+        if m:
+            return m.group(1)
+    sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden")
+
+
+TOKEN = load_token()
+HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json",
+        "Content-Type": "application/json"}
+
+
+def req(method: str, path: str, body=None):
+    url = f"{NB}{path}"
+    data = json.dumps(body).encode() if body is not None else None
+    r = urllib.request.Request(url, data=data, headers=HDRS, method=method)
+    try:
+        with urllib.request.urlopen(r, timeout=20) as resp:
+            return resp.status, json.loads(resp.read().decode() or "{}")
+    except urllib.error.HTTPError as e:
+        return e.code, json.loads(e.read().decode() or "{}")
+
+
+def find(endpoint: str, params: dict, match_key: str, match_val: str):
+    q = urllib.parse.urlencode(dict(params, depth=0))
+    st, d = req("GET", f"/api/{endpoint}/?{q}")
+    for row in d.get("results", []):
+        if row.get(match_key) == match_val:
+            return row
+    return None
+
+
+# ---------------------------------------------------------------- CustomField-Definitionen
+# Beide CFs werden auf ContentType ipam.service gesetzt.
+
+CF_FQDN = {
+    "key": "caddy_fqdn",
+    "label": "Caddy FQDN",
+    "type": "text",
+    "description": "Caddy-Hostname fuer diesen Dienst (z.B. vw.emsenhuber.net)",
+    "content_types": ["ipam.service"],
+    "required": False,
+    "default": None,
+}
+
+CF_GROUP = {
+    "key": "caddy_group",
+    "label": "Caddy Group",
+    "type": "select",
+    "description": "Caddy-Abschnitt (public/internal/mcp/https-backend)",
+    "content_types": ["ipam.service"],
+    "required": False,
+    "default": None,
+    "choices": [
+        {"value": "public",       "display": "Oeffentlich (CF Tunnel)"},
+        {"value": "internal",     "display": "Intern (Mesh/LAN)"},
+        {"value": "mcp",          "display": "MCP-Endpunkt"},
+        {"value": "https-backend", "display": "HTTPS-Backend (Selbstsignat)"},
+    ],
+}
+
+
+def ensure_custom_field(cf_def: dict) -> bool:
+    """Legt einen CustomField an wenn er nicht existiert. Gibt True zurueck wenn ok."""
+    key = cf_def["key"]
+    existing = find("extras/custom-fields", {"key": key}, "key", key)
+    if existing:
+        print(f"  = CF '{key}' existiert ({existing['id']})")
+        return True
+    if not COMMIT:
+        print(f"  + CF '{key}'  [DRY-RUN]")
+        return True
+    payload = {
+        "key": cf_def["key"],
+        "label": cf_def["label"],
+        "type": cf_def["type"],
+        "description": cf_def.get("description", ""),
+        "content_types": cf_def["content_types"],
+        "required": cf_def.get("required", False),
+    }
+    if cf_def.get("choices"):
+        payload["choices"] = cf_def["choices"]
+    st, d = req("POST", "/api/extras/custom-fields/", payload)
+    if st in (200, 201):
+        print(f"  + CF '{key}' angelegt ({d.get('id')})")
+        return True
+    print(f"  ! CF '{key}' FEHLER {st}: {json.dumps(d)[:300]}")
+    return False
+
+
+# ---------------------------------------------------------------- Caddyfile-Routen-Inventar
+# (vm_name, svc_name, fqdn, group)
+# Basiert auf caddy/Caddyfile Stand 2026-09-07 (terpene als Quick-Win schon drin).
+ROUTES = [
+    # Oeffentlich (via CF Tunnel)
+    ("vaultwarden",  "vaultwarden-http",  "vw.emsenhuber.net",               "public"),
+    ("haos",         "home-assistant",    "hass.emsenhuber.net",              "public"),
+    ("n8n",          "n8n-web",           "n8n.emsenhuber.net",               "public"),
+    ("gitea",        "gitea-http",        "gitea.emsenhuber.net",             "public"),
+    # terpene: kein ipam.Service in TP37 P2 → hier als Hinweis, wird separat geseedet
+    # ("macmini",    "terpene-worker",    "terpene.teilzeitladen.xyz",        "public"),
+    # Intern
+    ("nautobot",     "nautobot-web",      "nautobot.bf.emsenhuber.net",       "internal"),
+    ("docker",       "portainer",         "portainer.bf.emsenhuber.net",      "https-backend"),
+    # ds-bf: kein eigener ipam.Service — NAS-DSM haengt am Port 8080 direkt
+    # ("nautobot",  "ds-bf",             "ds-bf.emsenhuber.net",             "internal"),
+    # MCP-Endpunkte
+    ("pve-mcp",      "pve-mcp",           "pve-mcp.bf.emsenhuber.net",        "mcp"),
+    # ha-mcp: kein Service in TP37 P2 (haos lauscht auf 9583, nicht als Service angelegt)
+    ("n8n-mcp",      "n8n-mcp",           "n8n-mcp.bf.emsenhuber.net",        "mcp"),
+    ("gitea-mcp",    "gitea-mcp",         "gitea-mcp.bf.emsenhuber.net",      "mcp"),
+    # HTTPS-Backends (Selbstsignat)
+    ("docker",       "portainer",         "portainer.bf.emsenhuber.net",      "https-backend"),
+]
+
+# Nur eindeutige Eintraege (portainer erscheint nur einmal)
+_seen = set()
+_unique_routes = []
+for r in ROUTES:
+    k = (r[0], r[1])
+    if k not in _seen:
+        _seen.add(k)
+        _unique_routes.append(r)
+ROUTES = _unique_routes
+
+
+def main():
+    print(f"== TP34 Caddy-CF-Seed  ({'COMMIT' if COMMIT else 'DRY-RUN'})  @ {NB} ==")
+
+    print("\n[1] CustomFields anlegen")
+    ensure_custom_field(CF_FQDN)
+    ensure_custom_field(CF_GROUP)
+
+    print(f"\n[2] ipam.Service-Objekte mit Caddy-CFs versehen ({len(ROUTES)} Routen)")
+    n_ok = n_skip = 0
+    for vm_name, svc_name, fqdn, group in ROUTES:
+        # Service-Objekt suchen
+        svc = find("ipam/services", {"name": svc_name, "virtual_machine": vm_name},
+                   "name", svc_name)
+        if not svc:
+            print(f"  ! {vm_name}/{svc_name}: Service nicht gefunden — uebersprungen")
+            n_skip += 1
+            continue
+        svc_id = svc["id"]
+        # Aktuelle CF-Werte pruefen
+        cfs = svc.get("custom_fields", {})
+        current_fqdn = cfs.get("caddy_fqdn")
+        current_group = cfs.get("caddy_group")
+        if current_fqdn == fqdn and current_group == group:
+            print(f"  = {vm_name}/{svc_name}: '{fqdn}' ({group}) — unveraendert")
+            n_ok += 1
+            continue
+        if not COMMIT:
+            print(f"  + {vm_name}/{svc_name}: '{fqdn}' ({group})  [DRY-RUN]"
+                  + (f"  (war: fqdn='{current_fqdn}', group='{current_group}')"
+                     if current_fqdn or current_group else ""))
+            n_ok += 1
+            continue
+        st, d = req("PATCH", f"/api/ipam/services/{svc_id}/",
+                    {"custom_fields": {"caddy_fqdn": fqdn, "caddy_group": group}})
+        if st == 200:
+            print(f"  + {vm_name}/{svc_name}: '{fqdn}' ({group}) gesetzt")
+            n_ok += 1
+        else:
+            print(f"  ! {vm_name}/{svc_name}: FEHLER {st}: {json.dumps(d)[:300]}")
+            n_skip += 1
+
+    print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN — nichts geschrieben'}) "
+          f"| gesetzt={n_ok} uebersprungen={n_skip} ==")
+
+
+if __name__ == "__main__":
+    main()

nautobot/tp34-caddy-render.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""
+TP34 — Caddyfile aus Nautobot-Daten rendern.
+
+Liest alle ipam.Service-Objekte mit gesetztem CustomField `caddy_fqdn` aus Nautobot
+und rendert daraus ein Caddyfile. Optional: Diff gegen das eingecheckte Caddyfile
+oder deploy via pct push + caddy reload.
+
+Aufruf:
+  python3 nautobot/tp34-caddy-render.py            # rendert + diff
+  python3 nautobot/tp34-caddy-render.py --deploy   # rendert + diff + push + reload
+  python3 nautobot/tp34-caddy-render.py --out /tmp/Caddyfile  # nur rendern nach Datei
+"""
+import difflib
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.request
+import urllib.parse
+import urllib.error
+from pathlib import Path
+
+NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080")
+REPO = Path(__file__).resolve().parent.parent
+YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml"
+CADDYFILE = REPO / "caddy" / "Caddyfile"
+DEPLOY = "--deploy" in sys.argv
+OUT = None
+for i, a in enumerate(sys.argv):
+    if a == "--out" and i + 1 < len(sys.argv):
+        OUT = Path(sys.argv[i + 1])
+
+
+def load_token() -> str:
+    for line in YML.read_text().splitlines():
+        m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line)
+        if m:
+            return m.group(1)
+    sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden")
+
+
+TOKEN = load_token()
+HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json",
+        "Content-Type": "application/json"}
+
+
+def req(method: str, path: str, body=None):
+    url = f"{NB}{path}"
+    data = json.dumps(body).encode() if body is not None else None
+    r = urllib.request.Request(url, data=data, headers=HDRS, method=method)
+    try:
+        with urllib.request.urlopen(r, timeout=20) as resp:
+            return resp.status, json.loads(resp.read().decode() or "{}")
+    except urllib.error.HTTPError as e:
+        return e.code, json.loads(e.read().decode() or "{}")
+
+
+def fetch_caddy_services() -> list[dict]:
+    """Alle ipam.Service-Objekte mit caddy_fqdn != None."""
+    page, out = 1, []
+    while True:
+        q = urllib.parse.urlencode({"limit": 100, "offset": (page - 1) * 100, "depth": 1})
+        st, d = req("GET", f"/api/ipam/services/?{q}")
+        results = d.get("results", [])
+        if not results:
+            break
+        for svc in results:
+            cf = svc.get("custom_fields", {})
+            if cf.get("caddy_fqdn"):
+                out.append({
+                    "fqdn": cf["caddy_fqdn"],
+                    "group": cf.get("caddy_group") or "internal",
+                    "vm": (svc.get("virtual_machine") or {}).get("name", "?"),
+                    "svc": svc.get("name", "?"),
+                    "ports": svc.get("ports", []),
+                    "protocol": svc.get("protocol", "tcp"),
+                })
+        if not d.get("next"):
+            break
+        page += 1
+    return out
+
+
+def get_vm_ip(vm_name: str) -> str | None:
+    """Primaere IP einer VM aus Nautobot."""
+    q = urllib.parse.urlencode({"name": vm_name, "depth": 1})
+    st, d = req("GET", f"/api/virtualization/virtual-machines/?{q}")
+    for vm in d.get("results", []):
+        pip = vm.get("primary_ip4")
+        if isinstance(pip, dict):
+            addr = pip.get("address", "")
+            return addr.split("/")[0] if addr else None
+    return None
+
+
+HEADER = """\
+# TP34 — Caddyfile (aus Nautobot generiert)
+# SSoT: Nautobot ipam.Service + CustomFields caddy_fqdn / caddy_group
+# Manueller Deploy: pct push 130 caddy/Caddyfile /etc/caddy/Caddyfile
+#                   ssh root@10.40.20.40 "caddy reload --config /etc/caddy/Caddyfile"
+{
+\temail christoph@emsenhuber.com
+\tacme_dns cloudflare {env.CF_DNS_API_TOKEN}
+}
+
+# --- Backends mit Selbstsignat: Caddy prueft deren Zertifikat nicht ---------
+(selfsigned) {
+\ttransport http {
+\t\ttls_insecure_skip_verify
+\t}
+}
+
+"""
+
+GROUP_HEADERS = {
+    "public":        "# =========================================================================\n"
+                     "# OEFFENTLICH  (via Cloudflare Tunnel + Access; Clients ohne WARP)\n"
+                     "# =========================================================================\n",
+    "internal":      "# =========================================================================\n"
+                     "# INTERN  (Mesh/LAN; Split-Horizon zeigt diese Namen auf 10.40.20.40)\n"
+                     "# =========================================================================\n",
+    "mcp":           "# --- MCP-Endpunkte: heute spricht ~/.claude.json diese IPs roh --------------\n",
+    "https-backend": "# --- HTTPS-Backends (Selbstsignat) -----------------------------------------\n",
+}
+
+
+def render_caddyfile(services: list[dict], vm_ips: dict) -> str:
+    groups: dict[str, list] = {"public": [], "internal": [], "mcp": [], "https-backend": []}
+    for svc in sorted(services, key=lambda s: (s["group"], s["fqdn"])):
+        groups.setdefault(svc["group"], []).append(svc)
+
+    out = [HEADER]
+    for group, label in [("public", None), ("internal", None), ("mcp", None),
+                          ("https-backend", None)]:
+        svcs = groups.get(group, [])
+        if not svcs:
+            continue
+        hdr = GROUP_HEADERS.get(group, f"# --- {group} ---\n")
+        out.append(hdr)
+        for svc in svcs:
+            ip = vm_ips.get(svc["vm"], "MISSING")
+            port = svc["ports"][0] if svc["ports"] else 80
+            scheme = "https" if group == "https-backend" else "http"
+            backend = f"{scheme}://{ip}:{port}"
+            out.append(f'{svc["fqdn"]} {{\n')
+            if group == "https-backend":
+                out.append(f"\treverse_proxy {backend} {{\n\t\timport selfsigned\n\t}}\n")
+            else:
+                out.append(f"\treverse_proxy {backend}\n")
+            out.append("}\n\n")
+    return "".join(out)
+
+
+def main():
+    print(f"== TP34 Caddy-Render  @ {NB} ==")
+
+    print("\n[0] Caddy-Services aus Nautobot laden...")
+    services = fetch_caddy_services()
+    print(f"  {len(services)} Dienste mit caddy_fqdn gefunden")
+
+    print("\n[1] VM-IPs holen...")
+    vm_names = sorted({s["vm"] for s in services})
+    vm_ips = {}
+    for name in vm_names:
+        ip = get_vm_ip(name)
+        vm_ips[name] = ip or "MISSING"
+        status = ip or "! NICHT GEFUNDEN"
+        print(f"  {name}: {status}")
+
+    print("\n[2] Caddyfile rendern...")
+    rendered = render_caddyfile(services, vm_ips)
+
+    if OUT:
+        OUT.write_text(rendered)
+        print(f"  → geschrieben nach {OUT}")
+        return
+
+    print("\n[3] Diff gegen eingechecktes Caddyfile (caddy/Caddyfile)...")
+    live = CADDYFILE.read_text() if CADDYFILE.exists() else ""
+    diff = list(difflib.unified_diff(
+        live.splitlines(keepends=True),
+        rendered.splitlines(keepends=True),
+        fromfile="caddy/Caddyfile (eingecheckt)",
+        tofile="gerendert (Nautobot)",
+    ))
+    if not diff:
+        print("  ✅ Kein Diff — Caddyfile ist in Sync mit Nautobot")
+    else:
+        print(f"  ⚠️  {len(diff)} Diff-Zeilen:")
+        for line in diff[:80]:
+            print("    " + line, end="")
+        if len(diff) > 80:
+            print(f"  … ({len(diff) - 80} weitere Zeilen)")
+
+    if DEPLOY:
+        if diff:
+            print("\n[4] Deploy...")
+            CADDYFILE.write_text(rendered)
+            ret = subprocess.call(
+                ["pct", "push", "130", str(CADDYFILE), "/etc/caddy/Caddyfile"])
+            if ret == 0:
+                subprocess.call(
+                    ["ssh", "root@10.40.20.40",
+                     "caddy reload --config /etc/caddy/Caddyfile"])
+            else:
+                print("  ! pct push fehlgeschlagen")
+        else:
+            print("\n[4] Deploy: nichts zu tun (kein Diff)")
+
+    print("\n== fertig ==")
+
+
+if __name__ == "__main__":
+    main()