From bca8f6e27e536725b9794ea7e292dffc2b896f9e Mon Sep 17 00:00:00 2001 From: Christoph Emsenhuber Date: Tue, 8 Sep 2026 14:00:02 +0200 Subject: [PATCH] =?UTF-8?q?TP34=20zur=C3=BCckgezogen:=20Caddyfile=20via=20?= =?UTF-8?q?GC=20verworfen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session lief auf veralteten Annahmen (Hostname terpene.* statt 420.*, zweiter CF-Token erfunden) — entgegen dem User-Constraint "raten verboten". 420.teilzeitladen.xyz ist ein Cloudflare-Worker (Projekt blaueblueten), nicht ein Caddy-Backend; :8890 gehoert zu terpene.* via Tunnel bf-mm. Caddyfile war netto unveraendert, nur die beiden neuen Skripte werden entfernt. Live LXC 130 ist bereits Baseline (e96fca5), Nautobot fuer TP34 unberuehrt (nur Dry-Run). Uebergabe: ~/.claude/plans/tp37-p2-seeden-dienste-silly-lerdorf.md Co-Authored-By: Claude Opus 4.8 --- nautobot/tp34-caddy-cf-seed.py | 209 ------------------------------- nautobot/tp34-caddy-render.py | 216 --------------------------------- 2 files changed, 425 deletions(-) delete mode 100644 nautobot/tp34-caddy-cf-seed.py delete mode 100644 nautobot/tp34-caddy-render.py diff --git a/nautobot/tp34-caddy-cf-seed.py b/nautobot/tp34-caddy-cf-seed.py deleted file mode 100644 index 9014636..0000000 --- a/nautobot/tp34-caddy-cf-seed.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python3 -""" -TP34 — Caddy-CustomFields + Werte auf ipam.Service-Objekte setzen. - -Idempotenter Seeder (get-or-create) fuer zwei CustomFields auf ipam.Service: - caddy_fqdn (text) — Caddy-Hostname (z.B. vw.emsenhuber.net) - caddy_group (choice) — Kategorie: public / internal / mcp / https-backend - -Danach werden die Werte aus dem aktuellen Caddyfile (Ist-Stand 2026-09-07) -auf die zugehoerigen ipam.Service-Objekte gesetzt. - -Vorbedingung: TP37 P2 muss gelaufen sein (ipam.Service-Objekte existieren). - -Token-Quelle: docs/nas-vlan20/stacks/210-nautobot.yml (gitignored), -Schluessel NAUTOBOT_SUPERUSER_API_TOKEN. - -Aufruf: - python3 nautobot/tp34-caddy-cf-seed.py # Dry-Run (default) - python3 nautobot/tp34-caddy-cf-seed.py --commit # schreibt nach Nautobot -""" -import json -import os -import re -import sys -import urllib.request -import urllib.parse -import urllib.error -from pathlib import Path - -NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080") -REPO = Path(__file__).resolve().parent.parent -YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml" -COMMIT = "--commit" in sys.argv - - -def load_token() -> str: - for line in YML.read_text().splitlines(): - m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line) - if m: - return m.group(1) - sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden") - - -TOKEN = load_token() -HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json", - "Content-Type": "application/json"} - - -def req(method: str, path: str, body=None): - url = f"{NB}{path}" - data = json.dumps(body).encode() if body is not None else None - r = urllib.request.Request(url, data=data, headers=HDRS, method=method) - try: - with urllib.request.urlopen(r, timeout=20) as resp: - return resp.status, json.loads(resp.read().decode() or "{}") - except urllib.error.HTTPError as e: - return e.code, json.loads(e.read().decode() or "{}") - - -def find(endpoint: str, params: dict, match_key: str, match_val: str): - q = urllib.parse.urlencode(dict(params, depth=0)) - st, d = req("GET", f"/api/{endpoint}/?{q}") - for row in d.get("results", []): - if row.get(match_key) == match_val: - return row - return None - - -# ---------------------------------------------------------------- CustomField-Definitionen -# Beide CFs werden auf ContentType ipam.service gesetzt. - -CF_FQDN = { - "key": "caddy_fqdn", - "label": "Caddy FQDN", - "type": "text", - "description": "Caddy-Hostname fuer diesen Dienst (z.B. vw.emsenhuber.net)", - "content_types": ["ipam.service"], - "required": False, - "default": None, -} - -CF_GROUP = { - "key": "caddy_group", - "label": "Caddy Group", - "type": "select", - "description": "Caddy-Abschnitt (public/internal/mcp/https-backend)", - "content_types": ["ipam.service"], - "required": False, - "default": None, - "choices": [ - {"value": "public", "display": "Oeffentlich (CF Tunnel)"}, - {"value": "internal", "display": "Intern (Mesh/LAN)"}, - {"value": "mcp", "display": "MCP-Endpunkt"}, - {"value": "https-backend", "display": "HTTPS-Backend (Selbstsignat)"}, - ], -} - - -def ensure_custom_field(cf_def: dict) -> bool: - """Legt einen CustomField an wenn er nicht existiert. Gibt True zurueck wenn ok.""" - key = cf_def["key"] - existing = find("extras/custom-fields", {"key": key}, "key", key) - if existing: - print(f" = CF '{key}' existiert ({existing['id']})") - return True - if not COMMIT: - print(f" + CF '{key}' [DRY-RUN]") - return True - payload = { - "key": cf_def["key"], - "label": cf_def["label"], - "type": cf_def["type"], - "description": cf_def.get("description", ""), - "content_types": cf_def["content_types"], - "required": cf_def.get("required", False), - } - if cf_def.get("choices"): - payload["choices"] = cf_def["choices"] - st, d = req("POST", "/api/extras/custom-fields/", payload) - if st in (200, 201): - print(f" + CF '{key}' angelegt ({d.get('id')})") - return True - print(f" ! CF '{key}' FEHLER {st}: {json.dumps(d)[:300]}") - return False - - -# ---------------------------------------------------------------- Caddyfile-Routen-Inventar -# (vm_name, svc_name, fqdn, group) -# Basiert auf caddy/Caddyfile Stand 2026-09-07 (terpene als Quick-Win schon drin). -ROUTES = [ - # Oeffentlich (via CF Tunnel) - ("vaultwarden", "vaultwarden-http", "vw.emsenhuber.net", "public"), - ("haos", "home-assistant", "hass.emsenhuber.net", "public"), - ("n8n", "n8n-web", "n8n.emsenhuber.net", "public"), - ("gitea", "gitea-http", "gitea.emsenhuber.net", "public"), - # 420.teilzeitladen.xyz: kein ipam.Service in TP37 P2 → hier als Hinweis, wird separat geseedet - # ("macmini", "terpene-worker", "420.teilzeitladen.xyz", "public"), - # Intern - ("nautobot", "nautobot-web", "nautobot.bf.emsenhuber.net", "internal"), - ("docker", "portainer", "portainer.bf.emsenhuber.net", "https-backend"), - # ds-bf: kein eigener ipam.Service — NAS-DSM haengt am Port 8080 direkt - # ("nautobot", "ds-bf", "ds-bf.emsenhuber.net", "internal"), - # MCP-Endpunkte - ("pve-mcp", "pve-mcp", "pve-mcp.bf.emsenhuber.net", "mcp"), - # ha-mcp: kein Service in TP37 P2 (haos lauscht auf 9583, nicht als Service angelegt) - ("n8n-mcp", "n8n-mcp", "n8n-mcp.bf.emsenhuber.net", "mcp"), - ("gitea-mcp", "gitea-mcp", "gitea-mcp.bf.emsenhuber.net", "mcp"), - # HTTPS-Backends (Selbstsignat) - ("docker", "portainer", "portainer.bf.emsenhuber.net", "https-backend"), -] - -# Nur eindeutige Eintraege (portainer erscheint nur einmal) -_seen = set() -_unique_routes = [] -for r in ROUTES: - k = (r[0], r[1]) - if k not in _seen: - _seen.add(k) - _unique_routes.append(r) -ROUTES = _unique_routes - - -def main(): - print(f"== TP34 Caddy-CF-Seed ({'COMMIT' if COMMIT else 'DRY-RUN'}) @ {NB} ==") - - print("\n[1] CustomFields anlegen") - ensure_custom_field(CF_FQDN) - ensure_custom_field(CF_GROUP) - - print(f"\n[2] ipam.Service-Objekte mit Caddy-CFs versehen ({len(ROUTES)} Routen)") - n_ok = n_skip = 0 - for vm_name, svc_name, fqdn, group in ROUTES: - # Service-Objekt suchen - svc = find("ipam/services", {"name": svc_name, "virtual_machine": vm_name}, - "name", svc_name) - if not svc: - print(f" ! {vm_name}/{svc_name}: Service nicht gefunden — uebersprungen") - n_skip += 1 - continue - svc_id = svc["id"] - # Aktuelle CF-Werte pruefen - cfs = svc.get("custom_fields", {}) - current_fqdn = cfs.get("caddy_fqdn") - current_group = cfs.get("caddy_group") - if current_fqdn == fqdn and current_group == group: - print(f" = {vm_name}/{svc_name}: '{fqdn}' ({group}) — unveraendert") - n_ok += 1 - continue - if not COMMIT: - print(f" + {vm_name}/{svc_name}: '{fqdn}' ({group}) [DRY-RUN]" - + (f" (war: fqdn='{current_fqdn}', group='{current_group}')" - if current_fqdn or current_group else "")) - n_ok += 1 - continue - st, d = req("PATCH", f"/api/ipam/services/{svc_id}/", - {"custom_fields": {"caddy_fqdn": fqdn, "caddy_group": group}}) - if st == 200: - print(f" + {vm_name}/{svc_name}: '{fqdn}' ({group}) gesetzt") - n_ok += 1 - else: - print(f" ! {vm_name}/{svc_name}: FEHLER {st}: {json.dumps(d)[:300]}") - n_skip += 1 - - print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN — nichts geschrieben'}) " - f"| gesetzt={n_ok} uebersprungen={n_skip} ==") - - -if __name__ == "__main__": - main() diff --git a/nautobot/tp34-caddy-render.py b/nautobot/tp34-caddy-render.py deleted file mode 100644 index 1844645..0000000 --- a/nautobot/tp34-caddy-render.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python3 -""" -TP34 — Caddyfile aus Nautobot-Daten rendern. - -Liest alle ipam.Service-Objekte mit gesetztem CustomField `caddy_fqdn` aus Nautobot -und rendert daraus ein Caddyfile. Optional: Diff gegen das eingecheckte Caddyfile -oder deploy via pct push + caddy reload. - -Aufruf: - python3 nautobot/tp34-caddy-render.py # rendert + diff - python3 nautobot/tp34-caddy-render.py --deploy # rendert + diff + push + reload - python3 nautobot/tp34-caddy-render.py --out /tmp/Caddyfile # nur rendern nach Datei -""" -import difflib -import json -import os -import re -import subprocess -import sys -import urllib.request -import urllib.parse -import urllib.error -from pathlib import Path - -NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080") -REPO = Path(__file__).resolve().parent.parent -YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml" -CADDYFILE = REPO / "caddy" / "Caddyfile" -DEPLOY = "--deploy" in sys.argv -OUT = None -for i, a in enumerate(sys.argv): - if a == "--out" and i + 1 < len(sys.argv): - OUT = Path(sys.argv[i + 1]) - - -def load_token() -> str: - for line in YML.read_text().splitlines(): - m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line) - if m: - return m.group(1) - sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden") - - -TOKEN = load_token() -HDRS = {"Authorization": f"Token {TOKEN}", "Accept": "application/json", - "Content-Type": "application/json"} - - -def req(method: str, path: str, body=None): - url = f"{NB}{path}" - data = json.dumps(body).encode() if body is not None else None - r = urllib.request.Request(url, data=data, headers=HDRS, method=method) - try: - with urllib.request.urlopen(r, timeout=20) as resp: - return resp.status, json.loads(resp.read().decode() or "{}") - except urllib.error.HTTPError as e: - return e.code, json.loads(e.read().decode() or "{}") - - -def fetch_caddy_services() -> list[dict]: - """Alle ipam.Service-Objekte mit caddy_fqdn != None.""" - page, out = 1, [] - while True: - q = urllib.parse.urlencode({"limit": 100, "offset": (page - 1) * 100, "depth": 1}) - st, d = req("GET", f"/api/ipam/services/?{q}") - results = d.get("results", []) - if not results: - break - for svc in results: - cf = svc.get("custom_fields", {}) - if cf.get("caddy_fqdn"): - out.append({ - "fqdn": cf["caddy_fqdn"], - "group": cf.get("caddy_group") or "internal", - "vm": (svc.get("virtual_machine") or {}).get("name", "?"), - "svc": svc.get("name", "?"), - "ports": svc.get("ports", []), - "protocol": svc.get("protocol", "tcp"), - }) - if not d.get("next"): - break - page += 1 - return out - - -def get_vm_ip(vm_name: str) -> str | None: - """Primaere IP einer VM aus Nautobot.""" - q = urllib.parse.urlencode({"name": vm_name, "depth": 1}) - st, d = req("GET", f"/api/virtualization/virtual-machines/?{q}") - for vm in d.get("results", []): - pip = vm.get("primary_ip4") - if isinstance(pip, dict): - addr = pip.get("address", "") - return addr.split("/")[0] if addr else None - return None - - -HEADER = """\ -# TP34 — Caddyfile (aus Nautobot generiert) -# SSoT: Nautobot ipam.Service + CustomFields caddy_fqdn / caddy_group -# Manueller Deploy: pct push 130 caddy/Caddyfile /etc/caddy/Caddyfile -# ssh root@10.40.20.40 "caddy reload --config /etc/caddy/Caddyfile" -{ -\temail christoph@emsenhuber.com -\tacme_dns cloudflare {env.CF_DNS_API_TOKEN} -} - -# --- Backends mit Selbstsignat: Caddy prueft deren Zertifikat nicht --------- -(selfsigned) { -\ttransport http { -\t\ttls_insecure_skip_verify -\t} -} - -""" - -GROUP_HEADERS = { - "public": "# =========================================================================\n" - "# OEFFENTLICH (via Cloudflare Tunnel + Access; Clients ohne WARP)\n" - "# =========================================================================\n", - "internal": "# =========================================================================\n" - "# INTERN (Mesh/LAN; Split-Horizon zeigt diese Namen auf 10.40.20.40)\n" - "# =========================================================================\n", - "mcp": "# --- MCP-Endpunkte: heute spricht ~/.claude.json diese IPs roh --------------\n", - "https-backend": "# --- HTTPS-Backends (Selbstsignat) -----------------------------------------\n", -} - - -def render_caddyfile(services: list[dict], vm_ips: dict) -> str: - groups: dict[str, list] = {"public": [], "internal": [], "mcp": [], "https-backend": []} - for svc in sorted(services, key=lambda s: (s["group"], s["fqdn"])): - groups.setdefault(svc["group"], []).append(svc) - - out = [HEADER] - for group, label in [("public", None), ("internal", None), ("mcp", None), - ("https-backend", None)]: - svcs = groups.get(group, []) - if not svcs: - continue - hdr = GROUP_HEADERS.get(group, f"# --- {group} ---\n") - out.append(hdr) - for svc in svcs: - ip = vm_ips.get(svc["vm"], "MISSING") - port = svc["ports"][0] if svc["ports"] else 80 - scheme = "https" if group == "https-backend" else "http" - backend = f"{scheme}://{ip}:{port}" - out.append(f'{svc["fqdn"]} {{\n') - if group == "https-backend": - out.append(f"\treverse_proxy {backend} {{\n\t\timport selfsigned\n\t}}\n") - else: - out.append(f"\treverse_proxy {backend}\n") - out.append("}\n\n") - return "".join(out) - - -def main(): - print(f"== TP34 Caddy-Render @ {NB} ==") - - print("\n[0] Caddy-Services aus Nautobot laden...") - services = fetch_caddy_services() - print(f" {len(services)} Dienste mit caddy_fqdn gefunden") - - print("\n[1] VM-IPs holen...") - vm_names = sorted({s["vm"] for s in services}) - vm_ips = {} - for name in vm_names: - ip = get_vm_ip(name) - vm_ips[name] = ip or "MISSING" - status = ip or "! NICHT GEFUNDEN" - print(f" {name}: {status}") - - print("\n[2] Caddyfile rendern...") - rendered = render_caddyfile(services, vm_ips) - - if OUT: - OUT.write_text(rendered) - print(f" → geschrieben nach {OUT}") - return - - print("\n[3] Diff gegen eingechecktes Caddyfile (caddy/Caddyfile)...") - live = CADDYFILE.read_text() if CADDYFILE.exists() else "" - diff = list(difflib.unified_diff( - live.splitlines(keepends=True), - rendered.splitlines(keepends=True), - fromfile="caddy/Caddyfile (eingecheckt)", - tofile="gerendert (Nautobot)", - )) - if not diff: - print(" ✅ Kein Diff — Caddyfile ist in Sync mit Nautobot") - else: - print(f" ⚠️ {len(diff)} Diff-Zeilen:") - for line in diff[:80]: - print(" " + line, end="") - if len(diff) > 80: - print(f" … ({len(diff) - 80} weitere Zeilen)") - - if DEPLOY: - if diff: - print("\n[4] Deploy...") - CADDYFILE.write_text(rendered) - ret = subprocess.call( - ["pct", "push", "130", str(CADDYFILE), "/etc/caddy/Caddyfile"]) - if ret == 0: - subprocess.call( - ["ssh", "root@10.40.20.40", - "caddy reload --config /etc/caddy/Caddyfile"]) - else: - print(" ! pct push fehlgeschlagen") - else: - print("\n[4] Deploy: nichts zu tun (kein Diff)") - - print("\n== fertig ==") - - -if __name__ == "__main__": - main()