TP44 Teil B: CF-API→Nautobot-Sync (WIP, Dry-Run verifiziert)
nautobot/tp44-cf-sync.py — periodischer Live-Pull der CF-Estate nach Nautobot (ersetzt den Inline-Snapshot des Seeders). Reuse: api() aus caddy/cf-sync.py, Nautobot-Helper des Seeders. - ID-Matching (tunnel_id/app-id/script/service_id) statt Name → rename-fest, keine Duplikate - Merge statt Replace → kuratierte extra_config-Keys bleiben - Verwaisung: extra_config.stale=true statt Hard-Delete; sicher uebersprungen, solange Token-Scope unvollstaendig - 403-tolerant: deckt die 15 lesbaren Objekte driftfrei ab, zieht Hyperdrive/VPC automatisch mit, sobald Token-Scope da ist Dry-Run: 11 Services unveraendert, 4 Networks Enrichment, 0 Duplikate. Offen: Token-Scope (Hyperdrive+VPC) erweitern, dann Full-Sync + Cron-Host. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
315
nautobot/tp44-cf-sync.py
Normal file
315
nautobot/tp44-cf-sync.py
Normal file
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
TP44 Etappe 2 Teil B — CF-Estate LIVE aus der Cloudflare-API nach Nautobot syncen.
|
||||
|
||||
Ersetzt den Inline-Snapshot von tp44-cf-estate-seed.py durch einen periodischen
|
||||
Live-Pull (get-or-create + Merge). Idempotent, Dry-Run default. Kein Push nach CF —
|
||||
read-only Pull, Nautobot bleibt Dokumentations-SSoT, CF operativer Master.
|
||||
|
||||
Deckung (Live-Pull, je Endpoint 403-tolerant):
|
||||
CloudNetwork cfd_tunnel + warp_connector (4)
|
||||
CloudService workers/scripts + access/apps (11) [Repo-Token OK]
|
||||
CloudService hyperdrive/configs + VPC directory (3) [braucht Token-Scope]
|
||||
|
||||
Verwaisung: ein CF-Objekt, das in Nautobot steht, aber im Live-Pull fehlt, wird
|
||||
NICHT geloescht, sondern extra_config.stale=true gesetzt (Doku-SSoT bewahrt Historie).
|
||||
Ein Objekt, das wieder auftaucht, verliert die Markierung.
|
||||
|
||||
Tokens:
|
||||
CF : CF_API_TOKEN + CF_ACCOUNT_ID aus ~/.secrets/credentials.env (nie im Output)
|
||||
Nautobot: NAUTOBOT_SUPERUSER_API_TOKEN aus docs/nas-vlan20/stacks/210-nautobot.yml (gitignored)
|
||||
|
||||
Aufruf:
|
||||
python3 nautobot/tp44-cf-sync.py # Dry-Run (default) — zeigt Diff
|
||||
python3 nautobot/tp44-cf-sync.py --commit # schreibt
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
NB = os.environ.get("NAUTOBOT_URL", "http://10.40.20.30:8080")
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
YML = REPO / "docs" / "nas-vlan20" / "stacks" / "210-nautobot.yml"
|
||||
COMMIT = "--commit" in sys.argv
|
||||
|
||||
# ------------------------------------------------------------ Cloudflare-Seite
|
||||
def cf_secrets():
|
||||
env = {}
|
||||
with open(os.path.expanduser("~/.secrets/credentials.env")) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env[k.strip()] = v.strip().strip("'\"")
|
||||
return env
|
||||
|
||||
|
||||
_CF = cf_secrets()
|
||||
CF_TOK, CF_ACCT = _CF.get("CF_API_TOKEN"), _CF.get("CF_ACCOUNT_ID")
|
||||
if not CF_TOK or not CF_ACCT:
|
||||
sys.exit("CF_API_TOKEN/CF_ACCOUNT_ID fehlen in ~/.secrets/credentials.env")
|
||||
|
||||
|
||||
def cf_get(path):
|
||||
"""(ok, result) — faengt 403/404 ab, damit fehlende Token-Scopes den Sync nicht kippen."""
|
||||
req = urllib.request.Request("https://api.cloudflare.com/client/v4" + path,
|
||||
headers={"Authorization": "Bearer " + CF_TOK})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return True, json.load(r).get("result")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (403, 404):
|
||||
return False, None
|
||||
raise
|
||||
|
||||
|
||||
def cf_ingress(tunnel_id):
|
||||
ok, res = cf_get(f"/accounts/{CF_ACCT}/cfd_tunnel/{tunnel_id}/configurations")
|
||||
rules = ((res or {}).get("config") or {}).get("ingress") or [] if ok else []
|
||||
return [r["hostname"] for r in rules if r.get("hostname")]
|
||||
|
||||
|
||||
def load_cf_estate():
|
||||
"""Baut (networks, services) aus dem Live-Pull. Jedes Objekt:
|
||||
(name, resource_type, description, extra_config, match_key, match_val).
|
||||
match_key/-val = stabile CF-ID → idempotentes Matching trotz kuratierter Namen.
|
||||
Live-Zaehl-/Statusfelder liegen unter eigenen Keys (connector_count, ingress_hostnames),
|
||||
damit der Merge kuratierte Strings (z.B. connectors='3x @ ap1/ap2/ap3') nicht ueberschreibt."""
|
||||
networks, services = [], []
|
||||
skipped = []
|
||||
|
||||
# --- CloudNetworks: cloudflared-Tunnel (+ Connector-Zahl, Ingress-Hostnames)
|
||||
ok, tuns = cf_get(f"/accounts/{CF_ACCT}/cfd_tunnel?is_deleted=false&per_page=50")
|
||||
for t in (tuns or []):
|
||||
networks.append((t["name"], "Cloudflare cloudflared Tunnel",
|
||||
f"cloudflared-Tunnel ({len(t.get('connections') or [])} Connectoren)",
|
||||
{"tunnel_id": t["id"], "kind": "cloudflared",
|
||||
"connector_count": len(t.get("connections") or []),
|
||||
"ingress_hostnames": cf_ingress(t["id"])},
|
||||
"tunnel_id", t["id"]))
|
||||
# --- CloudNetworks: WARP-Connector
|
||||
ok, warps = cf_get(f"/accounts/{CF_ACCT}/warp_connector?per_page=50")
|
||||
for w in (warps or []):
|
||||
networks.append((w["name"], "Cloudflare WARP Connector",
|
||||
f"WARP-Connector ({len(w.get('connections') or [])} Connectoren)",
|
||||
{"tunnel_id": w["id"], "kind": "warp_connector",
|
||||
"connector_count": len(w.get("connections") or [])},
|
||||
"tunnel_id", w["id"]))
|
||||
|
||||
# --- CloudServices: Workers
|
||||
ok, scripts = cf_get(f"/accounts/{CF_ACCT}/workers/scripts")
|
||||
for s in (scripts or []):
|
||||
services.append((s["id"], "Cloudflare Worker", "Cloudflare Worker",
|
||||
{"script": s["id"]}, "script", s["id"]))
|
||||
|
||||
# --- CloudServices: Access-Apps
|
||||
ok, apps = cf_get(f"/accounts/{CF_ACCT}/access/apps?per_page=100")
|
||||
if not ok:
|
||||
skipped.append("access/apps")
|
||||
for a in (apps or []):
|
||||
services.append((a.get("name") or a.get("id"), "Cloudflare Access Application",
|
||||
f"Access-App ({a.get('type')})",
|
||||
{"id": a.get("id"), "type": a.get("type"), "domain": a.get("domain")},
|
||||
"id", a.get("id")))
|
||||
|
||||
# --- CloudServices: Hyperdrive (braucht Scope)
|
||||
ok, hyper = cf_get(f"/accounts/{CF_ACCT}/hyperdrive/configs")
|
||||
if not ok:
|
||||
skipped.append("hyperdrive/configs")
|
||||
for h in (hyper or []):
|
||||
origin = h.get("origin") or {}
|
||||
services.append((h.get("name") or h.get("id"), "Cloudflare Hyperdrive",
|
||||
"Hyperdrive-Config",
|
||||
{"id": h.get("id"), "database": origin.get("database"),
|
||||
"user": origin.get("user")}, "id", h.get("id")))
|
||||
|
||||
# --- CloudServices: VPC-Service (braucht Scope)
|
||||
ok, vpc = cf_get(f"/accounts/{CF_ACCT}/connectivity/directory/services")
|
||||
if not ok:
|
||||
skipped.append("connectivity/directory/services")
|
||||
for v in (vpc or []):
|
||||
services.append((v.get("name") or v.get("id"), "Cloudflare VPC Service",
|
||||
"VPC-Service",
|
||||
{"service_id": v.get("id"), "type": v.get("type")},
|
||||
"service_id", v.get("id")))
|
||||
|
||||
return networks, services, skipped
|
||||
|
||||
|
||||
# ------------------------------------------------------------ Nautobot-Seite
|
||||
def nb_token():
|
||||
for line in YML.read_text().splitlines():
|
||||
m = re.search(r"NAUTOBOT_SUPERUSER_API_TOKEN[=:]\s*([^\s\"']+)", line)
|
||||
if m:
|
||||
return m.group(1)
|
||||
sys.exit(f"NAUTOBOT_SUPERUSER_API_TOKEN nicht in {YML} gefunden")
|
||||
|
||||
|
||||
NB_HDRS = {"Authorization": f"Token {nb_token()}", "Accept": "application/json",
|
||||
"Content-Type": "application/json"}
|
||||
|
||||
|
||||
def nb(method, path, body=None):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
r = urllib.request.Request(f"{NB}{path}", data=data, headers=NB_HDRS, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(r, timeout=20) as resp:
|
||||
return resp.status, json.loads(resp.read().decode() or "{}")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read().decode() or "{}")
|
||||
|
||||
|
||||
def nb_all(endpoint, params):
|
||||
"""Alle Rows eines Endpoints (paginiert)."""
|
||||
out, url = [], f"/api/{endpoint}/?{urllib.parse.urlencode(dict(params, depth=0, limit=200))}"
|
||||
while url:
|
||||
st, d = nb("GET", url)
|
||||
out += d.get("results", [])
|
||||
nxt = d.get("next")
|
||||
url = nxt.replace(NB, "") if nxt else None
|
||||
return out
|
||||
|
||||
|
||||
_INDEX = {} # endpoint -> [rows] (einmal je Account geladen)
|
||||
|
||||
|
||||
def index(endpoint, acct_id):
|
||||
if endpoint not in _INDEX:
|
||||
_INDEX[endpoint] = nb_all(endpoint, {"cloud_account": acct_id})
|
||||
return _INDEX[endpoint]
|
||||
|
||||
|
||||
def upsert(endpoint, name, payload, label, seen, acct_id=None,
|
||||
match_key=None, match_val=None):
|
||||
"""get-or-create + Merge von extra_config.
|
||||
Matching: bei (match_key,match_val) nach der stabilen CF-ID in extra_config
|
||||
(Rename-fest), sonst nach Name. Trackt die getroffene id in seen[endpoint]."""
|
||||
ex = None
|
||||
if match_val is not None:
|
||||
for row in index(endpoint, acct_id):
|
||||
if (row.get("extra_config") or {}).get(match_key) == match_val:
|
||||
ex = row
|
||||
break
|
||||
else:
|
||||
for row in (index(endpoint, acct_id) if acct_id else
|
||||
nb_all(endpoint, {"name": name})):
|
||||
if row.get("name") == name:
|
||||
ex = row
|
||||
break
|
||||
if ex:
|
||||
seen.setdefault(endpoint, set()).add(ex["id"])
|
||||
merged = dict(ex.get("extra_config") or {})
|
||||
merged.update(payload.get("extra_config") or {})
|
||||
if merged == (ex.get("extra_config") or {}):
|
||||
print(f" = {label}: '{name}' unveraendert")
|
||||
return ex
|
||||
if not COMMIT:
|
||||
print(f" ~ {label}: '{name}' extra_config-Update [DRY-RUN]")
|
||||
return ex
|
||||
st, d = nb("PATCH", f"/api/{endpoint}/{ex['id']}/", {"extra_config": merged})
|
||||
print(f" ~ {label}: '{name}' aktualisiert" if st == 200
|
||||
else f" ! {label}: '{name}' PATCH {st}: {json.dumps(d)[:200]}")
|
||||
return d if st == 200 else ex
|
||||
if not COMMIT:
|
||||
print(f" + {label}: '{name}' [DRY-RUN]")
|
||||
return {"id": f"<dry:{name}>"}
|
||||
st, d = nb("POST", f"/api/{endpoint}/", payload)
|
||||
if st in (200, 201):
|
||||
seen.setdefault(endpoint, set()).add(d["id"])
|
||||
print(f" + {label}: '{name}' angelegt")
|
||||
return d
|
||||
print(f" ! {label}: '{name}' FEHLER {st}: {json.dumps(d)[:200]}")
|
||||
return None
|
||||
|
||||
|
||||
def mark_stale(endpoint, acct_id, seen_ids, label):
|
||||
"""CF-Objekte in Nautobot (per id), die im Live-Pull fehlen → extra_config.stale=true.
|
||||
Ein wieder aufgetauchtes Objekt verliert die Markierung."""
|
||||
for row in index(endpoint, acct_id):
|
||||
extra = row.get("extra_config") or {}
|
||||
nm = row.get("name")
|
||||
if row["id"] in seen_ids:
|
||||
if extra.get("stale") and COMMIT:
|
||||
nb("PATCH", f"/api/{endpoint}/{row['id']}/",
|
||||
{"extra_config": {k: v for k, v in extra.items() if k != "stale"}})
|
||||
print(f" ↩ {label}: '{nm}' stale-Markierung entfernt (wieder da)")
|
||||
continue
|
||||
if extra.get("stale"):
|
||||
continue
|
||||
if not COMMIT:
|
||||
print(f" ⚠ {label}: '{nm}' fehlt im Live-Pull → wuerde stale markiert [DRY-RUN]")
|
||||
continue
|
||||
nb("PATCH", f"/api/{endpoint}/{row['id']}/", {"extra_config": dict(extra, stale=True)})
|
||||
print(f" ⚠ {label}: '{nm}' als stale markiert (nicht mehr in CF)")
|
||||
|
||||
|
||||
RESOURCE_TYPES = [
|
||||
("Cloudflare cloudflared Tunnel", "cloud.cloudnetwork"),
|
||||
("Cloudflare WARP Connector", "cloud.cloudnetwork"),
|
||||
("Cloudflare VPC Service", "cloud.cloudservice"),
|
||||
("Cloudflare Hyperdrive", "cloud.cloudservice"),
|
||||
("Cloudflare Worker", "cloud.cloudservice"),
|
||||
("Cloudflare Access Application", "cloud.cloudservice"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
print(f"== TP44 CF-Sync ({'COMMIT' if COMMIT else 'DRY-RUN'}) CF→{NB} ==")
|
||||
networks, services, skipped = load_cf_estate()
|
||||
print(f"Live-Pull: {len(networks)} CloudNetworks · {len(services)} CloudServices"
|
||||
+ (f" · SKIP (Token-Scope): {', '.join(skipped)}" if skipped else " · alle Endpoints lesbar"))
|
||||
|
||||
print("\n[1] Manufacturer + CloudAccount")
|
||||
mfr = upsert("dcim/manufacturers", "Cloudflare", {"name": "Cloudflare"}, "Manufacturer", {})
|
||||
mfr_id = mfr["id"]
|
||||
acct = upsert("cloud/cloud-accounts", "Cloudflare",
|
||||
{"name": "Cloudflare", "account_number": CF_ACCT, "provider": mfr_id},
|
||||
"CloudAccount", {})
|
||||
acct_id = acct["id"]
|
||||
|
||||
print("\n[2] CloudResourceTypes")
|
||||
rt_ids = {}
|
||||
for name, ct in RESOURCE_TYPES:
|
||||
rt = upsert("cloud/cloud-resource-types", name,
|
||||
{"name": name, "provider": mfr_id, "content_types": [ct]}, "ResourceType", {})
|
||||
rt_ids[name] = rt["id"]
|
||||
|
||||
seen = {}
|
||||
|
||||
print("\n[3] CloudNetworks")
|
||||
for name, rt_name, desc, extra, mk, mv in networks:
|
||||
upsert("cloud/cloud-networks", name,
|
||||
{"name": name, "cloud_resource_type": rt_ids.get(rt_name), "cloud_account": acct_id,
|
||||
"description": desc, "extra_config": extra}, "CloudNetwork", seen,
|
||||
acct_id=acct_id, match_key=mk, match_val=mv)
|
||||
|
||||
print("\n[4] CloudServices")
|
||||
for name, rt_name, desc, extra, mk, mv in services:
|
||||
upsert("cloud/cloud-services", name,
|
||||
{"name": name, "cloud_resource_type": rt_ids.get(rt_name), "cloud_account": acct_id,
|
||||
"description": desc, "extra_config": extra}, "CloudService", seen,
|
||||
acct_id=acct_id, match_key=mk, match_val=mv)
|
||||
|
||||
print("\n[5] Verwaisung (stale-Markierung; nur Endpoints, die gelesen wurden)")
|
||||
# Nur markieren, wenn der zugehoerige Endpoint erreichbar war — sonst wuerde ein
|
||||
# fehlender Token-Scope echte Objekte faelschlich als verwaist markieren.
|
||||
if "access/apps" not in skipped and "hyperdrive/configs" not in skipped \
|
||||
and "connectivity/directory/services" not in skipped:
|
||||
mark_stale("cloud/cloud-services", acct_id, seen.get("cloud/cloud-services", set()), "CloudService")
|
||||
else:
|
||||
print(" · CloudService-Verwaisung uebersprungen (Token-Scope unvollstaendig — kein sicherer Vergleich)")
|
||||
mark_stale("cloud/cloud-networks", acct_id, seen.get("cloud/cloud-networks", set()), "CloudNetwork")
|
||||
|
||||
print(f"\n== fertig ({'COMMIT' if COMMIT else 'DRY-RUN — nichts geschrieben'}) ==")
|
||||
if skipped:
|
||||
print(f"Hinweis: {', '.join(skipped)} nicht lesbar → Token-Scope erweitern, dann syncen sie mit.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user