TP23 Stufe 4: AGH CIDR-Gruppen+Policy auf .36/.37 + Sync/Exception-Tooling

- A0: API-User 'automation' auf agh-bf1/.36 + agh-bf2/.37 (PW in ~/.secrets, TP30-Merkposten)
- A: 8 clients.persistent (7 VLAN-CIDR-Gruppen + dns-bypass-exempt), Policy-Matrix
  (INFRA/MEDIA/MGMT filtering=aus, GUEST safe-search+parental an), live verifiziert
- B: agh-sync.py (check/push), Repo-SSoT AdGuardHome.yaml mit REDIGIERTEN Hashes
  (Alt-Hash-Exposure im Git entfernt); Invariante .36==.37 (AGH normalisiert beim Restart)
- C: agh-exception.py (allow-domain/unallow/unfilter/refilter/status) gegen beide
  Instanzen, end-to-end getestet; Contract in adguard/README.md
- Stufe 6 (Bypass-Sperre) NICHT geschaltet; kein DHCP-/Firewall-Write

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

adguard/AdGuardHome.yaml

@@ -13,7 +13,9 @@ http:
   session_ttl: 30d
 users:
   - name: admin
-    password: $2a$10$ue8PEedG4t2JRyT83XiGHeqIU4PFRzhn8rjDOZGYGt0sFC2qju2fi
+    password: '!on-device (siehe ~/.secrets + Gerät)'
+  - name: automation
+    password: '!on-device (siehe ~/.secrets + Gerät)'
 auth_attempts: 5
 block_auth_min: 15
 http_proxy: ""
@@ -202,69 +204,9 @@ filtering:
     - domain: gitea.emsenhuber.net
       answer: 10.40.20.40
       enabled: true
-    - domain: nautobot.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
     - domain: ds-bf.emsenhuber.net
       answer: 10.40.20.40
       enabled: true
-    - domain: ccu.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: iobroker.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: z2m25.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: pve-mcp.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: ha-mcp.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: n8n-mcp.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: gitea-mcp.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: pve.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: portainer.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: pbs.emsenhuber.net
-      answer: 10.40.20.40
-      enabled: true
-    - domain: macmini.emsenhuber.net
-      answer: 10.40.20.167
-      enabled: true
-    - domain: mfc.emsenhuber.net
-      answer: 10.40.10.6
-      enabled: true
-    - domain: tv.emsenhuber.net
-      answer: 10.40.40.11
-      enabled: true
-    - domain: docker.emsenhuber.net
-      answer: 10.40.20.126
-      enabled: true
-    - domain: platform-pg.emsenhuber.net
-      answer: 10.40.20.20
-      enabled: true
-    - domain: core.emsenhuber.net
-      answer: 10.40.99.1
-      enabled: true
-    - domain: ap1.emsenhuber.net
-      answer: 10.40.99.11
-      enabled: true
-    - domain: ap2.emsenhuber.net
-      answer: 10.40.99.12
-      enabled: true
-    - domain: branch.emsenhuber.net
-      answer: 10.40.99.2
-      enabled: true
     - domain: nas.bf.emsenhuber.net
       answer: 10.40.20.10
       enabled: true
@@ -316,7 +258,232 @@ clients:
     rdns: true
     dhcp: true
     hosts: true
-  persistent: []
+  persistent:
+    - safe_search:
+        enabled: false
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: dns-bypass-exempt
+      ids:
+        - 10.40.10.82
+        - 10.40.10.183
+      tags: []
+      upstreams: []
+      uid: 381d431a-1a43-41ea-adaa-2d67ada9aab0
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: false
+      filtering_enabled: false
+      parental_enabled: false
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
+    - safe_search:
+        enabled: true
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: grp-guest
+      ids:
+        - 10.40.90.0/24
+      tags: []
+      upstreams: []
+      uid: af70dd86-43a9-4165-8228-26c7b90db5e6
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: false
+      filtering_enabled: true
+      parental_enabled: true
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
+    - safe_search:
+        enabled: false
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: grp-infra
+      ids:
+        - 10.40.20.0/24
+      tags: []
+      upstreams: []
+      uid: 78b2da82-1946-41ee-b386-b04eaee49bc3
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: false
+      filtering_enabled: false
+      parental_enabled: false
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
+    - safe_search:
+        enabled: false
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: grp-iot
+      ids:
+        - 10.40.30.0/24
+      tags: []
+      upstreams: []
+      uid: 7a8ac674-4365-4b06-ad51-2516400903a1
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: true
+      filtering_enabled: true
+      parental_enabled: false
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
+    - safe_search:
+        enabled: false
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: grp-main
+      ids:
+        - 10.40.10.0/24
+      tags: []
+      upstreams: []
+      uid: 39955cc1-6ec2-4cf6-a757-233672b9dc3d
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: true
+      filtering_enabled: true
+      parental_enabled: false
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
+    - safe_search:
+        enabled: false
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: grp-media
+      ids:
+        - 10.40.40.0/24
+      tags: []
+      upstreams: []
+      uid: 44e4a1d7-d1b6-446c-a753-65e714ec93aa
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: false
+      filtering_enabled: false
+      parental_enabled: false
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
+    - safe_search:
+        enabled: false
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: grp-mgmt
+      ids:
+        - 10.40.99.0/24
+      tags: []
+      upstreams: []
+      uid: 4c5eeafb-f7f2-4fb1-ab87-fd2a2366d57a
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: false
+      filtering_enabled: false
+      parental_enabled: false
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
+    - safe_search:
+        enabled: false
+        bing: false
+        duckduckgo: false
+        ecosia: false
+        google: false
+        pixabay: false
+        yandex: false
+        youtube: false
+      blocked_services:
+        schedule:
+          time_zone: Local
+        ids: []
+      name: grp-untrusted
+      ids:
+        - 10.40.50.0/24
+      tags: []
+      upstreams: []
+      uid: 7f182498-7b82-4580-b1b0-0a2d46c205ef
+      upstreams_cache_size: 0
+      upstreams_cache_enabled: false
+      use_global_settings: true
+      filtering_enabled: true
+      parental_enabled: false
+      safebrowsing_enabled: false
+      use_global_blocked_services: true
+      ignore_querylog: false
+      ignore_statistics: false
 log:
   enabled: true
   file: ""

adguard/README.md

@@ -0,0 +1,57 @@
+# AdGuard Home — `agh-bf1` (.36) / `agh-bf2` (.37)  ·  TP23 Stufe 2/4/5
+
+Das AGH-**Paar** (pve LXC 110/111, `10.40.20.36` / `.37`) ist der Client-Resolver des BF-Netzes
+(DHCP liefert beide in allen 7 VLANs). Upstream: DoH zum CF-Gateway; conditional
+`[/bf.emsenhuber.net/] → 10.40.20.1`. **Kein natives HA** → Sync ist Betriebsaufwand.
+
+## Dateien
+| Datei | Zweck |
+|---|---|
+| `AdGuardHome.yaml` | **Repo-SSoT** der Config. **Passwort-Hashes REDIGIERT** (`'<redacted>'`) — keine Secrets im Git. Echte Hashes leben auf den Geräten; Klartext des API-Users in `~/.secrets`. |
+| `agh-sync.py` | `check` (Drift `.36`↔`.37`) · `push` (SSoT auf beide, gestaffelter Restart, reinjiziert Hashes live). |
+| `agh-exception.py` | Exception-Primitive (Stufe 5) gegen **beide** Instanzen. |
+
+## Client-Policy (Stufe 4, `clients.persistent`)
+Ein VLAN = eine CIDR-Gruppe. Alle nutzen denselben CF-Upstream-Filter; Divergenz = per-Client-Toggle.
+
+| Gruppe | CIDR | filtering | safe_search | parental |
+|---|---|---|---|---|
+| grp-main | 10.40.10.0/24 | an (global) | – | – |
+| grp-infra | 10.40.20.0/24 | **aus** | – | – |
+| grp-iot | 10.40.30.0/24 | an | – | – |
+| grp-media | 10.40.40.0/24 | **aus** | – | – |
+| grp-untrusted | 10.40.50.0/24 | an | – | – |
+| grp-guest | 10.40.90.0/24 | an | **on** | **on** |
+| grp-mgmt | 10.40.99.0/24 | **aus** | – | – |
+| dns-bypass-exempt | 10.40.10.183 + .82 | **aus** | – | – |
+
+`filtering aus` bei INFRA/MEDIA/MGMT = bewusste Reduktion (Streaming/Server/Mgmt nicht brechen), per Gruppe rückholbar.
+
+## Sync-Invariante
+AGH **schreibt die YAML beim Restart um** (normalisiert) → **nicht** gegen die Repo-Datei diffen,
+sondern **`.36` == `.37`**. Genau das prüft `agh-sync.py check`.
+
+## Exception-Contract (Stufe 5 — was die HA-Schicht auslöst)
+Auth: Basic, User `automation` (`~/.secrets` → `AGH_AUTOMATION_*`). **Immer auf `.36` UND `.37`.**
+
+🔴 **Wichtig für D:** die beiden AGH-Endpunkte sind **read-modify-write**, kein simples Append —
+`set_rules` **ersetzt die ganze Liste**, `clients/update` braucht das **volle Client-Objekt**.
+Ein zustandsloses HA-`rest_command` reicht deshalb **nicht**; `agh-exception.py` ist die
+Referenz-RMW-Implementierung, die D aufruft (Deploy-Weg auf HAOS in Etappe D zu entscheiden:
+`shell_command`/`command_line` gegen die deployte CLI **oder** ein dünner Wrapper-Endpunkt).
+
+| Aktion | CLI | AGH-API (RMW) |
+|---|---|---|
+| Domain global entsperren | `allow-domain <d>` | `GET /control/filtering/status` → +`@@\|\|<d>^` → `POST /control/filtering/set_rules` |
+| Entsperrung zurück | `unallow-domain <d>` | dito, Regel entfernen |
+| Client entfiltern | `unfilter <ip>` | `GET /control/clients` → Objekt, `filtering_enabled=false` → `POST /control/clients/update` |
+| Client wieder filtern | `refilter <ip>` | dito, zurück auf global |
+
+**Kein Timing in der CLI** — Ablauf/Revert macht der Aufrufer (HA-Timer) und, ab Stufe 6, der
+RouterOS-Backstop.
+
+## RouterOS-Backstop (Stufe 6, noch NICHT gebaut)
+Analog `jit-admin` (`routeros/tp5-etappe4-input-mgmt.rsc`): Address-List `jit-dns-bypass` mit
+`timeout` + Forward-Regel, die dieser Liste 53/853 nach außen erlaubt — der Fail-safe, falls HA
+den Revert verpasst. Gebaut via **Nautobot GC** (kein Direkt-Write), erst wenn die Bypass-Sperre
+scharf geschaltet wird.

adguard/agh-exception.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+agh-exception.py — TP23 Stufe 5 Exception-Primitive gegen das AGH-Paar (.36 + .37).
+
+Von der HA-Self-Service-Schicht (rest_command) ODER von Hand aufrufbar. Wirkt IMMER auf
+BEIDE Instanzen (kein Sync im Datenpfad). KEIN eigenes Timing — Ablauf/Revert macht der
+Aufrufer (HA-Timer) bzw. der RouterOS-Backstop (Stufe 6). Diese CLI ist reines add/remove.
+
+  allow-domain   <domain>   → user_rules += @@||domain^   (Domain für ALLE entsperren)
+  unallow-domain <domain>   → user_rules -= @@||domain^
+  unfilter       <ip>       → Client (dessen ids <ip> enthält) filtering_enabled=false
+  refilter       <ip>       → denselben Client wieder filtern (use_global_settings=true)
+  status                    → user_rules + Client-Policy beider Instanzen
+
+Auth: AGH_AUTOMATION_USER/_PASS aus ~/.secrets/credentials.env (nie in argv).
+"""
+import sys, os, json, base64, urllib.request, urllib.error, pathlib
+
+INST = ["10.40.20.36", "10.40.20.37"]
+SECRETS = pathlib.Path.home() / ".secrets/credentials.env"
+
+
+def creds():
+    u = p = None
+    for line in SECRETS.read_text().splitlines():
+        if line.startswith("AGH_AUTOMATION_USER="): u = line.split("=", 1)[1].strip()
+        if line.startswith("AGH_AUTOMATION_PASS="): p = line.split("=", 1)[1].strip()
+    if not (u and p): sys.exit("AGH_AUTOMATION_USER/_PASS fehlen in ~/.secrets")
+    return u, p
+
+
+AUTH = "Basic " + base64.b64encode(":".join(creds()).encode()).decode()
+
+
+def api(ip, path, method="GET", body=None):
+    url = f"http://{ip}{path}"
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, method=method,
+                                 headers={"Authorization": AUTH,
+                                          "Content-Type": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=8) as r:
+            raw = r.read()
+            return json.loads(raw) if raw else {}
+    except urllib.error.HTTPError as e:
+        sys.exit(f"{ip} {path} → HTTP {e.code}: {e.read().decode()[:200]}")
+    except Exception as e:
+        sys.exit(f"{ip} {path} → {e}")
+
+
+def rule(domain):
+    return f"@@||{domain}^"
+
+
+def allow(domain, remove=False):
+    r = rule(domain)
+    for ip in INST:
+        st = api(ip, "/control/filtering/status")
+        rules = st.get("user_rules", [])
+        if remove:
+            rules = [x for x in rules if x != r]
+        elif r not in rules:
+            rules = rules + [r]
+        api(ip, "/control/filtering/set_rules", "POST", {"rules": rules})
+        print(f"{ip}: {'entfernt' if remove else 'gesetzt'} {r} (user_rules={len(rules)})")
+
+
+def set_filter(ip_addr, enabled):
+    for ip in INST:
+        data = api(ip, "/control/clients")
+        c = None
+        for cand in data.get("clients", []):
+            if ip_addr in cand.get("ids", []):
+                c = cand; break
+        if not c:
+            sys.exit(f"{ip}: kein Client mit id exakt {ip_addr} — für Gruppen die CIDR angeben")
+        c = dict(c)
+        c["use_global_settings"] = enabled
+        c["filtering_enabled"] = enabled
+        api(ip, "/control/clients/update", "POST", {"name": c["name"], "data": c})
+        print(f"{ip}: {c['name']} filtering={'an' if enabled else 'AUS'}")
+
+
+def status():
+    for ip in INST:
+        st = api(ip, "/control/filtering/status")
+        allows = [r for r in st.get("user_rules", []) if r.startswith("@@")]
+        cl = api(ip, "/control/clients").get("clients", [])
+        print(f"== {ip} ==  allow-rules={allows or '—'}")
+        for c in cl:
+            print(f"   {c['name']:16} ids={c['ids']} global={c['use_global_settings']} filt={c['filtering_enabled']}")
+
+
+if __name__ == "__main__":
+    a = sys.argv[1:]
+    if not a: sys.exit(__doc__)
+    cmd = a[0]
+    if cmd == "allow-domain":   allow(a[1])
+    elif cmd == "unallow-domain": allow(a[1], remove=True)
+    elif cmd == "unfilter":     set_filter(a[1], False)
+    elif cmd == "refilter":     set_filter(a[1], True)
+    elif cmd == "status":       status()
+    else: sys.exit(f"unbekannt: {cmd}\n{__doc__}")

adguard/agh-sync.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+agh-sync.py — AdGuard-Home-Paar agh-bf1 (.36) / agh-bf2 (.37) synchron halten.
+
+TP23 Stufe 2/4: die beiden AGH-Instanzen (pve LXC 110/111) haben KEIN natives HA →
+Config-Sync ist Betriebsaufwand. Dieses Werkzeug ist der Sync.
+
+SSoT   = adguard/AdGuardHome.yaml (Passwort-Hashes REDIGIERT — keine Secrets im Git).
+Wahrheit im Datenpfad = die beiden Live-Instanzen; die Invariante ist .36 == .37
+(AGH normalisiert die YAML beim Restart, daher NICHT gegen die Repo-Datei vergleichen,
+sondern die beiden Live-Configs gegeneinander).
+
+  check   pull .36 + .37, Passwörter redigiert diffen → meldet Drift (exit 1 bei Drift)
+  push    Repo-SSoT nehmen, echte users-Hashes aus .36 (live) reinjizieren, validieren,
+          auf beide pushen, GESTAFFELT neu starten (110 → prüfen → 111), .36==.37 + API prüfen
+
+Kein Direkt-`sshpass -p` (Leak). SSH-Key zu root@pve vorausgesetzt.
+"""
+import subprocess, sys, tempfile, base64, os, re, pathlib, urllib.request
+
+PVE = "root@10.40.20.100"
+INST = {"110": "10.40.20.36", "111": "10.40.20.37"}  # ct-id -> ip
+PRIMARY = "110"                                        # Reihenfolge: primary zuerst
+SVC = "AdGuardHome.service"
+CFG = "/opt/AdGuardHome/AdGuardHome.yaml"
+REPO = pathlib.Path(__file__).resolve().parent
+SSOT = REPO / "AdGuardHome.yaml"
+PW_RE = re.compile(r"^(    password: ).*$")
+
+
+def sh(*args, inp=None):
+    return subprocess.run(args, input=inp, capture_output=True, text=True)
+
+
+def pull(ct):
+    r = sh("ssh", "-o", "ConnectTimeout=8", PVE, f"pct exec {ct} -- cat {CFG}")
+    if r.returncode:
+        sys.exit(f"pull {ct} fehlgeschlagen: {r.stderr.strip()}")
+    return r.stdout
+
+
+def redact(text):
+    return "\n".join(PW_RE.sub(r"\1'<redacted>'", l) for l in text.splitlines())
+
+
+def live_users_hashes(ct):
+    """echte password-Zeilen (in Reihenfolge) aus der Live-Config von ct."""
+    out, inusers = [], False
+    for l in pull(ct).splitlines():
+        if l.startswith("users:"): inusers = True; continue
+        if inusers and l and not l.startswith((" ", "\t")): break
+        if inusers:
+            m = PW_RE.match(l)
+            if m: out.append(l)
+    return out
+
+
+def cmd_check():
+    a, b = redact(pull("110")), redact(pull("111"))
+    if a == b:
+        print("✅ .36 == .37 (synchron)")
+        return 0
+    print("🔴 DRIFT zwischen .36 und .37:")
+    import difflib
+    for line in difflib.unified_diff(a.splitlines(), b.splitlines(),
+                                     ".36", ".37", lineterm=""):
+        print(line)
+    return 1
+
+
+def cmd_push():
+    if not SSOT.exists():
+        sys.exit(f"SSoT fehlt: {SSOT}")
+    ssot = SSOT.read_text()
+    # echte Hashes aus dem primary reinjizieren (SSoT trägt nur Redaktions-Platzhalter)
+    hashes = live_users_hashes(PRIMARY)
+    red = [i for i, l in enumerate(ssot.splitlines()) if PW_RE.match(l)]
+    if len(red) != len(hashes):
+        sys.exit(f"users-Anzahl SSoT({len(red)}) != live({len(hashes)}) — manuell prüfen")
+    lines = ssot.splitlines()
+    for idx, real in zip(red, hashes):
+        lines[idx] = real
+    merged = "\n".join(lines) + "\n"
+    # validieren
+    try:
+        import yaml
+        doc = yaml.safe_load(merged)
+        assert doc["users"] and doc["clients"]["persistent"]
+    except Exception as e:
+        sys.exit(f"merge invalide: {e}")
+    b64 = base64.b64encode(merged.encode()).decode()
+    order = [PRIMARY] + [c for c in INST if c != PRIMARY]
+    for ct in order:
+        print(f"→ push + restart {ct} ({INST[ct]}) …")
+        r = sh("ssh", "-o", "ConnectTimeout=10", PVE,
+               f"pct exec {ct} -- sh -c 'base64 -d > {CFG}.new && "
+               f"cp -a {CFG} {CFG}.presync && mv {CFG}.new {CFG} && "
+               f"systemctl restart {SVC} && sleep 2 && systemctl is-active {SVC}'",
+               inp=b64)
+        if r.returncode or "active" not in r.stdout:
+            sys.exit(f"push {ct} fehlgeschlagen: {r.stdout} {r.stderr}")
+        print(f"  {ct} active")
+    print(cmd_check_msg())
+    return cmd_check()
+
+
+def cmd_check_msg():
+    return "— Verifikation:"
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1] if len(sys.argv) > 1 else "check"
+    sys.exit({"check": cmd_check, "push": cmd_push}.get(cmd, cmd_check)())