feat: initial implementation of NSX Security Group IP Sync tool
- Added main script (main.py) for syncing IPs between NSX managers. - Implemented logging setup and YAML configuration loading. - Created NSXClient for interacting with NSX Policy API. - Developed SyncEngine for comparing and syncing IPs. - Added interactive group selection feature. - Implemented data export functionality for JSON and CSV formats. - Created requirements.txt for dependencies. - Added __init__.py for package versioning. - Included error handling and validation for configuration.
This commit is contained in:
brandon committed
commit
5b76457ad8
9 files changed
+1579
No files matched your search
@@ -0,0 +1,3 @@
|
||||
"""NSX Security Group IP Sync package."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,272 @@
|
||||
"""
|
||||
Export source/target group IP data to JSON or CSV for auditing and testing.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .nsx_client import NSXClient
|
||||
from .sync_engine import GroupDiff
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
return datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
|
||||
def _ensure_dir(path: str) -> None:
|
||||
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Data collection helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def collect_group_data(
|
||||
client: NSXClient,
|
||||
group_names: List[str],
|
||||
domain: str = "default",
|
||||
include_effective_ips: bool = True,
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Build a list of dicts, one per group, with metadata and IP lists.
|
||||
Effective IP fetching is optional (slow for large environments).
|
||||
"""
|
||||
records = []
|
||||
for name in group_names:
|
||||
record: Dict = {
|
||||
"group_name": name,
|
||||
"domain": domain,
|
||||
"nsx_manager": client.label,
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"static_ips": [],
|
||||
"effective_ips": [],
|
||||
"group_found": True,
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
group = client.get_group(name, domain)
|
||||
record["group_id"] = group.get("id", "")
|
||||
record["display_name"] = group.get("display_name", "")
|
||||
record["description"] = group.get("description", "")
|
||||
|
||||
for expr in group.get("expression", []):
|
||||
if expr.get("resource_type") == "IPAddressExpression":
|
||||
record["static_ips"].extend(expr.get("ip_addresses", []))
|
||||
record["static_ips"] = sorted(set(record["static_ips"]))
|
||||
|
||||
if include_effective_ips:
|
||||
record["effective_ips"] = client.get_group_effective_ips(name, domain)
|
||||
|
||||
except Exception as exc:
|
||||
msg = str(exc)
|
||||
if "not found" in msg.lower():
|
||||
record["group_found"] = False
|
||||
record["error"] = msg
|
||||
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
|
||||
def collect_diff_data(diffs: List[GroupDiff]) -> List[Dict]:
|
||||
"""Serialize a list of GroupDiff objects to plain dicts for export."""
|
||||
return [
|
||||
{
|
||||
"group_name": d.group_name,
|
||||
"source_ip_count": len(d.source_ips),
|
||||
"target_static_ip_count": len(d.target_static_ips),
|
||||
"ips_to_add_count": len(d.ips_to_add),
|
||||
"ips_to_remove_count": len(d.ips_to_remove),
|
||||
"source_found": d.source_found,
|
||||
"target_found": d.target_found,
|
||||
"source_ips": d.source_ips,
|
||||
"target_static_ips": d.target_static_ips,
|
||||
"ips_to_add": d.ips_to_add,
|
||||
"ips_to_remove": d.ips_to_remove,
|
||||
"error": d.error,
|
||||
}
|
||||
for d in diffs
|
||||
]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# JSON export
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def export_json(
|
||||
data: List[Dict],
|
||||
output_path: Optional[str] = None,
|
||||
label: str = "export",
|
||||
) -> str:
|
||||
if not output_path:
|
||||
output_path = os.path.join("exports", f"{label}_{_timestamp()}.json")
|
||||
_ensure_dir(output_path)
|
||||
with open(output_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, indent=2, default=str)
|
||||
logger.info("JSON export written to: %s", output_path)
|
||||
return output_path
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CSV export
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _flatten_for_csv(record: Dict) -> List[Dict]:
|
||||
"""
|
||||
Expand a group record so each IP occupies its own row.
|
||||
If there are no IPs we still emit one row.
|
||||
"""
|
||||
base = {
|
||||
"group_name": record.get("group_name", ""),
|
||||
"group_id": record.get("group_id", ""),
|
||||
"display_name": record.get("display_name", ""),
|
||||
"nsx_manager": record.get("nsx_manager", ""),
|
||||
"domain": record.get("domain", ""),
|
||||
"timestamp": record.get("timestamp", ""),
|
||||
"group_found": record.get("group_found", ""),
|
||||
"error": record.get("error", ""),
|
||||
}
|
||||
all_ips = sorted(
|
||||
set(record.get("effective_ips", []) + record.get("static_ips", []))
|
||||
)
|
||||
if not all_ips:
|
||||
return [{**base, "ip_address": "", "ip_type": ""}]
|
||||
|
||||
rows = []
|
||||
static_set = set(record.get("static_ips", []))
|
||||
effective_set = set(record.get("effective_ips", []))
|
||||
for ip in all_ips:
|
||||
types = []
|
||||
if ip in static_set:
|
||||
types.append("static")
|
||||
if ip in effective_set:
|
||||
types.append("effective")
|
||||
rows.append({**base, "ip_address": ip, "ip_type": "/".join(types)})
|
||||
return rows
|
||||
|
||||
|
||||
def _flatten_diff_for_csv(record: Dict) -> List[Dict]:
|
||||
"""Expand a diff record so each IP occupies its own row."""
|
||||
base = {
|
||||
"group_name": record.get("group_name", ""),
|
||||
"source_found": record.get("source_found", ""),
|
||||
"target_found": record.get("target_found", ""),
|
||||
"error": record.get("error", ""),
|
||||
}
|
||||
all_ips = sorted(
|
||||
set(
|
||||
record.get("source_ips", [])
|
||||
+ record.get("target_static_ips", [])
|
||||
+ record.get("ips_to_add", [])
|
||||
+ record.get("ips_to_remove", [])
|
||||
)
|
||||
)
|
||||
if not all_ips:
|
||||
return [{**base, "ip_address": "", "action": "no_change"}]
|
||||
|
||||
src = set(record.get("source_ips", []))
|
||||
tgt = set(record.get("target_static_ips", []))
|
||||
add = set(record.get("ips_to_add", []))
|
||||
rem = set(record.get("ips_to_remove", []))
|
||||
|
||||
rows = []
|
||||
for ip in all_ips:
|
||||
if ip in add:
|
||||
action = "add"
|
||||
elif ip in rem:
|
||||
action = "remove"
|
||||
else:
|
||||
action = "in_sync"
|
||||
rows.append(
|
||||
{
|
||||
**base,
|
||||
"ip_address": ip,
|
||||
"in_source": ip in src,
|
||||
"in_target": ip in tgt,
|
||||
"action": action,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def export_csv(
|
||||
data: List[Dict],
|
||||
output_path: Optional[str] = None,
|
||||
label: str = "export",
|
||||
is_diff: bool = False,
|
||||
) -> str:
|
||||
if not output_path:
|
||||
output_path = os.path.join("exports", f"{label}_{_timestamp()}.csv")
|
||||
_ensure_dir(output_path)
|
||||
|
||||
rows: List[Dict] = []
|
||||
for record in data:
|
||||
if is_diff:
|
||||
rows.extend(_flatten_diff_for_csv(record))
|
||||
else:
|
||||
rows.extend(_flatten_for_csv(record))
|
||||
|
||||
if not rows:
|
||||
logger.warning("No data to write to CSV.")
|
||||
return output_path
|
||||
|
||||
with open(output_path, "w", newline="", encoding="utf-8") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
logger.info("CSV export written to: %s", output_path)
|
||||
return output_path
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Convenience: export both source and target snapshots + diff
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def export_full_report(
|
||||
source_client: NSXClient,
|
||||
target_client: NSXClient,
|
||||
group_names: List[str],
|
||||
diffs: List[GroupDiff],
|
||||
output_dir: str = "exports",
|
||||
fmt: str = "both",
|
||||
source_domain: str = "default",
|
||||
target_domain: str = "default",
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
Write source snapshot, target snapshot, and diff report.
|
||||
fmt: 'json' | 'csv' | 'both'
|
||||
Returns a dict of {report_name: file_path}.
|
||||
"""
|
||||
ts = _timestamp()
|
||||
written: Dict[str, str] = {}
|
||||
|
||||
src_data = collect_group_data(source_client, group_names, source_domain)
|
||||
tgt_data = collect_group_data(target_client, group_names, target_domain)
|
||||
diff_data = collect_diff_data(diffs)
|
||||
|
||||
def _path(name: str, ext: str) -> str:
|
||||
return os.path.join(output_dir, f"{name}_{ts}.{ext}")
|
||||
|
||||
if fmt in ("json", "both"):
|
||||
written["source_json"] = export_json(src_data, _path("source", "json"), "source")
|
||||
written["target_json"] = export_json(tgt_data, _path("target", "json"), "target")
|
||||
written["diff_json"] = export_json(diff_data, _path("diff", "json"), "diff")
|
||||
|
||||
if fmt in ("csv", "both"):
|
||||
written["source_csv"] = export_csv(src_data, _path("source", "csv"), "source")
|
||||
written["target_csv"] = export_csv(tgt_data, _path("target", "csv"), "target")
|
||||
written["diff_csv"] = export_csv(
|
||||
diff_data, _path("diff", "csv"), "diff", is_diff=True
|
||||
)
|
||||
|
||||
return written
|
||||
@@ -0,0 +1,172 @@
|
||||
"""
|
||||
Interactive mode: list all Security Groups from the source manager and
|
||||
allow the user to select which ones to sync via a terminal UI.
|
||||
|
||||
This is a nice-to-have feature and only runs when --interactive is passed.
|
||||
Requires no additional dependencies beyond the standard library.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
from .nsx_client import NSXClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PAGE = 20 # groups per screen page
|
||||
|
||||
|
||||
def _print_page(groups: List[dict], page: int, selected: set) -> None:
|
||||
start = page * _PAGE
|
||||
end = min(start + _PAGE, len(groups))
|
||||
total_pages = (len(groups) + _PAGE - 1) // _PAGE
|
||||
|
||||
print(f"\n Security Groups (page {page + 1}/{total_pages}) "
|
||||
f"– {len(selected)} selected")
|
||||
print(f" {'#':<5} {'*':<3} {'Display Name':<50} {'ID'}")
|
||||
print(f" {'-'*5} {'-'*3} {'-'*50} {'-'*36}")
|
||||
|
||||
for i, g in enumerate(groups[start:end], start=start):
|
||||
marker = "*" if g["id"] in selected else " "
|
||||
name = g.get("display_name", g["id"])[:50]
|
||||
print(f" {i:<5} {marker:<3} {name:<50} {g['id']}")
|
||||
|
||||
|
||||
def _prompt(msg: str, default: str = "") -> str:
|
||||
try:
|
||||
val = input(msg).strip()
|
||||
return val if val else default
|
||||
except (KeyboardInterrupt, EOFError):
|
||||
return "q"
|
||||
|
||||
|
||||
def select_groups(
|
||||
source: NSXClient,
|
||||
domain: str = "default",
|
||||
preselected: Optional[List[str]] = None,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Interactive terminal selector. Returns a list of group names (display_name
|
||||
if unique, otherwise id) the user wants to sync.
|
||||
|
||||
Navigation:
|
||||
<number(s)> – toggle selection by row index (comma-separated)
|
||||
a – select all on current page
|
||||
n – next page
|
||||
p – previous page
|
||||
s – show currently selected list
|
||||
d – done / confirm
|
||||
q – quit without selecting
|
||||
"""
|
||||
print(f"\nFetching Security Groups from [{source.label}]...")
|
||||
try:
|
||||
all_groups = source.list_groups(domain)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to list groups: %s", exc)
|
||||
print(f"\nError: {exc}")
|
||||
return []
|
||||
|
||||
if not all_groups:
|
||||
print("No Security Groups found.")
|
||||
return []
|
||||
|
||||
all_groups.sort(key=lambda g: g.get("display_name", g["id"]).lower())
|
||||
|
||||
selected: set = set()
|
||||
if preselected:
|
||||
for g in all_groups:
|
||||
if g.get("display_name") in preselected or g["id"] in preselected:
|
||||
selected.add(g["id"])
|
||||
|
||||
page = 0
|
||||
total_pages = (len(all_groups) + _PAGE - 1) // _PAGE
|
||||
|
||||
print(
|
||||
"\nControls: enter row number(s) to toggle | 'a' all on page | "
|
||||
"'n' next | 'p' prev | 's' show selected | 'd' done | 'q' quit"
|
||||
)
|
||||
|
||||
while True:
|
||||
_print_page(all_groups, page, selected)
|
||||
cmd = _prompt("\n > ").lower()
|
||||
|
||||
if cmd == "q":
|
||||
print("Cancelled.")
|
||||
return []
|
||||
|
||||
if cmd == "d":
|
||||
break
|
||||
|
||||
if cmd == "n":
|
||||
if page < total_pages - 1:
|
||||
page += 1
|
||||
else:
|
||||
print(" Already on last page.")
|
||||
continue
|
||||
|
||||
if cmd == "p":
|
||||
if page > 0:
|
||||
page -= 1
|
||||
else:
|
||||
print(" Already on first page.")
|
||||
continue
|
||||
|
||||
if cmd == "a":
|
||||
start = page * _PAGE
|
||||
end = min(start + _PAGE, len(all_groups))
|
||||
for g in all_groups[start:end]:
|
||||
selected.add(g["id"])
|
||||
continue
|
||||
|
||||
if cmd == "s":
|
||||
if not selected:
|
||||
print(" Nothing selected yet.")
|
||||
else:
|
||||
print(f"\n Selected ({len(selected)}):")
|
||||
for gid in sorted(selected):
|
||||
for g in all_groups:
|
||||
if g["id"] == gid:
|
||||
print(f" - {g.get('display_name', gid)}")
|
||||
break
|
||||
continue
|
||||
|
||||
# Try to parse as comma-separated indices
|
||||
tokens = [t.strip() for t in cmd.replace(" ", ",").split(",") if t.strip()]
|
||||
valid = True
|
||||
indices = []
|
||||
for tok in tokens:
|
||||
if tok.isdigit():
|
||||
idx = int(tok)
|
||||
if 0 <= idx < len(all_groups):
|
||||
indices.append(idx)
|
||||
else:
|
||||
print(f" Index {idx} is out of range.")
|
||||
valid = False
|
||||
else:
|
||||
print(f" Unknown command or index: '{tok}'")
|
||||
valid = False
|
||||
|
||||
if valid and indices:
|
||||
for idx in indices:
|
||||
gid = all_groups[idx]["id"]
|
||||
if gid in selected:
|
||||
selected.discard(gid)
|
||||
print(
|
||||
f" Deselected: {all_groups[idx].get('display_name', gid)}"
|
||||
)
|
||||
else:
|
||||
selected.add(gid)
|
||||
print(
|
||||
f" Selected: {all_groups[idx].get('display_name', gid)}"
|
||||
)
|
||||
|
||||
if not selected:
|
||||
print("No groups selected – nothing to sync.")
|
||||
return []
|
||||
|
||||
# Return display_names if they're unique, otherwise IDs
|
||||
display_names = [g.get("display_name", g["id"]) for g in all_groups if g["id"] in selected]
|
||||
print(f"\nConfirmed {len(display_names)} group(s) for sync:")
|
||||
for name in display_names:
|
||||
print(f" - {name}")
|
||||
return display_names
|
||||
@@ -0,0 +1,300 @@
|
||||
"""
|
||||
NSX Policy API client supporting NSX 3.x, 4.x, and 9.x managers.
|
||||
Uses the Policy API (preferred over the deprecated Manager API).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_POLICY_BASE = "/policy/api/v1"
|
||||
_PAGE_SIZE = 1000
|
||||
_MAX_RETRIES = 3
|
||||
_RETRY_BACKOFF = 2 # seconds
|
||||
|
||||
|
||||
class NSXAuthError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NSXAPIError(Exception):
|
||||
def __init__(self, message: str, status_code: int = 0, body: str = ""):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.body = body
|
||||
|
||||
|
||||
class NSXClient:
|
||||
"""
|
||||
Thin wrapper around the NSX Policy REST API.
|
||||
|
||||
Supports basic auth (username + password) and handles:
|
||||
- Cursor-based pagination
|
||||
- Automatic retries on transient errors
|
||||
- Optional SSL verification bypass (for self-signed certs)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
username: str,
|
||||
password: str,
|
||||
verify_ssl: bool = True,
|
||||
timeout: int = 30,
|
||||
label: str = "",
|
||||
):
|
||||
self.base_url = f"https://{host.rstrip('/')}{_POLICY_BASE}"
|
||||
self.timeout = timeout
|
||||
self.label = label or host
|
||||
|
||||
if not verify_ssl:
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
self._session = requests.Session()
|
||||
self._session.auth = (username, password)
|
||||
self._session.verify = verify_ssl
|
||||
self._session.headers.update(
|
||||
{"Content-Type": "application/json", "Accept": "application/json"}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
return f"{self.base_url}/{path.lstrip('/')}"
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
**kwargs,
|
||||
) -> requests.Response:
|
||||
url = self._url(path)
|
||||
last_exc: Optional[Exception] = None
|
||||
|
||||
for attempt in range(1, _MAX_RETRIES + 1):
|
||||
try:
|
||||
resp = self._session.request(
|
||||
method, url, timeout=self.timeout, **kwargs
|
||||
)
|
||||
if resp.status_code == 401:
|
||||
raise NSXAuthError(
|
||||
f"[{self.label}] Authentication failed – check credentials."
|
||||
)
|
||||
if resp.status_code == 403:
|
||||
raise NSXAuthError(
|
||||
f"[{self.label}] Access denied to {path} – check permissions."
|
||||
)
|
||||
if resp.status_code >= 500 and attempt < _MAX_RETRIES:
|
||||
logger.warning(
|
||||
"[%s] HTTP %s on %s, retry %d/%d",
|
||||
self.label, resp.status_code, path, attempt, _MAX_RETRIES,
|
||||
)
|
||||
time.sleep(_RETRY_BACKOFF * attempt)
|
||||
continue
|
||||
if not resp.ok:
|
||||
raise NSXAPIError(
|
||||
f"[{self.label}] HTTP {resp.status_code} on {method} {path}",
|
||||
status_code=resp.status_code,
|
||||
body=resp.text,
|
||||
)
|
||||
return resp
|
||||
except (requests.ConnectionError, requests.Timeout) as exc:
|
||||
last_exc = exc
|
||||
if attempt < _MAX_RETRIES:
|
||||
logger.warning(
|
||||
"[%s] Connection error on %s, retry %d/%d: %s",
|
||||
self.label, path, attempt, _MAX_RETRIES, exc,
|
||||
)
|
||||
time.sleep(_RETRY_BACKOFF * attempt)
|
||||
|
||||
raise NSXAPIError(
|
||||
f"[{self.label}] Failed to reach {path} after {_MAX_RETRIES} attempts: {last_exc}"
|
||||
)
|
||||
|
||||
def _get_paged(self, path: str, params: Optional[Dict] = None) -> List[Dict]:
|
||||
"""Collect all pages for a list endpoint using cursor-based pagination."""
|
||||
params = dict(params or {})
|
||||
params["page_size"] = _PAGE_SIZE
|
||||
results: List[Dict] = []
|
||||
|
||||
while True:
|
||||
resp = self._request("GET", path, params=params)
|
||||
data = resp.json()
|
||||
results.extend(data.get("results", []))
|
||||
|
||||
cursor = data.get("cursor")
|
||||
if not cursor:
|
||||
break
|
||||
params["cursor"] = cursor
|
||||
|
||||
return results
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Version / connectivity check
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_version(self) -> str:
|
||||
"""Return the NSX manager version string."""
|
||||
resp = self._request("GET", "/infra/sites")
|
||||
# Version is available via a different endpoint
|
||||
try:
|
||||
resp2 = self._session.get(
|
||||
f"https://{self.base_url.split('//')[1].split('/')[0]}/api/v1/node/version",
|
||||
timeout=self.timeout,
|
||||
)
|
||||
if resp2.ok:
|
||||
return resp2.json().get("product_version", "unknown")
|
||||
except Exception:
|
||||
pass
|
||||
return "unknown"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Groups
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def list_groups(self, domain: str = "default") -> List[Dict]:
|
||||
"""Return all security groups in the given policy domain."""
|
||||
logger.debug("[%s] Listing groups in domain '%s'", self.label, domain)
|
||||
return self._get_paged(f"/infra/domains/{domain}/groups")
|
||||
|
||||
def get_group(self, group_id: str, domain: str = "default") -> Dict:
|
||||
"""Return a single group definition by ID or display_name."""
|
||||
groups = self.list_groups(domain)
|
||||
# Try exact ID match first, then display_name
|
||||
for g in groups:
|
||||
if g.get("id") == group_id or g.get("display_name") == group_id:
|
||||
return g
|
||||
raise NSXAPIError(
|
||||
f"[{self.label}] Group '{group_id}' not found in domain '{domain}'"
|
||||
)
|
||||
|
||||
def get_group_effective_ips(
|
||||
self, group_id: str, domain: str = "default"
|
||||
) -> List[str]:
|
||||
"""
|
||||
Return the list of effective (computed) IP addresses that are members
|
||||
of the group. This resolves VM-based, tag-based, etc. criteria.
|
||||
"""
|
||||
logger.debug(
|
||||
"[%s] Fetching effective IPs for group '%s'", self.label, group_id
|
||||
)
|
||||
path = f"/infra/domains/{domain}/groups/{group_id}/members/ip-addresses"
|
||||
raw = self._get_paged(path)
|
||||
# Each item is a plain string (the IP / CIDR)
|
||||
ips = [item if isinstance(item, str) else item.get("ip_address", "") for item in raw]
|
||||
return sorted(ip for ip in ips if ip)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# IP Address Expression helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _extract_ip_expressions(
|
||||
expressions: List[Dict],
|
||||
) -> Tuple[List[Dict], List[str]]:
|
||||
"""
|
||||
Split expressions into (non-ip-entries, existing-static-ips).
|
||||
Removes IPAddressExpression items and their preceding ConjunctionOperators.
|
||||
Returns a clean expression list ready for re-injection and the existing IPs.
|
||||
"""
|
||||
existing_ips: List[str] = []
|
||||
clean: List[Dict] = []
|
||||
|
||||
skip_next_conjunction = False
|
||||
for expr in expressions:
|
||||
rtype = expr.get("resource_type", "")
|
||||
if rtype == "IPAddressExpression":
|
||||
existing_ips.extend(expr.get("ip_addresses", []))
|
||||
# Remove the conjunction that precedes this entry (if we added it)
|
||||
if clean and clean[-1].get("resource_type") == "ConjunctionOperator":
|
||||
clean.pop()
|
||||
continue
|
||||
clean.append(expr)
|
||||
|
||||
_ = skip_next_conjunction # unused, kept for clarity
|
||||
return clean, existing_ips
|
||||
|
||||
def patch_group_ips(
|
||||
self,
|
||||
group_id: str,
|
||||
ips_to_add: List[str],
|
||||
ips_to_remove: List[str],
|
||||
domain: str = "default",
|
||||
whatif: bool = False,
|
||||
) -> Dict:
|
||||
"""
|
||||
Modify the static IP address entries in a target group:
|
||||
- Adds ips_to_add as static entries
|
||||
- Removes ips_to_remove from static entries
|
||||
- Leaves all non-IP expressions (Conditions, Tags, etc.) untouched
|
||||
|
||||
Returns a result dict describing what was (or would have been) done.
|
||||
"""
|
||||
group = self.get_group(group_id, domain)
|
||||
gid = group["id"]
|
||||
display = group.get("display_name", gid)
|
||||
expressions = list(group.get("expression", []))
|
||||
|
||||
base_exprs, current_static_ips = self._extract_ip_expressions(expressions)
|
||||
current_set = set(current_static_ips)
|
||||
desired_set = (current_set | set(ips_to_add)) - set(ips_to_remove)
|
||||
|
||||
actually_added = sorted(set(ips_to_add) - current_set)
|
||||
actually_removed = sorted(current_set & set(ips_to_remove))
|
||||
|
||||
result = {
|
||||
"group_id": gid,
|
||||
"display_name": display,
|
||||
"ips_added": actually_added,
|
||||
"ips_removed": actually_removed,
|
||||
"whatif": whatif,
|
||||
}
|
||||
|
||||
if not actually_added and not actually_removed:
|
||||
logger.info("[%s] Group '%s': no changes needed.", self.label, display)
|
||||
result["status"] = "no_change"
|
||||
return result
|
||||
|
||||
if whatif:
|
||||
logger.info(
|
||||
"[%s] [WHATIF] Group '%s': would add %d, remove %d IPs.",
|
||||
self.label, display, len(actually_added), len(actually_removed),
|
||||
)
|
||||
result["status"] = "whatif"
|
||||
return result
|
||||
|
||||
# Build the new expression list
|
||||
new_exprs = list(base_exprs)
|
||||
if desired_set:
|
||||
if new_exprs:
|
||||
new_exprs.append(
|
||||
{"resource_type": "ConjunctionOperator", "conjunction_operator": "OR"}
|
||||
)
|
||||
new_exprs.append(
|
||||
{
|
||||
"resource_type": "IPAddressExpression",
|
||||
"ip_addresses": sorted(desired_set),
|
||||
}
|
||||
)
|
||||
|
||||
patch_body = dict(group)
|
||||
patch_body["expression"] = new_exprs
|
||||
|
||||
self._request(
|
||||
"PATCH",
|
||||
f"/infra/domains/{domain}/groups/{gid}",
|
||||
json=patch_body,
|
||||
)
|
||||
logger.info(
|
||||
"[%s] Group '%s': added %d, removed %d IPs.",
|
||||
self.label, display, len(actually_added), len(actually_removed),
|
||||
)
|
||||
result["status"] = "updated"
|
||||
return result
|
||||
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Core sync logic: compare source effective IPs to target static IPs and
|
||||
produce a change plan, then optionally apply it.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .nsx_client import NSXClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GroupDiff:
|
||||
group_name: str
|
||||
source_ips: List[str] = field(default_factory=list)
|
||||
target_static_ips: List[str] = field(default_factory=list)
|
||||
ips_to_add: List[str] = field(default_factory=list)
|
||||
ips_to_remove: List[str] = field(default_factory=list)
|
||||
source_found: bool = True
|
||||
target_found: bool = True
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
group_name: str
|
||||
status: str # updated | no_change | whatif | error | skipped
|
||||
ips_added: List[str] = field(default_factory=list)
|
||||
ips_removed: List[str] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class SyncEngine:
|
||||
"""
|
||||
Orchestrates a full source→target sync for a list of Security Group names.
|
||||
|
||||
Strategy
|
||||
--------
|
||||
1. Fetch effective IPs from each group on the source manager.
|
||||
2. Fetch the current static IP entries from matching groups on the target.
|
||||
3. Diff: source effective IPs are the desired state for static entries on target.
|
||||
4. Apply: PATCH target groups to add/remove IPs (skipped in whatif mode).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
source: NSXClient,
|
||||
target: NSXClient,
|
||||
source_domain: str = "default",
|
||||
target_domain: str = "default",
|
||||
max_workers: int = 4,
|
||||
):
|
||||
self.source = source
|
||||
self.target = target
|
||||
self.source_domain = source_domain
|
||||
self.target_domain = target_domain
|
||||
self.max_workers = max_workers
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Phase 1: gather data concurrently
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _fetch_source_ips(self, group_name: str) -> tuple:
|
||||
"""Return (group_name, ips_list, error_or_None)."""
|
||||
try:
|
||||
ips = self.source.get_group_effective_ips(group_name, self.source_domain)
|
||||
return group_name, ips, None
|
||||
except Exception as exc:
|
||||
return group_name, [], str(exc)
|
||||
|
||||
def _fetch_target_static_ips(self, group_name: str) -> tuple:
|
||||
"""Return (group_name, static_ips_list, found_bool, error_or_None)."""
|
||||
try:
|
||||
group = self.target.get_group(group_name, self.target_domain)
|
||||
expressions = group.get("expression", [])
|
||||
ips: List[str] = []
|
||||
for expr in expressions:
|
||||
if expr.get("resource_type") == "IPAddressExpression":
|
||||
ips.extend(expr.get("ip_addresses", []))
|
||||
return group_name, sorted(ips), True, None
|
||||
except Exception as exc:
|
||||
msg = str(exc)
|
||||
if "not found" in msg.lower():
|
||||
return group_name, [], False, None
|
||||
return group_name, [], True, msg
|
||||
|
||||
def build_diff(self, group_names: List[str]) -> List[GroupDiff]:
|
||||
"""
|
||||
Concurrently fetch source and target data, then compute per-group diffs.
|
||||
"""
|
||||
source_map: Dict[str, tuple] = {}
|
||||
target_map: Dict[str, tuple] = {}
|
||||
|
||||
logger.info(
|
||||
"Fetching data from source [%s] and target [%s] for %d group(s)...",
|
||||
self.source.label, self.target.label, len(group_names),
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.max_workers) as pool:
|
||||
src_futures = {
|
||||
pool.submit(self._fetch_source_ips, g): g for g in group_names
|
||||
}
|
||||
tgt_futures = {
|
||||
pool.submit(self._fetch_target_static_ips, g): g for g in group_names
|
||||
}
|
||||
for fut in as_completed(src_futures):
|
||||
name, ips, err = fut.result()
|
||||
source_map[name] = (ips, err)
|
||||
for fut in as_completed(tgt_futures):
|
||||
name, ips, found, err = fut.result()
|
||||
target_map[name] = (ips, found, err)
|
||||
|
||||
diffs: List[GroupDiff] = []
|
||||
for name in group_names:
|
||||
src_ips, src_err = source_map.get(name, ([], "not fetched"))
|
||||
tgt_ips, tgt_found, tgt_err = target_map.get(name, ([], True, "not fetched"))
|
||||
|
||||
diff = GroupDiff(group_name=name)
|
||||
|
||||
if src_err:
|
||||
diff.error = f"Source error: {src_err}"
|
||||
diff.source_found = False
|
||||
diffs.append(diff)
|
||||
continue
|
||||
|
||||
if tgt_err:
|
||||
diff.error = f"Target error: {tgt_err}"
|
||||
diffs.append(diff)
|
||||
continue
|
||||
|
||||
if not tgt_found:
|
||||
diff.target_found = False
|
||||
diff.error = f"Group '{name}' not found on target – skipping."
|
||||
diffs.append(diff)
|
||||
continue
|
||||
|
||||
diff.source_ips = src_ips
|
||||
diff.target_static_ips = tgt_ips
|
||||
|
||||
src_set = set(src_ips)
|
||||
tgt_set = set(tgt_ips)
|
||||
diff.ips_to_add = sorted(src_set - tgt_set)
|
||||
diff.ips_to_remove = sorted(tgt_set - src_set)
|
||||
|
||||
diffs.append(diff)
|
||||
|
||||
return diffs
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Phase 2: apply changes
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _apply_diff(self, diff: GroupDiff, whatif: bool) -> SyncResult:
|
||||
if diff.error:
|
||||
return SyncResult(
|
||||
group_name=diff.group_name,
|
||||
status="skipped",
|
||||
error=diff.error,
|
||||
)
|
||||
if not diff.ips_to_add and not diff.ips_to_remove:
|
||||
logger.info("Group '%s': already in sync.", diff.group_name)
|
||||
return SyncResult(group_name=diff.group_name, status="no_change")
|
||||
|
||||
try:
|
||||
patch_result = self.target.patch_group_ips(
|
||||
group_id=diff.group_name,
|
||||
ips_to_add=diff.ips_to_add,
|
||||
ips_to_remove=diff.ips_to_remove,
|
||||
domain=self.target_domain,
|
||||
whatif=whatif,
|
||||
)
|
||||
return SyncResult(
|
||||
group_name=diff.group_name,
|
||||
status=patch_result["status"],
|
||||
ips_added=patch_result.get("ips_added", []),
|
||||
ips_removed=patch_result.get("ips_removed", []),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Group '%s': error applying changes: %s", diff.group_name, exc)
|
||||
return SyncResult(
|
||||
group_name=diff.group_name,
|
||||
status="error",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
def sync(
|
||||
self, group_names: List[str], whatif: bool = False
|
||||
) -> tuple[List[GroupDiff], List[SyncResult]]:
|
||||
"""
|
||||
Full pipeline: build diffs, then apply (or simulate) changes.
|
||||
|
||||
Returns (diffs, results) so callers can inspect both.
|
||||
"""
|
||||
diffs = self.build_diff(group_names)
|
||||
|
||||
if whatif:
|
||||
logger.info("--- WHATIF MODE: no changes will be written ---")
|
||||
|
||||
results: List[SyncResult] = []
|
||||
with ThreadPoolExecutor(max_workers=self.max_workers) as pool:
|
||||
futures = {
|
||||
pool.submit(self._apply_diff, diff, whatif): diff.group_name
|
||||
for diff in diffs
|
||||
}
|
||||
for fut in as_completed(futures):
|
||||
results.append(fut.result())
|
||||
|
||||
# Sort results to match original group order
|
||||
order = {name: i for i, name in enumerate(group_names)}
|
||||
results.sort(key=lambda r: order.get(r.group_name, 9999))
|
||||
|
||||
return diffs, results
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Summary printer
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def print_summary(
|
||||
diffs: List[GroupDiff],
|
||||
results: List[SyncResult],
|
||||
whatif: bool = False,
|
||||
) -> None:
|
||||
label = "[WHATIF] " if whatif else ""
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {label}Sync Summary")
|
||||
print(f"{'='*60}")
|
||||
|
||||
result_map = {r.group_name: r for r in results}
|
||||
|
||||
for diff in diffs:
|
||||
r = result_map.get(diff.group_name)
|
||||
print(f"\n Group: {diff.group_name}")
|
||||
if diff.error:
|
||||
print(f" STATUS : SKIPPED – {diff.error}")
|
||||
continue
|
||||
|
||||
print(f" Source IPs : {len(diff.source_ips)}")
|
||||
print(f" Target IPs : {len(diff.target_static_ips)}")
|
||||
print(f" To Add : {len(diff.ips_to_add)}")
|
||||
print(f" To Remove : {len(diff.ips_to_remove)}")
|
||||
|
||||
if r:
|
||||
status_label = r.status.upper()
|
||||
if r.status == "no_change":
|
||||
status_label = "IN SYNC"
|
||||
print(f" STATUS : {status_label}")
|
||||
if r.error:
|
||||
print(f" ERROR : {r.error}")
|
||||
if r.ips_added:
|
||||
for ip in r.ips_added:
|
||||
print(f" + {ip}")
|
||||
if r.ips_removed:
|
||||
for ip in r.ips_removed:
|
||||
print(f" - {ip}")
|
||||
|
||||
print(f"\n{'='*60}\n")
|
||||
Reference in new issue
Block a user