Merge pull request #379 from SUNET/feature.dist_mirror_interface

New ifclass "mirror" on dist devices which copies settings from

docs/reporef/index.rst

@@ -397,7 +397,7 @@ Keys for interfaces.yml or interfaces_<model>.yml:
 * interfaces: List of dicctionaries with keys:
 
   * name: Interface name, like "Ethernet1". Can also be an interface range like "Ethernet[1-4]".
-  * ifclass: Interface class, one of: downlink, fabric, custom, port_template_*
+  * ifclass: Interface class, one of: downlink, fabric, custom, mirror, port_template_*
   * config: Optional. Raw CLI config used in case "custom" ifclass was selected
 
 * Additional interface options for port_template type:
@@ -420,6 +420,21 @@ Keys for interfaces.yml or interfaces_<model>.yml:
   * metric: Optional integer specifying metric for this interface.
   * cli_append_str: Optional. Custom configuration to append to this interface.
 
+* For downlink and fabric type ports these options are available with same function as above:
+
+  * aggregate_id
+  * enabled
+  * cli_append_str
+  * metric
+  * mtu
+  * tags
+
+*  For mirror type ports these options are available with same function as above:
+
+  * description
+  * enabled
+
+
 The "downlink" ifclass is used on DIST devices to specify that this interface
 is used to connect access devices. The "fabric" ifclass is used to specify that
 this interface is used to connect DIST or CORE devices with each other to form
@@ -431,6 +446,13 @@ providing DHCP (relay) access.
 be used to apply some site-specific configuration via Jinja templates. For
 example specify "port_template_hypervisor" and build a corresponding Jinja
 template by matching on that ifclass.
+"mirror" ifclass can be used on DIST devices that are part of the same management
+domain to copy interface settings from one device to the other, this way you
+don't have to maintain the list of allowed vlans on both devices for example.
+On one device you configure ports normally with port_template or custom ifclass,
+and on the other device in the same management domain you can use ifclass mirror
+without any other settings (but you can optionally override description and enabled
+status).
 
 base_system.yml:
 

src/cnaas_nms/db/git.py

@@ -13,8 +13,10 @@
 
 from cnaas_nms.app_settings import app_settings
 from cnaas_nms.db.device import Device, DeviceType
+from cnaas_nms.db.device_vars import expand_interface_settings
 from cnaas_nms.db.exceptions import ConfigException, RepoStructureException
 from cnaas_nms.db.git_worktrees import WorktreeError, find_templates_worktree_path, refresh_existing_templates_worktrees
+from cnaas_nms.db.helper import MgmtdomainNotFoundError, find_mgmtdomain_peer
 from cnaas_nms.db.job import Job, JobStatus
 from cnaas_nms.db.joblock import Joblock, JoblockError
 from cnaas_nms.db.session import redis_session, sqla_session
@@ -26,6 +28,7 @@
     get_group_settings_asdict,
     get_group_templates_branch,
     get_groups,
+    get_settings,
     rebuild_settings_cache,
 )
 from cnaas_nms.devicehandler.sync_history import add_sync_event
@@ -186,6 +189,30 @@ def reset_repo(local_repo: Repo, remote_repo_path: str):
     local_repo.head.reset(index=True, working_tree=True)
 
 
+def get_peer_with_mirror_interfaces(session, dev: Device) -> Optional[Device]:
+    """Returns peer device of management domain if it has any ifclass mirror interfaces"""
+    logger = get_logger()
+    if dev.device_type == DeviceType.ACCESS:
+        return None
+    try:
+        peer_device = find_mgmtdomain_peer(session, dev)
+    except MgmtdomainNotFoundError:
+        return None
+    except Exception:
+        logger.exception("Error while finding peer device for mirrored interfaces")
+        return None
+
+    peer_settings, _ = get_settings(peer_device.hostname, peer_device.device_type, peer_device.model)
+
+    try:
+        for intf in expand_interface_settings(peer_settings["interfaces"]):
+            if intf["ifclass"] == "mirror":
+                return peer_device
+    except KeyError:
+        pass
+    return None
+
+
 def _refresh_repo_task_settings(job_id: Optional[int] = None) -> str:
     logger = get_logger()
     local_repo_path = app_settings.SETTINGS_LOCAL
@@ -233,6 +260,10 @@ def _refresh_repo_task_settings(job_id: Optional[int] = None) -> str:
             if dev:
                 dev.synchronized = False
                 add_sync_event(hostname, "refresh_settings", ret, job_id)
+                peer_device = get_peer_with_mirror_interfaces(session, dev)
+                if peer_device:
+                    peer_device.synchronized = False
+                    add_sync_event(peer_device.hostname, "refresh_settings", ret, job_id)
             else:
                 logger.warn("Settings updated for unknown device: {}".format(hostname))
 
@@ -395,8 +426,13 @@ def settings_syncstatus(updated_settings: set) -> Tuple[Set[DeviceType], Set[str
         elif basedir.startswith("devices"):
             try:
                 hostname = filename.split(os.path.sep)[1]
-                if Device.valid_hostname(hostname):
-                    unsynced_hostnames.add(hostname)
+                if not Device.valid_hostname(hostname):
+                    continue
+                unsynced_hostnames.add(hostname)
+                # determine mirror device syncstatus
+                mirror_device: Optional[Device] = None
+                if mirror_device:
+                    unsynced_hostnames.add(mirror_device.hostname)
             except Exception as e:
                 logger.exception("Error in settings devices directory: {}".format(str(e)))
         elif basedir.startswith("groups"):

src/cnaas_nms/db/helper.py

@@ -10,6 +10,10 @@
 from cnaas_nms.db.mgmtdomain import Mgmtdomain
 
 
+class MgmtdomainNotFoundError(Exception):
+    pass
+
+
 def canonical_mac(mac):
     """Return a standardized format of MAC-addresses for CNaaS to
     store in databases etc."""
@@ -18,7 +22,16 @@ def canonical_mac(mac):
     return str(na_mac)
 
 
-def find_mgmtdomain_one_device(session, device0: Device) -> Optional[Mgmtdomain]:
+def find_mgmtdomain_peer(session, device0: Device) -> Device:
+    """Find the peer device in a mgmtdomain for a given device"""
+    mgmtdomain: Mgmtdomain = find_mgmtdomain_one_device(session, device0)
+    if mgmtdomain.device_a == device0:
+        return mgmtdomain.device_b
+    else:
+        return mgmtdomain.device_a
+
+
+def find_mgmtdomain_one_device(session, device0: Device) -> Mgmtdomain:
     if device0.device_type == DeviceType.DIST:
         mgmtdomain = (
             session.query(Mgmtdomain)
@@ -27,18 +40,18 @@ def find_mgmtdomain_one_device(session, device0: Device) -> Optional[Mgmtdomain]
             .one_or_none()
         )
         if not mgmtdomain:
-            raise Exception("No mgmtdomain found for uplink device: {}".format(device0.hostname))
+            raise MgmtdomainNotFoundError("No mgmtdomain found for uplink device: {}".format(device0.hostname))
     elif device0.device_type == DeviceType.ACCESS:
         if device0.management_ip:
             mgmtdomain = find_mgmtdomain_by_ip(session, IPv4Address(device0.management_ip))
         else:
-            raise Exception("No mgmtdomain found for uplink device: {}".format(device0.hostname))
+            raise MgmtdomainNotFoundError("No mgmtdomain found for uplink device: {}".format(device0.hostname))
     else:
         raise Exception("Unexpected uplink device type: {}".format(device0.device_type))
     return mgmtdomain
 
 
-def find_mgmtdomain_two_devices(session, device0: Device, device1: Device) -> Optional[Mgmtdomain]:
+def find_mgmtdomain_two_devices(session, device0: Device, device1: Device) -> Mgmtdomain:
     if device0.device_type != device1.device_type:
         raise ValueError(
             "Both uplink devices must be of same device type: {}, {}".format(device0.hostname, device1.hostname)
@@ -89,7 +102,7 @@ def find_mgmtdomain_two_devices(session, device0: Device, device1: Device) -> Op
     return mgmtdomain
 
 
-def find_mgmtdomain(session, hostnames: List[str]) -> Optional[Mgmtdomain]:
+def find_mgmtdomain(session, hostnames: List[str]) -> Mgmtdomain:
     """Find the corresponding management domain for a pair of
     distribution switches.
 

src/cnaas_nms/db/settings_fields.py

@@ -55,7 +55,7 @@
 ifname_range_schema = Field(
     None, pattern=f"^{IFNAME_RANGE_REGEX}$", description="Interface range pattern or interface name"
 )
-IFCLASS_REGEX = r"(custom|downlink|fabric|port_template_[a-zA-Z0-9_]+)"
+IFCLASS_REGEX = r"(custom|downlink|fabric|mirror|port_template_[a-zA-Z0-9_]+)"
 ifclass_schema = Field(None, pattern=f"^{IFCLASS_REGEX}$", description="Interface class: custom, downlink or uplink")
 ifdescr_schema = Field(None, max_length=64, description="Interface description, 0-64 characters")
 tcpudp_port_schema = Field(None, ge=0, lt=65536, description="TCP or UDP port number, 0-65535")

src/cnaas_nms/db/tests/test_mgmtdomain.py

@@ -71,6 +71,13 @@ def test_find_mgmtdomain_invalid(self):
             self.assertRaises(ValueError, cnaas_nms.db.helper.find_mgmtdomain, session, [])
             self.assertRaises(ValueError, cnaas_nms.db.helper.find_mgmtdomain, session, [1, 2, 3])
 
+    def test_find_mgmtdomain_peer_device(self):
+        with sqla_session() as session:  # type: ignore
+            d_a = session.query(Device).filter(Device.hostname == "mgmtdomaintest1").one()
+            d_b = session.query(Device).filter(Device.hostname == "mgmtdomaintest2").one()
+            found_peer: Device = cnaas_nms.db.helper.find_mgmtdomain_peer(session, d_a)
+            self.assertEqual(d_b, found_peer, "Peer device not found")
+
     def test_find_mgmtdomain_twodist(self):
         with sqla_session() as session:  # type: ignore
             mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain(session, ["eosdist1", "eosdist2"])

src/cnaas_nms/devicehandler/sync_devices.py

@@ -319,6 +319,25 @@ def populate_device_vars(
                                 "peer_asn": None,
                             }
                         )
+                elif intf["ifclass"] == "mirror":
+                    # Copy interface settings from mgmtdomain peer device
+                    if_dict = {"indexnum": ifindexnum}
+                    peer_device = cnaas_nms.db.helper.find_mgmtdomain_peer(session, dev)
+                    peer_settings, _ = get_settings(peer_device.hostname, devtype, peer_device.model)
+                    if "interfaces" in peer_settings and peer_settings["interfaces"]:
+                        for peer_intf in peer_settings["interfaces"]:
+                            if peer_intf["name"] == intf["name"]:
+                                if peer_intf["ifclass"] in ["fabric", "downlink"]:
+                                    raise Exception(f"Cannot mirror {peer_intf['ifclass']} interface")
+                                for copied_key_name, value in peer_intf.items():
+                                    if_dict[copied_key_name] = value
+                                break
+                    # Description and enabled can be set separately from mirrored interface
+                    if "description" in intf:
+                        if_dict["description"] = intf["description"]
+                    if "enabled" in intf:
+                        if_dict["enabled"] = intf["enabled"]
+                    fabric_device_variables["interfaces"].append(if_dict)
                 else:
                     if_dict = {"indexnum": ifindexnum}
                     for extra_key_name, value in intf.items():