diff --git a/src/cnaas_nms/db/device.py b/src/cnaas_nms/db/device.py index b9cc7b9e..ac265ac9 100644 --- a/src/cnaas_nms/db/device.py +++ b/src/cnaas_nms/db/device.py @@ -7,6 +7,7 @@ import re from typing import List, Optional, Set +from nornir.core.inventory import Group as NornirGroup from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, String, Unicode, UniqueConstraint, event from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy_utils import IPAddressType @@ -499,6 +500,16 @@ def validate(cls, new_entry=True, **kwargs): return data, errors + @classmethod + def nornir_groups_to_devicetype(cls, groups: List[NornirGroup]) -> DeviceType: + """Parse list of groups from nornir (task.host.groups) and return DeviceType""" + devtype: DeviceType = DeviceType.UNKNOWN + # Get the first group that starts with T_ and use that name to determine DeviceType + # Eg group name T_DIST -> DeviceType.DIST + devtype_name = next(filter(lambda x: x.name.startswith("T_"), groups)).name[2:] + devtype = DeviceType[devtype_name] + return devtype + @event.listens_for(Device, "after_update") def after_update_device(mapper, connection, target: Device): diff --git a/src/cnaas_nms/db/git.py b/src/cnaas_nms/db/git.py index bb0b39be..62f86296 100644 --- a/src/cnaas_nms/db/git.py +++ b/src/cnaas_nms/db/git.py @@ -12,7 +12,7 @@ from cnaas_nms.app_settings import app_settings from cnaas_nms.db.device import Device, DeviceType from cnaas_nms.db.exceptions import ConfigException, RepoStructureException -from cnaas_nms.db.git_worktrees import WorktreeError, refresh_existing_templates_worktrees +from cnaas_nms.db.git_worktrees import WorktreeError, find_templates_worktree_path, refresh_existing_templates_worktrees from cnaas_nms.db.job import Job, JobStatus from cnaas_nms.db.joblock import Joblock, JoblockError from cnaas_nms.db.session import redis_session, sqla_session @@ -22,6 +22,7 @@ VlanConflictError, get_device_primary_groups, get_group_settings_asdict, + get_group_templates_branch, get_groups, rebuild_settings_cache, ) @@ -409,3 +410,17 @@ def parse_repo_url(url: str) -> Tuple[str, Optional[str]]: """Parses a URL to a repository, returning the path and branch refspec separately""" path, branch = urldefrag(url) return path, branch if branch else None + + +def get_template_repo_path(hostname: str): + local_repo_path = app_settings.TEMPLATES_LOCAL + + # override template path if primary group template path is set + primary_group = get_device_primary_groups().get(hostname) + if primary_group: + templates_branch = get_group_templates_branch(primary_group) + if templates_branch: + primary_group_template_path = find_templates_worktree_path(templates_branch) + if primary_group_template_path: + local_repo_path = primary_group_template_path + return local_repo_path diff --git a/src/cnaas_nms/db/git_worktrees.py b/src/cnaas_nms/db/git_worktrees.py index 1e4e036f..0b2a3770 100644 --- a/src/cnaas_nms/db/git_worktrees.py +++ b/src/cnaas_nms/db/git_worktrees.py @@ -21,6 +21,7 @@ def refresh_existing_templates_worktrees(job_id: int, group_settings: dict, devi """Look for existing worktrees and refresh them""" logger = get_logger() updated_groups: Set[str] = set() + commit_by: str = "" if os.path.isdir("/tmp/worktrees"): for subdir in os.listdir("/tmp/worktrees"): try: @@ -31,9 +32,9 @@ def refresh_existing_templates_worktrees(job_id: int, group_settings: dict, devi if not diff: continue - ret: str changed_files: Set[str] - ret, changed_files = parse_git_changed_files(diff, prev_commit, wt_repo) + commit_by_new, changed_files = parse_git_changed_files(diff, prev_commit, wt_repo) + commit_by += commit_by_new # don't update updated_groups if changes were only in other branches if not changed_files: continue @@ -52,7 +53,7 @@ def refresh_existing_templates_worktrees(job_id: int, group_settings: dict, devi dev: Device = session.query(Device).filter_by(hostname=hostname).one_or_none() if dev: dev.synchronized = False - add_sync_event(hostname, "refresh_templates", ret, job_id) + add_sync_event(hostname, "refresh_templates", commit_by, job_id) updated_hostnames.add(hostname) if updated_hostnames: logger.debug( diff --git a/src/cnaas_nms/devicehandler/get.py b/src/cnaas_nms/devicehandler/get.py index b5a0f457..b669bef3 100644 --- a/src/cnaas_nms/devicehandler/get.py +++ b/src/cnaas_nms/devicehandler/get.py @@ -1,7 +1,9 @@ import hashlib +import os import re from typing import Dict, List, Optional +import yaml from netutils.config import compliance from netutils.lib_mapper import NAPALM_LIB_MAPPER from nornir.core.filter import F @@ -12,8 +14,11 @@ import cnaas_nms.devicehandler.nornir_helper from cnaas_nms.db.device import Device, DeviceType from cnaas_nms.db.device_vars import expand_interface_settings +from cnaas_nms.db.exceptions import RepoStructureException +from cnaas_nms.db.git import get_template_repo_path from cnaas_nms.db.interface import Interface, InterfaceConfigType, InterfaceError from cnaas_nms.db.session import sqla_session +from cnaas_nms.tools.jinja_filters import get_config_section from cnaas_nms.tools.log import get_logger @@ -53,7 +58,29 @@ def get_running_config_interface(session: sqla_session, hostname: str, interface return "\n".join(ret) -def calc_config_hash(hostname, config): +def get_unmanaged_config_sections(hostname: str, platform: str, devtype: DeviceType) -> List[str]: + local_repo_path = get_template_repo_path(hostname) + + mapfile = os.path.join(local_repo_path, platform, "mapping.yml") + if not os.path.isfile(mapfile): + raise RepoStructureException("File {} not found in template repo".format(mapfile)) + with open(mapfile, "r") as f: + mapping = yaml.safe_load(f) + if ( + "unmanaged_config_sections" in mapping[devtype.name] + and type(mapping[devtype.name]["unmanaged_config_sections"]) is list + ): + return mapping[devtype.name]["unmanaged_config_sections"] + return [] + + +def calc_config_hash(hostname: str, config: str, platform: str, devtype: DeviceType): + ignore_config_sections: List[str] = get_unmanaged_config_sections(hostname, platform, devtype) + for section in ignore_config_sections: + skip_section = get_config_section(config, section, platform) + if skip_section: + config = config.replace(skip_section, "") + config = config.replace("\n", "") try: hash_object = hashlib.sha256(config.encode()) except Exception: diff --git a/src/cnaas_nms/devicehandler/sync_devices.py b/src/cnaas_nms/devicehandler/sync_devices.py index e3d8e493..2d4083c1 100644 --- a/src/cnaas_nms/devicehandler/sync_devices.py +++ b/src/cnaas_nms/devicehandler/sync_devices.py @@ -1,7 +1,6 @@ import datetime import os import time -from hashlib import sha256 from ipaddress import IPv4Address, IPv4Interface, ip_interface from typing import List, Optional, Tuple @@ -15,16 +14,15 @@ from nornir_utils.plugins.functions import print_result import cnaas_nms.db.helper -from cnaas_nms.app_settings import api_settings, app_settings +from cnaas_nms.app_settings import api_settings from cnaas_nms.db.device import Device, DeviceState, DeviceType from cnaas_nms.db.device_vars import expand_interface_settings -from cnaas_nms.db.git import RepoStructureException -from cnaas_nms.db.git_worktrees import find_templates_worktree_path +from cnaas_nms.db.git import RepoStructureException, get_template_repo_path from cnaas_nms.db.interface import Interface from cnaas_nms.db.job import Job from cnaas_nms.db.joblock import Joblock, JoblockError from cnaas_nms.db.session import redis_session, sqla_session -from cnaas_nms.db.settings import get_device_primary_groups, get_group_templates_branch, get_settings +from cnaas_nms.db.settings import get_settings from cnaas_nms.devicehandler.changescore import calculate_score from cnaas_nms.devicehandler.get import calc_config_hash from cnaas_nms.devicehandler.nornir_helper import NornirJobResult, cnaas_init, get_jinja_env, inventory_selector @@ -532,16 +530,7 @@ def push_sync_device( platform = dev.platform devtype = dev.device_type - local_repo_path = app_settings.TEMPLATES_LOCAL - - # override template path if primary group template path is set - primary_group = get_device_primary_groups().get(hostname) - if primary_group: - templates_branch = get_group_templates_branch(primary_group) - if templates_branch: - primary_group_template_path = find_templates_worktree_path(templates_branch) - if primary_group_template_path: - local_repo_path = primary_group_template_path + local_repo_path = get_template_repo_path(hostname) mapfile = os.path.join(local_repo_path, platform, "mapping.yml") if not os.path.isfile(mapfile): @@ -674,6 +663,7 @@ def sync_check_hash(task, force=False, job_id=None): task: Nornir task force: Ignore device hash """ + logger = get_logger() set_thread_data(job_id) if force is True: return @@ -686,11 +676,12 @@ def sync_check_hash(task, force=False, job_id=None): res = task.run(task=napalm_get, getters=["config"]) task.host.close_connection("napalm") - running_config = dict(res.result)["config"]["running"].encode() - if running_config is None: - raise Exception("Failed to get running configuration") - hash_obj = sha256(running_config) - running_hash = hash_obj.hexdigest() + try: + devtype = Device.nornir_groups_to_devicetype(task.host.groups) + except Exception as e: + logger.error("Unable to determine device type") + logger.exception(e) + running_hash = calc_config_hash(task.host.name, dict(res.result)["config"]["running"], task.host.platform, devtype) if stored_hash != running_hash: raise Exception("Device {} configuration is altered outside of CNaaS!".format(task.host.name)) @@ -706,7 +697,16 @@ def update_config_hash(task): or "config" not in res[0].result ): raise Exception("Unable to get config from device") - new_config_hash = calc_config_hash(task.host.name, res[0].result["config"]["running"]) + + try: + devtype = Device.nornir_groups_to_devicetype(task.host.groups) + except Exception as e: + logger.error("Unable to determine device type") + logger.exception(e) + + new_config_hash = calc_config_hash( + task.host.name, res[0].result["config"]["running"], task.host.platform, devtype + ) if not new_config_hash: raise ValueError("Empty config hash") except Exception as e: