Skip to content

Commit

Permalink
Allow specifying unmanaged_config_sections in mapping.yml file which
Browse files Browse the repository at this point in the history
makes the config hash ignore those parts of the config
This can be used together with get_config_section jinja filter to have
sections of the configuration handled by an external tool
  • Loading branch information
indy-independence committed Oct 30, 2024
1 parent 6e3d9ff commit 7c969fe
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 26 deletions.
11 changes: 11 additions & 0 deletions src/cnaas_nms/db/device.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import re
from typing import List, Optional, Set

from nornir.core.inventory import Group as NornirGroup
from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, String, Unicode, UniqueConstraint, event
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy_utils import IPAddressType
Expand Down Expand Up @@ -499,6 +500,16 @@ def validate(cls, new_entry=True, **kwargs):

return data, errors

@classmethod
def nornir_groups_to_devicetype(cls, groups: List[NornirGroup]) -> DeviceType:
"""Parse list of groups from nornir (task.host.groups) and return DeviceType"""
devtype: DeviceType = DeviceType.UNKNOWN
# Get the first group that starts with T_ and use that name to determine DeviceType
# Eg group name T_DIST -> DeviceType.DIST
devtype_name = next(filter(lambda x: x.name.startswith("T_"), groups)).name[2:]
devtype = DeviceType[devtype_name]
return devtype


@event.listens_for(Device, "after_update")
def after_update_device(mapper, connection, target: Device):
Expand Down
17 changes: 16 additions & 1 deletion src/cnaas_nms/db/git.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from cnaas_nms.app_settings import app_settings
from cnaas_nms.db.device import Device, DeviceType
from cnaas_nms.db.exceptions import ConfigException, RepoStructureException
from cnaas_nms.db.git_worktrees import WorktreeError, refresh_existing_templates_worktrees
from cnaas_nms.db.git_worktrees import WorktreeError, find_templates_worktree_path, refresh_existing_templates_worktrees
from cnaas_nms.db.job import Job, JobStatus
from cnaas_nms.db.joblock import Joblock, JoblockError
from cnaas_nms.db.session import redis_session, sqla_session
Expand All @@ -22,6 +22,7 @@
VlanConflictError,
get_device_primary_groups,
get_group_settings_asdict,
get_group_templates_branch,
get_groups,
rebuild_settings_cache,
)
Expand Down Expand Up @@ -409,3 +410,17 @@ def parse_repo_url(url: str) -> Tuple[str, Optional[str]]:
"""Parses a URL to a repository, returning the path and branch refspec separately"""
path, branch = urldefrag(url)
return path, branch if branch else None


def get_template_repo_path(hostname: str):
local_repo_path = app_settings.TEMPLATES_LOCAL

# override template path if primary group template path is set
primary_group = get_device_primary_groups().get(hostname)
if primary_group:
templates_branch = get_group_templates_branch(primary_group)
if templates_branch:
primary_group_template_path = find_templates_worktree_path(templates_branch)
if primary_group_template_path:
local_repo_path = primary_group_template_path
return local_repo_path
7 changes: 4 additions & 3 deletions src/cnaas_nms/db/git_worktrees.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ def refresh_existing_templates_worktrees(job_id: int, group_settings: dict, devi
"""Look for existing worktrees and refresh them"""
logger = get_logger()
updated_groups: Set[str] = set()
commit_by: str = ""
if os.path.isdir("/tmp/worktrees"):
for subdir in os.listdir("/tmp/worktrees"):
try:
Expand All @@ -31,9 +32,9 @@ def refresh_existing_templates_worktrees(job_id: int, group_settings: dict, devi
if not diff:
continue

ret: str
changed_files: Set[str]
ret, changed_files = parse_git_changed_files(diff, prev_commit, wt_repo)
commit_by_new, changed_files = parse_git_changed_files(diff, prev_commit, wt_repo)
commit_by += commit_by_new
# don't update updated_groups if changes were only in other branches
if not changed_files:
continue
Expand All @@ -52,7 +53,7 @@ def refresh_existing_templates_worktrees(job_id: int, group_settings: dict, devi
dev: Device = session.query(Device).filter_by(hostname=hostname).one_or_none()
if dev:
dev.synchronized = False
add_sync_event(hostname, "refresh_templates", ret, job_id)
add_sync_event(hostname, "refresh_templates", commit_by, job_id)
updated_hostnames.add(hostname)
if updated_hostnames:
logger.debug(
Expand Down
29 changes: 28 additions & 1 deletion src/cnaas_nms/devicehandler/get.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import hashlib
import os
import re
from typing import Dict, List, Optional

import yaml
from netutils.config import compliance
from netutils.lib_mapper import NAPALM_LIB_MAPPER
from nornir.core.filter import F
Expand All @@ -12,8 +14,11 @@
import cnaas_nms.devicehandler.nornir_helper
from cnaas_nms.db.device import Device, DeviceType
from cnaas_nms.db.device_vars import expand_interface_settings
from cnaas_nms.db.exceptions import RepoStructureException
from cnaas_nms.db.git import get_template_repo_path
from cnaas_nms.db.interface import Interface, InterfaceConfigType, InterfaceError
from cnaas_nms.db.session import sqla_session
from cnaas_nms.tools.jinja_filters import get_config_section
from cnaas_nms.tools.log import get_logger


Expand Down Expand Up @@ -53,7 +58,29 @@ def get_running_config_interface(session: sqla_session, hostname: str, interface
return "\n".join(ret)


def calc_config_hash(hostname, config):
def get_unmanaged_config_sections(hostname: str, platform: str, devtype: DeviceType) -> List[str]:
local_repo_path = get_template_repo_path(hostname)

mapfile = os.path.join(local_repo_path, platform, "mapping.yml")
if not os.path.isfile(mapfile):
raise RepoStructureException("File {} not found in template repo".format(mapfile))
with open(mapfile, "r") as f:
mapping = yaml.safe_load(f)
if (
"unmanaged_config_sections" in mapping[devtype.name]
and type(mapping[devtype.name]["unmanaged_config_sections"]) is list
):
return mapping[devtype.name]["unmanaged_config_sections"]
return []


def calc_config_hash(hostname: str, config: str, platform: str, devtype: DeviceType):
ignore_config_sections: List[str] = get_unmanaged_config_sections(hostname, platform, devtype)
for section in ignore_config_sections:
skip_section = get_config_section(config, section, platform)
if skip_section:
config = config.replace(skip_section, "")
config = config.replace("\n", "")
try:
hash_object = hashlib.sha256(config.encode())
except Exception:
Expand Down
42 changes: 21 additions & 21 deletions src/cnaas_nms/devicehandler/sync_devices.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import datetime
import os
import time
from hashlib import sha256
from ipaddress import IPv4Address, IPv4Interface, ip_interface
from typing import List, Optional, Tuple

Expand All @@ -15,16 +14,15 @@
from nornir_utils.plugins.functions import print_result

import cnaas_nms.db.helper
from cnaas_nms.app_settings import api_settings, app_settings
from cnaas_nms.app_settings import api_settings
from cnaas_nms.db.device import Device, DeviceState, DeviceType
from cnaas_nms.db.device_vars import expand_interface_settings
from cnaas_nms.db.git import RepoStructureException
from cnaas_nms.db.git_worktrees import find_templates_worktree_path
from cnaas_nms.db.git import RepoStructureException, get_template_repo_path
from cnaas_nms.db.interface import Interface
from cnaas_nms.db.job import Job
from cnaas_nms.db.joblock import Joblock, JoblockError
from cnaas_nms.db.session import redis_session, sqla_session
from cnaas_nms.db.settings import get_device_primary_groups, get_group_templates_branch, get_settings
from cnaas_nms.db.settings import get_settings
from cnaas_nms.devicehandler.changescore import calculate_score
from cnaas_nms.devicehandler.get import calc_config_hash
from cnaas_nms.devicehandler.nornir_helper import NornirJobResult, cnaas_init, get_jinja_env, inventory_selector
Expand Down Expand Up @@ -532,16 +530,7 @@ def push_sync_device(
platform = dev.platform
devtype = dev.device_type

local_repo_path = app_settings.TEMPLATES_LOCAL

# override template path if primary group template path is set
primary_group = get_device_primary_groups().get(hostname)
if primary_group:
templates_branch = get_group_templates_branch(primary_group)
if templates_branch:
primary_group_template_path = find_templates_worktree_path(templates_branch)
if primary_group_template_path:
local_repo_path = primary_group_template_path
local_repo_path = get_template_repo_path(hostname)

mapfile = os.path.join(local_repo_path, platform, "mapping.yml")
if not os.path.isfile(mapfile):
Expand Down Expand Up @@ -674,6 +663,7 @@ def sync_check_hash(task, force=False, job_id=None):
task: Nornir task
force: Ignore device hash
"""
logger = get_logger()
set_thread_data(job_id)
if force is True:
return
Expand All @@ -686,11 +676,12 @@ def sync_check_hash(task, force=False, job_id=None):
res = task.run(task=napalm_get, getters=["config"])
task.host.close_connection("napalm")

running_config = dict(res.result)["config"]["running"].encode()
if running_config is None:
raise Exception("Failed to get running configuration")
hash_obj = sha256(running_config)
running_hash = hash_obj.hexdigest()
try:
devtype = Device.nornir_groups_to_devicetype(task.host.groups)
except Exception as e:
logger.error("Unable to determine device type")
logger.exception(e)
running_hash = calc_config_hash(task.host.name, dict(res.result)["config"]["running"], task.host.platform, devtype)
if stored_hash != running_hash:
raise Exception("Device {} configuration is altered outside of CNaaS!".format(task.host.name))

Expand All @@ -706,7 +697,16 @@ def update_config_hash(task):
or "config" not in res[0].result
):
raise Exception("Unable to get config from device")
new_config_hash = calc_config_hash(task.host.name, res[0].result["config"]["running"])

try:
devtype = Device.nornir_groups_to_devicetype(task.host.groups)
except Exception as e:
logger.error("Unable to determine device type")
logger.exception(e)

new_config_hash = calc_config_hash(
task.host.name, res[0].result["config"]["running"], task.host.platform, devtype
)
if not new_config_hash:
raise ValueError("Empty config hash")
except Exception as e:
Expand Down

0 comments on commit 7c969fe

Please sign in to comment.