Merge pull request #369 from SUNET/feature.jinja_get_config_section

Add get_config_block_regex to extract config section from config

docs/configuration/index.rst

@@ -47,6 +47,8 @@ Defines parameters for the API:
   specified in seconds. Defaults to 300.
 - commit_confirmed_wait: Time to wait between comitting configuration and checking
   that the device is still reachable, specified in seconds. Defaults to 1.
+- napalm_timeout: Timeout for NAPALM operations, specified in seconds. Defaults to 60.
+  Increase if you get errors like "jnpr.junos.exception.RpcTimeoutError: RpcTimeoutError" on jobs.
 
 /etc/cnaas-nms/auth_config.yml
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

src/cnaas_nms/app_settings.py

@@ -57,6 +57,7 @@ class ApiSettings(BaseSettings):
     COMMIT_CONFIRMED_TIMEOUT: int = 300
     COMMIT_CONFIRMED_WAIT: int = 1
     SETTINGS_OVERRIDE: Optional[dict] = None
+    NAPALM_TIMEOUT: int = 60
 
     @field_validator("MGMTDOMAIN_PRIMARY_IP_VERSION")
     @classmethod
@@ -118,6 +119,7 @@ def construct_api_settings() -> ApiSettings:
             COMMIT_CONFIRMED_TIMEOUT=config.get("commit_confirmed_timeout", 300),
             COMMIT_CONFIRMED_WAIT=config.get("commit_confirmed_wait", 1),
             SETTINGS_OVERRIDE=config.get("settings_override", None),
+            NAPALM_TIMEOUT=config.get("napalm_timeout", 60),
         )
     else:
         return ApiSettings()

src/cnaas_nms/db/device.py

@@ -7,6 +7,7 @@
 import re
 from typing import List, Optional, Set
 
+from nornir.core.inventory import Group as NornirGroup
 from sqlalchemy import Boolean, DateTime, Enum, ForeignKey, Integer, String, Unicode, UniqueConstraint, event
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy_utils import IPAddressType
@@ -499,6 +500,16 @@ def validate(cls, new_entry=True, **kwargs):
 
         return data, errors
 
+    @classmethod
+    def nornir_groups_to_devicetype(cls, groups: List[NornirGroup]) -> DeviceType:
+        """Parse list of groups from nornir (task.host.groups) and return DeviceType"""
+        devtype: DeviceType = DeviceType.UNKNOWN
+        # Get the first group that starts with T_ and use that name to determine DeviceType
+        # Eg group name T_DIST -> DeviceType.DIST
+        devtype_name = next(filter(lambda x: x.name.startswith("T_"), groups)).name[2:]
+        devtype = DeviceType[devtype_name]
+        return devtype
+
 
 @event.listens_for(Device, "after_update")
 def after_update_device(mapper, connection, target: Device):

src/cnaas_nms/db/git.py

@@ -3,15 +3,16 @@
 import json
 import os
 import shutil
-from typing import Dict, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set, Tuple
 from urllib.parse import urldefrag
 
 import yaml
 
+import git.remote
 from cnaas_nms.app_settings import app_settings
 from cnaas_nms.db.device import Device, DeviceType
 from cnaas_nms.db.exceptions import ConfigException, RepoStructureException
-from cnaas_nms.db.git_worktrees import WorktreeError, clean_templates_worktree
+from cnaas_nms.db.git_worktrees import WorktreeError, find_templates_worktree_path, refresh_existing_templates_worktrees
 from cnaas_nms.db.job import Job, JobStatus
 from cnaas_nms.db.joblock import Joblock, JoblockError
 from cnaas_nms.db.session import redis_session, sqla_session
@@ -20,12 +21,15 @@
     SettingsSyntaxError,
     VlanConflictError,
     get_device_primary_groups,
+    get_group_settings_asdict,
+    get_group_templates_branch,
     get_groups,
     rebuild_settings_cache,
 )
 from cnaas_nms.devicehandler.sync_history import add_sync_event
 from cnaas_nms.scheduler.thread_data import set_thread_data
 from cnaas_nms.tools.event import add_event
+from cnaas_nms.tools.githelpers import parse_git_changed_files
 from cnaas_nms.tools.log import get_logger
 from git import InvalidGitRepositoryError, Repo
 from git.exc import GitCommandError, NoSuchPathError
@@ -223,17 +227,9 @@ def _refresh_repo_task(repo_type: RepoType = RepoType.TEMPLATES, job_id: Optiona
         prev_commit = local_repo.commit().hexsha
         logger.debug("git pull from {}".format(remote_repo_path))
 
-        diff = local_repo.remotes.origin.pull()
-        for item in diff:
-            if item.ref.remote_head != local_repo.head.ref.name:
-                continue
+        diff: List[git.remote.FetchInfo] = local_repo.remotes.origin.pull()
+        ret, changed_files = parse_git_changed_files(diff, prev_commit, local_repo)
 
-            ret += "Commit {} by {} at {}\n".format(
-                item.commit.name_rev, item.commit.committer, item.commit.committed_datetime
-            )
-            diff_files = local_repo.git.diff("{}..{}".format(prev_commit, item.commit.hexsha), name_only=True).split()
-            changed_files.update(diff_files)
-            prev_commit = item.commit.hexsha
     except (InvalidGitRepositoryError, NoSuchPathError):  # noqa: S110
         logger.info("Local repository {} not found, cloning from remote".format(local_repo_path))
         try:
@@ -302,7 +298,7 @@ def _refresh_repo_task(repo_type: RepoType = RepoType.TEMPLATES, job_id: Optiona
             devtype: DeviceType
             for devtype, platform in updated_devtypes:
                 Device.set_devtype_syncstatus(session, devtype, ret, "templates", platform, job_id)
-        clean_templates_worktree()
+        refresh_existing_templates_worktrees(job_id, get_group_settings_asdict(), get_device_primary_groups())
 
     return ret
 
@@ -414,3 +410,17 @@ def parse_repo_url(url: str) -> Tuple[str, Optional[str]]:
     """Parses a URL to a repository, returning the path and branch refspec separately"""
     path, branch = urldefrag(url)
     return path, branch if branch else None
+
+
+def get_template_repo_path(hostname: str):
+    local_repo_path = app_settings.TEMPLATES_LOCAL
+
+    # override template path if primary group template path is set
+    primary_group = get_device_primary_groups().get(hostname)
+    if primary_group:
+        templates_branch = get_group_templates_branch(primary_group)
+        if templates_branch:
+            primary_group_template_path = find_templates_worktree_path(templates_branch)
+            if primary_group_template_path:
+                local_repo_path = primary_group_template_path
+    return local_repo_path

src/cnaas_nms/db/git_worktrees.py

@@ -1,9 +1,14 @@
 import os
 import shutil
-from typing import Optional
+from typing import Optional, Set
 
 import git.exc
 from cnaas_nms.app_settings import app_settings
+from cnaas_nms.db.device import Device
+from cnaas_nms.db.groups import get_groups_using_branch
+from cnaas_nms.db.session import sqla_session
+from cnaas_nms.devicehandler.sync_history import add_sync_event
+from cnaas_nms.tools.githelpers import parse_git_changed_files
 from cnaas_nms.tools.log import get_logger
 from git import Repo
 
@@ -12,10 +17,50 @@ class WorktreeError(Exception):
     pass
 
 
-def clean_templates_worktree():
+def refresh_existing_templates_worktrees(job_id: int, group_settings: dict, device_primary_groups: dict):
+    """Look for existing worktrees and refresh them"""
+    logger = get_logger()
+    updated_groups: Set[str] = set()
+    commit_by: str = ""
     if os.path.isdir("/tmp/worktrees"):
         for subdir in os.listdir("/tmp/worktrees"):
-            shutil.rmtree("/tmp/worktrees/" + subdir, ignore_errors=True)
+            try:
+                logger.info("Pulling worktree for branch {}".format(subdir))
+                wt_repo = Repo("/tmp/worktrees/" + subdir)
+                prev_commit = wt_repo.commit().hexsha
+                diff = wt_repo.remotes.origin.pull()
+                if not diff:
+                    continue
+
+                changed_files: Set[str]
+                commit_by_new, changed_files = parse_git_changed_files(diff, prev_commit, wt_repo)
+                commit_by += commit_by_new
+                # don't update updated_groups if changes were only in other branches
+                if not changed_files:
+                    continue
+            except Exception as e:
+                logger.exception(e)
+                shutil.rmtree("/tmp/worktrees/" + subdir, ignore_errors=True)
+            updated_groups.update(get_groups_using_branch(subdir, group_settings))
+
+    # find all devices that are using these branches and mark them as unsynchronized
+    updated_hostnames: Set[str] = set()
+    with sqla_session() as session:
+        for hostname, primary_group in device_primary_groups.items():
+            if hostname in updated_hostnames:
+                continue
+            if primary_group in updated_groups:
+                dev: Device = session.query(Device).filter_by(hostname=hostname).one_or_none()
+                if dev:
+                    dev.synchronized = False
+                    add_sync_event(hostname, "refresh_templates", commit_by, job_id)
+                    updated_hostnames.add(hostname)
+    if updated_hostnames:
+        logger.debug(
+            "Devices marked as unsynchronized because git worktree branches were refreshed: {}".format(
+                ", ".join(updated_hostnames)
+            )
+        )
 
     local_repo = Repo(app_settings.TEMPLATES_LOCAL)
     local_repo.git.worktree("prune")

src/cnaas_nms/db/groups.py

@@ -0,0 +1,14 @@
+from typing import List
+
+# TODO: move all group related things here from settings
+#  make new settings_helper.py with (verify_dir_structure etc) and separate settings_groups for get_settigns groups?
+#  use get_group_settings_asdict instead of passing dict in get_groups_using_branch below
+
+
+def get_groups_using_branch(branch_name: str, group_settings: dict) -> List[str]:
+    """Returns a list of group names that use the specified branch name"""
+    groups = []
+    for group_name, group_data in group_settings.items():
+        if group_data.get("templates_branch") == branch_name:
+            groups.append(group_name)
+    return groups

src/cnaas_nms/devicehandler/get.py

@@ -1,7 +1,9 @@
 import hashlib
+import os
 import re
 from typing import Dict, List, Optional
 
+import yaml
 from netutils.config import compliance
 from netutils.lib_mapper import NAPALM_LIB_MAPPER
 from nornir.core.filter import F
@@ -12,8 +14,11 @@
 import cnaas_nms.devicehandler.nornir_helper
 from cnaas_nms.db.device import Device, DeviceType
 from cnaas_nms.db.device_vars import expand_interface_settings
+from cnaas_nms.db.exceptions import RepoStructureException
+from cnaas_nms.db.git import get_template_repo_path
 from cnaas_nms.db.interface import Interface, InterfaceConfigType, InterfaceError
 from cnaas_nms.db.session import sqla_session
+from cnaas_nms.tools.jinja_filters import get_config_section
 from cnaas_nms.tools.log import get_logger
 
 
@@ -53,7 +58,32 @@ def get_running_config_interface(session: sqla_session, hostname: str, interface
     return "\n".join(ret)
 
 
-def calc_config_hash(hostname, config):
+def get_unmanaged_config_sections(hostname: str, platform: str, devtype: DeviceType) -> List[str]:
+    local_repo_path = get_template_repo_path(hostname)
+
+    mapfile = os.path.join(local_repo_path, platform, "mapping.yml")
+    if not os.path.isfile(mapfile):
+        raise RepoStructureException("File {} not found in template repo".format(mapfile))
+    with open(mapfile, "r") as f:
+        mapping = yaml.safe_load(f)
+        if (
+            "unmanaged_config_sections" in mapping[devtype.name]
+            and type(mapping[devtype.name]["unmanaged_config_sections"]) is list
+        ):
+            return mapping[devtype.name]["unmanaged_config_sections"]
+    return []
+
+
+def calc_config_hash(hostname: str, config: str, platform: str, devtype: DeviceType):
+    ignore_config_sections: List[str] = get_unmanaged_config_sections(hostname, platform, devtype)
+    for section in ignore_config_sections:
+        skip_section = get_config_section(config, section, platform)
+        if skip_section:
+            config = config.replace(skip_section, "")
+    if platform == "junos":
+        # remove line starting with "## Last commit" from config string so we don't get config hash mismatch
+        config = re.sub(r"^#{2}.*\n", "", config, flags=re.MULTILINE)
+    config = config.replace("\n", "")
     try:
         hash_object = hashlib.sha256(config.encode())
     except Exception:

src/cnaas_nms/devicehandler/nornir_plugins/cnaas_inventory.py

@@ -3,7 +3,7 @@
 from nornir.core.inventory import ConnectionOptions, Defaults, Group, Groups, Host, Hosts, Inventory, ParentGroups
 
 import cnaas_nms.db.session
-from cnaas_nms.app_settings import app_settings
+from cnaas_nms.app_settings import api_settings, app_settings
 from cnaas_nms.db.device import Device, DeviceState, DeviceType
 from cnaas_nms.db.settings import get_groups
 from cnaas_nms.tools.pki import ssl_context
@@ -41,11 +41,12 @@ def load(self) -> Inventory:
             connection_options={
                 "napalm": ConnectionOptions(
                     extras={
+                        "timeout": api_settings.NAPALM_TIMEOUT,
                         "optional_args": {
                             # args to eAPI HttpsEapiConnection for EOS
                             "enforce_verification": True,
                             "context": ssl_context,
-                        }
+                        },
                     }
                 ),
                 "netmiko": ConnectionOptions(extras={}),