Add FQDN support for device onboarding (#251)

* update to fix mtu int error
* added fqdn support, formatting
---------

Co-authored-by: Susan Hooks <susan.hooks@chevron.com>

changes/241.added

@@ -0,0 +1 @@
+Added FQDN support to the sync network device job.

nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py

@@ -1,5 +1,6 @@
 """DiffSync adapters."""
 
+import socket
 from collections import defaultdict
 from typing import DefaultDict, Dict, FrozenSet, Hashable, Tuple, Type
 
@@ -11,7 +12,9 @@
 from nautobot.dcim.models import Device, DeviceType, Manufacturer, Platform
 
 from nautobot_device_onboarding.diffsync.models import sync_devices_models
-from nautobot_device_onboarding.nornir_plays.command_getter import sync_devices_command_getter
+from nautobot_device_onboarding.nornir_plays.command_getter import (
+    sync_devices_command_getter,
+)
 from nautobot_device_onboarding.utils import diffsync_utils
 
 ParameterSet = FrozenSet[Tuple[str, Hashable]]
@@ -76,8 +79,8 @@ def load_platforms(self):
                 adapter=self,
                 pk=platform.pk,
                 name=platform.name,
-                network_driver=platform.network_driver if platform.network_driver else "",
-                manufacturer__name=platform.manufacturer.name if platform.manufacturer else None,
+                network_driver=(platform.network_driver if platform.network_driver else ""),
+                manufacturer__name=(platform.manufacturer.name if platform.manufacturer else None),
             )
             self.add(onboarding_platform)
             if self.job.debug:
@@ -125,12 +128,12 @@ def load_devices(self):
                 name=device.name,
                 platform__name=device.platform.name if device.platform else "",
                 primary_ip4__host=device.primary_ip4.host if device.primary_ip4 else "",
-                primary_ip4__status__name=device.primary_ip4.status.name if device.primary_ip4 else "",
+                primary_ip4__status__name=(device.primary_ip4.status.name if device.primary_ip4 else ""),
                 role__name=device.role.name,
                 status__name=device.status.name,
-                secrets_group__name=device.secrets_group.name if device.secrets_group else "",
+                secrets_group__name=(device.secrets_group.name if device.secrets_group else ""),
                 interfaces=interfaces,
-                mask_length=device.primary_ip4.mask_length if device.primary_ip4 else None,
+                mask_length=(device.primary_ip4.mask_length if device.primary_ip4 else None),
                 serial=device.serial,
             )
             self.add(onboarding_device)
@@ -167,12 +170,17 @@ def _validate_ip_addresses(self, ip_addresses):
         """Validate the format of each IP Address in a list of IP Addresses."""
         # Validate IP Addresses
         validation_successful = True
-        for ip_address in ip_addresses:
+        for i, ip_address in enumerate(ip_addresses):
             try:
                 netaddr.IPAddress(ip_address)
             except netaddr.AddrFormatError:
-                self.job.logger.error(f"[{ip_address}] is not a valid IP Address ")
-                validation_successful = False
+                try:
+                    resolved_ip = socket.gethostbyname(ip_address)
+                    self.job.logger.info(f"[{ip_address}] resolved to [{resolved_ip}]")
+                    ip_addresses[i] = resolved_ip
+                except socket.gaierror:
+                    self.job.logger.error(f"[{ip_address}] is not a valid IP Address or name.")
+                    validation_successful = False
         if validation_successful:
             return True
         raise netaddr.AddrConversionError
@@ -203,10 +211,14 @@ def execute_command_getter(self):
                         f"The selected platform, {self.job.platform} "
                         "does not have a network driver, please update the Platform."
                     )
-                    raise Exception("Platform.network_driver missing")  # pylint: disable=broad-exception-raised
+                    raise Exception(  # pylint: disable=broad-exception-raised
+                        "Platform.network_driver missing"
+                    )
 
         result = sync_devices_command_getter(
-            self.job.job_result, self.job.logger.getEffectiveLevel(), self.job.job_result.task_kwargs
+            self.job.job_result,
+            self.job.logger.getEffectiveLevel(),
+            self.job.job_result.task_kwargs,
         )
         if self.job.debug:
             self.job.logger.debug(f"Command Getter Result: {result}")
@@ -297,7 +309,13 @@ def load_device_types(self):
     def _fields_missing_data(self, device_data, ip_address, platform):
         """Verify that all of the fields returned from a device actually contain data."""
         fields_missing_data = []
-        required_fields_from_device = ["device_type", "hostname", "mgmt_interface", "mask_length", "serial"]
+        required_fields_from_device = [
+            "device_type",
+            "hostname",
+            "mgmt_interface",
+            "mask_length",
+            "serial",
+        ]
         if platform:  # platform is only returned with device data if not provided on the job form/csv
             required_fields_from_device.append("platform")
         for field in required_fields_from_device:
@@ -321,7 +339,9 @@ def load_devices(self):
                     job=self.job, ip_address=ip_address, query_string="platform"
                 )
                 primary_ip4__status = diffsync_utils.retrieve_submitted_value(
-                    job=self.job, ip_address=ip_address, query_string="ip_address_status"
+                    job=self.job,
+                    ip_address=ip_address,
+                    query_string="ip_address_status",
                 )
                 device_role = diffsync_utils.retrieve_submitted_value(
                     job=self.job, ip_address=ip_address, query_string="device_role"
@@ -338,7 +358,7 @@ def load_devices(self):
                     device_type__model=self.device_data[ip_address]["device_type"],
                     location__name=location.name,
                     name=self.device_data[ip_address]["hostname"],
-                    platform__name=platform.name if platform else self.device_data[ip_address]["platform"],
+                    platform__name=(platform.name if platform else self.device_data[ip_address]["platform"]),
                     primary_ip4__host=ip_address,
                     primary_ip4__status__name=primary_ip4__status.name,
                     role__name=device_role.name,

nautobot_device_onboarding/jobs.py

@@ -10,11 +10,30 @@
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
-from nautobot.apps.jobs import BooleanVar, ChoiceVar, FileVar, IntegerVar, Job, MultiObjectVar, ObjectVar, StringVar
+from nautobot.apps.jobs import (
+    BooleanVar,
+    ChoiceVar,
+    FileVar,
+    IntegerVar,
+    Job,
+    MultiObjectVar,
+    ObjectVar,
+    StringVar,
+)
 from nautobot.core.celery import register_jobs
 from nautobot.dcim.models import Device, DeviceType, Location, Platform
-from nautobot.extras.choices import CustomFieldTypeChoices, SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices
-from nautobot.extras.models import CustomField, Role, SecretsGroup, SecretsGroupAssociation, Status
+from nautobot.extras.choices import (
+    CustomFieldTypeChoices,
+    SecretsGroupAccessTypeChoices,
+    SecretsGroupSecretTypeChoices,
+)
+from nautobot.extras.models import (
+    CustomField,
+    Role,
+    SecretsGroup,
+    SecretsGroupAssociation,
+    Status,
+)
 from nautobot.ipam.models import Namespace
 from nautobot_plugin_nornir.constants import NORNIR_SETTINGS
 from nautobot_ssot.jobs.base import DataSource
@@ -32,7 +51,10 @@
 )
 from nautobot_device_onboarding.exceptions import OnboardException
 from nautobot_device_onboarding.netdev_keeper import NetdevKeeper
-from nautobot_device_onboarding.nornir_plays.command_getter import _parse_credentials, netmiko_send_commands
+from nautobot_device_onboarding.nornir_plays.command_getter import (
+    _parse_credentials,
+    netmiko_send_commands,
+)
 from nautobot_device_onboarding.nornir_plays.empty_inventory import EmptyInventory
 from nautobot_device_onboarding.nornir_plays.inventory_creator import _set_inventory
 from nautobot_device_onboarding.nornir_plays.logger import NornirLogger
@@ -62,7 +84,9 @@ class OnboardingTask(Job):  # pylint: disable=too-many-instance-attributes
     port = IntegerVar(default=22)
     timeout = IntegerVar(default=30)
     credentials = ObjectVar(
-        model=SecretsGroup, required=False, description="SecretsGroup for Device connection credentials."
+        model=SecretsGroup,
+        required=False,
+        description="SecretsGroup for Device connection credentials.",
     )
     platform = ObjectVar(
         model=Platform,
@@ -128,7 +152,9 @@ def run(self, *args, **data):
                 self._onboard(address=address)
             except OnboardException as err:
                 self.logger.exception(
-                    "The following exception occurred when attempting to onboard %s: %s", address, str(err)
+                    "The following exception occurred when attempting to onboard %s: %s",
+                    address,
+                    str(err),
                 )
                 if not data["continue_on_failure"]:
                     raise OnboardException(
@@ -146,7 +172,7 @@ def _onboard(self, address):
             username=self.username,
             password=self.password,
             secret=self.secret,
-            napalm_driver=self.platform.napalm_driver if self.platform and self.platform.napalm_driver else None,
+            napalm_driver=(self.platform.napalm_driver if self.platform and self.platform.napalm_driver else None),
             optional_args=(
                 self.platform.napalm_args if self.platform and self.platform.napalm_args else settings.NAPALM_ARGS
             ),
@@ -159,10 +185,10 @@ def _onboard(self, address):
             "netdev_mgmt_ip_address": address,
             "netdev_nb_location_name": self.location.name,
             "netdev_nb_device_type_name": self.device_type,
-            "netdev_nb_role_name": self.role.name if self.role else PLUGIN_SETTINGS["default_device_role"],
+            "netdev_nb_role_name": (self.role.name if self.role else PLUGIN_SETTINGS["default_device_role"]),
             "netdev_nb_role_color": PLUGIN_SETTINGS["default_device_role_color"],
             "netdev_nb_platform_name": self.platform.name if self.platform else None,
-            "netdev_nb_credentials": self.credentials if PLUGIN_SETTINGS["assign_secrets_group"] else None,
+            "netdev_nb_credentials": (self.credentials if PLUGIN_SETTINGS["assign_secrets_group"] else None),
             # Kwargs discovered on the Onboarded Device:
             "netdev_hostname": netdev_dict["netdev_hostname"],
             "netdev_vendor": netdev_dict["netdev_vendor"],
@@ -175,10 +201,16 @@ def _onboard(self, address):
             "driver_addon_result": netdev_dict["driver_addon_result"],
         }
         onboarding_cls = netdev_dict["onboarding_class"]()
-        onboarding_cls.credentials = {"username": self.username, "password": self.password, "secret": self.secret}
+        onboarding_cls.credentials = {
+            "username": self.username,
+            "password": self.password,
+            "secret": self.secret,
+        }
         onboarding_cls.run(onboarding_kwargs=onboarding_kwargs)
         self.logger.info(
-            "Successfully onboarded %s with a management IP of %s", netdev_dict["netdev_hostname"], address
+            "Successfully onboarded %s with a management IP of %s",
+            netdev_dict["netdev_hostname"],
+            address,
         )
 
     def _parse_credentials(self, credentials):
@@ -236,7 +268,9 @@ class Meta:
         description="Enable for more verbose logging.",
     )
     csv_file = FileVar(
-        label="CSV File", required=False, description="If a file is provided all the options below will be ignored."
+        label="CSV File",
+        required=False,
+        description="If a file is provided all the options below will be ignored.",
     )
     location = ObjectVar(
         model=Location,
@@ -247,7 +281,7 @@ class Meta:
     namespace = ObjectVar(model=Namespace, required=False, description="Namespace ip addresses belong to.")
     ip_addresses = StringVar(
         required=False,
-        description="IP address of the device to sync, specify in a comma separated list for multiple devices.",
+        description="IP address or FQDN of the device to sync, specify in a comma separated list for multiple devices.",
         label="IPv4 addresses",
     )
     port = IntegerVar(required=False, default=22)
@@ -288,7 +322,9 @@ class Meta:
         description="Status to be applied to all new synced IP addresses. This value does not update with additional syncs.",
     )
     secrets_group = ObjectVar(
-        model=SecretsGroup, required=False, description="SecretsGroup for device connection credentials."
+        model=SecretsGroup,
+        required=False,
+        description="SecretsGroup for device connection credentials.",
     )
     platform = ObjectVar(
         model=Platform,
@@ -333,7 +369,8 @@ def _process_csv_data(self, csv_file):
                 query = f"location_name: {row.get('location_name')}, location_parent_name: {row.get('location_parent_name')}"
                 if row.get("location_parent_name"):
                     location = Location.objects.get(
-                        name=row["location_name"].strip(), parent__name=row["location_parent_name"].strip()
+                        name=row["location_name"].strip(),
+                        parent__name=row["location_parent_name"].strip(),
                     )
                 else:
                     query = query = f"location_name: {row.get('location_name')}"
@@ -452,7 +489,10 @@ def run(
                 for ip_address in self.processed_csv_data:
                     self.ip_addresses.append(ip_address)
                 # prepare the task_kwargs needed by the CommandGetterDO job
-                self.job_result.task_kwargs = {"debug": debug, "csv_file": self.task_kwargs_csv_data}
+                self.job_result.task_kwargs = {
+                    "debug": debug,
+                    "csv_file": self.task_kwargs_csv_data,
+                }
             else:
                 raise ValidationError(message="CSV check failed. No devices will be synced.")
 
@@ -536,7 +576,9 @@ class Meta:
     sync_vrfs = BooleanVar(default=False, description="Sync VRFs and interface VRF assignments.")
     sync_cables = BooleanVar(default=False, description="Sync cables between interfaces via a LLDP or CDP.")
     namespace = ObjectVar(
-        model=Namespace, required=True, description="The namespace for all IP addresses created or updated in the sync."
+        model=Namespace,
+        required=True,
+        description="The namespace for all IP addresses created or updated in the sync.",
     )
     interface_status = ObjectVar(
         model=Status,
@@ -632,7 +674,9 @@ def run(
         if self.debug:
             self.logger.debug("Checking for last_network_data_sync custom field")
         try:
-            cf = CustomField.objects.get(key="last_network_data_sync")  # pylint:disable=invalid-name
+            cf = CustomField.objects.get(  # pylint:disable=invalid-name
+                key="last_network_data_sync"
+            )
         except ObjectDoesNotExist:
             cf, _ = CustomField.objects.get_or_create(  # pylint:disable=invalid-name
                 label="Last Network Data Sync",
@@ -716,7 +760,9 @@ def run(self, *args, **kwargs):  # pragma: no cover
         ip_addresses = kwargs["ip_addresses"].replace(" ", "").split(",")
         port = kwargs["port"]
         platform = kwargs["platform"]
-        username, password, secret = _parse_credentials(kwargs["secrets_group"])  # pylint:disable=unused-variable
+        username, password, secret = (  # pylint:disable=unused-variable
+            _parse_credentials(kwargs["secrets_group"])
+        )
 
         # Initiate Nornir instance with empty inventory
         compiled_results = {}
@@ -771,5 +817,10 @@ def run(self, *args, **kwargs):  # pragma: no cover
         return f"Successfully ran the following commands: {', '.join(list(compiled_results.keys()))}"
 
 
-jobs = [OnboardingTask, SSOTSyncDevices, SSOTSyncNetworkData, DeviceOnboardingTroubleshootingJob]
+jobs = [
+    OnboardingTask,
+    SSOTSyncDevices,
+    SSOTSyncNetworkData,
+    DeviceOnboardingTroubleshootingJob,
+]
 register_jobs(*jobs)