Update ssot to v3.0 (#235)

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ on: # yamllint disable-line rule:truthy rule:comments
   pull_request: ~
 
 env:
-  APP_NAME: "nautobot-app-device-onboarding"
+  APP_NAME: "nautobot-device-onboarding"
 
 jobs:
   ruff-format:
@@ -91,6 +91,10 @@ jobs:
         uses: "actions/checkout@v4"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v6"
+      - name: "Constrain Nautobot version and regenerate lock file"
+        env:
+          INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "true"
+        run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v3"
@@ -108,6 +112,7 @@ jobs:
           build-args: |
             NAUTOBOT_VER=${{ matrix.nautobot-version }}
             PYTHON_VER=${{ matrix.python-version }}
+            CI=true
       - name: "Copy credentials"
         run: "cp development/creds.example.env development/creds.env"
       - name: "Linting: pylint"
@@ -141,6 +146,10 @@ jobs:
         uses: "actions/checkout@v4"
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v6"
+      - name: "Constrain Nautobot version and regenerate lock file"
+        env:
+          INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL: "true"
+        run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v3"
@@ -158,6 +167,7 @@ jobs:
           build-args: |
             NAUTOBOT_VER=${{ matrix.nautobot-version }}
             PYTHON_VER=${{ matrix.python-version }}
+            CI=true
       - name: "Copy credentials"
         run: "cp development/creds.example.env development/creds.env"
       - name: "Use Mysql invoke settings when needed"

changes/235.dependencies

@@ -0,0 +1,2 @@
+Updated Nautobot App SSoT to v3.0.0.
+Pinned griffe to v1.1.1.

development/Dockerfile

@@ -53,29 +53,18 @@ RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \
 WORKDIR /source
 COPY . /source
 
-# Get container's installed Nautobot version as a forced constraint
-# NAUTOBOT_VER may be a branch name and not a published release therefor we need to get the installed version
-#  so pip can use it to recognize local constraints.
-RUN pip show nautobot | grep "^Version: " | sed -e 's/Version: /nautobot==/' > constraints.txt
+# Build args must be declared in each stage
+ARG PYTHON_VER
 
-# Use Poetry to grab dev dependencies from the lock file
-# Can be improved in Poetry 1.2 which allows `poetry install --only dev`
-#
-# We can't use the entire freeze as it takes forever to resolve with rigidly fixed non-direct dependencies,
-#   especially those that are only direct to Nautobot but the container included versions slightly mismatch
-RUN poetry export -f requirements.txt --without-hashes --extras all --output poetry_freeze_base.txt
-RUN poetry export -f requirements.txt --without-hashes --extras all --with dev --output poetry_freeze_all.txt
-RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_dev.txt
-
-# Install all local project as editable, constrained on Nautobot version, to get any additional
-#   direct dependencies of the app
-RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \
-    pip install -c constraints.txt -e .[all]
+# Constrain the Nautobot version to NAUTOBOT_VER
+# In CI, this should be done outside of the Dockerfile to prevent cross-compile build failures
+ARG CI
+RUN if [ -z "${CI+x}" ]; then \
+    INSTALLED_NAUTOBOT_VER=$(pip show nautobot | grep "^Version" | sed "s/Version: //"); \
+    poetry add --lock nautobot@${INSTALLED_NAUTOBOT_VER} --python ${PYTHON_VER}; fi
 
-# Install any dev dependencies frozen from Poetry
-# Can be improved in Poetry 1.2 which allows `poetry install --only dev`
-RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \
-    pip install -c constraints.txt -r poetry_freeze_dev.txt
+# Install the app
+RUN poetry install --extras all --with dev
 
 COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py
 # !!! USE CAUTION WHEN MODIFYING LINES ABOVE

docs/requirements.txt

@@ -1,5 +1,6 @@
-mkdocs==1.5.2
-mkdocs-material==9.1.15
+mkdocs==1.6.0
+mkdocs-material==9.5.32
 markdown-version-annotations==1.0.1
-mkdocstrings-python==1.5.2
-mkdocstrings==0.22.0
+griffe==1.1.1
+mkdocstrings-python==1.10.8
+mkdocstrings==0.25.2

nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py

@@ -17,7 +17,7 @@
 ParameterSet = FrozenSet[Tuple[str, Hashable]]
 
 
-class SyncDevicesNautobotAdapter(diffsync.DiffSync):
+class SyncDevicesNautobotAdapter(diffsync.Adapter):
     """Adapter for loading Nautobot data."""
 
     manufacturer = sync_devices_models.SyncDevicesManufacturer
@@ -62,7 +62,7 @@ def load_manufacturers(self):
         for manufacturer in Manufacturer.objects.all():
             if self.job.debug:
                 self.job.logger.debug("Loading Manufacturer data from Nautobot...")
-            onboarding_manufacturer = self.manufacturer(diffsync=self, pk=manufacturer.pk, name=manufacturer.name)
+            onboarding_manufacturer = self.manufacturer(adapter=self, pk=manufacturer.pk, name=manufacturer.name)
             self.add(onboarding_manufacturer)
             if self.job.debug:
                 self.job.logger.debug(f"Manufacturer: {manufacturer.name} loaded.")
@@ -73,7 +73,7 @@ def load_platforms(self):
             self.job.logger.debug("Loading Platform data from Nautobot...")
         for platform in Platform.objects.all():
             onboarding_platform = self.platform(
-                diffsync=self,
+                adapter=self,
                 pk=platform.pk,
                 name=platform.name,
                 network_driver=platform.network_driver if platform.network_driver else "",
@@ -89,7 +89,7 @@ def load_device_types(self):
             self.job.logger.debug("Loading DeviceType data from Nautobot...")
         for device_type in DeviceType.objects.all():
             onboarding_device_type = self.device_type(
-                diffsync=self,
+                adapter=self,
                 pk=device_type.pk,
                 model=device_type.model,
                 part_number=device_type.part_number,
@@ -118,7 +118,7 @@ def load_devices(self):
             else:
                 interfaces = []
             onboarding_device = self.device(
-                diffsync=self,
+                adapter=self,
                 pk=device.pk,
                 device_type__model=device.device_type.model,
                 location__name=device.location.name,
@@ -145,7 +145,7 @@ def load(self):
         self.load_devices()
 
 
-class SyncDevicesNetworkAdapter(diffsync.DiffSync):
+class SyncDevicesNetworkAdapter(diffsync.Adapter):
     """Adapter for loading device data from a network."""
 
     manufacturer = sync_devices_models.SyncDevicesManufacturer
@@ -235,7 +235,7 @@ def load_manufacturers(self):
             onboarding_manufacturer = None
             try:
                 onboarding_manufacturer = self.manufacturer(
-                    diffsync=self,
+                    adapter=self,
                     name=self.device_data[ip_address]["manufacturer"],
                 )
             except KeyError as err:
@@ -256,7 +256,7 @@ def load_platforms(self):
             onboarding_platform = None
             try:
                 onboarding_platform = self.platform(
-                    diffsync=self,
+                    adapter=self,
                     name=self.device_data[ip_address]["platform"],
                     manufacturer__name=self.device_data[ip_address]["manufacturer"],
                     network_driver=self.device_data[ip_address]["network_driver"],
@@ -279,7 +279,7 @@ def load_device_types(self):
             onboarding_device_type = None
             try:
                 onboarding_device_type = self.device_type(
-                    diffsync=self,
+                    adapter=self,
                     model=self.device_data[ip_address]["device_type"],
                     part_number=self.device_data[ip_address]["device_type"],
                     manufacturer__name=self.device_data[ip_address]["manufacturer"],
@@ -334,7 +334,7 @@ def load_devices(self):
                 )
 
                 onboarding_device = self.device(
-                    diffsync=self,
+                    adapter=self,
                     device_type__model=self.device_data[ip_address]["device_type"],
                     location__name=location.name,
                     name=self.device_data[ip_address]["hostname"],

nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py

@@ -29,7 +29,7 @@ class FilteredNautobotAdapter(NautobotAdapter):
     def _load_objects(self, diffsync_model):  # pylint: disable=protected-access
         """Given a diffsync model class, load a list of models from the database and return them."""
         parameter_names = self._get_parameter_names(diffsync_model)
-        for database_object in diffsync_model._get_queryset(diffsync=self):  # pylint: disable=protected-access
+        for database_object in diffsync_model._get_queryset(adapter=self):  # pylint: disable=protected-access
             self._load_single_object(database_object, diffsync_model, parameter_names)
 
 
@@ -99,7 +99,7 @@ def load_ip_addresses(self):
             parent__namespace__name=self.job.namespace.name,
         ):
             network_ip_address = self.ip_address(
-                diffsync=self,
+                adapter=self,
                 host=ip_address.host,
                 mask_length=ip_address.mask_length,
                 type=ip_address.type,
@@ -123,7 +123,7 @@ def load_vlans(self):
         """
         for vlan in VLAN.objects.all():
             network_vlan = self.vlan(
-                diffsync=self,
+                adapter=self,
                 name=vlan.name,
                 vid=vlan.vid,
                 location__name=vlan.location.name if vlan.location else "",
@@ -149,7 +149,7 @@ def load_tagged_vlans_to_interface(self):
                 tagged_vlans.append(vlan_dict)
 
             network_tagged_vlans_to_interface = self.tagged_vlans_to_interface(
-                diffsync=self,
+                adapter=self,
                 device__name=interface.device.name,
                 name=interface.name,
                 tagged_vlans=tagged_vlans,
@@ -170,7 +170,7 @@ def load_untagged_vlan_to_interface(self):
                 untagged_vlan["id"] = str(interface.untagged_vlan.vid)
 
             network_untagged_vlan_to_interface = self.untagged_vlan_to_interface(
-                diffsync=self,
+                adapter=self,
                 device__name=interface.device.name,
                 name=interface.name,
                 untagged_vlan=untagged_vlan,
@@ -186,7 +186,7 @@ def load_lag_to_interface(self):
         """
         for interface in Interface.objects.filter(device__in=self.job.devices_to_load):
             network_lag_to_interface = self.lag_to_interface(
-                diffsync=self,
+                adapter=self,
                 device__name=interface.device.name,
                 name=interface.name,
                 lag__interface__name=interface.lag.name if interface.lag else "",
@@ -202,7 +202,7 @@ def load_vrfs(self):
         """
         for vrf in VRF.objects.all():
             network_vrf = self.vrf(
-                diffsync=self,
+                adapter=self,
                 name=vrf.name,
                 namespace__name=vrf.namespace.name,
             )
@@ -224,7 +224,7 @@ def load_vrf_to_interface(self):
                 vrf["name"] = interface.vrf.name
 
             network_vrf_to_interface = self.vrf_to_interface(
-                diffsync=self,
+                adapter=self,
                 device__name=interface.device.name,
                 name=interface.name,
                 vrf=vrf,
@@ -258,7 +258,7 @@ def load_cables(self):
                     termination_b_interface = cable.termination_a.name
 
                 network_cable = self.cable(
-                    diffsync=self,
+                    adapter=self,
                     status__name=cable.status.name,
                     termination_a__app_label="dcim",
                     termination_a__model="interface",
@@ -362,7 +362,7 @@ class MacUnixExpandedUppercase(mac_unix_expanded):
     word_fmt = "%.2X"
 
 
-class SyncNetworkDataNetworkAdapter(diffsync.DiffSync):
+class SyncNetworkDataNetworkAdapter(diffsync.Adapter):
     """Adapter for loading Network data."""
 
     def __init__(self, *args, job, sync=None, **kwargs):
@@ -465,7 +465,7 @@ def load_devices(self):
         for hostname, device_data in self.job.command_getter_result.items():
             try:
                 network_device = self.device(
-                    diffsync=self,
+                    adapter=self,
                     name=hostname,
                     serial=device_data["serial"],
                     last_network_data_sync=datetime.datetime.now().date().isoformat(),
@@ -489,7 +489,7 @@ def load_devices(self):
     def load_interface(self, hostname, interface_name, interface_data):
         """Load an interface into the DiffSync store."""
         network_interface = self.interface(
-            diffsync=self,
+            adapter=self,
             name=interface_name,
             device__name=hostname,
             status__name=self.job.interface_status.name,
@@ -499,6 +499,8 @@ def load_interface(self, hostname, interface_name, interface_data):
             description=interface_data["description"],
             enabled=interface_data["link_status"],
             mode=interface_data["802.1Q_mode"],
+            parent_interface__name=None,
+            lag__name=None,
             # untagged_vlan__name=self._get_vlan_name(interface_data=interface_data),
         )
         self.add(network_interface)
@@ -518,7 +520,7 @@ def load_ip_addresses(self):
                                 self.job.logger.debug(f"Loading {ip_address} from {interface_name} on {hostname}")
                             try:
                                 network_ip_address = self.ip_address(
-                                    diffsync=self,
+                                    adapter=self,
                                     host=ip_address["ip_address"],
                                     mask_length=int(ip_address["prefix_length"]),
                                     type="host",
@@ -553,7 +555,7 @@ def load_vlans(self):
                 for tagged_vlan in interface_data["tagged_vlans"]:
                     try:
                         network_vlan = self.vlan(
-                            diffsync=self,
+                            adapter=self,
                             name=tagged_vlan["name"],
                             vid=tagged_vlan["id"],
                             location__name=location_names.get(hostname, ""),
@@ -570,7 +572,7 @@ def load_vlans(self):
                 if interface_data["untagged_vlan"]:
                     try:
                         network_vlan = self.vlan(
-                            diffsync=self,
+                            adapter=self,
                             name=interface_data["untagged_vlan"]["name"],
                             vid=interface_data["untagged_vlan"]["id"],
                             location__name=location_names.get(hostname, ""),
@@ -594,7 +596,7 @@ def load_vrfs(self):
                 if interface_data["vrf"]:
                     try:
                         network_vrf = self.vrf(
-                            diffsync=self,
+                            adapter=self,
                             name=interface_data["vrf"]["name"],
                             namespace__name=self.job.namespace.name,
                         )
@@ -615,7 +617,7 @@ def load_ip_address_to_interfaces(self):
                     if ip_address["ip_address"]:  # the ip_address and mask_length may be empty, skip these
                         try:
                             network_ip_address_to_interface = self.ipaddress_to_interface(
-                                diffsync=self,
+                                adapter=self,
                                 interface__device__name=hostname,
                                 interface__name=interface_name,
                                 ip_address__host=ip_address["ip_address"],
@@ -637,7 +639,7 @@ def load_tagged_vlans_to_interface(self):
             for interface_name, interface_data in device_data["interfaces"].items():
                 try:
                     network_tagged_vlans_to_interface = self.tagged_vlans_to_interface(
-                        diffsync=self,
+                        adapter=self,
                         device__name=hostname,
                         name=interface_name,
                         tagged_vlans=interface_data["tagged_vlans"],
@@ -656,7 +658,7 @@ def load_untagged_vlan_to_interface(self):
             for interface_name, interface_data in device_data["interfaces"].items():
                 try:
                     network_untagged_vlan_to_interface = self.untagged_vlan_to_interface(
-                        diffsync=self,
+                        adapter=self,
                         device__name=hostname,
                         name=interface_name,
                         untagged_vlan=interface_data["untagged_vlan"],
@@ -675,7 +677,7 @@ def load_lag_to_interface(self):
             for interface_name, interface_data in device_data["interfaces"].items():
                 try:
                     network_lag_to_interface = self.lag_to_interface(
-                        diffsync=self,
+                        adapter=self,
                         device__name=hostname,
                         name=interface_name,
                         lag__interface__name=interface_data["lag"] if interface_data["lag"] else "",
@@ -694,7 +696,7 @@ def load_vrf_to_interface(self):
             for interface_name, interface_data in device_data["interfaces"].items():
                 try:
                     network_vrf_to_interface = self.vrf_to_interface(
-                        diffsync=self,
+                        adapter=self,
                         device__name=hostname,
                         name=interface_name,
                         vrf=interface_data["vrf"],
@@ -772,7 +774,7 @@ def load_cables(self):  # pylint: disable=inconsistent-return-statements
                         termination_b_interface = local_interface
 
                     network_cable = self.cable(
-                        diffsync=self,
+                        adapter=self,
                         status__name="Connected",  # ask for default status in the job form
                         termination_a__app_label="dcim",
                         termination_a__model="interface",