Merge branch 'develop'

Author: glennmatthews

.dockerignore

@@ -0,0 +1,4 @@
+.git
+dist
+dumps
+**/__pycache__/

.gitignore

@@ -171,6 +171,7 @@ ehthumbs_vista.db
 
 # Dump file
 *.stackdump
+dumps/
 
 # Folder config file
 [Dd]esktop.ini

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v1.1.0 (2021-04-07)
+
+### Added
+
+- Now supports import from NetBox versions up to 2.10.8
+- Now compatible with Nautobot 1.0.0b3
+
+### Changed
+
+- #28 - Rework of internal data representations to use primary keys instead of natural keys for most models.
+  This should fix many "duplicate object" problems reported by earlier versions of this plugin (#11, #19, #25, #26, #27)
+
+### Fixed
+
+- #10 - Catch `ObjectDoesNotExist` exceptions instead of erroring out
+- #12 - Duplicate object reports should include primary key
+- #13 - Allow import of objects with custom field data referencing custom fields that no longer exist
+- #14 - Allow import of objects with old custom field data not matching latest requirements
+- #24 - Allow import of EUI MACAddress records
+
+### Removed
+
+- No longer compatible with Nautobot 1.0.0b2 and earlier
+
+
 ## v1.0.1 (2021-03-09)
 
 ### Added

README.md

@@ -10,7 +10,7 @@ The plugin is available as a Python package in PyPI and can be installed with pi
 pip install nautobot-netbox-importer
 ```
 
-> The plugin is compatible with Nautobot 1.0 and can handle JSON data exported from NetBox 2.10.3 through 2.10.5 at present.
+> The plugin is compatible with Nautobot 1.0.0b3 and later and can handle JSON data exported from NetBox 2.10.x at present.
 
 Once installed, the plugin needs to be enabled in your `nautobot_config.py`:
 
@@ -22,7 +22,7 @@ PLUGINS = ["nautobot_netbox_importer"]
 
 ### Getting a data export from NetBox
 
-From the NetBox root directory, run the following command:
+From the NetBox root directory, run the following command to produce a JSON file (here, `/tmp/netbox_data.json`) describing the contents of your NetBox database:
 
 ```shell
 python netbox/manage.py dumpdata \
@@ -34,7 +34,7 @@ python netbox/manage.py dumpdata \
 
 ### Importing the data into Nautobot
 
-From the Nautobot root directory, run `nautobot-server import_netbox_json <json_file> <netbox_version>`, for example `nautobot-server import_netbox_json /tmp/netbox_data.json 2.10.3`.
+From within the Nautobot application environment, run `nautobot-server import_netbox_json <json_file> <netbox_version>`, for example `nautobot-server import_netbox_json /tmp/netbox_data.json 2.10.3`.
 
 ## Contributing
 

development/nautobot_config.py

@@ -78,6 +78,22 @@
         "SSL": False,
         "DEFAULT_TIMEOUT": 300,
     },
+    "custom_fields": {
+        "HOST": os.environ.get("REDIS_HOST", "localhost"),
+        "PORT": os.environ.get("REDIS_PORT", 6379),
+        "DB": 0,
+        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
+        "SSL": False,
+        "DEFAULT_TIMEOUT": 300,
+    },
+    "webhooks": {
+        "HOST": os.environ.get("REDIS_HOST", "localhost"),
+        "PORT": os.environ.get("REDIS_PORT", 6379),
+        "DB": 0,
+        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
+        "SSL": False,
+        "DEFAULT_TIMEOUT": 300,
+    },
 }
 
 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file.

nautobot_netbox_importer/__init__.py

@@ -22,6 +22,7 @@ class NautobotNetboxImporterConfig(PluginConfig):
     description = "Data importer from NetBox 2.10.x to Nautobot."
     base_url = "netbox-importer"
     required_settings = []
+    min_version = "1.0.0b3"
     max_version = "1.9999"
     default_settings = {}
     caching_config = {}

nautobot_netbox_importer/diffsync/adapters/__init__.py

@@ -9,6 +9,9 @@
     version.parse("2.10.3"): NetBox210DiffSync,
     version.parse("2.10.4"): NetBox210DiffSync,
     version.parse("2.10.5"): NetBox210DiffSync,
+    version.parse("2.10.6"): NetBox210DiffSync,
+    version.parse("2.10.7"): NetBox210DiffSync,
+    version.parse("2.10.8"): NetBox210DiffSync,
 }
 
 __all__ = (

nautobot_netbox_importer/diffsync/adapters/abstract.py

@@ -5,11 +5,12 @@
 from uuid import UUID
 
 from diffsync import Diff, DiffSync, DiffSyncFlags, DiffSyncModel
-from diffsync.exceptions import ObjectAlreadyExists
+from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound
 from pydantic.error_wrappers import ValidationError
 import structlog
 
 import nautobot_netbox_importer.diffsync.models as n2nmodels
+from nautobot_netbox_importer.diffsync.models.validation import netbox_pk_to_nautobot_pk
 
 
 class N2NDiffSync(DiffSync):
@@ -31,7 +32,6 @@ class N2NDiffSync(DiffSync):
     permission = n2nmodels.Permission
     token = n2nmodels.Token
     user = n2nmodels.User
-    userconfig = n2nmodels.UserConfig
 
     # Circuits
     circuit = n2nmodels.Circuit
@@ -76,6 +76,7 @@ class N2NDiffSync(DiffSync):
     # Extras
     configcontext = n2nmodels.ConfigContext
     customfield = n2nmodels.CustomField
+    customfieldchoice = n2nmodels.CustomFieldChoice
     customlink = n2nmodels.CustomLink
     exporttemplate = n2nmodels.ExportTemplate
     jobresult = n2nmodels.JobResult
@@ -121,17 +122,18 @@ class N2NDiffSync(DiffSync):
     # The specific order of models below is constructed empirically, but basically attempts to place all models
     # in sequence so that if model A has a hard dependency on a reference to model B, model B gets processed first.
     #
+    # Note: with the latest changes in design for this plugin (using deterministic UUIDs in Nautobot to allow
+    # direct mapping of NetBox PKs to Nautobot PKs), this order is now far less critical than it was previously.
+    #
 
     top_level = (
         # "contenttype", Not synced, as these are hard-coded in NetBox/Nautobot
-        "customfield",
-        "permission",
+        # "permission", Not synced, as these are superseded by "objectpermission"
         "group",
-        "user",
+        "user",  # Includes NetBox "userconfig" model as well
         "objectpermission",
         "token",
-        "userconfig",
-        # "status", Not synced, as these are hard-coded in NetBox/Nautobot
+        # "status", Not synced, as these are hard-coded in NetBox and autogenerated in Nautobot
         # Need Tenant and TenantGroup before we can populate Sites
         "tenantgroup",
         "tenant",  # Not all Tenants belong to a TenantGroup
@@ -194,7 +196,6 @@ class N2NDiffSync(DiffSync):
         #   Interface/VMInterface -> Device/VirtualMachine (device)
         # Interface comes after Device because it MUST have a Device to be created;
         # IPAddress comes after Interface because we use the assigned_object as part of the IP's unique ID.
-        # We will fixup the Device->primary_ip reference in fixup_data_relations()
         "ipaddress",
         "cable",
         "service",
@@ -207,6 +208,10 @@ class N2NDiffSync(DiffSync):
         "webhook",
         "taggeditem",
         "jobresult",
+        # Imported last so that any "required=True" CustomFields do not cause Nautobot to reject
+        # NetBox records that predate the creation of those CustomFields
+        "customfield",
+        "customfieldchoice",
     )
 
     def __init__(self, *args, **kwargs):
@@ -229,41 +234,10 @@ def add(self, obj: DiffSyncModel):
         self._data_by_pk[modelname][obj.pk] = obj
         super().add(obj)
 
-    def fixup_data_relations(self):
-        """Iterate once more over all models and fix up any leftover FK relations."""
-        for name in self.top_level:
-            instances = self.get_all(name)
-            if not instances:
-                self.logger.info("No instances to review", model=name)
-            else:
-                self.logger.info(f"Reviewing all {len(instances)} instances", model=name)
-            for diffsync_instance in instances:
-                for fk_field, target_name in diffsync_instance.fk_associations().items():
-                    value = getattr(diffsync_instance, fk_field)
-                    if not value:
-                        continue
-                    if "*" in target_name:
-                        target_content_type_field = target_name[1:]
-                        target_content_type = getattr(diffsync_instance, target_content_type_field)
-                        target_name = target_content_type["model"]
-                    target_class = getattr(self, target_name)
-                    if "pk" in value:
-                        new_value = self.get_fk_identifiers(diffsync_instance, target_class, value["pk"])
-                        if isinstance(new_value, (UUID, int)):
-                            self.logger.error(
-                                "Still unable to resolve reference?",
-                                source=diffsync_instance,
-                                target=target_name,
-                                pk=new_value,
-                            )
-                        else:
-                            self.logger.debug(
-                                "Replacing forward reference with identifiers", pk=value["pk"], identifiers=new_value
-                            )
-                            setattr(diffsync_instance, fk_field, new_value)
-
     def get_fk_identifiers(self, source_object, target_class, pk):
         """Helper to load_record: given a class and a PK, get the identifiers of the given instance."""
+        if isinstance(pk, int):
+            pk = netbox_pk_to_nautobot_pk(target_class.get_type(), pk)
         target_record = self.get_by_pk(target_class, pk)
         if not target_record:
             self.logger.debug(
@@ -281,6 +255,8 @@ def get_by_pk(self, obj, pk):
             modelname = obj
         else:
             modelname = obj.get_type()
+        if pk not in self._data_by_pk[modelname]:
+            raise ObjectNotFound(f"PK {pk} not found in stored {modelname} instances")
         return self._data_by_pk[modelname].get(pk)
 
     def make_model(self, diffsync_model, data):
@@ -293,18 +269,21 @@ def make_model(self, diffsync_model, data):
                 "This may be an issue with your source data or may reflect a bug in this plugin.",
                 action="load",
                 exception=str(exc),
-                model=diffsync_model,
+                model=diffsync_model.get_type(),
                 model_data=data,
             )
             return None
         try:
             self.add(instance)
         except ObjectAlreadyExists:
+            existing_instance = self.get(diffsync_model, instance.get_unique_id())
             self.logger.warning(
-                "Apparent duplicate object encountered. "
+                "Apparent duplicate object encountered? "
                 "This may be an issue with your source data or may reflect a bug in this plugin.",
-                model=instance,
-                model_id=instance.get_unique_id(),
+                duplicate_id=instance.get_identifiers(),
+                model=diffsync_model.get_type(),
+                pk_1=existing_instance.pk,
+                pk_2=instance.pk,
             )
         return instance
 

nautobot_netbox_importer/diffsync/adapters/nautobot.py

@@ -7,6 +7,7 @@
 import structlog
 
 from .abstract import N2NDiffSync
+from ..models.abstract import NautobotBaseModel
 
 
 IGNORED_FIELD_CLASSES = (GenericRel, GenericForeignKey, models.ManyToManyRel, models.ManyToOneRel)
@@ -23,7 +24,7 @@ class NautobotDiffSync(N2NDiffSync):
 
     logger = structlog.get_logger()
 
-    def load_model(self, diffsync_model, record):
+    def load_model(self, diffsync_model, record):  # pylint: disable=too-many-branches
         """Instantiate the given DiffSync model class from the given Django record."""
         data = {}
 
@@ -46,7 +47,8 @@ def load_model(self, diffsync_model, record):
 
             # If we got here, the field is some sort of foreign-key reference(s).
             if not value:
-                # It's a null reference though, so we don't need to do anything special with it.
+                # It's a null or empty list reference though, so we don't need to do anything special with it.
+                data[field.name] = value
                 continue
 
             # What's the name of the model that this is a reference to?
@@ -69,16 +71,24 @@ def load_model(self, diffsync_model, record):
                 continue
 
             if isinstance(value, list):
-                # This field is a one-to-many or many-to-many field, a list of foreign key references.
-                # For each foreign key, find the corresponding DiffSync record, and use its
-                # natural keys (identifiers) in the data in place of the foreign key value.
-                data[field.name] = [
-                    self.get_fk_identifiers(diffsync_model, target_class, foreign_record.pk) for foreign_record in value
-                ]
-            elif isinstance(value, (UUID, int)):
-                # Look up the DiffSync record corresponding to this foreign key,
-                # and store its natural keys (identifiers) in the data in place of the foreign key value.
-                data[field.name] = self.get_fk_identifiers(diffsync_model, target_class, value)
+                # This field is a one-to-many or many-to-many field, a list of object references.
+                if issubclass(target_class, NautobotBaseModel):
+                    # Replace each object reference with its appropriate primary key value
+                    data[field.name] = [foreign_record.pk for foreign_record in value]
+                else:
+                    # Since the PKs of these built-in Django models may differ between NetBox and Nautobot,
+                    # e.g., ContentTypes, replace each reference with the natural key (not PK) of the referenced model.
+                    data[field.name] = [
+                        self.get_by_pk(target_name, foreign_record.pk).get_identifiers() for foreign_record in value
+                    ]
+            elif isinstance(value, UUID):
+                # Standard Nautobot UUID foreign-key reference, no transformation needed.
+                data[field.name] = value
+            elif isinstance(value, int):
+                # Reference to a built-in model by its integer primary key.
+                # Since this may not be the same value between NetBox and Nautobot (e.g., ContentType references)
+                # replace the PK with the natural keys of the referenced model.
+                data[field.name] = self.get_by_pk(target_name, value).get_identifiers()
             else:
                 self.logger.error(f"Invalid PK value {value}")
                 data[field.name] = None
@@ -89,13 +99,10 @@ def load_model(self, diffsync_model, record):
     def load(self):
         """Load all available and relevant data from Nautobot in the appropriate sequence."""
         self.logger.info("Loading data from Nautobot into DiffSync...")
-        for modelname in ("contenttype", "status", *self.top_level):
+        for modelname in ("contenttype", "permission", "status", *self.top_level):
             diffsync_model = getattr(self, modelname)
             self.logger.info(f"Loading all {modelname} records...")
             for instance in diffsync_model.nautobot_model().objects.all():
                 self.load_model(diffsync_model, instance)
 
-        self.logger.info("Fixing up any previously unresolved object relations...")
-        self.fixup_data_relations()
-
         self.logger.info("Data loading from Nautobot complete.")