From f29d50a5c1a93545911753dbdd182093608a7c05 Mon Sep 17 00:00:00 2001
From: Roald Teunissen <Roaldteunissen@gmail.com>
Date: Wed, 24 Aug 2022 11:27:45 +0000
Subject: [PATCH 01/21] Improved and documented pytests

---
 docs/how-to.md | 23 ++++++++++++++++++++++-
 pytest.ini     |  2 +-
 run-tests.sh   |  2 +-
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/docs/how-to.md b/docs/how-to.md
index e8315475..c1ee6e6c 100644
--- a/docs/how-to.md
+++ b/docs/how-to.md
@@ -1,4 +1,25 @@
 # How to use..
 ## Sphinx
 
-## Tests
+## Tests using pytest
+
+pytest.ini is called before running a pytest.
+This file specifies what django settings (ds) to use, which annotated to exclude using the mark (-m) keyword and other functions.
+
+#### Run all tests
+Important: make sure to be outside of an enviroment (deactivate).
+```
+./run-tests.sh
+```
+
+#### Run all test until one fails
+```
+pipenv run pytest -x
+```
+
+## Gitpod environment set up steps
+1. Make sure there is a branch available in the Github repository
+2. Go to the workspace overview in Gitpod of TGWF
+3. Run pre-build
+    Click on the pre-build option in the workspace overview of the workspace you want to prepare.
+4. After this preperation, open the workspace and it's ready to be used
\ No newline at end of file
diff --git a/pytest.ini b/pytest.ini
index 25c37836..8629049c 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -3,7 +3,7 @@
 ; so it can take precedent over any environment variables
 ; that might point DJANGO_SETTINGS_MODULE to development 
 ; or production, instead of the testings module
-addopts = --reuse-db --maxfail=0 -m "not smoke_test and not dramatiq and not flaky" --ds="greenweb.settings.testing"
+addopts = --create-db --maxfail=0 -m "not smoke_test and not dramatiq and not flaky" --ds="greenweb.settings.testing"
 python_files = tests.py test_*.py *_tests.py
 markers =
     only: Convenience method, so we can run a focussed test in pytest-watch
diff --git a/run-tests.sh b/run-tests.sh
index 08efada0..3e9f63ab 100755
--- a/run-tests.sh
+++ b/run-tests.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-pipenv run pytest
\ No newline at end of file
+pipenv run pytest 
\ No newline at end of file

From 43495964043b0a69f37be5053b2ae7ce14a40d50 Mon Sep 17 00:00:00 2001
From: Roald Teunissen <Roaldteunissen@gmail.com>
Date: Wed, 24 Aug 2022 14:45:23 +0000
Subject: [PATCH 02/21] WIP: Reading and validating a csv file

---
 apps/greencheck/fixtures/test_dataset_csv.csv |  3 +
 apps/greencheck/importers/importer_csv.py     | 65 ++++++++++++++++
 apps/greencheck/tests/test_importer_csv.py    | 75 +++++++++++++++++++
 3 files changed, 143 insertions(+)
 create mode 100644 apps/greencheck/fixtures/test_dataset_csv.csv
 create mode 100644 apps/greencheck/importers/importer_csv.py
 create mode 100644 apps/greencheck/tests/test_importer_csv.py

diff --git a/apps/greencheck/fixtures/test_dataset_csv.csv b/apps/greencheck/fixtures/test_dataset_csv.csv
new file mode 100644
index 00000000..d439671e
--- /dev/null
+++ b/apps/greencheck/fixtures/test_dataset_csv.csv
@@ -0,0 +1,3 @@
+31.15.8.0/21
+45.81.42.0/23
+81.95.96.0/20
diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
new file mode 100644
index 00000000..ef4c594e
--- /dev/null
+++ b/apps/greencheck/importers/importer_csv.py
@@ -0,0 +1,65 @@
+import requests
+import logging
+import pandas as pd
+import ipdb
+import re
+import ipaddress
+
+from apps.greencheck.importers.importer_interface import BaseImporter, Importer
+
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+
+class CsvImporter(BaseImporter):
+    # def __init__(cls):
+    # cls.hosting_provider_id = settings.Csv_PROVIDER_ID
+
+    def fetch_data_from_source(cls):
+        # TODO: fetch data from website dropping
+        # For now: expect that a csv is saved somewhere and fetch in the following way:
+        raw_data = pd.read_csv(
+            "apps/greencheck/fixtures/test_dataset_csv.csv", header=None
+        )
+        return cls.parse_to_list(raw_data)
+
+    def parse_to_list(cls, raw_data):
+        try:
+            list_of_ips = []
+
+            validate_csv_file(raw_data)
+
+            # Assume this single column holds ASN or network (ip with subnet)
+            if len(raw_data.columns) == 1:
+                list_of_ips = list(raw_data.iloc[:, 0])
+            # elif len(data.columns) == 2:
+            # start_ip = data.iloc[:,0]
+            # end_ip = data.iloc[:,1]
+            # Create network from these
+            # for index, row in data.iterrows():
+            #   list_of_ips.append(row[index,0])
+            #     print(row['c1'], row['c2'])
+            return list_of_ips
+        except Exception as e:
+            logger.exception("Something really unexpected happened. Aborting")
+
+    def validate_csv_file(cls, data):
+        # Loop through dataset to validate the data format
+        if len(data.columns) == 1:
+            validate_column_in_csv_file(data)
+        elif len(data.columns) == 2:
+            validate_column_in_csv_file(data.iloc[:, 0])
+            validate_column_in_csv_file(data.iloc[:, 1])
+        else:
+            # exception: wrong format
+            logger.exception("Number of columns in CSV are not as expected")
+
+    def validate_column_in_csv_file(cls, column):
+        column = list(column)
+        for address in column:
+            if not re.search("(AS)[0-9]+$", address) or not isinstance(
+                ipaddress.ip_network(address),
+                (ipaddress.IPv4Network, ipaddress.IPv6Network),
+            ):
+                logger.exception("Value of %s is in an incorrect format", address)
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
new file mode 100644
index 00000000..f6e8a875
--- /dev/null
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -0,0 +1,75 @@
+import pytest
+import pathlib
+import re
+import pandas as pd
+import ipdb
+from io import StringIO
+
+from django.core.management import call_command
+from apps.greencheck.importers.importer_csv import CsvImporter
+
+from django.conf import settings
+
+
+@pytest.fixture
+def sample_data_raw():
+    """
+    Retrieve a locally saved sample of the population to use for this test
+    Return: CSV
+    """
+    # this_file = pathlib.Path(__file__)
+    # csv_path = this_file.parent.parent.joinpath("fixtures", "test_dataset_csv.csv")
+    return pd.read_csv("apps/greencheck/fixtures/test_dataset_csv.csv", header=None)
+    # return pd.read_csv(csv_path, header=None)
+
+@pytest.fixture
+def sample_data_as_list(sample_data_raw):
+    """
+    Retrieve a locally saved sample of the population to use for this test and parse it to a list
+    Return: List
+    """
+    importer = CsvImporter()
+    return importer.parse_to_list(sample_data_raw)
+
+
+@pytest.mark.django_db
+class TestCsvImporter:
+    def test_parse_to_list(self, sample_data_raw):
+        """
+        Test the parsing function.
+        """
+        # Initialize Csv importer
+        importer = CsvImporter()
+
+        # Run parse list with sample data
+        list_of_addresses = importer.parse_to_list(sample_data_raw)
+        # ipdb.set_trace()
+
+        # Test: resulting list contains items
+        assert len(list_of_addresses) > 0
+
+@pytest.mark.django_db
+class TestCsvImportCommand:
+    """
+    This just tests that we have a management command that can run.
+    We _could_ mock the call to fetch ip ranges, if this turns out to be a slow test.
+    """
+
+    def test_handle(self, mocker, sample_data_as_list):
+        # mock the call to retrieve from source, to a locally stored
+        # testing sample. By instead using the test sample,
+        # we avoid unnecessary network requests.
+
+        # identify method we want to mock
+        path_to_mock = (
+            "apps.greencheck.importers.importer_csv."
+            "CsvImporter.fetch_data_from_source"
+        )
+
+        # define a different return when the targeted mock
+        # method is called
+        mocker.patch(
+            path_to_mock, return_value=sample_data_as_list,
+        )
+        
+        # call_command("update_networks_in_db_csv")

From 3a88e2853ba44fe00ca7a89ab6e2aa80645e6477 Mon Sep 17 00:00:00 2001
From: Roald Teunissen <Roaldteunissen@gmail.com>
Date: Thu, 25 Aug 2022 11:36:11 +0000
Subject: [PATCH 03/21] Fixed function call and provided some todo's

---
 apps/greencheck/importers/importer_csv.py  | 19 +++++++++----------
 apps/greencheck/tests/test_importer_csv.py | 22 +++++++++++++++-------
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index ef4c594e..a17ef070 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -28,12 +28,14 @@ def parse_to_list(cls, raw_data):
         try:
             list_of_ips = []
 
-            validate_csv_file(raw_data)
+            cls.validate_csv_file(raw_data)
 
-            # Assume this single column holds ASN or network (ip with subnet)
             if len(raw_data.columns) == 1:
+                # ASN or network (ip with subnet)
                 list_of_ips = list(raw_data.iloc[:, 0])
             # elif len(data.columns) == 2:
+            # TODO: Implement a way to convert two ip addresses to a network
+            # # Start and ending ip range
             # start_ip = data.iloc[:,0]
             # end_ip = data.iloc[:,1]
             # Create network from these
@@ -45,18 +47,15 @@ def parse_to_list(cls, raw_data):
             logger.exception("Something really unexpected happened. Aborting")
 
     def validate_csv_file(cls, data):
-        # Loop through dataset to validate the data format
-        if len(data.columns) == 1:
-            validate_column_in_csv_file(data)
-        elif len(data.columns) == 2:
-            validate_column_in_csv_file(data.iloc[:, 0])
-            validate_column_in_csv_file(data.iloc[:, 1])
+        if len(data.columns) == 1 or len(data.columns) == 2:
+            cls.validate_column_in_csv_file(data.iloc[:, 0].values.tolist())
+
+            if len(data.columns) == 2:
+                cls.validate_column_in_csv_file(data.iloc[:, 1].values.tolist())
         else:
-            # exception: wrong format
             logger.exception("Number of columns in CSV are not as expected")
 
     def validate_column_in_csv_file(cls, column):
-        column = list(column)
         for address in column:
             if not re.search("(AS)[0-9]+$", address) or not isinstance(
                 ipaddress.ip_network(address),
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index f6e8a875..50d0d8d9 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -17,10 +17,10 @@ def sample_data_raw():
     Retrieve a locally saved sample of the population to use for this test
     Return: CSV
     """
-    # this_file = pathlib.Path(__file__)
-    # csv_path = this_file.parent.parent.joinpath("fixtures", "test_dataset_csv.csv")
-    return pd.read_csv("apps/greencheck/fixtures/test_dataset_csv.csv", header=None)
-    # return pd.read_csv(csv_path, header=None)
+    this_file = pathlib.Path(__file__)
+    csv_path = this_file.parent.parent.joinpath("fixtures", "test_dataset_csv.csv")
+    return pd.read_csv(csv_path, header=None)
+
 
 @pytest.fixture
 def sample_data_as_list(sample_data_raw):
@@ -43,11 +43,17 @@ def test_parse_to_list(self, sample_data_raw):
 
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
-        # ipdb.set_trace()
 
         # Test: resulting list contains items
         assert len(list_of_addresses) > 0
 
+    # def test_validate_csv_file(self):
+    #     # TODO: Implement this test
+
+    # def test_validate_column_in_csv_file(self):
+    #     # TODO: Implement this test
+
+
 @pytest.mark.django_db
 class TestCsvImportCommand:
     """
@@ -69,7 +75,9 @@ def test_handle(self, mocker, sample_data_as_list):
         # define a different return when the targeted mock
         # method is called
         mocker.patch(
-            path_to_mock, return_value=sample_data_as_list,
+            path_to_mock,
+            return_value=sample_data_as_list,
         )
-        
+
+        # TODO: Do we need this call command?
         # call_command("update_networks_in_db_csv")

From 043929b2f50c3e00a134dee3b42cf44db895cf67 Mon Sep 17 00:00:00 2001
From: Roald Teunissen <Roaldteunissen@gmail.com>
Date: Thu, 25 Aug 2022 14:34:47 +0000
Subject: [PATCH 04/21] WIP: Included bulk import files, creating a csv upload
 page

---
 Pipfile                                       |   1 +
 Pipfile.lock                                  | 512 +++++++++---------
 apps/accounts/admin_site.py                   |  14 +
 .../templates/import_csv_preview.html         |  79 +++
 .../templates/import_csv_results.html         |  42 ++
 apps/accounts/templates/import_csv_start.html |  76 +++
 apps/accounts/templates/import_ip_ranges.html |  86 +++
 apps/greencheck/forms.py                      |  60 ++
 makefile                                      |   2 +
 9 files changed, 623 insertions(+), 249 deletions(-)
 create mode 100644 apps/accounts/templates/import_csv_preview.html
 create mode 100644 apps/accounts/templates/import_csv_results.html
 create mode 100644 apps/accounts/templates/import_csv_start.html
 create mode 100644 apps/accounts/templates/import_ip_ranges.html

diff --git a/Pipfile b/Pipfile
index 5cd21a0d..4204a712 100644
--- a/Pipfile
+++ b/Pipfile
@@ -76,6 +76,7 @@ python-dateutil = "*"
 geoip2 = "*"
 iso3166 = "*"
 django-logentry-admin = "*"
+pandas = "*"
 
 [pipenv]
 # Needed for `black`. See https://github.com/microsoft/vscode-python/pull/5967.
diff --git a/Pipfile.lock b/Pipfile.lock
index 342dcd3e..fad5958a 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "01b663466247d744bc35c06abaec815b56696bcda8af2397389b584cb61cca96"
+            "sha256": "86b55b33255e8df8b5e8ac87462ef278ca359396df92e4f8636a58dce7d23f7d"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -126,35 +126,36 @@
         },
         "bcrypt": {
             "hashes": [
-                "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521",
-                "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb",
-                "sha256:4e029cef560967fb0cf4a802bcf4d562d3d6b4b1bf81de5ec1abbe0f1adb027e",
-                "sha256:61bae49580dce88095d669226d5076d0b9d927754cedbdf76c6c9f5099ad6f26",
-                "sha256:6d2cb9d969bfca5bc08e45864137276e4c3d3d7de2b162171def3d188bf9d34a",
-                "sha256:7180d98a96f00b1050e93f5b0f556e658605dd9f524d0b0e68ae7944673f525e",
-                "sha256:7d9ba2e41e330d2af4af6b1b6ec9e6128e91343d0b4afb9282e54e5508f31baa",
-                "sha256:7ff2069240c6bbe49109fe84ca80508773a904f5a8cb960e02a977f7f519b129",
-                "sha256:88273d806ab3a50d06bc6a2fc7c87d737dd669b76ad955f449c43095389bc8fb",
-                "sha256:a2c46100e315c3a5b90fdc53e429c006c5f962529bc27e1dfd656292c20ccc40",
-                "sha256:cd43303d6b8a165c29ec6756afd169faba9396a9472cdff753fe9f19b96ce2fa"
-            ],
-            "version": "==3.2.2"
+                "sha256:0b0f0c7141622a31e9734b7f649451147c04ebb5122327ac0bd23744df84be90",
+                "sha256:1c3334446fac200499e8bc04a530ce3cf0b3d7151e0e4ac5c0dddd3d95e97843",
+                "sha256:2d0dd19aad87e4ab882ef1d12df505f4c52b28b69666ce83c528f42c07379227",
+                "sha256:594780b364fb45f2634c46ec8d3e61c1c0f1811c4f2da60e8eb15594ecbf93ed",
+                "sha256:7c7dd6c1f05bf89e65261d97ac3a6520f34c2acb369afb57e3ea4449be6ff8fd",
+                "sha256:845b1daf4df2dd94d2fdbc9454953ca9dd0e12970a0bfc9f3dcc6faea3fa96e4",
+                "sha256:8780e69f9deec9d60f947b169507d2c9816e4f11548f1f7ebee2af38b9b22ae4",
+                "sha256:bf413f2a9b0a2950fc750998899013f2e718d20fa4a58b85ca50b6df5ed1bbf9",
+                "sha256:bfb67f6a6c72dfb0a02f3df51550aa1862708e55128b22543e2b42c74f3620d7",
+                "sha256:c59c170fc9225faad04dde1ba61d85b413946e8ce2e5f5f5ff30dfd67283f319",
+                "sha256:dc6ec3dc19b1c193b2f7cf279d3e32e7caf447532fbcb7af0906fe4398900c33",
+                "sha256:ede0f506554571c8eda80db22b83c139303ec6b595b8f60c4c8157bdd0bdee36"
+            ],
+            "version": "==4.0.0"
         },
         "boto3": {
             "hashes": [
-                "sha256:44026e44549148dbc5b261ead5f6b339e785680c350ef621bf85f7e2fca05b49",
-                "sha256:b2d9d55f123a9a91eea2fd8e379d90abf37634420fbb45c22d67e10b324ec71b"
+                "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575",
+                "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"
             ],
             "index": "pypi",
-            "version": "==1.24.46"
+            "version": "==1.24.59"
         },
         "botocore": {
             "hashes": [
-                "sha256:747b7e94aef41498f063fc0be79c5af102d940beea713965179e1ead89c7e9ec",
-                "sha256:f66d8305d1f59d83334df9b11b6512bb1e14698ec4d5d6d42f833f39f3304ca7"
+                "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da",
+                "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.27.46"
+            "version": "==1.27.59"
         },
         "brotli": {
             "hashes": [
@@ -231,82 +232,13 @@
             "markers": "python_version >= '3.6'",
             "version": "==2022.6.15"
         },
-        "cffi": {
-            "hashes": [
-                "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5",
-                "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef",
-                "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104",
-                "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426",
-                "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405",
-                "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375",
-                "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a",
-                "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e",
-                "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc",
-                "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf",
-                "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185",
-                "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497",
-                "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3",
-                "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35",
-                "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c",
-                "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83",
-                "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21",
-                "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca",
-                "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984",
-                "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac",
-                "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd",
-                "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee",
-                "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a",
-                "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2",
-                "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192",
-                "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7",
-                "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585",
-                "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f",
-                "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e",
-                "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27",
-                "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b",
-                "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e",
-                "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e",
-                "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d",
-                "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c",
-                "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415",
-                "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82",
-                "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02",
-                "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314",
-                "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325",
-                "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c",
-                "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3",
-                "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914",
-                "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045",
-                "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d",
-                "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9",
-                "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5",
-                "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2",
-                "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c",
-                "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3",
-                "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2",
-                "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8",
-                "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d",
-                "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d",
-                "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9",
-                "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162",
-                "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76",
-                "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4",
-                "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e",
-                "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9",
-                "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6",
-                "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b",
-                "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01",
-                "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"
-            ],
-            "version": "==1.15.1"
-        },
         "charset-normalizer": {
             "hashes": [
-                "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5",
-                "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"
+                "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
+                "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.1.0"
+            "version": "==2.1.1"
         },
         "click": {
             "hashes": [
@@ -471,11 +403,11 @@
         },
         "django-mysql": {
             "hashes": [
-                "sha256:13d640413680b57c0ea3a49c4a0a9d1ac2b4a6e679baa5ac299dd45f3880a583",
-                "sha256:26ba087d9163b411d404d44bc53dfbb4b321b4692eee3061a373040e9482000e"
+                "sha256:ec809868113c652930874c055dd59baf93b3ba8c6d7e90bd891e28db15a4c840",
+                "sha256:f66b02fabd48f858b9fd70e57f4d66f1c6c807047a2eec8b7dce2b0aa1dcd4c1"
             ],
             "index": "pypi",
-            "version": "==4.7.0"
+            "version": "==4.7.1"
         },
         "django-redis": {
             "hashes": [
@@ -495,11 +427,11 @@
         },
         "django-storages": {
             "hashes": [
-                "sha256:204a99f218b747c46edbfeeb1310d357f83f90fa6a6024d8d0a3f422570cee84",
-                "sha256:a475edb2f0f04c4f7e548919a751ecd50117270833956ed5bd585c0575d2a5e7"
+                "sha256:3540b45618b04be2c867c0982e8d2bd8e34f84dae922267fcebe4691fb93daf0",
+                "sha256:b3d98ecc09f1b1627c2b2cf430964322ce4e08617dbf9b4236c16a32875a1e0b"
             ],
             "index": "pypi",
-            "version": "==1.12.3"
+            "version": "==1.13.1"
         },
         "django-taggit": {
             "hashes": [
@@ -542,11 +474,11 @@
         },
         "django-waffle": {
             "hashes": [
-                "sha256:150c0867d0f30c650a5c1e59a56b164780b62ab3bd48c4c65b6ae3481cf9dcfa",
-                "sha256:b0fef2cab15e739cadaffaf5b71206e813604bad093dc547959f1e3ee76c556d"
+                "sha256:3f4bd5e265c27ff3346b8379f5165a43e0791c515c8c08e7a94faf6281c758db",
+                "sha256:da1b76d4ca42b5c59c798ffec022a6209eb3dd2eb8fe0ce30da6dd9d9a5a5e24"
             ],
             "index": "pypi",
-            "version": "==2.6.0"
+            "version": "==2.7.0"
         },
         "django-widget-tweaks": {
             "hashes": [
@@ -949,6 +881,40 @@
             "index": "pypi",
             "version": "==2.1.1"
         },
+        "numpy": {
+            "hashes": [
+                "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0",
+                "sha256:2bd879d3ca4b6f39b7770829f73278b7c5e248c91d538aab1e506c628353e47f",
+                "sha256:4f41f5bf20d9a521f8cab3a34557cd77b6f205ab2116651f12959714494268b0",
+                "sha256:5593f67e66dea4e237f5af998d31a43e447786b2154ba1ad833676c788f37cde",
+                "sha256:5e28cd64624dc2354a349152599e55308eb6ca95a13ce6a7d5679ebff2962913",
+                "sha256:633679a472934b1c20a12ed0c9a6c9eb167fbb4cb89031939bfd03dd9dbc62b8",
+                "sha256:806970e69106556d1dd200e26647e9bee5e2b3f1814f9da104a943e8d548ca38",
+                "sha256:806cc25d5c43e240db709875e947076b2826f47c2c340a5a2f36da5bb10c58d6",
+                "sha256:8247f01c4721479e482cc2f9f7d973f3f47810cbc8c65e38fd1bbd3141cc9842",
+                "sha256:8ebf7e194b89bc66b78475bd3624d92980fca4e5bb86dda08d677d786fefc414",
+                "sha256:8ecb818231afe5f0f568c81f12ce50f2b828ff2b27487520d85eb44c71313b9e",
+                "sha256:8f9d84a24889ebb4c641a9b99e54adb8cab50972f0166a3abc14c3b93163f074",
+                "sha256:909c56c4d4341ec8315291a105169d8aae732cfb4c250fbc375a1efb7a844f8f",
+                "sha256:9b83d48e464f393d46e8dd8171687394d39bc5abfe2978896b77dc2604e8635d",
+                "sha256:ac987b35df8c2a2eab495ee206658117e9ce867acf3ccb376a19e83070e69418",
+                "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01",
+                "sha256:b8b97a8a87cadcd3f94659b4ef6ec056261fa1e1c3317f4193ac231d4df70215",
+                "sha256:bd5b7ccae24e3d8501ee5563e82febc1771e73bd268eef82a1e8d2b4d556ae66",
+                "sha256:bdc02c0235b261925102b1bd586579b7158e9d0d07ecb61148a1799214a4afd5",
+                "sha256:be6b350dfbc7f708d9d853663772a9310783ea58f6035eec649fb9c4371b5389",
+                "sha256:c403c81bb8ffb1c993d0165a11493fd4bf1353d258f6997b3ee288b0a48fce77",
+                "sha256:cf8c6aed12a935abf2e290860af8e77b26a042eb7f2582ff83dc7ed5f963340c",
+                "sha256:d98addfd3c8728ee8b2c49126f3c44c703e2b005d4a95998e2167af176a9e722",
+                "sha256:dc76bca1ca98f4b122114435f83f1fcf3c0fe48e4e6f660e07996abf2f53903c",
+                "sha256:dec198619b7dbd6db58603cd256e092bcadef22a796f778bf87f8592b468441d",
+                "sha256:df28dda02c9328e122661f399f7655cdcbcf22ea42daa3650a26bce08a187450",
+                "sha256:e603ca1fb47b913942f3e660a15e55a9ebca906857edfea476ae5f0fe9b457d5",
+                "sha256:ecfdd68d334a6b97472ed032b5b37a30d8217c097acfff15e8452c710e775524"
+            ],
+            "markers": "python_version < '3.10'",
+            "version": "==1.23.2"
+        },
         "packaging": {
             "hashes": [
                 "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
@@ -957,6 +923,33 @@
             "markers": "python_version >= '3.6'",
             "version": "==21.3"
         },
+        "pandas": {
+            "hashes": [
+                "sha256:0329e22df7e85fd4cd4df5d186fad200fa879abc62583a2b0fa5919032b3e7d5",
+                "sha256:0a3cfd8777f3eb9fa6df12ea6f4aea338a9205507eb12d8dca3b3f00cf4ffe17",
+                "sha256:1002500cd99fbd00b4421f252ee7f8be5b6aabfeff29d5a744ec73ba3a917beb",
+                "sha256:1db16e716aac37e0d77278526c59027bdd02701994586912d83f4bf10ef80a06",
+                "sha256:1e2b9186b4d960043c47f62aeb489d88f74bfa574b486d4d2fd52bf496f94c23",
+                "sha256:4517b8f426b152620143ada0059e3a4142802f9582372885eb56a509283ca216",
+                "sha256:53400419ae6d9cec1cc8df719c0cbf7f76da9afd1725a1428993b112adb7f268",
+                "sha256:61fbe9e0e638a606ad47e362af5d8c71e9d33e4e5824e5b8d3811cdb45aabc89",
+                "sha256:6b842469bbdf1e7733c70b4e9bceee90ff8f421ad871cefb09d7e4d326557112",
+                "sha256:774c07952f1da5c8813ca082744161797dfd05c9bd16f0afae8c9774e82e06e6",
+                "sha256:7c0bd042e4d43c185f21a5616b98eaa0f5370ac816e633949a2c3ce922d35c0f",
+                "sha256:a9a8b27f08c7fe9b9bdb8631e663af140cee0deeac102dee84c867633b460e2a",
+                "sha256:b0cb459061105bc0927ebba493cb4cb745e43a18532b92e49000c4f34ea30eb9",
+                "sha256:bc626256a6d430e96b6e47208c0e5f9ad0403558a2237873e68ec37ce321bf70",
+                "sha256:bf99d2ba08dd476c3adf2baaa6e6ed16f78a35832146555260c08915e0633e3f",
+                "sha256:c0df2515125d83489521e1edfebe7cab2276c9b2984a21dcc5e8ef563e4b8e0d",
+                "sha256:c8867fc7722fc5f10c61ee7e8b76295c41d20d0f26b1f7341837476efca66601",
+                "sha256:cba004d0d33a28722c89b41512d90398f8041d0c24492f794863db1ddcc7932f",
+                "sha256:dac53ad9e6c45e1dae2263a6789aab79aa3b24b8974760ce22febe05b0b63865",
+                "sha256:f972af3232d41f6c755682f3f11ba0e5f0cf546a7eb6b3beba36677c5fcb2ca0",
+                "sha256:f9f763d6a88c21690a5cad79c6f3074c3b71e04f8d949edf1970d27f0d746cf5"
+            ],
+            "index": "pypi",
+            "version": "==1.5.0rc0"
+        },
         "phpserialize": {
             "hashes": [
                 "sha256:bf672d312d203d09a84c26366fab8f438a3ffb355c407e69974b7ef2d39a0fa7"
@@ -1043,13 +1036,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.14.1"
         },
-        "pycparser": {
-            "hashes": [
-                "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
-                "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
-            ],
-            "version": "==2.21"
-        },
         "pyparsing": {
             "hashes": [
                 "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
@@ -1068,10 +1054,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7",
-                "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"
+                "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197",
+                "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5"
             ],
-            "version": "==2022.1"
+            "version": "==2022.2.1"
         },
         "pyyaml": {
             "hashes": [
@@ -1138,13 +1124,17 @@
         },
         "ruamel.yaml.clib": {
             "hashes": [
+                "sha256:066f886bc90cc2ce44df8b5f7acfc6a7e2b2e672713f027136464492b0c34d7c",
                 "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd",
                 "sha256:1070ba9dd7f9370d0513d649420c3b362ac2d687fe78c6e888f5b12bf8bc7bee",
                 "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0",
+                "sha256:1b4139a6ffbca8ef60fdaf9b33dec05143ba746a6f0ae0f9d11d38239211d335",
+                "sha256:210c8fcfeff90514b7133010bf14e3bad652c8efde6b20e00c43854bf94fa5a6",
                 "sha256:221eca6f35076c6ae472a531afa1c223b9c29377e62936f61bc8e6e8bdc5f9e7",
                 "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277",
                 "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104",
                 "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd",
+                "sha256:61bc5e5ca632d95925907c569daa559ea194a4d16084ba86084be98ab1cec1c6",
                 "sha256:6e7be2c5bcb297f5b82fee9c665eb2eb7001d1050deaba8471842979293a80b0",
                 "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78",
                 "sha256:77df077d32921ad46f34816a9a16e6356d8100374579bc35e15bab5d4e9377de",
@@ -1160,6 +1150,7 @@
                 "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5",
                 "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe",
                 "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751",
+                "sha256:d3c620a54748a3d4cf0bcfe623e388407c8e85a4b06b8188e126302bcab93ea8",
                 "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502",
                 "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed",
                 "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"
@@ -1177,11 +1168,11 @@
         },
         "sentry-sdk": {
             "hashes": [
-                "sha256:60b13757d6344a94bf0ccb3c0a006c4de77daab09871b30fbbd05d5ec24e54fb",
-                "sha256:f185c53496d79b280fe5d9d21e6572aee1ab802d3354eb12314d216cfbaa8d30"
+                "sha256:2d7ec7bc88ebbdf2c4b6b2650b3257893d386325a96c9b723adcd31033469b63",
+                "sha256:b4b41f90951ed83e7b4c176eef021b19ecba39da5b73aca106c97a9b7e279a90"
             ],
             "index": "pypi",
-            "version": "==1.9.0"
+            "version": "==1.9.5"
         },
         "six": {
             "hashes": [
@@ -1193,45 +1184,45 @@
         },
         "sqlalchemy": {
             "hashes": [
-                "sha256:047ef5ccd8860f6147b8ac6c45a4bc573d4e030267b45d9a1c47b55962ff0e6f",
-                "sha256:05a05771617bfa723ba4cef58d5b25ac028b0d68f28f403edebed5b8243b3a87",
-                "sha256:0ec54460475f0c42512895c99c63d90dd2d9cbd0c13491a184182e85074b04c5",
-                "sha256:107df519eb33d7f8e0d0d052128af2f25066c1a0f6b648fd1a9612ab66800b86",
-                "sha256:14ea8ff2d33c48f8e6c3c472111d893b9e356284d1482102da9678195e5a8eac",
-                "sha256:1745987ada1890b0e7978abdb22c133eca2e89ab98dc17939042240063e1ef21",
-                "sha256:1962dfee37b7fb17d3d4889bf84c4ea08b1c36707194c578f61e6e06d12ab90f",
-                "sha256:20bf65bcce65c538e68d5df27402b39341fabeecf01de7e0e72b9d9836c13c52",
-                "sha256:26146c59576dfe9c546c9f45397a7c7c4a90c25679492ff610a7500afc7d03a6",
-                "sha256:365b75938049ae31cf2176efd3d598213ddb9eb883fbc82086efa019a5f649df",
-                "sha256:4770eb3ba69ec5fa41c681a75e53e0e342ac24c1f9220d883458b5596888e43a",
-                "sha256:50e7569637e2e02253295527ff34666706dbb2bc5f6c61a5a7f44b9610c9bb09",
-                "sha256:5c2d19bfb33262bf987ef0062345efd0f54c4189c2d95159c72995457bf4a359",
-                "sha256:621f050e72cc7dfd9ad4594ff0abeaad954d6e4a2891545e8f1a53dcdfbef445",
-                "sha256:6d81de54e45f1d756785405c9d06cd17918c2eecc2d4262dc2d276ca612c2f61",
-                "sha256:6f95706da857e6e79b54c33c1214f5467aab10600aa508ddd1239d5df271986e",
-                "sha256:752ef2e8dbaa3c5d419f322e3632f00ba6b1c3230f65bc97c2ff5c5c6c08f441",
-                "sha256:7b2785dd2a0c044a36836857ac27310dc7a99166253551ee8f5408930958cc60",
-                "sha256:7f13644b15665f7322f9e0635129e0ef2098409484df67fcd225d954c5861559",
-                "sha256:8194896038753b46b08a0b0ae89a5d80c897fb601dd51e243ed5720f1f155d27",
-                "sha256:864d4f89f054819cb95e93100b7d251e4d114d1c60bc7576db07b046432af280",
-                "sha256:8b773c9974c272aae0fa7e95b576d98d17ee65f69d8644f9b6ffc90ee96b4d19",
-                "sha256:8f901be74f00a13bf375241a778455ee864c2c21c79154aad196b7a994e1144f",
-                "sha256:91d2b89bb0c302f89e753bea008936acfa4e18c156fb264fe41eb6bbb2bbcdeb",
-                "sha256:b0538b66f959771c56ff996d828081908a6a52a47c5548faed4a3d0a027a5368",
-                "sha256:b30e70f1594ee3c8902978fd71900d7312453922827c4ce0012fa6a8278d6df4",
-                "sha256:b71be98ef6e180217d1797185c75507060a57ab9cd835653e0112db16a710f0d",
-                "sha256:c6d00cb9da8d0cbfaba18cad046e94b06de6d4d0ffd9d4095a3ad1838af22528",
-                "sha256:d1f665e50592caf4cad3caed3ed86f93227bffe0680218ccbb293bd5a6734ca8",
-                "sha256:e6e2c8581c6620136b9530137954a8376efffd57fe19802182c7561b0ab48b48",
-                "sha256:e7a7667d928ba6ee361a3176e1bef6847c1062b37726b33505cc84136f657e0d",
-                "sha256:ec3985c883d6d217cf2013028afc6e3c82b8907192ba6195d6e49885bfc4b19d",
-                "sha256:ede13a472caa85a13abe5095e71676af985d7690eaa8461aeac5c74f6600b6c0",
-                "sha256:f24d4d6ec301688c59b0c4bb1c1c94c5d0bff4ecad33bb8f5d9efdfb8d8bc925",
-                "sha256:f2a42acc01568b9701665e85562bbff78ec3e21981c7d51d56717c22e5d3d58b",
-                "sha256:fbc076f79d830ae4c9d49926180a1140b49fa675d0f0d555b44c9a15b29f4c80"
+                "sha256:00dd998b43b282c71de46b061627b5edb9332510eb1edfc5017b9e4356ed44ea",
+                "sha256:08b47c971327e733ffd6bae2d4f50a7b761793efe69d41067fcba86282819eea",
+                "sha256:0992f3cc640ec0f88f721e426da884c34ff0a60eb73d3d64172e23dfadfc8a0b",
+                "sha256:0c956a5d1adb49a35d78ef0fae26717afc48a36262359bb5b0cbd7a3a247c26f",
+                "sha256:1ab08141d93de83559f6a7d9a962830f918623a885b3759ec2b9d1a531ff28fe",
+                "sha256:1cf03d37819dc17a388d313919daf32058d19ba1e592efdf14ce8cbd997e6023",
+                "sha256:2026632051a93997cf8f6fda14360f99230be1725b7ab2ef15be205a4b8a5430",
+                "sha256:23b693876ac7963b6bc7b1a5f3a2642f38d2624af834faad5933913928089d1b",
+                "sha256:26ee4dbac5dd7abf18bf3cd8f04e51f72c339caf702f68172d308888cd26c6c9",
+                "sha256:28b1791a30d62fc104070965f1a2866699c45bbf5adc0be0cf5f22935edcac58",
+                "sha256:2b64955850a14b9d481c17becf0d3f62fb1bb31ac2c45c2caf5ad06d9e811187",
+                "sha256:2cf50611ef4221ad587fb7a1708e61ff72966f84330c6317642e08d6db4138fd",
+                "sha256:44a660506080cc975e1dfa5776fe5f6315ddc626a77b50bf0eee18b0389ea265",
+                "sha256:4ec440990ab00650d0c7ea2c75bc225087afdd7ddcb248e3d934def4dff62762",
+                "sha256:63ad778f4e80913fb171247e4fa82123d0068615ae1d51a9791fc4284cb81748",
+                "sha256:69deec3a94de10062080d91e1ba69595efeafeafe68b996426dec9720031fb25",
+                "sha256:6b70d02bbe1adbbf715d2249cacf9ac17c6f8d22dfcb3f1a4fbc5bf64364da8a",
+                "sha256:885e11638946472b4a0a7db8e6df604b2cf64d23dc40eedc3806d869fcb18fae",
+                "sha256:959bf4390766a8696aa01285016c766b4eb676f712878aac5fce956dd49695d9",
+                "sha256:9ced2450c9fd016f9232d976661623e54c450679eeefc7aa48a3d29924a63189",
+                "sha256:a0b9e3d81f86ba04007f0349e373a5b8c81ec2047aadb8d669caf8c54a092461",
+                "sha256:a62c0ecbb9976550f26f7bf75569f425e661e7249349487f1483115e5fc893a6",
+                "sha256:b07fc38e6392a65935dc8b486229679142b2ea33c94059366b4d8b56f1e35a97",
+                "sha256:b41b87b929118838bafc4bb18cf3c5cd1b3be4b61cd9042e75174df79e8ac7a2",
+                "sha256:b7ccdca6cd167611f4a62a8c2c0c4285c2535640d77108f782ce3f3cccb70f3a",
+                "sha256:b7ff0a8bf0aec1908b92b8dfa1246128bf4f94adbdd3da6730e9c542e112542d",
+                "sha256:bb342c0e25cc8f78a0e7c692da3b984f072666b316fbbec2a0e371cb4dfef5f0",
+                "sha256:bf073c619b5a7f7cd731507d0fdc7329bee14b247a63b0419929e4acd24afea8",
+                "sha256:c8d974c991eef0cd29418a5957ae544559dc326685a6f26b3a914c87759bf2f4",
+                "sha256:c9d0f1a9538cc5e75f2ea0cb6c3d70155a1b7f18092c052e0d84105622a41b63",
+                "sha256:cdee4d475e35684d210dc6b430ff8ca2ed0636378ac19b457e2f6f350d1f5acc",
+                "sha256:cfa8ab4ba0c97ab6bcae1f0948497d14c11b6c6ecd1b32b8a79546a0823d8211",
+                "sha256:d259fa08e4b3ed952c01711268bcf6cd2442b0c54866d64aece122f83da77c6d",
+                "sha256:f2aa85aebc0ef6b342d5d3542f969caa8c6a63c8d36cf5098769158a9fa2123c",
+                "sha256:fa9e0d7832b7511b3b3fd0e67fac85ff11fd752834c143ca2364c9b778c0485a",
+                "sha256:fb4edb6c354eac0fcc07cb91797e142f702532dbb16c1d62839d6eec35f814cf"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.4.39"
+            "version": "==1.4.40"
         },
         "sqlite-fts4": {
             "hashes": [
@@ -1310,11 +1301,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc",
-                "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"
+                "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
+                "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
-            "version": "==1.26.11"
+            "markers": "python_version >= '3.5'",
+            "version": "==1.26.12"
         },
         "whitenoise": {
             "extras": [
@@ -1479,14 +1470,6 @@
             ],
             "version": "==0.7.12"
         },
-        "appnope": {
-            "hashes": [
-                "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24",
-                "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"
-            ],
-            "markers": "sys_platform == 'darwin'",
-            "version": "==0.1.3"
-        },
         "asgiref": {
             "hashes": [
                 "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4",
@@ -1505,10 +1488,10 @@
         },
         "asttokens": {
             "hashes": [
-                "sha256:0844691e88552595a6f4a4281a9f7f79b8dd45ca4ccea82e5e05b4bbdb76705c",
-                "sha256:9a54c114f02c7a9480d56550932546a3f1fe71d8a02f1bc7ccd0ee3ee35cf4d5"
+                "sha256:c61e16246ecfb2cde2958406b4c8ebc043c9e6d73aaa83c941673b35e5d3a76b",
+                "sha256:e3305297c744ae53ffa032c45dc347286165e4ffce6875dc662b205db0623d86"
             ],
-            "version": "==2.0.5"
+            "version": "==2.0.8"
         },
         "attrs": {
             "hashes": [
@@ -1533,6 +1516,28 @@
             ],
             "version": "==0.2.0"
         },
+        "backports.zoneinfo": {
+            "hashes": [
+                "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf",
+                "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328",
+                "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546",
+                "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6",
+                "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570",
+                "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9",
+                "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7",
+                "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987",
+                "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722",
+                "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582",
+                "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc",
+                "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b",
+                "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1",
+                "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08",
+                "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac",
+                "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"
+            ],
+            "markers": "python_version < '3.9'",
+            "version": "==0.2.1"
+        },
         "beautifulsoup4": {
             "hashes": [
                 "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
@@ -1580,11 +1585,11 @@
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5",
-                "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"
+                "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
+                "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.1.0"
+            "version": "==2.1.1"
         },
         "click": {
             "hashes": [
@@ -1623,50 +1628,59 @@
                 "toml"
             ],
             "hashes": [
-                "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32",
-                "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7",
-                "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996",
-                "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55",
-                "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46",
-                "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de",
-                "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039",
-                "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee",
-                "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1",
-                "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f",
-                "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63",
-                "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083",
-                "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe",
-                "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0",
-                "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6",
-                "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe",
-                "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933",
-                "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0",
-                "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c",
-                "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07",
-                "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8",
-                "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b",
-                "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e",
-                "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120",
-                "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f",
-                "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e",
-                "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd",
-                "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f",
-                "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386",
-                "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8",
-                "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae",
-                "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc",
-                "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783",
-                "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d",
-                "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c",
-                "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97",
-                "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978",
-                "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf",
-                "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29",
-                "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39",
-                "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452"
-            ],
-            "index": "pypi",
-            "version": "==6.4.2"
+                "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2",
+                "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820",
+                "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827",
+                "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3",
+                "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d",
+                "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145",
+                "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875",
+                "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2",
+                "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74",
+                "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f",
+                "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c",
+                "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973",
+                "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1",
+                "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782",
+                "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0",
+                "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760",
+                "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a",
+                "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3",
+                "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7",
+                "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a",
+                "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f",
+                "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e",
+                "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86",
+                "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa",
+                "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa",
+                "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796",
+                "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a",
+                "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928",
+                "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0",
+                "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac",
+                "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c",
+                "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685",
+                "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d",
+                "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e",
+                "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f",
+                "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558",
+                "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58",
+                "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781",
+                "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a",
+                "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa",
+                "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc",
+                "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892",
+                "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d",
+                "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817",
+                "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1",
+                "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c",
+                "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908",
+                "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19",
+                "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60",
+                "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"
+            ],
+            "index": "pypi",
+            "version": "==6.4.4"
         },
         "decorator": {
             "hashes": [
@@ -1697,11 +1711,11 @@
         },
         "django-debug-toolbar": {
             "hashes": [
-                "sha256:89a52128309eb4da12738801ff0c202d2ff8730d1c3225fac6acf630c303e661",
-                "sha256:97965f2630692de316ea0c1ca5bfa81660d7ba13146dbc6be2059cf55b35d0e5"
+                "sha256:95fc2fd29c56cc86678aae9f6919ececefe892f2a78c4004b193a223a8380c3d",
+                "sha256:fe7fe3f21865218827e2162ecc06eba386dfe8cffe4f3501c49bb4359e06a0e6"
             ],
             "index": "pypi",
-            "version": "==3.5.0"
+            "version": "==3.6.0"
         },
         "docutils": {
             "hashes": [
@@ -1729,10 +1743,10 @@
         },
         "executing": {
             "hashes": [
-                "sha256:4ce4d6082d99361c0231fc31ac1a0f56979363cc6819de0b1410784f99e49105",
-                "sha256:ea278e2cf90cbbacd24f1080dd1f0ac25b71b2e21f50ab439b7ba45dd3195587"
+                "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e",
+                "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"
             ],
-            "version": "==0.9.1"
+            "version": "==0.10.0"
         },
         "factory-boy": {
             "hashes": [
@@ -1744,11 +1758,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:172e45220b7a46743f4fb58cf380adb306d5c3ab1c0b0d97062508474cec5ff8",
-                "sha256:7c3f8ee807d3916415568169a172bf0893ea9cc3371ab55e4e5f5170d2185bea"
+                "sha256:067a03f64e555261610e69277536072997b4576dbf84b113faef3c06d85b466b",
+                "sha256:0e00bfa1eadf1493f15662edb181222fea4847764cf3f9ff3e66ee0f95c9a644"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==13.15.1"
+            "version": "==14.1.0"
         },
         "flake8": {
             "hashes": [
@@ -1768,11 +1782,11 @@
         },
         "hypothesis": {
             "hashes": [
-                "sha256:b60b59d36e76626a38cf893d70d0c3c39c5fa02c6c5895966ff5fead8fb9ff0a",
-                "sha256:de63c34309181875e71d0f5d1c1051c9320a1fe0517ea6733af8cedf818191f4"
+                "sha256:469125235b4fdd6b6a4b6a6fa13e528d3e8de4dbd3bd9236b03323fefa3a9b32",
+                "sha256:ee42fe4d2ff96c49910085780d6b8f34cbcf4c44427616e22833869d451116bb"
             ],
             "index": "pypi",
-            "version": "==6.54.1"
+            "version": "==6.54.4"
         },
         "idna": {
             "hashes": [
@@ -1833,7 +1847,7 @@
                 "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7",
                 "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"
             ],
-            "markers": "python_version < '4' and python_full_version >= '3.6.1'",
+            "markers": "python_version < '4.0' and python_full_version >= '3.6.1'",
             "version": "==5.10.1"
         },
         "jedi": {
@@ -1957,11 +1971,11 @@
         },
         "matplotlib-inline": {
             "hashes": [
-                "sha256:a04bfba22e0d1395479f866853ec1ee28eea1485c1d69a6faf00dc3e24ff34ee",
-                "sha256:aed605ba3b72462d64d475a21a9296f400a19c4f74a31b59103d2a99ffd5aa5c"
+                "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311",
+                "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==0.1.3"
+            "version": "==0.1.6"
         },
         "mccabe": {
             "hashes": [
@@ -1981,11 +1995,11 @@
         },
         "mdurl": {
             "hashes": [
-                "sha256:6a8f6804087b7128040b2fb2ebe242bdc2affaeaa034d5fc9feeed30b443651b",
-                "sha256:f79c9709944df218a4cdb0fcc0b0c7ead2f44594e3e84dc566606f04ad749c20"
+                "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8",
+                "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.1.1"
+            "version": "==0.1.2"
         },
         "mypy-extensions": {
             "hashes": [
@@ -2104,11 +2118,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb",
-                "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"
+                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
+                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.12.0"
+            "version": "==2.13.0"
         },
         "pylint": {
             "hashes": [
@@ -2208,10 +2222,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7",
-                "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"
+                "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197",
+                "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5"
             ],
-            "version": "==2022.1"
+            "version": "==2022.2.1"
         },
         "pyyaml": {
             "hashes": [
@@ -2380,10 +2394,10 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:77bec1402dcd0987e9022326473fdbcc767304892a533ed8c29888dacb7dddbc",
-                "sha256:aa1d52d14d09c7a9a12bb740e6bdfffe0f5e8f4f9218d85e7c73a8c37f7ae38d"
+                "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7",
+                "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa"
             ],
-            "version": "==0.3.0"
+            "version": "==0.4.0"
         },
         "toml": {
             "hashes": [
@@ -2402,11 +2416,11 @@
         },
         "tomlkit": {
             "hashes": [
-                "sha256:1c5bebdf19d5051e2e1de6cf70adfc5948d47221f097fcff7a3ffc91e953eaf5",
-                "sha256:61901f81ff4017951119cd0d1ed9b7af31c821d6845c8c477587bbdcd5e5854e"
+                "sha256:25d4e2e446c453be6360c67ddfb88838cfc42026322770ba13d1fbd403a93a5c",
+                "sha256:3235a9010fae54323e727c3ac06fb720752fe6635b3426e379daec60fbd44a83"
             ],
-            "markers": "python_version >= '3.6' and python_version < '4'",
-            "version": "==0.11.1"
+            "markers": "python_version >= '3.6' and python_version < '4.0'",
+            "version": "==0.11.4"
         },
         "tornado": {
             "hashes": [
@@ -2443,11 +2457,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc",
-                "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"
+                "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
+                "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
-            "version": "==1.26.11"
+            "markers": "python_version >= '3.5'",
+            "version": "==1.26.12"
         },
         "wcwidth": {
             "hashes": [
diff --git a/apps/accounts/admin_site.py b/apps/accounts/admin_site.py
index d13d9cd7..567bfabf 100644
--- a/apps/accounts/admin_site.py
+++ b/apps/accounts/admin_site.py
@@ -139,6 +139,7 @@ def get_urls(self):
         patterns = [
             path("try_out/", CheckUrlView.as_view(), name="check_url"),
             path("green-urls", GreenUrlsView.as_view(), name="green_urls"),
+            path("import-ip-ranges", GreenUrlsView.as_view(), name="import_ip_ranges"),
         ]
         return patterns + urls
 
@@ -171,6 +172,19 @@ def get_app_list(self, request):
                     }
                 ],
             },
+            {
+                "name": "Upload ip ranges",
+                "app_label": "greencheck",
+                "app_url": reverse("admin:import_ip_ranges"),
+                "models": [
+                    {
+                        "name": "As CSV",
+                        "object_name": "greencheck_url",
+                        "admin_url": reverse("admin:import_ip_ranges"),
+                        "view_only": True,
+                    }
+                ],
+            },
         ]
         return app_list
 
diff --git a/apps/accounts/templates/import_csv_preview.html b/apps/accounts/templates/import_csv_preview.html
new file mode 100644
index 00000000..79a404ad
--- /dev/null
+++ b/apps/accounts/templates/import_csv_preview.html
@@ -0,0 +1,79 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
+
+<p> If you have a large number of CSV files, you can import them in bulk.</p>
+
+<p>Make sure the form follows the sample format</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.199</td>
+
+</tr>
+</table>
+
+<p>If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.</p>
+<p>So, uploading a CSV file with a single column IP addresses would be the equivalent to:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.197</td>
+
+</tr>
+</table>
+
+{% endblock %}
+
+
+{% block content %}
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import Preview</h2>
+
+<p>The following IP ranges would be imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+
+{% for ip in ip_ranges %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<form enctype="multipart/form-data" action="{% url 'greenweb_admin:accounts_hostingprovider_save_import_from_csv' provider.id %}" method="post">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="Submit"
+        style="margin-top: 0px; padding: 6px 15px">
+
+      <a href="{% url 'greenweb_admin:accounts_hostingprovider_start_import_from_csv' provider.id %}">Back to start</a>
+</form>
+
+
+{% endblock content %}
\ No newline at end of file
diff --git a/apps/accounts/templates/import_csv_results.html b/apps/accounts/templates/import_csv_results.html
new file mode 100644
index 00000000..081f9289
--- /dev/null
+++ b/apps/accounts/templates/import_csv_results.html
@@ -0,0 +1,42 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
+
+<p> If you have a large number of CSV files, you can import them in bulk.</p>
+{% endblock %}
+
+{% block content %}
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import Preview</h2>
+
+<p>The following IP ranges have been created, and updated for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+
+{% for ip in ip_ranges %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+{% endblock content %}
\ No newline at end of file
diff --git a/apps/accounts/templates/import_csv_start.html b/apps/accounts/templates/import_csv_start.html
new file mode 100644
index 00000000..6795e1a3
--- /dev/null
+++ b/apps/accounts/templates/import_csv_start.html
@@ -0,0 +1,76 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
+
+<p> If you have a large number of Ip addresses or IP ranges, you can import them in bulk.</p>
+
+<p>Make sure the uploaded file follows the sample format</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.199</td>
+
+</tr>
+</table>
+
+<p>If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.</p>
+<p>So, uploading a CSV file with a single column IP addresses would be the equivalent to:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.197</td>
+
+</tr>
+</table>
+
+{% endblock %}
+
+
+{% block content %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<form enctype="multipart/form-data" action="{% url 'greenweb_admin:accounts_hostingprovider_save_import_from_csv' provider.id %}" method="post">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="Submit"
+        style="margin-top: 0px; padding: 6px 15px">
+</form>
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import Preview</h2>
+
+<p>The following IP ranges would be imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+
+{% for ip in ip_ranges %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+{% endblock content %}
\ No newline at end of file
diff --git a/apps/accounts/templates/import_ip_ranges.html b/apps/accounts/templates/import_ip_ranges.html
new file mode 100644
index 00000000..1290fb6a
--- /dev/null
+++ b/apps/accounts/templates/import_ip_ranges.html
@@ -0,0 +1,86 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block title %}
+    The Green Web Foundation Member portal: import ip ranges {{ domain }}
+{% endblock %}
+
+
+{% block pretitle %}
+<h1>Try out the greencheck with a URL of your choice</h1>
+{% endblock %}
+<!-- 
+
+{% block content %}
+{% if green_status %}
+<div>
+    <p>From the submitted url of <a href="{{ form.url.value }}">{{ form.url.value }}</a> we looked up the domain <code>{{ domain }}</code></p>
+
+    <p>This domain pointed to IP address <code>{{ ip_lookup }}</code>. This returned a <strong>{{ green_status}}</strong> result in our records.</p>
+</div>
+{% endif %}
+
+{% if matching_green_ip %}
+
+<p>This showed as green because we were able to link this IP address to to the IP range <em>{{ matching_green_ip.ip_start }} to {{ matching_green_ip.ip_start }}</em>, that we have for {{ matching_green_ip.hostingprovider }}.</p>
+
+{% endif %}
+
+{% if matching_green_as %}
+
+<p>This showed as green because we were able to link this IP address to autonomous network AS{{ matching_green_as.asn }} we have registered for {{ matching_green_as.hostingprovider }}.</p>
+
+{% endif %}
+
+{% if whois_info %}
+
+    <hr />
+
+    <h3>More info</h3>
+
+    {% if not matching_green_as and not matching_green_ip and green_status == "gray" %}
+
+    <p>We couldn't find a green match for the provided url: <a href="{{ form.url.value }}">{{ form.url.value }}</a>, but the info below shows the information we <em>can</em> find for it.</p>
+
+    {% endif %}
+
+    <p><a href="{{ form.url.value }}">{{ form.url.value }}</a> points to the IP address <code>{{ whois_info.query }}</code>.</p>
+    <p>This IP address appears to belong to the Autonomous Network {{ whois_info.asn }}, which is assigned to <code>{{ whois_info.network.name }}</code>.<p>
+
+
+    <p>You can look up <a href="https://stat.ripe.net/app/launchpad/AS{{ whois_info.asn }}">AS {{ whois_info.asn }}, on RIPE Stat, an free network anaylsis tool from RIPE<a/>, to learn who is repsonsible for the autonomous network, and where it is primaily located. <small><a href="https://stat.ripe.net/about/">What is RIPE Stat?</a></small></p>
+
+    <p>Look up the domain name <code>{{ domain }}</code> as well as the IP address <code>{{ whois_info.query }}</code> on the <a href="https://lookup.icann.org/">this registration data lookup tool</a> from ICANN. <small><a href="https://en.wikipedia.org/wiki/ICANN">Who is ICANN?</a></small>
+
+
+{% endif %}
+
+<h3>Making corrections</h3>
+
+<p>If you think this result should be showing green when it ought to be gray, or vice versa, please <a href="https://www.thegreenwebfoundation.org/faq/">check our FAQ for reasons why this might be, and to get help with making corrections.</a>.</p>
+
+
+{% if green_status %}
+
+<hr / >
+
+<h2>Try another lookup</h2>
+{% endif %}
+
+<style>
+    form.try_out input[type='url'] {
+        min-width:25rem;
+    }
+</style>
+
+<form action="{{ form_url }}" method="post" class="try_out">
+    {% csrf_token %}
+    {{ form.url.errors }}
+    <label for="{{ form.url.id_for_label }}">Url: </label>
+    {{ form.url }}
+
+    <input type="submit" value="Submit"
+        style="margin-top: 0px; padding: 6px 15px">
+</form>
+
+{% endblock content %} -->
diff --git a/apps/greencheck/forms.py b/apps/greencheck/forms.py
index 864718a7..0a246f29 100644
--- a/apps/greencheck/forms.py
+++ b/apps/greencheck/forms.py
@@ -1,4 +1,9 @@
+
+from django.db.models.fields import CharField
+from django.forms.fields import BooleanField, FileField, IntegerField
+from apps.accounts.models.hosting import ProviderLabel
 import logging
+import io
 
 from django import forms
 from django.forms import ModelForm
@@ -10,6 +15,8 @@
 from .models import GreencheckIp
 from .models import GreencheckIpApprove
 from .models import GreencheckASN, GreencheckASNapprove
+from . import bulk_importers
+from ..accounts import models as ac_models
 
 User = get_user_model()
 
@@ -100,6 +107,59 @@ def clean_is_staff(self):
         except KeyError:
             raise ValidationError("Alert staff: a bug has occurred.")
 
+class ImporterCSVForm(forms.Form):
+    """
+    A form for handling bulk IP range submissions in the django admin.
+    Uses the ImporterCSV class to handle imports
+    """
+
+    provider = forms.ModelChoiceField(
+        empty_label="Choose a Provider",
+        queryset=ac_models.Hostingprovider.objects.all(),
+    )
+    csv_file = FileField(required=False)
+    skip_preview = BooleanField(
+        required=False, help_text=("Do not show the preview of what would happen. "),
+    )
+
+    ip_ranges = []
+    importer = None
+
+    def initialize_importer(self):
+        """
+        Clean our form, and return our importer with the
+        """
+
+        self.is_valid()
+
+        uploaded_csv_string = self.cleaned_data["csv_file"].read().decode("utf-8")
+        csv_file = io.StringIO(uploaded_csv_string)
+
+        importer = bulk_importers.ImporterCSV(self.cleaned_data["provider"])
+        importer.ips_from_file(csv_file)
+
+        self.importer = importer
+        return importer
+
+    def get_ip_ranges(self):
+        """
+        Return a list of the IP Ranges, showing which ones would be updated, and
+        which ones would be created with this submission.
+        """
+        importer = self.initialize_importer()
+        provider = self.cleaned_data["provider"]
+        # make preview of generated ips
+        ips = importer.preview(provider)
+        return ips
+
+    def save(self):
+        """Save our list of IP ranges to the database"""
+        if self.importer is None:
+            self.initialize_importer()
+        logger.info("Skipping preview, running import")
+        self.importer.run()
+        return self.importer
+
 
 class GreencheckAsnForm(ModelForm, ApprovalMixin):
     ApprovalModel = GreencheckASNapprove
diff --git a/makefile b/makefile
index a7ae704f..e235c205 100644
--- a/makefile
+++ b/makefile
@@ -11,6 +11,8 @@ release:
 	PIPENV_DOTENV_LOCATION=.env.prod pipenv run ansible-playbook ansible/deploy.yml -i ansible/inventories/prod.yml
 	PIPENV_DOTENV_LOCATION=.env.prod pipenv run sentry-cli releases finalize $(shell sentry-cli releases propose-version)
 
+dev.runserver:
+	python manage.py runserver
 
 dev.test:
 	pytest -s --create-db --looponfail --ds=greenweb.settings.testing

From 59885dbd89a91f23becb740f85e7bc45ae094982 Mon Sep 17 00:00:00 2001
From: Roald Teunissen <Roaldteunissen@gmail.com>
Date: Mon, 29 Aug 2022 07:40:52 +0000
Subject: [PATCH 05/21] Expanded the base importer to allow ip ranges

---
 apps/greencheck/importers/importer_csv.py     | 31 ++++++-----
 .../importers/importer_interface.py           | 55 +++++++++++++------
 apps/greencheck/tests/test_base_importer.py   | 28 ++++++++--
 3 files changed, 77 insertions(+), 37 deletions(-)

diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index a17ef070..91adfb5c 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -33,15 +33,11 @@ def parse_to_list(cls, raw_data):
             if len(raw_data.columns) == 1:
                 # ASN or network (ip with subnet)
                 list_of_ips = list(raw_data.iloc[:, 0])
-            # elif len(data.columns) == 2:
-            # TODO: Implement a way to convert two ip addresses to a network
-            # # Start and ending ip range
-            # start_ip = data.iloc[:,0]
-            # end_ip = data.iloc[:,1]
-            # Create network from these
-            # for index, row in data.iterrows():
-            #   list_of_ips.append(row[index,0])
-            #     print(row['c1'], row['c2'])
+            elif len(data.columns) == 2:
+                # IP range (start and ending IP)
+                
+                for index, row in data.iterrows():
+                    list_of_ips.append((row[0], row[1]))
             return list_of_ips
         except Exception as e:
             logger.exception("Something really unexpected happened. Aborting")
@@ -53,12 +49,19 @@ def validate_csv_file(cls, data):
             if len(data.columns) == 2:
                 cls.validate_column_in_csv_file(data.iloc[:, 1].values.tolist())
         else:
-            logger.exception("Number of columns in CSV are not as expected")
+            logger.exception("Number of columns in CSV are not as expected.")
 
     def validate_column_in_csv_file(cls, column):
         for address in column:
-            if not re.search("(AS)[0-9]+$", address) or not isinstance(
-                ipaddress.ip_network(address),
-                (ipaddress.IPv4Network, ipaddress.IPv6Network),
+            if (
+                not re.search("(AS)[0-9]+$", address)
+                or not isinstance(
+                    ipaddress.ip_network(address),
+                    (ipaddress.IPv4Network, ipaddress.IPv6Network),
+                )
+                or not isinstance(
+                    ipaddress.ip_address(address),
+                    (ipaddress.IPv4Address, ipaddress.IPv6Address),
+                )
             ):
-                logger.exception("Value of %s is in an incorrect format", address)
+                logger.exception("Value of %s has an incorrect format.", address)
diff --git a/apps/greencheck/importers/importer_interface.py b/apps/greencheck/importers/importer_interface.py
index 74cacab2..04e1594d 100644
--- a/apps/greencheck/importers/importer_interface.py
+++ b/apps/greencheck/importers/importer_interface.py
@@ -9,6 +9,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 @runtime_checkable
 class Importer(Protocol):
     def fetch_data_from_source(cls) -> list:
@@ -17,29 +18,41 @@ def fetch_data_from_source(cls) -> list:
     def parse_to_list(cls) -> list:
         raise NotImplementedError
 
-class BaseImporter():
-    hosting_provider_id:int
+
+class BaseImporter:
+    hosting_provider_id: int
 
     def process_addresses(cls, list_of_addresses: list):
         count_asn = 0
         count_ip = 0
-        
+
         # Determine the type of address (IPv4, IPv6 or ASN)for address in list_of_addresses:
         try:
             for address in list_of_addresses:
                 if re.search("(AS)[0-9]+$", address):
                     # Address is ASN
+
                     cls.save_asn(cls, address)
                     count_asn += 1
-                elif isinstance(
+                elif isinstance(address, str) and isinstance(
                     ipaddress.ip_network(address),
                     (ipaddress.IPv4Network, ipaddress.IPv6Network),
                 ):
-                    # Address is IPv4 or IPv6
+                    # Address is IPv4 or IPv6 network
+                    network = ipaddress.ip_network(address)
+
+                    cls.save_ip(cls, (network[1], network[-1]))
+                    count_ip += 1
+                elif isinstance(address, tuple) and isinstance(
+                    ipaddress.ip_address(address),
+                    (ipaddress.IPv4Address, ipaddress.IPv6Address),
+                ):
+                    # Address is IPv4 or IPv6 range
+
                     cls.save_ip(cls, address)
                     count_ip += 1
-            
-            return f"Processing complete. Added {count_asn} ASN's and {count_ip} IP's (either IPv4 and/or IPv6)"
+
+            return f"Processing complete. Added {count_asn} ASN's and {count_ip} IP ranges (either IPv4 and/or IPv6)"
         except ValueError:
             logger.exception(
                 "Value has invalid structure. Must be IPv4 or IPv6 with subnetmask (101.102.103.104/27) or AS number (AS123456)."
@@ -51,9 +64,7 @@ def process_addresses(cls, list_of_addresses: list):
             return f"An error occurred while adding new entries. Added {count_asn} ASN's and {count_ip} IP's (either IPv4 and/or IPv6)"
 
     def save_asn(cls, address: str):
-        hoster = Hostingprovider.objects.get(
-            pk=cls.hosting_provider_id
-        )
+        hoster = Hostingprovider.objects.get(pk=cls.hosting_provider_id)
 
         gc_asn, created = GreencheckASN.objects.update_or_create(
             active=True, asn=int(address.replace("AS", "")), hostingprovider=hoster
@@ -65,19 +76,27 @@ def save_asn(cls, address: str):
             logger.debug(gc_asn)
             return gc_asn
 
-    def save_ip(cls, address: str):
-        # Convert to IPv4 network with it's respective range
-        network = ipaddress.ip_network(address)
-        hoster = Hostingprovider.objects.get(
-            pk=cls.hosting_provider_id
-        )
+    def save_ip(cls, address):
+        if isinstance(address, tuple):
+            start_address = ipaddress.ip_address(address[0])
+            ending_address = ipaddress.ip_address(address[1])
+        elif isinstance(address, str):
+            network = ipaddress.ip_network(address)
+
+            start_address = network[1]
+            ending_address = network[-1]
+
+        hoster = Hostingprovider.objects.get(pk=cls.hosting_provider_id)
 
         gc_ip, created = GreencheckIp.objects.update_or_create(
-            active=True, ip_start=network[1], ip_end=network[-1], hostingprovider=hoster
+            active=True,
+            ip_start=start_address,
+            ip_end=ending_address,
+            hostingprovider=hoster,
         )
         gc_ip.save()  # Save the newly created or updated object
 
         if created:
             # Only log and return when a new object was created
             logger.debug(gc_ip)
-            return gc_ip
\ No newline at end of file
+            return gc_ip
diff --git a/apps/greencheck/tests/test_base_importer.py b/apps/greencheck/tests/test_base_importer.py
index 4e6d628f..377fcfd2 100644
--- a/apps/greencheck/tests/test_base_importer.py
+++ b/apps/greencheck/tests/test_base_importer.py
@@ -63,8 +63,10 @@ def test_save_ip(self, hosting_provider, base_importer):
         """
         Test saving IPv4 and IPv6 networks to the database
         """
-        testing_ipv4 = "191.233.8.24/29"
-        testing_ipv6 = "2603:1010:304::140/123"
+        testing_ipv4_range = ("191.233.8.25", "191.233.8.30")
+        testing_ipv6_range = ("2603:1010:0304:0000:0000:0000:0000:0140", "2603:1010:0304:0000:0000:0000:0000:015f")
+        testing_ipv4_network = "191.233.8.24/29"
+        testing_ipv6_network = "2603:1010:304::140/123"
 
         assert (
             GreencheckIp.objects.all().count() == 0
@@ -72,17 +74,33 @@ def test_save_ip(self, hosting_provider, base_importer):
         hosting_provider.save()  # Initialize hosting provider in database
 
         # Import a single IPv4 network
-        BaseImporter.save_ip(base_importer, testing_ipv4)
+        BaseImporter.save_ip(base_importer, testing_ipv4_range)
 
         assert (
             GreencheckIp.objects.all().count() == 1
+        )  # Test: database is empty (for IP)
+        hosting_provider.save()  # Initialize hosting provider in database
+
+        # Import a single IPv4 network
+        BaseImporter.save_ip(base_importer, testing_ipv6_range)
+
+        assert (
+            GreencheckIp.objects.all().count() == 2
+        )  # Test: database is empty (for IP)
+        hosting_provider.save()  # Initialize hosting provider in database
+
+        # Import a single IPv4 network
+        BaseImporter.save_ip(base_importer, testing_ipv4_network)
+
+        assert (
+            GreencheckIp.objects.all().count() == 3
         )  # Test: IPv4 is saved after insertion
 
         # Import a single IPv6 network
-        BaseImporter.save_ip(base_importer, testing_ipv6)
+        BaseImporter.save_ip(base_importer, testing_ipv6_network)
 
         assert (
-            GreencheckIp.objects.all().count() == 2
+            GreencheckIp.objects.all().count() == 4
         )  # Test: IPv6 is saved after insertion
 
     def test_save_asn(self, hosting_provider, base_importer):

From 02ad311d5cb553486e80bf65a039c3ec7a8302ac Mon Sep 17 00:00:00 2001
From: Roald Teunissen <Roaldteunissen@gmail.com>
Date: Tue, 30 Aug 2022 16:18:35 +0000
Subject: [PATCH 06/21] WIP: Implemented an option to import csv files

---
 apps/accounts/admin.py | 112 ++++++++++++++++++++++++++++++++++++++++-
 makefile               |   4 ++
 2 files changed, 114 insertions(+), 2 deletions(-)

diff --git a/apps/accounts/admin.py b/apps/accounts/admin.py
index b7bfee2a..09139674 100644
--- a/apps/accounts/admin.py
+++ b/apps/accounts/admin.py
@@ -5,6 +5,7 @@
 from django.contrib import admin
 from django.urls import reverse
 from django.contrib.auth.admin import UserAdmin, GroupAdmin, Group
+import django.forms as dj_forms
 from django.contrib.auth.forms import UserCreationForm
 from django.utils.safestring import mark_safe
 from django.shortcuts import redirect, render
@@ -41,6 +42,8 @@
 from apps.greencheck.models import GreencheckASNapprove
 from apps.greencheck.choices import StatusApproval
 
+from apps.greencheck.forms import ImporterCSVForm
+
 
 from .utils import get_admin_name, reverse_admin_name
 from .admin_site import greenweb_admin
@@ -290,7 +293,7 @@ class HostingAdmin(admin.ModelAdmin):
     ]
     # these are not really fields, but buttons
     # see the corresponding methods
-    readonly_fields = ["preview_email_button"]
+    readonly_fields = ["preview_email_button", "start_csv_import_button"]
     ordering = ("name",)
 
     # Factories
@@ -357,6 +360,82 @@ def preview_email(self, request, *args, **kwargs):
 
         return render(request, "preview_email.html", context)
 
+    def start_import_from_csv(self, request, *args, **kwargs):
+        """
+        Show the form, and preview required formate for the importer
+        for the given hosting provider.
+        """
+
+        # get our provider
+        provider = Hostingprovider.objects.get(pk=kwargs["provider"])
+
+        # get our document
+        data = {"provider": provider.id}
+        form = ImporterCSVForm(data)
+        form.fields["provider"].widget = dj_forms.widgets.HiddenInput()
+
+        return render(
+            request,
+            "import_csv_start.html",
+            {"form": form, "ip_ranges": [], "provider": provider},
+        )
+
+    def save_import_from_csv(self, request, *args, **kwargs):
+        """
+        Process the contents of the uploaded file, and either
+        show a preview of the IP ranges that would be created, or
+        create them, based on submitted form value
+        """
+        provider = Hostingprovider.objects.get(pk=kwargs["provider"])
+
+        if request.method == "POST":
+            # get our provider
+            data = {"provider": provider.id}
+
+            # try to get our document
+            form = ImporterCSVForm(request.POST, request.FILES)
+            form.fields["provider"].widget = dj_forms.widgets.HiddenInput()
+
+            valid = form.is_valid()
+            skip_preview = form.cleaned_data["skip_preview"]
+
+            rich.inspect(request.POST)
+            rich.inspect(form.cleaned_data)
+            if valid and skip_preview:
+                # not doing preview. Run the import
+                form.save()
+
+                ip_ranges = form.get_ip_ranges()
+                context = {
+                    "ip_ranges": ip_ranges,
+                    "provider": provider,
+                }
+                return render(request, "import_csv_preview.html", context,)
+
+            if valid:
+                # the save default we don't save the contents
+                # just showing what would happen
+                ip_ranges = form.get_ip_ranges()
+                context = {
+                    "form": form,
+                    "ip_ranges": ip_ranges,
+                    "provider": provider,
+                }
+                return render(request, "import_csv_preview.html", context,)
+
+            # otherwise fallback to showing the form with errors,
+            # ready for another attempted submission
+
+            context = {
+                "form": form,
+                "ip_ranges": None,
+                "provider": provider,
+            }
+
+            return render(request, "import_csv_preview.html", context,)
+
+        return redirect("greenweb_admin:accounts_hostingprovider_change", provider.id)
+
     def send_email(self, request, *args, **kwargs):
         """
         Send the given email, log the outbound request in the admin, and
@@ -537,6 +616,16 @@ def get_urls(self):
                 self.send_email,
                 name=get_admin_name(self.model, "send_email"),
             ),
+            path(
+                "<provider>/start_import_from_csv",
+                self.start_import_from_csv,
+                name=get_admin_name(self.model, "start_import_from_csv"),
+            ),
+            path(
+                "<provider>/save_import_from_csv",
+                self.save_import_from_csv,
+                name=get_admin_name(self.model, "save_import_from_csv"),
+            ),
             path(
                 "<provider>/preview_email",
                 self.preview_email,
@@ -565,7 +654,12 @@ def get_fieldsets(self, request, obj=None):
         fieldset = [
             (
                 "Hostingprovider info",
-                {"fields": (("name", "website",), "country", "services")},
+                {
+                    "fields": (
+                        ("name", "website",), "country", "services",
+                        ("start_csv_import_button"),
+                    )
+                },
             )
         ]
 
@@ -658,6 +752,20 @@ def preview_email_button(self, obj):
 
     preview_email_button.short_description = "Support Messages"
 
+    @mark_safe
+    def start_csv_import_button(self, obj):
+        """
+        Create clickable link to begin process of bulk import
+        of IP ranges.
+        """
+        url = reverse_admin_name(
+            Hostingprovider, name="start_import_from_csv", kwargs={"provider": obj.pk},
+        )
+        link = f'<a href="{url}" class="start_csv_import">Import IP Ranges from CSV</a>'
+        return link
+
+    send_button.short_description = "Import IP Ranges from a CSV file"
+    
     @mark_safe
     def html_website(self, obj):
         html = f'<a href="{obj.website}" target="_blank">{obj.website}</a>'
diff --git a/makefile b/makefile
index e235c205..c68e343d 100644
--- a/makefile
+++ b/makefile
@@ -11,6 +11,10 @@ release:
 	PIPENV_DOTENV_LOCATION=.env.prod pipenv run ansible-playbook ansible/deploy.yml -i ansible/inventories/prod.yml
 	PIPENV_DOTENV_LOCATION=.env.prod pipenv run sentry-cli releases finalize $(shell sentry-cli releases propose-version)
 
+dev.createsuperuser:
+	python ./manage.py createsuperuser --username admin --email admin@admin.commits --noinput
+	python ./manage.py set_fake_passwords
+
 dev.runserver:
 	python manage.py runserver
 

From a2ca335e6e432fdc098ea33bc431c0c418675efb Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Tue, 30 Aug 2022 18:13:59 +0000
Subject: [PATCH 07/21] Update tests so we have our failing examples

---
 Pipfile                                       |  1 +
 apps/accounts/admin.py                        |  2 +-
 apps/greencheck/fixtures/test_dataset_csv.csv |  9 +-
 apps/greencheck/importers/importer_csv.py     | 87 ++++++++++---------
 apps/greencheck/tests/test_importer_csv.py    | 41 +++++++--
 5 files changed, 88 insertions(+), 52 deletions(-)

diff --git a/Pipfile b/Pipfile
index 4204a712..a43b9c3c 100644
--- a/Pipfile
+++ b/Pipfile
@@ -77,6 +77,7 @@ geoip2 = "*"
 iso3166 = "*"
 django-logentry-admin = "*"
 pandas = "*"
+rich = "*"
 
 [pipenv]
 # Needed for `black`. See https://github.com/microsoft/vscode-python/pull/5967.
diff --git a/apps/accounts/admin.py b/apps/accounts/admin.py
index 09139674..7922f315 100644
--- a/apps/accounts/admin.py
+++ b/apps/accounts/admin.py
@@ -25,7 +25,7 @@
     ActionListFilter,
     UserListFilter,
 )
-
+import rich
 
 from taggit.models import Tag
 import logging
diff --git a/apps/greencheck/fixtures/test_dataset_csv.csv b/apps/greencheck/fixtures/test_dataset_csv.csv
index d439671e..e45d4af7 100644
--- a/apps/greencheck/fixtures/test_dataset_csv.csv
+++ b/apps/greencheck/fixtures/test_dataset_csv.csv
@@ -1,3 +1,6 @@
-31.15.8.0/21
-45.81.42.0/23
-81.95.96.0/20
+104.21.2.197, 104.21.2.199
+104.21.2.0/24
+104.21.2.192/24
+AS234
+AS
+
diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index 91adfb5c..7ed9b967 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -4,6 +4,7 @@
 import ipdb
 import re
 import ipaddress
+import rich
 
 from apps.greencheck.importers.importer_interface import BaseImporter, Importer
 
@@ -14,54 +15,58 @@
 
 class CsvImporter(BaseImporter):
     # def __init__(cls):
+        
+        
     # cls.hosting_provider_id = settings.Csv_PROVIDER_ID
 
-    def fetch_data_from_source(cls):
-        # TODO: fetch data from website dropping
-        # For now: expect that a csv is saved somewhere and fetch in the following way:
+    def fetch_data_from_source(cls, filepath_or_buffer):
         raw_data = pd.read_csv(
-            "apps/greencheck/fixtures/test_dataset_csv.csv", header=None
+            filepath_or_buffer, header=None
         )
         return cls.parse_to_list(raw_data)
 
     def parse_to_list(cls, raw_data):
-        try:
-            list_of_ips = []
-
-            cls.validate_csv_file(raw_data)
-
-            if len(raw_data.columns) == 1:
-                # ASN or network (ip with subnet)
-                list_of_ips = list(raw_data.iloc[:, 0])
-            elif len(data.columns) == 2:
-                # IP range (start and ending IP)
+        rows = raw_data.values
+        imported_networks  = {
+            "asns": [],
+            "ip_networks": [],
+            "ip_ranges": []            
+        }
+        for row in rows:
                 
-                for index, row in data.iterrows():
-                    list_of_ips.append((row[0], row[1]))
-            return list_of_ips
-        except Exception as e:
-            logger.exception("Something really unexpected happened. Aborting")
-
-    def validate_csv_file(cls, data):
-        if len(data.columns) == 1 or len(data.columns) == 2:
-            cls.validate_column_in_csv_file(data.iloc[:, 0].values.tolist())
+            # just one column? it's probably an AS or a IP network
+            if pd.isnull(row[1]):
+            
+                # is it a AS number?
+                if row[0].startswith("AS"):
+                    # as_number = row[0].split("AS ")[1]
+                    imported_networks["asns"].append(row[0])
+                else: 
+                    # if it isn't an AS number it's probably an IP network
+                    try:
+                        ip_network = ipaddress.ip_network(row[0])
+                        imported_networks['ip_networks'].append(row[0])
+                    except Exception:
+                        logger.warn(f"Item {row[0]} was not an ip network. Not importing.")
+            else:
+                # import ipdb ; ipdb.set_trace()                    
+                try:
+                    first_ip, last_ip = row[0].strip(), row[1].strip()
+                    ip_begin = ipaddress.ip_address(first_ip)
+                    ip_end = ipaddress.ip_address(last_ip)
+                    imported_networks['ip_ranges'].append((first_ip, last_ip))
+                except Exception:
+                    logger.warn(f"Row {row} does not look like an IP address. Not importing")
+                
+        flattened_network_list = [
+            *imported_networks['asns'], 
+            *imported_networks['ip_networks'],
+            *imported_networks['ip_ranges']
+        ]
+        rich.print(flattened_network_list)
+        
+        return flattened_network_list
+        
+        
 
-            if len(data.columns) == 2:
-                cls.validate_column_in_csv_file(data.iloc[:, 1].values.tolist())
-        else:
-            logger.exception("Number of columns in CSV are not as expected.")
 
-    def validate_column_in_csv_file(cls, column):
-        for address in column:
-            if (
-                not re.search("(AS)[0-9]+$", address)
-                or not isinstance(
-                    ipaddress.ip_network(address),
-                    (ipaddress.IPv4Network, ipaddress.IPv6Network),
-                )
-                or not isinstance(
-                    ipaddress.ip_address(address),
-                    (ipaddress.IPv4Address, ipaddress.IPv6Address),
-                )
-            ):
-                logger.exception("Value of %s has an incorrect format.", address)
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index 50d0d8d9..a7954489 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -40,18 +40,45 @@ def test_parse_to_list(self, sample_data_raw):
         """
         # Initialize Csv importer
         importer = CsvImporter()
-
+        
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
 
-        # Test: resulting list contains items
+        # Test: resulting list contains some items
         assert len(list_of_addresses) > 0
+        
+        # do we have an ip network?
+        assert "104.21.2.0/24" in list_of_addresses
+                
+        # have we filtered out our incorrect IP network?
+        assert "104.21.2.192/24" not in list_of_addresses
+        
+                
+        # do we have our expected AS number?
+        assert "AS234" in list_of_addresses
+        
+        # have we filtered out our bad AS line?
+        assert "AS" not in list_of_addresses
+        
+        # do we have an IP range in our list
+        expected_ip_range = ('104.21.2.197', '104.21.2.199')
+        assert expected_ip_range in list_of_addresses
+        
+
+
+    def test_process_imports(self, sample_data_raw):
+        # Initialize Csv importer
+        importer = CsvImporter()
+        
+        # Run parse list with sample data
+        list_of_addresses = importer.parse_to_list(sample_data_raw)
+        created_networks = importer.process_addresses(list_of_addresses)
 
-    # def test_validate_csv_file(self):
-    #     # TODO: Implement this test
-
-    # def test_validate_column_in_csv_file(self):
-    #     # TODO: Implement this test
+        # we should have seen one AS network added
+        assert "1 ASN" in created_networks
+        # have we created two new IP ranges?
+        assert "2 IP" in created_networks
+        
 
 
 @pytest.mark.django_db

From 633fb3f8bea2c2edf248a976bf2bfcccb56285f3 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Tue, 30 Aug 2022 18:15:24 +0000
Subject: [PATCH 08/21] Add missing pipffile.lock

---
 Pipfile.lock | 107 +++++++++++++++++++++++++++++++--------------------
 1 file changed, 65 insertions(+), 42 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index fad5958a..92ab8ca5 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "86b55b33255e8df8b5e8ac87462ef278ca359396df92e4f8636a58dce7d23f7d"
+            "sha256": "612fda4ff92d3308f840b5005d3acb36a5daaaf2020283d0d5e5c0df109d9739"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -143,19 +143,19 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575",
-                "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"
+                "sha256:818a40b82e4f66b4bdd4fa38fcc3ed0fb26542f7d8c4d15279d4ba1d4762cd95",
+                "sha256:84b962f18506ad495dc973aeb5168697ee882497c01430dddeba5e8339db7932"
             ],
             "index": "pypi",
-            "version": "==1.24.59"
+            "version": "==1.24.62"
         },
         "botocore": {
             "hashes": [
-                "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da",
-                "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"
+                "sha256:69682c874dc8ed1856bffd203786c9591fb76a1946d3dcc516bda1ee6a6989f3",
+                "sha256:8563c7d8b80e8041667cf35b397f1c399537f6c887e1c501f0064bfd7ae541ba"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.27.59"
+            "version": "==1.27.62"
         },
         "brotli": {
             "hashes": [
@@ -255,6 +255,13 @@
             ],
             "version": "==1.2.2"
         },
+        "commonmark": {
+            "hashes": [
+                "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60",
+                "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"
+            ],
+            "version": "==0.9.1"
+        },
         "confusable-homoglyphs": {
             "hashes": [
                 "sha256:3b4a0d9fa510669498820c91a0bfc0c327568cecec90648cf3819d4a6fc6a751",
@@ -1036,6 +1043,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.14.1"
         },
+        "pygments": {
+            "hashes": [
+                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
+                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.13.0"
+        },
         "pyparsing": {
             "hashes": [
                 "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
@@ -1114,6 +1129,14 @@
             "index": "pypi",
             "version": "==2.28.1"
         },
+        "rich": {
+            "hashes": [
+                "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb",
+                "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"
+            ],
+            "index": "pypi",
+            "version": "==12.5.1"
+        },
         "ruamel.yaml": {
             "hashes": [
                 "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7",
@@ -1233,11 +1256,11 @@
         },
         "sqlite-utils": {
             "hashes": [
-                "sha256:790b01f4be16c32756b9b5eae07b6b7c905b6613ca538d646877c50b05b0a53a",
-                "sha256:e8841bff7fcf29b0789c374a4ccca42107de1592836aa6899a04e3aebd304277"
+                "sha256:9e6b2bf7ae89e02e107557b0cff1121f1fbb06128a815122743332197693b272",
+                "sha256:d9ea1026a9c007a895cdd04abdcbe3cd2ac03515c2a2ebbad9233939aa111f5b"
             ],
             "index": "pypi",
-            "version": "==3.28"
+            "version": "==3.29"
         },
         "sqlparse": {
             "hashes": [
@@ -1282,7 +1305,7 @@
                 "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02",
                 "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version < '3.9'",
             "version": "==4.3.0"
         },
         "unicodecsv": {
@@ -1480,11 +1503,11 @@
         },
         "astroid": {
             "hashes": [
-                "sha256:86b0a340a512c65abf4368b80252754cda17c02cdbbd3f587dddf98112233e7b",
-                "sha256:bb24615c77f4837c707669d16907331374ae8a964650a66999da3f5ca68dc946"
+                "sha256:396c88d0a58d7f8daadf730b2ce90838bf338c6752558db719ec6f99c18ec20e",
+                "sha256:d612609242996c4365aeb0345e61edba34363eaaba55f1c0addf6a98f073bef6"
             ],
-            "markers": "python_full_version >= '3.6.2'",
-            "version": "==2.11.7"
+            "markers": "python_full_version >= '3.7.2'",
+            "version": "==2.12.5"
         },
         "asttokens": {
             "hashes": [
@@ -1506,7 +1529,7 @@
                 "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51",
                 "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==2.10.3"
         },
         "backcall": {
@@ -1543,7 +1566,7 @@
                 "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
                 "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==4.11.1"
         },
         "black": {
@@ -1727,11 +1750,11 @@
         },
         "exceptiongroup": {
             "hashes": [
-                "sha256:6990c24f06b8d33c8065cfe43e5e8a4bfa384e0358be036af9cc60b6321bd11a",
-                "sha256:ab0a968e1ef769e55d9a596f4a89f7be9ffedbc9fdefdb77cc68cf5c33ce1035"
+                "sha256:2e3c3fc1538a094aab74fad52d6c33fc94de3dfee3ee01f187c0e0c72aec5337",
+                "sha256:9086a4a21ef9b31c72181c77c040a074ba0889ee56a7b289ff0afb0d97655f96"
             ],
             "markers": "python_version < '3.11'",
-            "version": "==1.0.0rc8"
+            "version": "==1.0.0rc9"
         },
         "execnet": {
             "hashes": [
@@ -1743,10 +1766,10 @@
         },
         "executing": {
             "hashes": [
-                "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e",
-                "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"
+                "sha256:550d581b497228b572235e633599133eeee67073c65914ca346100ad56775349",
+                "sha256:98daefa9d1916a4f0d944880d5aeaf079e05585689bebd9ff9b32e31dd5e1017"
             ],
-            "version": "==0.10.0"
+            "version": "==1.0.0"
         },
         "factory-boy": {
             "hashes": [
@@ -1761,7 +1784,7 @@
                 "sha256:067a03f64e555261610e69277536072997b4576dbf84b113faef3c06d85b466b",
                 "sha256:0e00bfa1eadf1493f15662edb181222fea4847764cf3f9ff3e66ee0f95c9a644"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==14.1.0"
         },
         "flake8": {
@@ -1855,7 +1878,7 @@
                 "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d",
                 "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==0.18.1"
         },
         "jinja2": {
@@ -1906,7 +1929,7 @@
                 "sha256:fccdf7c2c5821a8cbd0a9440a456f5050492f2270bd54e94360cac663398739b",
                 "sha256:fd45683c3caddf83abbb1249b653a266e7069a09f486daa8863fb0e7496a9fdb"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==1.7.1"
         },
         "livereload": {
@@ -1982,7 +2005,7 @@
                 "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
                 "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==0.7.0"
         },
         "mdit-py-plugins": {
@@ -2029,7 +2052,7 @@
                 "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0",
                 "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==0.8.3"
         },
         "pathspec": {
@@ -2067,7 +2090,7 @@
                 "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
                 "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==1.0.0"
         },
         "prompt-toolkit": {
@@ -2105,7 +2128,7 @@
                 "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785",
                 "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==2.9.1"
         },
         "pyflakes": {
@@ -2113,7 +2136,7 @@
                 "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2",
                 "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==2.5.0"
         },
         "pygments": {
@@ -2126,11 +2149,11 @@
         },
         "pylint": {
             "hashes": [
-                "sha256:487ce2192eee48211269a0e976421f334cf94de1806ca9d0a99449adcdf0285e",
-                "sha256:fabe30000de7d07636d2e82c9a518ad5ad7908590fe135ace169b44839c15f90"
+                "sha256:4b124affc198b7f7c9b5f9ab690d85db48282a025ef9333f51d2d7281b92a6c3",
+                "sha256:4f3f7e869646b0bd63b3dfb79f3c0f28fc3d2d923ea220d52620fd625aed92b0"
             ],
             "index": "pypi",
-            "version": "==2.14.5"
+            "version": "==2.15.0"
         },
         "pylint-django": {
             "hashes": [
@@ -2193,7 +2216,7 @@
                 "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e",
                 "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==1.4.0"
         },
         "pytest-mock": {
@@ -2309,7 +2332,7 @@
                 "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759",
                 "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==2.3.2.post1"
         },
         "sphinx": {
@@ -2357,7 +2380,7 @@
                 "sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07",
                 "sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2"
             ],
-            "markers": "python_version >= '3.6'",
+            "markers": "python_full_version >= '3.6.0'",
             "version": "==2.0.0"
         },
         "sphinxcontrib-jsmath": {
@@ -2394,10 +2417,10 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7",
-                "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa"
+                "sha256:66d2ebd3d7f29047612ead465b6cae5371006a71f45037c7e2507d01367bce3b",
+                "sha256:715c8855fbf5c43587b141e46cc9d9339cc0d1f8d6e0f98ed0d01c6cb974e29f"
             ],
-            "version": "==0.4.0"
+            "version": "==0.5.0"
         },
         "toml": {
             "hashes": [
@@ -2419,7 +2442,7 @@
                 "sha256:25d4e2e446c453be6360c67ddfb88838cfc42026322770ba13d1fbd403a93a5c",
                 "sha256:3235a9010fae54323e727c3ac06fb720752fe6635b3426e379daec60fbd44a83"
             ],
-            "markers": "python_version >= '3.6' and python_version < '4.0'",
+            "markers": "python_version < '4.0' and python_full_version >= '3.6.0'",
             "version": "==0.11.4"
         },
         "tornado": {
@@ -2452,7 +2475,7 @@
                 "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02",
                 "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version < '3.9'",
             "version": "==4.3.0"
         },
         "urllib3": {

From c51aeafd7f256b47918f34ee56427515c554902e Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 04:09:29 +0000
Subject: [PATCH 09/21] Get CSV importer parsing info as intended

Add tests and refactor importer
---
 apps/greencheck/importers/importer_csv.py     |  74 +++++-----
 .../importers/importer_interface.py           | 127 +++++++++++++-----
 apps/greencheck/tests/test_importer_csv.py    |  32 ++---
 3 files changed, 150 insertions(+), 83 deletions(-)

diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index 7ed9b967..42652319 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -5,6 +5,7 @@
 import re
 import ipaddress
 import rich
+from typing import List
 
 from apps.greencheck.importers.importer_interface import BaseImporter, Importer
 
@@ -14,59 +15,62 @@
 
 
 class CsvImporter(BaseImporter):
-    # def __init__(cls):
-        
-        
-    # cls.hosting_provider_id = settings.Csv_PROVIDER_ID
+    def __init__(self, provider_id: int):
+        self.hosting_provider_id = provider_id
 
-    def fetch_data_from_source(cls, filepath_or_buffer):
-        raw_data = pd.read_csv(
-            filepath_or_buffer, header=None
-        )
+    def fetch_data_from_source(cls, filepath_or_buffer) -> List:
+        """
+        Return a list of the valid values from the provided CSV
+        for importing
+        """
+        raw_data = pd.read_csv(filepath_or_buffer, header=None)
         return cls.parse_to_list(raw_data)
 
-    def parse_to_list(cls, raw_data):
+    def parse_to_list(self, raw_data: pd.DataFrame) -> List:
+        """
+        Parse the provided pandas DataFrame, and return a flattened list
+        of ip ranges, or importable IP networks, or AS numbers
+        """
         rows = raw_data.values
-        imported_networks  = {
-            "asns": [],
-            "ip_networks": [],
-            "ip_ranges": []            
-        }
+        imported_networks = {"asns": [], "ip_networks": [], "ip_ranges": []}
         for row in rows:
-                
+
             # just one column? it's probably an AS or a IP network
             if pd.isnull(row[1]):
-            
-                # is it a AS number?
+
+                # is it an AS number?
                 if row[0].startswith("AS"):
-                    # as_number = row[0].split("AS ")[1]
-                    imported_networks["asns"].append(row[0])
-                else: 
+                    # split out the as number from the row,
+                    # add check for people getting AS number
+                    as_number = row[0].split("AS ")[0]
+                    just_as_with_no_number = as_number.lower() == "as"
+
+                    if as_number and not just_as_with_no_number:
+                        imported_networks["asns"].append(row[0])
+                else:
                     # if it isn't an AS number it's probably an IP network
                     try:
                         ip_network = ipaddress.ip_network(row[0])
-                        imported_networks['ip_networks'].append(row[0])
+                        imported_networks["ip_networks"].append(row[0])
                     except Exception:
-                        logger.warn(f"Item {row[0]} was not an ip network. Not importing.")
+                        logger.warn(
+                            f"Item {row[0]} was not an ip network. Not importing."
+                        )
             else:
-                # import ipdb ; ipdb.set_trace()                    
                 try:
                     first_ip, last_ip = row[0].strip(), row[1].strip()
                     ip_begin = ipaddress.ip_address(first_ip)
                     ip_end = ipaddress.ip_address(last_ip)
-                    imported_networks['ip_ranges'].append((first_ip, last_ip))
+                    imported_networks["ip_ranges"].append((first_ip, last_ip))
                 except Exception:
-                    logger.warn(f"Row {row} does not look like an IP address. Not importing")
-                
+                    logger.warn(
+                        f"Row {row} does not look like an IP address. Not importing"
+                    )
+
         flattened_network_list = [
-            *imported_networks['asns'], 
-            *imported_networks['ip_networks'],
-            *imported_networks['ip_ranges']
+            *imported_networks["asns"],
+            *imported_networks["ip_networks"],
+            *imported_networks["ip_ranges"],
         ]
-        rich.print(flattened_network_list)
-        
-        return flattened_network_list
-        
-        
-
 
+        return flattened_network_list
diff --git a/apps/greencheck/importers/importer_interface.py b/apps/greencheck/importers/importer_interface.py
index 04e1594d..8ad49d72 100644
--- a/apps/greencheck/importers/importer_interface.py
+++ b/apps/greencheck/importers/importer_interface.py
@@ -1,8 +1,9 @@
 import ipaddress
 import re
 import logging
+import rich
 
-from typing import Protocol, runtime_checkable
+from typing import Protocol, runtime_checkable, Union
 
 from apps.greencheck.models import GreencheckIp, GreencheckASN
 from apps.accounts.models import Hostingprovider
@@ -12,59 +13,121 @@
 
 @runtime_checkable
 class Importer(Protocol):
-    def fetch_data_from_source(cls) -> list:
+    def fetch_data_from_source(self) -> list:
+        """
+        Fetches the data, and returns a data structure for parsing
+        with `parse_to_list`
+        """
         raise NotImplementedError
 
-    def parse_to_list(cls) -> list:
+    def parse_to_list(self) -> list:
+        """
+        Returns a list of either strings that can be parsed as AS numbers
+        or IP networks
+        """
         raise NotImplementedError
 
 
 class BaseImporter:
     hosting_provider_id: int
 
-    def process_addresses(cls, list_of_addresses: list):
+    def is_ip_range(self, address: Union[str, tuple]) -> Union[tuple, False]:
+        """
+        Check if "address" is a usable ip range, and return the tuple containing
+        the required ip addresses, ready for importing as an ip range
+        """
+        # return early if not a tuple we can parse
+        if not isinstance(address, tuple):
+            return
+
+        first_ip = ipaddress.ip_address(address[0])
+        last_ip = ipaddress.ip_address(address[-1])
+        if first_ip and last_ip:
+            return address
+
+    def is_ip_network(self, address: Union[str, tuple]):
+        """
+        Chck that "address" is a string we can parse to an ip network,
+        ready for saving as an ip range.
+        """
+        # return early if not a string we can parse
+        if not isinstance(address, str):
+            return
+
+        try:
+            network = ipaddress.ip_network(address)
+        except ValueError:
+            logger.exception(
+                (
+                    f"Value for address: {address} has an invalid structure. "
+                    "Must be IPv4 or IPv6 with subnetmask (101.102.103.104/27) "
+                    "or AS number (AS123456)."
+                )
+            )
+            return
+
+        if network:
+            return network
+
+    def is_as_number(self, address: Union[str, tuple]) -> Union[str, False]:
+        """
+        Check that "address" is a string suitable for saving as a AS number
+        """
+        # return early if not a string we can parse
+        if not isinstance(address, str):
+            return
+
+        if address.startswith("AS"):
+            return address
+
+    def process_addresses(self, list_of_addresses: list):
         count_asn = 0
         count_ip = 0
 
-        # Determine the type of address (IPv4, IPv6 or ASN)for address in list_of_addresses:
+        # Determine the type of address (IPv4, IPv6 or ASN) for
+        # address in list_of_addresses:
         try:
             for address in list_of_addresses:
-                if re.search("(AS)[0-9]+$", address):
-                    # Address is ASN
 
-                    cls.save_asn(cls, address)
-                    count_asn += 1
-                elif isinstance(address, str) and isinstance(
-                    ipaddress.ip_network(address),
-                    (ipaddress.IPv4Network, ipaddress.IPv6Network),
-                ):
-                    # Address is IPv4 or IPv6 network
-                    network = ipaddress.ip_network(address)
-
-                    cls.save_ip(cls, (network[1], network[-1]))
+                if self.is_ip_range(address):
+                    # address looks like an IPv4 or IPv6 range
+                    self.save_ip(address)
                     count_ip += 1
-                elif isinstance(address, tuple) and isinstance(
-                    ipaddress.ip_address(address),
-                    (ipaddress.IPv4Address, ipaddress.IPv6Address),
-                ):
-                    # Address is IPv4 or IPv6 range
+                    continue
 
-                    cls.save_ip(cls, address)
+                if self.is_ip_network(address):
+                    # address looks like IPv4 or IPv6 network
+                    network = ipaddress.ip_network(address)
+                    self.save_ip((network[1], network[-1]))
                     count_ip += 1
+                    continue
 
-            return f"Processing complete. Added {count_asn} ASN's and {count_ip} IP ranges (either IPv4 and/or IPv6)"
+                if self.is_as_number(address):
+                    # address is ASN
+                    self.save_asn(address)
+                    count_asn += 1
+                    continue
+
+            return (
+                f"Processing complete. Added {count_asn} ASN's and {count_ip} IP "
+                "ranges (either IPv4 and/or IPv6)"
+            )
         except ValueError:
-            logger.exception(
-                "Value has invalid structure. Must be IPv4 or IPv6 with subnetmask (101.102.103.104/27) or AS number (AS123456)."
+            return (
+                f"An error occurred while adding new entries. Added {count_asn} AS "
+                f"numbers and {count_ip} ip ranges (either IPv4 and/or IPv6)"
             )
-            return f"An error occurred while adding new entries. Added {count_asn} ASN's and {count_ip} IP's (either IPv4 and/or IPv6)"
         except Exception as e:
             logger.exception("Something really unexpected happened. Aborting")
             logger.exception(e)
-            return f"An error occurred while adding new entries. Added {count_asn} ASN's and {count_ip} IP's (either IPv4 and/or IPv6)"
+            return (
+                "An error occurred while adding new entries. "
+                f"Added {count_asn} ASN's and {count_ip} IP's "
+                "(either IPv4 and/or IPv6)"
+            )
 
-    def save_asn(cls, address: str):
-        hoster = Hostingprovider.objects.get(pk=cls.hosting_provider_id)
+    def save_asn(self, address: str):
+        hoster = Hostingprovider.objects.get(pk=self.hosting_provider_id)
 
         gc_asn, created = GreencheckASN.objects.update_or_create(
             active=True, asn=int(address.replace("AS", "")), hostingprovider=hoster
@@ -76,7 +139,7 @@ def save_asn(cls, address: str):
             logger.debug(gc_asn)
             return gc_asn
 
-    def save_ip(cls, address):
+    def save_ip(self, address):
         if isinstance(address, tuple):
             start_address = ipaddress.ip_address(address[0])
             ending_address = ipaddress.ip_address(address[1])
@@ -86,7 +149,7 @@ def save_ip(cls, address):
             start_address = network[1]
             ending_address = network[-1]
 
-        hoster = Hostingprovider.objects.get(pk=cls.hosting_provider_id)
+        hoster = Hostingprovider.objects.get(pk=self.hosting_provider_id)
 
         gc_ip, created = GreencheckIp.objects.update_or_create(
             active=True,
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index a7954489..af9c75dd 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -6,6 +6,7 @@
 from io import StringIO
 
 from django.core.management import call_command
+from apps.accounts.models.hosting import Hostingprovider
 from apps.greencheck.importers.importer_csv import CsvImporter
 
 from django.conf import settings
@@ -34,42 +35,42 @@ def sample_data_as_list(sample_data_raw):
 
 @pytest.mark.django_db
 class TestCsvImporter:
-    def test_parse_to_list(self, sample_data_raw):
+    def test_parse_to_list(self, sample_data_raw, hosting_provider: Hostingprovider):
         """
         Test the parsing function.
         """
         # Initialize Csv importer
-        importer = CsvImporter()
-        
+        hosting_provider.save()
+        importer = CsvImporter(hosting_provider.id)
+
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
 
         # Test: resulting list contains some items
         assert len(list_of_addresses) > 0
-        
+
         # do we have an ip network?
         assert "104.21.2.0/24" in list_of_addresses
-                
-        # have we filtered out our incorrect IP network?
+
+        # have we filtered out our incorrect IP network?
         assert "104.21.2.192/24" not in list_of_addresses
-        
-                
+
         # do we have our expected AS number?
         assert "AS234" in list_of_addresses
-        
+
         # have we filtered out our bad AS line?
         assert "AS" not in list_of_addresses
-        
+
         # do we have an IP range in our list
-        expected_ip_range = ('104.21.2.197', '104.21.2.199')
+        expected_ip_range = ("104.21.2.197", "104.21.2.199")
         assert expected_ip_range in list_of_addresses
-        
 
+    def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
 
-    def test_process_imports(self, sample_data_raw):
         # Initialize Csv importer
-        importer = CsvImporter()
-        
+        hosting_provider.save()
+        importer = CsvImporter(hosting_provider.id)
+
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
         created_networks = importer.process_addresses(list_of_addresses)
@@ -78,7 +79,6 @@ def test_process_imports(self, sample_data_raw):
         assert "1 ASN" in created_networks
         # have we created two new IP ranges?
         assert "2 IP" in created_networks
-        
 
 
 @pytest.mark.django_db

From 53165738e18000d18c63d30fe66064df45406d59 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 04:12:55 +0000
Subject: [PATCH 10/21] Pass a type, not a value in the method signature

---
 apps/greencheck/importers/importer_interface.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/apps/greencheck/importers/importer_interface.py b/apps/greencheck/importers/importer_interface.py
index 8ad49d72..75f1c97d 100644
--- a/apps/greencheck/importers/importer_interface.py
+++ b/apps/greencheck/importers/importer_interface.py
@@ -31,7 +31,7 @@ def parse_to_list(self) -> list:
 class BaseImporter:
     hosting_provider_id: int
 
-    def is_ip_range(self, address: Union[str, tuple]) -> Union[tuple, False]:
+    def is_ip_range(self, address: Union[str, tuple]) -> Union[tuple, bool]:
         """
         Check if "address" is a usable ip range, and return the tuple containing
         the required ip addresses, ready for importing as an ip range
@@ -54,6 +54,10 @@ def is_ip_network(self, address: Union[str, tuple]):
         if not isinstance(address, str):
             return
 
+        # this looks an AS number return early
+        if address.startswith("AS"):
+            return
+
         try:
             network = ipaddress.ip_network(address)
         except ValueError:
@@ -69,7 +73,7 @@ def is_ip_network(self, address: Union[str, tuple]):
         if network:
             return network
 
-    def is_as_number(self, address: Union[str, tuple]) -> Union[str, False]:
+    def is_as_number(self, address: Union[str, tuple]) -> Union[str, bool]:
         """
         Check that "address" is a string suitable for saving as a AS number
         """

From 57eba2bcf695f3001ce29fd26cf60fca9ffe5f15 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 05:25:24 +0000
Subject: [PATCH 11/21] Add tests and code for import preview in admin

---
 apps/greencheck/importers/importer_csv.py  | 70 +++++++++++++++++++++
 apps/greencheck/tests/test_importer_csv.py | 71 +++++++++++++---------
 2 files changed, 113 insertions(+), 28 deletions(-)

diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index 42652319..32151909 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -6,8 +6,10 @@
 import ipaddress
 import rich
 from typing import List
+from apps.accounts.models.hosting import Hostingprovider
 
 from apps.greencheck.importers.importer_interface import BaseImporter, Importer
+from apps.greencheck.models import GreencheckIp, GreencheckASN
 
 from django.conf import settings
 
@@ -74,3 +76,71 @@ def parse_to_list(self, raw_data: pd.DataFrame) -> List:
         ]
 
         return flattened_network_list
+
+    def preview(self, provider: Hostingprovider, list_of_networks: List) -> List:
+        """
+        Return a list of the GreencheckIPs that would be updated
+        or created based on the current provided file.
+
+        Return a preview of the networks to import, suitable for displaying
+        in a webpage.
+        """
+
+        green_ips = []
+        green_asns = []
+        # try to find a GreenIP
+        for network in list_of_networks:
+
+            if self.is_as_number(network):
+                # this looks like an AS number
+                try:
+                    as_number = network.split("AS")[1]
+                    green_asn = GreencheckASN.objects.get(
+                        asn=as_number, active=True, hostingprovider=provider
+                    )
+                    green_asns.append(green_asn)
+                except GreencheckASN.DoesNotExist:
+                    green_asn = GreencheckASN(
+                        active=True, asn=as_number, hostingprovider=provider
+                    )
+                    green_asns.append(green_asn)
+
+            if ip_network := self.is_ip_network(network):
+                try:
+                    green_ip = GreencheckIp.objects.get(
+                        active=True,
+                        ip_start=ip_network[1],
+                        ip_end=ip_network[-1],
+                        hostingprovider=provider,
+                    )
+                    green_ips.append(green_ip)
+
+                except GreencheckIp.DoesNotExist:
+                    green_ip = GreencheckIp(
+                        active=True,
+                        ip_start=ip_network[1],
+                        ip_end=ip_network[-1],
+                        hostingprovider=provider,
+                    )
+                    green_ips.append(green_ip)
+
+            if ip_range := self.is_ip_range(network):
+                try:
+                    green_ip = GreencheckIp.objects.get(
+                        active=True,
+                        hostingprovider=provider,
+                        ip_start=ip_range[0],
+                        ip_end=ip_range[1],
+                    )
+                    green_ips.append(green_ip)
+                except GreencheckIp.DoesNotExist:
+                    green_ip = GreencheckIp(
+                        active=True,
+                        hostingprovider=provider,
+                        ip_start=ip_range[0],
+                        ip_end=ip_range[1],
+                    )
+                    green_ips.append(green_ip)
+
+        # or make a new one, in memory
+        return {"green_ips": green_ips, "green_asns": green_asns}
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index af9c75dd..ca9e0621 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -24,7 +24,7 @@ def sample_data_raw():
 
 
 @pytest.fixture
-def sample_data_as_list(sample_data_raw):
+def sample_data_as_list(sample_data_raw, hosting_provider: Hostingprovider):
     """
     Retrieve a locally saved sample of the population to use for this test and parse it to a list
     Return: List
@@ -80,31 +80,46 @@ def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovide
         # have we created two new IP ranges?
         assert "2 IP" in created_networks
 
+    def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
+        """
+        Can we see a representation of the data we would import before
+        we run the import it?
+        """
+        # Initialize Csv importer
+        hosting_provider.save()
+        importer = CsvImporter(hosting_provider.id)
 
-@pytest.mark.django_db
-class TestCsvImportCommand:
-    """
-    This just tests that we have a management command that can run.
-    We _could_ mock the call to fetch ip ranges, if this turns out to be a slow test.
-    """
-
-    def test_handle(self, mocker, sample_data_as_list):
-        # mock the call to retrieve from source, to a locally stored
-        # testing sample. By instead using the test sample,
-        # we avoid unnecessary network requests.
-
-        # identify method we want to mock
-        path_to_mock = (
-            "apps.greencheck.importers.importer_csv."
-            "CsvImporter.fetch_data_from_source"
-        )
-
-        # define a different return when the targeted mock
-        # method is called
-        mocker.patch(
-            path_to_mock,
-            return_value=sample_data_as_list,
-        )
-
-        # TODO: Do we need this call command?
-        # call_command("update_networks_in_db_csv")
+        # Run parse list with sample data
+        list_of_addresses = importer.parse_to_list(sample_data_raw)
+        preview = importer.preview(hosting_provider, list_of_addresses)
+
+        assert len(preview["green_ips"]) == 2
+        assert len(preview["green_asns"]) == 1
+
+
+# @pytest.mark.django_db
+# class TestCsvImportCommand:
+#     """
+#     This just tests that we have a management command that can run.
+#     """
+
+#     def test_handle(self, mocker, sample_data_as_list):
+#         # mock the call to retrieve from source, to a locally stored
+#         # testing sample. By instead using the test sample,
+#         # we avoid unnecessary network requests.
+
+#         # identify method we want to mock
+#         path_to_mock = (
+#             "apps.greencheck.importers.importer_csv."
+#             "CsvImporter.fetch_data_from_source"
+#         )
+
+#         # define a different return when the targeted mock
+#         # method is called
+#         mocker.patch(
+#             path_to_mock,
+#             return_value=sample_data_as_list,
+#         )
+
+#         # TODO: Do we need this call command?
+#         # call_command("update_networks_in_db_csv")

From 347ff9eafb1c479a32ffcfe3551af36915d35825 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 06:25:16 +0000
Subject: [PATCH 12/21] Add indication of ip range length before import

---
 .../templates/import_csv_preview.html         | 64 +++++++++++++++++--
 1 file changed, 59 insertions(+), 5 deletions(-)

diff --git a/apps/accounts/templates/import_csv_preview.html b/apps/accounts/templates/import_csv_preview.html
index 79a404ad..d1c8dd95 100644
--- a/apps/accounts/templates/import_csv_preview.html
+++ b/apps/accounts/templates/import_csv_preview.html
@@ -6,7 +6,7 @@ <h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
 
 <p> If you have a large number of CSV files, you can import them in bulk.</p>
 
-<p>Make sure the form follows the sample format</p>
+<p>Make sure the form follows the sample format, without the header row</p>
 
 <table>
 <th>IP Range start</th><th>Ip Range End</th>
@@ -15,11 +15,21 @@ <h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
   <td>104.21.2.199</td>
 
 </tr>
+<tr>
+  <td>104.21.2.0/24</td>
+  <td></td>
+</tr>
+<tr>
+  <td>AS12345</td>
+  <td></td>
+</tr>
+
 </table>
 
 <p>If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.</p>
 <p>So, uploading a CSV file with a single column IP addresses would be the equivalent to:</p>
 
+
 <table>
 <th>IP Range start</th><th>Ip Range End</th>
 <tr>
@@ -29,8 +39,26 @@ <h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
 </tr>
 </table>
 
-{% endblock %}
+<p> If you have an an IP range to import, add it all in the first column. It will be expanded into a full IP range</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.0/24</td>
+  <td></td>
+</tr>
+</table>
 
+<p>Likewise with AS numbers. Do not as a space between 'AS' and the number itself.</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>AS12345</td>
+  <td></td>
+</tr>
+</table>
+{% endblock %}
 
 {% block content %}
 
@@ -43,9 +71,9 @@ <h2>IP Range Import Preview</h2>
 <p>The following IP ranges would be imported for <strong>{{ provider }}</strong>:</p>
 
 <table>
-<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th><th>Length</th>
 
-{% for ip in ip_ranges %}
+{% for ip in ip_ranges.green_ips %}
 
   <tr>
     <td>{{ ip.ip_start }}</td>
@@ -57,6 +85,32 @@ <h2>IP Range Import Preview</h2>
           Created
       {% endif %}
     </td>
+    <td>
+      {{  ip.ip_range_length }}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+<h2>AS Import Preview</h2>
+
+<p>The following AS Numbers ranges would be imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>AS number</th><th>Created / Updated</th>
+
+{% for as in ip_ranges.green_asns %}
+
+  <tr>
+    <td>{{ as.asn }}</td>
+    <td>
+      {% if as.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
   </tr>
 
 {% endfor %}
@@ -76,4 +130,4 @@ <h2>IP Range Import Preview</h2>
 </form>
 
 
-{% endblock content %}
\ No newline at end of file
+{% endblock content %}

From 8d4695c8cc7ce1f27f9be4f56dd4a08c59f5e831 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 06:26:59 +0000
Subject: [PATCH 13/21] Update method signature for updating tests

---
 apps/greencheck/importers/importer_interface.py | 2 +-
 apps/greencheck/tests/test_base_importer.py     | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/apps/greencheck/importers/importer_interface.py b/apps/greencheck/importers/importer_interface.py
index 75f1c97d..c75d92b9 100644
--- a/apps/greencheck/importers/importer_interface.py
+++ b/apps/greencheck/importers/importer_interface.py
@@ -134,7 +134,7 @@ def save_asn(self, address: str):
         hoster = Hostingprovider.objects.get(pk=self.hosting_provider_id)
 
         gc_asn, created = GreencheckASN.objects.update_or_create(
-            active=True, asn=int(address.replace("AS", "")), hostingprovider=hoster
+            active=True, asn=int(address.replace("AS", "")), hostingprovider=hoster.id
         )
         gc_asn.save()  # Save the newly created or updated object
 
diff --git a/apps/greencheck/tests/test_base_importer.py b/apps/greencheck/tests/test_base_importer.py
index 377fcfd2..404eeb7e 100644
--- a/apps/greencheck/tests/test_base_importer.py
+++ b/apps/greencheck/tests/test_base_importer.py
@@ -64,7 +64,10 @@ def test_save_ip(self, hosting_provider, base_importer):
         Test saving IPv4 and IPv6 networks to the database
         """
         testing_ipv4_range = ("191.233.8.25", "191.233.8.30")
-        testing_ipv6_range = ("2603:1010:0304:0000:0000:0000:0000:0140", "2603:1010:0304:0000:0000:0000:0000:015f")
+        testing_ipv6_range = (
+            "2603:1010:0304:0000:0000:0000:0000:0140",
+            "2603:1010:0304:0000:0000:0000:0000:015f",
+        )
         testing_ipv4_network = "191.233.8.24/29"
         testing_ipv6_network = "2603:1010:304::140/123"
 
@@ -135,7 +138,8 @@ def test_process_addresses(self, hosting_provider, base_importer, sample_data):
         hosting_provider.save()  # Initialize hosting provider in database
 
         # Process list of addresses in JSON file
-        BaseImporter.process_addresses(base_importer, sample_data)
+        importer = BaseImporter()
+        importer.process_addresses(sample_data)
 
         assert (
             GreencheckIp.objects.all().count() == 63

From da5ec238ba9d93521ecccd6ca53c1e09c8273424 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 06:28:38 +0000
Subject: [PATCH 14/21] Update form to use new CSV importer

---
 apps/greencheck/forms.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/apps/greencheck/forms.py b/apps/greencheck/forms.py
index 0a246f29..8e7be957 100644
--- a/apps/greencheck/forms.py
+++ b/apps/greencheck/forms.py
@@ -1,8 +1,8 @@
-
 from django.db.models.fields import CharField
 from django.forms.fields import BooleanField, FileField, IntegerField
 from apps.accounts.models.hosting import ProviderLabel
 import logging
+import rich
 import io
 
 from django import forms
@@ -15,7 +15,7 @@
 from .models import GreencheckIp
 from .models import GreencheckIpApprove
 from .models import GreencheckASN, GreencheckASNapprove
-from . import bulk_importers
+from .importers import CsvImporter
 from ..accounts import models as ac_models
 
 User = get_user_model()
@@ -107,6 +107,7 @@ def clean_is_staff(self):
         except KeyError:
             raise ValidationError("Alert staff: a bug has occurred.")
 
+
 class ImporterCSVForm(forms.Form):
     """
     A form for handling bulk IP range submissions in the django admin.
@@ -119,7 +120,8 @@ class ImporterCSVForm(forms.Form):
     )
     csv_file = FileField(required=False)
     skip_preview = BooleanField(
-        required=False, help_text=("Do not show the preview of what would happen. "),
+        required=False,
+        help_text=("Do not show the preview of what would happen. "),
     )
 
     ip_ranges = []
@@ -134,9 +136,11 @@ def initialize_importer(self):
 
         uploaded_csv_string = self.cleaned_data["csv_file"].read().decode("utf-8")
         csv_file = io.StringIO(uploaded_csv_string)
+        importer = CsvImporter(self.cleaned_data["provider"])
 
-        importer = bulk_importers.ImporterCSV(self.cleaned_data["provider"])
-        importer.ips_from_file(csv_file)
+        importable_values = importer.fetch_data_from_source(csv_file)
+
+        self.ip_ranges = importable_values
 
         self.importer = importer
         return importer
@@ -146,10 +150,11 @@ def get_ip_ranges(self):
         Return a list of the IP Ranges, showing which ones would be updated, and
         which ones would be created with this submission.
         """
-        importer = self.initialize_importer()
+        self.initialize_importer()
         provider = self.cleaned_data["provider"]
         # make preview of generated ips
-        ips = importer.preview(provider)
+        ips = self.importer.preview(provider, self.ip_ranges)
+        rich.print(ips)
         return ips
 
     def save(self):
@@ -157,7 +162,10 @@ def save(self):
         if self.importer is None:
             self.initialize_importer()
         logger.info("Skipping preview, running import")
-        self.importer.run()
+        import ipdb
+
+        ipdb.set_trace()
+        self.importer.process_addresses(self.ip_ranges)
         return self.importer
 
 
@@ -176,8 +184,7 @@ class Meta:
         )
 
     def save(self, commit=True):
-        """
-        """
+        """ """
         # Like the GreencheckIpForm, we non-staff user creates an ip, instead of saving
         self._save_approval()
         return super().save(commit=True)
@@ -246,8 +253,7 @@ class Meta:
         fields = "__all__"
 
     def save(self, commit=True):
-        """
-        """
+        """ """
         ip_instance = self.instance.greencheck_ip
         if commit is True:
             if ip_instance:

From a5cfeafdd28d429fc687610cf4979e6b0f5af931 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 19:34:28 +0000
Subject: [PATCH 15/21] Add __init__ file for  CSV Importer

---
 apps/greencheck/forms.py                   |   4 +-
 apps/greencheck/importers/__init__.py      |   1 +
 apps/greencheck/importers/importer_csv.py  |   2 +-
 apps/greencheck/test_importer_csv.py       | 125 +++++++++++++++++++++
 apps/greencheck/tests/test_importer_csv.py |  14 +--
 5 files changed, 136 insertions(+), 10 deletions(-)
 create mode 100644 apps/greencheck/importers/__init__.py
 create mode 100644 apps/greencheck/test_importer_csv.py

diff --git a/apps/greencheck/forms.py b/apps/greencheck/forms.py
index 8e7be957..7e9e1ca7 100644
--- a/apps/greencheck/forms.py
+++ b/apps/greencheck/forms.py
@@ -15,7 +15,7 @@
 from .models import GreencheckIp
 from .models import GreencheckIpApprove
 from .models import GreencheckASN, GreencheckASNapprove
-from .importers import CsvImporter
+from .importers import CSVImporter
 from ..accounts import models as ac_models
 
 User = get_user_model()
@@ -136,7 +136,7 @@ def initialize_importer(self):
 
         uploaded_csv_string = self.cleaned_data["csv_file"].read().decode("utf-8")
         csv_file = io.StringIO(uploaded_csv_string)
-        importer = CsvImporter(self.cleaned_data["provider"])
+        importer = CSVImporter(self.cleaned_data["provider"])
 
         importable_values = importer.fetch_data_from_source(csv_file)
 
diff --git a/apps/greencheck/importers/__init__.py b/apps/greencheck/importers/__init__.py
new file mode 100644
index 00000000..235e23c8
--- /dev/null
+++ b/apps/greencheck/importers/__init__.py
@@ -0,0 +1 @@
+from .importer_csv import CSVImporter
diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index 32151909..b1101869 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -16,7 +16,7 @@
 logger = logging.getLogger(__name__)
 
 
-class CsvImporter(BaseImporter):
+class CSVImporter(BaseImporter):
     def __init__(self, provider_id: int):
         self.hosting_provider_id = provider_id
 
diff --git a/apps/greencheck/test_importer_csv.py b/apps/greencheck/test_importer_csv.py
new file mode 100644
index 00000000..f30832ee
--- /dev/null
+++ b/apps/greencheck/test_importer_csv.py
@@ -0,0 +1,125 @@
+import pytest
+import pathlib
+import re
+import pandas as pd
+import ipdb
+from io import StringIO
+
+from django.core.management import call_command
+from apps.accounts.models.hosting import Hostingprovider
+from apps.greencheck.importers.importer_csv import CSVImporter
+
+from django.conf import settings
+
+
+@pytest.fixture
+def sample_data_raw():
+    """
+    Retrieve a locally saved sample of the population to use for this test
+    Return: CSV
+    """
+    this_file = pathlib.Path(__file__)
+    csv_path = this_file.parent.parent.joinpath("fixtures", "test_dataset_csv.csv")
+    return pd.read_csv(csv_path, header=None)
+
+
+@pytest.fixture
+def sample_data_as_list(sample_data_raw, hosting_provider: Hostingprovider):
+    """
+    Retrieve a locally saved sample of the population to use for this test and parse it to a list
+    Return: List
+    """
+    importer = CSVImporter()
+    return importer.parse_to_list(sample_data_raw)
+
+
+@pytest.mark.django_db
+class TestCSVImporter:
+    def test_parse_to_list(self, sample_data_raw, hosting_provider: Hostingprovider):
+        """
+        Test the parsing function.
+        """
+        # Initialize Csv importer
+        hosting_provider.save()
+        importer = CSVImporter(hosting_provider.id)
+
+        # Run parse list with sample data
+        list_of_addresses = importer.parse_to_list(sample_data_raw)
+
+        # Test: resulting list contains some items
+        assert len(list_of_addresses) > 0
+
+        # do we have an ip network?
+        assert "104.21.2.0/24" in list_of_addresses
+
+        # have we filtered out our incorrect IP network?
+        assert "104.21.2.192/24" not in list_of_addresses
+
+        # do we have our expected AS number?
+        assert "AS234" in list_of_addresses
+
+        # have we filtered out our bad AS line?
+        assert "AS" not in list_of_addresses
+
+        # do we have an IP range in our list
+        expected_ip_range = ("104.21.2.197", "104.21.2.199")
+        assert expected_ip_range in list_of_addresses
+
+    def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
+
+        # Initialize Csv importer
+        hosting_provider.save()
+        importer = CSVImporter(hosting_provider.id)
+
+        # Run parse list with sample data
+        list_of_addresses = importer.parse_to_list(sample_data_raw)
+        created_networks = importer.process_addresses(list_of_addresses)
+
+        # we should have seen one AS network added
+        assert "1 ASN" in created_networks
+        # have we created two new IP ranges?
+        assert "2 IP" in created_networks
+
+    def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
+        """
+        Can we see a representation of the data we would import before
+        we run the import it?
+        """
+        # Initialize Csv importer
+        hosting_provider.save()
+        importer = CSVImporter(hosting_provider.id)
+
+        # Run parse list with sample data
+        list_of_addresses = importer.parse_to_list(sample_data_raw)
+        preview = importer.preview(hosting_provider, list_of_addresses)
+
+        assert len(preview["green_ips"]) == 2
+        assert len(preview["green_asns"]) == 1
+
+
+# @pytest.mark.django_db
+# class TestCsvImportCommand:
+#     """
+#     This just tests that we have a management command that can run.
+#     """
+
+#     def test_handle(self, mocker, sample_data_as_list):
+#         # mock the call to retrieve from source, to a locally stored
+#         # testing sample. By instead using the test sample,
+#         # we avoid unnecessary network requests.
+
+#         # identify method we want to mock
+#         path_to_mock = (
+#             "apps.greencheck.importers.importer_csv."
+#             "CSVImporter.fetch_data_from_source"
+#         )
+
+#         # define a different return when the targeted mock
+#         # method is called
+#         mocker.patch(
+#             path_to_mock,
+#             return_value=sample_data_as_list,
+#         )
+
+#         # TODO: Do we need this call command?
+#         # call_command("update_networks_in_db_csv")
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index ca9e0621..f30832ee 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -7,7 +7,7 @@
 
 from django.core.management import call_command
 from apps.accounts.models.hosting import Hostingprovider
-from apps.greencheck.importers.importer_csv import CsvImporter
+from apps.greencheck.importers.importer_csv import CSVImporter
 
 from django.conf import settings
 
@@ -29,19 +29,19 @@ def sample_data_as_list(sample_data_raw, hosting_provider: Hostingprovider):
     Retrieve a locally saved sample of the population to use for this test and parse it to a list
     Return: List
     """
-    importer = CsvImporter()
+    importer = CSVImporter()
     return importer.parse_to_list(sample_data_raw)
 
 
 @pytest.mark.django_db
-class TestCsvImporter:
+class TestCSVImporter:
     def test_parse_to_list(self, sample_data_raw, hosting_provider: Hostingprovider):
         """
         Test the parsing function.
         """
         # Initialize Csv importer
         hosting_provider.save()
-        importer = CsvImporter(hosting_provider.id)
+        importer = CSVImporter(hosting_provider.id)
 
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
@@ -69,7 +69,7 @@ def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovide
 
         # Initialize Csv importer
         hosting_provider.save()
-        importer = CsvImporter(hosting_provider.id)
+        importer = CSVImporter(hosting_provider.id)
 
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
@@ -87,7 +87,7 @@ def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovide
         """
         # Initialize Csv importer
         hosting_provider.save()
-        importer = CsvImporter(hosting_provider.id)
+        importer = CSVImporter(hosting_provider.id)
 
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
@@ -111,7 +111,7 @@ def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovide
 #         # identify method we want to mock
 #         path_to_mock = (
 #             "apps.greencheck.importers.importer_csv."
-#             "CsvImporter.fetch_data_from_source"
+#             "CSVImporter.fetch_data_from_source"
 #         )
 
 #         # define a different return when the targeted mock

From 0c7534d37ded0da4de6179ed329972367ffa0ece Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Thu, 1 Sep 2022 20:35:45 +0000
Subject: [PATCH 16/21] Update the importer to work with hosting provider
 objects not ids

---
 apps/accounts/admin.py                        |   9 +-
 apps/greencheck/forms.py                      |  10 +-
 apps/greencheck/importers/importer_csv.py     |   4 +-
 .../importers/importer_interface.py           |  12 +-
 apps/greencheck/test_importer_csv.py          | 125 ------------------
 apps/greencheck/tests/test_importer_csv.py    |  10 +-
 6 files changed, 22 insertions(+), 148 deletions(-)
 delete mode 100644 apps/greencheck/test_importer_csv.py

diff --git a/apps/accounts/admin.py b/apps/accounts/admin.py
index 7922f315..edae3573 100644
--- a/apps/accounts/admin.py
+++ b/apps/accounts/admin.py
@@ -399,15 +399,14 @@ def save_import_from_csv(self, request, *args, **kwargs):
             valid = form.is_valid()
             skip_preview = form.cleaned_data["skip_preview"]
 
-            rich.inspect(request.POST)
-            rich.inspect(form.cleaned_data)
             if valid and skip_preview:
                 # not doing preview. Run the import
-                form.save()
+                completed_importer = form.save()
 
-                ip_ranges = form.get_ip_ranges()
+                # TODO show the updated files
+                
                 context = {
-                    "ip_ranges": ip_ranges,
+                    "ip_ranges": [],
                     "provider": provider,
                 }
                 return render(request, "import_csv_preview.html", context,)
diff --git a/apps/greencheck/forms.py b/apps/greencheck/forms.py
index 7e9e1ca7..41e4f575 100644
--- a/apps/greencheck/forms.py
+++ b/apps/greencheck/forms.py
@@ -154,7 +154,6 @@ def get_ip_ranges(self):
         provider = self.cleaned_data["provider"]
         # make preview of generated ips
         ips = self.importer.preview(provider, self.ip_ranges)
-        rich.print(ips)
         return ips
 
     def save(self):
@@ -162,11 +161,12 @@ def save(self):
         if self.importer is None:
             self.initialize_importer()
         logger.info("Skipping preview, running import")
-        import ipdb
-
-        ipdb.set_trace()
+        
+        provider = self.cleaned_data["provider"]
+        
         self.importer.process_addresses(self.ip_ranges)
-        return self.importer
+        
+        return self.importer.preview(provider, self.ip_ranges)
 
 
 class GreencheckAsnForm(ModelForm, ApprovalMixin):
diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index b1101869..67a620cd 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -17,8 +17,8 @@
 
 
 class CSVImporter(BaseImporter):
-    def __init__(self, provider_id: int):
-        self.hosting_provider_id = provider_id
+    def __init__(self, provider: Hostingprovider):
+        self.hosting_provider = provider
 
     def fetch_data_from_source(cls, filepath_or_buffer) -> List:
         """
diff --git a/apps/greencheck/importers/importer_interface.py b/apps/greencheck/importers/importer_interface.py
index c75d92b9..47af7452 100644
--- a/apps/greencheck/importers/importer_interface.py
+++ b/apps/greencheck/importers/importer_interface.py
@@ -29,7 +29,7 @@ def parse_to_list(self) -> list:
 
 
 class BaseImporter:
-    hosting_provider_id: int
+    hosting_provider: Hostingprovider
 
     def is_ip_range(self, address: Union[str, tuple]) -> Union[tuple, bool]:
         """
@@ -131,10 +131,12 @@ def process_addresses(self, list_of_addresses: list):
             )
 
     def save_asn(self, address: str):
-        hoster = Hostingprovider.objects.get(pk=self.hosting_provider_id)
+        
 
         gc_asn, created = GreencheckASN.objects.update_or_create(
-            active=True, asn=int(address.replace("AS", "")), hostingprovider=hoster.id
+            active=True, 
+            asn=int(address.replace("AS", "")),
+            hostingprovider=self.hosting_provider
         )
         gc_asn.save()  # Save the newly created or updated object
 
@@ -153,13 +155,11 @@ def save_ip(self, address):
             start_address = network[1]
             ending_address = network[-1]
 
-        hoster = Hostingprovider.objects.get(pk=self.hosting_provider_id)
-
         gc_ip, created = GreencheckIp.objects.update_or_create(
             active=True,
             ip_start=start_address,
             ip_end=ending_address,
-            hostingprovider=hoster,
+            hostingprovider=self.hosting_provider,
         )
         gc_ip.save()  # Save the newly created or updated object
 
diff --git a/apps/greencheck/test_importer_csv.py b/apps/greencheck/test_importer_csv.py
deleted file mode 100644
index f30832ee..00000000
--- a/apps/greencheck/test_importer_csv.py
+++ /dev/null
@@ -1,125 +0,0 @@
-import pytest
-import pathlib
-import re
-import pandas as pd
-import ipdb
-from io import StringIO
-
-from django.core.management import call_command
-from apps.accounts.models.hosting import Hostingprovider
-from apps.greencheck.importers.importer_csv import CSVImporter
-
-from django.conf import settings
-
-
-@pytest.fixture
-def sample_data_raw():
-    """
-    Retrieve a locally saved sample of the population to use for this test
-    Return: CSV
-    """
-    this_file = pathlib.Path(__file__)
-    csv_path = this_file.parent.parent.joinpath("fixtures", "test_dataset_csv.csv")
-    return pd.read_csv(csv_path, header=None)
-
-
-@pytest.fixture
-def sample_data_as_list(sample_data_raw, hosting_provider: Hostingprovider):
-    """
-    Retrieve a locally saved sample of the population to use for this test and parse it to a list
-    Return: List
-    """
-    importer = CSVImporter()
-    return importer.parse_to_list(sample_data_raw)
-
-
-@pytest.mark.django_db
-class TestCSVImporter:
-    def test_parse_to_list(self, sample_data_raw, hosting_provider: Hostingprovider):
-        """
-        Test the parsing function.
-        """
-        # Initialize Csv importer
-        hosting_provider.save()
-        importer = CSVImporter(hosting_provider.id)
-
-        # Run parse list with sample data
-        list_of_addresses = importer.parse_to_list(sample_data_raw)
-
-        # Test: resulting list contains some items
-        assert len(list_of_addresses) > 0
-
-        # do we have an ip network?
-        assert "104.21.2.0/24" in list_of_addresses
-
-        # have we filtered out our incorrect IP network?
-        assert "104.21.2.192/24" not in list_of_addresses
-
-        # do we have our expected AS number?
-        assert "AS234" in list_of_addresses
-
-        # have we filtered out our bad AS line?
-        assert "AS" not in list_of_addresses
-
-        # do we have an IP range in our list
-        expected_ip_range = ("104.21.2.197", "104.21.2.199")
-        assert expected_ip_range in list_of_addresses
-
-    def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
-
-        # Initialize Csv importer
-        hosting_provider.save()
-        importer = CSVImporter(hosting_provider.id)
-
-        # Run parse list with sample data
-        list_of_addresses = importer.parse_to_list(sample_data_raw)
-        created_networks = importer.process_addresses(list_of_addresses)
-
-        # we should have seen one AS network added
-        assert "1 ASN" in created_networks
-        # have we created two new IP ranges?
-        assert "2 IP" in created_networks
-
-    def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
-        """
-        Can we see a representation of the data we would import before
-        we run the import it?
-        """
-        # Initialize Csv importer
-        hosting_provider.save()
-        importer = CSVImporter(hosting_provider.id)
-
-        # Run parse list with sample data
-        list_of_addresses = importer.parse_to_list(sample_data_raw)
-        preview = importer.preview(hosting_provider, list_of_addresses)
-
-        assert len(preview["green_ips"]) == 2
-        assert len(preview["green_asns"]) == 1
-
-
-# @pytest.mark.django_db
-# class TestCsvImportCommand:
-#     """
-#     This just tests that we have a management command that can run.
-#     """
-
-#     def test_handle(self, mocker, sample_data_as_list):
-#         # mock the call to retrieve from source, to a locally stored
-#         # testing sample. By instead using the test sample,
-#         # we avoid unnecessary network requests.
-
-#         # identify method we want to mock
-#         path_to_mock = (
-#             "apps.greencheck.importers.importer_csv."
-#             "CSVImporter.fetch_data_from_source"
-#         )
-
-#         # define a different return when the targeted mock
-#         # method is called
-#         mocker.patch(
-#             path_to_mock,
-#             return_value=sample_data_as_list,
-#         )
-
-#         # TODO: Do we need this call command?
-#         # call_command("update_networks_in_db_csv")
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index f30832ee..56010cc5 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -18,8 +18,7 @@ def sample_data_raw():
     Retrieve a locally saved sample of the population to use for this test
     Return: CSV
     """
-    this_file = pathlib.Path(__file__)
-    csv_path = this_file.parent.parent.joinpath("fixtures", "test_dataset_csv.csv")
+    csv_path = pathlib.Path(settings.ROOT) / "apps" / "greencheck" / "fixtures" / "test_dataset_csv.csv"
     return pd.read_csv(csv_path, header=None)
 
 
@@ -41,7 +40,7 @@ def test_parse_to_list(self, sample_data_raw, hosting_provider: Hostingprovider)
         """
         # Initialize Csv importer
         hosting_provider.save()
-        importer = CSVImporter(hosting_provider.id)
+        importer = CSVImporter(hosting_provider)
 
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
@@ -69,12 +68,13 @@ def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovide
 
         # Initialize Csv importer
         hosting_provider.save()
-        importer = CSVImporter(hosting_provider.id)
+        importer = CSVImporter(hosting_provider)
 
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
         created_networks = importer.process_addresses(list_of_addresses)
-
+        import rich
+        rich.print(created_networks)
         # we should have seen one AS network added
         assert "1 ASN" in created_networks
         # have we created two new IP ranges?

From 9da0b04e6e15f7326b908ed507c720b743504225 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Fri, 2 Sep 2022 04:35:40 +0000
Subject: [PATCH 17/21] Add tests to check that the importer saves to db

---
 apps/greencheck/tests/test_importer_csv.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index 56010cc5..12326c51 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -79,6 +79,23 @@ def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovide
         assert "1 ASN" in created_networks
         # have we created two new IP ranges?
         assert "2 IP" in created_networks
+        
+        # have we created the new Green ASN in the db?
+        green_asns = hosting_provider.greencheckasn_set.all()
+        assert green_asns .first().asn == 234
+
+        # have we created the green ip ranges in the db?
+        green_ips = hosting_provider.greencheckip_set.all().order_by('ip_start')
+        
+        # have we converted a network to a range?
+        assert green_ips[0].ip_start == "104.21.2.1"
+        assert green_ips[0].ip_end == "104.21.2.255"
+        
+        # have do we have the range added as well?
+        assert green_ips[1].ip_start == "104.21.2.197"
+        assert green_ips[1].ip_end == "104.21.2.199"
+        
+        
 
     def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
         """

From fbc00682b968ce91c34c48b70ec85ca0f147079e Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Fri, 2 Sep 2022 05:55:48 +0000
Subject: [PATCH 18/21] Update flow for import show imported networks

---
 apps/accounts/admin.py                        |  6 +-
 .../templates/import_csv_preview.html         | 60 ++-----------------
 .../templates/import_csv_results.html         | 60 ++++++++++++++++---
 apps/accounts/templates/import_csv_start.html | 50 ++++++++++++++--
 apps/greencheck/forms.py                      | 12 +++-
 apps/greencheck/importers/importer_csv.py     |  1 +
 apps/greencheck/tests/test_importer_csv.py    | 36 +++++++++--
 7 files changed, 148 insertions(+), 77 deletions(-)

diff --git a/apps/accounts/admin.py b/apps/accounts/admin.py
index edae3573..37944493 100644
--- a/apps/accounts/admin.py
+++ b/apps/accounts/admin.py
@@ -404,12 +404,12 @@ def save_import_from_csv(self, request, *args, **kwargs):
                 completed_importer = form.save()
 
                 # TODO show the updated files
-                
+        
                 context = {
-                    "ip_ranges": [],
+                    "ip_ranges": completed_importer,
                     "provider": provider,
                 }
-                return render(request, "import_csv_preview.html", context,)
+                return render(request, "import_csv_results.html", context,)
 
             if valid:
                 # the save default we don't save the contents
diff --git a/apps/accounts/templates/import_csv_preview.html b/apps/accounts/templates/import_csv_preview.html
index d1c8dd95..929f9b64 100644
--- a/apps/accounts/templates/import_csv_preview.html
+++ b/apps/accounts/templates/import_csv_preview.html
@@ -4,60 +4,7 @@
 {% block pretitle %}
 <h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
 
-<p> If you have a large number of CSV files, you can import them in bulk.</p>
 
-<p>Make sure the form follows the sample format, without the header row</p>
-
-<table>
-<th>IP Range start</th><th>Ip Range End</th>
-<tr>
-  <td>104.21.2.197</td>
-  <td>104.21.2.199</td>
-
-</tr>
-<tr>
-  <td>104.21.2.0/24</td>
-  <td></td>
-</tr>
-<tr>
-  <td>AS12345</td>
-  <td></td>
-</tr>
-
-</table>
-
-<p>If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.</p>
-<p>So, uploading a CSV file with a single column IP addresses would be the equivalent to:</p>
-
-
-<table>
-<th>IP Range start</th><th>Ip Range End</th>
-<tr>
-  <td>104.21.2.197</td>
-  <td>104.21.2.197</td>
-
-</tr>
-</table>
-
-<p> If you have an an IP range to import, add it all in the first column. It will be expanded into a full IP range</p>
-
-<table>
-<th>IP Range start</th><th>Ip Range End</th>
-<tr>
-  <td>104.21.2.0/24</td>
-  <td></td>
-</tr>
-</table>
-
-<p>Likewise with AS numbers. Do not as a space between 'AS' and the number itself.</p>
-
-<table>
-<th>IP Range start</th><th>Ip Range End</th>
-<tr>
-  <td>AS12345</td>
-  <td></td>
-</tr>
-</table>
 {% endblock %}
 
 {% block content %}
@@ -126,8 +73,13 @@ <h2>AS Import Preview</h2>
     <input type="submit" value="Submit"
         style="margin-top: 0px; padding: 6px 15px">
 
-      <a href="{% url 'greenweb_admin:accounts_hostingprovider_start_import_from_csv' provider.id %}">Back to start</a>
+
+
+      <a style="margin-left:1rem;margin-right:1rem;" href="{% url 'greenweb_admin:accounts_hostingprovider_start_import_from_csv' provider.id %}">Back to import start</a>
+      
+      <a style="margin-left:1rem;margin-right:1rem;" class="" href="{% url 'greenweb_admin:accounts_hostingprovider_change' provider.id %}">Back to hosting provider</a>
 </form>
 
 
+
 {% endblock content %}
diff --git a/apps/accounts/templates/import_csv_results.html b/apps/accounts/templates/import_csv_results.html
index 081f9289..ec259fde 100644
--- a/apps/accounts/templates/import_csv_results.html
+++ b/apps/accounts/templates/import_csv_results.html
@@ -2,25 +2,24 @@
 {% load i18n static %}
 
 {% block pretitle %}
-<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
-
-<p> If you have a large number of CSV files, you can import them in bulk.</p>
+<h1>CSV Import Results for {{ provider }}</h1>
 {% endblock %}
 
 {% block content %}
 
+
 {% if ip_ranges %}
 
 <hr / style="margin-top:3rem; margin-bottom:3rem;">
 
-<h2>IP Range Import Preview</h2>
+<h2>IP Range Import </h2>
 
-<p>The following IP ranges have been created, and updated for <strong>{{ provider }}</strong>:</p>
+<p>The following IP ranges were imported for <strong>{{ provider }}</strong>:</p>
 
 <table>
-<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th><th>Length</th>
 
-{% for ip in ip_ranges %}
+{% for ip in ip_ranges.green_ips %}
 
   <tr>
     <td>{{ ip.ip_start }}</td>
@@ -32,6 +31,32 @@ <h2>IP Range Import Preview</h2>
           Created
       {% endif %}
     </td>
+    <td>
+      {{  ip.ip_range_length }}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+<h2>AS Import</h2>
+
+<p>The following AS Numbers ranges were imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>AS number</th><th>Created / Updated</th>
+
+{% for as in ip_ranges.green_asns %}
+
+  <tr>
+    <td>{{ as.asn }}</td>
+    <td>
+      {% if as.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
   </tr>
 
 {% endfor %}
@@ -39,4 +64,23 @@ <h2>IP Range Import Preview</h2>
 
 {% endif %}
 
-{% endblock content %}
\ No newline at end of file
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<style>
+.submit-row a.button{ 
+  margin-right: 1rem;
+}
+.submit-row a.button:visited{
+  color: #fff;
+}
+</style>
+
+<div class="submit-row">
+      <a class="button" href="{% url 'greenweb_admin:accounts_hostingprovider_change' provider.id %}">Back to hosting provider</a>
+
+      |
+
+      <a style="margin-left:1rem;" class="button" href="{% url 'greenweb_admin:accounts_hostingprovider_start_import_from_csv' provider.id %}">Make another CSV Import</a>
+</form>
+
+{% endblock content %}
diff --git a/apps/accounts/templates/import_csv_start.html b/apps/accounts/templates/import_csv_start.html
index 6795e1a3..e5052f9b 100644
--- a/apps/accounts/templates/import_csv_start.html
+++ b/apps/accounts/templates/import_csv_start.html
@@ -4,9 +4,13 @@
 {% block pretitle %}
 <h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
 
-<p> If you have a large number of Ip addresses or IP ranges, you can import them in bulk.</p>
+<details>
+<summary>Formatting notes for uploading CSVs</summary>
 
-<p>Make sure the uploaded file follows the sample format</p>
+<div style="padding:1rem; background-color:#eee; border-radius: 5px; margin-top: 1rem;">
+  <p> If you have a large number of CSV files, you can import them in bulk.</p>
+
+<p>Make sure the form follows the sample format, without the header row</p>
 
 <table>
 <th>IP Range start</th><th>Ip Range End</th>
@@ -15,11 +19,21 @@ <h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
   <td>104.21.2.199</td>
 
 </tr>
+<tr>
+  <td>104.21.2.0/24</td>
+  <td></td>
+</tr>
+<tr>
+  <td>AS12345</td>
+  <td></td>
+</tr>
+
 </table>
 
 <p>If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.</p>
 <p>So, uploading a CSV file with a single column IP addresses would be the equivalent to:</p>
 
+
 <table>
 <th>IP Range start</th><th>Ip Range End</th>
 <tr>
@@ -29,20 +43,48 @@ <h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
 </tr>
 </table>
 
+<p> If you have an an IP range to import, add it all in the first column. It will be expanded into a full IP range</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.0/24</td>
+  <td></td>
+</tr>
+</table>
+
+<p>Likewise with AS numbers. Do not as a space between 'AS' and the number itself.</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>AS12345</td>
+  <td></td>
+</tr>
+</table>
+</div>
+
+</details>
+
+
 {% endblock %}
 
 
 {% block content %}
 
-<hr / style="margin-top:3rem; margin-bottom:3rem;">
+<hr / style="margin-top:1rem; margin-bottom:1rem;">
 
 <form enctype="multipart/form-data" action="{% url 'greenweb_admin:accounts_hostingprovider_save_import_from_csv' provider.id %}" method="post">
     {% csrf_token %}
     {{ form.as_p }}
     <input type="submit" value="Submit"
         style="margin-top: 0px; padding: 6px 15px">
+    <a class="" href="{% url 'greenweb_admin:accounts_hostingprovider_change' provider.id %}">Back to hosting provider</a>
+
 </form>
 
+
+
 {% if ip_ranges %}
 
 <hr / style="margin-top:3rem; margin-bottom:3rem;">
@@ -73,4 +115,4 @@ <h2>IP Range Import Preview</h2>
 
 {% endif %}
 
-{% endblock content %}
\ No newline at end of file
+{% endblock content %}
diff --git a/apps/greencheck/forms.py b/apps/greencheck/forms.py
index 41e4f575..a9cf2329 100644
--- a/apps/greencheck/forms.py
+++ b/apps/greencheck/forms.py
@@ -121,10 +121,16 @@ class ImporterCSVForm(forms.Form):
     csv_file = FileField(required=False)
     skip_preview = BooleanField(
         required=False,
-        help_text=("Do not show the preview of what would happen. "),
+        help_text=("Do not show the preview of what would happen. Save to the import to the database"),
+    )
+    replace_with_import = BooleanField(
+        required=False,
+        label=("Replace networks with this import"),
+        help_text=("Replace all the networks assigned to this hoster with the networks in this import"),
     )
 
     ip_ranges = []
+    processed_ips = []
     importer = None
 
     def initialize_importer(self):
@@ -165,8 +171,8 @@ def save(self):
         provider = self.cleaned_data["provider"]
         
         self.importer.process_addresses(self.ip_ranges)
-        
-        return self.importer.preview(provider, self.ip_ranges)
+        self.processed_ips = self.importer.preview(provider, self.ip_ranges)
+        return self.processed_ips
 
 
 class GreencheckAsnForm(ModelForm, ApprovalMixin):
diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py
index 67a620cd..17e34260 100644
--- a/apps/greencheck/importers/importer_csv.py
+++ b/apps/greencheck/importers/importer_csv.py
@@ -19,6 +19,7 @@
 class CSVImporter(BaseImporter):
     def __init__(self, provider: Hostingprovider):
         self.hosting_provider = provider
+        self.processed_ips = []
 
     def fetch_data_from_source(cls, filepath_or_buffer) -> List:
         """
diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py
index 12326c51..bed0e697 100644
--- a/apps/greencheck/tests/test_importer_csv.py
+++ b/apps/greencheck/tests/test_importer_csv.py
@@ -73,8 +73,7 @@ def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovide
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
         created_networks = importer.process_addresses(list_of_addresses)
-        import rich
-        rich.print(created_networks)
+        
         # we should have seen one AS network added
         assert "1 ASN" in created_networks
         # have we created two new IP ranges?
@@ -95,8 +94,6 @@ def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovide
         assert green_ips[1].ip_start == "104.21.2.197"
         assert green_ips[1].ip_end == "104.21.2.199"
         
-        
-
     def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
         """
         Can we see a representation of the data we would import before
@@ -104,7 +101,7 @@ def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovide
         """
         # Initialize Csv importer
         hosting_provider.save()
-        importer = CSVImporter(hosting_provider.id)
+        importer = CSVImporter(hosting_provider)
 
         # Run parse list with sample data
         list_of_addresses = importer.parse_to_list(sample_data_raw)
@@ -113,6 +110,35 @@ def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovide
         assert len(preview["green_ips"]) == 2
         assert len(preview["green_asns"]) == 1
 
+    def test_view_processed_imports(self, sample_data_raw, hosting_provider: Hostingprovider):
+        """
+        Can we compare the state of an import to the networks already in the database
+        for this provider?
+        """
+        # Initialize Csv importer
+        hosting_provider.save()
+        importer = CSVImporter(hosting_provider)
+
+        # Run parse list with sample data
+        list_of_addresses = importer.parse_to_list(sample_data_raw)
+        # Run our import to save them to the database, simulating saving 
+        # via our the form
+        created_networks = importer.process_addresses(list_of_addresses)
+        # Generate a view of the data, to check if we are fetching from 
+        # the database now
+        preview = importer.preview(hosting_provider, list_of_addresses)
+        
+        green_ips = [gip for gip in preview['green_ips']]
+        green_asns = [gip for gip in preview['green_asns']]
+        
+        # are these the IPs checked against the database?
+        for green_ip in green_ips:
+            assert green_ip.id is not None
+            
+        # are these the ASNs checked against the database?
+        for green_asn in green_asns:
+            assert green_asn.id is not None
+
 
 # @pytest.mark.django_db
 # class TestCsvImportCommand:

From 360185cf51767c5997254f4c77beb40d4eb932ad Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Fri, 2 Sep 2022 05:56:58 +0000
Subject: [PATCH 19/21] Comment out unimplemented "replace" feature

---
 apps/greencheck/forms.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/greencheck/forms.py b/apps/greencheck/forms.py
index a9cf2329..1d759275 100644
--- a/apps/greencheck/forms.py
+++ b/apps/greencheck/forms.py
@@ -123,11 +123,11 @@ class ImporterCSVForm(forms.Form):
         required=False,
         help_text=("Do not show the preview of what would happen. Save to the import to the database"),
     )
-    replace_with_import = BooleanField(
-        required=False,
-        label=("Replace networks with this import"),
-        help_text=("Replace all the networks assigned to this hoster with the networks in this import"),
-    )
+    # replace_with_import = BooleanField(
+    #     required=False,
+    #     label=("Replace networks with this import"),
+    #     help_text=("Replace all the networks assigned to this hoster with the networks in this import"),
+    # )
 
     ip_ranges = []
     processed_ips = []

From 94dd85234df03e89fc12003d0c28c012b2f9d3e5 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Fri, 2 Sep 2022 06:16:57 +0000
Subject: [PATCH 20/21] update importer to use sightly more idiomatic python

---
 apps/greencheck/importers/importer_interface.py |  3 +++
 apps/greencheck/tests/test_base_importer.py     | 17 ++++++++---------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/apps/greencheck/importers/importer_interface.py b/apps/greencheck/importers/importer_interface.py
index 47af7452..2474ae86 100644
--- a/apps/greencheck/importers/importer_interface.py
+++ b/apps/greencheck/importers/importer_interface.py
@@ -30,6 +30,9 @@ def parse_to_list(self) -> list:
 
 class BaseImporter:
     hosting_provider: Hostingprovider
+    def __init__(self, provider):
+        self.hosting_provider = provider
+    
 
     def is_ip_range(self, address: Union[str, tuple]) -> Union[tuple, bool]:
         """
diff --git a/apps/greencheck/tests/test_base_importer.py b/apps/greencheck/tests/test_base_importer.py
index 404eeb7e..21ba26e9 100644
--- a/apps/greencheck/tests/test_base_importer.py
+++ b/apps/greencheck/tests/test_base_importer.py
@@ -33,13 +33,12 @@ def hosting_provider():
 
 
 @pytest.fixture
-def base_importer():
+def base_importer(hosting_provider: Hostingprovider):
     """
     Initialize a BaseImporter object
     Return: BaseImporter
     """
-    importer = BaseImporter
-    importer.hosting_provider_id = 1234
+    importer = BaseImporter(hosting_provider)
     return importer
 
 
@@ -77,7 +76,7 @@ def test_save_ip(self, hosting_provider, base_importer):
         hosting_provider.save()  # Initialize hosting provider in database
 
         # Import a single IPv4 network
-        BaseImporter.save_ip(base_importer, testing_ipv4_range)
+        base_importer.save_ip(testing_ipv4_range)
 
         assert (
             GreencheckIp.objects.all().count() == 1
@@ -85,7 +84,7 @@ def test_save_ip(self, hosting_provider, base_importer):
         hosting_provider.save()  # Initialize hosting provider in database
 
         # Import a single IPv4 network
-        BaseImporter.save_ip(base_importer, testing_ipv6_range)
+        base_importer.save_ip(testing_ipv6_range)
 
         assert (
             GreencheckIp.objects.all().count() == 2
@@ -93,14 +92,14 @@ def test_save_ip(self, hosting_provider, base_importer):
         hosting_provider.save()  # Initialize hosting provider in database
 
         # Import a single IPv4 network
-        BaseImporter.save_ip(base_importer, testing_ipv4_network)
+        base_importer.save_ip(testing_ipv4_network)
 
         assert (
             GreencheckIp.objects.all().count() == 3
         )  # Test: IPv4 is saved after insertion
 
         # Import a single IPv6 network
-        BaseImporter.save_ip(base_importer, testing_ipv6_network)
+        base_importer.save_ip(testing_ipv6_network)
 
         assert (
             GreencheckIp.objects.all().count() == 4
@@ -138,8 +137,8 @@ def test_process_addresses(self, hosting_provider, base_importer, sample_data):
         hosting_provider.save()  # Initialize hosting provider in database
 
         # Process list of addresses in JSON file
-        importer = BaseImporter()
-        importer.process_addresses(sample_data)
+        
+        base_importer.process_addresses(sample_data)
 
         assert (
             GreencheckIp.objects.all().count() == 63

From 5d068e36409ad07ac4a4aed571a0c524dae0c453 Mon Sep 17 00:00:00 2001
From: Chris Adams <chris@productscience.co.uk>
Date: Fri, 2 Sep 2022 06:36:27 +0000
Subject: [PATCH 21/21] Move upload CSV option to admin section

So only admins can see it and use it
---
 apps/accounts/admin.py      |  4 +---
 apps/accounts/admin_site.py | 13 -------------
 2 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/apps/accounts/admin.py b/apps/accounts/admin.py
index 37944493..1e51ad0d 100644
--- a/apps/accounts/admin.py
+++ b/apps/accounts/admin.py
@@ -402,8 +402,6 @@ def save_import_from_csv(self, request, *args, **kwargs):
             if valid and skip_preview:
                 # not doing preview. Run the import
                 completed_importer = form.save()
-
-                # TODO show the updated files
         
                 context = {
                     "ip_ranges": completed_importer,
@@ -656,7 +654,6 @@ def get_fieldsets(self, request, obj=None):
                 {
                     "fields": (
                         ("name", "website",), "country", "services",
-                        ("start_csv_import_button"),
                     )
                 },
             )
@@ -670,6 +667,7 @@ def get_fieldsets(self, request, obj=None):
                     ("partner", "model"),
                     ("staff_labels",),
                     ("email_template", "preview_email_button"),
+                    ("start_csv_import_button"),
                 )
             },
         )
diff --git a/apps/accounts/admin_site.py b/apps/accounts/admin_site.py
index 567bfabf..de54e0ea 100644
--- a/apps/accounts/admin_site.py
+++ b/apps/accounts/admin_site.py
@@ -172,19 +172,6 @@ def get_app_list(self, request):
                     }
                 ],
             },
-            {
-                "name": "Upload ip ranges",
-                "app_label": "greencheck",
-                "app_url": reverse("admin:import_ip_ranges"),
-                "models": [
-                    {
-                        "name": "As CSV",
-                        "object_name": "greencheck_url",
-                        "admin_url": reverse("admin:import_ip_ranges"),
-                        "view_only": True,
-                    }
-                ],
-            },
         ]
         return app_list