Improve command add jsondecodeerror

georiviere/observations/management/commands/__init__.py

@@ -1,3 +1,4 @@
+import json
 import requests
 
 from django.core.management.base import BaseCommand, CommandError
@@ -47,9 +48,11 @@ def handle(self, *args, **options):
 
         if verbosity >= 2:
             self.stdout.write('Get station from API {0}'.format(response.url))
-
-        response_content = response.json()
-
+        try:
+            response_content = response.json()
+        except json.decoder.JSONDecodeError:
+            self.stdout.write(f'Response is not a json')
+            return
         if verbosity >= 1:
             self.stdout.write('Import {1} stations from API {0}'.format(response.url, response_content['count']))
 
@@ -60,6 +63,10 @@ def handle(self, *args, **options):
             response = requests.get(response_content['next'])
             if verbosity >= 2:
                 self.stdout.write('Import next page from {0}'.format(response.url))
-            response_content = response.json()
+            try:
+                response_content = response.json()
+            except requests.exceptions.JSONDecodeError:
+                self.stdout.write(f'Response is not a json')
+                return
             results = response_content['data']
             self.create_or_update_stations(results, verbosity, with_parameters)

georiviere/observations/management/commands/import_hydrobiologie_stations.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import json
 import requests
 from django.contrib.gis.geos import Point
 from georiviere.observations.models import Station, StationProfile, Parameter, ParameterTracking, Unit
@@ -57,7 +58,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False):
                 except requests.exceptions.ConnectionError as e:
                     self.stdout.write(f'Limit of connection has been exceeded {e}')
                     continue
-                response_firstpage_content = response_firstpage.json()
+                try:
+                    response_firstpage_content = response_firstpage.json()
+                except json.JSONDecodeError:
+                    self.stdout.write(f'Response is not a json')
+                    continue
                 indices_data = response_firstpage_content['data']
 
                 for indice in indices_data:

georiviere/observations/management/commands/import_pcquality_stations.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import json
 import requests
 from django.contrib.gis.geos import Point
 from georiviere.observations.models import Station, StationProfile, Parameter, ParameterTracking, Unit
@@ -67,7 +68,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False):
                 except requests.exceptions.ConnectionError as e:
                     self.stdout.write(f'Limit of connection has been exceeded {e}')
                     continue
-                response_firstpage_content = response_firstpage.json()
+                try:
+                    response_firstpage_content = response_firstpage.json()
+                except json.JSONDecodeError:
+                    self.stdout.write(f'Response is not a json')
+                    continue
                 analysepc_data = response_firstpage_content['data']
 
                 # If there is more than one page, get data desc sorted
@@ -78,7 +83,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False):
                     except requests.exceptions.ConnectionError as e:
                         self.stdout.write(f'Limit of connection has been exceeded {e}')
                         continue
-                    response_desc_data = response_desc_results.json()['data']
+                    try:
+                        response_desc_data = response_desc_results.json()['data']
+                    except json.JSONDecodeError:
+                        self.stdout.write(f'Response is not a json')
+                        continue
                     analysepc_data = analysepc_data + response_desc_data
 
                 for measure in analysepc_data:

georiviere/observations/tests/test_commands.py

@@ -45,6 +45,51 @@ def requests_get_mock_response(*args, **kwargs):
     return mock_response
 
 
+def requests_get_mock_response_error_initial_url(*args, **kwargs):
+    mock_response = mock.Mock()
+    mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1)
+    mock_response.status_code = 200
+    return mock_response
+
+
+def requests_get_mock_response_error(*args, **kwargs):
+    """Build mock_response with test file data, according to api_url"""
+    # Get filename from api_url
+    mock_response = mock.Mock()
+    filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations.json'
+    if "urf" in args[0]:
+        filename = TEST_DATA_PATH / 'response_sandre_urf.json'
+    if "hydrometrie" in args[0]:
+        filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations.json'
+    if "temperature" in args[0]:
+        filename = TEST_DATA_PATH / 'response_api_temperature_stations.json'
+    if "station_pc" in args[0]:
+        filename = TEST_DATA_PATH / 'response_api_pcquality_stations.json'
+    if "analyse_pc" in args[0]:
+        if "sort" in kwargs['params']:
+            mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1)
+        else:
+            filename = TEST_DATA_PATH / 'response_api_pcquality_analyse.json'
+            if kwargs.get('params', {})["code_station"] == "05137100":
+                mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1)
+    if 'stations_hydrobio' in args[0]:
+        if kwargs.get('params', {}).get("code_station") == "06000874":
+            filename = TEST_DATA_PATH / 'response_api_hydrobio_stations.json'
+        else:
+            mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1)
+    if 'indices' in args[0]:
+        mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1)
+    if 'taxons' in args[0]:
+        mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1)
+    # Build response
+
+    with open(filename, 'r') as f:
+        expected_dict = json.load(f)
+    mock_response.json.return_value = expected_dict
+    mock_response.status_code = 200
+    return mock_response
+
+
 def requests_get_mock_response_500(*args, **kwargs):
     """Build mock_response like if server is down"""
     # Build response
@@ -75,6 +120,61 @@ def test_urf_imported(self, mock_get):
         self.assertIn('Created unit mg(Cl2)/L', out.getvalue())
 
 
+@mock.patch.object(requests, 'get', side_effect=requests_get_mock_response_error_initial_url)
+class ImportStationErrorTest(TestCase):
+    """Test import_station command
+    Test datas are from LABERGEMENT-STE-MARIE
+    """
+
+    def test_import_hydrometric_stations_error_json(self, mock_get):
+        out = StringIO()
+
+        # Call command
+        call_command('import_hydrometric_stations', with_parameters=True, stdout=out)
+
+        # Check stations imported
+        stations = Station.objects.all()
+        self.assertEqual(stations.count(), 0)
+        self.assertIn("Response is not a json", out.getvalue())
+        mock_get.side_effect = requests_get_mock_response_error
+        # Call command 2nd time
+        call_command('import_hydrometric_stations', with_parameters=True, stdout=out)
+
+        self.assertEqual(stations.count(), 2)
+
+    def test_import_pcquality_stations_error_json(self, mock_get):
+        out = StringIO()
+
+        # Call command
+        call_command('import_pcquality_stations', with_parameters=True, stdout=out)
+
+        # Check stations imported
+        stations = Station.objects.all()
+        self.assertEqual(stations.count(), 0)
+        self.assertIn("Response is not a json", out.getvalue())
+        mock_get.side_effect = requests_get_mock_response_error
+        # Call command 2nd time
+        call_command('import_pcquality_stations', with_parameters=True, stdout=out)
+
+        self.assertEqual(stations.count(), 28)
+
+    def test_import_temperature_stations_error_json(self, mock_get):
+        out = StringIO()
+
+        # Call command
+        call_command('import_temperature_stations', stdout=out)
+
+        # Check stations imported
+        stations = Station.objects.all()
+        self.assertEqual(stations.count(), 0)
+        self.assertIn("Response is not a json", out.getvalue())
+        mock_get.side_effect = requests_get_mock_response_error
+        # Call command 2nd time
+        call_command('import_temperature_stations', with_parameters=True, stdout=out)
+
+        self.assertEqual(stations.count(), 2)
+
+
 @mock.patch.object(requests, 'get', side_effect=requests_get_mock_response)
 class ImportStationTest(TestCase):
     """Test import_station command