diff --git a/georiviere/observations/management/commands/__init__.py b/georiviere/observations/management/commands/__init__.py index 8422fff0..66133c9f 100644 --- a/georiviere/observations/management/commands/__init__.py +++ b/georiviere/observations/management/commands/__init__.py @@ -1,3 +1,4 @@ +import json import requests from django.core.management.base import BaseCommand, CommandError @@ -47,9 +48,11 @@ def handle(self, *args, **options): if verbosity >= 2: self.stdout.write('Get station from API {0}'.format(response.url)) - - response_content = response.json() - + try: + response_content = response.json() + except json.JSONDecodeError: + self.stdout.write('Response is not a json') + return if verbosity >= 1: self.stdout.write('Import {1} stations from API {0}'.format(response.url, response_content['count'])) @@ -60,6 +63,10 @@ def handle(self, *args, **options): response = requests.get(response_content['next']) if verbosity >= 2: self.stdout.write('Import next page from {0}'.format(response.url)) - response_content = response.json() + try: + response_content = response.json() + except json.JSONDecodeError: + self.stdout.write('Response is not a json') + return results = response_content['data'] self.create_or_update_stations(results, verbosity, with_parameters) diff --git a/georiviere/observations/management/commands/import_hydrobiologie_stations.py b/georiviere/observations/management/commands/import_hydrobiologie_stations.py index 5f0e788d..88f626e4 100644 --- a/georiviere/observations/management/commands/import_hydrobiologie_stations.py +++ b/georiviere/observations/management/commands/import_hydrobiologie_stations.py @@ -1,4 +1,5 @@ from datetime import datetime +import json import requests from django.contrib.gis.geos import Point from georiviere.observations.models import Station, StationProfile, Parameter, ParameterTracking, Unit @@ -57,7 +58,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False): except requests.exceptions.ConnectionError as e: self.stdout.write(f'Limit of connection has been exceeded {e}') continue - response_firstpage_content = response_firstpage.json() + try: + response_firstpage_content = response_firstpage.json() + except json.JSONDecodeError: + self.stdout.write('Response is not a json') + continue indices_data = response_firstpage_content['data'] for indice in indices_data: diff --git a/georiviere/observations/management/commands/import_pcquality_stations.py b/georiviere/observations/management/commands/import_pcquality_stations.py index d4f16c6a..25ec7f02 100644 --- a/georiviere/observations/management/commands/import_pcquality_stations.py +++ b/georiviere/observations/management/commands/import_pcquality_stations.py @@ -1,4 +1,5 @@ from datetime import datetime +import json import requests from django.contrib.gis.geos import Point from georiviere.observations.models import Station, StationProfile, Parameter, ParameterTracking, Unit @@ -67,7 +68,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False): except requests.exceptions.ConnectionError as e: self.stdout.write(f'Limit of connection has been exceeded {e}') continue - response_firstpage_content = response_firstpage.json() + try: + response_firstpage_content = response_firstpage.json() + except json.JSONDecodeError: + self.stdout.write('Response is not a json') + continue analysepc_data = response_firstpage_content['data'] # If there is more than one page, get data desc sorted @@ -78,7 +83,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False): except requests.exceptions.ConnectionError as e: self.stdout.write(f'Limit of connection has been exceeded {e}') continue - response_desc_data = response_desc_results.json()['data'] + try: + response_desc_data = response_desc_results.json()['data'] + except json.JSONDecodeError: + self.stdout.write('Response is not a json') + continue analysepc_data = analysepc_data + response_desc_data for measure in analysepc_data: diff --git a/georiviere/observations/tests/data/response_api_hydrobio_indices.json b/georiviere/observations/tests/data/response_api_hydrobio_indices.json index 87cd8076..276540e0 100644 --- a/georiviere/observations/tests/data/response_api_hydrobio_indices.json +++ b/georiviere/observations/tests/data/response_api_hydrobio_indices.json @@ -3,7 +3,7 @@ "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/indices?code_station_hydrobio=06000874&page=1&size=1000", "last": null, "prev": null, - "next": null, + "next": "https://hubeau.eaufrance.fr/api/v1/hydrobio/indices?code_station_hydrobio=06000874&page=3&size=1000", "api_version": "1.0.0", "data": [ { diff --git a/georiviere/observations/tests/data/response_api_hydrobio_indices_page3.json b/georiviere/observations/tests/data/response_api_hydrobio_indices_page3.json new file mode 100644 index 00000000..d946f16a --- /dev/null +++ b/georiviere/observations/tests/data/response_api_hydrobio_indices_page3.json @@ -0,0 +1,9 @@ +{ + "count": 0, + "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/indices?code_station_hydrobio=06000874&page=2&size=1000", + "last": null, + "prev": null, + "next": "", + "api_version": "1.0.0", + "data": [] +} diff --git a/georiviere/observations/tests/data/response_api_hydrobio_stations.json b/georiviere/observations/tests/data/response_api_hydrobio_stations.json index 6c26db05..5f07f52c 100644 --- a/georiviere/observations/tests/data/response_api_hydrobio_stations.json +++ b/georiviere/observations/tests/data/response_api_hydrobio_stations.json @@ -3,7 +3,7 @@ "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/stations_hydrobio?code_station_hydrobio=06000874&page=1&size=1000", "last": null, "prev": null, - "next": null, + "next": "https://hubeau.eaufrance.fr/api/v1/hydrobio/stations_hydrobio?code_station_hydrobio=06000874&page=3&size=1000", "api_version": "1.0.0", "data": [ { diff --git a/georiviere/observations/tests/data/response_api_hydrobio_stations_page3.json b/georiviere/observations/tests/data/response_api_hydrobio_stations_page3.json new file mode 100644 index 00000000..1f19f46a --- /dev/null +++ b/georiviere/observations/tests/data/response_api_hydrobio_stations_page3.json @@ -0,0 +1,9 @@ +{ + "count": 0, + "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/stations_hydrobio?code_station_hydrobio=06000874&page=2&size=1000", + "last": null, + "prev": null, + "next": "", + "api_version": "1.0.0", + "data": [] +} diff --git a/georiviere/observations/tests/data/response_api_hydrobio_taxons.json b/georiviere/observations/tests/data/response_api_hydrobio_taxons.json index c657886e..a4dcec7a 100644 --- a/georiviere/observations/tests/data/response_api_hydrobio_taxons.json +++ b/georiviere/observations/tests/data/response_api_hydrobio_taxons.json @@ -3,7 +3,7 @@ "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/taxons?code_station_hydrobio=06000874&page=1&size=1000", "last": null, "prev": null, - "next": null, + "next": "https://hubeau.eaufrance.fr/api/v1/hydrobio/taxons?code_station_hydrobio=06000874&page=3&size=1000", "api_version": "1.0.0", "data": [ { diff --git a/georiviere/observations/tests/data/response_api_hydrobio_taxons_page3.json b/georiviere/observations/tests/data/response_api_hydrobio_taxons_page3.json new file mode 100644 index 00000000..1a8a6a9c --- /dev/null +++ b/georiviere/observations/tests/data/response_api_hydrobio_taxons_page3.json @@ -0,0 +1,9 @@ +{ + "count": 45, + "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/taxons?code_station_hydrobio=06000874&page=1&size=1000", + "last": null, + "prev": null, + "next": null, + "api_version": "1.0.0", + "data": [] +} \ No newline at end of file diff --git a/georiviere/observations/tests/data/response_api_hydrometrie_stations.json b/georiviere/observations/tests/data/response_api_hydrometrie_stations.json index 5bdf87a4..4a089cdf 100644 --- a/georiviere/observations/tests/data/response_api_hydrometrie_stations.json +++ b/georiviere/observations/tests/data/response_api_hydrometrie_stations.json @@ -3,7 +3,7 @@ "first": "https://hubeau.eaufrance.fr/api/v1/hydrometrie/referentiel/stations?code_site=U2012010&code_site=U2015410&format=json&page=1&size=20", "last": null, "prev": null, - "next": null, + "next": "https://hubeau.eaufrance.fr/api/v1/hydrometrie/referentiel/stations?code_site=U2012010&code_site=U2015410&format=json&page=3&size=20", "api_version": "1.0.1", "data": [ { diff --git a/georiviere/observations/tests/data/response_api_hydrometrie_stations_page3.json b/georiviere/observations/tests/data/response_api_hydrometrie_stations_page3.json new file mode 100644 index 00000000..f16e6170 --- /dev/null +++ b/georiviere/observations/tests/data/response_api_hydrometrie_stations_page3.json @@ -0,0 +1,10 @@ +{ + "count": 2, + "first": "https://hubeau.eaufrance.fr/api/v1/hydrometrie/referentiel/stations?code_site=U2012010&code_site=U2015410&format=json&page=2&size=20", + "last": null, + "prev": null, + "next": null, + "api_version": "1.0.1", + "data": [ + ] +} \ No newline at end of file diff --git a/georiviere/observations/tests/data/response_api_pcquality_stations.json b/georiviere/observations/tests/data/response_api_pcquality_stations.json index bd191043..da3e55f9 100644 --- a/georiviere/observations/tests/data/response_api_pcquality_stations.json +++ b/georiviere/observations/tests/data/response_api_pcquality_stations.json @@ -3,7 +3,7 @@ "first": "https://hubeau.eaufrance.fr/api/v1/qualite_rivieres/station_pc?format=json&code_departement=11&page=1&size=1000", "last": null, "prev": null, - "next": null, + "next": "https://hubeau.eaufrance.fr/api/v1/qualite_rivieres/station_pc?format=json&code_departement=11&page=3&size=1000", "api_version": "1.1.0", "data": [ { diff --git a/georiviere/observations/tests/data/response_api_pcquality_stations_page3.json b/georiviere/observations/tests/data/response_api_pcquality_stations_page3.json new file mode 100644 index 00000000..ba27c718 --- /dev/null +++ b/georiviere/observations/tests/data/response_api_pcquality_stations_page3.json @@ -0,0 +1,9 @@ +{ + "count": 28, + "first": "https://hubeau.eaufrance.fr/api/v1/qualite_rivieres/station_pc?format=json&code_departement=11&page=2&size=1000", + "last": null, + "prev": null, + "next": null, + "api_version": "1.1.0", + "data": [] +} \ No newline at end of file diff --git a/georiviere/observations/tests/data/response_api_temperature_stations.json b/georiviere/observations/tests/data/response_api_temperature_stations.json index c5f949d3..104813b8 100644 --- a/georiviere/observations/tests/data/response_api_temperature_stations.json +++ b/georiviere/observations/tests/data/response_api_temperature_stations.json @@ -3,7 +3,7 @@ "first": "https://hubeau.eaufrance.fr/api/v1/temperature/station?format=json&code_commune=25320&code_commune=25415&page=1&size=1000", "last": null, "prev": null, - "next": null, + "next": "https://hubeau.eaufrance.fr/api/v1/temperature/station?format=json&code_commune=25320&code_commune=25415&page=3&size=1000", "api_version": "1.0.0", "data": [ { diff --git a/georiviere/observations/tests/data/response_api_temperature_stations_page3.json b/georiviere/observations/tests/data/response_api_temperature_stations_page3.json new file mode 100644 index 00000000..9329ca6b --- /dev/null +++ b/georiviere/observations/tests/data/response_api_temperature_stations_page3.json @@ -0,0 +1,9 @@ +{ + "count": 0, + "first": "https://hubeau.eaufrance.fr/api/v1/temperature/station?format=json&code_commune=25320&code_commune=25415&page=2&size=1000", + "last": null, + "prev": null, + "next": "", + "api_version": "1.0.0", + "data": [] +} diff --git a/georiviere/observations/tests/test_commands.py b/georiviere/observations/tests/test_commands.py index 71432e46..a20337e8 100644 --- a/georiviere/observations/tests/test_commands.py +++ b/georiviere/observations/tests/test_commands.py @@ -17,25 +17,43 @@ def requests_get_mock_response(*args, **kwargs): """Build mock_response with test file data, according to api_url""" # Get filename from api_url filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations_page3.json' if "urf" in args[0]: filename = TEST_DATA_PATH / 'response_sandre_urf.json' if "hydrometrie" in args[0]: filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations_page3.json' if "temperature" in args[0]: filename = TEST_DATA_PATH / 'response_api_temperature_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_temperature_stations_page3.json' if "station_pc" in args[0]: filename = TEST_DATA_PATH / 'response_api_pcquality_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_stations_page3.json' if "analyse_pc" in args[0]: if "sort" in kwargs['params']: filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_desc.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_page3.json' else: filename = TEST_DATA_PATH / 'response_api_pcquality_analyse.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_page3.json' if 'stations_hydrobio' in args[0]: filename = TEST_DATA_PATH / 'response_api_hydrobio_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrobio_stations_page3.json' if 'indices' in args[0]: filename = TEST_DATA_PATH / 'response_api_hydrobio_indices.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrobio_indices_page3.json' if 'taxons' in args[0]: filename = TEST_DATA_PATH / 'response_api_hydrobio_taxons.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrobio_taxons_page3.json' # Build response mock_response = mock.Mock() with open(filename, 'r') as f: @@ -45,6 +63,66 @@ def requests_get_mock_response(*args, **kwargs): return mock_response +def requests_get_mock_response_error_initial_url(*args, **kwargs): + mock_response = mock.Mock() + mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1) + mock_response.status_code = 200 + return mock_response + + +def requests_get_mock_response_error(*args, **kwargs): + """Build mock_response with test file data, according to api_url""" + # Get filename from api_url + mock_response = mock.Mock() + filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations_page3.json' + if "urf" in args[0]: + filename = TEST_DATA_PATH / 'response_sandre_urf.json' + if "hydrometrie" in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrometrie_stations_page3.json' + if "temperature" in args[0]: + filename = TEST_DATA_PATH / 'response_api_temperature_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_temperature_stations_page3.json' + if "station_pc" in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_stations_page3.json' + if "analyse_pc" in args[0]: + if "sort" in kwargs['params']: + mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1) + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_page3.json' + else: + filename = TEST_DATA_PATH / 'response_api_pcquality_analyse.json' + if kwargs.get('params', {})["code_station"] == "05137100": + mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1) + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_page3.json' + if 'stations_hydrobio' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrobio_stations.json' + if 'page=3' in args[0]: + mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1) + if 'indices' in args[0]: + mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1) + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrobio_indices_page3.json' + if 'taxons' in args[0]: + mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1) + if 'page=3' in args[0]: + mock_response.json.side_effect = json.JSONDecodeError('msg', 'doc', 1) + # Build response + + with open(filename, 'r') as f: + expected_dict = json.load(f) + mock_response.json.return_value = expected_dict + mock_response.status_code = 200 + return mock_response + + def requests_get_mock_response_500(*args, **kwargs): """Build mock_response like if server is down""" # Build response @@ -75,6 +153,77 @@ def test_urf_imported(self, mock_get): self.assertIn('Created unit mg(Cl2)/L', out.getvalue()) +@mock.patch.object(requests, 'get', side_effect=requests_get_mock_response_error_initial_url) +class ImportStationErrorTest(TestCase): + """Test import_station command + Test datas are from LABERGEMENT-STE-MARIE + """ + + def test_import_hydrometric_stations_error_json(self, mock_get): + out = StringIO() + + # Call command + call_command('import_hydrometric_stations', with_parameters=True, stdout=out) + + # Check stations imported + stations = Station.objects.all() + self.assertEqual(stations.count(), 0) + self.assertIn("Response is not a json", out.getvalue()) + mock_get.side_effect = requests_get_mock_response_error + # Call command 2nd time + call_command('import_hydrometric_stations', with_parameters=True, stdout=out) + + self.assertEqual(stations.count(), 2) + + def test_import_pcquality_stations_error_json(self, mock_get): + out = StringIO() + + # Call command + call_command('import_pcquality_stations', with_parameters=True, stdout=out) + + # Check stations imported + stations = Station.objects.all() + self.assertEqual(stations.count(), 0) + self.assertIn("Response is not a json", out.getvalue()) + mock_get.side_effect = requests_get_mock_response_error + # Call command 2nd time + call_command('import_pcquality_stations', with_parameters=True, stdout=out) + + self.assertEqual(stations.count(), 28) + + def test_import_temperature_stations_error_json(self, mock_get): + out = StringIO() + + # Call command + call_command('import_temperature_stations', stdout=out) + + # Check stations imported + stations = Station.objects.all() + self.assertEqual(stations.count(), 0) + self.assertIn("Response is not a json", out.getvalue()) + mock_get.side_effect = requests_get_mock_response_error + # Call command 2nd time + call_command('import_temperature_stations', with_parameters=True, stdout=out) + + self.assertEqual(stations.count(), 2) + + def test_import_hydrobio_stations_error_json(self, mock_get): + out = StringIO() + + # Call command + call_command('import_hydrobiologie_stations', stdout=out) + + # Check stations imported + stations = Station.objects.all() + self.assertEqual(stations.count(), 0) + self.assertIn("Response is not a json", out.getvalue()) + mock_get.side_effect = requests_get_mock_response_error + # Call command 2nd time + call_command('import_hydrobiologie_stations', with_parameters=True, stdout=out) + + self.assertEqual(stations.count(), 1) + + @mock.patch.object(requests, 'get', side_effect=requests_get_mock_response) class ImportStationTest(TestCase): """Test import_station command @@ -85,7 +234,7 @@ def test_hydrometric_stations_imported(self, mock_get): out = StringIO() # Call command - call_command('import_hydrometric_stations', stdout=out) + call_command('import_hydrometric_stations', stdout=out, size=100, department="49") # Check stations imported stations = Station.objects.all() @@ -332,7 +481,9 @@ def requests_get_mock_response_fail_connectionerror_asc(*args, **kwargs): """Build mock_response with test file data, according to api_url""" # Get filename from api_url filename = TEST_DATA_PATH / 'response_api_pcquality_stations.json' - if kwargs.get('params').get('code_station') == "05134550": + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_stations_page3.json' + if kwargs.get('params', {}).get('code_station') == "05134550": raise requests.exceptions.ConnectionError if "analyse_pc" in args[0]: filename = TEST_DATA_PATH / 'response_api_pcquality_analyse.json' @@ -349,13 +500,19 @@ def requests_get_mock_response_fail_connectionerror_desc(*args, **kwargs): """Build mock_response with test file data, according to api_url""" # Get filename from api_url filename = TEST_DATA_PATH / 'response_api_pcquality_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_stations_page3.json' if "analyse_pc" in args[0]: if "sort" in kwargs['params']: filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_desc.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_page3.json' if kwargs.get('params').get('code_station') == "05134550": raise requests.exceptions.ConnectionError else: filename = TEST_DATA_PATH / 'response_api_pcquality_analyse.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_pcquality_analyse_page3.json' # Build response mock_response = mock.Mock() @@ -428,6 +585,8 @@ def requests_get_mock_response_fail_connectionerror_taxons(*args, **kwargs): """Build mock_response with test file data, according to api_url""" # Get filename from api_url filename = TEST_DATA_PATH / 'response_api_hydrobio_stations.json' + if 'page=3' in args[0]: + filename = TEST_DATA_PATH / 'response_api_hydrobio_stations_page3.json' if "taxons" in args[0]: raise requests.exceptions.ConnectionError if "indices" in args[0]: