Merge pull request #190 from Georiviere/improve_commands_import_stations

Improve command add jsondecodeerror

georiviere/observations/management/commands/__init__.py

@@ -1,3 +1,4 @@
+import json
 import requests
 
 from django.core.management.base import BaseCommand, CommandError
@@ -47,9 +48,11 @@ def handle(self, *args, **options):
 
         if verbosity >= 2:
             self.stdout.write('Get station from API {0}'.format(response.url))
-
-        response_content = response.json()
-
+        try:
+            response_content = response.json()
+        except json.JSONDecodeError:
+            self.stdout.write('Response is not a json')
+            return
         if verbosity >= 1:
             self.stdout.write('Import {1} stations from API {0}'.format(response.url, response_content['count']))
 
@@ -60,6 +63,10 @@ def handle(self, *args, **options):
             response = requests.get(response_content['next'])
             if verbosity >= 2:
                 self.stdout.write('Import next page from {0}'.format(response.url))
-            response_content = response.json()
+            try:
+                response_content = response.json()
+            except json.JSONDecodeError:
+                self.stdout.write('Response is not a json')
+                return
             results = response_content['data']
             self.create_or_update_stations(results, verbosity, with_parameters)

georiviere/observations/management/commands/import_hydrobiologie_stations.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import json
 import requests
 from django.contrib.gis.geos import Point
 from georiviere.observations.models import Station, StationProfile, Parameter, ParameterTracking, Unit
@@ -57,7 +58,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False):
                 except requests.exceptions.ConnectionError as e:
                     self.stdout.write(f'Limit of connection has been exceeded {e}')
                     continue
-                response_firstpage_content = response_firstpage.json()
+                try:
+                    response_firstpage_content = response_firstpage.json()
+                except json.JSONDecodeError:
+                    self.stdout.write('Response is not a json')
+                    continue
                 indices_data = response_firstpage_content['data']
 
                 for indice in indices_data:

georiviere/observations/management/commands/import_pcquality_stations.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import json
 import requests
 from django.contrib.gis.geos import Point
 from georiviere.observations.models import Station, StationProfile, Parameter, ParameterTracking, Unit
@@ -67,7 +68,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False):
                 except requests.exceptions.ConnectionError as e:
                     self.stdout.write(f'Limit of connection has been exceeded {e}')
                     continue
-                response_firstpage_content = response_firstpage.json()
+                try:
+                    response_firstpage_content = response_firstpage.json()
+                except json.JSONDecodeError:
+                    self.stdout.write('Response is not a json')
+                    continue
                 analysepc_data = response_firstpage_content['data']
 
                 # If there is more than one page, get data desc sorted
@@ -78,7 +83,11 @@ def create_or_update_stations(self, results, verbosity, with_parameters=False):
                     except requests.exceptions.ConnectionError as e:
                         self.stdout.write(f'Limit of connection has been exceeded {e}')
                         continue
-                    response_desc_data = response_desc_results.json()['data']
+                    try:
+                        response_desc_data = response_desc_results.json()['data']
+                    except json.JSONDecodeError:
+                        self.stdout.write('Response is not a json')
+                        continue
                     analysepc_data = analysepc_data + response_desc_data
 
                 for measure in analysepc_data:

georiviere/observations/tests/data/response_api_hydrobio_indices.json

@@ -3,7 +3,7 @@
   "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/indices?code_station_hydrobio=06000874&page=1&size=1000",
   "last": null,
   "prev": null,
-  "next": null,
+  "next": "https://hubeau.eaufrance.fr/api/v1/hydrobio/indices?code_station_hydrobio=06000874&page=3&size=1000",
   "api_version": "1.0.0",
   "data": [
     {

georiviere/observations/tests/data/response_api_hydrobio_indices_page3.json

@@ -0,0 +1,9 @@
+{
+  "count": 0,
+  "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/indices?code_station_hydrobio=06000874&page=2&size=1000",
+  "last": null,
+  "prev": null,
+  "next": "",
+  "api_version": "1.0.0",
+  "data": []
+}

georiviere/observations/tests/data/response_api_hydrobio_stations.json

@@ -3,7 +3,7 @@
   "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/stations_hydrobio?code_station_hydrobio=06000874&page=1&size=1000",
   "last": null,
   "prev": null,
-  "next": null,
+  "next": "https://hubeau.eaufrance.fr/api/v1/hydrobio/stations_hydrobio?code_station_hydrobio=06000874&page=3&size=1000",
   "api_version": "1.0.0",
   "data": [
     {

georiviere/observations/tests/data/response_api_hydrobio_stations_page3.json

@@ -0,0 +1,9 @@
+{
+  "count": 0,
+  "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/stations_hydrobio?code_station_hydrobio=06000874&page=2&size=1000",
+  "last": null,
+  "prev": null,
+  "next": "",
+  "api_version": "1.0.0",
+  "data": []
+}

georiviere/observations/tests/data/response_api_hydrobio_taxons.json

@@ -3,7 +3,7 @@
   "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/taxons?code_station_hydrobio=06000874&page=1&size=1000",
   "last": null,
   "prev": null,
-  "next": null,
+  "next": "https://hubeau.eaufrance.fr/api/v1/hydrobio/taxons?code_station_hydrobio=06000874&page=3&size=1000",
   "api_version": "1.0.0",
   "data": [
     {

georiviere/observations/tests/data/response_api_hydrobio_taxons_page3.json

@@ -0,0 +1,9 @@
+{
+  "count": 45,
+  "first": "https://hubeau.eaufrance.fr/api/v1/hydrobio/taxons?code_station_hydrobio=06000874&page=1&size=1000",
+  "last": null,
+  "prev": null,
+  "next": null,
+  "api_version": "1.0.0",
+  "data": []
+}

georiviere/observations/tests/data/response_api_hydrometrie_stations.json

@@ -3,7 +3,7 @@
   "first": "https://hubeau.eaufrance.fr/api/v1/hydrometrie/referentiel/stations?code_site=U2012010&code_site=U2015410&format=json&page=1&size=20",
   "last": null,
   "prev": null,
-  "next": null,
+  "next": "https://hubeau.eaufrance.fr/api/v1/hydrometrie/referentiel/stations?code_site=U2012010&code_site=U2015410&format=json&page=3&size=20",
   "api_version": "1.0.1",
   "data": [
     {

georiviere/observations/tests/data/response_api_hydrometrie_stations_page3.json

@@ -0,0 +1,10 @@
+{
+  "count": 2,
+  "first": "https://hubeau.eaufrance.fr/api/v1/hydrometrie/referentiel/stations?code_site=U2012010&code_site=U2015410&format=json&page=2&size=20",
+  "last": null,
+  "prev": null,
+  "next": null,
+  "api_version": "1.0.1",
+  "data": [
+  ]
+}

georiviere/observations/tests/data/response_api_pcquality_stations.json

@@ -3,7 +3,7 @@
   "first": "https://hubeau.eaufrance.fr/api/v1/qualite_rivieres/station_pc?format=json&code_departement=11&page=1&size=1000",
   "last": null,
   "prev": null,
-  "next": null,
+  "next": "https://hubeau.eaufrance.fr/api/v1/qualite_rivieres/station_pc?format=json&code_departement=11&page=3&size=1000",
   "api_version": "1.1.0",
   "data": [
     {

georiviere/observations/tests/data/response_api_pcquality_stations_page3.json

@@ -0,0 +1,9 @@
+{
+  "count": 28,
+  "first": "https://hubeau.eaufrance.fr/api/v1/qualite_rivieres/station_pc?format=json&code_departement=11&page=2&size=1000",
+  "last": null,
+  "prev": null,
+  "next": null,
+  "api_version": "1.1.0",
+  "data": []
+}

georiviere/observations/tests/data/response_api_temperature_stations.json

@@ -3,7 +3,7 @@
   "first": "https://hubeau.eaufrance.fr/api/v1/temperature/station?format=json&code_commune=25320&code_commune=25415&page=1&size=1000",
   "last": null,
   "prev": null,
-  "next": null,
+  "next": "https://hubeau.eaufrance.fr/api/v1/temperature/station?format=json&code_commune=25320&code_commune=25415&page=3&size=1000",
   "api_version": "1.0.0",
   "data": [
     {

georiviere/observations/tests/data/response_api_temperature_stations_page3.json

@@ -0,0 +1,9 @@
+{
+  "count": 0,
+  "first": "https://hubeau.eaufrance.fr/api/v1/temperature/station?format=json&code_commune=25320&code_commune=25415&page=2&size=1000",
+  "last": null,
+  "prev": null,
+  "next": "",
+  "api_version": "1.0.0",
+  "data": []
+}