change product_type (#221)

* change product_type * update postman collection * fix tests
kartoza · Oct 24, 2024 · 9907b2d · 9907b2d
1 parent a94d5c5
commit 9907b2d
Show file tree

Hide file tree

Showing 12 changed files with 54 additions and 50 deletions.
diff --git a/django_project/gap/fixtures/4.dataset_type.json b/django_project/gap/fixtures/4.dataset_type.json
@@ -3,37 +3,37 @@
     "model": "gap.datasettype",
     "pk": 1,
     "fields": {
-      "name": "Historical Reanalysis",
+      "name": "CBAM Daily Historical Reanalysis (2012 - 2023)",
       "description": "",
       "type": "historical",
-      "variable_name": "historical_reanalysis"
+      "variable_name": "cbam_historical_analysis"
     }
   },
   {
     "model": "gap.datasettype",
     "pk": 2,
     "fields": {
-      "name": "Short-term Forecast",
+      "name": "CBAM Short-Term Weather Forecast (14-days)",
       "description": "",
       "type": "forecast",
-      "variable_name": "shortterm_forecast"
+      "variable_name": "cbam_shortterm_forecast"
     }
   },
   {
     "model": "gap.datasettype",
     "pk": 3,
     "fields": {
-      "name": "Seasonal Forecast",
+      "name": "Salient Seasonal Weather Forecast (3-months)",
       "description": "",
       "type": "forecast",
-      "variable_name": "seasonal_forecast"
+      "variable_name": "salient_seasonal_forecast"
     }
   },
   {
     "model": "gap.datasettype",
     "pk": 4,
     "fields": {
-      "name": "Tahmo Ground Observation",
+      "name": "Ground Observations (TAHMO stations)",
       "description": "",
       "type": "historical",
       "variable_name": "tahmo_ground_observation"
@@ -43,10 +43,10 @@
     "model": "gap.datasettype",
     "pk": 5,
     "fields": {
-      "name": "WindBorne Observational",
+      "name": "Radiosonde Observations (Windborne)",
       "description": "",
       "type": "historical",
-      "variable_name": "windborne_observational"
+      "variable_name": "windborne_radiosonde_observation"
     }
   },
   {
@@ -63,17 +63,17 @@
     "model": "gap.datasettype",
     "pk": 7,
     "fields": {
-      "name": "Tahmo Disdrometer Observation",
+      "name": "Disdrometer Ground Observation",
       "description": "",
       "type": "historical",
-      "variable_name": "tahmo_disdrometer_observation"
+      "variable_name": "disdrometer_ground_observation"
     }
   },
   {
     "model": "gap.datasettype",
     "pk": 8,
     "fields": {
-      "name": "Arable Ground Observation",
+      "name": "Ground Observations (Arable stations)",
       "description": "",
       "type": "historical",
       "variable_name": "arable_ground_observation"

diff --git a/django_project/gap/ingestor/arable.py b/django_project/gap/ingestor/arable.py
@@ -20,7 +20,7 @@
 
 PROVIDER = 'Arable'
 STATION_TYPE = 'Ground Observations'
-DATASET_TYPE = 'Arable Ground Observation'
+DATASET_TYPE = 'arable_ground_observation'
 DATASET_NAME = 'Arable Ground Observational'
 API_KEY_ENV_NAME = 'ARABLE_API_KEY'
 
@@ -54,7 +54,7 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):
             name=STATION_TYPE
         )
         self.dataset_type = DatasetType.objects.get(
-            name=DATASET_TYPE
+            variable_name=DATASET_TYPE
         )
         self.dataset, _ = Dataset.objects.get_or_create(
             name=DATASET_NAME,

diff --git a/django_project/gap/ingestor/tahmo.py b/django_project/gap/ingestor/tahmo.py
@@ -50,7 +50,7 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):
             name='Ground Observations'
         )
         self.dataset_type = DatasetType.objects.get(
-            name='Tahmo Ground Observation'
+            variable_name='tahmo_ground_observation'
         )
         self.dataset, _ = Dataset.objects.get_or_create(
             name='Tahmo Ground Observational',

diff --git a/django_project/gap/ingestor/tahmo_api.py b/django_project/gap/ingestor/tahmo_api.py
@@ -24,7 +24,7 @@
 
 PROVIDER = 'Tahmo'
 STATION_TYPE = 'Disdrometer'
-DATASET_TYPE = 'Tahmo Disdrometer Observation'
+DATASET_TYPE = 'disdrometer_ground_observation'
 DATASET_NAME = 'Tahmo Disdrometer Observational'
 TAHMO_API_USERNAME_ENV_NAME = 'TAHMO_API_USERNAME'
 TAHMO_API_PASSWORD_ENV_NAME = 'TAHMO_API_PASSWORD'
@@ -108,7 +108,7 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):
             name=STATION_TYPE
         )
         self.dataset_type = DatasetType.objects.get(
-            name=DATASET_TYPE
+            variable_name=DATASET_TYPE
         )
         self.dataset, _ = Dataset.objects.get_or_create(
             name=DATASET_NAME,

diff --git a/django_project/gap/ingestor/wind_borne_systems.py b/django_project/gap/ingestor/wind_borne_systems.py
@@ -23,7 +23,7 @@
 
 PROVIDER = 'WindBorne Systems'
 STATION_TYPE = 'Balloon'
-DATASET_TYPE = 'WindBorne Observational'
+DATASET_TYPE = 'windborne_radiosonde_observation'
 DATASET_NAME = 'WindBorne Balloons Observations'
 USERNAME_ENV_NAME = 'WIND_BORNE_SYSTEMS_USERNAME'
 PASSWORD_ENV_NAME = 'WIND_BORNE_SYSTEMS_PASSWORD'
@@ -95,7 +95,7 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):
             name=STATION_TYPE
         )
         self.dataset_type = DatasetType.objects.get(
-            name=DATASET_TYPE
+            variable_name=DATASET_TYPE
         )
         self.dataset, _ = Dataset.objects.get_or_create(
             name=DATASET_NAME,

diff --git a/django_project/gap/providers/tio.py b/django_project/gap/providers/tio.py
@@ -105,13 +105,13 @@ def tomorrowio_shortterm_forecast_dataset() -> Dataset:
     """Return dataset object for tomorrow.io Dataset for sort term forecast."""
     provider, _ = Provider.objects.get_or_create(name='Tomorrow.io')
     dt_shorttermforecast, _ = DatasetType.objects.get_or_create(
-        name='Short-term Forecast',
+        variable_name='cbam_shortterm_forecast',
         defaults={
             'type': CastType.FORECAST
         }
     )
     ds_forecast, _ = Dataset.objects.get_or_create(
-        name=f'{provider.name} {dt_shorttermforecast.name}',
+        name='Tomorrow.io Short-term Forecast',
         provider=provider,
         type=dt_shorttermforecast,
         store_type=DatasetStore.EXT_API,
@@ -151,13 +151,13 @@ def init_provider(cls):
         """Init Tomorrow.io provider and variables."""
         provider, _ = Provider.objects.get_or_create(name='Tomorrow.io')
         dt_historical, _ = DatasetType.objects.get_or_create(
-            name='Historical Reanalysis',
+            variable_name='cbam_historical_analysis',
             defaults={
                 'type': CastType.HISTORICAL
             }
         )
         ds_historical, _ = Dataset.objects.get_or_create(
-            name=f'{provider.name} {dt_historical.name}',
+            name='Tomorrow.io Historical Reanalysis',
             provider=provider,
             type=dt_historical,
             store_type=DatasetStore.EXT_API,
@@ -174,7 +174,7 @@ def init_provider(cls):
             }
         )
         ds_ltn, _ = Dataset.objects.get_or_create(
-            name=f'{provider.name} {dt_ltn.name}',
+            name='Tomorrow.io Long Term Normals (20 years)',
             provider=provider,
             type=dt_ltn,
             store_type=DatasetStore.EXT_API,

diff --git a/django_project/gap/tests/providers/test_tio.py b/django_project/gap/tests/providers/test_tio.py
@@ -16,7 +16,8 @@
     Dataset,
     DatasetAttribute,
     DatasetType,
-    CastType
+    CastType,
+    DatasetStore
 )
 from gap.utils.reader import (
     DatasetReaderInput
@@ -41,7 +42,8 @@ def setUp(self):
         """Set test class."""
         TomorrowIODatasetReader.init_provider()
         self.dataset = Dataset.objects.filter(
-            provider__name='Tomorrow.io'
+            provider__name='Tomorrow.io',
+            store_type=DatasetStore.EXT_API
         ).first()
 
         attr = DatasetAttribute.objects.filter(

diff --git a/django_project/gap_api/api_views/measurement.py b/django_project/gap_api/api_views/measurement.py
@@ -109,13 +109,15 @@ class MeasurementAPI(GAPAPILoggingMixin, APIView):
             description='Product type',
             type=openapi.TYPE_STRING,
             enum=[
-                'historical_reanalysis',
-                'shortterm_forecast',
-                'seasonal_forecast',
-                'observations',
-                'windborne_observational'
+                'cbam_historical_analysis',
+                'arable_ground_observation',
+                'disdrometer_ground_observation',
+                'tahmo_ground_observation',
+                'windborne_radiosonde_observation',
+                'cbam_shortterm_forecast',
+                'salient_seasonal_forecast'
             ],
-            default='historical_reanalysis'
+            default='cbam_historical_analysis'
         ),
         openapi.Parameter(
             'output_type', openapi.IN_QUERY,
@@ -229,7 +231,7 @@ def _get_product_filter(self):
         """
         product = self.request.GET.get('product', None)
         if product is None:
-            return ['historical_reanalysis']
+            return ['cbam_historical_analysis']
         return [product.lower()]
 
     def _get_format_filter(self):

diff --git a/django_project/gap_api/tests/test_measurement_api.py b/django_project/gap_api/tests/test_measurement_api.py
@@ -354,7 +354,7 @@ def test_validate_dataset_attributes(self, mocked_reader):
         ]
         request = self._get_measurement_request_point(
             attributes=','.join(attribs),
-            product='seasonal_forecast',
+            product='salient_seasonal_forecast',
             output_type='csv'
         )
         response = view(request)

diff --git a/django_project/gap_api/tests/test_measurement_api_airborne.py b/django_project/gap_api/tests/test_measurement_api_airborne.py
@@ -87,7 +87,7 @@ def setUp(self):
             name=STATION_TYPE
         )
         dataset_type = DatasetType.objects.get(
-            name=DATASET_TYPE
+            variable_name=DATASET_TYPE
         )
         dataset = Dataset.objects.get(
             name=DATASET_NAME,
@@ -138,7 +138,7 @@ def test_read_point(self):
             lat=0, lon=0,
             start_dt='2000-01-01', end_dt='2000-03-01',
             attributes=','.join(['atmospheric_pressure', 'temperature']),
-            product='windborne_observational',
+            product='windborne_radiosonde_observation',
             output_type='json',
         )
         response = view(request)
@@ -161,7 +161,7 @@ def test_read_point(self):
             lat=10, lon=10,
             start_dt='2000-01-01', end_dt='2000-03-01',
             attributes=','.join(['atmospheric_pressure', 'temperature']),
-            product='windborne_observational',
+            product='windborne_radiosonde_observation',
             output_type='json',
         )
         response = view(request)
@@ -206,7 +206,7 @@ def test_read_with_bbox(self):
             bbox='0,0,100,100',
             start_dt='2000-02-01', end_dt='2000-03-01',
             attributes=','.join(['atmospheric_pressure', 'temperature']),
-            product='windborne_observational',
+            product='windborne_radiosonde_observation',
             output_type='csv',
         )
         response = view(request)
@@ -234,7 +234,7 @@ def test_read_with_bbox(self):
             bbox='5,5,20,20',
             start_dt='2000-01-01', end_dt='2000-03-01',
             attributes=','.join(['atmospheric_pressure', 'temperature']),
-            product='windborne_observational',
+            product='windborne_radiosonde_observation',
             output_type='csv',
         )
         response = view(request)
@@ -258,7 +258,7 @@ def test_read_with_bbox(self):
             altitudes='1.5,5',
             start_dt='2000-01-01', end_dt='2000-03-01',
             attributes=','.join(['atmospheric_pressure', 'temperature']),
-            product='windborne_observational',
+            product='windborne_radiosonde_observation',
             output_type='csv',
         )
         response = view(request)
@@ -287,7 +287,7 @@ def test_read_to_netcdf(self):
             bbox='0,0,100,100',
             start_dt='2000-02-01', end_dt='2000-03-01',
             attributes=','.join(['atmospheric_pressure', 'temperature']),
-            product='windborne_observational',
+            product='windborne_radiosonde_observation',
             output_type='netcdf',
         )
         response = view(request)

diff --git a/docs/src/developer/api/guide/assets/tngap_api.postman_collection.zip b/docs/src/developer/api/guide/assets/tngap_api.postman_collection.zip
diff --git a/docs/src/developer/api/guide/measurement.md b/docs/src/developer/api/guide/measurement.md
@@ -41,14 +41,14 @@ TomorrowNow provides access to the data through a RESTful API, available at http
 | Product | Provider | Resolution | Source | Version | API product_type |
 |---------|----------|------------|--------|---------|------------------|
 | **Historical Data** |
-| CBAM Daily Historical Reanalysis (2012 - 2023) | Tomorrow.io | 4km² | Tomorrow.io CBAM 1F enhanced bias-corrected reanalysis | 2012-2023 | historical_reanalysis |
+| CBAM Daily Historical Reanalysis (2012 - 2023) | Tomorrow.io | 4km² | Tomorrow.io CBAM 1F enhanced bias-corrected reanalysis | 2012-2023 | cbam_historical_analysis |
 | Ground Observations (TAHMO stations) | TAHMO weather stations | 300+ stations across East Africa | TAHMO Gap Filled Data (csv) | 2018-2024 | tahmo_ground_observation |
 | Ground Observations (Arable stations) | Arable weather stations | 300+ stations across East Africa | Arable (API) | | arable_ground_observation |
-| Disdrometer Observation Data | disdrometers | | Tahmo (API) | | tahmo_disdrometer_observation |
-| Radiosonde Observations (Windborne) | WindBorne Systems | 100 weather balloons| Windborne Systems | | windborne_observational |
+| Disdrometer Observation Data | disdrometers | | Tahmo (API) | | disdrometer_ground_observation |
+| Radiosonde Observations (Windborne) | WindBorne Systems | 100 weather balloons| Windborne Systems | | windborne_radiosonde_observation |
 | **Weather Forecasts** |
-| CBAM Short-Term weather forecast (14-days) | Tomorrow.io | 4km² | Tomorrow.io CBAM satellite enhanced short-term weather forecasts | | shortterm_forecast |
-| Salient Seasonal weather forecast (3-months) | Salient | 9km² | Salient (API) | v9 | seasonal_forecast |
+| CBAM Short-Term weather forecast (14-days) | Tomorrow.io | 4km² | Tomorrow.io CBAM satellite enhanced short-term weather forecasts | | cbam_shortterm_forecast |
+| Salient Seasonal weather forecast (3-months) | Salient | 9km² | Salient (API) | v9 | salient_seasonal_forecast |
 | |
 
 
@@ -210,7 +210,7 @@ https://docs.xarray.dev/en/stable/user-guide/io.html#netcdf
 import requests
 from requests.auth import HTTPBasicAuth
 
-url = "https://tngap.sta.do.kartoza.com/api/v1/measurement/?lat=-1.404244&lon=35.008688&attributes=max_temperature,min_temperature&start_date=2019-11-01&end_date=2019-11-02&product=historical_reanalysis&output_type=json"
+url = "https://tngap.sta.do.kartoza.com/api/v1/measurement/?lat=-1.404244&lon=35.008688&attributes=max_temperature,min_temperature&start_date=2019-11-01&end_date=2019-11-02&product=cbam_historical_analysis&output_type=json"
 
 payload={}
 headers = {}
@@ -224,14 +224,14 @@ print(response.json())
 ### CURL
 
 ```
-curl --location 'https://tngap.sta.do.kartoza.com/api/v1/measurement/?lat=-1.404244&lon=35.008688&attributes=max_temperature%2Cmin_temperature&start_date=2019-11-01&end_date=2019-11-02&product=historical_reanalysis&output_type=json' -u 'YOUR_USERNAME:YOUR_PASSWORD' -H 'User-Agent: PostmanRuntime/7.42.0'
+curl --location 'https://tngap.sta.do.kartoza.com/api/v1/measurement/?lat=-1.404244&lon=35.008688&attributes=max_temperature%2Cmin_temperature&start_date=2019-11-01&end_date=2019-11-02&product=cbam_historical_analysis&output_type=json' -u 'YOUR_USERNAME:YOUR_PASSWORD' -H 'User-Agent: PostmanRuntime/7.42.0'
 ```
 
 ### JavaScript-JQuery
 
 ```js
 var settings = {
-  "url": "https://tngap.sta.do.kartoza.com/api/v1/measurement/?lat=-1.404244&lon=35.008688&attributes=max_temperature,min_temperature&start_date=2019-11-01&end_date=2019-11-02&product=historical_reanalysis&output_type=json",
+  "url": "https://tngap.sta.do.kartoza.com/api/v1/measurement/?lat=-1.404244&lon=35.008688&attributes=max_temperature,min_temperature&start_date=2019-11-01&end_date=2019-11-02&product=cbam_historical_analysis&output_type=json",
   "method": "GET",
   "timeout": 0,
   "headers": {
@@ -274,6 +274,6 @@ You can download the postman collection below and import the API collection usin
 | 400 | Unknown geometry type! | Use geometry with type Polygon/Multipolygon/MultiPoint to make a request using POST method |
 | 400 | Output format json is only available for single point query! | JSON output is only available for GET method with singe point query. Please use csv/netcdf output format! |
 | 400 | No matching attribute found! | The attribute list cannot be found in the product type. |
-| 400 | Attribute with ensemble cannot be mixed with non-ensemble | When requesting for product type seasonal_forecast and output is csv, the attribute that is in ensemble (50-values) cannot be requested with the attribute that does not have ensemble. Please use netcdf output format instead! |
+| 400 | Attribute with ensemble cannot be mixed with non-ensemble | When requesting for product type salient_seasonal_forecast and output is csv, the attribute that is in ensemble (50-values) cannot be requested with the attribute that does not have ensemble. Please use netcdf output format instead! |
 | 400 | Incorrect output type | Use either json, csv, netcdf or ascii |
 | 404 | No weather data is found for given queries | |