Fix api issues

django_project/gap/admin/main.py

@@ -90,6 +90,12 @@
     )
     list_filter = ('station',)
     search_fields = ('name',)
+    # fix N+1 issues for dataset_attribute dropdown
+    readonly_fields = ('dataset_attribute', 'station_history',)
+
+    def has_add_permission(self, request, obj=None):
+        """Disable add measurement from admin page."""
+        return False
 
 
 class IngestorSessionProgressInline(admin.TabularInline):

django_project/gap/admin/preferences.py

@@ -59,4 +59,11 @@ class PreferencesAdmin(admin.ModelAdmin):
                 )
             }
         ),
+        (
+            'Logging', {
+                'fields': (
+                    'api_log_batch_size',
+                )
+            }
+        ),
     )

django_project/gap/fixtures/5.dataset.json

@@ -9,7 +9,8 @@
       "type": 4,
       "time_step": "DAILY",
       "store_type": "TABLE",
-      "is_internal_use": false
+      "is_internal_use": false,
+      "observation_type": "GROUND_OBSERVATION"
     }
   },
   {
@@ -22,7 +23,8 @@
       "type": 1,
       "time_step": "DAILY",
       "store_type": "ZARR",
-      "is_internal_use": false
+      "is_internal_use": false,
+      "observation_type": "NOT_SPECIFIED"
     }
   },
   {
@@ -35,7 +37,8 @@
       "type": 3,
       "time_step": "DAILY",
       "store_type": "ZARR",
-      "is_internal_use": false
+      "is_internal_use": false,
+      "observation_type": "NOT_SPECIFIED"
     }
   },
   {
@@ -48,7 +51,8 @@
       "type": 2,
       "time_step": "DAILY",
       "store_type": "ZARR",
-      "is_internal_use": false
+      "is_internal_use": false,
+      "observation_type": "NOT_SPECIFIED"
     }
   },
   {
@@ -61,7 +65,8 @@
       "type": 1,
       "time_step": "DAILY",
       "store_type": "EXT_API",
-      "is_internal_use": true
+      "is_internal_use": true,
+      "observation_type": "NOT_SPECIFIED"
     }
   },
   {
@@ -74,7 +79,8 @@
       "type": 2,
       "time_step": "DAILY",
       "store_type": "EXT_API",
-      "is_internal_use": true
+      "is_internal_use": true,
+      "observation_type": "NOT_SPECIFIED"
     }
   },
   {
@@ -87,7 +93,8 @@
       "type": 6,
       "time_step": "DAILY",
       "store_type": "EXT_API",
-      "is_internal_use": true
+      "is_internal_use": true,
+      "observation_type": "NOT_SPECIFIED"
     }
   },
   {
@@ -100,7 +107,8 @@
       "type": 8,
       "time_step": "DAILY",
       "store_type": "TABLE",
-      "is_internal_use": false
+      "is_internal_use": false,
+      "observation_type": "GROUND_OBSERVATION"
     }
   },
   {
@@ -111,9 +119,10 @@
       "description": "Tahmo data that coming from API : https://datahub.tahmo.org/",
       "provider": 1,
       "type": 7,
-      "time_step": "DAILY",
+      "time_step": "QUARTER_HOURLY",
       "store_type": "TABLE",
-      "is_internal_use": false
+      "is_internal_use": false,
+      "observation_type": "NOT_SPECIFIED"
     }
   },
   {
@@ -124,9 +133,10 @@
       "description": "",
       "provider": 6,
       "type": 5,
-      "time_step": "DAILY",
+      "time_step": "OTHER",
       "store_type": "TABLE",
-      "is_internal_use": false
+      "is_internal_use": false,
+      "observation_type": "UPPER_AIR_OBSERVATION"
     }
   }
 ]

django_project/gap/ingestor/tahmo_api.py

@@ -114,7 +114,7 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):
             name=DATASET_NAME,
             provider=self.provider,
             type=self.dataset_type,
-            time_step=DatasetTimeStep.DAILY,
+            time_step=DatasetTimeStep.QUARTER_HOURLY,
             store_type=DatasetStore.TABLE
         )
 

django_project/gap/ingestor/wind_borne_systems.py

@@ -101,7 +101,7 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):
             name=DATASET_NAME,
             provider=self.provider,
             type=self.dataset_type,
-            time_step=DatasetTimeStep.DAILY,
+            time_step=DatasetTimeStep.OTHER,
             store_type=DatasetStore.TABLE
         )
 

django_project/gap/migrations/0033_dataset_observation_type_alter_dataset_time_step.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.7 on 2024-10-24 07:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0032_preferences_api_log_batch_size'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dataset',
+            name='observation_type',
+            field=models.CharField(choices=[('GROUND_OBSERVATION', 'GROUND_OBSERVATION'), ('UPPER_AIR_OBSERVATION', 'UPPER_AIR_OBSERVATION'), ('NOT_SPECIFIED', 'NOT_SPECIFIED')], default='NOT_SPECIFIED', max_length=512),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='time_step',
+            field=models.CharField(choices=[('DAILY', 'DAILY'), ('HOURLY', 'HOURLY'), ('QUARTER_HOURLY', 'QUARTER_HOURLY'), ('OTHER', 'OTHER')], max_length=512),
+        ),
+    ]

django_project/gap/models/dataset.py

@@ -46,8 +46,18 @@ class DatasetStore:
 class DatasetTimeStep:
     """Dataset Time Step."""
 
+    QUARTER_HOURLY = 'QUARTER_HOURLY'
     HOURLY = 'HOURLY'
     DAILY = 'DAILY'
+    OTHER = 'OTHER'
+
+
+class DatasetObservationType:
+    """Observation type of data source."""
+
+    GROUND_OBSERVATION = 'GROUND_OBSERVATION'
+    UPPER_AIR_OBSERVATION = 'UPPER_AIR_OBSERVATION'
+    NOT_SPECIFIED = 'NOT_SPECIFIED'
 
 
 class Dataset(Definition):
@@ -63,6 +73,8 @@ class Dataset(Definition):
         choices=(
             (DatasetTimeStep.DAILY, DatasetTimeStep.DAILY),
             (DatasetTimeStep.HOURLY, DatasetTimeStep.HOURLY),
+            (DatasetTimeStep.QUARTER_HOURLY, DatasetTimeStep.QUARTER_HOURLY),
+            (DatasetTimeStep.OTHER, DatasetTimeStep.OTHER),
         ),
         max_length=512
     )
@@ -82,6 +94,24 @@ class Dataset(Definition):
             'not exposed through API.'
         ),
     )
+    observation_type = models.CharField(
+        choices=(
+            (
+                DatasetObservationType.GROUND_OBSERVATION,
+                DatasetObservationType.GROUND_OBSERVATION
+            ),
+            (
+                DatasetObservationType.UPPER_AIR_OBSERVATION,
+                DatasetObservationType.UPPER_AIR_OBSERVATION
+            ),
+            (
+                DatasetObservationType.NOT_SPECIFIED,
+                DatasetObservationType.NOT_SPECIFIED
+            ),
+        ),
+        max_length=512,
+        default=DatasetObservationType.NOT_SPECIFIED
+    )
 
 
 class DataSourceFile(models.Model):

django_project/gap/providers/__init__.py

@@ -5,7 +5,7 @@
 .. note:: Helper for reading NetCDF File
 """
 
-from gap.ingestor.wind_borne_systems import PROVIDER
+from gap.ingestor.wind_borne_systems import PROVIDER as WINBORNE_PROVIDER
 from gap.models import Dataset, DatasetStore
 from gap.providers.airborne_observation import ObservationAirborneDatasetReader
 from gap.providers.cbam import CBAMZarrReader, CBAMNetCDFReader  # noqa
@@ -36,7 +36,7 @@ def get_reader_from_dataset(dataset: Dataset):
         return SalientZarrReader
     elif dataset.provider.name in ['Tahmo', 'Arable']:
         return ObservationDatasetReader
-    elif dataset.provider.name in [PROVIDER]:
+    elif dataset.provider.name in [WINBORNE_PROVIDER]:
         return ObservationAirborneDatasetReader
     elif (
         dataset.provider.name == TIO_PROVIDER and

django_project/gap/providers/airborne_observation.py

@@ -79,7 +79,7 @@ def _find_nearest_station_by_points(self):
     def get_measurements(self, start_date: datetime, end_date: datetime):
         """Return measurements."""
         nearest_histories = self.get_nearest_stations()
-        if nearest_histories is None:
+        if nearest_histories is None or len(nearest_histories) == 0:
             return None
 
         return Measurement.objects.select_related(

django_project/gap/providers/observation.py

@@ -19,7 +19,9 @@
     Dataset,
     DatasetAttribute,
     Station,
-    Measurement
+    Measurement,
+    DatasetTimeStep,
+    DatasetObservationType
 )
 from gap.utils.reader import (
     LocationInputType,
@@ -74,29 +76,57 @@ def to_csv_stream(self, suffix='.csv', separator=','):
         :yield: bytes of csv file
         :rtype: bytes
         """
-        headers = [
-            'date',
-            'lat',
-            'lon',
-            'altitude'
-        ]
+        dataset = self.attributes[0].dataset
+        time_col_exists = dataset.time_step != DatasetTimeStep.DAILY
+        alt_col_exists = (
+            dataset.observation_type ==
+            DatasetObservationType.UPPER_AIR_OBSERVATION
+        )
+        headers = ['date']
+
+        # add time if time_step is not daily
+        if time_col_exists:
+            headers.append('time')
+
+        # add lat and lon
+        headers.extend(['lat', 'lon'])
+
+        # add altitude if it's upper air observation
+        if alt_col_exists:
+            headers.append('altitude')
+
+        # write headers
         for attr in self.attributes:
             headers.append(attr.attribute.variable_name)
         yield bytes(','.join(headers) + '\n', 'utf-8')
 
         for val in self.values:
-            data = [
-                val.get_datetime_repr('%Y-%m-%d'),
+            data = [val.get_datetime_repr('%Y-%m-%d')]
+
+            # add time if time_step is not daily
+            if time_col_exists:
+                data.append(val.get_datetime_repr('%H:%M:%S'))
+
+            # add lat and lon
+            data.extend([
                 str(val.location.y),
-                str(val.location.x),
-                str(val.altitude) if val.altitude else '',
-            ]
+                str(val.location.x)
+            ])
+
+            # add altitude if it's upper air observation
+            if alt_col_exists:
+                data.append(
+                    str(val.altitude) if val.altitude else ''
+                )
+
             for attr in self.attributes:
                 var_name = attr.attribute.variable_name
                 if var_name in val.values:
                     data.append(str(val.values[var_name]))
                 else:
                     data.append('')
+
+            # write row
             yield bytes(','.join(data) + '\n', 'utf-8')
 
     def to_netcdf_stream(self):