add api tracking logs

deployment/docker/requirements.txt

@@ -44,7 +44,7 @@ django-storages==1.14.2
 django-storages[s3]
 
 # Python client for Redis database and key-value store
-redis==4.3.4
+redis==5.1.1
 
 psycopg2-binary==2.9.9
 # drf yasg
@@ -69,3 +69,6 @@ numpy==1.26.4
 salientsdk==0.3.5
 
 python-geohash==0.8.5
+
+# api tracking
+drf-api-tracking==1.8.4

django_project/_version.txt

@@ -1 +1 @@
-0.0.5
+0.0.6

django_project/core/celery.py

@@ -9,9 +9,12 @@
 from celery.schedules import crontab
 from celery.utils.serialization import strtobool
 from celery.worker.control import inspect_command
+from celery.worker import strategy
 from django.utils import timezone
 
 logger = logging.getLogger(__name__)
+# Disable info log from Celery MainProcess
+strategy.logger.setLevel(logging.WARNING)
 
 # set the default Django settings module for the 'celery' program.
 # this is also used in manage.py
@@ -69,6 +72,11 @@
         # Run everyday at 00:00 UTC
         'schedule': crontab(minute='00', hour='00'),
     },
+    'store-api-logs': {
+        'task': 'store_api_logs',
+        # Run every 5minutes
+        'schedule': crontab(minute='*/5'),
+    },
     'cleanup-r-execution-logs': {
         'task': 'cleanup_r_execution_logs',
         # Run every first day of each month at 00:00 UTC
@@ -151,7 +159,8 @@ def update_task_progress(
 
 
 EXCLUDED_TASK_LIST = [
-    'celery.backend_cleanup'
+    'celery.backend_cleanup',
+    'store_api_logs'
 ]
 
 

django_project/core/settings/base.py

@@ -93,6 +93,7 @@
         'DIRS': [
             # Put Templates
             absolute_path('core', 'templates'),
+            absolute_path('gap_api', 'templates'),
         ],
         'OPTIONS': {
             'loaders': [
@@ -141,7 +142,11 @@
         'LOCATION': (
             f'redis://default:{os.environ.get("REDIS_PASSWORD", "")}'
             f'@{os.environ.get("REDIS_HOST", "")}',
-        )
+        ),
+        'OPTIONS': {
+            'socket_timeout': 2,
+            'socket_connect_timeout': 2
+        }
     }
 }
 

django_project/core/settings/contrib.py

@@ -17,7 +17,8 @@
     'django_cleanup.apps.CleanupConfig',
     'django_celery_beat',
     'django_celery_results',
-    'drf_yasg'
+    'drf_yasg',
+    'rest_framework_tracking'
 )
 
 WEBPACK_LOADER = {
@@ -58,3 +59,6 @@
 ]
 
 SENTRY_DSN = os.environ.get('SENTRY_DSN', '')
+
+# Disable log API request body
+DRF_TRACKING_DECODE_REQUEST_BODY = False

django_project/gap/migrations/0032_preferences_api_log_batch_size.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-10-19 06:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0031_measurement_station_history'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='preferences',
+            name='api_log_batch_size',
+            field=models.IntegerField(default=500, help_text='Number of API Request logs to be saved in a batch.'),
+        ),
+    ]

django_project/gap/models/preferences.py

@@ -120,6 +120,12 @@ class Preferences(SingletonModel):
         )
     )
 
+    # api log batch size
+    api_log_batch_size = models.IntegerField(
+        default=500,
+        help_text='Number of API Request logs to be saved in a batch.'
+    )
+
     class Meta:  # noqa: D106
         verbose_name_plural = "preferences"
 

django_project/gap_api/admin.py

@@ -0,0 +1,172 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP API.
+
+.. note:: Admin for API Tracking
+"""
+
+import random
+from django.contrib import admin
+from django.db.models import Count, TextField
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import TruncDay, Cast
+from rest_framework_tracking.admin import APIRequestLogAdmin
+from rest_framework_tracking.models import APIRequestLog as BaseAPIRequestLog
+
+from gap.models import DatasetType
+from gap_api.models import APIRequestLog
+
+
+admin.site.unregister(BaseAPIRequestLog)
+
+
+def generate_random_color():
+    """Generate random color for product type."""
+    return "#{:06x}".format(random.randint(0, 0xFFFFFF))
+
+
+class ProductTypeFilter(admin.SimpleListFilter):
+    """Custom filter for product type field."""
+
+    title = 'Product Type'
+    parameter_name = 'product_type'
+
+    def lookups(self, request, model_admin):
+        """Get list of product type."""
+        dataset_types = DatasetType.objects.exclude(
+            variable_name='default'
+        ).order_by('variable_name')
+        return [(dt.variable_name, dt.variable_name) for dt in dataset_types]
+
+    def queryset(self, request, queryset):
+        """Filter queryset using product type."""
+        if self.value():
+            return queryset.filter(query_params__product=self.value())
+        return queryset
+
+
+class GapAPIRequestLogAdmin(APIRequestLogAdmin):
+    """Admin class for APIRequestLog model."""
+
+    list_display = (
+        "id",
+        "product_type",
+        "requested_at",
+        "response_ms",
+        "status_code",
+        "user",
+        "view_method",
+        "path",
+        "remote_addr",
+        "host",
+    )
+    list_filter = (ProductTypeFilter, "user", "status_code")
+    search_fields = ()
+
+    def product_type(self, obj: APIRequestLog):
+        """Display product from query_params.
+
+        :param obj: current row
+        :type obj: APIRequestLog
+        :return: product in json query_params
+        :rtype: str
+        """
+        return obj.query_params.get('product', '-')
+
+    product_type.short_description = 'Product Type'
+
+    def changelist_view(self, request, extra_context=None):
+        """Render the changelist view.
+
+        :param request: request
+        :type request: Request object
+        :param extra_context: extra context, defaults to None
+        :type extra_context: any, optional
+        :return: Rendered view
+        :rtype: any
+        """
+        # Aggregate api logs per day
+        chart_data = self._generate_chart_data(request)
+
+        # generate color for products
+        product_counts = []
+        for product in chart_data['product']:
+            product['color'] = generate_random_color()
+            product_counts.append(product)
+
+        extra_context = extra_context or {
+            "chart_data": list(chart_data['total_requests']),
+            "product_chart_data": product_counts
+        }
+
+        # Call the superclass changelist_view to render the page
+        return super().changelist_view(request, extra_context=extra_context)
+
+    def _generate_chart_data(self, request):
+        """Generate chart data and construct the filter from request object.
+
+        :param request: request
+        :type request: Request object
+        :return: APIRequestLog group by Date and the count
+        :rtype: list
+        """
+        product_type = request.GET.get("product_type", None)
+        user_id = request.GET.get("user__id__exact", None)
+
+        # handle requested_at__day, requested_at__month, requested_at__year
+        other_filters = {}
+        for key, val in request.GET.items():
+            if key.startswith('requested_at__'):
+                other_filters[key] = val
+
+        return self._do_query_chart_data(
+            product_type, user_id, other_filters)
+
+    def _do_query_chart_data(
+            self, product_type, user_id, other_filters):
+        """Get chart data by filters.
+
+        :param product_type: product type
+        :type product_type: str
+        :param user_id: user ID
+        :type user_id: int
+        :param other_filters: Dictionary of valid filter
+        :type other_filters: dict
+        :return: APIRequestLog group by Date and the count
+        :rtype: list
+        """
+        filters = {}
+        if product_type:
+            filters['query_params__product'] = product_type
+
+        if user_id:
+            filters['user__id'] = user_id
+
+        filters.update(other_filters)
+        return {
+            'total_requests': (
+                APIRequestLog.objects.filter(
+                    **filters
+                )
+                .annotate(date=TruncDay("requested_at"))
+                .values("date")
+                .annotate(y=Count("id"))
+                .order_by("-date")
+            ),
+            'product': (
+                APIRequestLog.objects.filter(
+                    **filters
+                ).annotate(
+                    product=Cast(
+                        KeyTextTransform('product', 'query_params'),
+                        TextField()
+                    )
+                )
+                .values('product')
+                .annotate(count=Count("id"))
+                .order_by('product')
+            )
+        }
+
+
+admin.site.register(APIRequestLog, GapAPIRequestLogAdmin)

django_project/gap_api/api_views/measurement.py

@@ -41,6 +41,7 @@
 )
 from gap_api.serializers.common import APIErrorSerializer
 from gap_api.utils.helper import ApiTag
+from gap_api.mixins import GAPAPILoggingMixin
 
 
 def attribute_list():
@@ -69,7 +70,7 @@ def default_attribute_list():
         pass
 
 
-class MeasurementAPI(APIView):
+class MeasurementAPI(GAPAPILoggingMixin, APIView):
     """API class for measurement."""
 
     date_format = '%Y-%m-%d'

django_project/gap_api/apps.py

@@ -13,3 +13,7 @@ class GapApiConfig(AppConfig):
 
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'gap_api'
+
+    def ready(self):
+        """App ready handler."""
+        from gap_api.tasks import store_api_logs  # noqa

django_project/gap_api/factories.py

@@ -0,0 +1,36 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP.
+
+.. note:: Factory classes for Models
+"""
+import factory
+from factory.django import DjangoModelFactory
+
+from core.factories import UserF
+from gap_api.models import APIRequestLog
+
+
+class APIRequestLogFactory(DjangoModelFactory):
+    """Factory class for APIRequestLog model."""
+
+    class Meta:  # noqa
+        model = APIRequestLog
+
+    user = factory.SubFactory(UserF)
+    username_persistent = factory.Faker('name')
+    path = '/api/v1/measurement'
+    host = 'http://localhost'
+    method = 'GET'
+    query_params = {
+        'lat': '-1.404244',
+        'lon': '35.008688',
+        'product': 'tahmo_ground_observation',
+        'end_date': '2019-11-10',
+        'attributes': [
+            'max_relative_humidity',
+            'min_relative_humidity'
+        ],
+        'start_date': '2019-11-01',
+        'output_type': 'csv'
+    }