diff --git a/deployment/docker/requirements.txt b/deployment/docker/requirements.txt index e1f217bc..37447f52 100644 --- a/deployment/docker/requirements.txt +++ b/deployment/docker/requirements.txt @@ -44,7 +44,7 @@ django-storages==1.14.2 django-storages[s3] # Python client for Redis database and key-value store -redis==4.3.4 +redis==5.1.1 psycopg2-binary==2.9.9 # drf yasg @@ -69,3 +69,6 @@ numpy==1.26.4 salientsdk==0.3.5 python-geohash==0.8.5 + +# api tracking +drf-api-tracking==1.8.4 diff --git a/django_project/_version.txt b/django_project/_version.txt index fa3de586..99d85ecd 100644 --- a/django_project/_version.txt +++ b/django_project/_version.txt @@ -1 +1 @@ -0.0.5 \ No newline at end of file +0.0.6 \ No newline at end of file diff --git a/django_project/core/celery.py b/django_project/core/celery.py index bc55a195..16112fb4 100644 --- a/django_project/core/celery.py +++ b/django_project/core/celery.py @@ -9,9 +9,12 @@ from celery.schedules import crontab from celery.utils.serialization import strtobool from celery.worker.control import inspect_command +from celery.worker import strategy from django.utils import timezone logger = logging.getLogger(__name__) +# Disable info log from Celery MainProcess +strategy.logger.setLevel(logging.WARNING) # set the default Django settings module for the 'celery' program. # this is also used in manage.py @@ -69,6 +72,11 @@ # Run everyday at 00:00 UTC 'schedule': crontab(minute='00', hour='00'), }, + 'store-api-logs': { + 'task': 'store_api_logs', + # Run every 5minutes + 'schedule': crontab(minute='*/5'), + }, 'cleanup-r-execution-logs': { 'task': 'cleanup_r_execution_logs', # Run every first day of each month at 00:00 UTC @@ -151,7 +159,8 @@ def update_task_progress( EXCLUDED_TASK_LIST = [ - 'celery.backend_cleanup' + 'celery.backend_cleanup', + 'store_api_logs' ] diff --git a/django_project/core/settings/base.py b/django_project/core/settings/base.py index f8f550b1..fd843149 100644 --- a/django_project/core/settings/base.py +++ b/django_project/core/settings/base.py @@ -93,6 +93,7 @@ 'DIRS': [ # Put Templates absolute_path('core', 'templates'), + absolute_path('gap_api', 'templates'), ], 'OPTIONS': { 'loaders': [ @@ -141,7 +142,11 @@ 'LOCATION': ( f'redis://default:{os.environ.get("REDIS_PASSWORD", "")}' f'@{os.environ.get("REDIS_HOST", "")}', - ) + ), + 'OPTIONS': { + 'socket_timeout': 2, + 'socket_connect_timeout': 2 + } } } diff --git a/django_project/core/settings/contrib.py b/django_project/core/settings/contrib.py index 15556a7b..29a4d2b8 100644 --- a/django_project/core/settings/contrib.py +++ b/django_project/core/settings/contrib.py @@ -17,7 +17,8 @@ 'django_cleanup.apps.CleanupConfig', 'django_celery_beat', 'django_celery_results', - 'drf_yasg' + 'drf_yasg', + 'rest_framework_tracking' ) WEBPACK_LOADER = { @@ -58,3 +59,6 @@ ] SENTRY_DSN = os.environ.get('SENTRY_DSN', '') + +# Disable log API request body +DRF_TRACKING_DECODE_REQUEST_BODY = False diff --git a/django_project/gap/migrations/0032_preferences_api_log_batch_size.py b/django_project/gap/migrations/0032_preferences_api_log_batch_size.py new file mode 100644 index 00000000..f3b12c5e --- /dev/null +++ b/django_project/gap/migrations/0032_preferences_api_log_batch_size.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-10-19 06:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gap', '0031_measurement_station_history'), + ] + + operations = [ + migrations.AddField( + model_name='preferences', + name='api_log_batch_size', + field=models.IntegerField(default=500, help_text='Number of API Request logs to be saved in a batch.'), + ), + ] diff --git a/django_project/gap/models/preferences.py b/django_project/gap/models/preferences.py index 86358307..de5b6e45 100644 --- a/django_project/gap/models/preferences.py +++ b/django_project/gap/models/preferences.py @@ -120,6 +120,12 @@ class Preferences(SingletonModel): ) ) + # api log batch size + api_log_batch_size = models.IntegerField( + default=500, + help_text='Number of API Request logs to be saved in a batch.' + ) + class Meta: # noqa: D106 verbose_name_plural = "preferences" diff --git a/django_project/gap_api/admin.py b/django_project/gap_api/admin.py new file mode 100644 index 00000000..7f780c14 --- /dev/null +++ b/django_project/gap_api/admin.py @@ -0,0 +1,172 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Admin for API Tracking +""" + +import random +from django.contrib import admin +from django.db.models import Count, TextField +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import TruncDay, Cast +from rest_framework_tracking.admin import APIRequestLogAdmin +from rest_framework_tracking.models import APIRequestLog as BaseAPIRequestLog + +from gap.models import DatasetType +from gap_api.models import APIRequestLog + + +admin.site.unregister(BaseAPIRequestLog) + + +def generate_random_color(): + """Generate random color for product type.""" + return "#{:06x}".format(random.randint(0, 0xFFFFFF)) + + +class ProductTypeFilter(admin.SimpleListFilter): + """Custom filter for product type field.""" + + title = 'Product Type' + parameter_name = 'product_type' + + def lookups(self, request, model_admin): + """Get list of product type.""" + dataset_types = DatasetType.objects.exclude( + variable_name='default' + ).order_by('variable_name') + return [(dt.variable_name, dt.variable_name) for dt in dataset_types] + + def queryset(self, request, queryset): + """Filter queryset using product type.""" + if self.value(): + return queryset.filter(query_params__product=self.value()) + return queryset + + +class GapAPIRequestLogAdmin(APIRequestLogAdmin): + """Admin class for APIRequestLog model.""" + + list_display = ( + "id", + "product_type", + "requested_at", + "response_ms", + "status_code", + "user", + "view_method", + "path", + "remote_addr", + "host", + ) + list_filter = (ProductTypeFilter, "user", "status_code") + search_fields = () + + def product_type(self, obj: APIRequestLog): + """Display product from query_params. + + :param obj: current row + :type obj: APIRequestLog + :return: product in json query_params + :rtype: str + """ + return obj.query_params.get('product', '-') + + product_type.short_description = 'Product Type' + + def changelist_view(self, request, extra_context=None): + """Render the changelist view. + + :param request: request + :type request: Request object + :param extra_context: extra context, defaults to None + :type extra_context: any, optional + :return: Rendered view + :rtype: any + """ + # Aggregate api logs per day + chart_data = self._generate_chart_data(request) + + # generate color for products + product_counts = [] + for product in chart_data['product']: + product['color'] = generate_random_color() + product_counts.append(product) + + extra_context = extra_context or { + "chart_data": list(chart_data['total_requests']), + "product_chart_data": product_counts + } + + # Call the superclass changelist_view to render the page + return super().changelist_view(request, extra_context=extra_context) + + def _generate_chart_data(self, request): + """Generate chart data and construct the filter from request object. + + :param request: request + :type request: Request object + :return: APIRequestLog group by Date and the count + :rtype: list + """ + product_type = request.GET.get("product_type", None) + user_id = request.GET.get("user__id__exact", None) + + # handle requested_at__day, requested_at__month, requested_at__year + other_filters = {} + for key, val in request.GET.items(): + if key.startswith('requested_at__'): + other_filters[key] = val + + return self._do_query_chart_data( + product_type, user_id, other_filters) + + def _do_query_chart_data( + self, product_type, user_id, other_filters): + """Get chart data by filters. + + :param product_type: product type + :type product_type: str + :param user_id: user ID + :type user_id: int + :param other_filters: Dictionary of valid filter + :type other_filters: dict + :return: APIRequestLog group by Date and the count + :rtype: list + """ + filters = {} + if product_type: + filters['query_params__product'] = product_type + + if user_id: + filters['user__id'] = user_id + + filters.update(other_filters) + return { + 'total_requests': ( + APIRequestLog.objects.filter( + **filters + ) + .annotate(date=TruncDay("requested_at")) + .values("date") + .annotate(y=Count("id")) + .order_by("-date") + ), + 'product': ( + APIRequestLog.objects.filter( + **filters + ).annotate( + product=Cast( + KeyTextTransform('product', 'query_params'), + TextField() + ) + ) + .values('product') + .annotate(count=Count("id")) + .order_by('product') + ) + } + + +admin.site.register(APIRequestLog, GapAPIRequestLogAdmin) diff --git a/django_project/gap_api/api_views/measurement.py b/django_project/gap_api/api_views/measurement.py index ad81d8b1..bb16ea55 100644 --- a/django_project/gap_api/api_views/measurement.py +++ b/django_project/gap_api/api_views/measurement.py @@ -41,6 +41,7 @@ ) from gap_api.serializers.common import APIErrorSerializer from gap_api.utils.helper import ApiTag +from gap_api.mixins import GAPAPILoggingMixin def attribute_list(): @@ -69,7 +70,7 @@ def default_attribute_list(): pass -class MeasurementAPI(APIView): +class MeasurementAPI(GAPAPILoggingMixin, APIView): """API class for measurement.""" date_format = '%Y-%m-%d' diff --git a/django_project/gap_api/apps.py b/django_project/gap_api/apps.py index 39f93c60..d05cb8e6 100644 --- a/django_project/gap_api/apps.py +++ b/django_project/gap_api/apps.py @@ -13,3 +13,7 @@ class GapApiConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'gap_api' + + def ready(self): + """App ready handler.""" + from gap_api.tasks import store_api_logs # noqa diff --git a/django_project/gap_api/factories.py b/django_project/gap_api/factories.py new file mode 100644 index 00000000..8c48e229 --- /dev/null +++ b/django_project/gap_api/factories.py @@ -0,0 +1,36 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Factory classes for Models +""" +import factory +from factory.django import DjangoModelFactory + +from core.factories import UserF +from gap_api.models import APIRequestLog + + +class APIRequestLogFactory(DjangoModelFactory): + """Factory class for APIRequestLog model.""" + + class Meta: # noqa + model = APIRequestLog + + user = factory.SubFactory(UserF) + username_persistent = factory.Faker('name') + path = '/api/v1/measurement' + host = 'http://localhost' + method = 'GET' + query_params = { + 'lat': '-1.404244', + 'lon': '35.008688', + 'product': 'tahmo_ground_observation', + 'end_date': '2019-11-10', + 'attributes': [ + 'max_relative_humidity', + 'min_relative_humidity' + ], + 'start_date': '2019-11-01', + 'output_type': 'csv' + } diff --git a/django_project/gap_api/migrations/0001_initial.py b/django_project/gap_api/migrations/0001_initial.py new file mode 100644 index 00000000..bbeeb5c8 --- /dev/null +++ b/django_project/gap_api/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 4.2.7 on 2024-10-17 16:50 + +import datetime +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='APIRequestLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('username_persistent', models.CharField(blank=True, max_length=200, null=True)), + ('requested_at', models.DateTimeField(db_index=True, default=datetime.datetime.now)), + ('response_ms', models.PositiveIntegerField(default=0)), + ('path', models.CharField(db_index=True, help_text='url path', max_length=200)), + ('view', models.CharField(blank=True, db_index=True, help_text='method called by this endpoint', max_length=200, null=True)), + ('view_method', models.CharField(blank=True, db_index=True, max_length=200, null=True)), + ('remote_addr', models.GenericIPAddressField(blank=True, null=True)), + ('host', models.URLField()), + ('method', models.CharField(max_length=10)), + ('user_agent', models.CharField(blank=True, max_length=255)), + ('data', models.TextField(blank=True, null=True)), + ('response', models.TextField(blank=True, null=True)), + ('errors', models.TextField(blank=True, null=True)), + ('status_code', models.PositiveIntegerField(blank=True, db_index=True, null=True)), + ('query_params', models.JSONField(blank=True, default=dict, null=True)), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='user_api', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'API Request Log', + 'abstract': False, + }, + ), + ] diff --git a/django_project/gap_api/mixins/__init__.py b/django_project/gap_api/mixins/__init__.py new file mode 100644 index 00000000..5fb58bae --- /dev/null +++ b/django_project/gap_api/mixins/__init__.py @@ -0,0 +1 @@ +from gap_api.mixins.logging import * # noqa diff --git a/django_project/gap_api/mixins/logging.py b/django_project/gap_api/mixins/logging.py new file mode 100644 index 00000000..dbbe66c0 --- /dev/null +++ b/django_project/gap_api/mixins/logging.py @@ -0,0 +1,33 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Mixin for API Tracking +""" + +import json +from django.core.cache import cache +from django.core.serializers.json import DjangoJSONEncoder +from rest_framework_tracking.base_mixins import BaseLoggingMixin + + +class GAPAPILoggingMixin(BaseLoggingMixin): + """Mixin to log GAP API request.""" + + CACHE_KEY = 'gap_logs_queue' + + def handle_log(self): + """Store log to redis queue cache.""" + # remove data and response + self.log['data'] = None + self.log['response'] = None + if self.log.get('user', None): + # replace with user id + self.log['user'] = self.log['user'].id + try: + cache._cache.get_client().rpush( + self.CACHE_KEY, + json.dumps(self.log, cls=DjangoJSONEncoder) + ) + except Exception: # noqa + pass diff --git a/django_project/gap_api/models/__init__.py b/django_project/gap_api/models/__init__.py new file mode 100644 index 00000000..04c9f084 --- /dev/null +++ b/django_project/gap_api/models/__init__.py @@ -0,0 +1 @@ +from gap_api.models.api_request_log import * # noqa diff --git a/django_project/gap_api/models/api_request_log.py b/django_project/gap_api/models/api_request_log.py new file mode 100644 index 00000000..a7e671f3 --- /dev/null +++ b/django_project/gap_api/models/api_request_log.py @@ -0,0 +1,28 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Models for API Tracking +""" + +from django.db import models +from django.conf import settings +from rest_framework_tracking.base_models import BaseAPIRequestLog + + +class APIRequestLog(BaseAPIRequestLog): + """Models that stores GAP API request log.""" + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='user_api' + ) + + query_params = models.JSONField( + default=dict, + null=True, + blank=True + ) diff --git a/django_project/gap_api/tasks.py b/django_project/gap_api/tasks.py new file mode 100644 index 00000000..ec9cffe5 --- /dev/null +++ b/django_project/gap_api/tasks.py @@ -0,0 +1,55 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Tasks for API Tracking +""" + +import json +from core.celery import app +from django.core.cache import cache +from django.contrib.auth import get_user_model +from django.utils.dateparse import parse_datetime + +from gap.models import Preferences +from gap_api.models import APIRequestLog +from gap_api.mixins import GAPAPILoggingMixin + + +UserModel = get_user_model() + + +@app.task(name='store_api_logs', ignore_result=True) +def store_api_logs(): + """Store API Logs from Redis into database.""" + # pull logs from Redis + batch_size = Preferences.load().api_log_batch_size + logs = [] + for _ in range(batch_size): + log_entry = cache._cache.get_client().lpop( + GAPAPILoggingMixin.CACHE_KEY + ) + if log_entry: + entry = json.loads(log_entry) + # parse requested_at + if entry.get('requested_at', None): + entry['requested_at'] = parse_datetime(entry['requested_at']) + + # parse user + if entry.get('user', None): + user_id = entry['user'] + entry['user'] = UserModel._default_manager.filter( + id=user_id).first() + + # parse attributes in query_params + if entry.get('query_params', None): + attributes = entry['query_params'].get('attributes', '') + entry['query_params']['attributes'] = ( + attributes.replace(' ', '').split(',') + ) + + logs.append(APIRequestLog(**entry)) + + # write logs to the database + if logs: + APIRequestLog.objects.bulk_create(logs) diff --git a/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html new file mode 100644 index 00000000..475ca937 --- /dev/null +++ b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html @@ -0,0 +1,129 @@ +{% extends "admin/change_list.html" %} +{% load static %} + + +{% block extrahead %} +{{ block.super }} + + + + + + + +{{ chart_data|json_script:"chartData" }} +{{ product_chart_data|json_script:"productChartData" }} + +{% endblock %} + +{% block content %} + +
+ +
+ +
+ +
+ +
+
+ +{{ block.super }} +{% endblock %} diff --git a/django_project/gap_api/tests/test_api_request_log.py b/django_project/gap_api/tests/test_api_request_log.py new file mode 100644 index 00000000..298a681b --- /dev/null +++ b/django_project/gap_api/tests/test_api_request_log.py @@ -0,0 +1,169 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Unit tests for API Request Log. +""" + +import json +import mock +import datetime +from django.test import TestCase, RequestFactory, override_settings +from django.contrib.admin import ModelAdmin + +from core.factories import UserF +from gap.models import DatasetType +from gap_api.models import APIRequestLog +from gap_api.tasks import store_api_logs +from gap_api.admin import ProductTypeFilter, GapAPIRequestLogAdmin +from gap_api.factories import APIRequestLogFactory + + +class MockRequestObj(object): + """Mock GET request object.""" + + GET = {} + + +@override_settings( + CACHES={ + 'default': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + } + } +) +class TestAPIRequestLog(TestCase): + """Test class for APIRequestLog model and task.""" + + fixtures = [ + '2.provider.json', + '3.station_type.json', + '4.dataset_type.json' + ] + + def setUp(self): + """Initialize test class.""" + self.factory = RequestFactory() + self.user = UserF.create() + self.admin_instance = GapAPIRequestLogAdmin( + APIRequestLog, admin_site=mock.MagicMock()) + + @mock.patch('django.core.cache.cache._cache.get_client') + def test_store_api_logs(self, mock_get_client): + """Test store api logs from cache.""" + mock_redis_client = mock_get_client.return_value + mock_redis_client.lpop.side_effect = [ + json.dumps({ + "requested_at": "2024-10-17 17:56:52.270889+00:00", + "data": None, + "remote_addr": "127.0.0.1", + "view": "gap_api.api_views.measurement.MeasurementAPI", + "view_method": "get", + "path": "/api/v1/measurement/", + "host": "localhost:8000", + "user_agent": "PostmanRuntime/7.42.0", + "method": "GET", + "query_params": { + "lat": "-1.404244", + "lon": "35.008688", + "attributes": ( + "max_relative_humidity,min_relative_humidity" + ), + "start_date": "2019-11-01", + "end_date": "2019-11-01", + "product": "tahmo_ground_observation", + "output_type": "csv" + }, + "user": f"{self.user.id}", + "username_persistent": "admin", + "response_ms": 182, + "response": None, + "status_code": 200 + }) + ] + [None] * 499 + store_api_logs() + # default batch size + self.assertEqual(mock_redis_client.lpop.call_count, 500) + logs = APIRequestLog.objects.all() + self.assertEqual(logs.count(), 1) + self.assertEqual(logs.first().user, self.user) + + def test_product_type_filter(self): + """Test ProductTypeFilter class.""" + f = ProductTypeFilter(None, {}, APIRequestLog, self.admin_instance) + names = f.lookups(None, self.admin_instance) + self.assertEqual( + len(names), + DatasetType.objects.exclude( + variable_name='default' + ).count() + ) + + def test_admin_product_type(self): + """Test product_type field in Admin class.""" + f = ProductTypeFilter(None, {}, APIRequestLog, self.admin_instance) + f.used_parameters = {} + + # create two logs + log1 = APIRequestLogFactory.create( + query_params={ + 'product': 'cbam_reanalysis_data' + } + ) + APIRequestLogFactory.create() + + qs = APIRequestLog.objects.all() + + # query with empty param should return all + res = f.queryset(None, qs) + self.assertEqual(res.count(), 2) + + # query with product_type cbam should return 1 + f.used_parameters = { + 'product_type': 'cbam_reanalysis_data' + } + res = f.queryset(None, qs) + self.assertEqual(res.count(), 1) + self.assertEqual(res.first().id, log1.id) + + @mock.patch.object(GapAPIRequestLogAdmin, '_generate_chart_data') + @mock.patch.object(ModelAdmin, 'changelist_view') + def test_change_list_view( + self, mock_super_changelist_view, mock_generate_chart_data): + """Test change_list view.""" + mock_chart_data = { + 'total_requests': [{"date": "2024-10-18", "count": 10}], + 'product': [{"product": "product a", "count": 10}] + } + mock_generate_chart_data.return_value = mock_chart_data + + # Simulate a GET request to the changelist view + request = self.factory.get('/admin/gap_api/apirequestlog/') + + # Call the changelist_view method + self.admin_instance.changelist_view(request) + + # Assert that _generate_chart_data was called + mock_generate_chart_data.assert_called_once_with(request) + + mock_super_changelist_view.assert_called_once() + + def test_generate_chart_data(self): + """Test generate chart data.""" + APIRequestLogFactory.create( + requested_at=datetime.datetime(2024, 10, 1, 0, 0, 0), + user=self.user + ) + APIRequestLogFactory.create( + requested_at=datetime.datetime(2023, 5, 1, 0, 0, 0), + user=self.user + ) + req = MockRequestObj() + req.GET = { + 'product_type': 'tahmo_ground_observation', + 'requested_at__year': '2023', + 'user__id__exact': f'{self.user.id}' + } + data = self.admin_instance._generate_chart_data(req) + self.assertEqual(len(data['total_requests']), 1) + self.assertEqual(len(data['product']), 1)