From 1f0feddbe33fcf5f4b2cdc911407da47da4902b9 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Thu, 17 Oct 2024 21:24:17 +0100 Subject: [PATCH 01/14] add api tracking logs --- deployment/docker/requirements.txt | 3 + django_project/core/celery.py | 8 + django_project/core/settings/base.py | 1 + django_project/core/settings/contrib.py | 6 +- .../0031_preferences_api_log_batch_size.py | 18 ++ django_project/gap/models/preferences.py | 6 + django_project/gap_api/admin.py | 171 ++++++++++++++++++ .../gap_api/api_views/measurement.py | 3 +- django_project/gap_api/apps.py | 4 + .../gap_api/migrations/0001_initial.py | 44 +++++ django_project/gap_api/mixins.py | 30 +++ django_project/gap_api/models.py | 28 +++ django_project/gap_api/tasks.py | 48 +++++ .../gap_api/apirequestlog/change_list.html | 136 ++++++++++++++ 14 files changed, 504 insertions(+), 2 deletions(-) create mode 100644 django_project/gap/migrations/0031_preferences_api_log_batch_size.py create mode 100644 django_project/gap_api/admin.py create mode 100644 django_project/gap_api/migrations/0001_initial.py create mode 100644 django_project/gap_api/mixins.py create mode 100644 django_project/gap_api/models.py create mode 100644 django_project/gap_api/tasks.py create mode 100644 django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html diff --git a/deployment/docker/requirements.txt b/deployment/docker/requirements.txt index e1f217bc..f9a4c9cc 100644 --- a/deployment/docker/requirements.txt +++ b/deployment/docker/requirements.txt @@ -69,3 +69,6 @@ numpy==1.26.4 salientsdk==0.3.5 python-geohash==0.8.5 + +# api tracking +drf-api-tracking==1.8.4 diff --git a/django_project/core/celery.py b/django_project/core/celery.py index 34093501..96658840 100644 --- a/django_project/core/celery.py +++ b/django_project/core/celery.py @@ -9,9 +9,12 @@ from celery.schedules import crontab from celery.utils.serialization import strtobool from celery.worker.control import inspect_command +from celery.worker import strategy from django.utils import timezone logger = logging.getLogger(__name__) +# Disable info log from Celery MainProcess +strategy.logger.setLevel(logging.WARNING) # set the default Django settings module for the 'celery' program. # this is also used in manage.py @@ -68,6 +71,11 @@ 'task': 'run_daily_ingestor', # Run everyday at 00:00 UTC 'schedule': crontab(minute='00', hour='00'), + }, + 'store-api-logs': { + 'task': 'store_api_logs', + # Run every 2minutes + 'schedule': crontab(minute='*/2'), } } diff --git a/django_project/core/settings/base.py b/django_project/core/settings/base.py index f8f550b1..dd86ba5c 100644 --- a/django_project/core/settings/base.py +++ b/django_project/core/settings/base.py @@ -93,6 +93,7 @@ 'DIRS': [ # Put Templates absolute_path('core', 'templates'), + absolute_path('gap_api', 'templates'), ], 'OPTIONS': { 'loaders': [ diff --git a/django_project/core/settings/contrib.py b/django_project/core/settings/contrib.py index 15556a7b..29a4d2b8 100644 --- a/django_project/core/settings/contrib.py +++ b/django_project/core/settings/contrib.py @@ -17,7 +17,8 @@ 'django_cleanup.apps.CleanupConfig', 'django_celery_beat', 'django_celery_results', - 'drf_yasg' + 'drf_yasg', + 'rest_framework_tracking' ) WEBPACK_LOADER = { @@ -58,3 +59,6 @@ ] SENTRY_DSN = os.environ.get('SENTRY_DSN', '') + +# Disable log API request body +DRF_TRACKING_DECODE_REQUEST_BODY = False diff --git a/django_project/gap/migrations/0031_preferences_api_log_batch_size.py b/django_project/gap/migrations/0031_preferences_api_log_batch_size.py new file mode 100644 index 00000000..7cf4bbec --- /dev/null +++ b/django_project/gap/migrations/0031_preferences_api_log_batch_size.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-10-17 19:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gap', '0030_preferences_ingestor_config'), + ] + + operations = [ + migrations.AddField( + model_name='preferences', + name='api_log_batch_size', + field=models.IntegerField(default=500, help_text='Number of API Request logs to be saved in a batch.'), + ), + ] diff --git a/django_project/gap/models/preferences.py b/django_project/gap/models/preferences.py index 86358307..de5b6e45 100644 --- a/django_project/gap/models/preferences.py +++ b/django_project/gap/models/preferences.py @@ -120,6 +120,12 @@ class Preferences(SingletonModel): ) ) + # api log batch size + api_log_batch_size = models.IntegerField( + default=500, + help_text='Number of API Request logs to be saved in a batch.' + ) + class Meta: # noqa: D106 verbose_name_plural = "preferences" diff --git a/django_project/gap_api/admin.py b/django_project/gap_api/admin.py new file mode 100644 index 00000000..04307c4e --- /dev/null +++ b/django_project/gap_api/admin.py @@ -0,0 +1,171 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Admin for API Tracking +""" + +import datetime +from django.contrib import admin +from django.http import JsonResponse +from django.db.models import Count +from django.db.models.functions import TruncDay +from rest_framework_tracking.admin import APIRequestLogAdmin +from rest_framework_tracking.models import APIRequestLog as BaseAPIRequestLog + +from gap.models import DatasetType +from gap_api.models import APIRequestLog + + +admin.site.unregister(BaseAPIRequestLog) + + +class ProductTypeFilter(admin.SimpleListFilter): + """Custom filter for product type field.""" + + title = 'Product Type' + parameter_name = 'product_type' + + def lookups(self, request, model_admin): + """Get list of product type.""" + dataset_types = DatasetType.objects.exclude( + variable_name='default' + ).order_by('variable_name') + return [(dt.variable_name, dt.variable_name) for dt in dataset_types] + + def queryset(self, request, queryset): + """Filter queryset using product type.""" + if self.value(): + return queryset.filter(query_params__product=self.value()) + return queryset + + +class GapAPIRequestLogAdmin(APIRequestLogAdmin): + """Admin class for APIRequestLog model.""" + + list_display = ( + "id", + "product_type", + "requested_at", + "response_ms", + "status_code", + "user", + "view_method", + "path", + "remote_addr", + "host", + ) + list_filter = (ProductTypeFilter, "user", "status_code") + search_fields = () + + def product_type(self, obj: APIRequestLog): + """Display product from query_params. + + :param obj: current row + :type obj: APIRequestLog + :return: product in json query_params + :rtype: str + """ + return obj.query_params.get('product', '-') + + product_type.short_description = 'Product Type' + + def changelist_view(self, request, extra_context=None): + """Render the changelist view. + + :param request: request + :type request: Request object + :param extra_context: extra context, defaults to None + :type extra_context: any, optional + :return: Rendered view + :rtype: any + """ + # Aggregate api logs per day + chart_data = self._generate_chart_data(request) + + extra_context = extra_context or {"chart_data": list(chart_data)} + + # Call the superclass changelist_view to render the page + return super().changelist_view(request, extra_context=extra_context) + + def chart_data_endpoint(self, request): + """Get response for chart data. + + :param request: request + :type request: Request object + :return: Chart data + :rtype: JsonResponse + """ + return JsonResponse( + list(self._generate_chart_data(request)), safe=False) + + def _generate_chart_data(self, request): + """Generate chart data and construct the filter from request object. + + :param request: request + :type request: Request object + :return: APIRequestLog group by Date and the count + :rtype: list + """ + start_date = request.GET.get("start_date", None) + end_date = request.GET.get("end_date", None) + product_type = request.GET.get("product_type", None) + user_id = request.GET.get("user__id__exact", None) + + # convert start_date and end_date to datetime objects + if start_date: + start_date = datetime.datetime.strptime( + start_date, "%Y-%m-%d").date() + if end_date: + end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d").date() + + # handle requested_at__day, requested_at__month, requested_at__year + other_filters = {} + for key, val in request.GET.items(): + if key.startswith('requested_at__'): + other_filters[key] = val + + return self._do_query_chart_data( + start_date, end_date, product_type, user_id, other_filters) + + def _do_query_chart_data( + self, start_date, end_date, product_type, user_id, other_filters): + """Get chart data by filters. + + :param start_date: start date filter + :type start_date: datetime + :param end_date: end date filter + :type end_date: datetime + :param product_type: product type + :type product_type: str + :param user_id: user ID + :type user_id: int + :param other_filters: Dictionary of valid filter + :type other_filters: dict + :return: APIRequestLog group by Date and the count + :rtype: list + """ + filters = {} + if start_date and end_date: + filters['requested_at__date__gte'] = start_date + filters['requested_at__date__lte'] = end_date + + if product_type: + filters['query_params__product'] = product_type + + if user_id: + filters['user__id'] = user_id + + filters.update(other_filters) + return ( + APIRequestLog.objects.filter( + **filters + ) + .annotate(date=TruncDay("requested_at")) + .values("date") + .annotate(y=Count("id")) + .order_by("-date") + ) + + +admin.site.register(APIRequestLog, GapAPIRequestLogAdmin) diff --git a/django_project/gap_api/api_views/measurement.py b/django_project/gap_api/api_views/measurement.py index bef76225..e1d11376 100644 --- a/django_project/gap_api/api_views/measurement.py +++ b/django_project/gap_api/api_views/measurement.py @@ -39,9 +39,10 @@ from gap_api.serializers.common import APIErrorSerializer from gap_api.utils.helper import ApiTag from gap.providers import get_reader_from_dataset +from gap_api.mixins import GAPAPILoggingMixin -class MeasurementAPI(APIView): +class MeasurementAPI(GAPAPILoggingMixin, APIView): """API class for measurement.""" date_format = '%Y-%m-%d' diff --git a/django_project/gap_api/apps.py b/django_project/gap_api/apps.py index 39f93c60..d05cb8e6 100644 --- a/django_project/gap_api/apps.py +++ b/django_project/gap_api/apps.py @@ -13,3 +13,7 @@ class GapApiConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'gap_api' + + def ready(self): + """App ready handler.""" + from gap_api.tasks import store_api_logs # noqa diff --git a/django_project/gap_api/migrations/0001_initial.py b/django_project/gap_api/migrations/0001_initial.py new file mode 100644 index 00000000..bbeeb5c8 --- /dev/null +++ b/django_project/gap_api/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 4.2.7 on 2024-10-17 16:50 + +import datetime +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='APIRequestLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('username_persistent', models.CharField(blank=True, max_length=200, null=True)), + ('requested_at', models.DateTimeField(db_index=True, default=datetime.datetime.now)), + ('response_ms', models.PositiveIntegerField(default=0)), + ('path', models.CharField(db_index=True, help_text='url path', max_length=200)), + ('view', models.CharField(blank=True, db_index=True, help_text='method called by this endpoint', max_length=200, null=True)), + ('view_method', models.CharField(blank=True, db_index=True, max_length=200, null=True)), + ('remote_addr', models.GenericIPAddressField(blank=True, null=True)), + ('host', models.URLField()), + ('method', models.CharField(max_length=10)), + ('user_agent', models.CharField(blank=True, max_length=255)), + ('data', models.TextField(blank=True, null=True)), + ('response', models.TextField(blank=True, null=True)), + ('errors', models.TextField(blank=True, null=True)), + ('status_code', models.PositiveIntegerField(blank=True, db_index=True, null=True)), + ('query_params', models.JSONField(blank=True, default=dict, null=True)), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='user_api', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'API Request Log', + 'abstract': False, + }, + ), + ] diff --git a/django_project/gap_api/mixins.py b/django_project/gap_api/mixins.py new file mode 100644 index 00000000..4062ba48 --- /dev/null +++ b/django_project/gap_api/mixins.py @@ -0,0 +1,30 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Mixins for API Tracking +""" + +import json +from django.core.cache import cache +from django.core.serializers.json import DjangoJSONEncoder +from rest_framework_tracking.base_mixins import BaseLoggingMixin + + +class GAPAPILoggingMixin(BaseLoggingMixin): + """Mixin to log GAP API request.""" + + CACHE_KEY = 'gap_logs_queue' + + def handle_log(self): + """Store log to redis queue cache.""" + # remove data and response + self.log['data'] = None + self.log['response'] = None + if self.log.get('user', None): + # replace with user id + self.log['user'] = self.log['user'].id + cache._cache.get_client().rpush( + self.CACHE_KEY, + json.dumps(self.log, cls=DjangoJSONEncoder) + ) diff --git a/django_project/gap_api/models.py b/django_project/gap_api/models.py new file mode 100644 index 00000000..a7e671f3 --- /dev/null +++ b/django_project/gap_api/models.py @@ -0,0 +1,28 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Models for API Tracking +""" + +from django.db import models +from django.conf import settings +from rest_framework_tracking.base_models import BaseAPIRequestLog + + +class APIRequestLog(BaseAPIRequestLog): + """Models that stores GAP API request log.""" + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='user_api' + ) + + query_params = models.JSONField( + default=dict, + null=True, + blank=True + ) diff --git a/django_project/gap_api/tasks.py b/django_project/gap_api/tasks.py new file mode 100644 index 00000000..3300e515 --- /dev/null +++ b/django_project/gap_api/tasks.py @@ -0,0 +1,48 @@ +# coding=utf-8 +""" +Tomorrow Now GAP API. + +.. note:: Tasks for API Tracking +""" + +import json +from core.celery import app +from django.core.cache import cache +from django.contrib.auth import get_user_model +from django.utils.dateparse import parse_datetime + +from gap.models import Preferences +from gap_api.models import APIRequestLog +from gap_api.mixins import GAPAPILoggingMixin + + +UserModel = get_user_model() + + +@app.task(name='store_api_logs', ignore_result=True) +def store_api_logs(): + """Store API Logs from Redis into database.""" + # pull logs from Redis + batch_size = Preferences.load().api_log_batch_size + logs = [] + for _ in range(batch_size): + log_entry = cache._cache.get_client().lpop( + GAPAPILoggingMixin.CACHE_KEY + ) + if log_entry: + entry = json.loads(log_entry) + # parse requested_at + if entry.get('requested_at', None): + entry['requested_at'] = parse_datetime(entry['requested_at']) + + # parse user + if entry.get('user', None): + user_id = entry['user'] + entry['user'] = UserModel._default_manager.filter( + id=user_id).first() + + logs.append(APIRequestLog(**entry)) + + # write logs to the database + if logs: + APIRequestLog.objects.bulk_create(logs) diff --git a/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html new file mode 100644 index 00000000..d598d751 --- /dev/null +++ b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html @@ -0,0 +1,136 @@ +{% extends "admin/change_list.html" %} +{% load static %} + + +{% block extrahead %} +{{ block.super }} + + + + + + + + + +{{ chart_data|json_script:"chartData" }} + +{% endblock %} + +{% block content %} + + + +
+ +
+ +{{ block.super }} +{% endblock %} From ebc27bb7a8672f58c0d787b6fd017ff6796dd7a6 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Fri, 18 Oct 2024 08:05:48 +0100 Subject: [PATCH 02/14] change the cron task to run every 5mins --- django_project/core/celery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_project/core/celery.py b/django_project/core/celery.py index 96658840..07f69293 100644 --- a/django_project/core/celery.py +++ b/django_project/core/celery.py @@ -74,8 +74,8 @@ }, 'store-api-logs': { 'task': 'store_api_logs', - # Run every 2minutes - 'schedule': crontab(minute='*/2'), + # Run every 5minutes + 'schedule': crontab(minute='*/5'), } } From 321f7635e4d01e32356e19a56bbc557e68322be9 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Fri, 18 Oct 2024 08:06:04 +0100 Subject: [PATCH 03/14] parse attributes in query_params to list --- django_project/gap_api/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/django_project/gap_api/tasks.py b/django_project/gap_api/tasks.py index 3300e515..ec9cffe5 100644 --- a/django_project/gap_api/tasks.py +++ b/django_project/gap_api/tasks.py @@ -41,6 +41,13 @@ def store_api_logs(): entry['user'] = UserModel._default_manager.filter( id=user_id).first() + # parse attributes in query_params + if entry.get('query_params', None): + attributes = entry['query_params'].get('attributes', '') + entry['query_params']['attributes'] = ( + attributes.replace(' ', '').split(',') + ) + logs.append(APIRequestLog(**entry)) # write logs to the database From 705d87e96c56d56e90413135cb5cd4b89c307bc7 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Fri, 18 Oct 2024 19:19:03 +0100 Subject: [PATCH 04/14] set redis timeout and ignore exception during push log --- django_project/core/settings/base.py | 6 +++++- django_project/gap_api/mixins.py | 11 +++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/django_project/core/settings/base.py b/django_project/core/settings/base.py index dd86ba5c..155e9bf8 100644 --- a/django_project/core/settings/base.py +++ b/django_project/core/settings/base.py @@ -142,7 +142,11 @@ 'LOCATION': ( f'redis://default:{os.environ.get("REDIS_PASSWORD", "")}' f'@{os.environ.get("REDIS_HOST", "")}', - ) + ), + 'OPTIONS': { + 'SOCKET_TIMEOUT': 2, + 'SOCKET_CONNECT_TIMEOUT': 2 + } } } diff --git a/django_project/gap_api/mixins.py b/django_project/gap_api/mixins.py index 4062ba48..59f1d14e 100644 --- a/django_project/gap_api/mixins.py +++ b/django_project/gap_api/mixins.py @@ -24,7 +24,10 @@ def handle_log(self): if self.log.get('user', None): # replace with user id self.log['user'] = self.log['user'].id - cache._cache.get_client().rpush( - self.CACHE_KEY, - json.dumps(self.log, cls=DjangoJSONEncoder) - ) + try: + cache._cache.get_client().rpush( + self.CACHE_KEY, + json.dumps(self.log, cls=DjangoJSONEncoder) + ) + except Exception: # noqa + pass From db6408e77ccfac0b7e4e1a6da05a36938044c775 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Fri, 18 Oct 2024 19:19:16 +0100 Subject: [PATCH 05/14] add factory for api request log model --- django_project/gap_api/factories.py | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 django_project/gap_api/factories.py diff --git a/django_project/gap_api/factories.py b/django_project/gap_api/factories.py new file mode 100644 index 00000000..463f9dd1 --- /dev/null +++ b/django_project/gap_api/factories.py @@ -0,0 +1,36 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Factory classes for Models +""" +import factory +from factory.django import DjangoModelFactory + +from core.factories import UserF +from gap_api.models import APIRequestLog + + +class APIRequestLogFactory(DjangoModelFactory): + """Factory class for APIRequestLog model.""" + + class Meta: # noqa + model = APIRequestLog + + user = factory.SubFactory(UserF) + username_persistent = factory.Faker('person') + path = '/api/v1/measurement' + host = 'http://localhost' + method = 'GET' + query_params = { + 'lat': '-1.404244', + 'lon': '35.008688', + 'product': 'tahmo_ground_observation', + 'end_date': '2019-11-10', + 'attributes': [ + 'max_relative_humidity', + 'min_relative_humidity' + ], + 'start_date': '2019-11-01', + 'output_type': 'csv' + } From 83c71796a45e087419b5a93b1843d8dcb8280e3f Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 07:14:41 +0100 Subject: [PATCH 06/14] update redis client lib --- deployment/docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/docker/requirements.txt b/deployment/docker/requirements.txt index f9a4c9cc..37447f52 100644 --- a/deployment/docker/requirements.txt +++ b/deployment/docker/requirements.txt @@ -44,7 +44,7 @@ django-storages==1.14.2 django-storages[s3] # Python client for Redis database and key-value store -redis==4.3.4 +redis==5.1.1 psycopg2-binary==2.9.9 # drf yasg From 7d1ce220461da142cce614c9a95b739bf3dde436 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 07:14:52 +0100 Subject: [PATCH 07/14] fix socket_timeout config --- django_project/core/settings/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_project/core/settings/base.py b/django_project/core/settings/base.py index 155e9bf8..fd843149 100644 --- a/django_project/core/settings/base.py +++ b/django_project/core/settings/base.py @@ -144,8 +144,8 @@ f'@{os.environ.get("REDIS_HOST", "")}', ), 'OPTIONS': { - 'SOCKET_TIMEOUT': 2, - 'SOCKET_CONNECT_TIMEOUT': 2 + 'socket_timeout': 2, + 'socket_connect_timeout': 2 } } } From d5ca72bba037524c8aedd459f6c9d45d0cf456b1 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 07:15:11 +0100 Subject: [PATCH 08/14] remove unused code --- .../gap_api/apirequestlog/change_list.html | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html index d598d751..55b4e8d5 100644 --- a/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html +++ b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html @@ -64,55 +64,6 @@ }, }, }); - - function onDateChange(newStart, newEnd) { - var formattedStart = newStart.format('YYYY-MM-D'); - var formattedEnd = newEnd.format('YYYY-MM-D'); - var url = "/admin/gap_api/apirequestlog/chart_data/?start_date=" + formattedStart + "&end_date=" + formattedEnd; - - var params = window.location.search; - if (params && params.startsWith('?')) { - url = url + '&' + params.substring(1) - } - - const res = fetch(url) - .then(resp => resp.json()) - .then(resp => { - resp.forEach((d) => { - d.x = new Date(d.date); - }); - - chart.data.datasets[0].data = resp; - chart.update(); - }) - - formatDateRangeInput(newStart, newEnd) - } - - function formatDateRangeInput(newStart, newEnd) { - document.querySelector('#daterangepicker span').innerHTML = newStart.format('MMMM D, YYYY') + ' - ' + newEnd.format('MMMM D, YYYY'); - } - - var start = chartData.length ? moment(chartData[chartData.length - 1].x) : moment(); - var end = moment(); - - - // Disable daterange picker because it doesn't refresh the table - // $('#daterangepicker').daterangepicker({ - // startDate: start, - // endDate: end, - // ranges: { - // 'Today': [moment(), moment()], - // 'Yesterday': [moment().subtract(1, 'days'), moment().subtract(1, 'days')], - // 'Last 7 Days': [moment().subtract(6, 'days'), moment()], - // 'Last 30 Days': [moment().subtract(29, 'days'), moment()], - // 'This Month': [moment().startOf('month'), moment().endOf('month')], - // 'Last Month': [moment().subtract(1, 'month').startOf('month'), moment().subtract(1, 'month').endOf('month')], - // 'Lifetime': [chartData.length ? moment(chartData[chartData.length - 1].x) : moment(), moment()], - // } - // }, onDateChange); - - // formatDateRangeInput(start, end); }); @@ -121,12 +72,6 @@ {% endblock %} {% block content %} - -
From dc5869dfe1f5ecf44ee87f8836a49987b9377256 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 07:15:22 +0100 Subject: [PATCH 09/14] refactor and add tests --- django_project/gap_api/admin.py | 34 +--- django_project/gap_api/factories.py | 2 +- django_project/gap_api/mixins/__init__.py | 1 + .../gap_api/{mixins.py => mixins/logging.py} | 2 +- django_project/gap_api/models/__init__.py | 1 + .../{models.py => models/api_request_log.py} | 0 .../gap_api/tests/test_api_request_log.py | 160 ++++++++++++++++++ 7 files changed, 166 insertions(+), 34 deletions(-) create mode 100644 django_project/gap_api/mixins/__init__.py rename django_project/gap_api/{mixins.py => mixins/logging.py} (96%) create mode 100644 django_project/gap_api/models/__init__.py rename django_project/gap_api/{models.py => models/api_request_log.py} (100%) create mode 100644 django_project/gap_api/tests/test_api_request_log.py diff --git a/django_project/gap_api/admin.py b/django_project/gap_api/admin.py index 04307c4e..f945b931 100644 --- a/django_project/gap_api/admin.py +++ b/django_project/gap_api/admin.py @@ -5,9 +5,7 @@ .. note:: Admin for API Tracking """ -import datetime from django.contrib import admin -from django.http import JsonResponse from django.db.models import Count from django.db.models.functions import TruncDay from rest_framework_tracking.admin import APIRequestLogAdmin @@ -88,17 +86,6 @@ def changelist_view(self, request, extra_context=None): # Call the superclass changelist_view to render the page return super().changelist_view(request, extra_context=extra_context) - def chart_data_endpoint(self, request): - """Get response for chart data. - - :param request: request - :type request: Request object - :return: Chart data - :rtype: JsonResponse - """ - return JsonResponse( - list(self._generate_chart_data(request)), safe=False) - def _generate_chart_data(self, request): """Generate chart data and construct the filter from request object. @@ -107,18 +94,9 @@ def _generate_chart_data(self, request): :return: APIRequestLog group by Date and the count :rtype: list """ - start_date = request.GET.get("start_date", None) - end_date = request.GET.get("end_date", None) product_type = request.GET.get("product_type", None) user_id = request.GET.get("user__id__exact", None) - # convert start_date and end_date to datetime objects - if start_date: - start_date = datetime.datetime.strptime( - start_date, "%Y-%m-%d").date() - if end_date: - end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d").date() - # handle requested_at__day, requested_at__month, requested_at__year other_filters = {} for key, val in request.GET.items(): @@ -126,16 +104,12 @@ def _generate_chart_data(self, request): other_filters[key] = val return self._do_query_chart_data( - start_date, end_date, product_type, user_id, other_filters) + product_type, user_id, other_filters) def _do_query_chart_data( - self, start_date, end_date, product_type, user_id, other_filters): + self, product_type, user_id, other_filters): """Get chart data by filters. - :param start_date: start date filter - :type start_date: datetime - :param end_date: end date filter - :type end_date: datetime :param product_type: product type :type product_type: str :param user_id: user ID @@ -146,10 +120,6 @@ def _do_query_chart_data( :rtype: list """ filters = {} - if start_date and end_date: - filters['requested_at__date__gte'] = start_date - filters['requested_at__date__lte'] = end_date - if product_type: filters['query_params__product'] = product_type diff --git a/django_project/gap_api/factories.py b/django_project/gap_api/factories.py index 463f9dd1..8c48e229 100644 --- a/django_project/gap_api/factories.py +++ b/django_project/gap_api/factories.py @@ -18,7 +18,7 @@ class Meta: # noqa model = APIRequestLog user = factory.SubFactory(UserF) - username_persistent = factory.Faker('person') + username_persistent = factory.Faker('name') path = '/api/v1/measurement' host = 'http://localhost' method = 'GET' diff --git a/django_project/gap_api/mixins/__init__.py b/django_project/gap_api/mixins/__init__.py new file mode 100644 index 00000000..0d0c85f4 --- /dev/null +++ b/django_project/gap_api/mixins/__init__.py @@ -0,0 +1 @@ +from .logging import * # noqa diff --git a/django_project/gap_api/mixins.py b/django_project/gap_api/mixins/logging.py similarity index 96% rename from django_project/gap_api/mixins.py rename to django_project/gap_api/mixins/logging.py index 59f1d14e..dbbe66c0 100644 --- a/django_project/gap_api/mixins.py +++ b/django_project/gap_api/mixins/logging.py @@ -2,7 +2,7 @@ """ Tomorrow Now GAP API. -.. note:: Mixins for API Tracking +.. note:: Mixin for API Tracking """ import json diff --git a/django_project/gap_api/models/__init__.py b/django_project/gap_api/models/__init__.py new file mode 100644 index 00000000..7ba41d23 --- /dev/null +++ b/django_project/gap_api/models/__init__.py @@ -0,0 +1 @@ +from .api_request_log import * # noqa diff --git a/django_project/gap_api/models.py b/django_project/gap_api/models/api_request_log.py similarity index 100% rename from django_project/gap_api/models.py rename to django_project/gap_api/models/api_request_log.py diff --git a/django_project/gap_api/tests/test_api_request_log.py b/django_project/gap_api/tests/test_api_request_log.py new file mode 100644 index 00000000..8d98a54c --- /dev/null +++ b/django_project/gap_api/tests/test_api_request_log.py @@ -0,0 +1,160 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Unit tests for API Request Log. +""" + +import json +import mock +import datetime +from django.test import TestCase, RequestFactory +from django.contrib.admin import ModelAdmin +from django.core.cache import cache + +from core.factories import UserF +from gap.models import DatasetType +from gap_api.models import APIRequestLog +from gap_api.mixins import GAPAPILoggingMixin +from gap_api.tasks import store_api_logs +from gap_api.admin import ProductTypeFilter, GapAPIRequestLogAdmin +from gap_api.factories import APIRequestLogFactory + + +class MockRequestObj(object): + """Mock GET request object.""" + + GET = {} + + +class TestAPIRequestLog(TestCase): + """Test class for APIRequestLog model and task.""" + + fixtures = [ + '2.provider.json', + '3.station_type.json', + '4.dataset_type.json' + ] + + def setUp(self): + """Initialize test class.""" + self.factory = RequestFactory() + self.user = UserF.create() + self.admin_instance = GapAPIRequestLogAdmin( + APIRequestLog, admin_site=mock.MagicMock()) + + def test_store_api_logs(self): + """Test store api logs from cache.""" + cache._cache.get_client().rpush( + GAPAPILoggingMixin.CACHE_KEY, + json.dumps({ + "requested_at": "2024-10-17 17:56:52.270889+00:00", + "data": None, + "remote_addr": "127.0.0.1", + "view": "gap_api.api_views.measurement.MeasurementAPI", + "view_method": "get", + "path": "/api/v1/measurement/", + "host": "localhost:8000", + "user_agent": "PostmanRuntime/7.42.0", + "method": "GET", + "query_params": { + "lat": "-1.404244", + "lon": "35.008688", + "attributes": ( + "max_relative_humidity,min_relative_humidity" + ), + "start_date": "2019-11-01", + "end_date": "2019-11-01", + "product": "tahmo_ground_observation", + "output_type": "csv" + }, + "user": f"{self.user.id}", + "username_persistent": "admin", + "response_ms": 182, + "response": None, + "status_code": 200 + }) + ) + store_api_logs() + logs = APIRequestLog.objects.all() + self.assertEqual(logs.count(), 1) + self.assertEqual(logs.first().user, self.user) + + def test_product_type_filter(self): + """Test ProductTypeFilter class.""" + f = ProductTypeFilter(None, {}, APIRequestLog, self.admin_instance) + names = f.lookups(None, self.admin_instance) + self.assertEqual( + len(names), + DatasetType.objects.exclude( + variable_name='default' + ).count() + ) + + def test_admin_product_type(self): + """Test product_type field in Admin class.""" + f = ProductTypeFilter(None, {}, APIRequestLog, self.admin_instance) + f.used_parameters = {} + + # create two logs + log1 = APIRequestLogFactory.create( + query_params={ + 'product': 'cbam_reanalysis_data' + } + ) + APIRequestLogFactory.create() + + qs = APIRequestLog.objects.all() + + # query with empty param should return all + res = f.queryset(None, qs) + self.assertEqual(res.count(), 2) + + # query with product_type cbam should return 1 + f.used_parameters = { + 'product_type': 'cbam_reanalysis_data' + } + res = f.queryset(None, qs) + self.assertEqual(res.count(), 1) + self.assertEqual(res.first().id, log1.id) + + @mock.patch.object(GapAPIRequestLogAdmin, '_generate_chart_data') + @mock.patch.object(ModelAdmin, 'changelist_view') + def test_change_list_view( + self, mock_super_changelist_view, mock_generate_chart_data): + """Test change_list view.""" + mock_chart_data = [{"date": "2024-10-18", "count": 10}] + mock_generate_chart_data.return_value = mock_chart_data + + # Simulate a GET request to the changelist view + request = self.factory.get('/admin/gap_api/apirequestlog/') + + # Call the changelist_view method + self.admin_instance.changelist_view(request) + + # Assert that _generate_chart_data was called + mock_generate_chart_data.assert_called_once_with(request) + + # Assert that the extra_context includes the correct chart data + expected_extra_context = {"chart_data": list(mock_chart_data)} + mock_super_changelist_view.assert_called_once_with( + request, extra_context=expected_extra_context) + + def test_generate_chart_data(self): + """Test generate chart data.""" + APIRequestLogFactory.create( + requested_at=datetime.datetime(2024, 10, 1, 0, 0, 0), + user=self.user + ) + APIRequestLogFactory.create( + requested_at=datetime.datetime(2023, 5, 1, 0, 0, 0), + user=self.user + ) + req = MockRequestObj() + req.GET = { + 'product_type': 'tahmo_ground_observation', + 'requested_at__year': '2023', + 'user__id__exact': f'{self.user.id}' + } + data = self.admin_instance._generate_chart_data(req) + self.assertEqual(len(data), 1) From 4db2faf5563236170229995cc7a1d174723161b0 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 07:21:09 +0100 Subject: [PATCH 10/14] fix migration --- ...g_batch_size.py => 0032_preferences_api_log_batch_size.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename django_project/gap/migrations/{0031_preferences_api_log_batch_size.py => 0032_preferences_api_log_batch_size.py} (78%) diff --git a/django_project/gap/migrations/0031_preferences_api_log_batch_size.py b/django_project/gap/migrations/0032_preferences_api_log_batch_size.py similarity index 78% rename from django_project/gap/migrations/0031_preferences_api_log_batch_size.py rename to django_project/gap/migrations/0032_preferences_api_log_batch_size.py index 7cf4bbec..f3b12c5e 100644 --- a/django_project/gap/migrations/0031_preferences_api_log_batch_size.py +++ b/django_project/gap/migrations/0032_preferences_api_log_batch_size.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2024-10-17 19:34 +# Generated by Django 4.2.7 on 2024-10-19 06:19 from django.db import migrations, models @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('gap', '0030_preferences_ingestor_config'), + ('gap', '0031_measurement_station_history'), ] operations = [ From a680ef51888798209a375b8f695f30da3e56d9a9 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 08:26:54 +0100 Subject: [PATCH 11/14] add pie chart for product type --- django_project/gap_api/admin.py | 53 +++++++++++--- .../gap_api/apirequestlog/change_list.html | 72 +++++++++++++++---- .../gap_api/tests/test_api_request_log.py | 26 +++---- 3 files changed, 116 insertions(+), 35 deletions(-) diff --git a/django_project/gap_api/admin.py b/django_project/gap_api/admin.py index f945b931..7f780c14 100644 --- a/django_project/gap_api/admin.py +++ b/django_project/gap_api/admin.py @@ -5,9 +5,11 @@ .. note:: Admin for API Tracking """ +import random from django.contrib import admin -from django.db.models import Count -from django.db.models.functions import TruncDay +from django.db.models import Count, TextField +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import TruncDay, Cast from rest_framework_tracking.admin import APIRequestLogAdmin from rest_framework_tracking.models import APIRequestLog as BaseAPIRequestLog @@ -18,6 +20,11 @@ admin.site.unregister(BaseAPIRequestLog) +def generate_random_color(): + """Generate random color for product type.""" + return "#{:06x}".format(random.randint(0, 0xFFFFFF)) + + class ProductTypeFilter(admin.SimpleListFilter): """Custom filter for product type field.""" @@ -81,7 +88,16 @@ def changelist_view(self, request, extra_context=None): # Aggregate api logs per day chart_data = self._generate_chart_data(request) - extra_context = extra_context or {"chart_data": list(chart_data)} + # generate color for products + product_counts = [] + for product in chart_data['product']: + product['color'] = generate_random_color() + product_counts.append(product) + + extra_context = extra_context or { + "chart_data": list(chart_data['total_requests']), + "product_chart_data": product_counts + } # Call the superclass changelist_view to render the page return super().changelist_view(request, extra_context=extra_context) @@ -127,15 +143,30 @@ def _do_query_chart_data( filters['user__id'] = user_id filters.update(other_filters) - return ( - APIRequestLog.objects.filter( - **filters + return { + 'total_requests': ( + APIRequestLog.objects.filter( + **filters + ) + .annotate(date=TruncDay("requested_at")) + .values("date") + .annotate(y=Count("id")) + .order_by("-date") + ), + 'product': ( + APIRequestLog.objects.filter( + **filters + ).annotate( + product=Cast( + KeyTextTransform('product', 'query_params'), + TextField() + ) + ) + .values('product') + .annotate(count=Count("id")) + .order_by('product') ) - .annotate(date=TruncDay("requested_at")) - .values("date") - .annotate(y=Count("id")) - .order_by("-date") - ) + } admin.site.register(APIRequestLog, GapAPIRequestLogAdmin) diff --git a/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html index 55b4e8d5..475ca937 100644 --- a/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html +++ b/django_project/gap_api/templates/admin/gap_api/apirequestlog/change_list.html @@ -5,25 +5,36 @@ {% block extrahead %} {{ block.super }} - - {{ chart_data|json_script:"chartData" }} +{{ product_chart_data|json_script:"productChartData" }} {% endblock %} {% block content %} -
- +
+ +
+ +
+ +
+ +
{{ block.super }} diff --git a/django_project/gap_api/tests/test_api_request_log.py b/django_project/gap_api/tests/test_api_request_log.py index 8d98a54c..41153ea6 100644 --- a/django_project/gap_api/tests/test_api_request_log.py +++ b/django_project/gap_api/tests/test_api_request_log.py @@ -10,12 +10,10 @@ import datetime from django.test import TestCase, RequestFactory from django.contrib.admin import ModelAdmin -from django.core.cache import cache from core.factories import UserF from gap.models import DatasetType from gap_api.models import APIRequestLog -from gap_api.mixins import GAPAPILoggingMixin from gap_api.tasks import store_api_logs from gap_api.admin import ProductTypeFilter, GapAPIRequestLogAdmin from gap_api.factories import APIRequestLogFactory @@ -43,10 +41,11 @@ def setUp(self): self.admin_instance = GapAPIRequestLogAdmin( APIRequestLog, admin_site=mock.MagicMock()) - def test_store_api_logs(self): + @mock.patch('django.core.cache.cache._cache.get_client') + def test_store_api_logs(self, mock_get_client): """Test store api logs from cache.""" - cache._cache.get_client().rpush( - GAPAPILoggingMixin.CACHE_KEY, + mock_redis_client = mock_get_client.return_value + mock_redis_client.lpop.side_effect = [ json.dumps({ "requested_at": "2024-10-17 17:56:52.270889+00:00", "data": None, @@ -74,8 +73,10 @@ def test_store_api_logs(self): "response": None, "status_code": 200 }) - ) + ] + [None] * 499 store_api_logs() + # default batch size + self.assertEqual(mock_redis_client.lpop.call_count, 500) logs = APIRequestLog.objects.all() self.assertEqual(logs.count(), 1) self.assertEqual(logs.first().user, self.user) @@ -123,7 +124,10 @@ def test_admin_product_type(self): def test_change_list_view( self, mock_super_changelist_view, mock_generate_chart_data): """Test change_list view.""" - mock_chart_data = [{"date": "2024-10-18", "count": 10}] + mock_chart_data = { + 'total_requests': [{"date": "2024-10-18", "count": 10}], + 'product': [{"product": "product a", "count": 10}] + } mock_generate_chart_data.return_value = mock_chart_data # Simulate a GET request to the changelist view @@ -135,10 +139,7 @@ def test_change_list_view( # Assert that _generate_chart_data was called mock_generate_chart_data.assert_called_once_with(request) - # Assert that the extra_context includes the correct chart data - expected_extra_context = {"chart_data": list(mock_chart_data)} - mock_super_changelist_view.assert_called_once_with( - request, extra_context=expected_extra_context) + mock_super_changelist_view.assert_called_once() def test_generate_chart_data(self): """Test generate chart data.""" @@ -157,4 +158,5 @@ def test_generate_chart_data(self): 'user__id__exact': f'{self.user.id}' } data = self.admin_instance._generate_chart_data(req) - self.assertEqual(len(data), 1) + self.assertEqual(len(data['total_requests']), 1) + self.assertEqual(len(data['product']), 1) From 91adcd71f0938be52f0f62ce2dd21c46dd49681d Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 08:33:55 +0100 Subject: [PATCH 12/14] exclude from background task --- django_project/core/celery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_project/core/celery.py b/django_project/core/celery.py index 3aab32c1..16112fb4 100644 --- a/django_project/core/celery.py +++ b/django_project/core/celery.py @@ -159,7 +159,8 @@ def update_task_progress( EXCLUDED_TASK_LIST = [ - 'celery.backend_cleanup' + 'celery.backend_cleanup', + 'store_api_logs' ] From 1e97266996e70c0451a5c461b2c1f19efffe1d9b Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 08:34:02 +0100 Subject: [PATCH 13/14] bump app version --- django_project/_version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_project/_version.txt b/django_project/_version.txt index fa3de586..99d85ecd 100644 --- a/django_project/_version.txt +++ b/django_project/_version.txt @@ -1 +1 @@ -0.0.5 \ No newline at end of file +0.0.6 \ No newline at end of file From d555202efdcad282f88593641c9d373ccb21c5df Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Sat, 19 Oct 2024 08:47:14 +0100 Subject: [PATCH 14/14] fix tests --- django_project/gap_api/mixins/__init__.py | 2 +- django_project/gap_api/models/__init__.py | 2 +- django_project/gap_api/tests/test_api_request_log.py | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/django_project/gap_api/mixins/__init__.py b/django_project/gap_api/mixins/__init__.py index 0d0c85f4..5fb58bae 100644 --- a/django_project/gap_api/mixins/__init__.py +++ b/django_project/gap_api/mixins/__init__.py @@ -1 +1 @@ -from .logging import * # noqa +from gap_api.mixins.logging import * # noqa diff --git a/django_project/gap_api/models/__init__.py b/django_project/gap_api/models/__init__.py index 7ba41d23..04c9f084 100644 --- a/django_project/gap_api/models/__init__.py +++ b/django_project/gap_api/models/__init__.py @@ -1 +1 @@ -from .api_request_log import * # noqa +from gap_api.models.api_request_log import * # noqa diff --git a/django_project/gap_api/tests/test_api_request_log.py b/django_project/gap_api/tests/test_api_request_log.py index 41153ea6..298a681b 100644 --- a/django_project/gap_api/tests/test_api_request_log.py +++ b/django_project/gap_api/tests/test_api_request_log.py @@ -8,7 +8,7 @@ import json import mock import datetime -from django.test import TestCase, RequestFactory +from django.test import TestCase, RequestFactory, override_settings from django.contrib.admin import ModelAdmin from core.factories import UserF @@ -25,6 +25,13 @@ class MockRequestObj(object): GET = {} +@override_settings( + CACHES={ + 'default': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + } + } +) class TestAPIRequestLog(TestCase): """Test class for APIRequestLog model and task."""