diff --git a/django_project/core/admin.py b/django_project/core/admin.py index 0b9292f..31ad517 100644 --- a/django_project/core/admin.py +++ b/django_project/core/admin.py @@ -20,6 +20,11 @@ class SitePreferencesAdmin(admin.ModelAdmin): 'output_group_to_keep', ) }), + ('Remove Layers Job', { + 'fields': ( + 'layer_days_to_keep', + ) + }), ) diff --git a/django_project/core/migrations/0005_sitepreferences_layer_days_to_keep.py b/django_project/core/migrations/0005_sitepreferences_layer_days_to_keep.py new file mode 100644 index 0000000..5170f8f --- /dev/null +++ b/django_project/core/migrations/0005_sitepreferences_layer_days_to_keep.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-05-31 07:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0004_sitepreferences_output_group_to_keep'), + ] + + operations = [ + migrations.AddField( + model_name='sitepreferences', + name='layer_days_to_keep', + field=models.IntegerField(default=14, help_text='Keep input/output layers until X days.'), + ), + ] diff --git a/django_project/core/models/preferences.py b/django_project/core/models/preferences.py index 91a65e2..f445997 100644 --- a/django_project/core/models/preferences.py +++ b/django_project/core/models/preferences.py @@ -47,6 +47,10 @@ class SitePreferences(SingletonModel): blank=True, help_text='Output group to keep from automatic removal.' ) + layer_days_to_keep = models.IntegerField( + default=14, + help_text='Keep input/output layers until X days.' + ) class Meta: # noqa: D106 verbose_name_plural = "site preferences" diff --git a/django_project/cplus_api/api_views/output.py b/django_project/cplus_api/api_views/output.py index 5e274ef..5ec3535 100644 --- a/django_project/cplus_api/api_views/output.py +++ b/django_project/cplus_api/api_views/output.py @@ -28,13 +28,6 @@ class UserScenarioAnalysisOutput(BaseScenarioReadAccess, APIView): """API to fetch output list of ScenarioAnalysis""" permission_classes = [IsAuthenticated] - param_all_outputs = openapi.Parameter( - 'download_all', openapi.IN_QUERY, - description='Whether to generate download URL for all outputs', - type=openapi.TYPE_BOOLEAN, - default=False, - required=False - ) param_group = openapi.Parameter( 'group', openapi.IN_QUERY, description='Filter the outputs by group', @@ -47,7 +40,6 @@ class UserScenarioAnalysisOutput(BaseScenarioReadAccess, APIView): tags=[SCENARIO_OUTPUT_API_TAG], manual_parameters=[ PARAM_SCENARIO_UUID_IN_PATH, - param_all_outputs, param_group ] + PARAMS_PAGINATION, responses={ @@ -60,9 +52,6 @@ class UserScenarioAnalysisOutput(BaseScenarioReadAccess, APIView): def get(self, request, *args, **kwargs): page = int(request.GET.get('page', '1')) page_size = get_page_size(request) - is_fetch_all = request.GET.get('download_all', None) - if is_fetch_all is not None: - is_fetch_all = is_fetch_all.lower() == 'true' group_filter = request.GET.get('group', None) scenario_uuid = kwargs.get('scenario_uuid') scenario_task = get_object_or_404( @@ -83,10 +72,7 @@ def get(self, request, *args, **kwargs): output = ( OutputLayerSerializer( paginated_entities, - many=True, - context={ - 'is_fetch_all': is_fetch_all - } + many=True ).data ) return Response(status=200, data={ @@ -132,9 +118,6 @@ def post(self, request, *args, **kwargs): ).order_by('id') return Response(status=200, data=( OutputLayerSerializer( - layers, many=True, - context={ - 'is_fetch_all': True - } + layers, many=True ).data )) diff --git a/django_project/cplus_api/serializers/layer.py b/django_project/cplus_api/serializers/layer.py index 12281ae..c1f2d3a 100644 --- a/django_project/cplus_api/serializers/layer.py +++ b/django_project/cplus_api/serializers/layer.py @@ -341,28 +341,14 @@ class Meta: class OutputLayerSerializer(serializers.ModelSerializer): - DEFAULT_GROUP_IN_OUTPUT_URL = ['weighted_activities'] filename = serializers.CharField(source='name') created_by = serializers.SerializerMethodField() url = serializers.SerializerMethodField() - def check_generate_all_outputs(self, obj: OutputLayer): - is_fetch_all = self.context.get('is_fetch_all', False) - if is_fetch_all: - return True - if ( - not obj.is_final_output and - obj.group not in self.DEFAULT_GROUP_IN_OUTPUT_URL - ): - return False - return True - def get_created_by(self, obj: OutputLayer): return obj.owner.email def get_url(self, obj: OutputLayer): - if not self.check_generate_all_outputs(obj): - return None if not obj.file.name: return None if not obj.file.storage.exists(obj.file.name): diff --git a/django_project/cplus_api/tasks/remove_layers.py b/django_project/cplus_api/tasks/remove_layers.py index 403de50..a02c5ef 100644 --- a/django_project/cplus_api/tasks/remove_layers.py +++ b/django_project/cplus_api/tasks/remove_layers.py @@ -27,10 +27,18 @@ def remove_layers(): } # Remove private Input Layer that is more 2 weeks - last_14_days_datetime = timezone.now() - timedelta(days=14) + last_x_days_datetime = ( + timezone.now() - + timedelta(days=SitePreferences.preferences().layer_days_to_keep) + ) input_layers = InputLayer.objects.filter( privacy_type=InputLayer.PrivacyTypes.PRIVATE, - created_on__lt=last_14_days_datetime + ).filter( + ( + Q(created_on__lt=last_x_days_datetime) & + Q(last_used_on__isnull=True) + ) | + Q(last_used_on__lt=last_x_days_datetime) ) results[InputLayer] = input_layers.count() input_layers.delete() @@ -38,7 +46,7 @@ def remove_layers(): # Remove private data that is more 2 weeks output_group_to_keep = SitePreferences.preferences().output_group_to_keep output_layers = OutputLayer.objects.filter( - created_on__lt=last_14_days_datetime + created_on__lt=last_x_days_datetime ).exclude( Q(is_final_output=True) | Q(group__in=output_group_to_keep) ) diff --git a/django_project/cplus_api/tests/test_output_api_view.py b/django_project/cplus_api/tests/test_output_api_view.py index 3769278..599996d 100644 --- a/django_project/cplus_api/tests/test_output_api_view.py +++ b/django_project/cplus_api/tests/test_output_api_view.py @@ -100,29 +100,7 @@ def test_fetch_output_list(self): response = view(request, **kwargs) self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data['results']), 5) - # 2 items should have url, 3 items should not have - filtered_layers = self.filter_layers_having_urls( - response.data['results']) - self.assertEqual(len(filtered_layers), 2) - find_layer = self.find_layer_from_response( - filtered_layers, output_layer_1.uuid) - self.assertTrue(find_layer) - find_layer = self.find_layer_from_response( - filtered_layers, output_layer_2.uuid) - self.assertTrue(find_layer) - # test with download_all, should return 5 items # 3 items should have url - request = self.factory.get( - reverse( - 'v1:scenario-output-list', - kwargs=kwargs - ) + '?download_all=true' - ) - request.resolver_match = FakeResolverMatchV1 - request.user = self.superuser - response = view(request, **kwargs) - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data['results']), 5) filtered_layers = self.filter_layers_having_urls( response.data['results']) self.assertEqual(len(filtered_layers), 3) @@ -140,7 +118,7 @@ def test_fetch_output_list(self): reverse( 'v1:scenario-output-list', kwargs=kwargs - ) + '?download_all=true&group=activities' + ) + '?group=activities' ) request.resolver_match = FakeResolverMatchV1 request.user = self.superuser