Merge pull request #40 from kartoza/fix-output-layer-api

Fix output layer api

django_project/core/admin.py

@@ -20,6 +20,11 @@ class SitePreferencesAdmin(admin.ModelAdmin):
                 'output_group_to_keep',
             )
         }),
+        ('Remove Layers Job', {
+            'fields': (
+                'layer_days_to_keep',
+            )
+        }),
     )
 
 

django_project/core/migrations/0005_sitepreferences_layer_days_to_keep.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-05-31 07:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0004_sitepreferences_output_group_to_keep'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sitepreferences',
+            name='layer_days_to_keep',
+            field=models.IntegerField(default=14, help_text='Keep input/output layers until X days.'),
+        ),
+    ]

django_project/core/models/preferences.py

@@ -47,6 +47,10 @@ class SitePreferences(SingletonModel):
         blank=True,
         help_text='Output group to keep from automatic removal.'
     )
+    layer_days_to_keep = models.IntegerField(
+        default=14,
+        help_text='Keep input/output layers until X days.'
+    )
 
     class Meta:  # noqa: D106
         verbose_name_plural = "site preferences"

django_project/cplus_api/api_views/output.py

@@ -28,13 +28,6 @@
 class UserScenarioAnalysisOutput(BaseScenarioReadAccess, APIView):
     """API to fetch output list of ScenarioAnalysis"""
     permission_classes = [IsAuthenticated]
-    param_all_outputs = openapi.Parameter(
-        'download_all', openapi.IN_QUERY,
-        description='Whether to generate download URL for all outputs',
-        type=openapi.TYPE_BOOLEAN,
-        default=False,
-        required=False
-    )
     param_group = openapi.Parameter(
         'group', openapi.IN_QUERY,
         description='Filter the outputs by group',
@@ -47,7 +40,6 @@ class UserScenarioAnalysisOutput(BaseScenarioReadAccess, APIView):
         tags=[SCENARIO_OUTPUT_API_TAG],
         manual_parameters=[
             PARAM_SCENARIO_UUID_IN_PATH,
-            param_all_outputs,
             param_group
         ] + PARAMS_PAGINATION,
         responses={
@@ -60,9 +52,6 @@ class UserScenarioAnalysisOutput(BaseScenarioReadAccess, APIView):
     def get(self, request, *args, **kwargs):
         page = int(request.GET.get('page', '1'))
         page_size = get_page_size(request)
-        is_fetch_all = request.GET.get('download_all', None)
-        if is_fetch_all is not None:
-            is_fetch_all = is_fetch_all.lower() == 'true'
         group_filter = request.GET.get('group', None)
         scenario_uuid = kwargs.get('scenario_uuid')
         scenario_task = get_object_or_404(
@@ -83,10 +72,7 @@ def get(self, request, *args, **kwargs):
             output = (
                 OutputLayerSerializer(
                     paginated_entities,
-                    many=True,
-                    context={
-                        'is_fetch_all': is_fetch_all
-                    }
+                    many=True
                 ).data
             )
         return Response(status=200, data={
@@ -132,9 +118,6 @@ def post(self, request, *args, **kwargs):
         ).order_by('id')
         return Response(status=200, data=(
             OutputLayerSerializer(
-                layers, many=True,
-                context={
-                    'is_fetch_all': True
-                }
+                layers, many=True
             ).data
         ))

django_project/cplus_api/serializers/layer.py

@@ -341,28 +341,14 @@ class Meta:
 
 
 class OutputLayerSerializer(serializers.ModelSerializer):
-    DEFAULT_GROUP_IN_OUTPUT_URL = ['weighted_activities']
     filename = serializers.CharField(source='name')
     created_by = serializers.SerializerMethodField()
     url = serializers.SerializerMethodField()
 
-    def check_generate_all_outputs(self, obj: OutputLayer):
-        is_fetch_all = self.context.get('is_fetch_all', False)
-        if is_fetch_all:
-            return True
-        if (
-            not obj.is_final_output and
-            obj.group not in self.DEFAULT_GROUP_IN_OUTPUT_URL
-        ):
-            return False
-        return True
-
     def get_created_by(self, obj: OutputLayer):
         return obj.owner.email
 
     def get_url(self, obj: OutputLayer):
-        if not self.check_generate_all_outputs(obj):
-            return None
         if not obj.file.name:
             return None
         if not obj.file.storage.exists(obj.file.name):

django_project/cplus_api/tasks/remove_layers.py

@@ -27,18 +27,26 @@ def remove_layers():
     }
 
     # Remove private Input Layer that is more 2 weeks
-    last_14_days_datetime = timezone.now() - timedelta(days=14)
+    last_x_days_datetime = (
+        timezone.now() -
+        timedelta(days=SitePreferences.preferences().layer_days_to_keep)
+    )
     input_layers = InputLayer.objects.filter(
         privacy_type=InputLayer.PrivacyTypes.PRIVATE,
-        created_on__lt=last_14_days_datetime
+    ).filter(
+        (
+            Q(created_on__lt=last_x_days_datetime) &
+            Q(last_used_on__isnull=True)
+        ) |
+        Q(last_used_on__lt=last_x_days_datetime)
     )
     results[InputLayer] = input_layers.count()
     input_layers.delete()
 
     # Remove private data that is more 2 weeks
     output_group_to_keep = SitePreferences.preferences().output_group_to_keep
     output_layers = OutputLayer.objects.filter(
-        created_on__lt=last_14_days_datetime
+        created_on__lt=last_x_days_datetime
     ).exclude(
         Q(is_final_output=True) | Q(group__in=output_group_to_keep)
     )

django_project/cplus_api/tests/test_output_api_view.py

@@ -100,29 +100,7 @@ def test_fetch_output_list(self):
         response = view(request, **kwargs)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(response.data['results']), 5)
-        # 2 items should have url, 3 items should not have
-        filtered_layers = self.filter_layers_having_urls(
-            response.data['results'])
-        self.assertEqual(len(filtered_layers), 2)
-        find_layer = self.find_layer_from_response(
-            filtered_layers, output_layer_1.uuid)
-        self.assertTrue(find_layer)
-        find_layer = self.find_layer_from_response(
-            filtered_layers, output_layer_2.uuid)
-        self.assertTrue(find_layer)
-        # test with download_all, should return 5 items
         # 3 items should have url
-        request = self.factory.get(
-            reverse(
-                'v1:scenario-output-list',
-                kwargs=kwargs
-            ) + '?download_all=true'
-        )
-        request.resolver_match = FakeResolverMatchV1
-        request.user = self.superuser
-        response = view(request, **kwargs)
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.data['results']), 5)
         filtered_layers = self.filter_layers_having_urls(
             response.data['results'])
         self.assertEqual(len(filtered_layers), 3)
@@ -140,7 +118,7 @@ def test_fetch_output_list(self):
             reverse(
                 'v1:scenario-output-list',
                 kwargs=kwargs
-            ) + '?download_all=true&group=activities'
+            ) + '?group=activities'
         )
         request.resolver_match = FakeResolverMatchV1
         request.user = self.superuser