Merge branch 'develop' of github.com:unicef/etools into ch33354-UNICEF-cash-unfunded

# Conflicts:
#	src/etools/applications/partners/locale/ar/LC_MESSAGES/django.mo
#	src/etools/applications/partners/locale/ar/LC_MESSAGES/django.po
#	src/etools/applications/partners/locale/es/LC_MESSAGES/django.mo
#	src/etools/applications/partners/locale/es/LC_MESSAGES/django.po
#	src/etools/applications/partners/locale/fr/LC_MESSAGES/django.mo
#	src/etools/applications/partners/locale/fr/LC_MESSAGES/django.po
#	src/etools/applications/partners/locale/pt/LC_MESSAGES/django.mo
#	src/etools/applications/partners/locale/pt/LC_MESSAGES/django.po
#	src/etools/applications/partners/locale/ru/LC_MESSAGES/django.mo
#	src/etools/applications/partners/locale/ru/LC_MESSAGES/django.po
#	src/etools/applications/partners/serializers/interventions_v3.py
#	src/etools/applications/reports/serializers/v2.py

Author: emaciupe

.flake8

@@ -6,7 +6,9 @@ ignore =
     F405,
     ; E501 maximum line length
     E501,
-    W504
+    W504,
+    ; E741 ambiguous variable name
+    E741
 
 exclude =
     */migrations,

Dockerfile-base

@@ -31,7 +31,7 @@ RUN apk add --no-cache --virtual .build-deps --update \
     gettext
 
 RUN pip install --no-cache-dir --upgrade \
-    pip \
+    pip==23.1.2 \
     pipenv==2022.10.4
 
 

Pipfile

@@ -22,16 +22,16 @@ azure-common = "==1.1.27"
 azure-storage-blob = "==2.1.0"
 azure-storage-common = "==2.1.0"
 carto = "==1.11.2"
-celery = "==5.1.2"
+celery = "==5.2.3"
 cryptography = "<3.4"
 dj-database-url = "==0.5"
 dj-static = "==0.0.6"
-Django = "==3.2.6"
+Django = "==3.2.19"
 django-admin-extra-urls = "*"
 django-appconf = "==1.0.4"
 django-celery-beat = "==2.2.1"
 django-celery-email = "==3.0.0"
-django-celery-results = "==2.2"
+django-celery-results = "==2.4.0"
 django-contrib-comments = "==2.1.0"
 django-cors-headers = "==3.7.0"
 django-debug-toolbar = "==3.2.1"
@@ -61,11 +61,11 @@ djangorestframework = "==3.12.4"
 drf-nested-routers = "==0.93.3"
 drf-querystringfilter = "==1.0.0"
 etools-validator = "==0.5.0"
-flower = "==0.9.5"  # issue when locking
+flower = "==1.2.0"  # issue when locking
 GDAL = "==3.0.2"
 gunicorn = "<20.0"
 newrelic = "*"
-Pillow = "==8.1.0"
+Pillow = "==9.3.0"
 psycopg2-binary = "==2.9.1"
 sentry-sdk = "*"
 requests = "==2.26"
@@ -83,8 +83,8 @@ xhtml2pdf = "==0.2.5"
 unicef-vision = "==0.6"
 etools-offline = "==0.1.0"
 openpyxl = "==3.0.5"
-pyyaml = "==5.4.1"
+pyyaml = "==5.3.1"
 reportlab = "==3.6.6"  # freeze: getStringIO from 'reportlab.lib.utils'
 
 [requires]
-python_version = "3.9"
+python_version = "3.9"

Pipfile.lock

@@ -1,2 +1,2 @@
-VERSION = __version__ = '10.1.1'
+VERSION = __version__ = '11.1.6'
 NAME = 'eTools'

src/etools/__init__.py

@@ -1,3 +1,4 @@
+from django.contrib.auth.models import AnonymousUser
 from django.urls import resolve, reverse
 
 from rest_framework import status
@@ -125,8 +126,9 @@ def test_unauthenticated_user_forbidden(self):
         factory = APIRequestFactory()
         view_info = resolve(self.url)
         request = factory.get(self.url)
+        request.user = AnonymousUser()
         response = view_info.func(request)
-        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+        self.assertEqual(response.status_code, status.HTTP_302_FOUND)
 
     def test_non_schema_user(self):
         user = UserFactory(profile=None, realms__data=[])

src/etools/applications/attachments/tests/test_views.py

@@ -29,7 +29,7 @@ class EngagementAdmin(admin.ModelAdmin):
     list_filter = [
         'status', 'start_date', 'end_date', 'status', 'engagement_type',
     ]
-    search_fields = 'partner__organization__name', 'agreement__auditor_firm__name',
+    search_fields = 'partner__organization__name', 'agreement__auditor_firm__organization__name',
     filter_horizontal = ('authorized_officers', 'active_pd', 'staff_members', 'users_notified', 'sections', 'offices')
     raw_id_fields = ('po_item', 'partner', 'active_pd', 'staff_members', 'authorized_officers', 'users_notified', )
 

src/etools/applications/audit/admin.py

@@ -1,11 +1,11 @@
-from django.contrib.auth import get_user_model
 from django.db import models
 from django.db.models.functions import TruncYear
 
 from django_filters import rest_framework as filters
 from rest_framework.filters import BaseFilterBackend, OrderingFilter
 
 from etools.applications.audit.models import Engagement
+from etools.applications.users.models import Country
 
 
 class DisplayStatusFilter(BaseFilterBackend):
@@ -87,12 +87,20 @@ class Meta:
         }
 
 
-class AuditorStaffMembersFilterSet(filters.FilterSet):
-    user__profile__countries_available__name = filters.CharFilter(field_name='realms__country__name', distinct=True)
+class StaffMembersCountriesAvailableFilter(BaseFilterBackend):
+    """
+    manual filtering on countries available without extra join
+    """
+    def filter_queryset(self, request, queryset, view):
+        country_name = request.query_params.get('user__profile__countries_available__name', '')
+        if not country_name:
+            return queryset
 
-    class Meta:
-        model = get_user_model()
-        fields = {}
+        country = Country.objects.filter(name=country_name).only('id').first()
+        if not country:
+            return queryset.none()
+
+        return queryset.filter(realms__country=country.id).distinct()
 
 
 class StaffMembersOrderingFilter(OrderingFilter):

src/etools/applications/audit/filters.py

@@ -349,20 +349,35 @@ def assign_permissions(self):
             'audit.engagement.cancel',
             condition=partner_contacted_condition
         )
+        self.add_permissions(
+            self.everybody, 'view',
+            'audit.engagement.send_back_comment',
+            condition=partner_contacted_condition
+        )
 
-        # report submitted. focal point can finalize. all can view
+        # report submitted. focal point can finalize or send back. all can view
         report_submitted_condition = self.engagement_status(Engagement.STATUSES.report_submitted)
         self.add_permissions(self.auditor, 'edit', self.report_attachments_block, condition=report_submitted_condition)
         self.add_permissions(
             self.focal_point, 'action',
             'audit.engagement.finalize',
             condition=report_submitted_condition
         )
+        self.add_permissions(
+            self.focal_point, 'action',
+            'audit.engagement.send_back',
+            condition=report_submitted_condition
+        )
         self.add_permissions(
             self.everybody, 'view',
             self.report_block,
             condition=report_submitted_condition
         )
+        self.add_permissions(
+            self.everybody, 'view',
+            'audit.engagement.send_back_comment',
+            condition=report_submitted_condition
+        )
 
         # final report. everybody can view. focal point can add action points
         final_engagement_condition = self.engagement_status(Engagement.STATUSES.final)

src/etools/applications/audit/management/commands/update_audit_permissions.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.19 on 2024-01-03 16:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audit', '0029_merge_20230523_1049'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='engagement',
+            name='send_back_comment',
+            field=models.TextField(blank=True, verbose_name='Send Back Comment'),
+        ),
+    ]

src/etools/applications/audit/migrations/0030_engagement_send_back_comment.py

@@ -28,7 +28,7 @@
     ValidateMARiskCategories,
     ValidateMARiskExtra,
 )
-from etools.applications.audit.transitions.serializers import EngagementCancelSerializer
+from etools.applications.audit.transitions.serializers import EngagementCancelSerializer, EngagementSendBackSerializer
 from etools.applications.audit.utils import generate_final_report
 from etools.applications.core.urlresolvers import build_frontend_url
 from etools.applications.environment.notifications import send_notification_with_template
@@ -174,6 +174,8 @@ class Engagement(InheritedModelMixin, TimeStampedModel, models.Model):
 
     cancel_comment = models.TextField(blank=True, verbose_name=_('Cancel Comment'))
 
+    send_back_comment = models.TextField(blank=True, verbose_name=_('Send Back Comment'))
+
     active_pd = models.ManyToManyField('partners.Intervention', verbose_name=_('Active PDs'), blank=True)
 
     authorized_officers = models.ManyToManyField(
@@ -282,6 +284,13 @@ def submit(self):
 
         self._notify_focal_points('audit/engagement/reported_by_auditor')
 
+    @transition(status, source=STATUSES.report_submitted, target=STATUSES.partner_contacted,
+                permission=has_action_permission(action='send_back'),
+                custom={'serializer': EngagementSendBackSerializer})
+    def send_back(self, send_back_comment):
+        self.date_of_report_submit = None
+        self.send_back_comment = send_back_comment
+
     @transition(status, source=[STATUSES.partner_contacted, STATUSES.report_submitted], target=STATUSES.cancelled,
                 permission=has_action_permission(action='cancel'),
                 custom={'serializer': EngagementCancelSerializer})

src/etools/applications/audit/models.py

@@ -26,57 +26,13 @@ def update(self, instance, validated_data):
 
 class AuditorStaffMemberSerializer(UserSerializer):
     user = UserSerializer(required=False, source='*')
-    # TODO: REALMS - do cleanup - we don't need this field since this serializer is not to be used for editing anymore
-    # user_pk = serializers.PrimaryKeyRelatedField(
-    #     write_only=True, required=False,
-    #     queryset=get_user_model().objects.all()
-    # )
     hidden = serializers.SerializerMethodField()
 
     def get_hidden(self, obj):
         return False
 
-    # TODO: REALMS - do cleanup
-    # # TODO: make sure email provided is lower_case
-    # def validate(self, attrs):
-    #     validated_data = super().validate(attrs)
-    #     user_pk = validated_data.pop('user_pk', None)
-    #
-    #     if user_pk:
-    #         if hasattr(user_pk, 'purchase_order_auditorstaffmember'):
-    #             firm = user_pk.purchase_order_auditorstaffmember.auditor_firm
-    #             raise serializers.ValidationError({'user': _('User is already assigned to ') + str(firm)})
-    #         if not self.instance:
-    #             validated_data['user'] = user_pk
-    #     elif 'user' not in validated_data:
-    #         raise serializers.ValidationError({'user': _('This field is required.')})
-    #     elif 'user' in validated_data:
-    #         email = validated_data['user'].get('email', None)
-    #         if not AuditorStaffMember.objects.filter(user__email=email).exists():
-    #             try:
-    #                 validated_data['user'] = get_user_model().objects.get(email=email, email__isnull=False)
-    #             except get_user_model().DoesNotExist:
-    #                 pass
-    #
-    #     return validated_data
-
-    # TODO: REALMS
-    # def update(self, instance, validated_data):
-    #     instance = super().update(instance, validated_data)
-    #     if 'hidden' in validated_data:
-    #         Realm.objects.update_or_create(
-    #             user=instance,
-    #             country=connection.tenant,
-    #             organization=self.context['firm'].organization,
-    #             group=Auditor.as_group(),
-    #             defaults={'is_active': not validated_data['hidden']}
-    #         )
-    #     return instance
-
     class Meta(UserSerializer.Meta):
         model = get_user_model()
-        # TODO: REALMS
-        # fields = ['id', 'user', 'user_pk', 'hidden']
         fields = ['id', 'user', 'hidden']
 
 

src/etools/applications/audit/serializers/auditor.py

@@ -292,7 +292,7 @@ class Meta(EngagementListSerializer.Meta):
             'start_date', 'end_date', 'partner_contacted_at', 'date_of_field_visit', 'date_of_draft_report_to_ip',
             'date_of_comments_by_ip', 'date_of_draft_report_to_unicef', 'date_of_comments_by_unicef',
             'date_of_report_submit', 'date_of_final_report', 'date_of_cancel',
-            'cancel_comment', 'specific_procedures',
+            'cancel_comment', 'send_back_comment', 'specific_procedures',
             'engagement_attachments',
             'report_attachments',
             'final_report',

src/etools/applications/audit/serializers/engagement.py

@@ -361,21 +361,42 @@ def calculate_risk(category):
         if category.applicable_questions:
             category.risk_score = category.risk_points / category.applicable_questions
 
-            lowest_score_possible = category.applicable_questions
-            highest_score_possible = 4 * lowest_score_possible
-            banding_width = highest_score_possible - lowest_score_possible
-            low_points_below = lowest_score_possible + banding_width * 0.15
-            moderate_points_below = lowest_score_possible + banding_width * 0.3
-            significant_points_below = lowest_score_possible + banding_width * 0.5
-
-            if category.risk_points < low_points_below:
-                category.risk_rating = 'low'
-            elif category.risk_points < moderate_points_below:
-                category.risk_rating = 'medium'
-            elif category.risk_points < significant_points_below:
-                category.risk_rating = 'significant'
+            if category.code in ['ma_questionnaire', 'ma_subject_areas']:
+                # v1 calculations
+                lowest_score_possible = 1
+                highest_score_possible = (4 * category.applicable_questions + 4 * category.applicable_key_questions)
+                highest_score_possible = highest_score_possible / category.applicable_questions
+                banding_width = (highest_score_possible - lowest_score_possible) / 4
+                low_scores_below = lowest_score_possible + banding_width
+                moderate_scores_below = low_scores_below + banding_width
+                significant_score_below = moderate_scores_below + banding_width
+
+                category.risk_rating = 0
+                if category.risk_score < low_scores_below:
+                    category.risk_rating = 'low'
+                elif category.risk_score < moderate_scores_below:
+                    category.risk_rating = 'medium'
+                elif category.risk_score < significant_score_below:
+                    category.risk_rating = 'significant'
+                else:
+                    category.risk_rating = 'high'
             else:
-                category.risk_rating = 'high'
+                # v2 calculations
+                lowest_score_possible = category.applicable_questions
+                highest_score_possible = 4 * lowest_score_possible
+                banding_width = highest_score_possible - lowest_score_possible
+                low_points_below = lowest_score_possible + banding_width * 0.15
+                moderate_points_below = lowest_score_possible + banding_width * 0.3
+                significant_points_below = lowest_score_possible + banding_width * 0.5
+
+                if category.risk_points < low_points_below:
+                    category.risk_rating = 'low'
+                elif category.risk_points < moderate_points_below:
+                    category.risk_rating = 'medium'
+                elif category.risk_points < significant_points_below:
+                    category.risk_rating = 'significant'
+                else:
+                    category.risk_rating = 'high'
         else:
             category.risk_score = None
             category.risk_rating = 0

src/etools/applications/audit/serializers/risks.py

@@ -19,7 +19,7 @@
           <td colspan="2">{{ serializer.fields.financial_findings_local.label }}</td>
           {% endif %}
           <td>{{ serializer.fields.percent_of_audited_expenditure.label }}</td>
-          <td>{{ serializer.fields.audit_opinion.label }}</td>
+          <td colspan="2">{{ serializer.fields.audit_opinion.label }}</td>
           <td>No. of Financial Findings</td>
           <td colspan="3" style="padding: 0">
             <table>
@@ -45,7 +45,7 @@
           <td colspan="2">{{ engagement.financial_findings_local|default_if_none:"0.00"|currency }}</td>
           {% endif %}
           <td>{{ engagement.percent_of_audited_expenditure|default_if_none:"0.00"|currency }}</td>
-          <td>{{ engagement.audit_opinion|default:"-"|title }}</td>
+          <td colspan="2">{{ engagement.audit_opinion|default:"-"|title }}</td>
           <td>{{ engagement.financial_finding_set|length }}</td>
           <td>{{ engagement.key_internal_weakness.high_risk_count }}</td>
           <td>{{ engagement.key_internal_weakness.medium_risk_count }}</td>