Merge pull request #39 from unicef/develop

Develop

.circleci/config.yml

@@ -3,7 +3,7 @@ jobs:
   build:
     working_directory: ~/code
     docker:
-      - image: python:3.7-slim
+      - image: unicef/etools-base:latest
         environment:
           PGHOST: 127.0.0.1
           DATABASE_URL: "postgis://postgres:postgres@127.0.0.1:5432/etools_datamart"
@@ -23,22 +23,6 @@ jobs:
           POSTGRES_DB: etools_datamart
           POSTGRES_PASSWORD: postgres
     steps:
-      - run:
-          name: Install dependencies
-          command: |
-            mkdir -p /usr/share/man/man1
-            mkdir -p /usr/share/man/man7
-            apt-get update
-            apt-get install -y \
-                  libc-bin \
-                  libzmq3-dev \
-                  git \
-                  gcc \
-                  curl \
-                  gdal-bin \
-                  python-dev \
-                  postgresql-client
-
       - restore_cache:
           keys:
             - source-{{ .Branch }}-{{ .Revision }}

src/etools_datamart/api/endpoints/datamart/__init__.py

@@ -2,10 +2,11 @@
 # flake8: noqa: F401
 from .actionpoint import ActionPointViewSet
 from .attachment import AttachmentViewSet
-from .audit_engagement import EngagementViewSet
+from .audit_engagement import EngagementDetailViewSet, EngagementViewSet
 from .audit_result import AuditResultViewSet
 from .audit_spotcheck import SpotCheckViewSet
 from .famindicator import FAMIndicatorViewSet
+from .fm_questions import FMOntrackViewSet, FMQuestionViewSet
 from .funds_grant import GrantViewSet
 from .funds_reservation import FundsReservationViewSet
 from .hact_aggregate import HACTAggreagateViewSet

src/etools_datamart/api/endpoints/datamart/actionpoint.py

@@ -97,6 +97,20 @@ class Meta(DataMartSerializer.Meta):
         exclude = ()
 
 
+class ActionPointSimpleSerializer(DataMartSerializer):
+    class Meta(DataMartSerializer.Meta):
+        model = models.ActionPoint
+        exclude = None
+        fields = (
+            'actions_taken',
+            'assigned_by_name',
+            'assigned_by_email',
+            'assigned_to_name',
+            'assigned_to_email',
+            'status',
+        )
+
+
 class ActionPointFilterForm(forms.Form):
     created = DateRangePickerField(label='Created between', required=False)
 

src/etools_datamart/api/endpoints/datamart/audit_engagement.py

@@ -85,6 +85,49 @@ def get_partner_name(self, obj):
             return 'N/A'
 
 
+class EngagementDetailSerializer(DataMartSerializer):
+    partner = serializers.SerializerMethodField()
+    vendor_number = serializers.SerializerMethodField()
+
+    class Meta(DataMartSerializer.Meta):
+        model = models.Engagement
+        exclude = None
+        fields = (
+            "po_item",
+            "engagement_type",
+            "start_date",
+            "end_date",
+            "partner",
+            "partner_contacted_at",
+            "vendor_number",
+            "auditor",
+            "total_value",  # Value of FACE forms
+            "spotcheck_total_amount_tested",
+            "date_of_field_visit",  # FACE Form period
+            "final_report",  # Findings
+            "agreement",  # Agreed actions by partner
+            "rating",
+            "rating_extra",
+            "spotcheck_total_amount_of_ineligible_expenditure",
+            "financial_findings",  # Financial findings
+            "action_points",
+            "status",
+            "audit_opinion",
+        )
+
+    def get_partner(self, obj):
+        try:
+            return obj.partner['name']
+        except KeyError:
+            return 'N/A'
+
+    def get_vendor_number(self, obj):
+        try:
+            return obj.partner['vendor_number']
+        except KeyError:
+            return 'N/A'
+
+
 class EngagementFilterForm(forms.Form):
     engagement_type__in = Select2MultipleChoiceField(label='Engagement Type',
                                                      choices=models.Engagement.TYPES,
@@ -129,3 +172,17 @@ class EngagementViewSet(DataMartViewSet):
 
     def get_querystringfilter_form(self, request, filter):
         return EngagementFilterForm(request.GET, filter.form_prefix)
+
+
+class EngagementDetailViewSet(DataMartViewSet):
+    querystringfilter_form_base_class = EngagementFilterForm
+
+    serializer_class = EngagementDetailSerializer
+    queryset = models.Engagement.objects.all()
+    filter_fields = ('engagement_type', 'partner_contacted_at',
+                     'start_date', 'end_date', 'status', 'audit_opinion')
+    serializers_fieldsets = {'std': None,
+                             'simple': EngagementSerializerSimple}
+
+    def get_querystringfilter_form(self, request, filter):
+        return EngagementFilterForm(request.GET, filter.form_prefix)

src/etools_datamart/api/endpoints/datamart/fm_questions.py

@@ -0,0 +1,23 @@
+from etools_datamart.api.endpoints.common import DataMartViewSet
+from etools_datamart.api.endpoints.datamart.serializers import DataMartSerializer
+from etools_datamart.apps.mart.data import models
+
+
+class FMQuestionSerializer(DataMartSerializer):
+    class Meta(DataMartSerializer.Meta):
+        model = models.FMQuestion
+
+
+class FMQuestionViewSet(DataMartViewSet):
+    serializer_class = FMQuestionSerializer
+    queryset = models.FMQuestion.objects.all()
+
+
+class FMOntrackSerializer(DataMartSerializer):
+    class Meta(DataMartSerializer.Meta):
+        model = models.FMOntrack
+
+
+class FMOntrackViewSet(DataMartViewSet):
+    serializer_class = FMOntrackSerializer
+    queryset = models.FMOntrack.objects.all()

src/etools_datamart/api/endpoints/datamart/partners_staffmember.py

@@ -17,6 +17,7 @@ class PartnerStaffMemberSerializer(DataMartSerializer):
     class Meta:
         model = models.PartnerStaffMember
         fields = ('partner',
+                  'user',
                   'vendor_number',
                   'position',
                   'first_name',

src/etools_datamart/api/urls.py

@@ -25,12 +25,15 @@ class ReadOnlyRouter(APIReadOnlyRouter):
 router.register(r'datamart/attachment/attachment', endpoints.AttachmentViewSet)
 router.register(r'datamart/funds/grants', endpoints.GrantViewSet)
 router.register(r'datamart/audit/engagements', endpoints.EngagementViewSet)
+router.register(r'datamart/audit/engagement-details', endpoints.EngagementDetailViewSet, basename="engagement-details")
 router.register(r'datamart/audit/results', endpoints.AuditResultViewSet)
 router.register(r'datamart/audit/spot-check', endpoints.SpotCheckViewSet)
 router.register(r'datamart/actionpoints', endpoints.ActionPointViewSet)
 router.register(r'datamart/locations', endpoints.LocationViewSet)
 router.register(r'datamart/office', endpoints.OfficeViewSet)
 router.register(r'datamart/fam-indicators', endpoints.FAMIndicatorViewSet)
+router.register(r'datamart/fm-questions', endpoints.FMQuestionViewSet)
+router.register(r'datamart/fm-ontrack', endpoints.FMOntrackViewSet)
 router.register(r'datamart/funds-reservation', endpoints.FundsReservationViewSet)
 router.register(r'datamart/hact/aggregate', endpoints.HACTAggreagateViewSet)
 router.register(r'datamart/hact/history', endpoints.HACTHistoryViewSet)

src/etools_datamart/apps/etl/loader.py

@@ -2,6 +2,7 @@
 import time
 from uuid import UUID
 
+from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.core.cache import caches
 from django.utils import timezone
@@ -314,6 +315,14 @@ def get_values(self, record):
                 continue
             else:
                 ret[k] = self.get_value(k, v, record, ret)
+
+            # enforce field size limit
+            try:
+                ret[k] = ret[k][:settings.FIELD_SIZE_LIMIT]
+            except TypeError:
+                # not subscriptable so ignoring
+                pass
+
         return ret
 
     def get_value(self, field_name, value_or_func, original_record, current_mapping):
@@ -394,8 +403,9 @@ def on_start(self, run_type):
         self.etl_task.update(**defs)
 
     def on_end(self, error=None, retry=False):
-        from etools_datamart.apps.subscriptions.models import Subscription
         from django.utils import timezone
+
+        from etools_datamart.apps.subscriptions.models import Subscription
         self.results.total_records = self.model.objects.count()
 
         cost = time.time() - self._start

src/etools_datamart/apps/mart/data/admin.py

@@ -153,6 +153,12 @@ class InterventionAdmin(DataModelAdmin, TruncateTableMixin):
     date_hierarchy = 'start_date'
 
 
+@register(models.GeoName)
+class GeoNameAdmin(ModelAdmin):
+    list_display = ("lat", "lng", "name", "geoname_id",)
+    search_fields = ("lat", "lng", "name",)
+
+
 @register(models.InterventionByLocation)
 class InterventionByLocationAdmin(DataModelAdmin, TruncateTableMixin):
     list_display = ('country_name', 'title', 'document_type',
@@ -203,7 +209,7 @@ class LocationAdmin(DataModelAdmin):
     # readonly_fields = ('parent', 'gateway')
     list_filter = ('level',)
     search_fields = ('name',)
-    autocomplete_fields = ('parent', 'gateway')
+    autocomplete_fields = ('parent', 'gateway', 'geoname',)
     actions = ['update_centroid', mass_update]
     mass_update_exclude = ['geom', 'id']
     mass_update_hints = []
@@ -247,7 +253,14 @@ class PartnerAdmin(DataModelAdmin):
 
 @register(models.PartnerStaffMember)
 class PartnerStaffMemberAdmin(DataModelAdmin):
-    list_display = ('title', 'last_name', 'first_name', 'email', 'phone')
+    list_display = (
+        'title',
+        'last_name',
+        'first_name',
+        'user',
+        'email',
+        'phone',
+    )
     date_hierarchy = 'created'
     list_filter = ('active',)
 

src/etools_datamart/apps/mart/data/loader.py

@@ -1,6 +1,7 @@
 import time
 from inspect import isclass
 
+from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import connections, models, transaction
 from django.utils import timezone
@@ -94,6 +95,13 @@ def xxget_values(self, record):
             else:
                 raise Exception("Invalid field name or mapping '%s:%s'" % (k, v))
 
+            # enforce field size limit
+            try:
+                ret[k] = ret[k][:settings.FIELD_SIZE_LIMIT]
+            except TypeError:
+                # not subscriptable so ignoring
+                pass
+
         return ret
 
     def get_value(self, field_name, value_or_func, original_record, current_mapping):
@@ -305,6 +313,13 @@ def ssget_values(self, record):
             else:
                 ret[k] = get_attr(record, v)
 
+            # enforce field size limit
+            try:
+                ret[k] = ret[k][:settings.FIELD_SIZE_LIMIT]
+            except TypeError:
+                # not subscriptable so ignoring
+                pass
+
         return ret
 
     def get_value(self, field_name, value_or_func, original_record, current_mapping):

src/etools_datamart/apps/mart/data/migrations/0119_engagement_auditor.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.2.9 on 2020-04-22 12:55
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0118_fundsreservation_pd_reference_number'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='engagement',
+            name='auditor',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='engagement',
+            name='action_points',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True),
+        ),
+    ]

src/etools_datamart/apps/mart/data/migrations/0120_auto_20200604_1749.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.2.11 on 2020-06-04 17:49
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0119_engagement_auditor'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='engagement',
+            name='rating',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='engagement',
+            name='rating_extra',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True),
+        ),
+    ]

src/etools_datamart/apps/mart/data/migrations/0121_auto_20200610_1701.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.11 on 2020-06-10 17:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0120_auto_20200604_1749'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='engagement',
+            name='rating',
+            field=models.CharField(blank=True, choices=[(0, 'N/A'), (1, 'Low'), (2, 'Medium'), (3, 'Significant'), (4, 'High')], max_length=100, null=True),
+        ),
+    ]

src/etools_datamart/apps/mart/data/migrations/0122_pdindicator_pd_reference_number.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.11 on 2020-06-29 17:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0121_auto_20200610_1701'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pdindicator',
+            name='pd_reference_number',
+            field=models.CharField(blank=True, max_length=256, null=True),
+        ),
+    ]