From f310ad1669905362aad84803ebb83768960cb6b3 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 16 Apr 2020 11:10:30 -0400 Subject: [PATCH 01/22] Update partners intervention budget source id --- .../apps/mart/data/models/partners_interventionbudget.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py index c9daa9767..abd33a0ed 100644 --- a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py +++ b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py @@ -12,7 +12,6 @@ class InterventionBudgetLoader(InterventionLoader): def process_country(self): qs = PartnersInterventionbudget.objects.all() for record in qs.all(): - record.intervention.budget = record filters = self.config.key(self, record) values = self.get_values(record.intervention) values['source_id'] = record.id @@ -57,7 +56,7 @@ class Options(InterventionAbstract.Options): model = PartnersInterventionbudget depends = (Location,) key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, - source_id=record.intervention.budget.pk) + source_id=record.pk) mapping = extend(InterventionAbstract.Options.mapping, dict( From 027f2f07819dfbfaff52c5de1276e513f0f3ebbf Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 17 Apr 2020 09:45:17 -0400 Subject: [PATCH 02/22] Add queryset to InterventionBudget loader so that gets used in later functions --- .../apps/mart/data/models/partners_interventionbudget.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py index abd33a0ed..0e1b39bba 100644 --- a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py +++ b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py @@ -9,9 +9,11 @@ class InterventionBudgetLoader(InterventionLoader): + def get_queryset(self): + return PartnersInterventionbudget.objects + def process_country(self): - qs = PartnersInterventionbudget.objects.all() - for record in qs.all(): + for record in self.get_queryset().all(): filters = self.config.key(self, record) values = self.get_values(record.intervention) values['source_id'] = record.id From 190f9dec5c423e03c22991613e350706b5fc5e16 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 17 Apr 2020 09:45:49 -0400 Subject: [PATCH 03/22] Update docker base dockerfile to use latest-stable Conflicts with libffi package --- docker/Dockerfile.alpine.base | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.alpine.base b/docker/Dockerfile.alpine.base index 3840eb439..f370e406c 100644 --- a/docker/Dockerfile.alpine.base +++ b/docker/Dockerfile.alpine.base @@ -11,9 +11,9 @@ ENV TEST ${TEST} RUN apk add --no-cache --virtual .build-deps \ - --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \ - --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \ - --repository http://dl-3.alpinelinux.org/alpine/edge/community/ \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/testing \ + --repository http://dl-3.alpinelinux.org/alpine/latest-stable/community/ \ gcc \ g++ \ gdal-dev \ From 8e1c0084506e25c029ae51d172c3d46e7c7d3057 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 17 Apr 2020 13:51:46 -0400 Subject: [PATCH 04/22] Add geonameid to locations endpoint --- .../migrations/0114_auto_20200417_1734.py | 37 +++++++++ .../apps/mart/data/models/location.py | 78 ++++++++++++++++++- src/etools_datamart/config/settings.py | 7 ++ 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 src/etools_datamart/apps/mart/data/migrations/0114_auto_20200417_1734.py diff --git a/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200417_1734.py b/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200417_1734.py new file mode 100644 index 000000000..54efabd7a --- /dev/null +++ b/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200417_1734.py @@ -0,0 +1,37 @@ +# Generated by Django 2.2.11 on 2020-04-17 17:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0113_auto_20200408_1747'), + ] + + operations = [ + migrations.AddField( + model_name='location', + name='geonameid', + field=models.CharField(max_length=50, null=True), + ), + migrations.CreateModel( + name='GeoName', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('toponym_name', models.CharField(max_length=254, null=True)), + ('name', models.CharField(max_length=254, null=True)), + ('lat', models.FloatField()), + ('lng', models.FloatField()), + ('geoname_id', models.CharField(max_length=50, null=True)), + ('country_code', models.CharField(max_length=20, null=True)), + ('country_name', models.CharField(max_length=150, null=True)), + ('fcl', models.CharField(max_length=50, null=True)), + ('fcode', models.CharField(max_length=50, null=True)), + ('distance', models.FloatField(null=True)), + ], + options={ + 'unique_together': {('lat', 'lng')}, + }, + ), + ] diff --git a/src/etools_datamart/apps/mart/data/models/location.py b/src/etools_datamart/apps/mart/data/models/location.py index b64dd77fe..51efd695e 100644 --- a/src/etools_datamart/apps/mart/data/models/location.py +++ b/src/etools_datamart/apps/mart/data/models/location.py @@ -1,7 +1,12 @@ +from xml.etree import ElementTree + +from django.conf import settings from django.contrib.gis.db import models as geomodels from django.contrib.gis.db.models.functions import Centroid from django.db import connection, models +import requests + from etools_datamart.apps.core.models import DataMartManager, DataMartQuerySet from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel from etools_datamart.apps.sources.etools.models import LocationsGatewaytype, LocationsLocation @@ -60,6 +65,10 @@ def load(self, **kwargs): finally: Location.objects.batch_update_centroid() + def get_geonameid(self, record: LocationsLocation, values: dict, **kwargs): + geoname = GeoName.objects.get_or_add(lat=record.lat, lng=record.lng) + return geoname.geoname_id + class Location(EtoolsDataMartModel): name = models.CharField(max_length=254, db_index=True) @@ -69,6 +78,7 @@ class Location(EtoolsDataMartModel): point = geomodels.PointField(blank=True, null=True) gateway = models.ForeignKey(GatewayType, models.DO_NOTHING, blank=True, null=True) geom = geomodels.MultiPolygonField(blank=True, null=True) + geonameid = models.CharField(max_length=50, null=True) level = models.IntegerField(db_index=True) lft = models.IntegerField() parent = models.ForeignKey('self', models.DO_NOTHING, blank=True, null=True) @@ -90,13 +100,77 @@ class Options: source = LocationsLocation queryset = lambda: LocationsLocation.objects.order_by('-parent') last_modify_field = 'modified' - exclude_from_compare = ['latitude', 'longitude', 'point'] + exclude_from_compare = ['latitude', 'longitude', 'point', 'geonameid'] # sync_deleted_records = False mapping = {'source_id': 'id', # 'area_code': lambda loader, record: loader.context['country'].business_area_code, 'parent': '__self__', - 'gateway': GatewayType + 'gateway': GatewayType, + 'geonameid': 'i', } def __str__(self): return self.name + + +# store geo names pulled from http://api.geonames.org +# Using lat and lng get geoname data with +# http://api.geonames.org/findNearby?lat=47.3&lng=9&username=ntrncic +class GeoNameManager(models.Manager): + def get_or_add(self, lat, lng): + # check if we have a matching lat, lng record + # if so, return that record + # otherwise create a new record based on results + # of request to geonames.org + try: + geoname = self.get_queryset().get(lat=lat, lng=lng) + except GeoName.DoesNotExist: + payload = { + "lat": lat, + "lng": lng, + "username": settings.GEONAMES_USERNAME, + } + res = requests.get( + settings.GEONAMES_URL, + params=payload, + timeout=settings.REQUEST_TIMEOUT, + ) + geoname = ElementTree.fromstring(res.content)[0] + mapping = [ + ("toponym_name", "toponymName"), + ("name", "name"), + ("lat", "lat"), + ("lng", "lng"), + ("geoname_id", "geonameId"), + ("country_code", "countryCode"), + ("country_name", "countryName"), + ("fcl", "fcl"), + ("fcode", "fcode"), + ("distance", "distance"), + ] + data = {} + for k, f in mapping: + data[k] = geoname.find(f).text + geoname = GeoName.objects.create(**data) + return geoname + + +class GeoName(models.Model): + toponym_name = models.CharField(max_length=254, null=True) + name = models.CharField(max_length=254, null=True) + lat = models.FloatField() + lng = models.FloatField() + geoname_id = models.CharField(max_length=50, null=True) + country_code = models.CharField(max_length=20, null=True) + country_name = models.CharField(max_length=150, null=True) + fcl = models.CharField(max_length=50, null=True) + fcode = models.CharField(max_length=50, null=True) + distance = models.FloatField(null=True) + + objects = GeoNameManager() + + class Meta: + unique_together = ('lat', 'lng') + + def __str__(self): + return f"{self.name} ({self.lat}, {self.lng})" diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index daa5e358c..caba29b52 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -79,6 +79,9 @@ URL_PREFIX=(str, ''), USE_X_FORWARDED_HOST=(bool, False), X_FRAME_OPTIONS=(str, 'DENY'), + GEONAMES_URL=(str, 'http://api.geonames.org/findNearby'), + GEONAMES_USERNAME=(str, 'ntrncic'), + REQUEST_TIMEOUT=(int, 300), ) DEBUG = env.bool('DEBUG') @@ -737,3 +740,7 @@ def before_send(event, hint): AZURE_AUTO_SIGN = env('AZURE_STORAGE_AUTO_SIGN') AZURE_ACCESS_MODE = env('AZURE_STORAGE_ACCESS_MODE') AZURE_ACCESS_TTL = env('AZURE_STORAGE_ACCESS_TTL') + +REQUEST_TIMEOUT = env('REQUEST_TIMEOUT') +GEONAMES_URL = env('GEONAMES_URL') +GEONAMES_USERNAME = env('GEONAMES_USERNAME') From 7588e473927af5eb76fe4f04f2e1c54f2b7f49d5 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 17 Apr 2020 13:52:08 -0400 Subject: [PATCH 05/22] Bump target and no longer test on build command --- docker/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 677c33142..184405668 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=1 DOCKER_PASS?= DOCKER_USER?= -TARGET?=2.13.1a +TARGET?=2.13.1b PUSH_BASE?=1 BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" ) #BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" | echo "`xargs`xx" ) @@ -55,7 +55,7 @@ build-test: TARGET=dev $(MAKE) .build test build: - $(MAKE) .build test + $(MAKE) .build .run: From cb95859d3dba318baeba2d7539442f9edfe23066 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 17 Apr 2020 14:50:47 -0400 Subject: [PATCH 06/22] Set the budget values in process_country and not through mapping --- .../data/models/partners_interventionbudget.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py index 0e1b39bba..76ed1e40f 100644 --- a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py +++ b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py @@ -17,6 +17,11 @@ def process_country(self): filters = self.config.key(self, record) values = self.get_values(record.intervention) values['source_id'] = record.id + values['budget_cso_contribution'] = record.partner_contribution_local + values['budget_unicef_cash'] = record.unicef_cash_local + values['budget_total'] = record.total_local + values['budget_currency'] = record.currency + values['budget_unicef_supply'] = record.in_kind_amount_local op = self.process_record(filters, values) self.increment_counter(op) @@ -59,12 +64,3 @@ class Options(InterventionAbstract.Options): depends = (Location,) key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, source_id=record.pk) - - mapping = extend(InterventionAbstract.Options.mapping, - dict( - budget_cso_contribution='budget.partner_contribution_local', - budget_unicef_cash='budget.unicef_cash_local', - budget_total='budget.total_local', - budget_currency='budget.currency', - budget_unicef_supply='budget.in_kind_amount_local', - )) From 7a41f38b9107893a7578a7c95107cbf8536ee55d Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 20 Apr 2020 10:08:46 -0400 Subject: [PATCH 07/22] Update get geoname functionality Handle no lat/lng values Handle duplicate records, happens for slightly different lat/lng values --- .../apps/mart/data/models/location.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/etools_datamart/apps/mart/data/models/location.py b/src/etools_datamart/apps/mart/data/models/location.py index 51efd695e..c3c7c4250 100644 --- a/src/etools_datamart/apps/mart/data/models/location.py +++ b/src/etools_datamart/apps/mart/data/models/location.py @@ -66,7 +66,12 @@ def load(self, **kwargs): Location.objects.batch_update_centroid() def get_geonameid(self, record: LocationsLocation, values: dict, **kwargs): - geoname = GeoName.objects.get_or_add(lat=record.lat, lng=record.lng) + if not record.latitude or not record.longitude: + return None + geoname = GeoName.objects.get_or_add( + lat=record.latitude, + lng=record.longitude, + ) return geoname.geoname_id @@ -106,7 +111,7 @@ class Options: # 'area_code': lambda loader, record: loader.context['country'].business_area_code, 'parent': '__self__', 'gateway': GatewayType, - 'geonameid': 'i', + 'geonameid': '-', } def __str__(self): @@ -122,6 +127,8 @@ def get_or_add(self, lat, lng): # if so, return that record # otherwise create a new record based on results # of request to geonames.org + if not lat or not lng: + return None try: geoname = self.get_queryset().get(lat=lat, lng=lng) except GeoName.DoesNotExist: @@ -151,7 +158,7 @@ def get_or_add(self, lat, lng): data = {} for k, f in mapping: data[k] = geoname.find(f).text - geoname = GeoName.objects.create(**data) + geoname, __ = GeoName.objects.get_or_create(**data) return geoname From b1e0611737ea10fcf19e0cfb42530a869713d1ab Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 20 Apr 2020 13:46:29 -0400 Subject: [PATCH 08/22] Update test record responses with geoname --- .../get/{'-serializer': 'geo'}.response.json | 17 ++++++++++++++--- .../get/{'-serializer': 'gis'}.response.json | 17 ++++++++++++++--- .../get/{'-serializer': 'latlng'}.response.json | 12 ++++++++++-- .../get/{'-serializer': 'light'}.response.json | 12 ++++++++++-- .../get/{'-serializer': 'std'}.response.json | 17 ++++++++++++++--- .../get/None.response.json | 13 +++++++++++-- 6 files changed, 73 insertions(+), 15 deletions(-) diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json index 976f346e3..961068e63 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"a68108482a49fa9d63bcfc2be35ff6e8\"" + "\"e55ef31bde4617bb74da202f1a868acf\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,19 +39,27 @@ ], "cache-key": [ "cache-key", - "a68108482a49fa9d63bcfc2be35ff6e8" + "e55ef31bde4617bb74da202f1a868acf" ], "cache-hit": [ "cache-hit", "False" ], + "qs_filter": [ + "qs_filter", + "{}" + ], + "qs_exclude": [ + "qs_exclude", + "{}" + ], "cache-ttl": [ "cache-ttl", "1y" ], "content-length": [ "Content-Length", - "1147" + "1198" ] }, "data": { @@ -77,6 +85,7 @@ "longitude": null, "p_code": "", "point": null, + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, @@ -98,6 +107,7 @@ "longitude": null, "p_code": "", "point": null, + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, @@ -119,6 +129,7 @@ "longitude": null, "p_code": "", "point": null, + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json index 0f171ed38..e7b54cb5f 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"1d8b75a006985f57eb41f0abe2b93b0d\"" + "\"c103699c1133008507285bf7143e88c9\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,19 +39,27 @@ ], "cache-key": [ "cache-key", - "1d8b75a006985f57eb41f0abe2b93b0d" + "c103699c1133008507285bf7143e88c9" ], "cache-hit": [ "cache-hit", "False" ], + "qs_filter": [ + "qs_filter", + "{}" + ], + "qs_exclude": [ + "qs_exclude", + "{}" + ], "cache-ttl": [ "cache-ttl", "1y" ], "content-length": [ "Content-Length", - "999" + "1050" ] }, "data": { @@ -73,6 +81,7 @@ "p_code": "", "point": null, "geom": null, + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, @@ -91,6 +100,7 @@ "p_code": "", "point": null, "geom": null, + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, @@ -109,6 +119,7 @@ "p_code": "", "point": null, "geom": null, + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json index 3f0c00e48..372cff8e5 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"af31155424c5e98a227bc738e4e90aed\"" + "\"3fef34d12dc9fa44cb6635d23439a315\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,12 +39,20 @@ ], "cache-key": [ "cache-key", - "af31155424c5e98a227bc738e4e90aed" + "3fef34d12dc9fa44cb6635d23439a315" ], "cache-hit": [ "cache-hit", "False" ], + "qs_filter": [ + "qs_filter", + "{}" + ], + "qs_exclude": [ + "qs_exclude", + "{}" + ], "cache-ttl": [ "cache-ttl", "1y" diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json index cdb776ef4..9356b7ea6 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"e3fea213e28ce892638e1e22af822fd4\"" + "\"5f6de07e807b72340507bbe541bc9061\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,12 +39,20 @@ ], "cache-key": [ "cache-key", - "e3fea213e28ce892638e1e22af822fd4" + "5f6de07e807b72340507bbe541bc9061" ], "cache-hit": [ "cache-hit", "False" ], + "qs_filter": [ + "qs_filter", + "{}" + ], + "qs_exclude": [ + "qs_exclude", + "{}" + ], "cache-ttl": [ "cache-ttl", "1y" diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json index a7b070a75..d680b5022 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"5ee8c76e2e1e6a3ff02fdc065c7fa81e\"" + "\"b201e0867d7223b2928b0db88f06cb90\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,19 +39,27 @@ ], "cache-key": [ "cache-key", - "5ee8c76e2e1e6a3ff02fdc065c7fa81e" + "b201e0867d7223b2928b0db88f06cb90" ], "cache-hit": [ "cache-hit", "False" ], + "qs_filter": [ + "qs_filter", + "{}" + ], + "qs_exclude": [ + "qs_exclude", + "{}" + ], "cache-ttl": [ "cache-ttl", "1y" ], "content-length": [ "Content-Length", - "825" + "876" ] }, "data": { @@ -69,6 +77,7 @@ "area_code": "", "name": "name000", "p_code": "", + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, @@ -83,6 +92,7 @@ "area_code": "", "name": "name001", "p_code": "", + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, @@ -97,6 +107,7 @@ "area_code": "", "name": "name002", "p_code": "", + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, diff --git a/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json b/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json index 7c878a822..ddec71e2e 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"79ff1e8816ef69c2d9436bbffae91cd2\"" + "\"db3891d8a2e812b90d8b2ffdbd8cb438\"" ], "x-frame-options": [ "X-Frame-Options", @@ -29,6 +29,14 @@ "service", "etools_datamart.api.endpoints.datamart.location.LocationViewSet" ], + "qs_filter": [ + "qs_filter", + "{}" + ], + "qs_exclude": [ + "qs_exclude", + "{}" + ], "cache-hit": [ "cache-hit", "False" @@ -39,7 +47,7 @@ ], "content-length": [ "Content-Length", - "247" + "264" ] }, "data": { @@ -50,6 +58,7 @@ "area_code": "", "name": "name003", "p_code": "", + "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, From b2eb32e5a85af2bf111c7c7fb2557d4b517ad3cb Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 20 Apr 2020 13:46:48 -0400 Subject: [PATCH 09/22] Exclude GeoName model from model loader tests --- tests/datamart/test_data_models.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/datamart/test_data_models.py b/tests/datamart/test_data_models.py index 4a4318919..a16a1c0c6 100644 --- a/tests/datamart/test_data_models.py +++ b/tests/datamart/test_data_models.py @@ -4,6 +4,10 @@ from django.apps import apps from django.db import connections +EXCLUDED_MODELS = [ + 'GeoName', +] + def pytest_generate_tests(metafunc): if 'context' in metafunc.fixturenames: @@ -11,13 +15,20 @@ def pytest_generate_tests(metafunc): models = [] ids = [] for m in config.get_models(): - ctx = {'country': 1, 'year': 2019} - models.append([m, ctx]) - ids.append(m.__name__) + if m.__name__ not in EXCLUDED_MODELS: + ctx = {'country': 1, 'year': 2019} + models.append([m, ctx]) + ids.append(m.__name__) metafunc.parametrize("model,context", models, ids=ids) elif 'model' in metafunc.fixturenames: config = apps.get_app_config('data') - metafunc.parametrize("model", config.get_models()) + metafunc.parametrize( + "model", + [ + m for m in config.get_models() + if m.__name__ not in EXCLUDED_MODELS + ], + ) def test_model_str(model): From cfb7e168b286acf85ea01fe9944d95dcbf606967 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 20 Apr 2020 13:59:48 -0400 Subject: [PATCH 10/22] Update Section name field length --- .../data/migrations/0114_auto_20200420_1755.py | 18 ++++++++++++++++++ .../apps/mart/data/models/report_sector.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 src/etools_datamart/apps/mart/data/migrations/0114_auto_20200420_1755.py diff --git a/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200420_1755.py b/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200420_1755.py new file mode 100644 index 000000000..5b0368b23 --- /dev/null +++ b/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200420_1755.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.9 on 2020-04-20 17:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0113_auto_20200408_1747'), + ] + + operations = [ + migrations.AlterField( + model_name='section', + name='name', + field=models.CharField(blank=True, max_length=128, null=True), + ), + ] diff --git a/src/etools_datamart/apps/mart/data/models/report_sector.py b/src/etools_datamart/apps/mart/data/models/report_sector.py index af82cde4e..dd2ad1993 100644 --- a/src/etools_datamart/apps/mart/data/models/report_sector.py +++ b/src/etools_datamart/apps/mart/data/models/report_sector.py @@ -5,7 +5,7 @@ class Section(EtoolsDataMartModel): - name = models.CharField(max_length=45, blank=True, null=True) + name = models.CharField(max_length=128, blank=True, null=True) description = models.CharField(max_length=256, blank=True, null=True) alternate_id = models.IntegerField(blank=True, null=True) alternate_name = models.CharField(max_length=255, blank=True, null=True) From 280e1379d60ced7733bd21641594f50c2dd632fa Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Tue, 21 Apr 2020 09:43:59 -0400 Subject: [PATCH 11/22] Add pd_url field to pdindicator endpoint --- .../data/migrations/0114_pdindicator_pd_url.py | 18 ++++++++++++++++++ .../apps/mart/data/models/pd_indicator.py | 10 ++++++++++ 2 files changed, 28 insertions(+) create mode 100644 src/etools_datamart/apps/mart/data/migrations/0114_pdindicator_pd_url.py diff --git a/src/etools_datamart/apps/mart/data/migrations/0114_pdindicator_pd_url.py b/src/etools_datamart/apps/mart/data/migrations/0114_pdindicator_pd_url.py new file mode 100644 index 000000000..35181a339 --- /dev/null +++ b/src/etools_datamart/apps/mart/data/migrations/0114_pdindicator_pd_url.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.9 on 2020-04-21 13:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0113_auto_20200408_1747'), + ] + + operations = [ + migrations.AddField( + model_name='pdindicator', + name='pd_url', + field=models.CharField(blank=True, max_length=254, null=True), + ), + ] diff --git a/src/etools_datamart/apps/mart/data/models/pd_indicator.py b/src/etools_datamart/apps/mart/data/models/pd_indicator.py index 6a22d63a3..b0ac622f6 100644 --- a/src/etools_datamart/apps/mart/data/models/pd_indicator.py +++ b/src/etools_datamart/apps/mart/data/models/pd_indicator.py @@ -1,3 +1,5 @@ +from django.shortcuts import reverse + from etools_datamart.apps.mart.data.fields import SafeDecimal from etools_datamart.apps.mart.data.loader import EtoolsLoader from etools_datamart.apps.mart.data.models import Location @@ -30,6 +32,12 @@ def process_country(self): op = self.process_record(filters, values) self.increment_counter(op) + def get_pd_url(self, record: ReportsAppliedindicator, values: dict, **kwargs): + return reverse( + "api:intervention-detail", + args=["latest", record.lower_result.result_link.intervention.pk], + ) + class PDIndicator(LocationMixin, EtoolsDataMartModel): context_code = models.CharField(max_length=50, blank=True, null=True) @@ -73,6 +81,7 @@ class PDIndicator(LocationMixin, EtoolsDataMartModel): # from lower_result lower_result_name = models.CharField(max_length=500, blank=True, null=True) result_link_intervention = models.IntegerField(blank=True, null=True) + pd_url = models.CharField(max_length=254, blank=True, null=True) # from section section_name = models.CharField(max_length=45, blank=True, null=True) @@ -126,6 +135,7 @@ class Options: section_name='section.name', lower_result_name='lower_result.name', result_link_intervention='lower_result.result_link.intervention.pk', + pd_url='-', disaggregation_name='disaggregation.name', disaggregation_active='disaggregation.active', From 8bc0a21431502af91af2cc2ccfe7a7b524a04988 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Tue, 21 Apr 2020 09:45:16 -0400 Subject: [PATCH 12/22] Lint cleanup --- .../apps/mart/data/models/partners_interventionbudget.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py index 76ed1e40f..f03b4e4fc 100644 --- a/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py +++ b/src/etools_datamart/apps/mart/data/models/partners_interventionbudget.py @@ -3,7 +3,6 @@ from etools_datamart.apps.mart.data.models import Location from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel from etools_datamart.apps.mart.data.models.intervention import InterventionAbstract, InterventionLoader -from etools_datamart.apps.mart.data.models.mixins import extend from etools_datamart.apps.sources.etools.models import (FundsFundsreservationheader, models, PartnersIntervention, PartnersInterventionbudget,) From ddacbbd17a332a70b18ce53a12cd845992127da5 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Tue, 21 Apr 2020 13:47:43 -0400 Subject: [PATCH 13/22] Update geoname field to foreign key relation on locations Update migration sequence --- ...r_pd_url.py => 0115_pdindicator_pd_url.py} | 2 +- ...417_1734.py => 0116_auto_20200417_1734.py} | 12 ++++++------ .../migrations/0117_auto_20200421_1544.py | 19 +++++++++++++++++++ .../apps/mart/data/models/location.py | 10 +++++----- 4 files changed, 31 insertions(+), 12 deletions(-) rename src/etools_datamart/apps/mart/data/migrations/{0114_pdindicator_pd_url.py => 0115_pdindicator_pd_url.py} (89%) rename src/etools_datamart/apps/mart/data/migrations/{0114_auto_20200417_1734.py => 0116_auto_20200417_1734.py} (86%) create mode 100644 src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py diff --git a/src/etools_datamart/apps/mart/data/migrations/0114_pdindicator_pd_url.py b/src/etools_datamart/apps/mart/data/migrations/0115_pdindicator_pd_url.py similarity index 89% rename from src/etools_datamart/apps/mart/data/migrations/0114_pdindicator_pd_url.py rename to src/etools_datamart/apps/mart/data/migrations/0115_pdindicator_pd_url.py index 35181a339..e88e5869e 100644 --- a/src/etools_datamart/apps/mart/data/migrations/0114_pdindicator_pd_url.py +++ b/src/etools_datamart/apps/mart/data/migrations/0115_pdindicator_pd_url.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('data', '0113_auto_20200408_1747'), + ('data', '0114_auto_20200420_1755'), ] operations = [ diff --git a/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200417_1734.py b/src/etools_datamart/apps/mart/data/migrations/0116_auto_20200417_1734.py similarity index 86% rename from src/etools_datamart/apps/mart/data/migrations/0114_auto_20200417_1734.py rename to src/etools_datamart/apps/mart/data/migrations/0116_auto_20200417_1734.py index 54efabd7a..9cd726a6d 100644 --- a/src/etools_datamart/apps/mart/data/migrations/0114_auto_20200417_1734.py +++ b/src/etools_datamart/apps/mart/data/migrations/0116_auto_20200417_1734.py @@ -6,15 +6,10 @@ class Migration(migrations.Migration): dependencies = [ - ('data', '0113_auto_20200408_1747'), + ('data', '0115_pdindicator_pd_url'), ] operations = [ - migrations.AddField( - model_name='location', - name='geonameid', - field=models.CharField(max_length=50, null=True), - ), migrations.CreateModel( name='GeoName', fields=[ @@ -34,4 +29,9 @@ class Migration(migrations.Migration): 'unique_together': {('lat', 'lng')}, }, ), + migrations.AddField( + model_name='location', + name='geoname', + field=models.ForeignKey(blank=True, null=True, on_delete=models.deletion.DO_NOTHING, to='data.GeoName'), + ), ] diff --git a/src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py b/src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py new file mode 100644 index 000000000..358184dc3 --- /dev/null +++ b/src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.9 on 2020-04-21 15:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0116_auto_20200417_1734'), + ] + + operations = [ + migrations.AlterField( + model_name='location', + name='geoname', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='data.GeoName'), + ), + ] diff --git a/src/etools_datamart/apps/mart/data/models/location.py b/src/etools_datamart/apps/mart/data/models/location.py index c3c7c4250..b396ee0e1 100644 --- a/src/etools_datamart/apps/mart/data/models/location.py +++ b/src/etools_datamart/apps/mart/data/models/location.py @@ -65,14 +65,14 @@ def load(self, **kwargs): finally: Location.objects.batch_update_centroid() - def get_geonameid(self, record: LocationsLocation, values: dict, **kwargs): + def get_geoname(self, record: LocationsLocation, values: dict, **kwargs): if not record.latitude or not record.longitude: return None geoname = GeoName.objects.get_or_add( lat=record.latitude, lng=record.longitude, ) - return geoname.geoname_id + return geoname class Location(EtoolsDataMartModel): @@ -83,7 +83,7 @@ class Location(EtoolsDataMartModel): point = geomodels.PointField(blank=True, null=True) gateway = models.ForeignKey(GatewayType, models.DO_NOTHING, blank=True, null=True) geom = geomodels.MultiPolygonField(blank=True, null=True) - geonameid = models.CharField(max_length=50, null=True) + geoname = models.ForeignKey("GeoName", models.DO_NOTHING, blank=True, null=True) level = models.IntegerField(db_index=True) lft = models.IntegerField() parent = models.ForeignKey('self', models.DO_NOTHING, blank=True, null=True) @@ -105,13 +105,13 @@ class Options: source = LocationsLocation queryset = lambda: LocationsLocation.objects.order_by('-parent') last_modify_field = 'modified' - exclude_from_compare = ['latitude', 'longitude', 'point', 'geonameid'] + exclude_from_compare = ['latitude', 'longitude', 'point', 'geoname'] # sync_deleted_records = False mapping = {'source_id': 'id', # 'area_code': lambda loader, record: loader.context['country'].business_area_code, 'parent': '__self__', 'gateway': GatewayType, - 'geonameid': '-', + 'geoname': '-', } def __str__(self): From cd41f85ad60f8541e2a660927ae158e0117366fb Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Tue, 21 Apr 2020 13:48:05 -0400 Subject: [PATCH 14/22] Add geonameid to location endpoint results --- src/etools_datamart/api/endpoints/datamart/location.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/etools_datamart/api/endpoints/datamart/location.py b/src/etools_datamart/api/endpoints/datamart/location.py index 75ca35c79..4de6b805d 100644 --- a/src/etools_datamart/api/endpoints/datamart/location.py +++ b/src/etools_datamart/api/endpoints/datamart/location.py @@ -26,11 +26,18 @@ class Meta: class LocationSerializer(serializers.ModelSerializer): + geonameid = serializers.SerializerMethodField() + class Meta: model = models.Location exclude = ('schema_name', 'tree_id', 'lft', 'rght', 'level', 'source_id', 'geom', 'point', 'latitude', 'longitude') + def get_geonameid(self, obj): + if obj.geoname: + return geoname.geoname_id + return None + class LocationViewSet(common.DataMartViewSet): serializer_class = LocationSerializer From 9b7a53e917adcb22a45528734fff5dea3279bab6 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Tue, 21 Apr 2020 13:48:24 -0400 Subject: [PATCH 15/22] Update test responses for locations --- .../get/{'-serializer': 'geo'}.response.json | 12 ++++++------ .../get/{'-serializer': 'gis'}.response.json | 12 ++++++------ .../get/{'-serializer': 'latlng'}.response.json | 4 ++-- .../get/{'-serializer': 'light'}.response.json | 4 ++-- .../get/{'-serializer': 'std'}.response.json | 15 +++++++++------ .../get/None.response.json | 7 ++++--- 6 files changed, 29 insertions(+), 25 deletions(-) diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json index 961068e63..eb6fe86be 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'geo'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"e55ef31bde4617bb74da202f1a868acf\"" + "\"7b1c8f06afa1b55cf60e751b25230b12\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,7 +39,7 @@ ], "cache-key": [ "cache-key", - "e55ef31bde4617bb74da202f1a868acf" + "7b1c8f06afa1b55cf60e751b25230b12" ], "cache-hit": [ "cache-hit", @@ -59,7 +59,7 @@ ], "content-length": [ "Content-Length", - "1198" + "1192" ] }, "data": { @@ -85,11 +85,11 @@ "longitude": null, "p_code": "", "point": null, - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1514, + "geoname": null, "parent": null } }, @@ -107,11 +107,11 @@ "longitude": null, "p_code": "", "point": null, - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1515, + "geoname": null, "parent": null } }, @@ -129,11 +129,11 @@ "longitude": null, "p_code": "", "point": null, - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1516, + "geoname": null, "parent": null } } diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json index e7b54cb5f..5cb74403b 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'gis'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"c103699c1133008507285bf7143e88c9\"" + "\"7ca2df9eb73a74de5a1ae12f402b4375\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,7 +39,7 @@ ], "cache-key": [ "cache-key", - "c103699c1133008507285bf7143e88c9" + "7ca2df9eb73a74de5a1ae12f402b4375" ], "cache-hit": [ "cache-hit", @@ -59,7 +59,7 @@ ], "content-length": [ "Content-Length", - "1050" + "1044" ] }, "data": { @@ -81,11 +81,11 @@ "p_code": "", "point": null, "geom": null, - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1514, + "geoname": null, "parent": null }, { @@ -100,11 +100,11 @@ "p_code": "", "point": null, "geom": null, - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1515, + "geoname": null, "parent": null }, { @@ -119,11 +119,11 @@ "p_code": "", "point": null, "geom": null, - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1516, + "geoname": null, "parent": null } ] diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json index 372cff8e5..6fd9aaa37 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'latlng'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"3fef34d12dc9fa44cb6635d23439a315\"" + "\"777205cfe9fa00154eca3971b7ccb10e\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,7 +39,7 @@ ], "cache-key": [ "cache-key", - "3fef34d12dc9fa44cb6635d23439a315" + "777205cfe9fa00154eca3971b7ccb10e" ], "cache-hit": [ "cache-hit", diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json index 9356b7ea6..dd7aede4c 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'light'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"5f6de07e807b72340507bbe541bc9061\"" + "\"26cd3e770361ce343fd412b1cb5b8e9e\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,7 +39,7 @@ ], "cache-key": [ "cache-key", - "5f6de07e807b72340507bbe541bc9061" + "26cd3e770361ce343fd412b1cb5b8e9e" ], "cache-hit": [ "cache-hit", diff --git a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json index d680b5022..603c715e4 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_list/_api_latest_datamart_locations_/get/{'-serializer': 'std'}.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"b201e0867d7223b2928b0db88f06cb90\"" + "\"de8daea5fda4f852ad3f4fff999eae2c\"" ], "x-frame-options": [ "X-Frame-Options", @@ -39,7 +39,7 @@ ], "cache-key": [ "cache-key", - "b201e0867d7223b2928b0db88f06cb90" + "de8daea5fda4f852ad3f4fff999eae2c" ], "cache-hit": [ "cache-hit", @@ -59,7 +59,7 @@ ], "content-length": [ "Content-Length", - "876" + "921" ] }, "data": { @@ -71,47 +71,50 @@ "results": [ { "id": 1502, + "geonameid": null, "last_modify_date": "16 Sep 2019 18:29:12", "seen": null, "country_name": "bolivia", "area_code": "", "name": "name000", "p_code": "", - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1514, + "geoname": null, "parent": null }, { "id": 1503, + "geonameid": null, "last_modify_date": "16 Sep 2019 18:29:12", "seen": null, "country_name": "chad", "area_code": "", "name": "name001", "p_code": "", - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1515, + "geoname": null, "parent": null }, { "id": 1504, + "geonameid": null, "last_modify_date": "16 Sep 2019 18:29:12", "seen": null, "country_name": "lebanon", "area_code": "", "name": "name002", "p_code": "", - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1516, + "geoname": null, "parent": null } ] diff --git a/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json b/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json index ddec71e2e..84cb8a972 100644 --- a/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json +++ b/tests/api/interfaces/_api_checker/test_data/test_record/_api_latest_datamart_locations_1505_/get/None.response.json @@ -15,7 +15,7 @@ ], "etag": [ "ETag", - "\"db3891d8a2e812b90d8b2ffdbd8cb438\"" + "\"9fc96177d97286f9f0ad7e47f1947f41\"" ], "x-frame-options": [ "X-Frame-Options", @@ -47,22 +47,23 @@ ], "content-length": [ "Content-Length", - "264" + "279" ] }, "data": { "id": 1505, + "geonameid": null, "last_modify_date": "16 Sep 2019 18:29:18", "seen": null, "country_name": "bolivia", "area_code": "", "name": "name003", "p_code": "", - "geonameid": null, "created": "16 Sep 2019 18:29:02", "modified": "16 Sep 2019 18:29:02", "is_active": true, "gateway": 1517, + "geoname": null, "parent": null }, "content_type": null From 5d7ed1670c4357982072fc356317fb639234e8b8 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Tue, 21 Apr 2020 13:49:21 -0400 Subject: [PATCH 16/22] Correct replace for geonameid --- src/etools_datamart/api/endpoints/datamart/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/api/endpoints/datamart/location.py b/src/etools_datamart/api/endpoints/datamart/location.py index 4de6b805d..1731572a3 100644 --- a/src/etools_datamart/api/endpoints/datamart/location.py +++ b/src/etools_datamart/api/endpoints/datamart/location.py @@ -35,7 +35,7 @@ class Meta: def get_geonameid(self, obj): if obj.geoname: - return geoname.geoname_id + return obj.geoname.geoname_id return None From a2ac036c6eb4319f495883d60650c94f1ca2973f Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Tue, 21 Apr 2020 13:49:32 -0400 Subject: [PATCH 17/22] lint cleanup --- .../apps/mart/data/migrations/0117_auto_20200421_1544.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py b/src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py index 358184dc3..8e39bab17 100644 --- a/src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py +++ b/src/etools_datamart/apps/mart/data/migrations/0117_auto_20200421_1544.py @@ -1,7 +1,7 @@ # Generated by Django 2.2.9 on 2020-04-21 15:44 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): From 02895ade87b3f068458904d232ce0c18a29926b5 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 23 Apr 2020 16:42:20 -0400 Subject: [PATCH 18/22] Update/set geoname in update_centroid method --- .../apps/mart/data/models/location.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/etools_datamart/apps/mart/data/models/location.py b/src/etools_datamart/apps/mart/data/models/location.py index b396ee0e1..98ae257a6 100644 --- a/src/etools_datamart/apps/mart/data/models/location.py +++ b/src/etools_datamart/apps/mart/data/models/location.py @@ -50,6 +50,10 @@ def update_centroid(self): each.point = each.cent each.latitude = each.point.y each.longitude = each.point.x + each.geoname = GeoName.objects.get_or_add( + lat=each.point.y, + lng=each.point.x, + ) each.save() @@ -65,15 +69,6 @@ def load(self, **kwargs): finally: Location.objects.batch_update_centroid() - def get_geoname(self, record: LocationsLocation, values: dict, **kwargs): - if not record.latitude or not record.longitude: - return None - geoname = GeoName.objects.get_or_add( - lat=record.latitude, - lng=record.longitude, - ) - return geoname - class Location(EtoolsDataMartModel): name = models.CharField(max_length=254, db_index=True) @@ -111,7 +106,6 @@ class Options: # 'area_code': lambda loader, record: loader.context['country'].business_area_code, 'parent': '__self__', 'gateway': GatewayType, - 'geoname': '-', } def __str__(self): From ce06d9815ab2d7b808996dc1dad1ab44e1caa86d Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 23 Apr 2020 16:53:33 -0400 Subject: [PATCH 19/22] Make get_or_add geoname function more robust Geoname is set in the batch update centroid method as well --- .../apps/mart/data/models/location.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/etools_datamart/apps/mart/data/models/location.py b/src/etools_datamart/apps/mart/data/models/location.py index 98ae257a6..803abe103 100644 --- a/src/etools_datamart/apps/mart/data/models/location.py +++ b/src/etools_datamart/apps/mart/data/models/location.py @@ -44,6 +44,17 @@ def batch_update_centroid(self): with connection.cursor() as cursor: cursor.execute(sql) + # need to update geoname + for record in super().filter( + latitude__isnull=False, + longitude__isnull=False, + ).all(): + record.geoname = GeoName.objects.get_or_add( + lat=record.latitude, + lng=record.longitude, + ) + record.save() + def update_centroid(self): clone = self._chain() for each in clone.annotate(cent=Centroid('geom')): @@ -152,7 +163,13 @@ def get_or_add(self, lat, lng): data = {} for k, f in mapping: data[k] = geoname.find(f).text - geoname, __ = GeoName.objects.get_or_create(**data) + lat = data.pop("lat") + lng = data.pop("lng") + geoname, __ = GeoName.objects.get_or_create( + lat=lat, + lng=lng, + defaults=data, + ) return geoname From 1aea7b78a79fb26a3e56b37b4316d75fcc4b1ff5 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Thu, 23 Apr 2020 17:01:06 -0400 Subject: [PATCH 20/22] Only update geoname if changed Ignore invalid fields on geoname response --- src/etools_datamart/apps/mart/data/models/location.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/etools_datamart/apps/mart/data/models/location.py b/src/etools_datamart/apps/mart/data/models/location.py index 803abe103..2ce8c8460 100644 --- a/src/etools_datamart/apps/mart/data/models/location.py +++ b/src/etools_datamart/apps/mart/data/models/location.py @@ -49,11 +49,13 @@ def batch_update_centroid(self): latitude__isnull=False, longitude__isnull=False, ).all(): - record.geoname = GeoName.objects.get_or_add( + geoname = GeoName.objects.get_or_add( lat=record.latitude, lng=record.longitude, ) - record.save() + if record.geoname != geoname: + record.geoname = geoname + record.save() def update_centroid(self): clone = self._chain() @@ -162,7 +164,10 @@ def get_or_add(self, lat, lng): ] data = {} for k, f in mapping: - data[k] = geoname.find(f).text + try: + data[k] = geoname.find(f).text + except AttributeError: + return None lat = data.pop("lat") lng = data.pop("lng") geoname, __ = GeoName.objects.get_or_create( From f7cadfcd21c4b81b0c5b4e0399517f6332014f44 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 27 Apr 2020 16:00:11 -0400 Subject: [PATCH 21/22] Update changelog --- CHANGES | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES b/CHANGES index f82eb8628..214f6a279 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,10 @@ +2.14 +---- +* Add geonameid to locations endpoint +* Add pd_url to pd indicator endpoint +* Set budget values in process_country and not through mapping +* Fix InterventionBudget loader queryset + 2.13 ---- * Add new office endpoint From bbeb2cb0dcefd53a18610d578d3c62fa13b9be18 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Mon, 27 Apr 2020 16:01:00 -0400 Subject: [PATCH 22/22] Bump version to 2.14 --- docker/Makefile | 2 +- src/etools_datamart/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 184405668..f0a36325d 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=1 DOCKER_PASS?= DOCKER_USER?= -TARGET?=2.13.1b +TARGET?=2.14.1a PUSH_BASE?=1 BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" ) #BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" | echo "`xargs`xx" ) diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index c3bba0f54..7987fc66f 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,7 +1,7 @@ import warnings NAME = 'etools-datamart' -VERSION = __version__ = '2.13' +VERSION = __version__ = '2.14' __author__ = '' # UserWarning: The psycopg2 wheel package will be renamed from release 2.11;