From 0d4beafb9f19bc0636fd6884372c91632d9cb6ef Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 21:01:45 +0100 Subject: [PATCH 01/86] update version number --- CHANGES | 4 ++++ docker/Makefile | 6 +++--- src/etools_datamart/__init__.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index 25efa5264..30e97d8f4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +1.8 (dev) +--------- +* + 1.7 --- * WARNINGS: migration reset diff --git a/docker/Makefile b/docker/Makefile index 408ac0103..29994e6d1 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -38,9 +38,9 @@ build: -f docker/${DOCKERFILE} . docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:dev # flatten image - docker create ${DOCKER_IMAGE_NAME}:${TARGET} foo - docker export foo | docker import - unicef/datamart:${TARGET} - docker rm foo +# docker create --name foo ${DOCKER_IMAGE_NAME}:${TARGET} +# docker export foo | docker import - unicef/datamart:${TARGET} +# docker rm foo docker images | grep ${DOCKER_IMAGE_NAME} diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index d8a3c921b..5e0f2abe6 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = 'etools-datamart' -VERSION = __version__ = '1.7' +VERSION = __version__ = '1.8a0' __author__ = '' From fc9461dc790ace191431a0156380cd4c8e2d0006 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 21:17:50 +0100 Subject: [PATCH 02/86] add some predefined Schedules --- .../apps/init/management/commands/init-setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index 953465f85..7a4792585 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -181,6 +181,10 @@ def handle(self, *args, **options): if options['tasks'] or _all or options['refresh']: midnight, __ = CrontabSchedule.objects.get_or_create(minute=0, hour=0) + CrontabSchedule.objects.get_or_create(hour=[0, 6, 12, 18]) + CrontabSchedule.objects.get_or_create(hour=[0, 12]) + IntervalSchedule.objects.get_or_create(every=1, period=IntervalSchedule.HOURS) + every_minute, __ = IntervalSchedule.objects.get_or_create(every=1, period=IntervalSchedule.MINUTES) tasks = app.get_all_etls() From 21ad394f16e48f9a5d196f1dc66d7dd12838b2d2 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 22:11:58 +0100 Subject: [PATCH 03/86] add ability to create 'per model' custom templates for html/pdf renderers --- CHANGES | 3 +- Pipfile | 2 + Pipfile.lock | 97 +++++++++++++-------- src/etools_datamart/apps/etl/admin.py | 3 +- src/etools_datamart/config/settings.py | 9 +- src/unicef_rest_framework/renderers/html.py | 1 + src/unicef_rest_framework/renderers/pdf.py | 1 + 7 files changed, 77 insertions(+), 39 deletions(-) diff --git a/CHANGES b/CHANGES index 30e97d8f4..61f6cfe2a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ 1.8 (dev) --------- -* +* add ability to create 'per model' custom templates for html/pdf renderers + new `TEMPLATE_CACHE_URL` enironment variable 1.7 --- diff --git a/Pipfile b/Pipfile index b84c8130c..67df73f6d 100644 --- a/Pipfile +++ b/Pipfile @@ -56,6 +56,8 @@ django-celery-email = "*" "xhtml2pdf" = "*" pisa = "*" django-crispy-forms = "*" +django-adminactions = "*" +django-dbtemplates = "*" [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index 2b97db8c8..7246653d3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2c79b29b6f0397ba9724ed2ec50a286a3178b5b335f0a3b70ff3594a60dc9a9d" + "sha256": "e7ddab109d76e6e54b7c5fe951f72be61e8d0dfb8788995ffd384a763aa535a9" }, "pipfile-spec": 6, "requires": { @@ -759,6 +759,13 @@ "index": "pypi", "version": "==2.1.3" }, + "django-adminactions": { + "hashes": [ + "sha256:15934cad0ee1ef198f715dc4c21599bc6add72ca667ba4118c4266d382ddd0e3" + ], + "index": "pypi", + "version": "==1.5" + }, "django-adminfilters": { "hashes": [ "sha256:adab7f927b0845f9d87902ef3148ba704214e5977e5da483e386e82e445cb879" @@ -857,6 +864,14 @@ "index": "pypi", "version": "==0.5.0" }, + "django-dbtemplates": { + "hashes": [ + "sha256:9f63208bde2b3395b1adf8dfbf8f405a2cc3b082652faa16d32ecebc0e20d72b", + "sha256:d17d01264277c6a744604e1fae52678c238d0695c47f67a6982f49b65cd117d6" + ], + "index": "pypi", + "version": "==2.0" + }, "django-environ": { "hashes": [ "sha256:6c9d87660142608f63ec7d5ce5564c49b603ea8ff25da595fd6098f6dc82afde", @@ -1393,36 +1408,36 @@ }, "reportlab": { "hashes": [ - "sha256:0fcc951899c1bb6e0fa90996d5ed9d24265c5a8eafabf2303570e079d0290c28", - "sha256:234e1790858f1608d2fc8f820fa5660b7838cb14f12dec81e3c1156a4891ee1a", - "sha256:23df47e5a4225728a6d0a16b1dbc2674fe8a535603c191ad9d33f8755d1ab08b", - "sha256:277e920f1f5aa48531d34a25ba83e4e5a201e15a6a75263d529c8996a84233ae", - "sha256:2c52b314f29a40b8ed863190947e556b75a719566d4e2e7d25b8aeb18f60b58c", - "sha256:2e38e61d8e32b003a4c50f4bf1593bc5784395a62d180edd7b206f7dfd77836c", - "sha256:2f42e2e6177756a8265126fbeb7be58fbc40affd471253c829ab165c13253f2e", - "sha256:463f446a96bd0ddaba8ca46747f251868334c119f28bffd42fc0add2b3bd986f", - "sha256:4be2e78b900a601028b4ff42c547096cc204a21ac2a9ce7d16768195106ffd12", - "sha256:4db5daa83aa5c74fba41c3212549ff62abc84921b9ce848bc04cda728e00ecf0", - "sha256:4e4669e49a4cbb323c6a3cb45297535e910d4627f796dba90362e456fb013e44", - "sha256:551bed92c44e302ac7d663242cda10bb8e459ccff0178f51d5693f846336961d", - "sha256:60995c16fa961a6576a67aaeb7a6657a081dabdea298d4c15ec68423171dcdc6", - "sha256:6228323d6a40355c20b4159bc927cb93b9c39537bd39c25b9071eedcf80b4a36", - "sha256:62aafd7769fcf36c9910a332ccee002ce4ff850c9b3ff89bbea8d7b78be979c1", - "sha256:64bf9f0778477a69563db3ca53db3312b7a65756c60e674de54e7527a0852fa0", - "sha256:678667a846ad4181b5a392c59e1ec37e03abc684854d66c7a05df32113a4d337", - "sha256:722bc6e77526cbe927f9a65f37b06840e6d99d6938462892558c6cbb90445d4c", - "sha256:8c967cf193ebeca8231e3c0c28d94c9d54dd1ee38284785c3a1cbf3aca637fd9", - "sha256:95bc72a157559b5963967cf8be17d9185ed64b5c408504b9efbd51ddcf4c145e", - "sha256:9b22737cbaff0e7d875bcb61cb593df908513f1b807551c71afc3a9d1fe83aeb", - "sha256:a2f26770e8af0d383d586fd95446ee246d0532395c9392463c07047cb89862bb", - "sha256:a417205a64cbec93219a8d7e268fe4ba4b7f3e037f7d1ca42f432b4388fb93dd", - "sha256:a6856b4118abbc783ce419e50185e69bceb4d49de39d8439f28571d5949fc34c", - "sha256:b04381abcb3c14afcef7faababbd620b2a352a2f2765f3bf18deabb2dfc7d617", - "sha256:b2348ec5f615d50913c1eaa8c3fc71d4478e4ea0e627950f6fabd70d2a38ff1d", - "sha256:f86f13f4e6d8c815c35ec75e3d3a94cf7de12d544b4d038269c8b7c73de6b906", - "sha256:fa7761cca303d4dbc2f5c5989b2be4baa4068fa4137a5fdc03eb47bfc1af0a5d" - ], - "version": "==3.5.11" + "sha256:00c4d275b14ccd77316c0e6b5ad12881c70edc97556b1177d90fe366163df786", + "sha256:03f968b21c41ff3364665b3f08426233e8255313c158a50f71745e66accd6d90", + "sha256:0a6bb96cc4ca7faaa5bcfb599eadd4afe6b90929cdc31e4913be4899f9ae4ac6", + "sha256:1857bb48c67b9278d5ada84a9c584f6bd61bb95d4f67d55799e8af9c8a5ef6e2", + "sha256:1b52d4d5ba9f5a1e939eed9441ca62a2ef7063a2610d001c70b39760b9d3314c", + "sha256:2637ea9a2ca093f10f0fc27ea02a8bdde0faadbeab9205ef1f5bf560510c6d52", + "sha256:2d63984f9593e5408cfc3da043cae871ab20006d494d5029a04ec5fa0d3044d6", + "sha256:55945d811655c52515067826f4f2a258e472fc34ad995acdc9a5f53910b8ebf6", + "sha256:63f1928c47acd9aa81ef75dd29ce74d292e50d77eaf2629ad0c99bbc1d2fa125", + "sha256:65a70673bc7d673ca52ee8b1b14e37959b01d45ef43729ed1507b6718351ec47", + "sha256:6dd39a260fd8e315f55e5f61c0a4b07994c3cd06d4893aacc9575a0e8804ba12", + "sha256:786e0d5b166fcfa1d0044ee411f820c3314b0bbc513985b0adc227118c2db009", + "sha256:7e10261065d0f926d9d83fd1f2edb8bec466f3c60b3e927ef40e2262805c069d", + "sha256:8474ed2a5d89a1546435294c91ae42fc18a29c3172af4b5f97d43bbf4d0a9018", + "sha256:8ce32d0878a38bc1f8b6ee1656f638a7629b2067f9b048787d4ad5fbd409093d", + "sha256:8fd604e791367038b673082c6e1d1801ef0998a68b0d436bcc0befc592f99541", + "sha256:9c25661f5089863c6976c0525593cdd3f3afb59092fbaa4ea9eb83c4e95a8c3c", + "sha256:a5c58f0ca4a4dd74fc8816a5240891614355d909b8c606febfb24375330b2a3b", + "sha256:a6b9a411ff87bca1ce33c902c25d25f11ea53d4941ad5d41b070a3eb95843b09", + "sha256:c1f0104d0a85d0db9bc98d5fa3679391ef052219b0f3ff0b741d22b0b78deb5f", + "sha256:c8f59aaa6989e111b0bfb46daae91c02cc4f5c23ce053fbd82ea6dac1c4c087d", + "sha256:cdca0b57cb5efccd946f79470598706fff661190627cb213e6eb7749aaeb02eb", + "sha256:d37854d9bce188b336dbf6de598924656d2ff0555dc720d347970ea8df097895", + "sha256:d6ab0ff7f2c3cf9ef33c163f5c85e3ebb1ced512d6ef8cdd022617f5951b1385", + "sha256:e7a145f376fcf56d0697b028fbd228dd6defe78bdf671a6ec6a8a87a930d8b6c", + "sha256:e94bd2457a5030df103d2f1b892d322076cd8c4b8db6c4331382f35431eb4ece", + "sha256:ecd7026ec27e6b33513fe53338ca33bd44dce6d8b00f6ec0ab4bfa10614a4503", + "sha256:fd697bc9afdad1e05d245da169a8c98f576a5c7b3d835fee76922f3e8b2d0388" + ], + "version": "==3.5.12" }, "requests": { "hashes": [ @@ -1573,6 +1588,20 @@ ], "index": "pypi", "version": "==0.2.3" + }, + "xlrd": { + "hashes": [ + "sha256:83a1d2f1091078fb3f65876753b5302c5cfb6a41de64b9587b74cefa75157148", + "sha256:8a21885513e6d915fe33a8ee5fdfa675433b61405ba13e2a69e62ee36828d7e2" + ], + "version": "==1.1.0" + }, + "xlwt": { + "hashes": [ + "sha256:a082260524678ba48a297d922cc385f58278b8aa68741596a87de01a9c628b2e", + "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88" + ], + "version": "==1.3.0" } }, "develop": { @@ -1820,11 +1849,11 @@ }, "ipython": { "hashes": [ - "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435", - "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd" + "sha256:6a9496209b76463f1dec126ab928919aaf1f55b38beb9219af3fe202f6bbdd12", + "sha256:f69932b1e806b38a7818d9a1e918e5821b685715040b48e59c657b3c7961b742" ], "index": "pypi", - "version": "==7.1.1" + "version": "==7.2.0" }, "ipython-genutils": { "hashes": [ diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index 2130f383d..8a5e16022 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from admin_extra_urls.extras import action, link from admin_extra_urls.mixins import _confirm_action +from adminactions.mass_update import mass_update from django.contrib import admin, messages from django.contrib.admin import register from django.http import HttpResponseRedirect @@ -27,7 +28,7 @@ class EtlTaskAdmin(TruncateTableMixin, admin.ModelAdmin): 'table_name', 'content_type', ) date_hierarchy = 'last_run' - actions = None + actions = [mass_update, ] def scheduling(self, obj): opts = PeriodicTask._meta diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 25e761071..314b3478d 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -14,8 +14,8 @@ ETOOLS_DUMP_LOCATION=(str, str(PACKAGE_DIR / 'apps' / 'multitenant' / 'postgresql')), CACHE_URL=(str, "redis://127.0.0.1:6379/1"), - # API_CACHE_URL=(str, "redis://127.0.0.1:6379/2"), - API_CACHE_URL=(str, "locmemcache://"), + API_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=api"), + TEMPLATE_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), # CACHE_URL=(str, "dummycache://"), # API_CACHE_URL=(str, "dummycache://"), ABSOLUTE_BASE_URL=(str, 'http://localhost:8000'), @@ -189,7 +189,8 @@ CACHES = { 'default': env.cache(), - 'api': env.cache('API_CACHE_URL') + 'api': env.cache('API_CACHE_URL'), + 'dbtemplates': env.cache('TEMPLATE_CACHE_URL') } ROOT_URLCONF = 'etools_datamart.config.urls' @@ -206,6 +207,7 @@ 'loaders': [ 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', + 'dbtemplates.loader.Loader', ], 'context_processors': [ 'django.template.context_processors.debug', @@ -275,6 +277,7 @@ 'month_field', 'drf_querystringfilter', 'crispy_forms', + 'dbtemplates', 'drf_yasg', 'adminfilters', diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index 59627a407..70419b063 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -20,6 +20,7 @@ class HTMLRenderer(BaseRenderer): def get_template(self, meta): return loader.select_template([ f'renderers/html/{meta.app_label}/{meta.model_name}.html', + f'renderers/html/{meta.app_label}/html.html', 'renderers/html/html.html']) def render(self, data, accepted_media_type=None, renderer_context=None): diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index 416e61531..4b4727579 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -48,6 +48,7 @@ class PDFRenderer(HTMLRenderer): def get_template(self, meta): return loader.select_template([ f'renderers/pdf/{meta.app_label}/{meta.model_name}.html', + f'renderers/pdf/{meta.app_label}/pdf.html', 'renderers/pdf/pdf.html']) def render(self, data, accepted_media_type=None, renderer_context=None): From 07230814ed5a617c1f9cb1ee9e2a0b8e64544d54 Mon Sep 17 00:00:00 2001 From: sax Date: Sat, 1 Dec 2018 11:21:54 +0100 Subject: [PATCH 04/86] fixes cached (304 HTTP_NOT_MODIFIED) response in some renderers --- src/etools_datamart/config/settings.py | 2 +- src/unicef_rest_framework/renderers/csv.py | 3 +++ src/unicef_rest_framework/renderers/html.py | 3 +++ src/unicef_rest_framework/renderers/pdf.py | 3 +++ src/unicef_rest_framework/renderers/xls.py | 3 +++ tests/api/test_api_cache.py | 18 ++++++++++++++++++ 6 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 314b3478d..72486cebd 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -207,7 +207,7 @@ 'loaders': [ 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', - 'dbtemplates.loader.Loader', + # 'dbtemplates.loader.Loader', ], 'context_processors': [ 'django.template.context_processors.debug', diff --git a/src/unicef_rest_framework/renderers/csv.py b/src/unicef_rest_framework/renderers/csv.py index 73b03479b..33b389775 100644 --- a/src/unicef_rest_framework/renderers/csv.py +++ b/src/unicef_rest_framework/renderers/csv.py @@ -9,6 +9,9 @@ class CSVRenderer(r.CSVRenderer): def render(self, data, media_type=None, renderer_context=None, writer_opts=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: data = dict(data)['results'] return super().render(data, media_type, renderer_context or {}, writer_opts) diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index 70419b063..b5d72caa7 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -24,6 +24,9 @@ def get_template(self, meta): 'renderers/html/html.html']) def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: model = renderer_context['view'].queryset.model opts = model._meta diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index 4b4727579..70578d8ab 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -52,6 +52,9 @@ def get_template(self, meta): 'renderers/pdf/pdf.html']) def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) diff --git a/src/unicef_rest_framework/renderers/xls.py b/src/unicef_rest_framework/renderers/xls.py index a70a53918..699f07feb 100644 --- a/src/unicef_rest_framework/renderers/xls.py +++ b/src/unicef_rest_framework/renderers/xls.py @@ -9,6 +9,9 @@ class XLSXRenderer(_XLSXRenderer): def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: if not data['results']: return '' diff --git a/tests/api/test_api_cache.py b/tests/api/test_api_cache.py index 6b6f42272..33aad342f 100644 --- a/tests/api/test_api_cache.py +++ b/tests/api/test_api_cache.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import pytest + from etools_datamart.api.endpoints import PartnerViewSet @@ -40,3 +42,19 @@ def test_etag(client, admin_user, data_service, django_assert_num_queries): res = client.get(url, HTTP_X_SCHEMA="bolivia", HTTP_IF_NONE_MATCH=etag) assert res.status_code == 304 assert res['etag'] == etag + + +@pytest.mark.parametrize("fmt", ["pdf", "csv", "xlsx", "xhtml", "json", "ms-xml", "xml", "ms-json"]) +def test_cache_renderers(fmt, client, admin_user, data_service, django_assert_num_queries): + url = f"{data_service.endpoint}?country_name=bolivia&format={fmt}" + client.force_authenticate(admin_user) + + res = client.get(url, HTTP_IF_NONE_MATCH='Not Set') + assert res.status_code == 200 + assert res['cache-version'] == str(data_service.cache_version) + assert res['etag'] + + etag = res['etag'] + res = client.get(url, HTTP_X_SCHEMA="bolivia", HTTP_IF_NONE_MATCH=etag) + assert res.status_code == 304 + assert res['etag'] == etag From 68620d2e34644394f380f778e8049a5f56834392 Mon Sep 17 00:00:00 2001 From: sax Date: Sat, 1 Dec 2018 14:55:34 +0100 Subject: [PATCH 05/86] updates requirements --- CHANGES | 2 ++ Pipfile | 3 +-- Pipfile.lock | 17 ++--------------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/CHANGES b/CHANGES index 61f6cfe2a..ca83efd0a 100644 --- a/CHANGES +++ b/CHANGES @@ -2,6 +2,8 @@ --------- * add ability to create 'per model' custom templates for html/pdf renderers new `TEMPLATE_CACHE_URL` enironment variable +* fixes error in some renderers with cached response + 1.7 --- diff --git a/Pipfile b/Pipfile index 67df73f6d..ce4b72dda 100644 --- a/Pipfile +++ b/Pipfile @@ -54,10 +54,9 @@ django-basicauth = "*" django-post-office = "*" django-celery-email = "*" "xhtml2pdf" = "*" -pisa = "*" django-crispy-forms = "*" django-adminactions = "*" -django-dbtemplates = "*" +django-dbtemplates = {file = "https://github.com/jazzband/django-dbtemplates/archive/2.0.1.tar.gz"} [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index 7246653d3..abb804866 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e7ddab109d76e6e54b7c5fe951f72be61e8d0dfb8788995ffd384a763aa535a9" + "sha256": "81eac07ad1f97c779e00281dd19958dd2bb27a51bda821d288bf0d6174b1e6ee" }, "pipfile-spec": 6, "requires": { @@ -865,12 +865,7 @@ "version": "==0.5.0" }, "django-dbtemplates": { - "hashes": [ - "sha256:9f63208bde2b3395b1adf8dfbf8f405a2cc3b082652faa16d32ecebc0e20d72b", - "sha256:d17d01264277c6a744604e1fae52678c238d0695c47f67a6982f49b65cd117d6" - ], - "index": "pypi", - "version": "==2.0" + "file": "https://github.com/jazzband/django-dbtemplates/archive/2.0.1.tar.gz" }, "django-environ": { "hashes": [ @@ -1258,14 +1253,6 @@ ], "version": "==5.3.0" }, - "pisa": { - "hashes": [ - "sha256:94c4ae0995c84bb0588ece4480486464612ed1526f0987fb1016b9c50e5d3327", - "sha256:a7164ac81ab5ea01fbae4f29d2c00183a31142ca30ad527f6ac96635819cbd12" - ], - "index": "pypi", - "version": "==3.0.33" - }, "psutil": { "hashes": [ "sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736", From bc3d8a41b830ba627063c995005ff7374bee2711 Mon Sep 17 00:00:00 2001 From: sax Date: Sat, 1 Dec 2018 15:14:26 +0100 Subject: [PATCH 06/86] minor css/style --- src/etools_datamart/apps/web/static/style.css | 26 +++++++-- .../apps/web/static/style.css.map | 2 +- .../apps/web/static/style.scss | 58 +++++++++++++------ .../apps/web/templates/index.html | 11 +++- 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/src/etools_datamart/apps/web/static/style.css b/src/etools_datamart/apps/web/static/style.css index feeacaa12..a07198c6c 100644 --- a/src/etools_datamart/apps/web/static/style.css +++ b/src/etools_datamart/apps/web/static/style.css @@ -57,12 +57,26 @@ body, html { text-align: center; } .right h2 { font-size: 14pt; } - .right ul { - padding: 20px 0 0 65px; - list-style: none; - font-size: 18pt; } - .right a { - color: #2090F8; } + +ul.index-menu { + padding: 20px 0 0 65px; + list-style: none; + font-size: 18pt; } + ul.index-menu a { + color: #2090F8; + text-transform: none; + text-decoration: none; } + ul.index-menu li.menu-sep { + width: 100%; + font-size: 2px; } + +.h10 { + display: block; + height: 10px; } + +.h30 { + display: block; + height: 30px; } /* [ login more ]*/ .left { diff --git a/src/etools_datamart/apps/web/static/style.css.map b/src/etools_datamart/apps/web/static/style.css.map index 16c09e0aa..4fd629469 100644 --- a/src/etools_datamart/apps/web/static/style.css.map +++ b/src/etools_datamart/apps/web/static/style.css.map @@ -1,6 +1,6 @@ { "version": 3, -"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEjB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKlB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAGhB,QAAS;EACN,qBAAqB,EAAE,IAAI;EAC3B,kBAAkB,EAAE,IAAI;EACxB,aAAa,EAAE,IAAI;;AAIpB,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAQ;MACN,WAAW,EAAE,IAAI;IAEnB,0BAAK;MACH,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,+BAAU;MACR,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK", +"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;;AAInB,aAAc;EACZ,OAAO,EAAE,aAAa;EACtB,UAAU,EAAE,IAAI;EAChB,SAAS,EAAE,IAAI;EACf,eAAE;IACA,KAAK,EAAE,OAAO;IACd,cAAc,EAAE,IAAI;IACpB,eAAe,EAAE,IAAI;EAEvB,yBAAY;IACV,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,GAAG;;AAKlB,IAAK;EACH,OAAO,EAAE,KAAK;EACd,MAAM,EAAE,IAAI;;AAGd,IAAK;EACH,OAAO,EAAE,KAAK;EACd,MAAM,EAAE,IAAI;;AAcd,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,qBAAqB,EAAE,IAAI;EAC3B,kBAAkB,EAAE,IAAI;EACxB,aAAa,EAAE,IAAI;;AAInB,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAS;MACP,WAAW,EAAE,IAAI;IAEnB,0BAAM;MACJ,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,+BAAW;MACT,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK", "sources": ["style.scss"], "names": [], "file": "style.css" diff --git a/src/etools_datamart/apps/web/static/style.scss b/src/etools_datamart/apps/web/static/style.scss index 3ac8fdc1e..d17e4eee7 100644 --- a/src/etools_datamart/apps/web/static/style.scss +++ b/src/etools_datamart/apps/web/static/style.scss @@ -77,22 +77,45 @@ body, html { h2 { font-size: 14pt; } - //h1 { - // width: 100%; - // font-size: 24pt; - // text-align: center; - //} - ul { - padding: 20px 0 0 65px; - list-style: none; - font-size: 18pt; - } +} + +ul.index-menu { + padding: 20px 0 0 65px; + list-style: none; + font-size: 18pt; a { color: #2090F8; + text-transform: none; + text-decoration: none; } + li.menu-sep { + width: 100%; + font-size: 2px; + //border: 1px solid black; + } +} + +.h10 { + display: block; + height: 10px; +} +.h30 { + display: block; + height: 30px; } +//.pad4 { +// padding: 4px; +//} +// +//.pad6 { +// padding: 6px; +//} +//.pad10 { +// padding: 10px; +//} + /* [ login more ]*/ .left { @@ -230,10 +253,11 @@ input.input100 { color: white; } } + .rounded { - -webkit-border-radius: 20px; - -moz-border-radius: 20px; - border-radius: 20px; + -webkit-border-radius: 20px; + -moz-border-radius: 20px; + border-radius: 20px; } .monitor { @@ -257,13 +281,13 @@ input.input100 { .row { width: 100%; padding: 5px; - &.header{ + &.header { font-weight: bold; } - &.odd{ + &.odd { background-color: #eeeeee; } - &.even{ + &.even { } } @@ -273,7 +297,7 @@ input.input100 { &.task { width: 30%; } - &.last_run{ + &.last_run { width: 200px; } &.last_changes { diff --git a/src/etools_datamart/apps/web/templates/index.html b/src/etools_datamart/apps/web/templates/index.html index a0be8b868..3ac0bbbfa 100644 --- a/src/etools_datamart/apps/web/templates/index.html +++ b/src/etools_datamart/apps/web/templates/index.html @@ -1,9 +1,11 @@ {% extends 'base.html' %}{% load static datamart %} {% block right %}

eTools Datamart

-
    +
      {% if request.user.is_anonymous %}
    • API (needs token)
      • +
    •  
      • +
    •  
    • Login
      • {% else %}
    • API
      • @@ -11,12 +13,17 @@

      eTools Datamart

    • Swagger
    • Monitor
      • {% if request.user.is_staff %} +
    •  
    • Admin
      • {% endif %} {% if request.user.is_superuser %} +
    •  
      • +
    • Flower
      • +
    • RabbitMQ
    • System Info
      • {% endif %} -
       
      +
    •  
      • +
    •  
    • Logout
    • Disconnect
      • From 24324ccd56951e0851d54ab90533c2c50e39825c Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 3 Dec 2018 22:35:55 +0100 Subject: [PATCH 07/86] bug fixing --- CHANGES | 1 + Makefile | 17 +-- src/etools_datamart/api/endpoints/common.py | 12 +- .../api/endpoints/datamart/pmpindicators.py | 2 +- .../api/endpoints/system/__init__.py | 2 +- .../api/endpoints/system/task_log.py | 2 +- src/etools_datamart/api/urls.py | 2 +- src/etools_datamart/apps/data/admin.py | 1 + src/etools_datamart/apps/etl/admin.py | 39 +++--- src/etools_datamart/apps/etl/lock.py | 8 +- src/etools_datamart/apps/etl/models.py | 9 +- src/etools_datamart/apps/etl/results.py | 2 +- src/etools_datamart/apps/etl/tasks/etl.py | 6 +- .../multitenant/postgresql/public.sqldump | Bin 4048686 -> 4047913 bytes .../apps/multitenant/postgresql/tenant.sql | 2 +- src/etools_datamart/celery.py | 45 +++++-- src/etools_datamart/config/settings.py | 1 + src/unicef_rest_framework/models/service.py | 3 +- src/unicef_rest_framework/renderers/csv.py | 5 +- src/unicef_rest_framework/renderers/html.py | 10 +- src/unicef_rest_framework/renderers/pdf.py | 2 +- src/unicef_rest_framework/renderers/xls.py | 6 +- src/unicef_rest_framework/views.py | 3 + tests/_test_lib/test_utilities/factories.py | 2 +- tests/api/test_api_cache.py | 2 +- tests/api/test_api_data.py | 117 ++++++++++++------ tests/etl/test_etl_tasklog.py | 8 ++ tests/test_subscription.py | 6 +- tests/unicef_security/test_azure.py | 10 +- .../vcr_cassettes/test_user_data.yml | 53 ++++++-- 30 files changed, 258 insertions(+), 120 deletions(-) diff --git a/CHANGES b/CHANGES index ca83efd0a..33022d661 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,7 @@ * add ability to create 'per model' custom templates for html/pdf renderers new `TEMPLATE_CACHE_URL` enironment variable * fixes error in some renderers with cached response +* Adopting of RabbitMQ to prevent message loss 1.7 diff --git a/Makefile b/Makefile index ddf55f8bd..2acaf02f7 100644 --- a/Makefile +++ b/Makefile @@ -59,14 +59,17 @@ urf: pipenv run pytest tests/urf --cov-config tests/urf/.coveragerc -demo: +stack: PYTHONPATH=./src pipenv run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid & PYTHONPATH=./src pipenv run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid & +# PYTHONPATH=./src pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & + pipenv run docker run -d -p 5555:5555 -e CELERY_BROKER_URL=${CELERY_BROKER_URL} --name datamart-flower --rm saxix/flower + +demo: stack PYTHONPATH=./src pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & - pipenv run docker run -d -p 5555:5555 -e CELERY_BROKER_URL=$CELERY_BROKER_URL --name datamart-flower --rm saxix/flower -stop-demo: - - kill `cat gunicorn.pid` - - kill `cat beat.pid` - - kill `cat celery.pid` - - docker stop datamart-flower +demo-stop: + -kill `cat gunicorn.pid` + -kill `cat beat.pid` + -kill `cat celery.pid` + -docker stop datamart-flower diff --git a/src/etools_datamart/api/endpoints/common.py b/src/etools_datamart/api/endpoints/common.py index 253e2e8e4..b29796e95 100644 --- a/src/etools_datamart/api/endpoints/common.py +++ b/src/etools_datamart/api/endpoints/common.py @@ -51,8 +51,12 @@ class UpdatesMixin: def updates(self, request, version): """ Returns only records changed from last ETL task""" task = EtlTask.objects.get_for_model(self.queryset.model) - offset = task.last_changes.strftime('%Y-%m-%d %H:%M') - queryset = self.queryset.filter(last_modify_date__gte=offset) + if task.last_changes: + offset = task.last_changes.strftime('%Y-%m-%d %H:%M') + queryset = self.queryset.filter(last_modify_date__gte=offset) + else: + offset = 'none' + queryset = self.queryset.all() serializer = self.get_serializer(queryset, many=True) return Response(serializer.data, @@ -79,7 +83,9 @@ def get_schema_fields(self): return ret def drf_ignore_filter(self, request, field): - return field in ['+serializer', 'cursor', '+fields', + return field in [self.serializer_field_param, + self.dynamic_fields_param, + 'cursor', 'ordering', 'page_size', 'format', ] def handle_exception(self, exc): diff --git a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py index 3289b3ec4..19e84c7dc 100644 --- a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py +++ b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py @@ -13,4 +13,4 @@ class PMPIndicatorsViewSet(common.DataMartViewSet): queryset = models.PMPIndicators.objects.all() filter_fields = ('country_name', 'business_area_code', 'vendor_number', 'partner_name', 'partner_type', 'last_modify_date', - 'pd_ssfa_ref', ) + 'cash_contribution', 'pd_ssfa_status', 'pd_ssfa_ref', ) diff --git a/src/etools_datamart/api/endpoints/system/__init__.py b/src/etools_datamart/api/endpoints/system/__init__.py index f4b1a17ab..cfa7965eb 100644 --- a/src/etools_datamart/api/endpoints/system/__init__.py +++ b/src/etools_datamart/api/endpoints/system/__init__.py @@ -1,2 +1,2 @@ # -*- coding: utf-8 -*- -from .task_log import TaskLogViewSet # noqa +from .task_log import MonitorViewSet # noqa diff --git a/src/etools_datamart/api/endpoints/system/task_log.py b/src/etools_datamart/api/endpoints/system/task_log.py index 25306b949..87baac819 100644 --- a/src/etools_datamart/api/endpoints/system/task_log.py +++ b/src/etools_datamart/api/endpoints/system/task_log.py @@ -5,7 +5,7 @@ from .. import common -class TaskLogViewSet(common.APIReadOnlyModelViewSet): +class MonitorViewSet(common.APIReadOnlyModelViewSet): """ """ diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index 2a2a498dc..c72b02955 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -32,7 +32,7 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'datamart/user-stats', endpoints.UserStatsViewSet) router.register(r'datamart/hact', endpoints.HACTViewSet) -router.register(r'system/tasks-log', endpoints.TaskLogViewSet) +router.register(r'system/monitor', endpoints.MonitorViewSet) # urlpatterns = router.urls diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index 445a83e74..6cc15b9ce 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -112,6 +112,7 @@ class PMPIndicatorsAdmin(DataModelAdmin, TruncateTableMixin): list_display = ('country_name', 'partner_name', 'partner_type', 'business_area_code') list_filter = (SchemaFilter, ('partner_type', AllValuesComboFilter), + ('pd_ssfa_status', AllValuesComboFilter), ) search_fields = ('partner_name',) date_hierarchy = 'pd_ssfa_creation_date' diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index 8a5e16022..e455c68c8 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -2,6 +2,7 @@ from admin_extra_urls.extras import action, link from admin_extra_urls.mixins import _confirm_action from adminactions.mass_update import mass_update +from crashlog.middleware import process_exception from django.contrib import admin, messages from django.contrib.admin import register from django.http import HttpResponseRedirect @@ -20,13 +21,15 @@ @register(models.EtlTask) class EtlTaskAdmin(TruncateTableMixin, admin.ModelAdmin): list_display = ('task', 'last_run', 'status', 'time', - 'last_success', 'last_failure', 'lock', 'scheduling', 'queue_task') - - readonly_fields = ('task', 'last_run', - 'last_success', 'last_failure', 'last_changes', - 'results', 'elapsed', 'time', 'status', - 'table_name', 'content_type', - ) + 'last_success', 'last_failure', 'locked', 'scheduling', + 'unlock_task', + 'queue_task') + + # readonly_fields = ('task', 'last_run', + # 'last_success', 'last_failure', 'last_changes', + # 'results', 'elapsed', 'time', 'status', + # 'table_name', 'content_type', + # ) date_hierarchy = 'last_run' actions = [mass_update, ] @@ -52,6 +55,14 @@ def queue_task(self, obj): queue_task.verbse_name = 'queue' + def unlock_task(self, obj): + opts = self.model._meta + url = reverse('admin:%s_%s_unlock' % (opts.app_label, + opts.model_name), args=[obj.id]) + return format_html(f'unlock') + + queue_task.verbse_name = 'unlock' + def has_add_permission(self, request): return False @@ -61,10 +72,10 @@ def has_delete_permission(self, request, obj=None): def time(self, obj): return naturaldelta(obj.elapsed) - def lock(self, obj): - return f"{obj.task}-lock" in cache + def locked(self, obj): + return obj.lock_key not in cache - lock.boolean = True + locked.boolean = True def changeform_view(self, request, object_id=None, form_url='', extra_context=None): if request.method == 'POST': @@ -77,22 +88,22 @@ def changeform_view(self, request, object_id=None, form_url='', extra_context=No def queue(self, request, pk): obj = self.get_object(request, pk) try: - task = app.tasks[obj.task] + task = app.tasks.get(obj.task) task.delay() self.message_user(request, f"Task '{obj.task}' queued", messages.SUCCESS) except Exception as e: # pragma: no cover + process_exception(e) self.message_user(request, f"Cannot queue '{obj.task}': {e}", messages.ERROR) return HttpResponseRedirect(reverse("admin:etl_etltask_changelist")) @action() def unlock(self, request, pk): obj = self.get_object(request, pk) - key = f"{obj.task}-lock" def _action(request): - cache.delete(key) + cache.delete(obj.lock_key) - return _confirm_action(self, request, _action, f"Continuing will unlock selected task. ({key})", + return _confirm_action(self, request, _action, f"Continuing will unlock selected task. ({obj.task})", "Successfully executed", ) @link() diff --git a/src/etools_datamart/apps/etl/lock.py b/src/etools_datamart/apps/etl/lock.py index 9f09dbafd..964e62ea3 100644 --- a/src/etools_datamart/apps/etl/lock.py +++ b/src/etools_datamart/apps/etl/lock.py @@ -5,7 +5,7 @@ from django.core.cache import caches from redis.exceptions import LockError -cache = caches['default'] +cache = caches['lock'] logger = logging.getLogger(__name__) @@ -16,7 +16,7 @@ class TaskExecutionOverlap(Exception): pass -def only_one(function=None, key="", timeout=None): +def only_one(function, key, timeout=None): """Enforce only one celery task at a time.""" def _unlock(key): @@ -40,8 +40,7 @@ def _caller(*args, **kwargs): have_lock = lock.acquire(blocking=False) if have_lock: ret_value = run_func(*args, **kwargs) - # else: - # raise TaskExecutionOverlap(key) + finally: if have_lock: try: @@ -52,5 +51,6 @@ def _caller(*args, **kwargs): return ret_value return _caller + function.unlock = partial(_unlock, key) return _dec(function) if function is not None else _dec diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index c950d129e..afd0dbebd 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -11,7 +11,10 @@ class TaskLogManager(models.Manager): def get_for_model(self, model: DataMartModel): - return self.get(content_type=ContentType.objects.get_for_model(model)) + try: + return self.get(content_type=ContentType.objects.get_for_model(model)) + except EtlTask.DoesNotExist: + raise EtlTask.DoesNotExist(f"EtlTask for model '{model.__name__}' does not exists") def get_for_task(self, task: ETLTask): return self.get_or_create(task=task.name, @@ -56,6 +59,10 @@ class Meta: def __str__(self): return f"{self.task} {self.status}" + @cached_property + def lock_key(self): + return f"{self.task}-lock" + @cached_property def verbose_name(self): return self.content_type.model_class()._meta.verbose_name diff --git a/src/etools_datamart/apps/etl/results.py b/src/etools_datamart/apps/etl/results.py index 5a4c8a11f..53ed5b34c 100644 --- a/src/etools_datamart/apps/etl/results.py +++ b/src/etools_datamart/apps/etl/results.py @@ -8,7 +8,7 @@ class EtlResult: __slots__ = [CREATED, UPDATED, UNCHANGED] - def __init__(self, updated=0, created=0, unchanged=0): + def __init__(self, updated=0, created=0, unchanged=0, **kwargs): self.created = created self.updated = updated self.unchanged = unchanged diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index d45bdc0fc..bc0733078 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -47,7 +47,7 @@ def process(Model, filters, values): else: op = UNCHANGED return op - except Exception as e: + except Exception as e: # pragma: no cover logging.exception(e) process_exception(e) raise @@ -350,8 +350,8 @@ def load_fam_indicator() -> EtlResult: return results -@app.etl(UserStats) -def load_user_report() -> EtlResult: +@app.etl(UserStats, bind=True) +def load_user_report(): connection = connections['etools'] countries = connection.get_tenants() today = date.today() diff --git a/src/etools_datamart/apps/multitenant/postgresql/public.sqldump b/src/etools_datamart/apps/multitenant/postgresql/public.sqldump index 0a761fec4ed2e11c36add887720aac29a7ea9f40..0ec2fa00419d218e71c19af840b9c9ae5543f83f 100644 GIT binary patch delta 40900 zcmaJ~2Y6IP*WQ_XlRy$eHa#Sp4WXBi?S+mYRUv?)NLQ>N#olQ4LchvbXv!~ALXj0m z1t}I3rNpkFG|OkB*+4<~-xy+PPge9e=uA z?R>VW0`p|myvdKS1G8wV@eHvW+2lUDpeizt^dOYsRgUosUF^KB1tjHv672DZr9!Ve zO(Gd!O@E5E)f zO!MYvEUs{c-L=vhuh8U;?8#_D95*XO0>&hJ6JxJ4%Q!T@$T*SdFdU_Wjq2Tn_wMs7kXXFP3##+KJ|}Hf>3fvCJ`pO}!4F{Q3!?09#y&w*GTQ z!^i_ADa3dpE#YODo#;ecMD{xEgt%k_St8TVX#739>PBie7Np0e1MWy!Sp_lHXC(aQ zi%e{Clo)?E%ubJeX3%YGWK3?7XN+#15_zVXPK@sM0=iTx{;+v9>(T*=hh@c;1ig_D zTC5?)2iXa~g;|%WTCOpvb+WO()mh_wP62&hs@T{19oDELRM5P5njyC>%NSFh8~Luy z2x3GE;)+5cHg~#KU~DN*j!Y@1#JJc|Kwqe0?jlmao|>*@i#Nsx?bfq9Z->(F3gghQ zi+$9X78&nU{9%+8C;aX;+Iq@ZtD8uhvGc}4{5?pN;-q}Ktctzdh5C*D zo{j8-PEZ>tNr3w!9&a%<9(UIK=5cu<t!|H z>T&sOWo*n!(#-f{OlIV8a0@X8Hi^H+NZZPMUEMy#m^mstaxUDEP`6h$=NDxecIfag z%bcp&jp5A_fF4^D0nCd`>)4qXRV@-;`E9M`E4Fnt$rnnEy)%l8ZCzR$UHT3-uD`!f z{@rG5fVhn@^YZC(k<7B;T?qT>eq{g7RtdU-HYeLYhn7Xwb-i9Q*0qVh^7w4c7Z?I`> zT=4ijkx>J~#8?uJtMvKNiN*}fi|iScMvMj>3Rv|lP0kcgbfZq=s#8Tqh+A- zm!%mKF1yhPb!-~zgC1YdW;bRG$%zzSelaoL>=e(C-y4}W#6gU9T?*)Gbc4~fz}R!e z4r6%Nc!vD`$lF)8Cq`b6I1atP=&E93tnHcbo6ndvyui45*zd$6+kF)jK0ykM`NLmf z4Gff0{odIPqvL`n5U>?sBs3b2DUdDqg%K-=v9wQuK-lIq1)PzYqcVt5uU`TCb+-I$ zOC&uXE1@vXJ9pCl|JJlVva>a<6Lif7dHU|ow*U4-N7$&9(025O1c;B7RB0uwd>cuP%%AoRG5U;)r{xY~;1X%EDcguWvUx^RWHBg5 z!dHaOWf>0;_QNXZe0Y2u<#F4Jjm$@!k&zF!AV%55cxoP>k?}~Y$gYQSQLvK==th~b zhL8MW44Yij0=%ApEzf8?w=mM<(JQI(-Ys#>UJpi!8Sl_0k%NynCx+wpgjarB3;D`$ z%}R+3m{~%oU)oFbRhh_hvre(sUqCZxb7wq1-e6?=?5@Pvd{_K83^~rZ-H4HVPXSvs zPfKNAT|qOArgNLIc8`%FHm)yht;|{Z?9fIn#aKTt*~pwf-e_=dP5bxyFfh(8HcFpN ziR_tQpBQPk70}m2g&t+cS0e*o-WSic&u1%U^WG!QNV}(sh;hlZgjYe#C-T*!3mo_u zv)EscmRKfR`X0$NerF%DU$>zAC(cMfhHVAxZS zee>fL8}vkSS1h2$@h9V%3WkhxtMiRRt4`8@SwmYS_WbIN?8Rz)RG$TL;gBoxZB>b8 z?0q`nw_wD%K7){eC=oVkr8&n9dwCNZzYUG$nP=lQ6bhqXOx{=+8NK0YVx%sNE5XWO z;>IS@G#4v}Q$SM7+3yZO;7f@ z#8@@sQ&P-kjlhuf)0@9h85buq>Z>2;^wnN#E%hBzaIcW`l>xl8gOL28r zwB|GNIcXGm>}@pc)vMx~VEdHFSH{{M^<+VefA??WqtylUZJ9G;$JfRyRdH2fjA0+` z*9s%Azc&EwWPMFx`e4{(?k+Z#evlmLy|aWEEjGlH@%n8|1+1|#aaVF=%?AZ?AzGSG z->G6BU4hx`)Q3N_hSjLJ(~$&?!ARzAbhq-C3+TIIBq%T@?D@>-^J+Xl-Y_f<@<~qQ z(Y-@8W9@5kurC-n|1k!Dkh!4R5j8scf4YNB{~QUm+7!>859KYiu=U4jy~vo){M1>rUyv-L;N5r*V8rAZ&mS&~Tz_CYF{Zy4Pa^0s zmK<(kJlo7%{4PCN&p3ZL7bP{`i2dF1u%Eikh4@ZcitNP2w4PCQco&;>6lQ0=o$-W% zK1}fB6wPLtn{_?1{RrC2bsr=o7c{ni-#9YoJD3>nd|1eySZJDu?fTJlW6$?Hj6=Jc z#mqrp5FJk_D~ufcp^_Mv?ahu2dcF`l_&+UgK~I_yne<~lqPfj=FRNN)b~lm55@Y92 zud*eFv@GNB$MGzLe35lOkETYm&k}wM8k3I|%Tf?)?4>8`v+1k!WaFWuTkz>-J?)AX zaXLHHlQv|>ug3tE*``rk*wX^*we$7nA4@S#AM3>K{tacb;EQwP0W4D9^(Rw|XPRXj&-q0*_KJcRmep>NI9t~XtE)GDTg9gSfRFlXfBXZ& z{>YQR_oha>0|~!{jLxTujeTF3j)UoE8nPw5X^JuQuh*i><~`9*dWW4p2otpBp&GE? zURY&bvA5)KDYIs6xQ-4($1KErr7BG&lc)39VO*UF?+w> z1+#a%Gas|+zc3nCewUy*6sb77ni$J}h%0rwZ3Tw=Y)&NOobdXXU9LLX<$9gp!M;3# z{8jvzfbgQB9VaAzyNzw8+?!EGn)Z zn|@qNX9FlE-hU?9MeUite6EqsZ0+C3dBdX#ibK4+PMQ&;*YWs|9yi;bjOlqv68RIw zDk|$^nO62Cw1DkPCd=4><|y@cCmpe+li%azRVkz;G4A+1K~aDmZ$%4vT79AsC5==Y+Uc6f#qV|V`_c*ABUsnMfA*=& z(541t3(F@MV>bRBSMK%Nve~>eEst-@AZV8MvkC74NLajMnFDAF$D{*Cj$76aJG$O< z0UouJYIfilwC?*i0U5#!bTWq&^VDqe9(k&S#6Q&MVFU9>b2Q9$Y?7TcV3oP#Q{p!3 z>2sOYhX=r~C{~Mm$C~wYB7-Cgxh%FTkKDli)HFNWOA?j(_%->YD`D{5BqSB&+Y3km zVK*d^c;@_KnmzOfKDl8dQjf4FlSu*nLatp7vu&49JDZ)7pv3Rt-HJ(?C`?hiUqTS8 zMK;wzUSY4-M@BxamniMyV@k-a8p}vcNHXBEHD-+qv_iJ@l$OMfm62p#P)4!{8<2*? zzmi|wb`WjOrZyp4*`?{EW{&a)JSMg&pWhUYX7+eSqGG?TrBK|IO=?b3*v~^qHup6r z4G4wNFFb+!r0R(+$Ukg!8ZtXIGZ7s`XLgnoCx5*q*+$r&EK)PZ`h$$LC9V0k)+Cn@ zk6AtYrM5!$l;M zjSZ8XtW#4G@1(&A-dTo z#OCxQi`k#9giO1|+8dcJIXX9*e$CJ9pPNVk>uyoWN2F2MiBv$T*uDM9n{4TMr2a=RppZ7>qARM0V@AM`dE0z+JORt0y z@MA*18YSS|E(ed5shGsEa52Vjmt|1MV!yC&#-I`SWh3~XERt?lqoZE?@6Jm?kni*d?oE8eB??raPkKBKDNg^h|y7_1k zKX3!43D##o;;S&5HiDKJ?K6c7u81AD39Bx#Hizq7{LdhusOTR23v7nDHxtU}VwhGfg%-RS9gM9Y{ z0>|W``wLj3P3E*+e-hcm&Rm($HDPVr+4BLK!|P8b-%z&vsstS&KU)%@ZFt3P#6?)& z5eXTBcd)I%wPbVdBq@CD?J#VxtQH;d7g+}#M^U%&M3*8~c_(>;eVv2KSUoyHLD&Vq z2^8e=%DYH1F%Camz}mct?+%Zqtt4>s-DExsU6atJ!yZ269&$frcVCx~%W#l8reM-y zE3QYUIBK?Wch`TW2Pv|Q#IhcW!$ zEoPY&K0rQUT}MLG#pC0ef*59t9wdV(n{ji3Cbt)piTyE>Yy41<$u>Mpeq-wa9o-s1uW+=%pAF+knU%b5;cT)`s2bb zL1AMeJt6IqU2NjzXr4dZnvfNYt9CYK76$N(W|H%4)a?mMJU%{ZmY8imzXK_rjLtS| zXJfW08I2T+?n)F6@MUwzJi`_DK{MSO;d`}AXSzZ8>TIg(`K!j}B3m(IeTDU@tCV>}_orVhWFdwy&j1y}tcJMx^&a{?Ar*3Wr zHz%$pcHVY1nr?eI_RKcpcGr&j+ZC$Z@J=JP*@RbDk@nCMs?k!LR+6pI5(~tNxFo+} zE$K|y2QT4+a}*daKe$eqUd>mMB4@5b5MD14SmegbBp-TLK|D_(?gz~VfkgiI28<}| zmMUZ;Um+gg-CrRMNhhx>_T`@1e3k+QJZLMzcq2lDe8oo6oUrBVkiS9&B7{&4!ho#J zYLd$)Y=Nh?%NwM=hy)&I7rhK{BL!H*84#%};LpD%M($M`AYW`PkB`cg;X9u32FW4R z8)hZl(bF6Xh?@`FM7)Fz+DMAZN)&>yrSa(^(w5J^-Hbj*eLiE!m_nyh0rFziDZ*QP z=oZq7wD;AtT3_u}TdL3^!tOFih9FuR|86Tf0QH3o*SJDw69vp~bBZV#e{ehT5LUJc zO{1woJiw2?g*6m~5juA>DReed$coSr!emYPWABhoB;=2c;BJ5I5xluVONgI;S1fDk z7G$l3LNIK&m*%telk_z9!fTk_pV>}YvTYM#U(J0V3(n~~h?iB~q9?On?~~`+`)hi& zzjm)~X`P7A1%pTgwRmVJT6PG|i0BC8uiZmhDYUqmYm=5QB3hYz%Ll@X@#x z=|$L8yCLXT2!{ByPle!HdmtE82!{Fj&j^Sw$<-K4Le_%PcP0WjEOI`U`=#HIbT;=F zSnQ*|BzCs#8#F`wZ`%Al$!z6%(vVF!2zJG~_jASj!8GJ$Gj?i)&afgCvB*IxrM$sc z#E&41$H>@F=xnb*crfB41jFa;!-(G=uAFGiglZclW7vsyi_+Lm=M1su@@8M5Hz{}YCr%WF(N zQt0fhK#10acm#juN8t)^`HfjW6gvAT03v`R!V&!JPhtrIj!7(|wT)m zf-E_`7pCZb3RypY?pF+Uf+9EK_d@3-3Q-Ys9mw^3t{)?LgzftQef&~|d=T<*`(r(I z9P$*tKxbcAqx}^S7&NB8|Em+|HG%+l9uP;R0utsOeuL3P5kX1UF|!V|mc>8|dd(kN zE`RoSm@w?>U-1Ql6!M@4)wtVrp8Th93N$(n&|n3~1NS3b>CM=_dep_X6>AOI^ri?o zOgn{9t@H;{!lwU;X|}p4Ni)6e)9b@wfBZB?xR~5v>hZJ58TgvZ6bX63C6r$Gwx?l( zwD;6B7EkTQa=Ai_57x#Kr`D9e`8TW$>It*9`AB$(0^$cm_{RB_XM}m)_ji)o&Rd&J zeT70nfbTsk)=QqhFrQqh5DaqfInsi#k555xs6sF#{LGEmw$@q-f8{)BN7%@}QBPMX zAR-(F&uk+eB(w=(|NH~_Vb<~?m+8rE%wMI{M_BPLy~sISArBsv>G?ghS5M{_>9nP= zYFWPkN_T_;C;ZC73Cg!5(NdVTG2_Bh$E+Qx(BefMK}%EKDTQ{z6n`FHH%cMz6RlRb zUQ6rKbdILm6~?NI?H+<2J6eGgZ2=zFrfl)2B!|yRrL71X;nC8{Yqy816-ry3VHKvvb|M z*Bu!U9H$WU@?UZ(#?GLd9SZ1$&hZLaAI47Mh~$0qDaOvX8qy-?%?f#}ou#*suPLBd zJb#>pyi8EY2l%N%w1H4CW^;LhwQa763V4wBDgvAagY4iKT3GC_!h%xn*%AM^>TI21 zYl`Wh@~AOJpRX_GS^O8YfO%rOU05Fn9n>ajgwd?jC@yS;3$gbcbaY!+)Zn6RV~7qG z2(Q=JlTpkjmC#C{%oQJfEG?Q&Co5El9W9~P0$~;!ZJrfHbg@AAylk42jsn7*E$9JX z6w%cR5jH+5$Yaz=E7`eXnp$>!iCJjfEdc1C*ncs(P1MSw$L=jP0T){U0zPA6lR|b` zV>*y+8BbH1(uS`@A-yah=01h-ZCvKkiSk#YfWH3?0O8yiHL#EkZi(`Bd!nWFQ4Dpd z1uEcX=ga8T#-g4n&1u{1(KH79x3n)tlwvEI(1FH;_v$sLkH#-TrD1-h7 zG(d0O9Zhbq6*R_hVqXH)7Az*|siiHyhypLS0D_ga?&n-Kt{JUt=Z>|5GcnW^R;U;Q ziT&G5CTFflXxB5*HtbIZ)Ky50g%L%(5*TGK)E z&!Um)-J6)08?7)gat#Z&p@W`kK=AFGrbI#GEI=?|cpG|k8+Y{EPcDlB##;e`OmJt| zh_-0Yosu*pGWO!8X3i#BVEjG?@=GcEM_@Xi)bKSg?QCUYCR<=?=q+qfIR^Gv$3D3` zio4Yc7vrw5j_v3mHVO=-rgVFC6m`1=D(GRLr!)rxMfB22iX1NgG>WQx@*j@krdZ%S z7tk8mU^iwvpScy0J{|*4vjCf{gqvA;d(oKg|0$Z*bSv=y15=?no9jWJ^+l(l++U-p z2P{w*@(5&gxA8?s|A%7Whb+K;8L}VS#vlEpjz^=ZJz@a}$e{fAa=+S}^tpKq&yBxV#MwX6bzDjLv_L5l^ zC^G^<6Z3}*tztLv=IlBfhyRrC zu(5YXPTfttf615Fh`RziP2KF+hqN%ZC3Z_r-HorOg}70kE4FWdy~sMy`+>t;vYQr) zZE<~zirI*c^f|d}l>Q@?V`u0}xgnILMI`gW$Y|W0Ep88dd1pFDZt$dMrQ#2pJK1A> z@S>m#eOPYLr2ke48pS7a`yxFj6=F-`>vB6HJum+sJJB{{*O{H#*2MOafroq%eGZ$+ z?Ci{|g0J-i_IS)6p3PQugZ6v6(ru1oc+Pd$@~ev zjIDiXZ%L`8sj+--k#tv@CjTGQ)isi}%Dh{(9I??G3{zRw-C5~TES$R5CrN^gn!<)% zO8bC%YR66`Gut0tN?(>NQ<@=lfgc*JrET57ijtYA{2i>@6UV_{C+_xNL6;3Y(q%x*aV8 zo09fPN*OJb05HAYlO!+NNd7-4Sf2{^m7NxuwGD>Vn{i$hEtXOsLCM^8V5P%QJAzfk zx(-8ebQq4}5bP-Gkb)o`tvC5bo%W+r1*uvvgf_>?LDVS)LFXx8(?+6L&RmJSPS|07 zAE-A+B;$rQmW_RnVA0rVnVB~*V@k{pG;$1mOmbwLx2`mY2%9|k04$4uT!8sN6U44G zuVh(NTn)!WU2etg)=+FsYbT%#&^-RZHMG%x;O|!8y|xbGAOZ*5eH|Rrmt2cQva60i z{a%Fvzbzyca46{gAK0l1Y!G`~j*kcZYt#+&A9rj)7(Z*_K?OE!>n%tftp7!N9{66g znC-d?Os8LNq&);vBo<&4o>&2coj8`JI-gO1ytaS|D!7eaMp(-ZdOG`lEX{Pz zRe<~mG)%aM=CiH0)1&{9%S8%o(AM7M!<@N;_9wOQ>c0sv=e`=*dJ`BF0kY|w3;df7`2x-9QV0Tnmg92{8Wd?=g zy7)Fw$<`_ay~zBw8MFy=PS@)3iysg^ND11Y0EvSJ+%J+mzT`pLjIe`F%#1Vd)l!|W zC_q7*Pl6ag*}UXo;rOdTI=BKE=F=ac0|@&Y2TC|LSqlc;{M@7HFRWo>WT^5kH2F6b z(mq>X!6<6ZAAVd^#Q*N2cIQ@we86@|oPopu$`%JLAXKuP>df^bhPk;jg$sLe7btpH zp(teQE%-;>+3wlc-_mC`?L~~mZS0QstT7<}Z=FM*kaER46>@CAQ!rAO_DJW8=Fv&$ zR~2^WhYES0?Q;21g2lvce+r)C;wMo(3k-TENeAglxZ2zg_89CSYqRSytL&+L*B2vyl6*72+X2cnL*-XYL#b91Xq-$E5?n2A zf06bfZ0jrxAU`X_J?PegoyA(Oz(i2`l$OM|E~j8bZdruSKB_?ac;*Vxt)<{`gWL-icqbv4C?qvkkE!ExFeV2QLBD6_p^V5X3ro`N!mi2!6so2*dn%Noz{R)-9#THtp95yt%fe_uA{TqS|{jXm&8eUtjAW` zl;z(EQyC+FG0S`de~BY5*s(Y1IU_XA?krL0^4PA7>*CP0j7Usl1;%IVDcF4M^jp}; za&S9MXT2XliQciDg4{QK0oq{`1v+5sEV+O{qdN(4N4Y{AY^rxKP8;dB+Z`3wa()-@ zvV;Ce*zC zwT>1FWPnZnlI9y-AJuta7wt~i*XuFLw^ATNHV-@b7Gm_lT{NHX-z~;62?|?-u-~z> zI7OqB_uoseAV!C2c4r5LxCgVY5N8l~u*#3InWo)G^j2^vW84UL9oAxZg+3oUev974 za4kw@(>_JZ=yd>%V#p_S3gWgdw)8E;#(OE?0a!vHJ$8bEbbiHWVl=FL2gTGyfeFEW z7eq-m`%5fQ8-4+E&d7Ms?&xNX3A#8cV~i=+S0U%&WS_8JGj~vXS$~BfW^pmT2reYM z?{+<%PumYWpB>zZF@LZE7vQ_Urr4=<;7Pl4m_jaO3$u>*>n)h;APN+v>Lcv~v5f-m zIxNT$)~dkDC=sek*tJ(^I56oTJ#1`w+U~qs0l|1Hh?6}3Fl|rR<~Pw@$122qeEJbu zOkBa*2lCyd0Eqz>M{qd!&)>pPz}oId$xT$qhxnN9vFu`tzef9*Y%Pf8vQM0pQN(lp zCnlE>JCOT36!HN+<45`txQ}kheY{&C8nShjY)Lk0v0i`^G+^uhLUmsKGm6Q`+hBLz zY6*t5h3+i~o&2(2MXBySgzuQHkoUmo6K7qlc}7p;2aXCuapAYXJfOgYFiOYH$Kc(L z(>bI^sD|{N_*bosDi2xflxA6tPWMU7hQ`b{?T$wk5D#DZ8@iFv`E9%NafO@@?M#p_ zOa2gZ`h@qgoU;|u0T?qiM_%xye+r8R@!u@RTm=Xu@4w(zFfQ7g<$Tgw4hxQ^f?4Tc zsejW{UUixhVq|}oEXP8Hcz|y{ zD+{o>yostgv~h*pT~M@5X`K$0@Q|Io}!_{ zJ#!wT(kg|#hd*0Sa}wj)-|fzbLe2-jDwv*aSbmz8&0D5wAu|80-LX*t3Gg{-T6ZbN z6=ESy8fe!OM5pb}E!J}2leadRooh2R?3_WM+3wt?koRDk6--bD`5abZ$3L=dbxOq< z2y?gSZu7ToncC|ic&ww1%UzqF|Arz7AKzft;P&~oAFA>#govvoPx(NdAE z!SF=r+wOQ*fei7vIoh+Pp!0oeLD*5fMH0Q()J~c|fs?xMOFf586cp7%El4(Nm#@9W z?(2_M@qt2_hdT?jDk24)dlZ7g6C?<#?AIb3Qn&7m)_{GLqZROxjWkSM2({at)e1Pw zYLnjzIP7XE|FJ?oWE(A*tle3L6O{ii9okIzEJBjb>iEnW2lHu>Q-j|GaeTYu3x%ME zk7%sDN7(tl(7L`-2x2*59v*REr=GzM_(%?0+Zf!+^-p2T{@x~9?tl0W_AB&Z;aXeY zFU>%e6pIvgXe!o7PNXOJ&q z4LxYhM_t+^coN*KWEUuue=5K}TcrSFr3FSX>+*@7%C~yZ@+i1?k`{bg0TveJg>FYD~rHn70~Vt0_$u;b-_(cR9mI2=5WA?^%_X1^;*w>vaz zLF@%S7}S15F0t2qk6!5b$2ym8&#UdV%ZVU>*q#3>|J)<{@NJq4fH$EvgROkUhg zgG*)06&M&Y6`%lLaFLjRbNk`53l)MPTMyQ)0fH>0J+!HW-IN4Ya(Dn($wk&6X{87d z11OKz>#3Cx78-z*vJ@b3*5=a}X->A{G>~(8Y2TZ<%T-`}2wF%?aU-nxs(WjoWXn%- zC_n)e7!HzX$~*Se9w376Vt14(K}qO98E zAVP;<0rK!k1GII7g-4)#S}Fvw#5Jkjm49QATR8|(nrQ>IhD0!F?9SE-WXL3N=P-ff zKMWEhp#-(H2Ejh5NJwt$WzoT#kH1XQqVNg@++*`6z`G8?*$iLjU?0oT%Q2V8)Ez+u z+|S)th;DgvKE5ri5Da3tgioIReLl*8`Rlj`vX!?j-UwS?K|VvMc5 z6<{A`GC@x_!E;AyI@^5IQAIR8chPf?$C=cJT!ca`f{< zH=@6^t+R)6vI3GhPV4_4Jdr!BAt3wXBma$nIYp3ecMDA4JeKxO&zhr{e zh&_A{rs=lJ&}VNL2NSsK1nsxBbyz3&S*zotCSti+UYB4oLxEX4NlW`LD#!iS7*7zp zz|mWhZqa_j;jgtl&W|XNA$htv+jA>?aN{s>vPa*+=U#iO_A0Vno0>7xS`qJan^s?Z zYHdQnlhznE{dVp5I31(tDTV0scWA#Qh(2pA%D3Ms*7)F06g{VqYIwH>Zi0}Sr;xhh z9Q%RmOx0@s1SDZmuHA| zNDY0V>@|gS^vsBL4`5-{<5py_WfPoPFnW%b!uma+MdEA)$Lk6$0apH?b}}yP+@=r? zW84+DH?S@bYxVg(4{11m77PRDc54h~{wQYQBf#u?ScB_LVBS(-+))f~_o#+b?1lK- z3UN5J&1@aQjVpZVV<)O*}fEp#`3w^?;?8d z{GS3K4&svm^*NX0q>H2Tw4Y>z-1(yd6UK>k65|9p)fE55S{z;$rzt-AN$tEDe?OrR zcN<+N7O+E4;atF~r?gMakorjlz{A!p(0-P3&fgSr;$RfS>l?9apM-}C{q}1U^QQvi z1Lq8wJXZXS_PLprzZBwr-uPL~9;E^Ntq=_GG0zEeU-AY1Q3!@)X^dKg(s*xSv^1P& z6aX3moj<=aH$c&ujmf@qbML^V%wEU?-MoS-g0eusH>9z?rJR`EaI+ zxPXE^@PFC?!5MI*DWv@zw@cve6CszOkPGmyp;O8^8!F_)S&c}&IlpuT)*w=CmO?zl z=dIL|NxM3SI7&+_0^Q_?@cL5HLaGf;s6(B+L1T~_BZC!_{ zPJam#%lVC37Ry|Xc?N8UboSL6?UlMZ3lus%SXjq&X7XRxqC?bG;8ZB^f-{}@NU5ME z)aRqt3m2MXCA3yxd>G-yZ5wRU1~kPU93Qed9W!L+t2jcZ`W>7TC0}&dpluaY1ZNwC zE`L-P?;pV$Suz)zD#QcwNTIoJgWkX56-)p!&hJtHg0|+kGo&$V_o`NpExp|=^IuE? zgIj?MnK<5&YxSahhh_@#u&ou6X~gHeCOVkNl_Ov+j;*CQZm5i(eO*hE+yh5YA?)T5 zhQ$buf}@8*&cml~);gL@0LR4&K`(<~OEzkYR*x5L5reHH4|G&Oe3-959w=t*wxJwv zYz0Gn-d1hDAP^LHQsCIM&FE$z8zk{9+r%INfU~y(fZL3WfiJ*1d`tV%q!aX0NW(-G z103iCIjqaun0N(|pzIO_0xRhn#H4q$?@a2!00kn<9(Y&VCzuD0N=vZ-Sj{tcpg@G! zK!uo_EqG7+L5f|b5c9Cr?_=l%;h^ktg_zftE52sH2bhgN-l@GW0Yel3?6ntwQ6B>E z?FZU>5^#kA;5Px2cLDJGhuTg7a9pVX1lTXTw1ZM?s6s5r{@$&9BgC8|6=ESAhBkR) zBi4AYc1#i~oTC)dxamqtUsbIgH>F2gOCvyUNb7K2niP9f+=hZWbcu-#u` z*;4YQ_N$2*ufT+0!^m4%spmb zOgLY*1|VjE3&F%q7sr3YCv5snJ1BS%uPG32w)1y5WCTgTu}LB4VF&)e&?ATq#hVp! z?3+Jer#k*ZXOSd`w-vGhtm$LKgHnFuDY2#(oCD_$1t{e3VHW}_6P`V=?IF==u90{`Gz_?oH9PHUl1xubR`7fNj0|EsQx!n5UX;L?jSyWg&dkyqv&A_m*5mtN z0pXT}#;i|{ZkS>51`3USmX@m@7a`=VVue_Mx60FTma7ak=P3lm4%!(b5IW-ds9zZZ z&sT_tV&caN^k1d8BimXW-h%p#bZ~V=HXIj{5iY2kkef{}(tnfzY)4~-LJvD!02 zV*ML4L|dj1_L{=!B`D|k)OQFcE-Wa~H<OY&G&|D!K z#=s>mdJs|jBtEg37*%8dzT6swW7o<}5Q66!{KMvYlPIXY0wm6VoPlc`SfHg2Z$vDL z?dYVy;KrL)sPK5ixVu6)02dyj)_H7wYt*}Re8A-SARe7|_E4a4V7Q6y+7{@Pb}*PS zoiLadwFad@{&rD2G{EYVq!f0(jXo`|*3ns^HpJJo)nQD=KcSC8S{$Q@TOo4U`Evb$ z8HMg^DINsjDk?s*0(ovzp?@whjsXe`j(v9N-^++~S*1b_oENcde%y@->P9z49uea% zyIg^Y9{ODB#kP9sE*a+13)zPr{dpORE*ql2;@)eK+F>tJ+vY`TBL3}IbpfL0ubX8y1gYv?YaH%*%Ma>HNKdc0%P_WcG`4TvUFXc{;R;nD z+fefWgGOOpqdmP|Tpn*~#7PV_f0>8S#5dCqv(|`{rv}xW$-vHc)LXJi)6tbX-vc+) zq_H%I8CO#nY2wB4$Mh@M%8vSm9$ud-cC000y*269j{1&Lq#NQ(JLwdk@Lz>eH|RXl zT9wah8xlJRA(zeRreo`Na#8^+c|?3dFFlE^>yBRKxQXI&8ocEPyJG4`*(IF25aXZM z4V#ZpBpfmm<~Y{jFHya&*irAuf2>gI#Nn^DR*0kGud3P3pT;h`SogEp7wHABUaT)i zv%5gQ+}`gTtxy)iraW=r0mFd@c6LW^y|q~`Pi)kSIm%@$R9;X9?CY`E5jDnIWgszs z`Rqg=y_IY&d~P58S((9vR$-n$aJ@oXh>acuGU|eUdM-tiiXW7ZW8I?N#9jM9_OaIb zf?j^&Qay>Hpc4BT>vk8u>KX+uWb2FTIhwJp17ofK%jBfu1#55=%k=&*o_>4`TNmU@ z*#p+B)6JKaD@ZC^JW!t^swVNvMT(B=tWym6Ileq38h?D-6sU_Dol(W5;hvo zyoha5 z0oZx7wL*9xuSwWYI$?xffF|O{c_6V&`Q>d9;}t6Wwu?*^a?k9n(OBBFx&j$$JzBq; z^ziyIipG@1|Ehh9#qHJ!;2Z($RIK3nSL=3>x%k2L{;;`EaiRk32XP*eq7JOXb!fn& zD8{~T_F?Q>x))oEWu&O+UE0QcJNHk0BwPA@!rN5V@pDijWV8nghWEr(O0oL(#QS8O z%PS^Z=LnS75eXYpC+(z3Z0q${)5vkTVsk8YyhvszZp45sL^wpksNM_5q$M1^cakD4 ze>5$QO|Yq=@B%e8>vuB&cTBK`)%*@b8S`A6{@iPLvGQ%^Fi4lE+|Cy{O9P=(=5?pT<;*!3>$oN`TmFXFX{{m zQpHq-if9IeiUPLu@z}h+v;74Ko-hn>U3|EF`^ zy#8Fda=FklDt$*QEWTf%gLj*+J1`~$YEH3thkQ)HoX=XrydHBWS2O;>lX{^TQgDr` z`7s_(z`UQ}5d}PG8yepy+>`0)D33sVCaPKJ)V+)4VQZ~Ek8PmP%CTel4^d_RsjxQJ z^&y4AASNDhJB`?HoySWS>c5DAC$XgDmhA_vRrozo6&xzkL^*qF>N?~S=;B8eXufr+ z{-gMm#57~ArVc~du+{|X{$LUJaPYKc`k(lUU`>1W;D$ zH@&DI6pxYRQCSM5XjW?QR*xqqpQzhQfGHXHgf=7wYI_2)&7;x}A{1M(7^ zISMtRHsvKY*!O(Wq7FT-+H3uJRvO_m5@jX#Emv;T-w^r1ZLzTrg!BJAOtiTQc#sd^ z<~)McP85f07X{8a-x}vekC)q;`TE!Ok82l$Y}Jk@74RT`Z@x95sVvD{-6k=l5Yd)g(ocfePZP3;$u2yVsPGN6u$Mq*ux9PLc5rR0QGgdY= zrCVpi^V8M|cs;!GExn5vuH7~5&*L%gv3W*;3&QM`_t^aTj@}Rp`3vB&Jk`0=hFNRG zsZPjATh^}tI|gUJryoWgTsSk!yLFyZXb58S8E%a4&Cl)Baptb@*s-@Y@g}H!1-#Lr5iT~$@KnDfeZ4( zU+C>cdI+RNKho!yx6C+Ru*QXOqR~FRwZPR3(pXxW+fkoa;ILzOpWcb5f202`s^x-2 z{3~(3BU93Srw4;>@T#&nd(d*Z#S9z_^3Tqrj72sO)6-Tk&$nb`0Rzx10 z?=Ha3)e3OX)`y+w2h!CgC-n-`G2}X-PlKWh2Z?z@plE}&BABV>4poL7s`bF*&_jIW zZ*sK}s=53Y5diaopO+Pi#AqYqYfuF9DZA@?ezmSpGYS9Li%)0JS&zqr}d})v+~y}(hD){xy|EfK1b?QhP)1GjWrlX zboZLpm%~n+#Ys-mRrkdieO#Ryk=m*h+Te+;(U#5E|BI=!cCAVs5o;arQ4f)OU>Wwn z=JD^&%O-YVe>m6&3z%0FIzljk#J*ULeX+mRsllZeqdUE74TmFopq?aKHdYV+(c+&ASz>|TeA)ZV;c05^lvhn2L$;Fe0Cm&A% zoXJ$FCChiEN=_Cgsn|aDrAG+!JPvGs`$KjO218`1FQHQ)upf3r-Q4Qdtd2C^ztgM z?=S7%gkDj_`O2|=$Ire9$;S5d#wz~K^d`RMbX*nxXL*y!&(rZ$kKEk5 zDV}C{n&WAKrzO9+cdKp>(uq}k{jOHMbvha1)vfkWdTSN`;FDJ0YV`IhzV3@wO9{QR ziVys%RYfYjyNccaZL2rvy;ZF9cdZ7~DOKF~uGL>DbXpaE;@4L5D4kx#N{-_w?+2>b z?BlIA(ub<}#V1;AOrwuf@v~=IJ(Elyi+38nT{bt^4v;Sgm%>Ro`FYei` zUoop^)9WzCZ204lf3=f;HIRR$%D)oiUw6vC%s;j9_qZt&C*E?qEf@cEyW0AlExL`n zX8hO*PWhiUp0<;2xbB9qtChRJrdtv|+cXXTy0HX*nSW$rE}L;-YVAc6Zs+4mjNhlG zF8XUGzsdYpWIl5-ezW;+<14=0Sd@{H7zc#q57B-%HHl5!pHD=IT*YZsH%BZ(BU~H;gk4^>jy7*xN?YdJ-Hm=Ky$#BSH z%u76O9BdT(n?FKz{?X6tDa!I_faR~wpLAI^ams7(+qB;wyD(bae3);2XAQ1p5K6G%w-wIt(i&2 zzZoAHBMW1t0#w$H=NiWrBpdazer0ZAwOIzATw<_4GK`klTj_$6*P-FFc!Ke$D;5}{ zjqQ0u%5KS$jjlNdjN-PL{5GotY--QbXwFqAz_i?%#uuKZ6}`aYcGE}pi zMy@|brP~{snHOTl6@d(XyQOJi2i^>Iy2igTjm%A!>vX;-?E|cNx?eiHX7xo^E=F@%P`(-xtTURtl5pSuCY*mWJN1<{5yNZ z{OEH>+-;l;3V#1Eo!@C?s%SRGPoPbfv8-(gJ+=rL+ugHruy}l4Tbgm+ok1Vo%#w^3 z3y&Jt_l{K=F#dBlHnaGx>$I_>PX@os)N7n`e{KBHH>Mtap$PNlaN~ylF?heI}5 z@%6H@jY)mtjNQI?V}L)-c*g&Zv2#EMzuW8`)W0*&FqQ{a8Gja6j&qMcXlqPe=JSS; z1;K;N7&a&->jAH=zn_w0Cmkvv31uJUuvr-(zJouKhV<$rUj=1Abe+ zh1)%sCmFAgO^V#q!NH71Lt>SOY%ODiim#0`N_wXl_so=qNFftheC;q}dZ&Ag;a6qw zd(FgNXx5q0hh0#~*IgZ>EaAcNg8D8S96})KD;JOzPfR)=;Efw|K@pX5-yw z<$ZGZ|IusDLEZm`#i|R#z?teqk4$EyjEs2~@>uU08mD^KizN5Df*G|&$G-R5nw#aq z`_xEDZU{G-jf?2Uz3tlZrMCxudO+(zA_1E2A>*}%AZETxELj2f27gC zFf(4fF($L&5XN_-r)297N@T{b<1_d)Qw6mw;(2r+jb+n(yP3f{RP5IijlwI&8z*k6 zY;3Q~WwRSaLsBCLuISE;_P12lyw??obQ+S&xHo7H;^~&=dR<_^V0fv)d{c3pv2ci; z!nv@Sj}EaLX;*imiBDoUemx;pRVXs^>UPY?n;i3_+ZXv}Xbv;(xh?j`V8lKgWqI%R z3_in3@mLq0ZX`^}rh2*26gQ91jk4i!#;_?V#`+OGjSW*OYs%{mM_eQKFk}CnF={;? zW9FEqvi}=HNB_%x9%;fQDhS)s7Qc$2e0Nb&WahYh&G>s- zOrqXEWXp{OjQLC(_n?)(jGMlu;tkN0H8Vz&H)LxFO_|2J9!b`mGy3KO#-|U)i1^&L zWcqz2Pm4^vmFdVY&o zW|kL;d#E`xUVSF^olH``GhDOcWZtPwOEj_Tk$jnX+Sn4!>w09Uk+v*@KWZjoh(~@i zik8O=qhL6)d)DpT*tjC5(}etyF?0H2OgtC+W60<{uMI69%-R^cDV|5a@cw{A{~%Xj$^&k65C?mv=U%zmi|! z9(lulD;;`vE2|xu_;>>20n0Ev7VR;sj6Kxg16Z@mR>o9sIB3g}`JdacN=C$gC_ ze062_^SO-4)H32)oW@wdY{+@#b(~}r&r6iB+SLDJRM6A(0nOMB{fkQ~_4`~NTL$!} zQRyCZ@7IPO7g*?>&z8}zhtU~3 zud8elpWAQjU-7%~&bzU>fPJDZn=nduZ$nY#2$HV1>*WTtYa5F*dOg3+NPI7YFSHyv z`e2imWQq2AVWpA1p|UUfJg$iIMJF@HZ;sLL@nZ5T-3O<8!%K0I`(M&gOItGdA}gP7 zyu91kwl!9VKeGCj#oT!DgP44K-I38N>oDWghZ+2FxlH-gIQvly$m@;Vz1pD}kAD)= zyguW2X_m3Q76WHPsjJ8B~N}DlSiKib{9aq;;xAZEokCk??TrOVqd5IbmP5No zu%yUc8~)(NynV3((!DYTTw^dHOS8Cd^Y=UyG}c^`&Mei(w|qw#e?ax!`(uznw=Lbs z`zR}t_CES_$-$VK2nHitJ}8tKlq*rPD%J7P0ea?7XnD0PR!i8{&{*?HW~A%K1DFy2 zeTan84xh+M>F^mD=+tLJ3-fVLtgTKjOhzu-!)V->i$~!&JRy4L%vIkJo80uxg?h@^|Y1vt}j2M z6MsS5qsL+qvZbUp+Pt6Di5&gP&yCF|Vg*8IDN~?mF`U1oeR^cX-Xvtz^5~u^A$xx~ z5YK*dkhYzL_R>?8b?JAzA}jY{>OKBPrQ7OvW9pS|>zD%#jne)9MK=zXM%QBB9oR%Y zf5Xat(CL_5xMh1e4`)VxI=F=ykNy>thR1I>4(A)co=P^({giJ^JDg@tP&i(cJ?s1tj&~meS2`nW|_MfndMklMhu8GJ>w%1<8OL{l`R@wFbChR+i@c;ZV&|JJVZYap`Y zRHj*aIm(`sE6Z=`cpTtGb*c!4A_;%?;&e}Z?5}=zWa8;`M(-qG)8hH4*)Hq}?|j#p z*GMF?%1Ol!tCVIeKbsNR{MUF!f7Yv_Cu}S|Yb~1Z{pU26BBRrZY_42?#V*Ey=>_ztj*Tb?g89R55p2w>@N9(0g(~MS}=MsH@2j~ zuq}l?Kdv$+zjEB%UQOdbBYTdc{iUrv$c}mgcj{X~ZA$iKB_k zj(%Wm(7kF&-gG%K+OZyciI%sPb(Rh{&et?HbD@Aps?WY=w6}ScpTpFpA9ot>ZcjEA zyp&DFNsz&U3ytxbrTPjxGI47*YHd$JEHdn-odrBkOipGWF&fr7wovfSoV2@u!!J%{ zapIm-2FvBK8e^qtxx#XVyMexIgWR6{2Q9p{O%>%q@o_^|#AviDMsEOiHklUwsileb z8Q2=8*W6X!Vm6g;8`6vcytXw>V10Cwdo4F;l~o|Q3Zhk6>{B|}1f|OL#v~kY`Ni|u zEK%-N$YHkHg6z*>>*zIqOhW`*L9r#5-L6sZU@YG46$2Zw5Ti#zXvfzrI5y{W+EvUO z)4|4U6U_|ADsuY?h-_MsswIk3`K%kGJKI-L84?4VvVM$8J3!?dmdfqT*jakDW6ZD( zcwAUj9B#w%L|O~Bnb8NGV=D(PSHAg{rnO>q-~uHJEG#reIbEQ&#L_yd75j~*T#26D z^|C5p*`83HFP64u%@{T97Mnp&NR+id3xgH0bKs%m-ZJaAyWJK68=smZhdo7xpu~ z*9o2ax1m*}Jz~sd>_bNLN5tkT5VARB64_#OHUWDN0;-o@N)J&_gmXGLh8<-saNqvGE6~-`myJ! zcRKX97+*z^SCsZ=9W;9I<|;pj$?oROMByMg3v9e4_FdQ`CJ#pE=6=h3tuu>ZcJJd? zpnL7fL@K}CS_R^xu|s*jwd3C=DVsK3#d7JsA#780zu}#zyK&1^Y!6+S7;6o}0WtAv zc0VKM?XeXb4%=GLNPA`Bmey@CAECZH0TbPLAyI{A9tIv=2$-6{{+Tw=et>Lnx3ZD zq1M;4_h{Nw@4bejxeIT=eD=r~nZ~r(+_^)xG;)r>(8#%w)n&AQMr;l|ZgPx( zseLya?&Y_x%48bQ!Vxe9rQ_K~ay)>vHdtxpjo^)F!%ggM%6l-TDS{raSaCCRFnadk zD*6Lr|1Fr$XwM@tb|dJ)EN|M4iCCOCC$KY=JR50jw9=?O5^=8)6WI$?@6ni19`w4g zsBkCoRC?LPlEmOiYzL=~b7Rx<2C1{3w-Ptrj_4yzd@Qymd@fsS32!0Frm#ZHhx3r+ zCNoLV;Z6jo=(+i^Me=#Xq`MgQg@cy&u-VGp`n%C}He8ENGG<{F^*$_t57~L9*m)1T zfz#Z_tGo?kn@_%_veD?1W_(NbcC%UAoY#}&?{T#0e)RjG=}~Mk`+b0RPGy3|T#xj; z*sZA8SP%N$W-p=V9$=|r`!v>o=QnGCN;UtOFSXWCHE+)_yM-7Fc>le$uin&{b{b!8y5W@WP`Y~g+d5>jLFnxHcK;R%}=5u53@fU z1uF3nb)Jn)J%kUzQdVaQi4 zu{J6_L9uH-%)O(nLMY@FQy0SHjqZPZWF@xXNmSpQg{+>jYE?F!TEylQh62@+E>vmt zi&r0KbD6`X5DL3R@e|V7#{P)xRv%ZSz0n=2-lKM}D(_per+NY|U$m#fGz8@42^aw4 ziN!37(O1v1Z0F}Hc@JDUPylj8ttG6v49B}@YzLm{_(BEpig8b23DLogO~_iv1A_Rj z>V4=-l>)dvl$VPE-D(9(qY@qd`-&S8x?cDU7ImASV$JEo)7XsNWRIh#H!;}L_gK2o zx^4okn1opfQiEv2^DM{tl`0WAZnoUQQ-o_Ni)WPjBFkyHM+FGj>Y^}X@0fDq=sRww zHE}$ho?OO`Xl^$uWxAH>_*#VyiZ#nwV_7NTib|>8bbD0_La=f&4lhtCtr=bRD$90! zqXLA*g6Ck2IW|`RYoF=(Rw0W~d**qTXD(D7MVZe1Dp@yXy`%?uLox0}mI6rF7}+D! zaX5H8Id$n-&#q*vsEpJXP8bc=b6qUDvXT0$auUAFN+zkvrdc- zzm0@`P=P`K&3}?N5jiESBcm|_B|54CgvFv#Ialvm2h~3+B*FE;ZrG4iQU6V8gS;W? z{~AodPb!F8Toqvr7%f|mZ2YW}^oZqev964E3!X@`R%l7iUsU2=giR1T6~VQvfYHtC zd7`VjrT);u~vgFVkYpws*HC`VdM{`%BpA@fS z(>Y?DWz#a9$5hJw7#RqaHWYd9U}VsiEvVV!DnP)NE~AwVpW{jrs^Fom8PhFyt~oD6ZciEAH%WR9y9W>Xb@B*w(Bf0x3>zWIb6J zK2X%W`>R{BKNYINE-_&<8_Xzf8*=?Wm89F2iKtShNZpE3whsi$(+gC0B~Gifc;t@5 zR^Eg%YI7GI+K!QRa$W-6|3SHdXz&5sOry4=tDaHm@fu_1J33Ls|D9 z?m#L2QsI1NY|IMv3GtCM9NQ1_#J1Hd?QfM5zi9Lc#z}iDZKIVQbXD^+{!!rrwx;Fr zyd+Am&C>+TC5Hb;pQG4kRmh;|ybW^$b^j6t{Z}O^qjiX;VGD6PYlw}Xub}!r6(B6m z?SQjL?Z1ZPxf&(GFmZ{ZU2;q<{TXKBoxLo{$ye9p=Rh1ChYmleXHn)JY<8F)g!2Kt zAsye%_R$-A;XK^5J;|Y|RCvVs&(X_i-x7p9p8hJyQA;Ha=kQBNw?}{|+B~6PwdRRa z;eEC&8PyUQdocU8NA$W}OSqbrIw~!Gj2A?;#Qd+(GkMr&oI0H8j8`E7;G!U&CF*>` z{EUhZp@|bz@`#L>(I7E*A0~tL;Xrvt!~SX+t*25W%~5&ODV|{cQmGW-!o@%Hx{vRx zM;-RFG9C_5UJpz(^;L?(V(rAoe`-kjl5!We=zHvUI)AQDlMci1jQJj$ zD5tu?t6jQ;)i+&O1bQkZ1|g{cE$@vANKqx_75^Sa1uU+OCFcDa%W$Nsq@oa{0{D@^+ zd$+GV6K-#{-jkuyB0J&c*Yz|p=O?+L?+#E-GRt&ksxTo;wQD#_7r*_Cr4M(9Xi-C! z>Bv%H!lL7^(&#My6Xq#fA?b06*}t)L_;)^9T8tSyMY`m|(wou_c((q#EUbH+1!%J7jA;$$OfY;ho&mipE*A~Wk*@1jv${vR9*-{0> zR3!J#v)*pV^Wo&V%k%56?r*eGY4Hnem0tyXxxeD^SFf+uDtutA#xeVrc18F7XE;xC zv{6Y1#dMvwXYDIoQjfp7ciUElm#ZgiYiH7!D_MeodxSQA>X@GBsy>MqsZ4M(UR3yl;O4pj`oQ1wCBQK81~ z64$583J+COc&PeB+ew9Y3y|82>Q1xo;tipRlY8LQ;3a{96kyjiq4?UTOtE@v$ z<-6+9b4)k44O*s*4&Wi!FDwNN!A5F<#6D;hZw{!V8Dtmy}Y06urMded!ln+u52Oo_j zqBU?N7=@(P0Er$Uy1vluveuS-)lKG+HJdHHCP>Bk9t!31fo&5igqZGF~o;qDKjspQvy@Ru{p=*GrKr>rNh)11#MM>VYpRn8Qn^FXyh z{hMwqzioFkx#l$iy&j|J#~h-3UTjRgP{+mFd>e%n)CdWhrw%9Gam_8`ucgr*KD>2# z&#>)}f?L%D_Ic>VCj5FD)mL`D?&qWOZEBPcQJ^X0>lg4k&054avYCGhYZ5P~YSF*~ zKDbr17QVVM3Uk*43%HHv{!KS)fzI`lIXh8~@z#hjNmn=1@&2+a?0Y+!hQB8Hpqpkj z=hxDhKQ+iV+z^!y)+isKv=;ojww~xf$le%5gla^T6RYUG7Ul z3J{o3ne=^AKCrdNva3A4N4^E?SQE_RJ56fsiCTrtGopyjH6nsEz9qi~3=|gEoS%6) zDqmECd?mYy>b2s7T6?2oXne0GR@S=K2n-UqNKRVW3PWhxd07psw-;ET?lpm8c}H}4 zYgFLt%{7_N-BD!EnvfMlBl=as`WHypM_)!^y=#P(lZfcCHhfT9uQf6ARePhrzBK_W zs4&u{j-!@s(Qdw|q3yRXitbkvI)HPzmtmLreZXky2!_g;b|eZKP!q@^xlCyt^__XU zAEJnXH6r{3>PjA+D@5-6QPaGj95%QnSjbCYq~y?(E@}P1PiV?#9gC)QMU5bnuW}dd z=q;N%>ts~^s+#1(0RpL{IrVFeM9vMApgHBBp*4WOp8TG(8#-IN==rPX^!)d&iZ!w2i~U4f3y_jQmw#6;n$Xx9fhPUpej3$ACbXq>dn;qt)g%x8n8lf(!%g|1l&Jg5 zYkk^ftx!Mv!w!kk2)bZS&76PGP@jp1$VWq68wG zq~3@V)X99i$>{w!)qPQLVMqA^jqT}6hSbi6lTZgP-$ zne{8k#2HfOlP05>AF_T0^EZ#44p=F*p`sI5&At&2<9D_%?=308c7E8B1oL-ICFz%c zZ~Y3=Zl0`|_H+o22bJ2vnOxhC_cDpR{D?`xJy6NMBfRLVjkIE91UXy)}+g{0qQYiE&n=H9{kGrGE; zZg);p0X((=CV&9Qp*?q@KG7M{K%3-)5WR3yaX&rlqM$*{F!KAv*}FNywlZ^$+f-=< zMX!4~C`HreV1P|gNrr8$sL#D1>wbJ62kBY@9Cs-I0k=q+%I71)KFL9H-lLNBfry_*!-|N=c?rW=-!Q9fg`aL zokhHc`&=015-NP20u=%ue>PvnD7X~*7pNpXw#!YHkwt2Xe-r~r-nc=_U*K6Z_!G=` zl1gR$cKihxbk{tPpG=Xg7ttM`e-9nATx2>eU4{goR3+$xeFLqjEA^kp>xj=DliT}$ zFUOI=jXWVW%2kZ%z&I$>c|MCH_kFaY$O_~?XS&sj*4-Xd@)U1DDbIl*HMi=EZ=T_!Uf6cR)0ImS4u`w`DjP zz5Sj`%7nz?6*2@NWt?v*5`=RRpcQYi9C7+Nn0_jI6YbS<-lZzan(@jBI>bqoauxR#FlI!T|wLC){ ze~UL})bSH^{lhBt0c59=mj~$Gj5>dYXxmX0Dg>|6ytt$7NIjb#7$;rJP1qD%u#O)x zW;|i1wi7RzQ+O1ED}Xtav>FSud)|e;ZuS$rgoJnbSk~QN9<_1D{;L|%;+P_LI4*X% znTWi0gkIjjn~JmR`4DD|T4HyePyv0mt1Ey4z!xpd?F>wAT+#xP}B;&^iw%}mN$nKX01#DMUfCNBmkdB`& zY=w*cmnwfDTL8FBa%_i<&J$WKQSupY#*Eij*y*!&mrM&cV|gS-0E7FJAnA;M-P#1Q znKuyPj6mhQi@2MulbJPnGYR$I#T_(stcK!m;Vq1$B}tTZi{8q*f28M5>>Kr2ZKtJu zE}gQ^c4dVs0adF(;;its1J(JDDXLrrY}Y_l8p-=gXwH|I?C<-Wmol6nqS76xwR9CY zWb0D_769MLXvHD)mj?YWnQt(j+nMt6@)Xf`5AVl}m22!yyCMxldS%O47p@R6S>oK+ z(vOwEEEUj)Ra2L3EM@T>dXl*J8@ZsEegsC^p#lZO8~fx0R+=o*qE-Kv%+EE)7*aNzW34WqII!lM|_+eVN zm)V^j6-b&}&@|hM`^tDz+|9Dx?(nJPK^i*5bKpIfVV-ZNl7wBwttGi4_&x8%jHkY` zJ37}a?g72@^&|WSX57Ee?(Cux^xAwTPjl2!-kBN8589nQRN{Wy5R+bc{6}5_LZ*j0 zA4Om7t%3w$PQb_{KyZTk7imd~^KpKy|4}W$(N%>EinQ4C$Z(Ra5;SN%_n$$s&|UNX_TC>JFZsg4T|HZj>#4JFim-`Z4Go ze`Bkp_%E)DV`n%{V$bOV^*5+cL6P$}yhBTJyh<`Gp8kj9oF?ME$j=)y@Y!8t*N^4fwD)rS82kJl6P|HHDGn$^;>os(23Ki0sK{+WATu7R`r z!OGpPk`Ia*7dXxhork3JPL*WX*4CtVzI|RpsSz8tJMLD<``ls#*U+Sh3frC2RFYn} z875nEy{6sFsNe&v7OzglIh7eIlpm`si6WrV#N1jM`0xk}+a33+P&kKJTf?5si^VAK zgDT0etvz<*vF*J-9(VWjjmIScBkE|l@8P57FuD&bpdj?MHlYI0mQlbg6~K$}D=*Oy z0~0m4^SxUmh50IgA4?9C3|dxKYsRQgBh>Li6(ET1F$o|5QfYJp{551M=AT*hwZ<%r zC5$A#V09S{MOSXm%~5F$i^B~xtO^nLwmTnJNa9#zJKFsx_RkL{YuRF?UBeL##ILcT zp+b3WJx!DV>fR_|i3;Gy4hWLT6u4spN9z&iwmY9z0fN{ClmK}tM_W1uE}>Z7P`f$` zU9N(LZGEh3HtKKH;lpHT_n{&Rkl+dh3L9IUVo=#x8rFdl<#=9&!o|gl^yc(iuI3Pj zvbELNg9y{k)|lyDQo;O~RODqIv^iUIig<^1Iis&zBa5%7P(fQUO>?mp;`KZY8(l?l z*h+m(1qdSNAWS1G_C-t{xV1aM<&&;-k@(StCem*;DpneQT9B`RHB zn+)=j{Q?GSep3xsJQQ7xasHMHpD!@FD)%V zlVrBgU~ThKDhoLsBkgdeJq9(0&ReX!-T+odgK^j#uK zBpJodPgE!$jGjahP}$<3M{A3n$l<8M&s8Y7hOOigkMg0zNp^(Y`K1aK!lJVRML;#C zLjj~ZETBCCPILLOH{8@*ZSnq@LMyhqu$Q63*%D!#`0g8mDV59)yK{#M<*{`(Q39yz zqkykf0J*Je-Uu@7fSxB7wwERY+zz|r8x;z-Ky`$nFeRM_RFZOY7gvBZ5lNlp{JUWk zN^nR4fJ4)g{yVDY(d#LCqS#ZU;XJ})L(m4ls8C*8HClY~Hzi%BW#Ymb ztr%+{p}-}QEs)L%=clb6HztnRB3M$Q%I~%+EX~W>b zwt>m}Tai1{p)v0+S@1UwChcmh*AkBm(2^Ooo(rpTRt5DE808H~0>#N+{G4i&rKscm$q}Ga3W@Ce`)pPi7=}4^w<&M_cU934rr8$7@lsDpPH5YG; z*7~3_Do$=(#KCu{)P#g>d?k(vP z(#NtJwRk#rgH~n{MY^ak9;`zp#bnBOoQUr{UK&o5UD87Z^5QB7`8kwVp1(w`sL##AI!aN#e*Ep->>&-KND` z%!8ck6mm4_cI{8e*~l5El3F-L`@@vFNg*Xl?vM_$WL7wCR>*n1wB`=YNsha;e@y9| zu?lI?^=@gSO%BB*g`C$fp1nufNRvZxhe|R4g9Dlab_VW4hjrYi9Wbd5_o^^KYH`2z zqserbu96Fh4pTMUk1r_^GgN|M!`&u>7EIH=HOUSSD`XL8Yi!JD+fab(0M+0|IA*KF z-DH12Yq)r(cEqA9JgO4*Na5ct!G$V8uW&u6ffghQ1CB*1NgqM7DNTP^t0UffNW;FG ziFsUw@tYWt^oW*di8~FIcmNafqKCA6F<}-gv#CFSxe5@pN%Ap1)X_-%Gg||Z!;I!X zr-Fopdyct!ka7Q&D#hpHwLu|k&|&mHDzE|EPC)=mmL-cTs@ zyTzFK(s4GU?%9j1z*jZ3@-rxh7GfiJ(n3%(KVG2yVur2Psq}iqcMIW~nS$$8f*Mx#h0t4@5k3CK$JAJZ_QY>bBju!ylO>$#)6hUp?zt|Z&k_Tett{-ro|}S z#wWF7RzQ6Jj!V`BOr|A%ahVT#Xs>3iogCD?Vu&T z{LCfA<=iMg-66`BYIUuMzjK!=FFtwT763Z2Tx&pGmTP~QGy%sKDv)1{eHPm$GSu(< zQY9FGi!ML>LGL{WkN4*la0^Y)w<-{B@sly@6e@iIyF7!QN4GMg{`*xRc{_SJD2b-O zp#5!vXn80awy)Y}1M=W`IZGe~W&q_j6Dd<9f&gxd3Q#I2C7MnptF@$8TE47V%mf99Orj57)qXcIw+_EFMqcF$jDt3< zgp--L64SJ#EZjQw(&Cb^3Go71>?GO{kxRrE(fUQJu;N>`N_$J97vB&ilT@v^pkGX1 zjooK6ftIr`oj}0mm5hKqD%#8wM58yb?Pbyk&Z;UGwi$8TCN#Aa6ULh*(s7YA0lK$C zG&Wo|$USV&egGe5Nh=c{CBf<*bMwTa}F!y0kJM_4rWx(Nfk#rOZ#mKGME3xdQo3Re}Lqnsj2G`xs%O86Rt( zSdeBaNYH|;`2>)KpJ*Rjkmf1~j*iJxHh&7pv!808T96hh2+rzD$mgE{Qu>+pp$Tym z)C>vY7R?v7VMdh%2}fI%pquh`;j2!cZP!ki?1(~@u*VYSJGGOhuuCP3ixzjm)=5F9 zUnS_X70B`L}xDGuNIGJ_QhXd%VhIz?GwqZa0XNg5b8D+WPX9+UG_N^xe^jo zL2y}_gb3%C8a~M*#X~A_T$pI4FzGApKU2EBN;+g~WJ*u?8g^*&9_^S3>7asy;og{E zt8pgj@l@6Tw(Go(`?hcWM$4iOd$si@aipU{1&-NARU{=t#pT~<|1&Y&R2XT`<>g!p z_sLnQf(7F2sRDT{5V`ha)OSB(%^r3-c?egd=Iz&ZR_bx|QR(qg$sw5HTMwX?+8w}n zmn;!SUlqh>LCzlpQ9EU^L#|N?`lMhQ<^GIXKlYRM zjR|p%Q$hUr5a{PW!8<+xGTZ+C zIcWP8XW+OOCu8lJlnP(1?6g*odYwkwO74VXu}Wn?6raHszr}%gMkN`#hROho=J<MT{wvk`w zpf}FJw08Ut?v9M-J6>1e++yH4xua-GmZ~H@wEDbu#u8kk67-7Q7i6?caswQ1sw90P ziRsvVGFb#~t0aSR*vRjMjNv{vrx!;K|xKe{!GGZlv`*jLt z|9xHXxrMsA{;0)9*q~5}O{-=mLx5V>f-OFw>t{`tfODe?gc;KUO|K15m)iOTi(2r$ z3Iq;nXK-xrRgc=VI}V?Lcrs4^-3<0SK2qVl7A`4XuTT5y=*KO7z)lqe&QwLDKaox* z=ygPs1RayRNfOwlf=UByk_0vhPPg(ZkE*sHR{4FS#f+vpKEx*D>W+FU$pF3COh0&W z{5eGhlKYwXFi69geI$5Jb}DX(`&tuP=&!pkim#`t^o7dxttdd#*DBEWT^yjVuTl^e zb&)hHyeL4QsX*cIL627Y>B5V`-&rb3WR%$q|edLtve~xt)ILYes2|c#kn58D}c84*t$qKh(C?R zNRPgQ)%W=-xn;N@EUA^A>FBFc5fl@A`m+GK@gXRS+Tq4=(~e)`Cf(KwpxYw`2K2@p zKwQmj0bSvs`7IG=D-|$k8zdw30+&G?V)cA(PsQ;e*WxKW%UP%Z;{@jbc^^TBsNY`y z`J!B#_ZhTR;e#MVnz44=$dD#(>ZtFPO5EjlI-)FOpMmRC+bERaLS0LVfRZy3k1yaw zs}7ecn0Sv0AG8gtxCtPUUb;;8)1l6=<8zac_o6tJqlm3o>dE2k=5#lwcIL|v3X9X&OP-4$~&%SfTN~AZRW7D6@&67 zgG^^op$^9oN6SOoO~ntr^fcLtDlfsuh0Ep#HGC?(qzucW^kQ&d{iIC4O79(BinC2# z1s=h$J{BHF@wZ)+e-j^2;e%HGTZ+8{^tv+tp~}(Yw(=iV;Bg|aedPgTGuS$1h@LB^ z57H+fH=fE4>GQ~sR5(H^WkKP-!YqZS;`(j3&u8M=DR2l96b;c=%k)4YjCNqZoBEgF z>e}`yU{JWO*3)INV+L=~D4d_^?5M!uJIB|`aFO}Vdl4C?XY%eo4;XFbX;+?Zt==~~ zsMH2U(n$Rq^m9+e)x>Vx>{g+*8pXDI&AP%csg;Kk#o*D%Vtt?2Rer;>8}5ueo9O7Q z(h(G;WA&E2uFqf59v<^!6wWRR7}|WK8B!@fb($kaUZ)?FowKq#z&@ELN{dvAg4GnI z(|{ZG7UIzLRwt>rde)5#?4&kFCxteThftc1qV;(FAK6u^pyerAXIB+EXd6);Co+$o zW>D}}y#+3;f$jMIW_^upwOEzqt#;8-(#sU7z#?^T_qZywA-ctdGAs#5>P_1a!U3!5SbC>=c zlz1yAv|z}}Lm!orAP&km+Rs6)zDHjx@%Y|qbo9a5lfHzrw*n7q+}Xs_zxS2TD{Jr5 z$6Zt$(<|wx(i9YeMtXSmNRyUo8bN`Fiz?LOk)n=AmhkK^e?K{l+@rHsL%jY+?{qG z;@M*1Z2eu?WL27c`L}2?zX~6;jgk|1zKG1x8_Jm=SYadaO#?ZBI|kQ?$LU<`VJyJ1 zFg#EHAL~|i`mX3Y)k&j+R0_qU1$r*;=JQq_M-NwI=_0TECWR`u*u6;af-?DhriLa9Sf3ixhRp4=j$6)JtW1AKFzrgz{`l`nhw7#D)Oob2It|&kJ zTbEkDp!=wQDJ=ED=PhFwE;m;AaEkdQ80S!hHaHfhHi4tWW@S{~#17|Qen`eSLIn?F z1%qS4x%BBPXxbZJ(Mx5GRNgA@^3r`8tkrOZl7LT)c+K>Cy;U-A0!ONV_=4an{Wn>M zu8LCQy53VKF+Yt`0E0fU@pb)AGpiMY!WFc>(lS~FMCDswX{leT|1R-WaxA~t;vB2M zqjvjRhjoX)sqZo~UXi{#OaoFe!HrSj!#M6&er(r4_15Y+V)I-2GO4NZi#hnRuhitY zMxhC}Jw3Npzi?3%$nUo}u2TWS;)iwm->iGpW{!^Fi@(Zpea*VuE}w9&*MET*QziH1 zx-L3RJ5Hr5jMJ`F&gjO8=^OM{kWqN$sEyl*i|cE zg!7|_L4KvT6#x4Iz6llx6$90cC|ylCoT5@5wv9Ez9E2l{iL`AGOz+iuus2fA=K~e1 zTqO-Wt7kcGQz(MvowOHgyY5vT{0a@#U6#ozU15T*jw1V8tgO%uF&U{CeHWb%y#pds@MoPcC}MU5uq-qWpx#}xI((Wgs!l-Nafz>3 z-gyFH6YscBg%69*kLgDNtZD~c>G01VPyk`Raq80}TAwt{uOFAXMF$BYGz&6-o~c5I z1<+x1!K&pofzuU0?ERq)b4BL=us7VZD!MWQ)%J6ysT9gXsO2X~Q|bH}Ty%BlSM=$X zXY{!;h1mM6F@>p$6kztdnkfhz9(`L5lM0pi%u9_b?^mgmH$9YRI)&2y)0@eyjCQnn zfabRTJEKmm`gG^N`al2Q^d3~Chbdu*d8Sn0IO-p)PYv~6Jsk3oLM8SUu8^S#dDfJ0 z)-*}9y@2I)4QBJAFEKr$QXQm8TCG-8cQ|-D-|<>mVkxV&P^!j0LU|{xa9S=3h&`-O z9l$PtR!e5H%6hz9>qWX|t8`&wpjNHDHRh}OZQHXHy0AO3IX+e67Jb<`uJSWY^-6>*q1JH58046;B$ThIrEP zWZ=oflZ7W6PY#}3@x9OSk%y<2%vyPa15chZE+6c`6c49EzTo}ylIJ;yuN@o|lQxtx1T#5Zp?Y9Gh_B_i$JMjCFb zD-l`i8#Ug;LnWfq!p5Jx$=jERI$iVMZpS;8h)q-Tea(315^?&4{K?PpqLNuRc5i~G zDV}C{n&W99ZtPyrpqI|OLwrZUPR@Ilh!1xc9MO3160!2Df)$MSEfJT0 zT~L_7`<2ktLj~{h0VNbZTrijqED`e$7o3UXgG Date: Mon, 3 Dec 2018 23:06:32 +0100 Subject: [PATCH 08/86] updates settings, add default for LOCK_CACHE_URL --- src/etools_datamart/config/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 401c329e3..e3f3d846a 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -15,6 +15,7 @@ CACHE_URL=(str, "redis://127.0.0.1:6379/1"), API_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=api"), + LOCK_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=lock"), TEMPLATE_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), # CACHE_URL=(str, "dummycache://"), # API_CACHE_URL=(str, "dummycache://"), From d4ac643b9a4f8aaf5c117a8c7b6949f77b7b895d Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 5 Dec 2018 07:31:13 +0100 Subject: [PATCH 09/86] updates requirements --- Pipfile | 2 +- Pipfile.lock | 24 ++++++++--------- docker/Dockerfile | 3 +-- src/etools_datamart/api/endpoints/openapi.py | 1 - .../init/management/commands/init-setup.py | 17 +++--------- .../apps/subscriptions/views.py | 2 +- .../apps/tracking/middleware.py | 27 +++++++++++-------- src/etools_datamart/config/settings.py | 14 +++++----- tests/test_commands.py | 7 ----- tests/test_subscription.py | 2 +- 10 files changed, 42 insertions(+), 57 deletions(-) diff --git a/Pipfile b/Pipfile index ce4b72dda..4d41fbbe5 100644 --- a/Pipfile +++ b/Pipfile @@ -8,7 +8,7 @@ psycopg2 = "*" admin-extra-urls = ">=2.1" celery = "*" coreapi = "*" -django = ">=2.1" +django = ">=2.1.4" django-adminfilters = ">=1.1" django-celery-beat = "==1.1.1" django-concurrency = "*" diff --git a/Pipfile.lock b/Pipfile.lock index abb804866..fae140675 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "81eac07ad1f97c779e00281dd19958dd2bb27a51bda821d288bf0d6174b1e6ee" + "sha256": "b1879bd53009e1f06a3807d199702cf1374021f4870a64405554612221101c57" }, "pipfile-spec": 6, "requires": { @@ -633,9 +633,9 @@ }, "billiard": { "hashes": [ - "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84" + "sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e" ], - "version": "==3.5.0.4" + "version": "==3.5.0.5" }, "celery": { "hashes": [ @@ -753,11 +753,11 @@ }, "django": { "hashes": [ - "sha256:1ffab268ada3d5684c05ba7ce776eaeedef360712358d6a6b340ae9f16486916", - "sha256:dd46d87af4c1bf54f4c926c3cfa41dc2b5c15782f15e4329752ce65f5dad1c37" + "sha256:068d51054083d06ceb32ce02b7203f1854256047a0d58682677dd4f81bceabd7", + "sha256:55409a056b27e6d1246f19ede41c6c610e4cab549c005b62cbeefabc6433356b" ], "index": "pypi", - "version": "==2.1.3" + "version": "==2.1.4" }, "django-adminactions": { "hashes": [ @@ -999,11 +999,11 @@ }, "djangorestframework-xml": { "hashes": [ - "sha256:caea8e446298b7fe1eb9a79306f35554db7531c2e637734d32de3cf99afbdc5a", - "sha256:f7d5efc26eabbca73db0ff0f0c15b59ca08e36660c02f96563a0d937321f519f" + "sha256:d8118580b6c0e94a6b908a78c8d842e9f349901dfff43d91adc2d73a54f4ba59", + "sha256:d85d5744e75fe01ea2af667b15f6aa7df97c710516477ba493558da8432f6b0f" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.4.0" }, "djangorestframework-yaml": { "hashes": [ @@ -1311,10 +1311,10 @@ }, "pyjwt": { "hashes": [ - "sha256:30b1380ff43b55441283cc2b2676b755cca45693ae3097325dea01f3d110628c", - "sha256:4ee413b357d53fd3fb44704577afac88e72e878716116270d722723d65b42176" + "sha256:00414bfef802aaecd8cc0d5258b6cb87bd8f553c2986c2c5f29b19dd5633aeb7", + "sha256:ddec8409c57e9d371c6006e388f91daf3b0b43bdf9fcbf99451fb7cf5ce0a86d" ], - "version": "==1.6.4" + "version": "==1.7.0" }, "pyparsing": { "hashes": [ diff --git a/docker/Dockerfile b/docker/Dockerfile index b99c917ca..84ba737b8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -55,7 +55,6 @@ LABEL org.label.name="eTools Datamart" \ ARG BUILD_DATE -ARG PIPENV_PYPI_MIRROR ARG PIPENV_ARGS ARG VERSION @@ -117,7 +116,7 @@ WORKDIR /code RUN set -ex \ ls -al /code \ - && pipenv install --system --deploy --ignore-pipfile $PIPENV_ARGS + && pipenv install --verbose --system --deploy --ignore-pipfile $PIPENV_ARGS RUN pip install . \ && rm -fr /code diff --git a/src/etools_datamart/api/endpoints/openapi.py b/src/etools_datamart/api/endpoints/openapi.py index 04c19d5a8..5d1affbeb 100644 --- a/src/etools_datamart/api/endpoints/openapi.py +++ b/src/etools_datamart/api/endpoints/openapi.py @@ -11,7 +11,6 @@ Each API endpoint allows filtering and/or ordering results. -Fiel ## Query lookups Any field where query functions are enabled allow to.... diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index 7a4792585..dc7b4a977 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -103,13 +103,6 @@ def add_arguments(self, parser): default=False, help='refresh datamart tables') - parser.add_argument( - '--async', - action='store_true', - dest='_async', - default=False, - help='use celery to refresh datamart') - def handle(self, *args, **options): verbosity = options['verbosity'] migrate = options['migrate'] @@ -228,10 +221,6 @@ def handle(self, *args, **options): self.stdout.write(f"Running {task.name}...", ending='\r') self.stdout.flush() - if options['_async']: - etl.delay() - self.stdout.write(f"{task.name} scheduled") - else: - etl.apply() - cost = naturaldelta(app.timers[task.name]) - self.stdout.write(f"{task.name} excuted in {cost}") + etl.apply() + cost = naturaldelta(app.timers[task.name]) + self.stdout.write(f"{task.name} excuted in {cost}") diff --git a/src/etools_datamart/apps/subscriptions/views.py b/src/etools_datamart/apps/subscriptions/views.py index af8d864e4..6b5a9d5fb 100644 --- a/src/etools_datamart/apps/subscriptions/views.py +++ b/src/etools_datamart/apps/subscriptions/views.py @@ -16,13 +16,13 @@ class Meta: @csrf_exempt def subscribe(request, etl_id): - code = 200 values = {'status': "", 'detail': ""} try: user = request.user payload = json.loads(request.body) form = SubscriptionForm(data=payload) if form.is_valid(): + code = 200 etl = EtlTask.objects.get(id=etl_id) s, created = Subscription.objects.update_or_create(user=user, content_type=etl.content_type, diff --git a/src/etools_datamart/apps/tracking/middleware.py b/src/etools_datamart/apps/tracking/middleware.py index fd55dc265..c33621c9e 100644 --- a/src/etools_datamart/apps/tracking/middleware.py +++ b/src/etools_datamart/apps/tracking/middleware.py @@ -10,6 +10,7 @@ from strategy_field.utils import fqn from etools_datamart.apps.tracking import config +from etools_datamart.apps.tracking.asyncqueue import AsyncQueue from .models import APIRequestLog, DailyCounter, MonthlyCounter, PathCounter, UserCounter @@ -118,10 +119,14 @@ def record_to_kwargs(request, response): cached=request.api_info.get('cache-hit', False), # see api.common.APICacheResponse content_type=media_type) + # -# class AsyncLogger(AsyncQueue): -# def _process(self, record): -# log_request(**record_to_kwargs(**record)) +class AsyncLogger(AsyncQueue): + def _process(self, record): + # import requests + # payload = 'v=1&t=event&tid=UA-XXXXXY&cid=555&ec=video&ea=play&el=holiday&ev=300' + # r = requests.post('http://www.google-analytics.com/collect', data=payload) + log_request(**record_to_kwargs(**record)) class StatsMiddleware(object): @@ -143,11 +148,11 @@ def __call__(self, request): self.log(request, response) return response -# -# class ThreadedStatsMiddleware(StatsMiddleware): -# def __init__(self, get_response): -# super(ThreadedStatsMiddleware, self).__init__(get_response) -# self.worker = AsyncLogger() -# -# def log(self, request, response): -# self.worker.queue({'request': request, 'response': response}) + +class ThreadedStatsMiddleware(StatsMiddleware): + def __init__(self, get_response): + super(ThreadedStatsMiddleware, self).__init__(get_response) + self.worker = AsyncLogger() + + def log(self, request, response): + self.worker.queue({'request': request, 'response': response}) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index e3f3d846a..4a9c9b0d6 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -14,11 +14,11 @@ ETOOLS_DUMP_LOCATION=(str, str(PACKAGE_DIR / 'apps' / 'multitenant' / 'postgresql')), CACHE_URL=(str, "redis://127.0.0.1:6379/1"), - API_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=api"), - LOCK_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=lock"), - TEMPLATE_CACHE_URL=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), + CACHE_URL_API=(str, "redis://127.0.0.1:6379/2?key_prefix=api"), + CACHE_URL_LOCK=(str, "redis://127.0.0.1:6379/2?key_prefix=lock"), + CACHE_URL_TEMPLATE=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), # CACHE_URL=(str, "dummycache://"), - # API_CACHE_URL=(str, "dummycache://"), + # CACHE_URL_API=(str, "dummycache://"), ABSOLUTE_BASE_URL=(str, 'http://localhost:8000'), DISCONNECT_URL=(str, 'https://login.microsoftonline.com/unicef.org/oauth2/logout'), ENABLE_LIVE_STATS=(bool, True), @@ -190,9 +190,9 @@ CACHES = { 'default': env.cache(), - 'lock': env.cache('LOCK_CACHE_URL'), - 'api': env.cache('API_CACHE_URL'), - 'dbtemplates': env.cache('TEMPLATE_CACHE_URL') + 'lock': env.cache('CACHE_URL_LOCK'), + 'api': env.cache('CACHE_URL_API'), + 'dbtemplates': env.cache('CACHE_URL_TEMPLATE') } ROOT_URLCONF = 'etools_datamart.config.urls' diff --git a/tests/test_commands.py b/tests/test_commands.py index 4900acd41..640527525 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -31,13 +31,6 @@ def test_init_setup_all(db, settings, autocreate_users, invalidate_cache): assert ModelUser.objects.exists() -@pytest.mark.django_db -def test_init_setup_refresh_async(db, settings): - call_command("init-setup", _async=True, refresh=True, stdout=StringIO()) - ModelUser = get_user_model() - assert ModelUser.objects.exists() - - @pytest.mark.django_db def test_init_setup_migrate(db, settings): settings.DEBUG = True diff --git a/tests/test_subscription.py b/tests/test_subscription.py index f68837538..b2c2e5063 100644 --- a/tests/test_subscription.py +++ b/tests/test_subscription.py @@ -65,7 +65,7 @@ def test_subscribe_404(rf, admin_user, etltask): request = rf.post(reverse("subscribe", args=[etltask.pk]), {"type": 1}, content_type='application/json') request.user = admin_user - res = subscribe(request, 21) + res = subscribe(request, -99) assert res.status_code == 404 From 5e556f4e7eb15e9a208a43ce2d093c64933304d8 Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 5 Dec 2018 14:20:44 +0100 Subject: [PATCH 10/86] add ability to sync users with GRAPH --- src/etools_datamart/config/settings.py | 6 +- src/unicef_rest_framework/auth.py | 2 +- src/unicef_security/admin.py | 44 +++++++++++++- src/unicef_security/config.py | 4 +- src/unicef_security/{azure.py => graph.py} | 58 +++++++++++++------ src/unicef_security/sync.py | 2 +- .../templates/admin/link_user.html | 39 +++++++++++++ tests/unicef_security/test_azure.py | 2 +- 8 files changed, 128 insertions(+), 29 deletions(-) rename src/unicef_security/{azure.py => graph.py} (85%) create mode 100644 src/unicef_security/templates/admin/link_user.html diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 4a9c9b0d6..769ab04a6 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -183,7 +183,7 @@ AUTHENTICATION_BACKENDS = [ # 'social_core.backends.azuread_tenant.AzureADTenantOAuth2', - 'unicef_security.azure.AzureADTenantOAuth2Ext', + 'unicef_security.graph.AzureADTenantOAuth2Ext', 'django.contrib.auth.backends.ModelBackend', 'django.contrib.auth.backends.RemoteUserBackend', ] @@ -460,7 +460,7 @@ SOCIAL_AUTH_REVOKE_TOKENS_ON_DISCONNECT = True SOCIAL_AUTH_PIPELINE = ( 'social_core.pipeline.social_auth.social_details', - 'unicef_security.azure.get_unicef_user', + 'unicef_security.graph.get_unicef_user', # 'unicef_security.azure.social_uid', # 'social_core.pipeline.social_auth.social_uid', # 'social_core.pipeline.social_auth.social_user', @@ -472,7 +472,7 @@ # 'social_core.pipeline.social_auth.load_extra_data', # 'social_core.pipeline.user.user_details', # 'social_core.pipeline.social_auth.associate_by_email', - 'unicef_security.azure.default_group', + 'unicef_security.graph.default_group', ) SOCIAL_AUTH_AZUREAD_TENANT_OAUTH2_KEY = env.str('AZURE_CLIENT_ID') diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index 73fdd3690..54ce59bec 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -7,7 +7,7 @@ from rest_framework import exceptions from rest_framework.exceptions import AuthenticationFailed, PermissionDenied from rest_framework_jwt import authentication -from unicef_security.azure import default_group, Synchronizer +from unicef_security.graph import default_group, Synchronizer logger = logging.getLogger() diff --git a/src/unicef_security/admin.py b/src/unicef_security/admin.py index 51bb8f9a3..88bcafbfc 100644 --- a/src/unicef_security/admin.py +++ b/src/unicef_security/admin.py @@ -5,9 +5,10 @@ from django.contrib import admin, messages from django.contrib.admin import ModelAdmin from django.contrib.auth.admin import UserAdmin +from django.forms import Form from django.template.response import TemplateResponse from django.utils.translation import gettext_lazy as _ -from unicef_security.azure import Synchronizer, SyncResult +from unicef_security.graph import default_group, Synchronizer, SyncResult from unicef_security.models import BusinessArea, Region, Role, User from unicef_security.sync import load_business_area, load_region @@ -42,13 +43,17 @@ class LoadUsersForm(forms.Form): emails = forms.CharField(widget=forms.Textarea) +class FF(Form): + selection = forms.CharField() + + @admin.register(User) class UserAdmin2(ExtraUrlMixin, UserAdmin): list_display = ['username', 'email', 'is_staff', 'is_superuser'] list_filter = ['is_superuser', 'is_staff'] search_fields = ['username', ] fieldsets = ( - (None, {'fields': ('username', 'password')}), + (None, {'fields': (('username', 'azure_id'), 'password')}), (_('Personal info'), {'fields': (('first_name', 'last_name',), ('email', 'display_name'), ('job_title',), @@ -81,6 +86,38 @@ def sync_user(self, request, pk): self.message_user(request, "User synchronized") + @action(label='Link user') + def link_user_data(self, request, pk): + opts = self.model._meta + ctx = { + 'opts': opts, + 'app_label': 'security', + 'change': True, + 'is_popup': False, + 'save_as': False, + 'has_delete_permission': False, + 'has_add_permission': False, + 'has_change_permission': True, + } + obj = self.get_object(request, pk) + syncronizer = Synchronizer() + try: + if request.method == 'POST': + if request.POST.get('selection'): + data = syncronizer.get_user(request.POST.get('selection')) + syncronizer.sync_user(obj, data['id']) + self.message_user(request, "User linked") + return None + else: + ctx['message'] = 'Select one entry to link' + + data = syncronizer.search_users(obj) + ctx['data'] = data + return TemplateResponse(request, 'admin/link_user.html', ctx) + + except Exception as e: + self.message_user(request, str(e), messages.ERROR) + @link() def load(self, request): opts = self.model._meta @@ -101,7 +138,8 @@ def load(self, request): emails = form.cleaned_data['emails'].split() total_results = SyncResult() for email in emails: - result = synchronizer.fetch_users("startswith(mail,'%s')" % email) + result = synchronizer.fetch_users("startswith(mail,'%s')" % email, + callback=default_group) total_results += result self.message_user(request, f"{len(total_results.created)} users have been created," f"{len(total_results.updated)} updated." diff --git a/src/unicef_security/config.py b/src/unicef_security/config.py index ba96c7be2..031ec6079 100644 --- a/src/unicef_security/config.py +++ b/src/unicef_security/config.py @@ -5,8 +5,8 @@ AZURE_CLIENT_ID = os.environ.get('AZURE_CLIENT_ID', '') AZURE_CLIENT_SECRET = os.environ.get('AZURE_CLIENT_SECRET', '') -UNICEF_AZURE_CLIENT_ID = os.environ.get('UNICEF_AZURE_CLIENT_ID', AZURE_CLIENT_ID) -UNICEF_AZURE_CLIENT_SECRET = os.environ.get('UNICEF_AZURE_CLIENT_SECRET', AZURE_CLIENT_SECRET) +GRAPH_CLIENT_ID = os.environ.get('GRAPH_CLIENT_ID', AZURE_CLIENT_ID) +GRAPH_CLIENT_SECRET = os.environ.get('GRAPH_CLIENT_SECRET', AZURE_CLIENT_SECRET) AZURE_SSL = True AZURE_URL_EXPIRATION_SECS = 10800 diff --git a/src/unicef_security/azure.py b/src/unicef_security/graph.py similarity index 85% rename from src/unicef_security/azure.py rename to src/unicef_security/graph.py index 292adeac6..7cfd34481 100644 --- a/src/unicef_security/azure.py +++ b/src/unicef_security/graph.py @@ -14,6 +14,7 @@ from social_core.backends.azuread_tenant import AzureADTenantOAuth2 from social_core.exceptions import AuthTokenError from social_django.models import UserSocialAuth +from unicef_security.config import GRAPH_CLIENT_ID, GRAPH_CLIENT_SECRET from . import config @@ -175,7 +176,10 @@ def __eq__(self, other): class Synchronizer: - def __init__(self, user_model=None, mapping=None, echo=None): + def __init__(self, user_model=None, mapping=None, echo=None, id=None, secret=None): + self.id = id or GRAPH_CLIENT_ID + self.secret = secret or GRAPH_CLIENT_SECRET + self.user_model = user_model or get_user_model() self.field_map = dict(mapping or DJANGOUSERMAP) self.user_pk_fields = self.field_map.pop('_pk') @@ -188,13 +192,13 @@ def __init__(self, user_model=None, mapping=None, echo=None): self.echo = echo or (lambda l: True) def get_token(self): - if not config.UNICEF_AZURE_CLIENT_ID and config.UNICEF_AZURE_CLIENT_SECRET: + if not self.id and self.secret: raise ValueError("Configure AZURE_CLIENT_ID and/or AZURE_CLIENT_SECRET") token = cache.get(AZURE_GRAPH_API_TOKEN_CACHE_KEY) if not token: post_dict = {'grant_type': 'client_credentials', - 'client_id': config.UNICEF_AZURE_CLIENT_ID, - 'client_secret': config.UNICEF_AZURE_CLIENT_SECRET, + 'client_id': self.id, + 'client_secret': self.secret, 'resource': config.AZURE_GRAPH_API_BASE_URL} response = requests.post(config.AZURE_TOKEN_URL, post_dict) if response.status_code != 200: # pragma: no cover @@ -266,19 +270,38 @@ def get_record(self, user_info: dict) -> (dict, dict): pk = {fieldname: data.pop(fieldname) for fieldname in self.user_pk_fields} return pk, data - def fetch_users(self, filter): + def fetch_users(self, filter, callback=None): self.startUrl = "%s?$filter=%s" % (self._baseurl, filter) - return self.syncronize() + return self.syncronize(callback=callback) + + def search_users(self, record): + url = "%s?$filter=" % self._baseurl + filters = [] + if record.email: + filters.append("mail eq '%s'" % record.email) + if record.last_name: + filters.append("surname eq '%s'" % record.last_name) + if record.first_name: + filters.append("givenName eq '%s'" % record.first_name) + + page = self.get_page(url + " or ".join(filters), single=True) + return page['value'] + + def filter_users_by_email(self, email): + """https://graph.microsoft.com/v1.0/users?$filter=mail eq 'sapostolico@unicef.org'""" + url = "%s?$filter=mail eq '%s'" % (self._baseurl, email) + page = self.get_page(url, single=True) + return page['value'] def get_user(self, username): url = "%s/%s" % (self._baseurl, username) user_info = self.get_page(url, single=True) return user_info - def sync_user(self, user): - if not user.azure_id: + def sync_user(self, user, azure_id=None): + if not azure_id or user.azure_id: raise ValueError("Cannot sync user without azure_id") - url = "%s/%s" % (self._baseurl, user.azure_id) + url = "%s/%s" % (self._baseurl, azure_id or user.azure_id) user_info = self.get_page(url, single=True) pk, values = self.get_record(user_info) user, __ = self.user_model.objects.update_or_create(**pk, @@ -291,22 +314,21 @@ def resume(self, *, delta_link=None, max_records=None): return self.syncronize(max_records) def is_valid(self, user_info): - return (user_info.get('email') and - user_info.get('first_name') and - user_info.get('last_name') and - 'noreply' not in user_info.get('email')) + return user_info.get('email') - def syncronize(self, max_records=None): + def syncronize(self, max_records=None, callback=None): logger.debug("Start Azure user synchronization") results = SyncResult() try: for i, user_info in enumerate(iter(self)): pk, values = self.get_record(user_info) if self.is_valid(values): - user_data = self.user_model.objects.update_or_create(**pk, - defaults=values) - self.echo(user_data) - results.log(user_data) + user, created = self.user_model.objects.update_or_create(**pk, + defaults=values) + if callback: + callback(user=user, is_new=created) + self.echo([user, created]) + results.log(user, created) else: results.log(user_info) if max_records and i > max_records: diff --git a/src/unicef_security/sync.py b/src/unicef_security/sync.py index d1ab75719..078d50857 100644 --- a/src/unicef_security/sync.py +++ b/src/unicef_security/sync.py @@ -5,7 +5,7 @@ from requests.auth import HTTPBasicAuth from . import config -from .azure import SyncResult +from .graph import SyncResult from .models import BusinessArea, Region logger = logging.getLogger(__name__) diff --git a/src/unicef_security/templates/admin/link_user.html b/src/unicef_security/templates/admin/link_user.html new file mode 100644 index 000000000..d03e47b33 --- /dev/null +++ b/src/unicef_security/templates/admin/link_user.html @@ -0,0 +1,39 @@ +{% extends "admin/change_form.html" %}{% load i18n admin_urls static admin_modify %} +{% block breadcrumbs %} +
      + {% trans 'Home' %} + › {{ opts.app_config.verbose_name }} + › {{ opts.verbose_name_plural|capfirst }} + › Link AD User +
      +{% endblock %} + +{% block content %} +

      Link User with AD

      +

      Select AD user to link with

      + {{ message }} +
      + {% csrf_token %} + + {% for entry in data %} + + + + {% for key,value in entry.items %} + + + + + {% endfor %} + {% endfor %} +
      +
      + + Select +
      {{ key }}{{ value }}
      + + +
      +{% endblock content %} + +{% block submit_buttons_bottom %}{% endblock %} diff --git a/tests/unicef_security/test_azure.py b/tests/unicef_security/test_azure.py index c711acd4c..380658f9f 100644 --- a/tests/unicef_security/test_azure.py +++ b/tests/unicef_security/test_azure.py @@ -1,7 +1,7 @@ from pathlib import Path import vcr -from unicef_security.azure import Synchronizer +from unicef_security.graph import Synchronizer def test_token(): From 6fd7ffab6e968cd984e1e7dd5153a749167380d3 Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 5 Dec 2018 14:43:47 +0100 Subject: [PATCH 11/86] fixes Synchronizer --- src/unicef_security/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/unicef_security/graph.py b/src/unicef_security/graph.py index 7cfd34481..271706e63 100644 --- a/src/unicef_security/graph.py +++ b/src/unicef_security/graph.py @@ -299,7 +299,7 @@ def get_user(self, username): return user_info def sync_user(self, user, azure_id=None): - if not azure_id or user.azure_id: + if not (azure_id or user.azure_id): raise ValueError("Cannot sync user without azure_id") url = "%s/%s" % (self._baseurl, azure_id or user.azure_id) user_info = self.get_page(url, single=True) From 30078e7f453963b0a058a37ee92fce176fd4f2f8 Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 5 Dec 2018 15:47:31 +0100 Subject: [PATCH 12/86] fixes tests --- tests/api/test_api_auth_jwt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/test_api_auth_jwt.py b/tests/api/test_api_auth_jwt.py index 8e359c618..07d8d854c 100644 --- a/tests/api/test_api_auth_jwt.py +++ b/tests/api/test_api_auth_jwt.py @@ -29,7 +29,7 @@ def user(db): def test_token(user, client): url = reverse('api:partners-list', args=['v1']) client.credentials(HTTP_AUTHORIZATION='jwt ' + TOKEN) - with mock.patch('unicef_security.azure.Synchronizer.get_user', + with mock.patch('unicef_security.graph.Synchronizer.get_user', return_value={'@odata.context': 'https://graph.microsoft.com/v1.0/$metadata#users/$entity', 'id': '21d2ecba-83e4-4e81-a93c-d44f55dd222e', 'businessPhones': [], 'displayName': 'Stefano Apostolico', 'givenName': 'Stefano', 'jobTitle': None, From e59c551910850670fdfd11d28449513717bda57b Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 5 Dec 2018 17:45:19 +0100 Subject: [PATCH 13/86] minor fixing --- src/etools_datamart/api/metadata.py | 8 ++++---- .../apps/init/management/commands/init-setup.py | 2 +- src/unicef_security/admin.py | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/etools_datamart/api/metadata.py b/src/etools_datamart/api/metadata.py index ef1e46caa..28691ec42 100644 --- a/src/etools_datamart/api/metadata.py +++ b/src/etools_datamart/api/metadata.py @@ -67,10 +67,10 @@ class SimpleMetadataWithFilters(SimpleMetadata): def determine_metadata(self, request, view): metadata = super(SimpleMetadataWithFilters, self).determine_metadata(request, view) - metadata['filters'] = getattr(view, 'filter_fields') - metadata['filter_blacklist'] = getattr(view, 'filter_blacklist') - metadata['ordering'] = getattr(view, 'ordering_fields') - metadata['serializers'] = getattr(view, 'serializers_fieldsets') + metadata['filters'] = getattr(view, 'filter_fields', '') + metadata['filter_blacklist'] = getattr(view, 'filter_blacklist', '') + metadata['ordering'] = getattr(view, 'ordering_fields', '') + metadata['serializers'] = getattr(view, 'serializers_fieldsets', '') # from django.db import connection # with connection.schema_editor() as editor: # sql = get_create_model(editor, view.queryset.model) diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index dc7b4a977..c10ee0c3c 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -150,7 +150,7 @@ def handle(self, *args, **options): loc = values.get('LOCATION', '') spec = urlparse(loc) if spec.scheme == 'redis': - RedisServer.objects.get_or_create(hostname=spec.netloc, + RedisServer.objects.get_or_create(hostname=spec.hostname, port=int(spec.port)) if os.environ.get('AUTOCREATE_USERS'): diff --git a/src/unicef_security/admin.py b/src/unicef_security/admin.py index 88bcafbfc..732b68f9c 100644 --- a/src/unicef_security/admin.py +++ b/src/unicef_security/admin.py @@ -49,7 +49,7 @@ class FF(Form): @admin.register(User) class UserAdmin2(ExtraUrlMixin, UserAdmin): - list_display = ['username', 'email', 'is_staff', 'is_superuser'] + list_display = ['username', 'email', 'is_staff', 'is_superuser', 'is_linked'] list_filter = ['is_superuser', 'is_staff'] search_fields = ['username', ] fieldsets = ( @@ -68,12 +68,12 @@ class UserAdmin2(ExtraUrlMixin, UserAdmin): 'fields': ('username', 'password1', 'password2'), }), ) + readonly_fields = ('azure_id', 'job_title', 'display_name') - # @link() - # def sync(self, request): - # from .tasks import sync_users - # sync_users.delay() - # self.message_user(request, "User synchronization scheduled") + def is_linked(self, obj): + return bool(obj.azure_id) + + is_linked.boolean = True @action(label='Sync') def sync_user(self, request, pk): From cfb7d51e8643db2d286c378250d61a175b58355d Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 6 Dec 2018 11:36:21 +0100 Subject: [PATCH 14/86] fixes creation of GroupAccessControl entries --- Makefile | 3 --- src/unicef_rest_framework/admin/acl.py | 21 +++++++++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 2acaf02f7..b7528c32b 100644 --- a/Makefile +++ b/Makefile @@ -30,9 +30,6 @@ lint: pipenv run pre-commit run --all-files pipenv run pre-commit run --all-files --hook-stage push pipenv run pre-commit run --all-files --hook-stage manual -# pipenv run flake8 src/ tests/ -# pipenv run isort -rc src/ --check-only -# pipenv run check-manifest clean: rm -fr ${BUILDDIR} dist *.egg-info .coverage coverage.xml .eggs diff --git a/src/unicef_rest_framework/admin/acl.py b/src/unicef_rest_framework/admin/acl.py index 66ffc0d75..1d3648528 100644 --- a/src/unicef_rest_framework/admin/acl.py +++ b/src/unicef_rest_framework/admin/acl.py @@ -39,6 +39,7 @@ def get_queryset(self, request): class GroupAccessControlForm(forms.Form): + overwrite_existing = forms.BooleanField(help_text="Overwrite existing entries", required=False) group = forms.ModelChoiceField(queryset=Group.objects.all()) policy = forms.ChoiceField(choices=AbstractAccessControl.POLICIES) services = forms.ModelMultipleChoiceField(queryset=Service.objects.all(), @@ -61,6 +62,9 @@ class GroupAccessControlAdmin(ExtraUrlMixin, admin.ModelAdmin): def get_queryset(self, request): return super(GroupAccessControlAdmin, self).get_queryset(request).select_related(*self.raw_id_fields) + def has_add_permission(self, request): + return False + @link() def add_acl(self, request): opts = self.model._meta @@ -82,9 +86,18 @@ def add_acl(self, request): form = GroupAccessControlForm(request.POST) if form.is_valid(): services = form.cleaned_data.pop('services') + group = form.cleaned_data.pop('group') + overwrite_existing = form.cleaned_data.pop('overwrite_existing') + for service in services: - GroupAccessControl.objects.get_or_create(service=service, - **form.cleaned_data) + if overwrite_existing: + GroupAccessControl.objects.update_or_create(service=service, + group=group, + defaults=form.cleaned_data) + else: + GroupAccessControl.objects.update_or_create(service=service, + group=group, + defaults=form.cleaned_data) self.message_user(request, 'ACLs created') else: @@ -92,8 +105,8 @@ def add_acl(self, request): 'policy': AbstractAccessControl.POLICY_ALLOW, 'serializers': 'std'}) ctx['adminform'] = AdminForm(form, - [(None, {'fields': [['group', - 'policy'], + [(None, {'fields': ['overwrite_existing', + ['group', 'policy'], 'services', ['rate', 'serializers']]})], {}) From 0f01f76a2ca2fc44c21ed5ce647ece25a8ff9e80 Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 6 Dec 2018 15:46:16 +0100 Subject: [PATCH 15/86] more consistent admin page --- .../migrations/0002_auto_20181206_1420.py | 38 ++++++++++ .../migrations/0003_auto_20181206_1420.py | 18 +++++ .../apps/web/templates/admin/base_site.html | 12 +++- .../apps/web/templates/admin/index_new.html | 29 ++++++++ src/etools_datamart/config/admin.py | 70 +++++++++++++++++++ src/etools_datamart/config/settings.py | 2 +- src/unicef_rest_framework/apps.py | 2 +- src/unicef_rest_framework/auth.py | 1 - src/unicef_security/admin.py | 5 +- 9 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py create mode 100644 src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py create mode 100644 src/etools_datamart/apps/web/templates/admin/index_new.html diff --git a/src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py b/src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py new file mode 100644 index 000000000..2e8f666e5 --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py @@ -0,0 +1,38 @@ +# Generated by Django 2.1.4 on 2018-12-06 14:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='famindicator', + name='last_modify_date', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='hact', + name='last_modify_date', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='intervention', + name='last_modify_date', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='pmpindicators', + name='last_modify_date', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='userstats', + name='last_modify_date', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py b/src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py new file mode 100644 index 000000000..45e24ff12 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.4 on 2018-12-06 14:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('subscriptions', '0002_auto_20181129_0824'), + ] + + operations = [ + migrations.AlterField( + model_name='subscription', + name='type', + field=models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel'), (3, 'Email+Pdf')]), + ), + ] diff --git a/src/etools_datamart/apps/web/templates/admin/base_site.html b/src/etools_datamart/apps/web/templates/admin/base_site.html index 0e505c3e8..294040670 100644 --- a/src/etools_datamart/apps/web/templates/admin/base_site.html +++ b/src/etools_datamart/apps/web/templates/admin/base_site.html @@ -3,7 +3,7 @@ {% block title %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} {% block branding %} -

      {{ site_header|default:_('Django administration') }}

      +

      {{ site_header|default:_('Django administration') }}

      {% endblock %} {% block userlinks %} @@ -11,8 +11,16 @@

      {{ site_header|default:_('D {% if user.is_superuser %} {% url 'sys-admin-info' as sysinfo %} {% if sysinfo %} - Sys Infos + / Sys Infos {% endif %} + / Switch index {% endif %} {% endblock %} {% block nav-global %}{% endblock %} +{#{% block footer %}#} +{# #} +{#{% endblock %}#} diff --git a/src/etools_datamart/apps/web/templates/admin/index_new.html b/src/etools_datamart/apps/web/templates/admin/index_new.html new file mode 100644 index 000000000..1459c4d38 --- /dev/null +++ b/src/etools_datamart/apps/web/templates/admin/index_new.html @@ -0,0 +1,29 @@ +{% extends "admin/index.html" %}{% load i18n %} +{% block content %} +
      + {% if groups %} + {% for section, apps in groups.items %} +
      + + + {% for model in apps %} + + {% if model.admin_url %} + + {% else %} + + {% endif %} + + {% endfor %} +
      + {{ section }} +
      {{ model.label }}{{ model.model_name }}
      +
      + {% endfor %} + {% else %} +

      {% trans "You don't have permission to view or edit anything." %}

      + {% endif %} +
      +{% endblock %} +{% block sidebar %} +{% endblock sidebar %} diff --git a/src/etools_datamart/config/admin.py b/src/etools_datamart/config/admin.py index 3c14752fa..97b9904bf 100644 --- a/src/etools_datamart/config/admin.py +++ b/src/etools_datamart/config/admin.py @@ -1,7 +1,12 @@ +from collections import OrderedDict + from django.contrib.admin import AdminSite from django.contrib.admin.apps import SimpleAdminConfig +from django.core.cache import cache from django.http import HttpResponseRedirect +from django.template.response import TemplateResponse from django.utils.translation import gettext_lazy +from django.views.decorators.cache import never_cache def reset_counters(request): @@ -37,6 +42,71 @@ def get_urls(self): ] return urls + @never_cache + def index(self, request, extra_context=None): + style = request.COOKIES.get('old_index_style', 0) + if style in [1, "1"]: + return super(DatamartAdminSite, self).index(request, {'index_style': 0}) + else: + return self.index_new(request, {'index_style': 1}) + + @never_cache + def index_new(self, request, extra_context=None): + key = f'apps_groups:{request.user.id}' + app_list = self.get_app_list(request) + groups = cache.get(key) + if not groups: + sections = { + 'Administration': ['unicef_rest_framework', 'constance', 'dbtemplates', 'subscriptions'], + 'Data': ['data', 'etools', 'etl'], + 'Security': ['auth', 'unicef_security', + 'unicef_rest_framework.GroupAccessControl', + 'unicef_rest_framework.UserAccessControl', + ], + 'Logs': ['tracking', 'django_db_logging', 'crashlog', ], + 'System': ['redisboard', 'django_celery_beat', 'post_office'], + 'Other': ['unicef_rest_framework.Application', ], + } + groups = OrderedDict([(k, []) for k in sections.keys()]) + + def get_section(model, app): + fqn = "%s.%s" % (app['app_label'], model['object_name']) + target = 'Other' + for sec, models in sections.items(): + if fqn in models: + return sec + elif app['app_label'] in models: + target = sec + return target + + for app in app_list: + for model in app['models']: + sec = get_section(model, app) + groups[sec].append( + {'app_label': app['app_label'], + 'app_name': app['name'], + 'app_url': app['app_url'], + 'label': "%s - %s" % (app['name'], model['object_name']), + 'model_name': model['name'], + 'admin_url': model['admin_url'], + 'perms': model['perms']}) + + for __, models in groups.items(): + models.sort(key=lambda x: x['label']) + cache.set(key, groups, 60 * 60) + + context = { + **self.each_context(request), + # 'title': self.index_title, + 'app_list': app_list, + 'groups': dict(groups), + **(extra_context or {}), + } + + request.current_app = self.name + + return TemplateResponse(request, 'admin/index_new.html', context) + class AdminConfig(SimpleAdminConfig): """The default AppConfig for admin which does autodiscovery.""" diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 769ab04a6..af383a3f6 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -268,7 +268,7 @@ 'etools_datamart.config.admin.AdminConfig', 'admin_extra_urls', - 'unicef_rest_framework', + 'unicef_rest_framework.apps.Config', 'rest_framework', 'oauth2_provider', 'social_django', diff --git a/src/unicef_rest_framework/apps.py b/src/unicef_rest_framework/apps.py index a694414c5..48e1ee0a3 100644 --- a/src/unicef_rest_framework/apps.py +++ b/src/unicef_rest_framework/apps.py @@ -4,4 +4,4 @@ class Config(AppConfig): name = 'unicef_rest_framework' - label = 'API Configuration' + verbose_name = 'API Configuration' diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index 54ce59bec..8a4d64414 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -40,7 +40,6 @@ def authenticate_credentials(self, payload): if not username: msg = _('Invalid payload.') raise exceptions.AuthenticationFailed(msg) - created = False try: user = User.objects.get_by_natural_key(username) except User.DoesNotExist: diff --git a/src/unicef_security/admin.py b/src/unicef_security/admin.py index 732b68f9c..f1ee95ac6 100644 --- a/src/unicef_security/admin.py +++ b/src/unicef_security/admin.py @@ -49,9 +49,10 @@ class FF(Form): @admin.register(User) class UserAdmin2(ExtraUrlMixin, UserAdmin): - list_display = ['username', 'email', 'is_staff', 'is_superuser', 'is_linked'] + list_display = ['username', 'display_name', 'email', 'is_staff', + 'is_superuser', 'is_linked', 'last_login'] list_filter = ['is_superuser', 'is_staff'] - search_fields = ['username', ] + search_fields = ['username', 'display_name'] fieldsets = ( (None, {'fields': (('username', 'azure_id'), 'password')}), (_('Personal info'), {'fields': (('first_name', 'last_name',), From dd92c5ffe29655ebdd5514bfdb0052a25eef6ca5 Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 6 Dec 2018 23:09:54 +0100 Subject: [PATCH 16/86] updates --- CHANGES | 2 + docker/Makefile | 2 +- src/drf_querystringfilter/backend.py | 2 + src/etools_datamart/api/endpoints/__init__.py | 1 + .../api/endpoints/unicef/__init__.py | 2 + .../api/endpoints/unicef/business_area.py | 13 ++++ .../api/endpoints/unicef/region.py | 13 ++++ .../api/endpoints/unicef/serializers.py | 15 ++++ src/etools_datamart/api/urls.py | 3 + .../apps/data/migrations/0001_initial.py | 12 ++-- .../migrations/0002_auto_20181206_1420.py | 38 ---------- .../apps/etl/migrations/0001_initial.py | 2 +- .../multitenant/postgresql/public.sqldump | Bin 4047913 -> 4047913 bytes .../subscriptions/migrations/0001_initial.py | 4 +- ...129_0824.py => 0002_auto_20181206_1447.py} | 2 +- .../migrations/0003_auto_20181206_1420.py | 18 ----- .../apps/tracking/management/__init__.py | 0 .../tracking/management/commands/__init__.py | 0 .../tracking/management/commands/track.py | 68 ++++++++++++++++++ .../apps/tracking/migrations/0001_initial.py | 2 +- ...129_0824.py => 0002_auto_20181206_1447.py} | 4 +- .../apps/web/templates/base.html | 14 +++- src/etools_datamart/config/settings.py | 14 +++- .../migrations/0001_initial.py | 2 +- ...129_0824.py => 0002_auto_20181206_1447.py} | 8 +-- src/unicef_rest_framework/renderers/html.py | 2 +- .../templates/renderers/html/html.html | 2 +- .../templates/rest_framework/base.html | 10 ++- .../migrations/0001_initial.py | 3 +- src/unicef_security/models.py | 1 + src/unicef_security/tasks.py | 7 ++ 31 files changed, 185 insertions(+), 81 deletions(-) create mode 100644 src/etools_datamart/api/endpoints/unicef/__init__.py create mode 100644 src/etools_datamart/api/endpoints/unicef/business_area.py create mode 100644 src/etools_datamart/api/endpoints/unicef/region.py create mode 100644 src/etools_datamart/api/endpoints/unicef/serializers.py delete mode 100644 src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py rename src/etools_datamart/apps/subscriptions/migrations/{0002_auto_20181129_0824.py => 0002_auto_20181206_1447.py} (93%) delete mode 100644 src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py create mode 100644 src/etools_datamart/apps/tracking/management/__init__.py create mode 100644 src/etools_datamart/apps/tracking/management/commands/__init__.py create mode 100644 src/etools_datamart/apps/tracking/management/commands/track.py rename src/etools_datamart/apps/tracking/migrations/{0002_auto_20181129_0824.py => 0002_auto_20181206_1447.py} (97%) rename src/unicef_rest_framework/migrations/{0002_auto_20181129_0824.py => 0002_auto_20181206_1447.py} (97%) create mode 100644 src/unicef_security/tasks.py diff --git a/CHANGES b/CHANGES index 33022d661..8b0502f0f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,9 +1,11 @@ 1.8 (dev) --------- +* WARNINGS: migration reset * add ability to create 'per model' custom templates for html/pdf renderers new `TEMPLATE_CACHE_URL` enironment variable * fixes error in some renderers with cached response * Adopting of RabbitMQ to prevent message loss +* new admin index page 1.7 diff --git a/docker/Makefile b/docker/Makefile index 29994e6d1..64e2ed5f4 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -80,7 +80,7 @@ local: $(MAKE) .run release: - docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} + echo ${DOCKER_PASS} | docker login -u ${DOCKER_USER} --password-stdin docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:latest docker push ${DOCKER_IMAGE_NAME}:latest docker push ${DOCKER_IMAGE_NAME}:${TARGET} diff --git a/src/drf_querystringfilter/backend.py b/src/drf_querystringfilter/backend.py index cf5629c6e..7671e1085 100644 --- a/src/drf_querystringfilter/backend.py +++ b/src/drf_querystringfilter/backend.py @@ -137,6 +137,8 @@ def _get_filters(self, request, queryset, view): # noqa for fieldname_arg in self.query_params: raw_value = self.query_params.get(fieldname_arg) + if not raw_value: + continue negate = fieldname_arg[-1] == "!" if negate: diff --git a/src/etools_datamart/api/endpoints/__init__.py b/src/etools_datamart/api/endpoints/__init__.py index b0965583a..4430b3d65 100644 --- a/src/etools_datamart/api/endpoints/__init__.py +++ b/src/etools_datamart/api/endpoints/__init__.py @@ -2,4 +2,5 @@ from .datamart import * # noqa from .system import * # noqa from .etools import * # noqa +from .unicef import * # noqa from .openapi import schema_view # noqa diff --git a/src/etools_datamart/api/endpoints/unicef/__init__.py b/src/etools_datamart/api/endpoints/unicef/__init__.py new file mode 100644 index 000000000..1976cdd76 --- /dev/null +++ b/src/etools_datamart/api/endpoints/unicef/__init__.py @@ -0,0 +1,2 @@ +from .region import RegionViewSet # noqa +from .business_area import BusinessAreaViewSet # noqa diff --git a/src/etools_datamart/api/endpoints/unicef/business_area.py b/src/etools_datamart/api/endpoints/unicef/business_area.py new file mode 100644 index 000000000..b2cf8ec3e --- /dev/null +++ b/src/etools_datamart/api/endpoints/unicef/business_area.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from unicef_security import models + +from . import serializers +from .. import common + + +class BusinessAreaViewSet(common.APIReadOnlyModelViewSet): + pagination_class = None + serializer_class = serializers.BusinessAreaSerializer + queryset = models.BusinessArea.objects.all() + filter_fields = ('code', 'name', 'long_name', + 'region', 'country', 'last_modify_date') diff --git a/src/etools_datamart/api/endpoints/unicef/region.py b/src/etools_datamart/api/endpoints/unicef/region.py new file mode 100644 index 000000000..f285c095c --- /dev/null +++ b/src/etools_datamart/api/endpoints/unicef/region.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from unicef_security import models + +from . import serializers +from .. import common + + +class RegionViewSet(common.APIReadOnlyModelViewSet): + pagination_class = None + serializer_class = serializers.RegionSerializer + queryset = models.Region.objects.all() + filter_fields = ('task', 'table_name', 'result', + 'last_success', 'last_failure') diff --git a/src/etools_datamart/api/endpoints/unicef/serializers.py b/src/etools_datamart/api/endpoints/unicef/serializers.py new file mode 100644 index 000000000..b2891bb67 --- /dev/null +++ b/src/etools_datamart/api/endpoints/unicef/serializers.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from rest_framework import serializers +from unicef_security import models + + +class BusinessAreaSerializer(serializers.ModelSerializer): + class Meta: + model = models.BusinessArea + exclude = ('country',) + + +class RegionSerializer(serializers.ModelSerializer): + class Meta: + model = models.Region + exclude = () diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index c72b02955..8b20694a8 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -32,6 +32,9 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'datamart/user-stats', endpoints.UserStatsViewSet) router.register(r'datamart/hact', endpoints.HACTViewSet) +router.register(r'unicef/business-areas', endpoints.BusinessAreaViewSet) +router.register(r'unicef/regions', endpoints.RegionViewSet) + router.register(r'system/monitor', endpoints.MonitorViewSet) # urlpatterns = router.urls diff --git a/src/etools_datamart/apps/data/migrations/0001_initial.py b/src/etools_datamart/apps/data/migrations/0001_initial.py index 9bbc51de0..8c4e74b38 100644 --- a/src/etools_datamart/apps/data/migrations/0001_initial.py +++ b/src/etools_datamart/apps/data/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.contrib.postgres.fields.jsonb import month_field.models @@ -19,7 +19,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('spotcheck_ip_contacted', models.IntegerField(default=0, verbose_name='Spot Check-IP Contacted')), ('spotcheck_report_submitted', models.IntegerField(default=0, verbose_name='Spot Check-Report Submitted')), @@ -49,7 +49,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('year', models.IntegerField()), ('microassessments_total', models.IntegerField(default=0, help_text='Total number of completed Microassessments in the business area in the past year')), ('programmaticvisits_total', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), @@ -69,7 +69,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('created', models.DateTimeField(auto_now=True)), ('updated', models.DateTimeField(null=True)), ('document_type', models.CharField(max_length=255, null=True)), @@ -122,7 +122,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('vendor_number', models.CharField(db_index=True, max_length=255, null=True)), ('business_area_code', models.CharField(db_index=True, max_length=100, null=True)), ('partner_name', models.CharField(db_index=True, max_length=255, null=True)), @@ -164,7 +164,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('total', models.IntegerField(default=0, verbose_name='Total users')), ('unicef', models.IntegerField(default=0, verbose_name='UNICEF uswers')), diff --git a/src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py b/src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py deleted file mode 100644 index 2e8f666e5..000000000 --- a/src/etools_datamart/apps/data/migrations/0002_auto_20181206_1420.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:20 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('data', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='famindicator', - name='last_modify_date', - field=models.DateTimeField(auto_now=True), - ), - migrations.AlterField( - model_name='hact', - name='last_modify_date', - field=models.DateTimeField(auto_now=True), - ), - migrations.AlterField( - model_name='intervention', - name='last_modify_date', - field=models.DateTimeField(auto_now=True), - ), - migrations.AlterField( - model_name='pmpindicators', - name='last_modify_date', - field=models.DateTimeField(auto_now=True), - ), - migrations.AlterField( - model_name='userstats', - name='last_modify_date', - field=models.DateTimeField(auto_now=True), - ), - ] diff --git a/src/etools_datamart/apps/etl/migrations/0001_initial.py b/src/etools_datamart/apps/etl/migrations/0001_initial.py index 034d0d20f..92dd230b3 100644 --- a/src/etools_datamart/apps/etl/migrations/0001_initial.py +++ b/src/etools_datamart/apps/etl/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.contrib.postgres.fields.jsonb import django.db.models.deletion diff --git a/src/etools_datamart/apps/multitenant/postgresql/public.sqldump b/src/etools_datamart/apps/multitenant/postgresql/public.sqldump index 0ec2fa00419d218e71c19af840b9c9ae5543f83f..9dd5980c33d637f5a36de54b87ddc94ef68ec7e1 100644 GIT binary patch delta 16017 zcmZ8|2YeM(_WzrbMj*WO-Uuz!d1YqaP-6!a38E+n5@dBjMS^7!5dt<41cD32wrK!3ie>)y^%#`qG$ZJYF zSHh!FcXWD%J7H{w*EiVR$6MaGjHy1}aZ|I4+|A7y?%ETDUNGxB2{%LD>FkY?-n8kH z5;ezNw5&7DU!vr>)ANJw$-Mmud;u1HIY2FOk1WsdIu#_#4px}94^(rgu$PkIomX^N zuEL523TVY3wb&b3(oc?J+1`TETm4Rs_j~z;5^jg-OsSghO{)0O2RO7~uv+HTcB_=s zv5@z8kGCZFQeAY^5i zT%)G)ZHwB09uRYP=BOTC)1`3&$15LQEqRz>9DVsCd{Zt9$qAUzK)PEsHp|<0`2=x1 zwqm?g zQE~#YP@v3RF}cW%Z{UnQg>GC;#rcXDYt^G# z2luHx~g3VecwQW-qz06rN#Gb@LQTO2N3UAXLF){yG$bw=4U6GeGXQdnfl2^t- z#ct=hiEhuUGhMzfljfDV?^lj-52WOa-#7ll^7hP)$u8kgph9qQYwzjc4X%?i9t&HL zQ_h^4UsqGTng#Rapy61ch-#aaQZMP=A(8_WT!jF+#=;~%DD~}y9Ww5(;G@#^UP``a z-2bRtL?q(1EL<%2G~#$Qi?5aNXxJUPv@fk2qx5xKD3=-7@@`ONF#KDPl>3itXf7O+K7b}nlTsAd*h7)>j^)s@I6%KR};ObYj zpm*1!=lNhWP#FiOdq*FeBbiuspj3cs_ozv3RYM|8dsxYI$4x5qn%8B^EwUVMZ$q+V zV?$o)jUAASmXEM6{B)t>UE`U~wEH6^$s6{B)R8ee>XkiZ`<-mu)TS?>?O%U7$-C?6 zUXrivc$+tDk-uZ6H)*3ZHL;lO-LcsecO~XT-D6Gp-pQkl2yYr>75)y*>;}Cb(t4ZF;SBCje zF3xrTP>QH!f)ey*y`G}+%jkr;W)pCuMk(L|?S0W)Ap3;PK(1T*Zh@Em)-z%@P9)^L z@%Chig-*DA0&F8ye~Df6AcHtf~w4&|6vlnT&QGH)i8)} z?Z>%p+ee+;;SY{+>+AE|zgLg4ypbQBFZsb5w|!jUy|PnYNhfLr(rN1fEz3*WT`Q*# zG0NkhZ11s8yUTf*e5IU zSI>?O1;N#G&AuG3X>X3~Z1csI9nAN>-T%3qob9+J2P@nk+A`cz-&VNu4rYt{xN9u# zQuEiz9N@ir;5s2E4isvZTXi_s8~T;Mld-5fl1Df~jjB@4<|>L@{%6`ci3ayM< zDpe@yEG3D}&QgT5ScVy5@8u|0N-9WT2G#$pWwU{KN}dnHrR1<2nm;iHfmvb0pzR-s%i=V^!8=x)kdE(w-l zM`14x_Ekz*c28xG9Ne}86#_={dMnB9&c+M|i*df+fv*W2D%hGn%3#Sc7G|w|6_S&S zMcG}3a=v60vq3#50_q<_%5VivpJh1I6jl1svRO(7HI2diUJff|G%11|dNrctNG?v8 z?J<=H_=Q4`SRfx4idJ;fGTA&^=`CnNhQS&g)=RyihzG7UE4 zD#ev+Fe8DUH1jH@E6o_MCR1HEr7x$?p#y`pj#N8dO=L^PV-{QhLZ*om>X(A$L9Lry zCt4vWx*ie+m41tytrca@UaM4UojHMR1hQ}-x8Hi*E$-GM6?E6bIL4uOD2Y^;ttYcS z*DEb@DzJ&!R5k&dUo=&zkaM+7Dw&{mp;ro(4ENv0*^Zs5eu9dlo2E!D6S8gAaHAr; z1uGOY*xs9z9zr5Q@E$XTM9fe!S=r4>l3Yy8gnkbN)hv3(P=ahst@5>;y%VCLrrMiL zy;ZqL&I~HB4=3%(T5nU9_*53Fouv$xydZSgv2^H?U9**W0v3kPSS(=FIte?=<1KL- zV(EQP>p>1rVmt1_xp2u18>ZMOT7S2a#dgk9x^oBK3d4mgh@$2jc42eC#j+2Kft?Ox`KYZ&r4x8B!z;fTYYeMosi_KBLT`e9&_ z?T8u0+;e#Z%^qH*T!OvipfHpOLg-?auv!_9J{%Td*F6FSAqk>TXQjos3%whZY--e@ z5*wx>Vy%BnNvGyVlpb{EQ6<&gk)A}4?oc2IuPM21pN<`dTy8Tw#nxy8km1~t8e;PL1x=y*mp9Xj4c|^=Gjca@Y+}QQX zWcd*`*e_2gqa`&w?`#U*qvo*do>DIG3Bs)DX=R%K5n-hp;=IGrKsOn-Ew?+;4f`CF7)A07jI*-Eu5D{54Dq=K^v zJ8VT0)-3ikg1gKELzeb}a-PIPEUxsFbNr7mTlaV6cAq!`UmS5Av$w&V$o_DkcpyjL zZNi#cpT?R;y{x2A?Mq5AKiVRCew+O4=x&*oL`ybfsnstjTV-z(n~YE{hn2pJO_ne# zkjulcJa+FZ%Edm68^rdYi?zL~T;?Nqz{3L}cKz$^Q*r{`+hZnne7iDRE(M1C=4RzS z*)fW|58V5Fmi1HiNkMVi>jA(2wF` z80C5h5dz$koystumBUVbEUYh16S}`@cYN;DrR`l1sXv{y;@C*-g(`{K*B3=9ZAnZ2_= zKA^=l4t!r`wL;^h@&RF?xJTfQ{x*m7)p36;$bE{ zTvqap!V@2SVtgl>x9Hhy@wdvkJ}d@#-l{6O?8oo006CJw2LBM(Dz?EE{RsC+aNyB{ zbWx8V0ZdTCWgBuVeqj+wK!?DqWN#f8J{^X&qpa&sxUrH0zf;1Yy!$^2&+u^qi|zVZ znd^VROFW@G;(x?g+b{7;Ve{0O@a`IZjcbsY0hsXN2L2A1U>6IqttXX8UB`BZa-(t+X1_!=(ez!?+n&a4wkh~~%OHN(?Mr4LTP@kvX70JJ+kRPu|`o#rR2 z&xmC>2J4feu9PD?aLPNV{wW~&lqVe*qoX>?Cy227I;p-TaH6awO&#ZBOx8cBKIeZ} z?8|h_T8@CN&Q!1SKVs~)EY&|H$H6JBxEi4d^oL|>!lS)D*Jo-PA@)ul=EM1zMwn$4 zsQfV7v)c9W)l92BeN9MVhdShS)@yHKmzoNdLZ zxm;byshxNPOjEn4vjij_)Y54{ICliGZq+wML>21a()qU&3UtOjuf6@+o2E^nNnv$7 zHGGTd%0md(ng^Qh`jfewbu~>t0v;cC7d#L^ ziWswA@Cci9w9d!>B8GkrRa;joz_b% z9MN7pz@tvR{k~R+5VV!i#@>G4KgVixNguI6e~QlNhjvgr{-Ou_N~*IF4!>(j)T^K5 z8498JaW0JTTXBF4WFFxY422y+q*6+!`uo%on-Go^(MrPy$dF`>pf(Lce9?nO_JKh3 z4y)Y+)QZyBu-u?fyQfcQ!g7&t-wEz}cX}e?bBq~O5>*%R4Go=j)!&Tz^wLRRFVJVb zZ^cr#^h%DIv1rH$M7ZZkEipZZKANniup>3N6A~E?^lL}5zh0}Jt0am}QPwy`eb^_m z0>}?cQEmF>kZMxdU(}1}^mXdLC8fi@yk14ply`&kH%xl04MQr|-Q8B^5rMv;sF_rKzMjD*%v2G2@`**Pfb}QDtGB3slS3k9Vc(|qmsC+B zFhF{`OKt~&=n3y;lzb7@*;E9W=VjOLmyhP9!Bo?JoPf4Aj&S9ua5CQEY`39=532B{`IHc)4qc{t{>Vk9QdA0Cmk!euAGyU=foG5FB8FnHTTY8jjMuv#n!iJ9!dmFmlWM>{Z*`}7x3(`q%9 zttIs?pUPpSkEp0j@p(aa_TfdK-sClaYf+tCQZiMWdMY(OqV}ZB_3+!5{4e73$Dp=E zf20vaU@&J5w?YDBvJcm)W5tJQL|FI7)f@Z|ldWn{QAYupiOiox4S!dP*opNj%31>E z1Y#m`xc5nQrUW69jEczNv1im=v8#k#&87jH)Na)FjB0UMKE3|5x}A@1BBWgKj5)h;Ns1`A8P3e?S0Jz5zr}U!@QS+D zr*k0YoE5JP(KD~B-PrH1s`Dfg@y%83NCxCiAI_x?6{0fs=nh{(;lKxmNZ9x{)H=CV z1R$fN1eQa8c)cq9Se(NHZ{^ z9m#-P=fmy500HOaBzEyQjQUeuw4gKPN9|449@%NaQVC9%>!4JMEgpY^Jqlg0wj&M&Jyx_z@ z-rD0pw&?^=IEDK>CtZo$#k@Ao^bMZn-c`WqZ`F&ST@Ru`urc|IXotMRcWcyKHO zR*I~JcLgySS&mNCq>{F9^Lxo)vbCco@2G`aKS00;uq>L?8UKx)3vpeTrd=*)g1o}P z&YH9b7Lo`9sO4v+g!X4>g=}KFCcPX)9f6Akf&uY_oS5mr&Vo3fp>FY%gyA-i5-?uN z=toBo-LZ8!n$-Rl?(_&*uSk0nS z1BIGY4wl7@H;;a)xlk)(DJ9x%k`eB5HO(`Wo@`60CVdPGKE{PIEM8l#N&nJniX7DV3iJiYvVk(NOvs97jbr4Ha?$cKFTVG4zAoHUH+|77h}X z3lT%pno3MIzo#ZWAy){yUrXWkuQ#E(qQt3JVOP70kKRDgquA~hW7LJ>7}fGpnre}4Owuo z_D`RRYZ#O;y0YL<&6P0t@4e`c!+I%g$<$NXXI0uD9}x+hF9LgN_(9L015*{9cApE4 zT63;;w@+dQjQ@{><(=gej+Wm(P8#-1?WH(=-$q>v&2r%SN z*tRRR40dX)Hbw3NqTqgf7epzf?kXrd$ub5lHCjmsvEXCleOFkrIso_kRTS@HC^k*=1FsB4(=<8DXoPLO9m|&8qlAw#Rx(Q~_aP>`;0{;;$&js^t#PZ#N3wBr2oPbk=3t9T z=3o$?sK;Ge=lJh|b0MRGw`sVczIklc)u2rqaYd$(~5hw3U>2C z2#Gih*q=)5Bp%$Av)>kJJmkdSR+xnrL-omz2>WP>mhIC=0|wvuK}$7*6)wf0OB$2a zEE9@|_vX+44%EZnsjb&Ku=kd0NQVoyHv6((++R)=VCBZ)&JaWrpbKUrkpbncW z9>UR@hqaSpn<39d5_H2Vu-(2=%aZFtf+kNQj<3?rNLpUbk+r)@cPkYllsKTo^{I8QJIl;sGALwjTQAy9S{OI07|P9^YhA z)01!qE_(u-BIGINuD=TvlP9$xt9w!_lY=22EuPVdM|+v<7OY$qGmJwkPiaU6ZGvWByGh$8HaF(5y3N7^!$KXTy?Ag|N>$Bj2Uh;P z@BlahQi-snE6r`fATKm(Avrp7*875tve@PhkFupz1{op}ZJZRgT5_ZxgO0XKssHmMC~7(!7^Ixr1rld@iv+BR_r_ z<1Fd}pM1jSS~?wIS}_UOz^QPRZrTb(DT2H{FKKDC{AKMau^kQ`vTWB%+?ur=Xv-^L zKkXF=5l7(-NIQys^r~>0IW!a~7Xcf8y2p&ywL+g7PhYlcJO{;zB5cPFe>Xw~e+R?g z;ZgrLpq)-OYdpN-RNUeS#H2TY==~Jw|xzXd@Q^09WBkrIBfj8 zTB6VkA!IPKg;IY{%clA7X>DA)gy8uI=+^fEt$$xTE}?ijA++}gfIj|#_LGF-*(`^C z@*$v2A8JR%@OaPi$sky>I>(j<5MMUnzy~c(99nJXjXgQ>e8wB;&NMr_=P$wE`NnM>`}2M{(UJ z3ufpk?2^y5BA>4X8RfOe`Y+G?nx)h619(YLx>x%}Y(O|hjRzoY z)AwOM1NOy_4q0&lKD{5{Yxl>G7nJ}3E^CGOKJcY>O7Mt;I5dx1TeT$G)T*79P$ZlW z#Mcei4QAedGMGZIeWfLD`%3E~4*?n5o`>RR6yZ26T-*oKWQgK zf0LGCKsNtBTC(JBM%l8Tv1MX-M3NQ{1dFNV1l~T2wcWKCYisxg8&RDB*Cv<+)zNxF z>%@kgh@YQnvFcyoP)Z(9S0XB8`~C~hN5WzV-$kU@mWS>`e@FLAf72%TQ*c<*@4}D8 zp};>V<MjJjB(#X2jxbT>kX zIkBNU@ViFUvn1Z8nZxm*s#@1OOJ%uhC6?GZfxFxs_QP#ZmS3C$l8(B*RV)Bzzq<(V zs80fJ$T3~_cMN9VN7N+(aZaMn{adWVju76!iPvd~)RKf}TPu@v={SY#sKfI6jsW zC4BT@8KwF#e?T~jWjfx=2@WXZ@zEQ~^$fWp7~HNrNXur3_IUD;j}r^@6)+yN6|zBH z~ zh4F~2jNhvPA-#;{hj5gsPB@BJX@PS%7{PNFHZiQHs)-yN4YQl0`hG=zpvZ3NHEJ@) zK&@QHBQAz2VTlj2kv>8JE{~-R)bYrhlcBaWw!OfY&2AX1XZr}-M=(_Wq7sHnG`_ub zmP${btDAISsE*pnBYM%QTh$J3urx@$s&pi+&ec~cVg!i&C8AQtcnME3!c&Q}jB;t{ zMS3Nb9@X=ZEzm!g-5{Eic(te-`+As;zaHX~g-2H{s$Oi&#d=>KW+UsxD`5dY>3UC8 zWwEti-KZZITN^<#VEq4(l@PmGFB2y~IDsPZc82X4PRHBSiVbP&hq)?rAwP|TKk{yH$94QN>DJA27KPgn2zE#fMWV!1Lib!gN|p_;KtwM zOn43#PJWnd`6eBI56LmmW81gr_)7=zfkEX1ue2t;zl6arzF6k%uYN)ABmF!zfx}|#jgR%4 zBp>KDiRT&RR5qbSHzW}rq<*Sj9sm8)KlRz-H)c0uk3LOI6@?_K-Gggi_;3H(;}J_> zvUC%W9v&il;IARR)Nd2RS$ND}BND@e#Bh*O4(a__!$G}+qztphL%6kkl6VC8+hKix z>;)BeMSJ=;NN2v&4gbb}r++5aX(2BrNf?sIy(J602Rov7l~YAp3fbH+>vJ6YECw*y zyr1+tBo=?J+jIyTIRF2A1BH47^m{yqn?rlopbo^OFGghQ%84|dL$}OygL%Ez=Se}?A zBxZxG-x*)4SnytIwS?Z(@oEGkqzONbh33Jf@@e(-0zY(+u>{wfX}Q3hDJ zIVk}x5iJQV87&2^16nFtN3>38X=t6%f@tYz8EBbkS!mg4IcT|Pd1(1)1!#q6MQFun zC1|B+WoYGSUC=7fx}tSM>yFj~ttVO~S}(NTXnoN7qV+@Tk7l5S(86dDv?!X1W})%5 V#LyhH0cZo!2C>aagXg~X{{U8x)|dbQ delta 16015 zcmZ8o2Y6LQ_J6me5lCKoZ-fA$zT5AkB`6}GNDxIqkf5szDiTx{5otCM6oOY6RKQqK ziV5D8QBjD!Aw+jwbO94YtRQh+T$T3!&CGpa_s{nY-@E6WIn&OZI(dHSf28Np%4#kg zHo8XY5>gW-NebcL0QR55{!3;5Dcj1XDe;EpYMM)G3*>zNdF9ViA_M4JsK8sL>{s;9Sjb=)FAn?fP2Z^TjqtBcUwsjFwB@xlsyx zPxn8?R}(e7Q(9NX$^U1hOpFw@e918QZ#|-Wi|rylQkVP!uWnR=*J37lW2^-KG3y_E znR>(wbtOYr$l3l%=idUPMM9MvL{dkp>0V@DoYyom-TyFp4WEpz>7jCgZ&@yfy{%Kj z{tbhypsy9`^zU&dUDVJ(u0;Us6^Qy;Z`UftX#Lq{! z41dV@!GgPG`HQa{E8vif&nFncZ_`h^Do@bZwjY^%G=N9F#kCcDTPIA32)v{DE2oYT z6L8E>skdrsp%>f0nfnU7@bo0V`r0%xAIA=5t$9w)^=_{Y`-`s|Ahymm{5PguE+||p zlnshJ@8#iPaTBK8_zSzVy6cA0$faZ#5slfaH}jP?489 zKi=zkRffm*Wy*q5@BPYgURzQg|9#`%Y=7T;H|SzQSQ!_u?(R1umhn9`4Bf9?c8!1=mRI#)U$SAG)YoevIi#Uc=|m#X2;Qi5<49{QcKCn6t?O(* z{lU><3WgIZ@`gT`??vk4{Us|bDGtYIm|p)?z5SyruM$%?bwNZPe^ZM0XFe1Sz-FkB z^(En()i{6e!@6Jzc`lEEhAgo|w(jq5NEB>r zh%0$x7sR6FBWw#hTH^z(kx>1Z@wpYpXvfc{&%8QSS zgNy2(^inx9IsQRkm?R@=`bpbI^KND&63ScisZ{9S@$&cp;)F^W#M`qY!9V%xhyco@ zdEwurLeerx3j1?jPg2-jG&QE!I6P7-Ai1$BvUnpeLutMEHRq8GJFoC>MAZ3v15s==x zk8`~CkGgmx9~kG=*XMP7uN`CiqdyuJ@N>Yg+<$eqxRIt24W*H7ZAzw}vbRo5A7Yft zKw17{pLQ4XGTBNQ6=~itCHhDHezAZ6aI zxtWrX7Rxm0d)d+z0hI$|R{x8VMF-|exdLY4QZiUJSu{qD=VDJ`*3!jNLt2qf@MeO*fL3Yc$Tdbd=!v|OeLANmrGZPdD<2oTOqCIl3>{m%*CO;QVGrK zDeV)3LnoDU7+KI;O7wO&rc)S)`Sk_bX^~5u;A!ew6(89#N-@@-WidG307_d z)PoFA|ENhL1+{CCZKl+hte7LElcsT);48XRN~Rdtq8)~mE!emg-DgSn^BZMFwNM^z z6j@cFWY7hU)LYO*4cZu$z7#`6q4UYMsFX>L4+N*Cfl?`-u?-7eK!*00b7}ivN%*&R z#G!|(q`91p9f{J~^P~fujjdUsVn$s+rwqfInVV{B4yo!dm($(DC7oL??Ch+OQg7OC zgmjf4(RI3Vl=K{5p{=`QmL%tq$Y`l^%*fOFi=}))YeZ<%CDJ&4Vm9O{O_RIR{+9+K zY#UKJYh19KwyBe;6XbI4+IxM&1!UJ1QW2duUiwK41xcEDrR0gZLznd=v#yl7k(m?a zM6$2~yA1kl(l%J>OzI}e@pSn_%!27{8(OZE^h?6BpwBI?3~huAJp>H8{1!1=%caj< zBULJ?tP@mQCJtiut=HY+Z97^{?tB=>S9QA-PZnmWiL}r4Qj3@h)Lj-SorFynf<1odph2>20Y(xsvadpzyV)m{@sw}xTop!5qp_mzb$3Bd-CvCk=S}v%f z5n4A#8Z3B4btuQSG$@fhbEO3V4E_s;k=7~LSr%yV)8I?*dqxd2cmmyZC(ec002|WC zHi|rPmy}6&FOa%($K8oop@INhKyw#M{rUI~#Buc9(sVwSqZ!Z#LzhZf-jWqzdiY)` ziSL60%~!P)P8>o%?&6X~_e-5gq^BBBYnDhy0wR|VygyJ4jvk>CmrH#FMmOmL4`7W< zq8;6)dshVF>KHnmUmqwt$1v%04@pmoK4yg0JPb^tVK~u<4P%bjnLfNmx)^)OKumie zcrK!GYo&?k!(cAG?h$APLBLFDNfGWs?*=K0G^)^r4bu>+);}htk>*FF9^~w!QnI%z zErC3`OM)Q0Cgpg2I(H%sbr4xl-OKqPqDv=l+$3kv$U5AUbT$yIMQ7~g)Y-R9n=<=W zvMoW*p{v(R*pA1*D~*ugs*su1DYN+*@<_$!wA`3LZls#nSx9s{j-Nh}qPlP@K} zamIgb&qH}ir#7HFmscRQqK;wI@&Vq);^d8k`X zzuzp~6u@G!EDIv(qODS005_n-SFD!1(ZWWF1t^TONmn&t$$U$7i+L2|Gil0;()j`2 zrdRmV(BQ+N8~!TI4v1a&;0WiavmItc^oPD>@f-PWGnU-?43<3R6)B0-y(|^6lPx4K zY!{!My{*#|$nq^%Y0b;hHs0HSbHF0FY+CXP_E^BQP!5Z}a_K#EdHU_zaE>CVTQVQgi7@IPHC=Kieb^4o27e$j&>*yZhaoje6xcWLJ4x{ zO}QJ%jh7>&Z8zNU;}3QscfK7vxzx9%fAK9eT(5RnA!)i!O&|r?awZ+}P6wALj~7Io zLg~9WHQoi6sO;Z*1>Otco1ETg@kVpTPw3i&lDUL! zC^JbD!iF{9$(pqqvu!3e4KcLl|6q zMCql6V@pTemrhn~Rx`xhOP+!#mo}=MH=g-g5=*q(M} zf$)O6eC4#^x0nWTEWo($Ht-L?1Us8L#&g)|!{eg8PRCB)afSPoOW(c7hNF{AJ~#otCnXL7(Bb!x;$NliWKp90EMEqk%RWi+>R@D>zSl_( zP63fouJBtTo#inBflKf0A_s=RjnbABd4j;eZ|EPEpASAX`ehns%}0o0t25;5f)9hf zmMI6P6g6>5tFA%_0{sy%jau~joDQZoeJ2<55qun)nJ=@$>_}(V|8L%!WERSEI2vx} zszUib{@XCTrj$YwEt3BtzFqQCk-VJ$HVrboSjIH~OOb4TfU|`6ClbcQEt1EQy?;>> z$^H_#m=lT=@k&nEk<5LeR1B?~2sp>)kzr+GINjDrbD6w^QJb;ImrUy_&*6|*FiXw= z!nh-N^{T%qB&1yaOB(ye*|JoA-Rx^_ntm;rqRSIW!?&2uqHbb3rW++scatCBoud}9 zE9B*jbw|)i4pj)&2r#`TPUbMyHDtz7@c6j9;1R@@^^6CCNV24dyp;de4U*MUzMu2! z2pGw0Jp+b@K@bk+kxMJZVl9IQeye$3(!)qM{Ll!sEqs&19ik*Re#M?pgxr)lcw5JVOxA6p^=L01>}D!X{|zIzgaP zLQeM&sEr6gG*U=bM`8dTk*wp?SPsIA9%OVM2t;pPuHaD1CF6CmL3W3uPtNLMk?`IL zy!Y1=CEv$(m%-;k~7!I{}hxC z{qlMlxl-OO8X6%S)`lt!);ditq%)_<4+y463rIJ}^98A4;JUP3k2vA+8|D0fz@ocu zl4(HTKtaK2Ehlr1Fc0`=IKQ>);00CBfUA)-Q+}6^k9hC1nes%@2mV2Ka%g~@L28Dn z>2%U88Rk=yDMmFxR|wmHb@*w*@V)`w+mplHZqM@T#{k zc>6>EZ2L|)jF)PGDF8y%5Jf3}M zk%&pVKQ7-8eAskNgN*VC$GDKuhQCUM^u!Y~s#y$%$j#-E!#z*QvjUJFGI`|i*t2pD z-&KNL%_0Lf%N3;kS=nZ=Jo5T8@=iXWX41va%5%hknoZAaloth@TogeNMO4zp=j8!{ z0738ZP4XSVhfZg1k(UJ@COx|~HVG*9(X6ghMGiE{Y4rU@`9Z-`cSGlM1YNfci{LXr z#-@ueZxjTm&*|qv5_08sIhP*$t2`>en&^HmU;%bhfUu#1c)_OMWVL|YbsbYgyQDn2 z@?{wn1h!X(8yeR^Qb@vE;LOZ@74q8sTU^&pugdEM9m;4SAthD5Kp1@amJf+IeJ^I?S_a>-t~m`s?eATRoc+|vuM4U_zv)ZVNr6i+VPBe(G_ zgdcu+M<0ql$PTitMdIojz_SP|{g7pU!$qC#=tokjOR^G|XL>C(<^wWcAfbF3Yfg^!?_5E&LSPW@LPG8&3e z{aDCI_CJlBTp0OV`r9wUB|vAFR-cGXS&!hDihqM~uKACw(xbo1y~M(h$u0a%MoN<{ z0vVG>e}`kv$Aik~#sb4~T6_|xDqwENVX(gB)G4_OU2zK5fg_OMi0~M(=nQNaXNEl6 zqO&rp&x|v275#W{_|h5W$^dWENwR`4f#Y3VOArWqPYs((UCX@3%GRTm{8XFe%WsSU6;crYvlR*0--x;PP5r=Vk#6``ax z-27f5m~8E=h&!s`)(_w?0$3)Ql8XPv&xg1!Nl`8nGeJy$C{+>mKtn(=fVBJ~6_bPM zN&%gmrf@Gu(=0c1AxBUko)iy4YkV(16DE5Xgd#T>*77CeR~>nbtXyq*g8 zgfxxa(Q{Y|>=%Mn)P+06B5X=}v-eJ5ucHQ6&0wXpdtc>VenfawL$~!)P6h-fR=8>u zOm<5|*%QE+rD8AwSOGn#E8_zMyVB=E@vF^X{W`AnIDhvm^Z74Ox&+-!xX66SWjShR+B#gR;zzC<@blP_4;`tL(NQCm z&0_xzWS&Oinlqh%mpo+Q&*}gYzC=kU$1lc7Z@EbMKrlx6u!b!DS#=pUn@qU`2D4>! zY__=OypEAcR{n-u)!wdZ9KCOhBD^5obO|yO*&IiY*C@7NiCf)Y)Ja$;dfBB4@~mtg zxZ2h^1ew`QO`!+IDafC5tcAlvNhF`_J*C2ij#uL8&6g`Y1Vdj3*Xr66g-KWzHPBt_UVo#(zz zO;QSoy&JCK?gsd@HItRzBx5ODiQZ4(vuU!@lN3!+uHo!)rK9A9gt#8`g((X6k~Q7Y zX=JJ*^skO<+>IQ&Rw*G1uT>K1&TEt&VqBQ!AFquaI&N(@Qh&WtLML3O^cMv1`&V4g z30MLY&x3gQxqh0GL{I-&Nxe11Ws=+>kzbgVGZW8LdfM zZiG@x<{wt5f^XLhrGhS+p%e%GY`ijKg%rB$W+f+pv6pAOxU!{AN!nhgWCSeSP%dN9 zJWI(XiHnu+nxB+3vhWrqj!c@RAn0XdM(Lnil!#yi!AJ6V0J7OjZ&g$&ju9AS)~&dw z&)uddVwSL8+h$|g0k&y2gt~Z+QWjuMdco~50s>1n%vG4tWPDhO2=QSy*)b10R6MU^ z0NUeDB{kMz;CzTEpP3o)w7$7?h;SP_Dc3bOECobqe;D3t+N<6s)TPcg(WxB)s@b&pa`Z(af+;fG-v zp-Su|-r1GW-?Dpc0`458}`TjYDf!a7Dy= zyU-(T_3(G<>XlCPy_E{m;hb%hep%1&FQ`z+FMfzC3HG7kZCfs#@5QzRmnVn6eRYvI59Y`VORd3j<&8- z%mAjbgf8CQk)JjwsdVTDC10?{{hazZH;h=bV?a}}cX#BqC!kNhYv8JYAxxC>d4SdQ z6x@MJpNwq^^40upT?PRnOcy?-lm-LZ7%&!+WzdhG#(;dRG(#6;Bqt49ZAHRZemiB>P zKIwBMjkHmvh;Z1zX>gZr+6HCG1HC>kD=B2 z{~ORyCz}-(VKFLJaN&q4ZvxT#O{gx>JxZ;&Aa?A-Mj$l!?LbK)B}<3Dqof2FolbmL ziRY@pju@dVuGimFvdE(Mly;_IY&<39&~5JnTK~RsTtF=WZT8HZ0k z1hnZx<(L@Wp-+8;>n1*2QnDBJ=EL0z%HVu}sDQ|OaJ@N1i%`%Aae)IGs$@_7$j%ms zI>pj<5N%Krh*N8`xF68!Qd0Nu)Iuf^kMw)5wg^m3%U2pK@3X9*vQO z)72z;@#jjR;0y1sJC9ZCzlf~{#+ZXhWWSO`E;uZ1SsFRshPMPI`;}k$1|S38*al&n zaRBofa3FSc$og~enS%gda}dYNa42sz_t zuk}m?GZUqkw>4ExJr~`p{y_Ijepe=mDOh-| z^#^xmAq}X3m#{m?9-@&KPAi@1;8U^dZlUCN8XhxW3Xo~w}>RZSHNbI)q5GBu95-VFA`ZBUqB48_x{&Z@eN zuM3ZlyYoPg^oheY`AJoSO@lQL5DVjh7#goKKNm}}vF0j7)_9~APg)Z2dTVupDm*9~ zaWcn+lMn{%PgI53u#uOq>Eu$#D+U$Xtr8Ndt}9y!E4lbo0dl>XJgknLpb$wa#WTk z!%(h9kL9Z2;6o?1rD_h%&sPI2iwBGhLRt&d#Ey#sDJ(@?$zcl%RFrlYBU4bWFH{pZ z-cTsahmGoUT~WtaHo-?9nqH!g2nK|=SgPX99Or<)U}B>;l&R@rH}TB78;jJkC_+A# zLBu$)!hJc61#Sg&P`B8oMK!vpLd6R(PG!*-yJHcei_Km|E`?v-VgK2EPa`$G)Xt<+ zPxUiF=ETSvDskFzm8ujlb7|*ZsxZZP?M8?9R$0jcFAiW3r}ydj(C7nwRpC22$UHpW zPaPDH8T3$pV3=)lUzwDBEPzASpfU^Od1xB-Hq z{AUUnf`Ew~^|NI1%z3Is+Nx9(P##eW*W4<1^1>xy(yLlU=IT6kwZun&mwz!Y*HKo- z6O7=ZpJS9msxDM3Ny#xa7byevbI}cP?i5xss-RzwQ1O>TY_jl#Yk294j=MhUTbVe$`)My^!xw>azr8J=sI^UD7Uh58fKD?|^?qT$IZ-ncU&nB;Rwb~_KwAF~nM1}*Q+v|3>tj~KxNA$tpI!%hX zpu9|wOPm>MPn2e{wO`$+9_L$&5`Spo|Bw|Edb3)}Pk`VA3i-P)y69#Ve=WjRgY4zG z)v%I5?wVV`Hsv(fZmt7czDuZeHr%3KAf^Iki&`Oa|J!G)ssKUxjb%<`Z?boeT1ji? zsI7uMa)#%Vvi;CPwR6>f3m8I#Dh?wj=f#9$`#kk2F_df5Df5Fn;NoW2EyT9*Nx+|7 zwOGaDa?Cp#(Fi{J($4p&xdFuH5K`NQeE8)1)G(hKdq8;sX=#%SX#G<4H$H(jtP>gcqq$1oX~`G$$fg;!|F(ZKr(Br zK%hjBOztGAMe~UIFTS)WY;@Wq>QuoHf8&S`mo`178lp2YOe4whcDa0G$_BBo?7`JI zk;m@apyDru*x)EAUlz;$cP4$FQcZG66He>klfkA(VOTf?MSURPjUaFgD+Cmg4>n>> z(>AJjTFtm25S#QoE}Zyq=*rD1{uq*BqB`BVRmGn<@DB_sUU{W7sr?xY8c+{iB$D@6 zy{L8xU=DJ`ETPYm_r;{kcG#rmzpDG$U`_;ujVHFNc%92X9QyUksx3ZrotExUdk7}F zgTy)eTY-)Yd;(qgx_U$~G4NRYZM6@fZ^Dfp(+qKI*`?-)p77o#3C9OH@zH`c$_AkS zlOVurmFsw}n;_R+O0(WmYXky`5m;OHupTMu43hdG9QC>5vA0z{lr;MN2e7kz(Kw_D zB9%P%BlUbaj;+L|Z+xuYB>3QT1fHdplj)=uH6n-{y6RK)s@U(J|E|szzg;?WpE_Mk z)pbbSK3w~Pe~N4G;DtwMs_+qzBOW4pMClt}s<-jsSWT*yXNL*0!(oziSnW?64ym0u zC4!*F!??93kJppD*ROe7a7G+rvc- zyFj~ttVO~S}(NTXnoN7 oqV+@Tj}}4G&~!8d%|x@%Y%~XrttW~$0Bs=JAi5=C@ch^Q5AvnWKmY&$ diff --git a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py index 8fee9683c..f04973063 100644 --- a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py +++ b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.db.models.deletion from django.db import migrations, models @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name='Subscription', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel')])), + ('type', models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel'), (3, 'Email+Pdf')])), ('kwargs', models.CharField(blank=True, default='', max_length=500)), ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), ], diff --git a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181206_1447.py similarity index 93% rename from src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py rename to src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181206_1447.py index da9944c95..7f73b93e7 100644 --- a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py +++ b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181206_1447.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.db.models.deletion from django.conf import settings diff --git a/src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py b/src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py deleted file mode 100644 index 45e24ff12..000000000 --- a/src/etools_datamart/apps/subscriptions/migrations/0003_auto_20181206_1420.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:20 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('subscriptions', '0002_auto_20181129_0824'), - ] - - operations = [ - migrations.AlterField( - model_name='subscription', - name='type', - field=models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel'), (3, 'Email+Pdf')]), - ), - ] diff --git a/src/etools_datamart/apps/tracking/management/__init__.py b/src/etools_datamart/apps/tracking/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/tracking/management/commands/__init__.py b/src/etools_datamart/apps/tracking/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/tracking/management/commands/track.py b/src/etools_datamart/apps/tracking/management/commands/track.py new file mode 100644 index 000000000..02d7d51f5 --- /dev/null +++ b/src/etools_datamart/apps/tracking/management/commands/track.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import logging +import os +from urllib.parse import urlencode + +from django.core.management import BaseCommand + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + args = '' + help = '' + requires_migrations_checks = False + requires_system_checks = False + + def add_arguments(self, parser): + + parser.add_argument( + '--tid', + action='store_true', + dest='all', + default=os.environ.get('ANALYTICS_CODE'), + help='select all options but `demo`') + parser.add_argument( + '--demo', + action='store_true', + dest='demo', + default=False, + help='create random demo .local') + + def echo(self, txt, st='SUCCESS'): + if self.verbosity == 0: + return + if self.verbosity == 1: + text = f"{txt}" + else: + text = f"\n{txt}" + self.stdout.write(getattr(self.style, st)(text)) + + def notify(self, model, created, name, tpl=" {op} {model} `{name}`"): + if self.verbosity > 2: + op = {True: "Created", False: "Updated"}[created] + self.stdout.write(tpl.format(op=op, model=model, name=name)) + elif self.verbosity > 1: + self.stdout.write('.', ending='') + + def handle(self, *args, **options): + self.verbosity = options['verbosity'] + tid = options['tid'] + import requests + # """https://www.google-analytics.com/r/collect?v=1&_v=j72&a=1254325655&t=pageview&_s=1&dl=http%3A%2F%2Flocalhost%2F&ul=en-gb&de=UTF-8&dt=Title&sd=24-bit&sr=1440x900&vp=1425x459&je=0&_u=IEBAAUAB~&jid=243210006&gjid=1351824934&cid=778822076.1544038618&tid=UA-130479575-1&_gid=909229133.1544038618&_r=1>m=2oubc0&z=118711575""" + values = { + "v": 1, + "t": "pageview", + # "t": "event", + # "ec": "video", # event category + # "ea": "play", # event action + # "el": "holiday", + "tid": tid, + "cid": 555, + "ev": 300, + "dl": "http://datamart.unicef.io/aaaa", + + } + qs = urlencode(values) + # payload = 'v=1&t=event&tid=UA-130479575-1&cid=555&ec=video&ea=play&el=holiday&ev=300&dl=http%3A%2F%2Flocalhost%2Fbbb' + requests.post('http://www.google-analytics.com/collect', data=qs) diff --git a/src/etools_datamart/apps/tracking/migrations/0001_initial.py b/src/etools_datamart/apps/tracking/migrations/0001_initial.py index 425d9911c..300391fcd 100644 --- a/src/etools_datamart/apps/tracking/migrations/0001_initial.py +++ b/src/etools_datamart/apps/tracking/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.utils.timezone import strategy_field.fields diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181206_1447.py similarity index 97% rename from src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py rename to src/etools_datamart/apps/tracking/migrations/0002_auto_20181206_1447.py index f98434cb8..271fa1e24 100644 --- a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py +++ b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181206_1447.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.db.models.deletion from django.conf import settings @@ -10,9 +10,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('unicef_rest_framework', '0001_initial'), ('tracking', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ diff --git a/src/etools_datamart/apps/web/templates/base.html b/src/etools_datamart/apps/web/templates/base.html index cfd19a527..0a47a4a25 100644 --- a/src/etools_datamart/apps/web/templates/base.html +++ b/src/etools_datamart/apps/web/templates/base.html @@ -7,7 +7,19 @@ {% endblock head %} - + +{# #} +{# #} diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index af383a3f6..7d172253d 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import datetime +import os from pathlib import Path import environ @@ -207,11 +208,12 @@ 'APP_DIRS': False, 'OPTIONS': { 'loaders': [ + 'dbtemplates.loader.Loader', 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', - # 'dbtemplates.loader.Loader', ], 'context_processors': [ + 'constance.context_processors.config', 'django.template.context_processors.debug', 'django.template.context_processors.request', 'etools_datamart.apps.multitenant.context_processors.schemas', @@ -383,6 +385,7 @@ CONSTANCE_CONFIG = { 'AZURE_USE_GRAPH': (True, 'Use MS Graph API to fetch user data', bool), 'DEFAULT_GROUP': ('Guests', 'Default group new users belong to', 'select_group'), + 'ANALYTICS_CODE': (env('ANALYTICS_CODE'), 'Google analytics code'), } CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler' @@ -606,3 +609,12 @@ BUSINESSAREA_MODEL = 'unicef_security.BusinessArea' AUTH_USER_MODEL = 'unicef_security.User' + + +def extra(r): + return {'AZURE_CLIENT_ID': os.environ['AZURE_CLIENT_ID'], + 'GRAPH_CLIENT_ID': os.environ['GRAPH_CLIENT_ID'], + 'AZURE_TENANT': os.environ['AZURE_TENANT']} + + +SYSINFO = {"extra": {'Azure': extra}} diff --git a/src/unicef_rest_framework/migrations/0001_initial.py b/src/unicef_rest_framework/migrations/0001_initial.py index 06eaedc84..b1c342065 100644 --- a/src/unicef_rest_framework/migrations/0001_initial.py +++ b/src/unicef_rest_framework/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import uuid diff --git a/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py b/src/unicef_rest_framework/migrations/0002_auto_20181206_1447.py similarity index 97% rename from src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py rename to src/unicef_rest_framework/migrations/0002_auto_20181206_1447.py index bf25f9213..699cb08ac 100644 --- a/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py +++ b/src/unicef_rest_framework/migrations/0002_auto_20181206_1447.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.db.models.deletion from django.conf import settings @@ -10,10 +10,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('auth', '0009_alter_user_last_name_max_length'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('contenttypes', '0002_remove_content_type_name'), + ('auth', '0009_alter_user_last_name_max_length'), ('unicef_rest_framework', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -131,7 +131,7 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='systemfilter', - unique_together={('service', 'group'), ('service', 'user')}, + unique_together={('service', 'user'), ('service', 'group')}, ), migrations.AlterUniqueTogether( name='groupaccesscontrol', diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index dde24133c..d47436ad6 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -12,7 +12,7 @@ def labelize(v): class HTMLRenderer(BaseRenderer): - media_type = 'text/xhtml' + media_type = 'text/html' format = 'xhtml' charset = 'utf-8' render_style = 'text' diff --git a/src/unicef_rest_framework/templates/renderers/html/html.html b/src/unicef_rest_framework/templates/renderers/html/html.html index a807290c9..d77cb7c80 100644 --- a/src/unicef_rest_framework/templates/renderers/html/html.html +++ b/src/unicef_rest_framework/templates/renderers/html/html.html @@ -33,7 +33,7 @@

      {{ opts.verbose_name }}

      {{ v }} {% endfor %} - {% for row in data.results %} + {% for row in data %} {% for k,v in row.items %} {{ v }} diff --git a/src/unicef_rest_framework/templates/rest_framework/base.html b/src/unicef_rest_framework/templates/rest_framework/base.html index 4751d9b04..751593d6b 100644 --- a/src/unicef_rest_framework/templates/rest_framework/base.html +++ b/src/unicef_rest_framework/templates/rest_framework/base.html @@ -25,7 +25,15 @@ {% if code_style %}{% endif %} {% endblock %} - + {% if config.ANALYTICS_CODE %} + + + {% endif %} {% endblock %} diff --git a/src/unicef_security/migrations/0001_initial.py b/src/unicef_security/migrations/0001_initial.py index b87338433..573525d7f 100644 --- a/src/unicef_security/migrations/0001_initial.py +++ b/src/unicef_security/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-06 14:47 import django.contrib.auth.models import django.contrib.auth.validators @@ -71,6 +71,7 @@ class Migration(migrations.Migration): name='Role', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('business_area', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.BUSINESSAREA_MODEL)), ('group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='auth.Group')), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], diff --git a/src/unicef_security/models.py b/src/unicef_security/models.py index a30120707..a2ece53b7 100644 --- a/src/unicef_security/models.py +++ b/src/unicef_security/models.py @@ -102,6 +102,7 @@ def save(self, *args, **kwargs): class Role(models.Model, TimeStampedModel): user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) group = models.ForeignKey(Group, on_delete=models.CASCADE) + business_area = models.ForeignKey(BusinessArea, on_delete=models.CASCADE) class Meta: app_label = 'unicef_security' diff --git a/src/unicef_security/tasks.py b/src/unicef_security/tasks.py new file mode 100644 index 000000000..6030e620e --- /dev/null +++ b/src/unicef_security/tasks.py @@ -0,0 +1,7 @@ +from celery.app import default_app +from unicef_security.sync import load_business_area + + +@default_app.task() +def sync_business_area(): + load_business_area() From fe8e84711ab0f74775a73638a37e13837550acef Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 6 Dec 2018 23:41:54 +0100 Subject: [PATCH 17/86] fixes settings --- src/etools_datamart/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 7d172253d..ad33784be 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -13,7 +13,7 @@ env = environ.Env(API_URL=(str, 'http://localhost:8000/api/'), ETOOLS_DUMP_LOCATION=(str, str(PACKAGE_DIR / 'apps' / 'multitenant' / 'postgresql')), - + ANALYTICS_CODE=(str, ""), CACHE_URL=(str, "redis://127.0.0.1:6379/1"), CACHE_URL_API=(str, "redis://127.0.0.1:6379/2?key_prefix=api"), CACHE_URL_LOCK=(str, "redis://127.0.0.1:6379/2?key_prefix=lock"), From f21e948c812baf6e7803efec8493438da11808c9 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 13:09:10 +0100 Subject: [PATCH 18/86] updates --- CHANGES | 1 + src/drf_querystringfilter/backend.py | 9 ++- src/etools_datamart/api/endpoints/common.py | 3 +- .../api/endpoints/etools/audit.py | 2 + src/etools_datamart/api/filtering.py | 14 ++--- src/etools_datamart/apps/etl/models.py | 3 + src/etools_datamart/apps/etl/results.py | 10 ++- src/etools_datamart/apps/multitenant/views.py | 38 ++++++------ .../apps/subscriptions/models.py | 14 ++--- .../apps/subscriptions/urls.py | 8 ++- src/etools_datamart/celery.py | 9 +-- src/etools_datamart/config/settings.py | 1 + src/unicef_rest_framework/auth.py | 32 +++++++++- src/unicef_rest_framework/negotiation.py | 14 +++++ .../renderers/__init__.py | 2 + src/unicef_rest_framework/renderers/html.py | 2 +- src/unicef_rest_framework/renderers/iqy.py | 50 +++++++++++++++ src/unicef_rest_framework/renderers/pdf.py | 2 +- src/unicef_rest_framework/renderers/txt.py | 50 +++++++++++++++ .../templates/renderers/{html => }/html.html | 0 .../templates/renderers/iqy.txt | 2 + .../templates/renderers/{pdf => }/pdf.html | 0 .../templates/renderers/text.txt | 2 + src/unicef_rest_framework/utils.py | 20 ++++++ src/unicef_rest_framework/views.py | 32 +++++++--- src/unicef_rest_framework/views_mixins.py | 27 ++++++++ tests/.coveragerc | 1 + tests/api/test_api_data.py | 15 ++++- tests/api/test_datamart_security.py | 4 +- tests/etl/test_etl_result.py | 25 ++++++++ tests/test_admin.py | 6 ++ tests/test_celery.py | 6 ++ tests/test_subscription.py | 61 +++++++++++++++++++ 33 files changed, 402 insertions(+), 63 deletions(-) create mode 100644 src/unicef_rest_framework/negotiation.py create mode 100644 src/unicef_rest_framework/renderers/iqy.py create mode 100644 src/unicef_rest_framework/renderers/txt.py rename src/unicef_rest_framework/templates/renderers/{html => }/html.html (100%) create mode 100644 src/unicef_rest_framework/templates/renderers/iqy.txt rename src/unicef_rest_framework/templates/renderers/{pdf => }/pdf.html (100%) create mode 100644 src/unicef_rest_framework/templates/renderers/text.txt create mode 100644 src/unicef_rest_framework/views_mixins.py create mode 100644 tests/test_celery.py diff --git a/CHANGES b/CHANGES index 8b0502f0f..27516b25c 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,7 @@ * fixes error in some renderers with cached response * Adopting of RabbitMQ to prevent message loss * new admin index page +* Excel IQY support (beta) 1.7 diff --git a/src/drf_querystringfilter/backend.py b/src/drf_querystringfilter/backend.py index 7671e1085..dac4e28c8 100644 --- a/src/drf_querystringfilter/backend.py +++ b/src/drf_querystringfilter/backend.py @@ -123,7 +123,7 @@ def _get_filters(self, request, queryset, view): # noqa - exclude null values: country__not=>< - only values in list: &country__id__in=176,20 - - exclude values in list: &country__id__not_in=176,20 + - exclude values in list: &country__id=176,20 """ self.opts = queryset.model._meta @@ -137,8 +137,9 @@ def _get_filters(self, request, queryset, view): # noqa for fieldname_arg in self.query_params: raw_value = self.query_params.get(fieldname_arg) - if not raw_value: - continue + if raw_value in ["''", '""']: + raw_value = "" + negate = fieldname_arg[-1] == "!" if negate: @@ -168,6 +169,8 @@ def _get_filters(self, request, queryset, view): # noqa processor = getattr(self, 'process_{}'.format(filter_field_name), None) if (filter_field_name not in filter_fields) and (not processor): raise InvalidQueryArgumentError(filter_field_name) + if raw_value is None and not processor: + continue # field is configured in Serializer # so we use 'source' attribute if filter_field_name in mapping: diff --git a/src/etools_datamart/api/endpoints/common.py b/src/etools_datamart/api/endpoints/common.py index b29796e95..84d1707b3 100644 --- a/src/etools_datamart/api/endpoints/common.py +++ b/src/etools_datamart/api/endpoints/common.py @@ -13,6 +13,7 @@ from rest_framework.response import Response from unicef_rest_framework.filtering import SystemFilterBackend from unicef_rest_framework.views import ReadOnlyModelViewSet +from unicef_rest_framework.views_mixins import IQYConnectionMixin from etools_datamart.api.filtering import DatamartQueryStringFilterBackend, TenantQueryStringFilterBackend from etools_datamart.apps.etl.models import EtlTask @@ -63,7 +64,7 @@ def updates(self, request, version): headers={'update-date': offset}) -class APIReadOnlyModelViewSet(ReadOnlyModelViewSet): +class APIReadOnlyModelViewSet(ReadOnlyModelViewSet, IQYConnectionMixin): filter_backends = [SystemFilterBackend, DatamartQueryStringFilterBackend, OrderingFilter] diff --git a/src/etools_datamart/api/endpoints/etools/audit.py b/src/etools_datamart/api/endpoints/etools/audit.py index 730c19254..77a95ffc4 100644 --- a/src/etools_datamart/api/endpoints/etools/audit.py +++ b/src/etools_datamart/api/endpoints/etools/audit.py @@ -7,3 +7,5 @@ class EngagementViewSet(common.APIMultiTenantReadOnlyModelViewSet): serializer_class = serializers.EngagementSerializer queryset = models.AuditEngagement.objects.all() + filter_fields = ['joint_audit', 'status', 'engagement_type', + 'cancel_comment'] diff --git a/src/etools_datamart/api/filtering.py b/src/etools_datamart/api/filtering.py index 51c9164de..ffccc81a6 100644 --- a/src/etools_datamart/api/filtering.py +++ b/src/etools_datamart/api/filtering.py @@ -20,13 +20,7 @@ class CountryNameProcessor: def process_country_name(self, efilters, eexclude, field, value, request, op, param, negate, **payload): filters = {} - if not value: - if not request.user.is_superuser: - allowed = get_etools_allowed_schemas(request.user) - if not allowed: # pragma: no cover - raise PermissionDenied("You don't have enabled schemas") - filters['country_name__iregex'] = r'(' + '|'.join(allowed) + ')' - else: + if value: value = set(value.lower().split(",")) validate_schemas(*value) if not request.user.is_superuser: @@ -34,6 +28,12 @@ def process_country_name(self, efilters, eexclude, field, value, request, if not user_schemas.issuperset(value): raise NotAuthorizedSchema(",".join(sorted(value - user_schemas))) filters['country_name__iregex'] = r'(' + '|'.join(value) + ')' + else: + if not request.user.is_superuser: + allowed = get_etools_allowed_schemas(request.user) + if not allowed: # pragma: no cover + raise PermissionDenied("You don't have enabled schemas") + filters['country_name__iregex'] = r'(' + '|'.join(allowed) + ')' if negate: return {}, filters diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index afd0dbebd..659a81384 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -10,6 +10,9 @@ class TaskLogManager(models.Manager): + def filter_for_models(self, *models): + return self.filter(content_type__in=ContentType.objects.get_for_models(*models).values()) + def get_for_model(self, model: DataMartModel): try: return self.get(content_type=ContentType.objects.get_for_model(model)) diff --git a/src/etools_datamart/apps/etl/results.py b/src/etools_datamart/apps/etl/results.py index 53ed5b34c..12cd7d393 100644 --- a/src/etools_datamart/apps/etl/results.py +++ b/src/etools_datamart/apps/etl/results.py @@ -1,4 +1,5 @@ import json +from json.decoder import WHITESPACE CREATED = 'created' UPDATED = 'updated' @@ -35,7 +36,14 @@ def __eq__(self, other): return False +class EtlDecoder(json.JSONDecoder): + + def decode(self, s, _w=WHITESPACE.match): + return super().decode(s, _w) + + class EtlEncoder(json.JSONEncoder): + def default(self, obj): if isinstance(obj, EtlResult): return { @@ -60,4 +68,4 @@ def etl_dumps(obj): # Decoder function def etl_loads(obj): - return json.loads(obj, object_hook=etl_decoder) + return json.loads(obj, cls=EtlDecoder, object_hook=etl_decoder) diff --git a/src/etools_datamart/apps/multitenant/views.py b/src/etools_datamart/apps/multitenant/views.py index 84602daab..eeab10e97 100644 --- a/src/etools_datamart/apps/multitenant/views.py +++ b/src/etools_datamart/apps/multitenant/views.py @@ -3,13 +3,12 @@ from django.http import HttpResponseRedirect from django.urls import reverse_lazy -from django.utils.http import urlencode from django.views.generic.edit import FormView +# from unicef_rest_framework.state import state +from unicef_rest_framework.utils import get_query_string from etools_datamart.apps.multitenant.forms import SchemasForm -# from unicef_rest_framework.state import state - logger = logging.getLogger(__name__) @@ -50,19 +49,20 @@ def form_valid(self, form): return response def get_query_string(self, new_params=None, remove=None): - if new_params is None: - new_params = {} - if remove is None: - remove = [] - p = self.params.copy() - for r in remove: - for k in list(p): - if k.startswith(r): - del p[k] - for k, v in new_params.items(): - if v is None: - if k in p: - del p[k] - else: - p[k] = v - return '?%s' % urlencode(sorted(p.items())) + return get_query_string(self.params, new_params, remove) + # if new_params is None: + # new_params = {} + # if remove is None: + # remove = [] + # p = self.params.copy() + # for r in remove: + # for k in list(p): + # if k.startswith(r): + # del p[k] + # for k, v in new_params.items(): + # if v is None: + # if k in p: + # del p[k] + # else: + # p[k] = v + # return '?%s' % urlencode(sorted(p.items())) diff --git a/src/etools_datamart/apps/subscriptions/models.py b/src/etools_datamart/apps/subscriptions/models.py index 5fba886f3..0537ed38e 100644 --- a/src/etools_datamart/apps/subscriptions/models.py +++ b/src/etools_datamart/apps/subscriptions/models.py @@ -91,13 +91,13 @@ class Meta: def __str__(self): return f"#{self.pk} {self.user} {self.get_type_display()} {self.content_type}" - @cached_property - def endpoint(self): - return self.content_type.model_class().service.endpoint - - @cached_property - def service(self): - return self.content_type.model_class().service + # @cached_property + # def endpoint(self): + # return self.content_type.model_class().service.endpoint + # + # @cached_property + # def service(self): + # return self.content_type.model_class().service @cached_property def viewset(self): diff --git a/src/etools_datamart/apps/subscriptions/urls.py b/src/etools_datamart/apps/subscriptions/urls.py index 1df0179d1..18656fb18 100644 --- a/src/etools_datamart/apps/subscriptions/urls.py +++ b/src/etools_datamart/apps/subscriptions/urls.py @@ -21,8 +21,12 @@ def _decorator(request, *args, **kwargs): if user: login(request, user, backend='django.contrib.auth.backends.RemoteUserBackend') else: - return HttpResponse(status=401) - return func(request, *args, **kwargs) + return HttpResponse(status=403) + + if request.user.is_authenticated: + return func(request, *args, **kwargs) + else: + return HttpResponse(status=401) return _decorator diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 2fc6f2f1d..fb179e8f3 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -4,11 +4,6 @@ from celery import Celery from celery.contrib.abortable import AbortableTask from celery.signals import task_postrun, task_prerun -from django.apps import apps -# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], -# related_name='tasks') -# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], -# related_name='etl') from kombu import Exchange, Queue from kombu.serialization import register @@ -62,8 +57,8 @@ def get_all_etls(self): app = DatamartCelery('datamart') app.config_from_object('django.conf:settings', namespace='CELERY') -# app.autodiscover_tasks() -app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) +app.autodiscover_tasks() +# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) # pragma app.timers = {} app.conf.task_queues = ( diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index ad33784be..808731f59 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -418,6 +418,7 @@ "rest_framework.renderers.BrowsableAPIRenderer", ), "PAGE_SIZE": 100, + 'DEFAULT_CONTENT_NEGOTIATION_CLASS': 'unicef_rest_framework.negotiation.CT', # "DEFAULT_PAGINATION_CLASS": 'rest_framework.pagination.CursorPagination', 'DEFAULT_PAGINATION_CLASS': 'unicef_rest_framework.pagination.APIPagination', 'DEFAULT_METADATA_CLASS': 'etools_datamart.api.metadata.SimpleMetadataWithFilters', diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index 8a4d64414..27ee5314d 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -5,6 +5,7 @@ from django.contrib.auth import get_user_model, login from django.utils.translation import ugettext as _ from rest_framework import exceptions +from rest_framework.authentication import BaseAuthentication from rest_framework.exceptions import AuthenticationFailed, PermissionDenied from rest_framework_jwt import authentication from unicef_security.graph import default_group, Synchronizer @@ -16,6 +17,36 @@ def jwt_get_username_from_payload(payload): return payload.get('preferred_username', payload.get('unique_name')) +def get_client_ip(environ): + """ + Naively yank the first IP address in an X-Forwarded-For header + and assume this is correct. + + Note: Don't use this in security sensitive situations since this + value may be forged from a client. + """ + try: + return environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip() + except (KeyError, IndexError): + return environ.get('REMOTE_ADDR') + + +class URLTokenAuthentication(BaseAuthentication): + def authenticate(self, request): + return None + + +class IPBasedAuthentication(BaseAuthentication): + def authenticate(self, request): + ip = get_client_ip(request.META) + if ip == '127.0.0.1': + User = get_user_model() + user = User.objects.get_by_natural_key('sax') + request.user = user + login(request, user, 'social_core.backends.azuread_tenant.AzureADTenantOAuth2') + return (user, None) + + class JWTAuthentication(authentication.JSONWebTokenAuthentication): def authenticate(self, request): @@ -68,5 +99,4 @@ def authenticate_credentials(self, payload): raise exceptions.AuthenticationFailed(msg) return user - # AQABAAIAAAC5una0EUFgTIF8ElaxtWjTE8f9OcsbHLLnobNloaTfC--E_fRoUrtiw2jul5yBV9rN3CO2C1BJ2IB99esAhsuRrzEowH3COPLFe5hkhovi4zfceFjwu6iSXpfgAFVGuo_fmep0osVwr0WkFzhWI5QEgNNnrf7d7gFm4iVC4gFE24R_JymglPADBvJIUMGAPHYg-IEyK1GKSkzpNSjJNZz6Pad_uVlDMrssFcrRqxKOJzqIhggLq7XQpJnmfUF5dJNdriDMkUjHBhDqlNpKTJZpnJg0jfIn7843kmKH0WXbJL0ss-tfgc_d8Q0240bdYXX6YSBV20NPx7MHy5V9i1RAtmr11cHBCw3uDuRriomgOhtIxTKYLox8iKYHbELA9Opvd-zLJm9krxoxlEHVO-PKl11No1mT8ZC83Ox37yxG5vrE7U7UxaLml9PmrjRZQoD1HvJ354IxZyP2pytYq2XhvIG_NDSDfuO5hwzPKb9F7G4Hytu96plKlu_yvdZ4Gghbp7z2sryeAiCnpYNlskGVUrQwF7BSHT73XuuOWeFelp-jn3tR4LQwqEGkg3zLqswcjbsRykSvS3cY6xTdBsCb7H70nygnhOgr_WlT9oY9KS2ElBVU-Q8OE8mkJ1rDV42hRb-haC7yzyUgtofbSQdVMIUgJRpuxYCrHNJ5oRsXmrWI0EVTdWFN25kwOMYPwOI8rzVf1oHikTQiHm3AN5wz0ill40IfjLB9niMEn4kntLDGJU1rIeALxv1s4lHxMZHpc1YEgLTf_3LnGtrsca3bIAA diff --git a/src/unicef_rest_framework/negotiation.py b/src/unicef_rest_framework/negotiation.py new file mode 100644 index 000000000..4b9543e8c --- /dev/null +++ b/src/unicef_rest_framework/negotiation.py @@ -0,0 +1,14 @@ +from rest_framework.negotiation import DefaultContentNegotiation + + +class CT(DefaultContentNegotiation): + + def select_renderer(self, request, renderers, format_suffix=None): + format_query_param = self.settings.URL_FORMAT_OVERRIDE + format = format_suffix or request.query_params.get(format_query_param) + if format == 'iqy': + for renderer in renderers: + if renderer.format == format: + return renderer, renderer.media_type + + return super().select_renderer(request, renderers, format_suffix) diff --git a/src/unicef_rest_framework/renderers/__init__.py b/src/unicef_rest_framework/renderers/__init__.py index 8cc3dadc9..ab3d1cd40 100644 --- a/src/unicef_rest_framework/renderers/__init__.py +++ b/src/unicef_rest_framework/renderers/__init__.py @@ -4,3 +4,5 @@ from .xls import XLSXRenderer # noqa from .html import HTMLRenderer # noqa from .pdf import PDFRenderer # noqa +from .txt import TextRenderer # noqa +from .iqy import IQYRenderer # noqa diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index d47436ad6..cb045c45b 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -21,7 +21,7 @@ def get_template(self, meta): return loader.select_template([ f'renderers/html/{meta.app_label}/{meta.model_name}.html', f'renderers/html/{meta.app_label}/html.html', - 'renderers/html/html.html']) + 'renderers/html.html']) def render(self, data, accepted_media_type=None, renderer_context=None): response = renderer_context['response'] diff --git a/src/unicef_rest_framework/renderers/iqy.py b/src/unicef_rest_framework/renderers/iqy.py new file mode 100644 index 000000000..c349e6495 --- /dev/null +++ b/src/unicef_rest_framework/renderers/iqy.py @@ -0,0 +1,50 @@ +import logging + +from crashlog.middleware import process_exception +from django.template import loader +from rest_framework.renderers import BaseRenderer + +logger = logging.getLogger(__name__) + + +def labelize(v): + return v.replace("_", " ").title() + + +class IQYRenderer(BaseRenderer): + media_type = 'text/plain' + format = 'iqy' + charset = 'utf-8' + render_style = 'text' + + def get_template(self, meta): + return loader.select_template([ + f'renderers/iqy/{meta.app_label}/{meta.model_name}.txt', + f'renderers/iqy/{meta.app_label}/iqy.txt', + 'renderers/iqy.txt']) + + def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' + try: + model = renderer_context['view'].queryset.model + opts = model._meta + template = self.get_template(opts) + if data and 'results' in data: + data = data['results'] + if data: + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': [labelize(v) for v in data[0].keys()]} + else: + c = {'data': {}, + 'model': model, + 'opts': opts, + 'headers': []} + return template.render(c) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index bb6010020..36f52a907 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -49,7 +49,7 @@ def get_template(self, meta): return loader.select_template([ f'renderers/pdf/{meta.app_label}/{meta.model_name}.html', f'renderers/pdf/{meta.app_label}/pdf.html', - 'renderers/pdf/pdf.html']) + 'renderers/pdf.html']) def render(self, data, accepted_media_type=None, renderer_context=None): response = renderer_context['response'] diff --git a/src/unicef_rest_framework/renderers/txt.py b/src/unicef_rest_framework/renderers/txt.py new file mode 100644 index 000000000..a51d395eb --- /dev/null +++ b/src/unicef_rest_framework/renderers/txt.py @@ -0,0 +1,50 @@ +import logging + +from crashlog.middleware import process_exception +from django.template import loader +from rest_framework.renderers import BaseRenderer + +logger = logging.getLogger(__name__) + + +def labelize(v): + return v.replace("_", " ").title() + + +class TextRenderer(BaseRenderer): + media_type = 'text/plain' + format = 'txt' + charset = 'utf-8' + render_style = 'text' + + def get_template(self, meta): + return loader.select_template([ + f'renderers/text/{meta.app_label}/{meta.model_name}.txt', + f'renderers/text/{meta.app_label}/text.txt', + 'renderers/text.txt']) + + def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' + try: + model = renderer_context['view'].queryset.model + opts = model._meta + template = self.get_template(opts) + if data and 'results' in data: + data = data['results'] + if data: + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': [labelize(v) for v in data[0].keys()]} + else: + c = {'data': {}, + 'model': model, + 'opts': opts, + 'headers': []} + return template.render(c) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/templates/renderers/html/html.html b/src/unicef_rest_framework/templates/renderers/html.html similarity index 100% rename from src/unicef_rest_framework/templates/renderers/html/html.html rename to src/unicef_rest_framework/templates/renderers/html.html diff --git a/src/unicef_rest_framework/templates/renderers/iqy.txt b/src/unicef_rest_framework/templates/renderers/iqy.txt new file mode 100644 index 000000000..33648c01b --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/iqy.txt @@ -0,0 +1,2 @@ +{% for v in headers %}{% endfor %}{% for row in data %} +{% for k,v in row.items %}{% endfor %}{% endfor %}
      {{ v }}
      {{ v }}
      diff --git a/src/unicef_rest_framework/templates/renderers/pdf/pdf.html b/src/unicef_rest_framework/templates/renderers/pdf.html similarity index 100% rename from src/unicef_rest_framework/templates/renderers/pdf/pdf.html rename to src/unicef_rest_framework/templates/renderers/pdf.html diff --git a/src/unicef_rest_framework/templates/renderers/text.txt b/src/unicef_rest_framework/templates/renderers/text.txt new file mode 100644 index 000000000..4a7bda5f3 --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/text.txt @@ -0,0 +1,2 @@ +{% for v in headers %}{{ v }} {% endfor %}{% for row in data %} +{% for k,v in row.items %}{{ v }} {% endfor %}{% endfor %} diff --git a/src/unicef_rest_framework/utils.py b/src/unicef_rest_framework/utils.py index 5de29f026..35428fa5d 100644 --- a/src/unicef_rest_framework/utils.py +++ b/src/unicef_rest_framework/utils.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +from urllib.parse import urlencode from rest_framework.settings import api_settings @@ -49,3 +50,22 @@ def humanize_size(num, suffix='B'): def get_hostname(): return os.environ.get('HOSTNAME') + + +def get_query_string(params, new_params=None, remove=None): + if new_params is None: + new_params = {} + if remove is None: + remove = [] + p = params.copy() + for r in remove: + for k in list(p): + if k.startswith(r): + del p[k] + for k, v in new_params.items(): + if v is None: + if k in p: + del p[k] + else: + p[k] = v + return '?%s' % urlencode(sorted(p.items())) diff --git a/src/unicef_rest_framework/views.py b/src/unicef_rest_framework/views.py index 2faae5e3a..ec23423c7 100644 --- a/src/unicef_rest_framework/views.py +++ b/src/unicef_rest_framework/views.py @@ -11,14 +11,16 @@ from rest_framework_xml.renderers import XMLRenderer from rest_framework_yaml.renderers import YAMLRenderer from strategy_field.utils import fqn -from unicef_rest_framework import acl -from unicef_rest_framework.auth import JWTAuthentication -from unicef_rest_framework.cache import cache_response, etag, ListKeyConstructor -from unicef_rest_framework.filtering import SystemFilterBackend -from unicef_rest_framework.permissions import ServicePermission -from unicef_rest_framework.renderers import (APIBrowsableAPIRenderer, HTMLRenderer, MSJSONRenderer, - MSXmlRenderer, PDFRenderer, XLSXRenderer,) -from unicef_rest_framework.renderers.csv import CSVRenderer + +from . import acl +from .auth import IPBasedAuthentication, JWTAuthentication, URLTokenAuthentication +from .cache import cache_response, etag, ListKeyConstructor +from .filtering import SystemFilterBackend +from .negotiation import CT +from .permissions import ServicePermission +from .renderers import (APIBrowsableAPIRenderer, HTMLRenderer, IQYRenderer, MSJSONRenderer, + MSXmlRenderer, PDFRenderer, TextRenderer, XLSXRenderer,) +from .renderers.csv import CSVRenderer class classproperty(object): @@ -40,10 +42,15 @@ class ReadOnlyModelViewSet(DynamicSerializerMixin, viewsets.ReadOnlyModelViewSet list_etag_func = ListKeyConstructor() authentication_classes = (SessionAuthentication, + IPBasedAuthentication, JWTAuthentication, BasicAuthentication, - TokenAuthentication) + TokenAuthentication, + IPBasedAuthentication, + URLTokenAuthentication, + ) default_access = acl.ACL_ACCESS_LOGIN + content_negotiation_class = CT filter_backends = [SystemFilterBackend, QueryStringFilterBackend, OrderingFilter] @@ -57,6 +64,8 @@ class ReadOnlyModelViewSet(DynamicSerializerMixin, viewsets.ReadOnlyModelViewSet MSJSONRenderer, XMLRenderer, MSXmlRenderer, + TextRenderer, + IQYRenderer, ] ordering_fields = ('id',) ordering = 'id' @@ -76,6 +85,11 @@ def dispatch(self, request, *args, **kwargs): return super().dispatch(request, *args, **kwargs) + # def filter_queryset(self, queryset): + # if hasattr(self.request, 'api_info'): + # self.request.api_info["sql"] = str(queryset.query) + # return super().filter_queryset(queryset) + @classproperty def label(cls): return cls.__name__.replace("ViewSet", "") diff --git a/src/unicef_rest_framework/views_mixins.py b/src/unicef_rest_framework/views_mixins.py new file mode 100644 index 000000000..088a753b0 --- /dev/null +++ b/src/unicef_rest_framework/views_mixins.py @@ -0,0 +1,27 @@ +from django.conf import settings +from django.http import HttpResponse +from rest_framework.decorators import action +from unicef_rest_framework.utils import get_query_string + + +class IQYConnectionMixin: + + @action(methods=['get'], detail=False) + def iqy(self, request, version): + qs = get_query_string(request.query_params, {'format': 'iqy'}, + remove=['format']) + url = f"{request.path}".replace('/iqy/', '/') + + iqy = """WEB +1 +{host}{url}{qs} + +Selection=AllTables +Formatting=html +PreFormattedTextToColumns=True +ConsecutiveDelimitersAsOne=True +SingleBlockTextImport=False +DisableDateRecognition=False + +""".format(host=settings.ABSOLUTE_BASE_URL, request=request, qs=qs, url=url) + return HttpResponse(iqy, content_type='text/plain') diff --git a/tests/.coveragerc b/tests/.coveragerc index 47792326e..59098792e 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -13,6 +13,7 @@ omit = **/multitenant/postgresql/creation.py **/apps/etools/models/public_old.py **/apps/etools/models/tenant_old.py + **/tracking/management/commands/track.py [report] # Regexes for lines to exclude from consideration diff --git a/tests/api/test_api_data.py b/tests/api/test_api_data.py index adee3a0e6..46fdb4e04 100644 --- a/tests/api/test_api_data.py +++ b/tests/api/test_api_data.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- import pytest +from dateutil.utils import today +from django.contrib.contenttypes.models import ContentType from test_utilities.factories import (FAMIndicatorFactory, HACTFactory, InterventionFactory, - PMPIndicatorFactory, UserStatsFactory,) + PMPIndicatorFactory, TaskLogFactory, UserStatsFactory,) from etools_datamart.api.endpoints import (FAMIndicatorViewSet, HACTViewSet, InterventionViewSet, PMPIndicatorsViewSet, UserStatsViewSet,) +from etools_datamart.apps.data.models import FAMIndicator from etools_datamart.apps.etl.models import EtlTask VIEWSETS = [ @@ -51,6 +54,16 @@ def test_list(client, action, viewset, format, ct, data): assert res.content assert res['Content-Type'] == ct + +def test_updates(client): + viewset = FAMIndicatorViewSet() + TaskLogFactory(last_changes=today(), + content_type=ContentType.objects.get_for_model(FAMIndicator)) + + url = f"{viewset.get_service().endpoint}updates/" + res = client.get(url) + assert res.status_code == 200, res + # @pytest.mark.parametrize("viewset", VIEWSETS) # def test_list_json(client, viewset): # res = client.get(viewset.get_service().endpoint) diff --git a/tests/api/test_datamart_security.py b/tests/api/test_datamart_security.py index 2ae628dcf..345fab2ee 100644 --- a/tests/api/test_datamart_security.py +++ b/tests/api/test_datamart_security.py @@ -29,10 +29,8 @@ def user_data(db): PARAMS = [("", 200, 3), - ("country_name=", 200, 1), ("country_name=lebanon", 200, 1), ("country_name!=lebanon", 200, 2), - ("country_name!=lebanon", 200, 2), ] @@ -48,7 +46,7 @@ def test_datamart_user_access_allowed_countries(user, url, code, expected, user_ with user_allow_service(user, UserStatsViewSet): res = client.get(f"{base}?{url}") assert res.status_code == code, res - assert len(res.json()['results']) == expected + assert len(res.json()['results']) == expected, res.json() def test_datamart_user_access_forbidden_countries(user, user_data): diff --git a/tests/etl/test_etl_result.py b/tests/etl/test_etl_result.py index e9ebac924..e1cae78f6 100644 --- a/tests/etl/test_etl_result.py +++ b/tests/etl/test_etl_result.py @@ -1,3 +1,4 @@ +from etools_datamart.apps.etl.results import etl_decoder, etl_dumps, etl_loads, EtlEncoder from etools_datamart.apps.etl.tasks.etl import EtlResult @@ -19,3 +20,27 @@ def test_result_ne_dict(): def test_result_ne_other(): assert not EtlResult() == 1 + + +def test_encoder(): + e = EtlEncoder() + assert e.encode( + EtlResult(1, 1, 1)) == '{"__type__": "__EtlResult__", "data": {"created": 1, "updated": 1, "unchanged": 1}}' + + +def test_encoder2(): + e = EtlEncoder() + assert e.encode({}) == '{}' + + +def test_encode(): + assert etl_decoder({}) == {} + + +def test_dumps(): + assert etl_dumps( + EtlResult(1, 1, 1)) == '{"__type__": "__EtlResult__", "data": {"created": 1, "updated": 1, "unchanged": 1}}' + + +def test_loads(): + assert etl_loads(etl_dumps({"a": 1})) == {"a": 1} diff --git a/tests/test_admin.py b/tests/test_admin.py index 2e4d80b52..871c1c7ac 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -9,3 +9,9 @@ def test_constance(django_app, admin_user): url = reverse("admin:constance_config_changelist") res = django_app.get(url, user=admin_user) assert res.status_code == 200 + + +def test_sysinfo(django_app, admin_user): + url = reverse("sys-admin-info") + res = django_app.get(url, user=admin_user) + assert res.status_code == 200 diff --git a/tests/test_celery.py b/tests/test_celery.py new file mode 100644 index 000000000..9c38a74a5 --- /dev/null +++ b/tests/test_celery.py @@ -0,0 +1,6 @@ +from etools_datamart.celery import app + + +def test_autodiscover(): + ret = app.tasks + assert 'etools_datamart.apps.etl.tasks.etl.load_hact' in ret diff --git a/tests/test_subscription.py b/tests/test_subscription.py index b2c2e5063..cb9a99c02 100644 --- a/tests/test_subscription.py +++ b/tests/test_subscription.py @@ -1,7 +1,10 @@ +import base64 import json +from unittest.mock import MagicMock import pytest from django.contrib.auth.models import AnonymousUser +from django.http import HttpResponse from django.urls import reverse from test_utilities.factories import EmailTemplateFactory, HACTFactory, SubscriptionFactory from unicef_rest_framework.test_utils import user_allow_service @@ -9,6 +12,7 @@ from etools_datamart.apps.data.models import HACT from etools_datamart.apps.etl.models import EtlTask from etools_datamart.apps.subscriptions.models import Subscription +from etools_datamart.apps.subscriptions.urls import http_basic_auth from etools_datamart.apps.subscriptions.views import subscribe @@ -103,3 +107,60 @@ def test_notification_email_attachment(subscription_attachment: Subscription, em assert len(emails) == 1 assert emails[0].to == [subscription_attachment.user.email] assert emails[0].attachments.count() == 1 + + +def test_http_basic_auth_401(rf): + request = rf.get('/') + request.user = AnonymousUser() + + def view(request): + return 11 + + f = http_basic_auth(view) + res = f(request) + assert res.status_code == 401 + + +def test_http_basic_auth_401b(rf, admin_user): + string = '%s:%s' % ('admin', '--') + base64string = base64.standard_b64encode(string.encode('utf-8')) + request = rf.get('/', HTTP_AUTHORIZATION="Digest %s" % base64string.decode('utf-8')) + request.user = AnonymousUser() + request.session = MagicMock() + + def view(request): + return HttpResponse("Ok") + + f = http_basic_auth(view) + res = f(request) + assert res.status_code == 401 + + +def test_http_basic_auth_400(rf, admin_user): + string = '%s:%s' % ('admin', '--') + base64string = base64.standard_b64encode(string.encode('utf-8')) + request = rf.get('/', HTTP_AUTHORIZATION="Basic %s" % base64string.decode('utf-8')) + request.user = AnonymousUser() + request.session = MagicMock() + + def view(request): + return HttpResponse("Ok") + + f = http_basic_auth(view) + res = f(request) + assert res.status_code == 403 + + +def test_http_basic_auth_200(rf, admin_user): + string = '%s:%s' % ('admin', 'password') + base64string = base64.standard_b64encode(string.encode('utf-8')) + request = rf.get('/', HTTP_AUTHORIZATION="Basic %s" % base64string.decode('utf-8')) + request.user = AnonymousUser() + request.session = MagicMock() + + def view(request): + return HttpResponse("Ok") + + f = http_basic_auth(view) + res = f(request) + assert res.status_code == 200 From 1ce59803daa501cfed8be06b47c6082ecae5cbf5 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 13:09:10 +0100 Subject: [PATCH 19/86] updates Debug Auth backends --- src/unicef_rest_framework/auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index 27ee5314d..343573f8f 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -2,6 +2,7 @@ from constance import config from crashlog.middleware import process_exception +from django.conf import settings from django.contrib.auth import get_user_model, login from django.utils.translation import ugettext as _ from rest_framework import exceptions @@ -98,5 +99,3 @@ def authenticate_credentials(self, payload): msg = _('User account is disabled.') raise exceptions.AuthenticationFailed(msg) return user - -# AQABAAIAAAC5una0EUFgTIF8ElaxtWjTE8f9OcsbHLLnobNloaTfC--E_fRoUrtiw2jul5yBV9rN3CO2C1BJ2IB99esAhsuRrzEowH3COPLFe5hkhovi4zfceFjwu6iSXpfgAFVGuo_fmep0osVwr0WkFzhWI5QEgNNnrf7d7gFm4iVC4gFE24R_JymglPADBvJIUMGAPHYg-IEyK1GKSkzpNSjJNZz6Pad_uVlDMrssFcrRqxKOJzqIhggLq7XQpJnmfUF5dJNdriDMkUjHBhDqlNpKTJZpnJg0jfIn7843kmKH0WXbJL0ss-tfgc_d8Q0240bdYXX6YSBV20NPx7MHy5V9i1RAtmr11cHBCw3uDuRriomgOhtIxTKYLox8iKYHbELA9Opvd-zLJm9krxoxlEHVO-PKl11No1mT8ZC83Ox37yxG5vrE7U7UxaLml9PmrjRZQoD1HvJ354IxZyP2pytYq2XhvIG_NDSDfuO5hwzPKb9F7G4Hytu96plKlu_yvdZ4Gghbp7z2sryeAiCnpYNlskGVUrQwF7BSHT73XuuOWeFelp-jn3tR4LQwqEGkg3zLqswcjbsRykSvS3cY6xTdBsCb7H70nygnhOgr_WlT9oY9KS2ElBVU-Q8OE8mkJ1rDV42hRb-haC7yzyUgtofbSQdVMIUgJRpuxYCrHNJ5oRsXmrWI0EVTdWFN25kwOMYPwOI8rzVf1oHikTQiHm3AN5wz0ill40IfjLB9niMEn4kntLDGJU1rIeALxv1s4lHxMZHpc1YEgLTf_3LnGtrsca3bIAA From 293cde825263f8c589df7ee26f79db59d1601f35 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 13:41:23 +0100 Subject: [PATCH 20/86] updates Debug Auth backends --- src/unicef_rest_framework/auth.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index 343573f8f..bcd3398a4 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -39,13 +39,14 @@ def authenticate(self, request): class IPBasedAuthentication(BaseAuthentication): def authenticate(self, request): - ip = get_client_ip(request.META) - if ip == '127.0.0.1': - User = get_user_model() - user = User.objects.get_by_natural_key('sax') - request.user = user - login(request, user, 'social_core.backends.azuread_tenant.AzureADTenantOAuth2') - return (user, None) + if settings.DEBUG: + ip = get_client_ip(request.META) + if ip == '127.0.0.1': + User = get_user_model() + user = User.objects.get_by_natural_key('sax') + request.user = user + login(request, user, 'social_core.backends.azuread_tenant.AzureADTenantOAuth2') + return (user, None) class JWTAuthentication(authentication.JSONWebTokenAuthentication): From f177657a690263fc633464ba451c50fb1977cd21 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 14:05:47 +0100 Subject: [PATCH 21/86] add UserServiceToken - reset migrations --- pyproject.toml | 3 +++ .../apps/data/migrations/0001_initial.py | 2 +- .../apps/etl/migrations/0001_initial.py | 2 +- .../apps/subscriptions/migrations/0001_initial.py | 2 +- ..._20181206_1447.py => 0002_auto_20181207_1305.py} | 4 ++-- .../apps/tracking/migrations/0001_initial.py | 2 +- ..._20181206_1447.py => 0002_auto_20181207_1305.py} | 2 +- src/unicef_rest_framework/auth.py | 2 +- .../migrations/0001_initial.py | 12 +++++++++++- ..._20181206_1447.py => 0002_auto_20181207_1305.py} | 13 +++++++++---- src/unicef_rest_framework/models/__init__.py | 1 + src/unicef_rest_framework/models/token.py | 11 +++++++++++ src/unicef_security/migrations/0001_initial.py | 2 +- 13 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 pyproject.toml rename src/etools_datamart/apps/subscriptions/migrations/{0002_auto_20181206_1447.py => 0002_auto_20181207_1305.py} (93%) rename src/etools_datamart/apps/tracking/migrations/{0002_auto_20181206_1447.py => 0002_auto_20181207_1305.py} (97%) rename src/unicef_rest_framework/migrations/{0002_auto_20181206_1447.py => 0002_auto_20181207_1305.py} (94%) create mode 100644 src/unicef_rest_framework/models/token.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..388a6acf1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel", "pipenv"] + diff --git a/src/etools_datamart/apps/data/migrations/0001_initial.py b/src/etools_datamart/apps/data/migrations/0001_initial.py index 8c4e74b38..a2605a968 100644 --- a/src/etools_datamart/apps/data/migrations/0001_initial.py +++ b/src/etools_datamart/apps/data/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.contrib.postgres.fields.jsonb import month_field.models diff --git a/src/etools_datamart/apps/etl/migrations/0001_initial.py b/src/etools_datamart/apps/etl/migrations/0001_initial.py index 92dd230b3..e666b9952 100644 --- a/src/etools_datamart/apps/etl/migrations/0001_initial.py +++ b/src/etools_datamart/apps/etl/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.contrib.postgres.fields.jsonb import django.db.models.deletion diff --git a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py index f04973063..07b473ed0 100644 --- a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py +++ b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.db.models.deletion from django.db import migrations, models diff --git a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181206_1447.py b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181207_1305.py similarity index 93% rename from src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181206_1447.py rename to src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181207_1305.py index 7f73b93e7..8fd005401 100644 --- a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181206_1447.py +++ b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181207_1305.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.db.models.deletion from django.conf import settings @@ -10,8 +10,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('subscriptions', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('subscriptions', '0001_initial'), ] operations = [ diff --git a/src/etools_datamart/apps/tracking/migrations/0001_initial.py b/src/etools_datamart/apps/tracking/migrations/0001_initial.py index 300391fcd..1d11761e3 100644 --- a/src/etools_datamart/apps/tracking/migrations/0001_initial.py +++ b/src/etools_datamart/apps/tracking/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.utils.timezone import strategy_field.fields diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181206_1447.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181207_1305.py similarity index 97% rename from src/etools_datamart/apps/tracking/migrations/0002_auto_20181206_1447.py rename to src/etools_datamart/apps/tracking/migrations/0002_auto_20181207_1305.py index 271fa1e24..9861b7e23 100644 --- a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181206_1447.py +++ b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181207_1305.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.db.models.deletion from django.conf import settings diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index bcd3398a4..94a44d11f 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -39,7 +39,7 @@ def authenticate(self, request): class IPBasedAuthentication(BaseAuthentication): def authenticate(self, request): - if settings.DEBUG: + if settings.DEBUG: # pragma: no cover ip = get_client_ip(request.META) if ip == '127.0.0.1': User = get_user_model() diff --git a/src/unicef_rest_framework/migrations/0001_initial.py b/src/unicef_rest_framework/migrations/0001_initial.py index b1c342065..3894e011d 100644 --- a/src/unicef_rest_framework/migrations/0001_initial.py +++ b/src/unicef_rest_framework/migrations/0001_initial.py @@ -1,9 +1,10 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import uuid import concurrency.fields import django.contrib.postgres.fields +import django.db.models.deletion import strategy_field.fields import unicef_rest_framework.models.acl from django.db import migrations, models @@ -113,4 +114,13 @@ class Migration(migrations.Migration): 'ordering': ('user', 'service'), }, ), + migrations.CreateModel( + name='UserServiceToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('token', models.CharField(max_length=1000)), + ('expires', models.DateField(blank=True, null=True)), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='unicef_rest_framework.Service')), + ], + ), ] diff --git a/src/unicef_rest_framework/migrations/0002_auto_20181206_1447.py b/src/unicef_rest_framework/migrations/0002_auto_20181207_1305.py similarity index 94% rename from src/unicef_rest_framework/migrations/0002_auto_20181206_1447.py rename to src/unicef_rest_framework/migrations/0002_auto_20181207_1305.py index 699cb08ac..d81ea7c2f 100644 --- a/src/unicef_rest_framework/migrations/0002_auto_20181206_1447.py +++ b/src/unicef_rest_framework/migrations/0002_auto_20181207_1305.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.db.models.deletion from django.conf import settings @@ -10,13 +10,18 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('auth', '0009_alter_user_last_name_max_length'), ('unicef_rest_framework', '0001_initial'), + ('auth', '0009_alter_user_last_name_max_length'), + ('contenttypes', '0002_remove_content_type_name'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ + migrations.AddField( + model_name='userservicetoken', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), migrations.AddField( model_name='useraccesscontrol', name='last_modify_user', @@ -131,7 +136,7 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='systemfilter', - unique_together={('service', 'user'), ('service', 'group')}, + unique_together={('service', 'group'), ('service', 'user')}, ), migrations.AlterUniqueTogether( name='groupaccesscontrol', diff --git a/src/unicef_rest_framework/models/__init__.py b/src/unicef_rest_framework/models/__init__.py index 9d8732d70..899b1a388 100644 --- a/src/unicef_rest_framework/models/__init__.py +++ b/src/unicef_rest_framework/models/__init__.py @@ -2,5 +2,6 @@ from .base import MasterDataModel # noqa from .acl import UserAccessControl # noqa from .application import Application # noqa +from .token import UserServiceToken # noqa from .service import Service, CacheVersion # noqa from .filter import SystemFilter, SystemFilterFieldRule # noqa diff --git a/src/unicef_rest_framework/models/token.py b/src/unicef_rest_framework/models/token.py new file mode 100644 index 000000000..04bd84cb9 --- /dev/null +++ b/src/unicef_rest_framework/models/token.py @@ -0,0 +1,11 @@ +from django.conf import settings +from django.db import models + +from .service import Service + + +class UserServiceToken(models.Model): + user = models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE) + service = models.ForeignKey(Service, models.CASCADE) + token = models.CharField(max_length=1000) + expires = models.DateField(blank=True, null=True) diff --git a/src/unicef_security/migrations/0001_initial.py b/src/unicef_security/migrations/0001_initial.py index 573525d7f..6445d7c73 100644 --- a/src/unicef_security/migrations/0001_initial.py +++ b/src/unicef_security/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.4 on 2018-12-06 14:47 +# Generated by Django 2.1.4 on 2018-12-07 13:05 import django.contrib.auth.models import django.contrib.auth.validators From 2fa6ad29e04b71a4773134a3dc52d7a1065ce90a Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 14:37:33 +0100 Subject: [PATCH 22/86] pyproject.toml --- .style.yapf | 4 ---- pyproject.toml | 3 --- 2 files changed, 7 deletions(-) delete mode 100644 .style.yapf delete mode 100644 pyproject.toml diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index bb30eb930..000000000 --- a/.style.yapf +++ /dev/null @@ -1,4 +0,0 @@ -[style] -based_on_style = pep8 -spaces_before_comment = 4 -split_before_logical_operator = true diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 388a6acf1..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel", "pipenv"] - From c246822ac006117614e8bec7d6df77f17228da81 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 17:36:07 +0100 Subject: [PATCH 23/86] updates redoc iqy docstring --- src/unicef_rest_framework/views_mixins.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/unicef_rest_framework/views_mixins.py b/src/unicef_rest_framework/views_mixins.py index 088a753b0..908fe7e1b 100644 --- a/src/unicef_rest_framework/views_mixins.py +++ b/src/unicef_rest_framework/views_mixins.py @@ -1,13 +1,29 @@ from django.conf import settings from django.http import HttpResponse +from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import action from unicef_rest_framework.utils import get_query_string +IQY = """WEB +1 +http://datamart.unicef.io/api/latest/hact/?format=iqy + +Selection=AllTables +Formatting=html +PreFormattedTextToColumns=True +ConsecutiveDelimitersAsOne=True +SingleBlockTextImport=False +DisableDateRecognition=False +DisableRedirections=True +""" + class IQYConnectionMixin: + @swagger_auto_schema(responses={200: "OK"}) @action(methods=['get'], detail=False) def iqy(self, request, version): + """Returns .iqy file to be used as Excel Web Connection """ qs = get_query_string(request.query_params, {'format': 'iqy'}, remove=['format']) url = f"{request.path}".replace('/iqy/', '/') @@ -22,6 +38,6 @@ def iqy(self, request, version): ConsecutiveDelimitersAsOne=True SingleBlockTextImport=False DisableDateRecognition=False - +DisableRedirections=True """.format(host=settings.ABSOLUTE_BASE_URL, request=request, qs=qs, url=url) return HttpResponse(iqy, content_type='text/plain') From 8eba2d24710af30fb8205d43b0c59e9d31097d5e Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 18:17:02 +0100 Subject: [PATCH 24/86] force casting to avoid PicklingError in admin index --- src/etools_datamart/config/admin.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/etools_datamart/config/admin.py b/src/etools_datamart/config/admin.py index 97b9904bf..2b10ef400 100644 --- a/src/etools_datamart/config/admin.py +++ b/src/etools_datamart/config/admin.py @@ -2,12 +2,14 @@ from django.contrib.admin import AdminSite from django.contrib.admin.apps import SimpleAdminConfig -from django.core.cache import cache +from django.core.cache import caches from django.http import HttpResponseRedirect from django.template.response import TemplateResponse from django.utils.translation import gettext_lazy from django.views.decorators.cache import never_cache +cache = caches['default'] + def reset_counters(request): from etools_datamart.apps.tracking.utils import reset_all_counters @@ -83,14 +85,13 @@ def get_section(model, app): for model in app['models']: sec = get_section(model, app) groups[sec].append( - {'app_label': app['app_label'], - 'app_name': app['name'], + {'app_label': str(app['app_label']), + 'app_name': str(app['name']), 'app_url': app['app_url'], 'label': "%s - %s" % (app['name'], model['object_name']), - 'model_name': model['name'], + 'model_name': str(model['name']), 'admin_url': model['admin_url'], 'perms': model['perms']}) - for __, models in groups.items(): models.sort(key=lambda x: x['label']) cache.set(key, groups, 60 * 60) From 18f822bdeda6e3da7ec14ea791a73239debf1f50 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 19:13:23 +0100 Subject: [PATCH 25/86] force content-disposition of iqy --- docker/Makefile | 2 +- src/unicef_rest_framework/views_mixins.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 64e2ed5f4..06fa2af75 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -80,7 +80,7 @@ local: $(MAKE) .run release: - echo ${DOCKER_PASS} | docker login -u ${DOCKER_USER} --password-stdin + @echo ${DOCKER_PASS} | docker login -u ${DOCKER_USER} --password-stdin docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:latest docker push ${DOCKER_IMAGE_NAME}:latest docker push ${DOCKER_IMAGE_NAME}:${TARGET} diff --git a/src/unicef_rest_framework/views_mixins.py b/src/unicef_rest_framework/views_mixins.py index 908fe7e1b..ecc96545e 100644 --- a/src/unicef_rest_framework/views_mixins.py +++ b/src/unicef_rest_framework/views_mixins.py @@ -24,8 +24,12 @@ class IQYConnectionMixin: @action(methods=['get'], detail=False) def iqy(self, request, version): """Returns .iqy file to be used as Excel Web Connection """ + try: + filename = self.get_service().name + except Exception: + filename = self.__class__.__name__ qs = get_query_string(request.query_params, {'format': 'iqy'}, - remove=['format']) + remove=['format', '_display']) url = f"{request.path}".replace('/iqy/', '/') iqy = """WEB @@ -40,4 +44,7 @@ def iqy(self, request, version): DisableDateRecognition=False DisableRedirections=True """.format(host=settings.ABSOLUTE_BASE_URL, request=request, qs=qs, url=url) - return HttpResponse(iqy, content_type='text/plain') + res = HttpResponse(iqy, content_type='text/plain') + if not request.query_params.get('_display', ''): + res['Content-Disposition'] = u'attachment; filename="%s.iqy"' % filename + return res From 8f04ac227e2eeaa5560ce78cc03a8e9b5665c267 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 20:40:23 +0100 Subject: [PATCH 26/86] allow index page customization --- src/etools_datamart/config/admin.py | 25 ++++++++++++++----------- src/etools_datamart/config/settings.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/etools_datamart/config/admin.py b/src/etools_datamart/config/admin.py index 2b10ef400..7440e759a 100644 --- a/src/etools_datamart/config/admin.py +++ b/src/etools_datamart/config/admin.py @@ -1,5 +1,6 @@ from collections import OrderedDict +from django.conf import settings from django.contrib.admin import AdminSite from django.contrib.admin.apps import SimpleAdminConfig from django.core.cache import caches @@ -10,6 +11,18 @@ cache = caches['default'] +DEFAULT_INDEX_SECTIONS = { + 'Administration': ['unicef_rest_framework', 'constance', 'dbtemplates', 'subscriptions'], + 'Data': ['data', 'etools', 'etl'], + 'Security': ['auth', 'unicef_security', + 'unicef_rest_framework.GroupAccessControl', + 'unicef_rest_framework.UserAccessControl', + ], + 'Logs': ['tracking', 'django_db_logging', 'crashlog', ], + 'System': ['redisboard', 'django_celery_beat', 'post_office'], + 'Other': ['unicef_rest_framework.Application', ], +} + def reset_counters(request): from etools_datamart.apps.tracking.utils import reset_all_counters @@ -58,17 +71,7 @@ def index_new(self, request, extra_context=None): app_list = self.get_app_list(request) groups = cache.get(key) if not groups: - sections = { - 'Administration': ['unicef_rest_framework', 'constance', 'dbtemplates', 'subscriptions'], - 'Data': ['data', 'etools', 'etl'], - 'Security': ['auth', 'unicef_security', - 'unicef_rest_framework.GroupAccessControl', - 'unicef_rest_framework.UserAccessControl', - ], - 'Logs': ['tracking', 'django_db_logging', 'crashlog', ], - 'System': ['redisboard', 'django_celery_beat', 'post_office'], - 'Other': ['unicef_rest_framework.Application', ], - } + sections = getattr(settings, 'INDEX_SECTIONS', DEFAULT_INDEX_SECTIONS) groups = OrderedDict([(k, []) for k in sections.keys()]) def get_section(model, app): diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 808731f59..1ba9c8f15 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -619,3 +619,15 @@ def extra(r): SYSINFO = {"extra": {'Azure': extra}} + +INDEX_SECTIONS = { + 'Administration': ['unicef_rest_framework', 'constance', 'dbtemplates', 'subscriptions'], + 'Data': ['data', 'etools', 'etl'], + 'Security': ['auth', 'unicef_security', + 'unicef_rest_framework.GroupAccessControl', + 'unicef_rest_framework.UserAccessControl', + ], + 'Logs': ['tracking', 'django_db_logging', 'crashlog', ], + 'System': ['redisboard', 'django_celery_beat', 'post_office'], + 'Other': ['unicef_rest_framework.Application', ], +} From 37bdb076d8afc2dbe319238c302eafd6b0993a69 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 20:48:58 +0100 Subject: [PATCH 27/86] fixes cache invalidation --- src/unicef_rest_framework/admin/cache.py | 7 +++++-- tox.ini | 26 +++++++++++++----------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/unicef_rest_framework/admin/cache.py b/src/unicef_rest_framework/admin/cache.py index bb3216fa6..c168b5b33 100644 --- a/src/unicef_rest_framework/admin/cache.py +++ b/src/unicef_rest_framework/admin/cache.py @@ -40,6 +40,9 @@ def get_queryset(self, request): def has_add_permission(self, request): return False + def has_delete_permission(self, request, obj=None): + return False + def get_cache_ttl(self, obj): if re.search(r'[smhdwy]', obj.cache_ttl): return "{} ({})".format(obj.cache_ttl, parse_ttl(obj.cache_ttl)) @@ -89,12 +92,12 @@ def generate_cache_token_single(self, request, pk): self.generate_cache_token(request, Service.objects.filter(id=pk)) def incr_version(self, request, queryset): - queryset.update(version=F("cache_version") + 1) + queryset.update(cache_version=F("cache_version") + 1) incr_version.short_description = "Increment version" def reset_version(self, request, queryset): - queryset.update(version=1) + queryset.update(cache_version=1) incr_version.short_description = "Increment version" diff --git a/tox.ini b/tox.ini index bbbb23a40..e5d69e794 100644 --- a/tox.ini +++ b/tox.ini @@ -23,35 +23,37 @@ filterwarnings = ignore::django.utils.deprecation.RemovedInDjango30Warning [tox] -envlist = py{36}-d{21} +envlist = py{36,37}-d{21} minversion = 3.5.2 [testenv] -basepython = python3.6 -passenv = PYTHONDONTWRITEBYTECODE USER PYTHONPATH DATABASE_URL DATABASE_URL_ETOOLS CIRCLECI CIRCLE_* CI +;basepython = python3.6 +passenv = + CI + CIRCLECI CIRCLE_* + DATABASE_URL + DATABASE_URL_ETOOLS + PYTHONDONTWRITEBYTECODE + PIPENV_VERBOSITY + PYTHONPATH + USER + setenv = PYTHONDONTWRITEBYTECODE=true PYTHONPATH={toxinidir}/src + PIPENV_VERBOSITY=-1 deps = pipenv==2018.10.13 -;PIPSI_HOME -;PIPSI_BIN_DIR commands = pipenv install -d --deploy --ignore-pipfile pipenv run pre-commit run --all-files pipenv run pre-commit run --all-files --hook-stage push pipenv run pre-commit run --all-files --hook-stage manual pipenv run py.test tests \ - --create-db \ --cov-report=term \ --cov-report=html \ --cov-config=tests/.coveragerc \ --cov=etools_datamart - - -[testenv:deps] -commands = - pipenv sync - pipenv run {toxinidir}/manage.py check --deploy + {posargs} From 7c7c92cd9be6a0dc0fbb9da6cb40d7c111eb6d0f Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 7 Dec 2018 23:55:37 +0100 Subject: [PATCH 28/86] admin helpers updates --- .../multitenant/postgresql/public.sqldump | Bin 4047913 -> 4047583 bytes src/unicef_rest_framework/admin/cache.py | 5 ++-- src/unicef_rest_framework/admin/service.py | 13 ++++++++--- src/unicef_rest_framework/models/service.py | 6 ++--- src/unicef_rest_framework/renderers/api.py | 14 +++++++++-- .../templates/rest_framework/base.html | 22 ++++++++++++++++-- 6 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/etools_datamart/apps/multitenant/postgresql/public.sqldump b/src/etools_datamart/apps/multitenant/postgresql/public.sqldump index 9dd5980c33d637f5a36de54b87ddc94ef68ec7e1..61254f3855112e0fd46e1c483cf9490d4bea8d9b 100644 GIT binary patch delta 18185 zcmZ`=2b2}X(!SF(%kC2I=A4(DZkRiFhO~>K0*Wl6C

      73X&F;hkr&Fn;5WB;D#9EcdC5ur)OOdY{Tr&1PAJ=Gyvn^NAA$RxtD11WL0`XFcn6R6`N^ zq?l!!_b+Tkv*$^<=ES_9c`|p8I~p|Kf^b9a_a1s$Yx>-@rl z-Za&a5y+<{Jy?-7ptyr4RSie1IVF?*-eJ}Hv+TSCHcn?sSe`Yq;s+lVQE5(3R%+F@ zZtZuAXx5W$>wS`F#9Y^*&}@ITJxlX+hiq$W`;QZ7)mqtcuK%fqtZ?TlZ;@0r638-7 zhjZz@@lw!S-Su~G#%j#`D_kzPTK^6e`LflxB}KdXpE1>(qZQe*TjUp*wf&RLU23X1 zFqUjR9{beSMobHoQ}2Z=+gcQF^r2cL(B6hptA0wl8S0*7*7Zxbwi(y>!_(A2g+sDy z9t)c5MhC5%dV~`k;(-olp=+*7HrMydFmInDhLS;cpPK82S+ja=PxLVYC8AHcbzAiv zo>i-H&8)qklRe#TeFpmS<2tO>CgvA*@>%F%2byg*_G@9K_q{QJQ3Gx50RfX~t?zfE zuPI%#N-r74S~@#MH6nqw*iYt^fo-k2OA};zO)mc7qShqqW4*A$f*qk$} zwY?ZaM|Se5B2jD6=!?Cv#Ufgu)Lb&U(Dc+k`HKQGI4;$yzOIEg)L29hWG#K3<(juu z2dz0{G*98NXv}(R>>!^m8V_WHF3)`R!l1JQM~*-4dXktL3Z&D|FS0Ca@`OA5;i#Hf zU0Y(#JCRNg=1I2brH{!?&DCEFrD9#m6Jrqo>x;=(`-4{XKqgID%W|wMro??{EF36s zpzCk;9TtzC2P^Hr~_)~E-s@v(Z`tXj~SRt%Opo4Y87R<4(u zQs`Z&iP?YoU}_kJwfJ9f!*b0^U(nAVj&1~s%-##~&Cuc|*4%|*pR*A$LrXeXM;2Y_ zPsmUmT6*$rsfjiDp;!W=1quZh3Qkgztep>c@%dxJRCt&sbm<4^*mdb(ucHbaR_gVAik5Ge}%I{Xiyjj z4lgtGY%AKiT}rn4uJV1m7B;NX)lt7|Bm!?yw+H9go6jU$)1PVM%ZRAf+UH*IKcgXQ z?yZofs;2Zxg*aoa z{g!_cYpQ0YZtUmxk3|A`OFxwgtvg=5ECEC)Ab{q_uP0k4H}&xm_NJNjnN&!-hD$+f z+M7wT>k2eYIAj|Y8YSh6ZDZ}J&-Q!9LxCK#z!5p5aXKl*1IDTGI7E%@xor>+w&^y^+RY zmx!Q1OZ9uD6zjXs`zBCv=)p!+FIAeE!99vK{0l#piiZsA#XScl_q2(J;j_ePGiqP9 zRkt_Y=MomGC_R{G{jmRYZ_x3GY8D@?Fb_ATo2P!LFlQdjvVFqGL$HnOzfBPU>-7Vp zJ*$i39LO}Qzss?zzV>w_^9NPW|8;PCTkv z?GNALk1%S$@ILuL&M?m$O}5AM#EA;3|5{2mX9KXc{RA;48(_^oI@sr~hOPa_zE6C{ ztdAO_{!kFbH2vjj|1%b{CZF(+*Eozd%Utniu2uirbe}^^H<$nE#6x%d;m1R9Eo41) zad@zEPd;!kk>r*BF5RdCo{#`Sv%;%=Z zc(A4Pqc2j|srLX@YHr$*ZZ3MJfY$`2-+iPJqL16N5S_aMO8Gz3o%ESMBDm5D8%#4hN-5@!_30cZWVPQdA|{P)6?}Oo$?&;H;=G}=WcuTd7<_t2 zs`lANBk0y6if%uJr3?IS(HLz|rOvc)nv_m;gE8>eB2p=hj6$EUN2PS1Uo_6Yh)MT) z8(UYSfjn$%TGCq1;4|Y=Tc1*m@%4uEZ(lM(9C#Q*%B17n!L_ctROn9zfldMS?#gm` zV^7J?J@lBt4^~N2eU@67k2+8K(r2l~1I40S0Uy~1s&O+1T{EbvE34o;E|mNeSdT{n zm4ddEj`f51&V8k;{0?!IFX}I?_O+phXo|%0C^SH7?gdAD@g-8W-$Re^x=W>j{_?|U z)55F`?|NBc^XNvHPZ@mHS{P9pJ(N}0xtiHISU_8@l#2MBL!`sLWFy8WUnMQ_XK3hw zb~NQGsgfoSV<|MNHP#5dvgtrixjEGiV@>$HVHlTtIvJr5mbXhPCXa(F?0I(sQAAq@ zj6-XZHwzKQ)?_Jzmrjyof11${&aiNhWzw@DDaZ%cN(cN&MK!95v5tJ~Ezi4N^}!>!U>pH>a?+G&#E^My4-wP|p8AKxzBkpSz6!v&ZcMq(92u5GWMt(wlK zm7stp^DWb{lJ1FT#6q?%>8ZP8WW_e)vk+=W5@&An2RXDEgir>grAJ)<6ACm%HX0jW8K+9^$V&0OhVg3jRGA4vEu zBOd0%=1J{*JWPN7d}zSkJ4RgNI~OLlf&oLHu{hza4PE8WKP0X2`x_x%^DywX0mftD zn1T2Qg&6+uQmL<_en7Pld>kq4c1Q}d>@vKHF`RC9b*QK;^%2`qz$ zmSan12@4kz)j)In8a?&au_LG6LK~7<4u9kcDLpZy7~il$8k884hNTx}6pkScOOVC+ zqNk)0iEo|%x=I?5=w@Jd1n*|qe9UU8w@-%SweA^dOyWDtOP)JR8k|&XCwpHMZ$+>F z6H558Y5ELLN8+2=Gh~R2io|VF+aXp znwprR&L`DNa}pekKpv8^Jf8VB_}bYV)MuDesK{(wkV-jCSeOp%Kn`_$K~tLkuD9T= z-j$AdYifkeQ4bbU-54dA3bI)y@AckUV?=33qu-w2&r`{|_+ zoyB2r3A@I5RJHvqyEa7a0SO*rx*bxV1lu@2^^q5ah}r18dgob+Ms|@$2k%z$DQ`Cd z0DEJX-lb&oqr0SJk4q#J;-^1JXf6^8^BJGv+g+{*g0y^*Tk+i8i9HhuAp#NE6kqtS zgbpGhghwK8;wfKBZkZTibqEKXowV?IdtjA5W3i}Snv+T}hD@qM_P6v9YyQxl7QDw^ zsYPOxaXxroLM4%q4*5u7_7kZXdk~8~>T8_#jeX$tmM@c9+AfY(9!*fpan~{u4)c=-F%bVd!Y}*gEX9O{3tX~B z$#5oA@-+5I>3XF(o&Hw(%bR*wMdbmh1~2|zaw}_MoI+3a>y#}1;15zSUr)&5a%cmS za`;b&Fe_hhocBDO@N$u`&hP&T!KE+5PN778aQuiDb3#ZY#Csh}c#KE{d*2S@`8&tG zcv+A|c;(M<86LNY*vobVf7gE`w{|VaRQ~ZV((V4B5KIg@Aw81#)_CKu*oN+gi$v(> z_EI?==p-f4L+?l-sxFpGY2F{s3KUc6AJP$Dqz;#72hH98geUaojo5m_NolA*GDIb? zE=0Jz!>O~ZD;jp9)LcSv+k~5^&zdvB@q7`L=D(872KkJmQI7cY-o&@MQD(kJi9}<3 zmcqs*(9|PDo*zts})j?AVMxJb_TMWfIKH4<)Ji zSE&unPGPIO8Af&9DU~fr48-6cG-ZjEMO`%42?9gS*~LB?!m4{)F#p(#s1d%aCA%Vl zkMgcT_DteiuN{$4Id z>#>c6cxFCxO~$`Sx~1|;+a^SSuonN>_fO)#P{0X`qo`Fn_ehjk$cC}4DO6j?a%`^P zqK9coAsbizwa2CVxw+_?S=X|Vj3PF)ysD{SjekAQO>2bdl_EB&d|%L|tNI%{jV>%^ zW7)$|sw?(~geFuG2lHe7#?}ToyeDCE_ zhu1~t=H=hxyj04Dl}~)drS5-jYFvz!4a%cFWo#&$`wBf-<_nC4qf}qUZmRrxi_2o* zIa$QL(h-d$@?ZaO>4yFdU7WPmY*OVO zr3`ZL@9J~Yc~v7i*cw)P@AcHPjjvUdiOi=?=7<^9(8MERou9m(cAZUNmxyjDe{h2< zW%RkF#6vW$_qpWspLgkLvT)ON3hR!P<(HrfsGK~w%ok`w!ZgM?F zs21hXW$j@(n|jl__K7(<#i85Er%bPKc#JkU!)EG+ecM2}E%C{eO zdE9+&9!88%P-#m8I}Rm=ce;(Aan-XzWjyX zT(%FKn;bC*>K$!pRTT5-bDB;?{rRBwLB%Q?UwO@Gm&g2b^FWN@RzBWVvC9{wQgJN7 z#i@PVQ+ac{?vC{>J~tP?&Owp}E*tCl2o3s4LEs)%V}Yp1;AxkxWYhN1ER`P_#V(W_ zWIWKtNAer5#cA5a?qu-w*Rh8kB2@Zxk?qIn`)^o`O0Q=Z(djYl-!^3|%D=jS^>Qg= zfxb?{zp@I#8pg6hK6xyg?=X$UxpWho;gX8?3 zUqREp6BUG?Q9Ky69Eo7{L{>?uliB;Wd@aUzPiDj1J|b;9*o|dSO&=wl51+y!4g%LW z+F6L#Z)X2+M8+dLek<$hQmKJ%PH|w~R1nx2La;EdFcGl z+gM-c0~v7NJJ{t;XC%5Ess9})-+gc=JKrTxd2j}+cL_Aqv6eiA2>kd=c9~0{^NVJ) z!OjQjF)Qb=vF=BNx4fH;ak?VaQ$iQ;BTq61!UA6cIP+tU<%D#f<`E=<5mcqBqXOmsY2CrJkEJt!U9Jo+W zmGi>IY?%v-2Ci^wC!%E0u!J3Q;W3BSek(zMm(w2)LU`3X5Z?F@E9K)JW<`z=)aK_u z!d`Pb8i4_#&~X9PEo0626U3%FR1s8d9%VZnDx@Qw!~=QL*>}T0+jyRQ{dX6Du62m~q-9c1P+Vk}RYaOn(~w_qh+JJ7Rlvex|1*V#-5i3bl?`A81r4i_GUwc%YuDSvE>XPNO> zpqB$1_7`xt={bntB$+I{b_f{pL7R~iqejkoLUpbR9c=!|6i8(Z}K!4C= zk$AYkAN-81aOB}ZQ8ybdo;vW0cO$QL=nZ?^&Jzd%JfENX7hIe`MsW{~%bV>|#?q-u zkl*no>tYLxhWNTYNOpzg;&<_0rn&U-fI*{YC`GhrmeO?Teyr=VeXi!Bco|phARq27 z=kXq2u~erQyh=n=sk-nx8`uDcYnXp@04bzB3RFdOJQpb9@vm8}hhF;)TkN3W2~@j0 zbV+ud=E9>A)43~I?zr&^2v&4!0*dJ`DBhNmuQJ|{e#V4_%_87ESATEklo$0T? zSPQ=JFZh245yvSd3ROj?aY{MN3_kk|8|mP&A-mYMsaH6Ww=kOGHl#3Pq zUtl?~uDR;M35g47KB$@OIN~U_UI(Xa)zDmawqq2pj=I?}0xXk8w!(iyX24kIwv;b- z49wsMTFE`U-(_j?Qb#Pd(M7cD7pa)#ii<&l?RQC3vQc-@E{q9WSvfT)(z&R_v5TFV1m35M7!qUtukjqL0d6*q@H?E7T9 zMnfsIyzPP)$fQY|?J>}0` zDiIsuy10@Dt7OxG;jXR&{q>zvLhCY=W_))w%+J;YZs+>g#XeejSV^Y?VOA=_mqZPO{vQd??c;7FRl|Ethx4IdMI3*D+`{dqBkZ+BKA594=M5LisLc!IBMiS# zEP3yKavAU6S6<`jA3^xZe%RQ;3#dpbXZ@^nqu`}-Ivu|RE4}t&`9p^>g7F%f^RuE0 zY&MO&6sO^?0p4hFMP*+KXVRkIQ6=A5t|amM2Fl$WzIX(HB4xIXGUxGeA(POzw-BIi=d967l3n4CtlZkCg1_!Rlyw#W$YezP2Q#7Fp)TjT=o_mW%X zeD8PNR2g}m7#7~qZzxPVlGXD((aP8H_nlpISk+i?()Wo za{S%$FqcrL(A{!tnsqPM^o@Ju_Ac14!8~}M9Cd`C_8*}o_sA7|(p;E?t$X-AQNF;V z@-qI%{c^++rSi~&@+02wZS&-8mm1zesJX`iIm8PVV1aF2sC?u?xu^I0sYPlE&=&;i}FRBp;QKO$#3vp~@_ z*FhX#DxYy^#j9)a*qZ=qLIant#8@cw zI65wSO#aJZD6SXnF2^2+@Yf%glU(6=+-tx3<_*hb)rE<>ChuI^vMMNROOP(_L zN2?*iVXNXO^$?jz4`qecuSMSu&tj)JBrycfASvS0o`WqqF!xz_5gmVCP9^9n$lrZl z&U3MGM_Ms$c@ZYn>_uOiC^CRmd%_5U^rj{J_i7mhn>Fy&Pppymxl@|CRz@}{=8f|p zDl(}Os;*~EdD%G@}kOUS5x#>A>+99shkpq_a)8T4WW&0TMU*pP1ALsFe06$NZb^$?lULiO6gOzlX1N%Tx|`Cv zO<+H66Kupr;b`(ueEaJ-2yI2_Jnc;!UheOj&GI0{Y1|K2N9`FIeARZji8Fr` zbxU^QP}sHu^RhL9lD-XLA7h`o^u;^jfb7ZP6yCWCSX0qP$NLBR6~^7uk$ZpOZer884XP_n^<^^KGW$$!M8PKj~jsoci6aN73;f zq1<^OUK&(Vsi76t{j{Q_(Znz1eCqLq{EZN&qj=_$1=E#Oe#w_|q03i;0g9Iki}$!z ztfMGqgDA9DPNfUJadt}@9Y26qPbGWhUxg0z5FYd%fGthjhw*gVhedP1h66smAK+{E z!%5oUaM%HtHo!O^_)0!yPcTd`9YE&T&>$z%ng;o_L#lBwu5^ZnYd0LfBHP5f?^Jr@ zYdK}(*K)bb)Zp#D!J62-kW2oU{m{2b{F-m&GzX8r>nZy&OTZR=hXd2*j{E((@8K+* z4_st^k7X}9gux#9L6#h$xXfs8Kfo!_dTy*h>w#%HsWhbgfuNF)i1eaw3CI zI_w?>Ixhb5?FZ4+^@yBG*hE3v{0o?D{z?AQUWrH?{vNLya{0z1hya{%=zQl<>}Kb~ zAbjLe&2b1TIPQ9HR2;BnkojfvUX60lwtgKauXsU8jXy&{6MvRZ*bJf(TKJ2c%V+;b zPH}V-Mlnb%J&J*??R1Ur`%Mmc__ROdN4?+c|CFzHhKO^!L<~{9*rk_F$<2AszvPw< zoyu!YAyu*Yt9;FAxy1Xu^9+t&don8O(URiqEFEbJbSILpYnb93sXG3`$$n{@OErqp z%5i)jKZ5zRN)nN;fE~USj&DzIye@67C>!i4Ac<*X=ULP#3H#xgqLe#x!)fj!W;Fq# zcM}C+rkHdrOb4rQ6^K{5O=wp#9;iN&thja*Lm@}tf~knY_ogUWF0Ux;$VgTMVI&-L0k@&ZllIlorl1EuM>Mz1U32p^pu@B?rO@ zUiFxUBnmw$`NWnAiVF64ku|nbMmrjfQC%ygE$@}47!Fno@fqn#j{AW!Nrp1Pk&e6z z&ryaWa&4RCO$4PA7d9drdbjy&Dxa34)Vd!rel%AJx*r;iDphiLe!i0ALgF@r8VZyY ze^0>TN|D&vuvrDl`wmy#p?soHNqO$3LdETG@Y*6z7OsTx(TS&*C{D1e~@j+`sz!qh-#8zwTR0^Xz2Qyc!;V0LSzlgmrzFSo%YY?Fx_ z;??;52$YXlaAJ#Zakh3E)pWosr>5GElm|UJEih8t(zNE^_EmZ?8aKd2pr%}7Upeu?7c1F} zN-y>h8fsMHE{fwWD%IA7fvT3xr33$Bpb}-2dAUPugtS2S#C1pQV!0{ZIz%brvo6C7 zsOe>n;tf2V3ttXqLh(jezGDh-II43r<5|Uuml9nv-lNq5!~Tc1nD`{6)LuQp>J{3LQTgmi$|6j7q}vPDlbA4}yq~-D zW(a6`3If*D3IRv|;f@hc%2(d3T;Qk_c7ppoRPr}YRgOw@_suRpT?-5nfoVtDIZbKL zM@>^2Y>P8s7w1#iUUIpnf==G)8QaD?mH844o@Gl`LwG?0P5c(?o)zd zE_h|71umdn2Ur1Le7|x~qLqE^p{XH60oMp4ui(~$N>4_YFLD{-etya#cuHe|QoyKj zz6*wd0E|E85M1F0P+^_vh=3UkP1N@Jtq&{xT!he<9fskCpczdkCC2ke*(=f2kGot^ zYfpPr>CfoD%RGoifatvLacmlU3RuQ|bi5HGZMj1EQKEY`x>LYLAMBL1JFisA89lSw zMPLCh_ihB5Q>UjDl`gHrqVHd&Y?Wx>DvwNa$T%(z3K>X(&YC2I4si@@8`0e+>wm|AR93VHoMm3JZgNk?`#qVcCTVztB~gt5cFeN`!8 z6nfJ|M>SshI#!@ZQZgFP3 zwa1+(R=f5K;TH#g;Xv@>n*iy_3;r$K+3wFBUJ-b8hZjdNyDg*QgDwJwFv_l96YAH^ z>G1|7O5+#68mk)MKj@jSJUZLHG(?%5W70fQM(Q?gf+Nei3TngVU4Is6Kc{y-sOz&tAphv+lnK2pj9g;b)@E3 zBQ*PpCFS$wQWBR4H9?2Kh;k61%c5itqVpw5NeEtUO>!X7P;~Cv33wGn@_*AaIY@w9WT{lZ$%;cR!X@M zQBf3h78(TsS6Ock1usBg-BnS6pzG=?!vELP^YX_3xA=X{_flP5-CbQ>UG?i8kKFUK zvhMz$mGb(5{VuLo+W53&MNxeCFIoIkBmT)0|0KRteyvLW8&oaX4CqGC@U^3W&T{Q% z)N2)op6Jjms?}L}^s}r?dtgEyi?;{`eEFnY%ZluKlkSr(97&M|{Y#A8DQDVF(B4_d>|r)waEdEgZI+(=PY28Fqc9cbs7>s$>^Md$!87 zzsox3X{rb8AG4qKG}TRdv6$u9Tk_I9Y=+<7mml)58Nq1p;v8q*deJd2XV@++8SEvP z_Kec$-lmp)y!=AR;i$w6`m&F2t=upU>F`!<`m7GR&#(+Gk zdqHVqpBi|*x5l!^4UT%+h67;DvfBDc$Ybi-FF>FMJJ zc&2Jb>~Rx@co-uAUs-g*gd%F*q$G(YjSg#77;U(=EP8bCsA%)feCPM6ai)Fznvl0$ z$d?^$nw)3%o;1L-XqMkzIC+HZ&>5f=^yLV?=wkyiqHpD4uw2QG*4_Adi~liU>6zr; zqGa3Mr_PR#HR2DD{|PnEJ}|9gTwuf>f`8k$OwF}RW|(q7!U!UMBl^?@A$yfvRzZdH=(>3`BaeFhBd#hshs z&EA{+h^NnhMd?qf*|hg*H6?oWoJ9MnwJm8Wqi1cgtC=G*J;yU|Ow z;LKNVP%G>UAGAC&!$Dt1L9e9xHqseNBxVF=)?j=PANM6 zp4N8R;(X8ebpwWedW%|Uk9+({j|3y=YcJ?kc3??wFJ1R_7W59$lI2NsW~7oCeZeXe zf}^XJ=R{X6O``fy%BX0+?)hF~^znxb`<-ROr6flQhH00txW=Ou!?GVh4 zX=lH<+8wde&J5do*SK3o5=^`GTF)sF!Lf1WGwVE(BYL#Y^SN~BR*d%5)`fQcbMBsY z+DA;ga(y7a*&V9Rg% z@}oz$7TM)nM#hI^1;WuUwsweqHnGOu^J&q&lX}@Pi zn~qdO{r^g)<|Ek3JHI~dQMnbcUp;=alqQVMvcl5nloL78K?iH?#_zxMvK#i)lV3>! zv~KwQ(bcC*#Bum&L1DD%Kgm)5=`?#=^M@^Mz!J3ShcPV!20^Ez{aI1;(6r3xn1e;p zvj4V?ru~#eLlo@bBL@p&1pClWL%niLJMi;j4+pp9d}*T6!^0B_QA0 zB2+qr74UhfN|J{h_VYz;l;a-3;Sisnro{b_;ScM4Plob~m&f9Jvy`~y8h$;%FU(Qm zTMd*jj_vXSC7W-`RoZzPBQ7s3P)eg#O^sM!`rx#h8vVgPnm?GYxcP-MXWgRp!&n|Y zalhg)^6Lwgw>@%<0FCIv%J>z<%5^Qf&+r>2H4bMT`N=XRzSlsu1i6roRw}7{QiXE4 zM-m)#Lp|#ztd1vCDWj!+9DSN$oJ{+=D5d=Vj><>gHYV-qqIBjDcUD|~DEY-XI-*9Y z;BBfE_aJp<6AAL+HA+0zH2je;|E^Z4lQNyg$O6i!ER4pZ14@5c;cNG)R+ zE~Lh=(vb?9lu`VSkg~%g0pV08?fX^D=2z;!sxJqkWX^@u3BRL8`xmRr2@xDxM7$2! zBTJ?-g(dN(fyx!0?t>OTJy?0l(?LjIe6iAj3VPt!MD$AA2P*BTF;PpT(unfBqu3Vvv_H?R;=-d_3C<#VcNOfbBPBf@WnJRxBs!|NT z@k+(@Xijgs4g=bord32+zn0FsU!|0{9Kc2Zp-@LrSxHBeSt4&5r^I79)P^@tRH{5v zH2u8WB&FP=S!|P40anOwzFsNu4Aj)A`39vu-!@sPmE6Mh1xy&TaD9DlRLXe%jY?}z zlL%x_RVuuc5N#d9@|`^BfenRx)lJF)S?x?LqT{ITQ>L|}Y`>DupPH^5m0dZtR=`&! zYO|^CRwaw`nMyy8Xe&%jSFvDpsg=nG%~EdiHWX2#({SThSP}v&kJAlUkKRZY15f>RmhFrLKu98dV=41Qx(51_7YV^ANU?9X7->FRUhz?j> zyGN;(!xs7rA{)*Z`kRprwIG&`o}5-lD<4u?Q~Lc%BGumub6Rn)@_Brw!C;urzhBAr zhz%Ni<%3EmFArSz0`Eo(h5|fup?AXsLnhy`$h-8xkj0NbqO9=9!J*I;1yAcl4AKng zVN5)i1hB+EdQ2Il#OXnJP{2pUye}!2DJd=dA^zm!aFKDwFruV$g*emBJ)sO!vRW9C z|Dlfwx6GPM<@;ew^Oh==9ytbIzf93R1e1TeTp8gJj6E1FFDd5NtW>V_kS%0F0BiUc ztKe)~#ufDQ>L-~eH_}QnFAs%`N=m4-3e*M!*PY*fl zt8%fCG`e*i&c$z@QO0`u)q!W^MFEY__)Vx?uud88X>B;bOMCi!GwN!dS1$3?nSfO$ z46fiC)+DrYTC3IOt zhevA_{JQN*dAuV;KTSK7`EiM1V4Wg6=KAa2j*vD~IN33^N?-xn_&&>}hBsIWEq_Bv zr?j1L%#9n7=p#q|pk;Jnk?Ss4>kV>Y}kHZC<8tG@U+N_vP3$Q#5W%Cs?!MZ#AAOTg!xtf@Ol?R zz*8^%mu2zyzQib6)NB~M^}oEUYl!Ljt1!K$ue{SUA{f^4bBJ`KFT+;1zoHa9)&OIA z%8*VX%*{}W_?suaD{6*$WwUqtn>w7Ki0UbGJN|ib4K|j4tCmH(PAg~Q;tWJR zeJT{7Vm~Sg{QDoUyW^E2ghOQ|pY?BLx`!SCCh3IQZT{nReMo65WJtaAXE?H!*&>KY z|F7~$TmbUhY$y1o3n@0Xg_UX+CH&?YTO{af=fu73&v+eOBqXBH0^0jKY+%9f%2l3@ z!o2w`%sSq)2q^QZu|`eV^Z=Xa8G!*ct{MV0W-@lUhYtTGqQHFqu*y6RECLrzr-N!G zPf1`1kzzX%pnoMH52@r&B{GkSEI;@~d`H-L*|hOzXn*c8t({V^5WF0j zsU%X}L?xN-NMVgF<3f4?lbxi{c8gLVRXx<~bXY^y8N4XGCNjK7dURe+eT) zw;9ybmSxC+{(cq9B!pdwH2beg8tr`v-Og*v9*uXV`#Q_T^>-U~Ons(@Wt zwYpUT27>?0_4voRuoM6Iea=w9-*VkX{{eqRW{S&iosZw3nT6~IR-HnJ3t6rl8RWMSK3ll2!Ia zm=+d$`j^*l)7jJ4Q)!8ZZiK0K3A?@Oody8vRhhu4HL{rD3HWG4hUU zRu$h|=1B2f^uNWJvCA4NDuX)r#n>-C54#be$IIB1s_NHWxx@cAqt$}IQSWlB>-{nQ z5$EBLp>MPw?Chow$lpFb9ZOU*BvMYSTd4vaby0inku_EJDm+_kOFoyi}j}B}q%h^ix9X%tEfY{86?oI9GG_F67 zfUutcdF4}aCpNl5>sukYZ#W;fP5{43i3{$nb6oBl&%><;DA<`zVW-#7iq0%o`ZB48-@DT9IgfO>t;_(-uICAA40r@cP-Ir8AAfNLJa8TX zUOa@x1fk=F=iDX_o~KE~PoD(YjaARa7#}(xV+%YoJA}plETxMyqleGK|3| zKPM-%0kVgn&bv=xy31<%deVAJ%j1h~WXoNe2+ahIKue{Xf1>|}n^?B2#o^tNPR#@e z^3*iML(ZT>A^zQTL>A7hfX^FrVmhm$?`E)e&e&-8E@;KSLP>w@@q~7?a~AT-DKh|7 zpPRwHkrE>O#7tHzC4~LH?gGuA4Ru;7A36(hqaztP>hKmifu2DNZe{thi|zwer;C(m zLrFUI>Z6r792^{_a+KjP-*_8BCZ|K-m}mdQE^!4%d|emc4k&icY}Qwjbic2Ugih{Y z&$^_buUe4ki#wq1SLUz_q$uQszt3f(TqeWUmCns&74+Jl)nxi~9g?2qvoZVa^H>`F zy^;OmOhu32@*9XtXMSov>n`OOeqMVQyI0BqlnLZiKraXGW@BAi*w?{ z{lQ}na7Z66WRJ^1MgqK{iM3(QEFvM)3s~maN7-Mb6u_7R97yK+zkuYZI%0BVG289d zTfTt~Hgkd)mn3Y}YC$59viT=ZK(CH;E9C3s;5C7zY_Jq%>AZ0nyG%*IRzZ?~a5+rF z{XJwQ>lyp~&?@$zBSZIFzK8?Rbbpe~kktXcaSfX9N_>6;5- zM}T{>$^R7VLWkC}55;KofbI(lR=)dbcAeYW#3JI^Mh>;T6e;?oXR(r!*Ye@zMoY!s zb!@n-z~(iaryKl&=P@3K85d~XByIP4cm`)WLEiO6b~N@oeIt9p<+Xfu)a^yqn`3zEJ{AGHKQGY8&#u!ZP{uTiDBPt>qgU zdytVzW49qU+xTisGGL=CS}Hjz-;?6Qyk2YXTq3!A

      Hw%UdxNFT9CpI zz3!Ttj(~OSA1kTnn}FRnzrnW1<_N;;1hT}NcCiIgy3WhqVo8qS>9_?j91v^JJFauk zbrb6YT#G=;YwQ|qt06)4r&=x&$09khW0o>m7*aSP8a~HOM z`a4SA;*VL4q|dt+iQA8J03u{0(}0zm@3Tx9->a5Y*fDlr&ZmjxVwAr>)7tZqpF~T%#Az_ z0K=IiQs+(*e%t%4>rxEA$xFXy#~lVkxC_Bx@_{GeW<(wm$aDtxGcLwm!2-806(4WTLUxC&nLn|oduK?~$#ZkTV9LDKL4e2~nQTw>uCN_#d(x|7cmGDJOo#$3qzG?}vRcq=f*)km9 z_avwTB?DmK9`r~Xv}{Voz3jO}^=7w1M@vz`Q6UdV1CAjgak!)uRk}a}{-BEkxh-x3 z7-U_lD*dPdFVNkgZ*HSLB1H;ZmX>kY>C0`^GMA=gNg!5Jwsq$Pja8TFNiCVzu&+o#n*iovc@`WYp64~11HD&6nQaX0x#pEwj z`}1&n3{`9!!{j?FRC($cIFvgHbdSupap+uHr5bWcurnU%pe~g{urn?bm&hf2WGD4A zmqF)&F6vz_gNc(w+*RY2xfMU$RlVD#TfR$L=mdHR)pb+z_|6(tdUn$OI58seLDWh!GjWA@DyY8VGMMoF00jHfVjatouB+1J<0USlR)HYW z)QFl#cW1$+UHk?7$-M4r4h=QcbHZlvK9`S(sLwf5GO-1f?&=OH87K5`2U4WnA8D!7 z`yh1GaHXR0w4Uk$m*3>8dSN7vq+o!L>#a_fjd2S16SyC)Ic@+vdbSQrBlfHr)UmPR z6?-mJ2e=jDDZ_-`YALPir)s>epW5E7un+<|6-_yC&|j~?dGtbmwWE?M- zjmtL=RG*h)!M1#95O$(7u#iq0PiR5jeX2;gXz;92Ks#@&Y&F`*s#xyRC~!ncf|Z=%P~_ApFlo z5KRG7Y|Yb0t1`SdaqxC=czMH>*y+v$aS5?;jM~rrL+8IHl`&DRX@ArsVkDa#D;)f3^UoL)?wpS%V&&ccdpm71?nOZlWpY6t8y z6UNJ%u2Z|je($*6aU_Bs#v4oVfdDPNL2bn++@O|1h^Ww!2RaoCZ&VBT_9^P1SnJfO z>R^}21nloz``&Vsnj5QsWSSa@{oXxYEs?*&{B(mFi2d$3Q!R}B9(c1_Amw3^E1W#& zg=uO&UpY%nbLn_X>d+V7ia{Q_Ma_@3{qU zBh9Gf8}3tsE-8qZP>|X@piYamXnZg>7K1NcfaOdTW5I=1nb?CTmWy!ZVeH->4`X4R z85!c{u4!^UHI2|x_?(4mhD$en<)9bvjf=2Wk`>oEf|Vna>L6zT^h&(g=AlPbCYek> z?!KBHg9{rKReyCslZnPQqs=H zv7y>L9&cUyM9fZ1K70vWjWa%2czcmq`ETfp~X*bQ71Xm!b3SawMFejAH0ggm=xMvuBuNkO!oRDjvENi!AsILyc@uKO2GEoXs%_T>pDUGp_u<&$5jipvw?>%LaUyMKgv+Hu!E z2ErnR6|WCy&bQFPkZ;jMO2S>ekc3ys*?j7EYL?sBM3{wXtoS}QjWAx8oPfDH8bwl4 zB(s|3ry%b4ldwx?l!%2=n$@)pl-U><~fchtI`!zhP}UtBLd%;}x$bnbu$OBE9JlOB<4DoCfdvl&Xne{}!f1$408&uKsHfUMhM$@IXO!^>A>qr|8z{u9N)!c1? zhZnokv~kjWBJz1ZP0!GBXjq1J#+iOZ=k=M|R4G1U0H{ZjkjD>YX*n*@M(sV7!0q}BMYSE_ z)0|Ur%(w7q&9zz*ZSA0aE;;c|tcCLwIQiC&n6s>dvPIqO&cOJc!*AQ!n1D8&wc}!D zK|h}0cZI%&bwMwMU9_{38sVT8yK?YMP{^5EjddMct^MFg$IVWepf0V^%J}RWEmtxF z1#+1EwO~HE6({aZ)mlCs?54dVn+0VvBJ+)|?Xa&RKvFZ1I4exG$yj{z6 zDB&Ql3~7gzWKoSUe6+|RN_i(;>&+Y*o~4b9-!JiHriKU2f>}5DbC%YbQOO0aEF;L9 zduZJlebggHF$okuo#?G~rBii)#>QU+(9gdFUh3!tT2vvWzsrheQcZQ*7%AH{`RNO_ zbVfJVxz&*%p}LE>Noh?#^~Z(Q$(teW#p6QU3gwS?Ev-2?CMeYLfmmykCJaL=x@4f% zlTqnVE!UYFp4iko*BHqcX}OF(9vq{axXZw+Qo{8{CjEOTa&f1(1^1!1qwDe3xIu5l z_YBn@a?~2aI~0f?$LQUWQbGt{aP<%vy@c}-z^C!z5!wq1?W}h>0zqHTILC%XKw&CI zX&*QX5kg21c3^uBJpV&U3kH2xIfy&GdpU3$F(5wfa_x{DLeS)WMr-q&H4BA;z6%|6 zog@1EyF$CBxt)eg50;!vo5yL9X9i!bF}nYHmjh?rcn1*YK+CHcqVQ#wV%hd^wK z+Y)*9l^!^J>P>ia)w0;U>?Un3W;E7ihDVR1<9YP%bab<_8Qn~nu6?S|nprPVsMi|65xx_G{Jp|kQ~eE5one(H6% z)|EHkso}$!E{(2GJW@wpUuQyLe5}#=KJBbRmko4B1Tf)h=T>d<1KI_QQWnPOmT$E4 zZmr*rCGia4%|(E zyHoj+$Hd;wSmKH`!Z0BhJkR0hNW(Yz&rz%rH#DM>16dy|`!tcwFVosG>b}WkLi#tV z1@}$j$5&`YjK-{q)#^YJ@fNj$-hWaH^E*}x158-$)*{bxYCBTTr=Z5g7r=OquhH!i>!6LBnEWUib>tJ!u$De;u3ouHu-ChyP50St) z@b7GT;AK3r9{iHFU7AJ0274Pt>wo{>3UWsw&uxIf+6{)&2V&mpx(<*vojUJz9I1#`|s? zoB;7Q0%XqSjr+7zMyEfIahSdd@_8t*Tzq_T+o!Su-y6cg0Y2_QDWAAbhn6qYiujI? zwJq=wAIJEFBXszteGb01`(Y~Zfp|Uou{MMRjj#Gl8==sGkK7hG8f2Fae9QqK!$Y!)4P5x19$M0y?@ISf8{Mj879uqiYs-%V=A>HxRWk2HBb4G{H2>~wW zaNxFnyOepAP{&PN#WiV*`TY-G&8Z2~LP)SV8K*61r~kCw!*RH|N|z zStbV+M`!=U1Sgsh|6dPWG~*E~2R@8bqKg)*z>BbqI5#(jM4?AD@rv=4N5nZS`G{l rI}58fi;X8-^I diff --git a/src/unicef_rest_framework/admin/cache.py b/src/unicef_rest_framework/admin/cache.py index c168b5b33..abbc9612d 100644 --- a/src/unicef_rest_framework/admin/cache.py +++ b/src/unicef_rest_framework/admin/cache.py @@ -28,9 +28,10 @@ class CacheVersionAdmin(ExtraUrlMixin, admin.ModelAdmin): list_display = ('name', 'cache_version', 'get_cache_ttl', 'cache_key') search_fields = ('name', 'viewset') actions = ['incr_version', 'reset_version', 'generate_cache_token'] - readonly_fields = ('cache_key', ) + readonly_fields = ('cache_key', 'name') list_filter = ('hidden',) form = CacheVersionForm + fieldsets = [("", {"fields": ('name', 'cache_version', 'cache_ttl', 'cache_key')})] def get_queryset(self, request): return super(CacheVersionAdmin, self).get_queryset(request). \ @@ -54,7 +55,7 @@ def get_cache_ttl(self, obj): @action(label='View Service') def goto_service(self, request, pk): - url = reverse("admin:core_service_change", args=[pk]) + url = reverse("admin:unicef_rest_framework_service_change", args=[pk]) return HttpResponseRedirect(url) @link(label='Reset cache', css_class="btn btn-danger", icon="fa fa-warning icon-white") diff --git a/src/unicef_rest_framework/admin/service.py b/src/unicef_rest_framework/admin/service.py index 397d2a082..4e4a4fcb3 100644 --- a/src/unicef_rest_framework/admin/service.py +++ b/src/unicef_rest_framework/admin/service.py @@ -39,15 +39,22 @@ def get_stash_url(obj, label=None, **kwargs): class ServiceAdmin(ExtraUrlMixin, admin.ModelAdmin): - list_display = ('name', 'visible', 'security', 'cache_version', 'source_model', - 'json', 'admin') + list_display = ('name', 'visible', 'cache_version', 'source_model', 'json', 'admin') list_filter = ('hidden', 'access') search_fields = ('name', 'viewset') readonly_fields = ('cache_version', 'cache_ttl', 'cache_key', 'viewset', 'name', 'uuid', - 'last_modify_user') + 'last_modify_user', 'source_model',) form = ServiceForm filter_horizontal = ('linked_models',) + fieldsets = [("", {"fields": ('name', + 'description', + # 'access', + # 'confidentiality', + # 'hidden', + 'source_model', + 'linked_models', + )})] # change_list_template = 'admin/unicef_rest_framework/service/change_list.html' diff --git a/src/unicef_rest_framework/models/service.py b/src/unicef_rest_framework/models/service.py index aa66f1f30..fbf792385 100644 --- a/src/unicef_rest_framework/models/service.py +++ b/src/unicef_rest_framework/models/service.py @@ -127,9 +127,8 @@ def get_access_level(self): def endpoint(self): for __, viewset, base_name in conf.ROUTER.registry: if viewset == self.viewset: - return reverse(f'api:{base_name}-list', args=['v1']) - else: - return None + return reverse(f'api:{base_name}-list', args=['latest']) + return None @cached_property def display_name(self): @@ -152,7 +151,6 @@ def save(self, force_insert=False, force_update=False, using=None, update_fields model = v.get_queryset().model ct = ContentType.objects.get_for_model(model) self.linked_models.add(ct) - self.viewset._service = None except Exception as e: logger.exception(e) super(Service, self).save(force_insert, force_update, using, update_fields) diff --git a/src/unicef_rest_framework/renderers/api.py b/src/unicef_rest_framework/renderers/api.py index 54b60c69c..4f78131f6 100644 --- a/src/unicef_rest_framework/renderers/api.py +++ b/src/unicef_rest_framework/renderers/api.py @@ -15,8 +15,13 @@ def get_context(self, data, accepted_media_type, renderer_context): # in the real flow, this is added by the MultiTenant Middleware # but this function is called before the middleware system is involved request = ctx['request'] - for key, value in request.api_info.items(): - ctx['response_headers'][key] = request.api_info.str(key) + view = ctx['view'] + for key, value in sorted(request.api_info.items()): + if key not in ['cache-hit']: + ctx['response_headers'][key] = request.api_info.str(key) + + ctx['extra_actions'] = view.get_extra_action_url_map() + ctx['base_action'] = reverse(f'api:{view.basename}-list', args=['latest']) if request.user.is_staff: try: @@ -25,4 +30,9 @@ def get_context(self, data, accepted_media_type, renderer_context): ctx['admin_url'] = admin_url except Exception: # pragma: no cover pass + + try: + ctx['iqy_url'] = ctx['extra_actions'].pop('Iqy') + except Exception: # pragma: no cover + pass return ctx diff --git a/src/unicef_rest_framework/templates/rest_framework/base.html b/src/unicef_rest_framework/templates/rest_framework/base.html index 751593d6b..778dca248 100644 --- a/src/unicef_rest_framework/templates/rest_framework/base.html +++ b/src/unicef_rest_framework/templates/rest_framework/base.html @@ -115,6 +115,14 @@ {% endif %} + {% if iqy_url %} + + IQY + + {% endif %} + {% if delete_form %} @@ -153,13 +161,23 @@

      {{ name }}

      {{ description }} {% endblock %} - {% if admin_url %} Admin {% endif %} - + {% for name,url in extra_actions.items %} + {% if url != request.build_absolute_uri %} + {{ name }} + {% endif %} + {% endfor %} + {% if request.path != base_action %} + List + {% endif %} {% if paginator %}