From 5ae2c053fda2960456e993a2dcb4dec31f761e41 Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 21 Nov 2018 11:07:22 +0100 Subject: [PATCH 01/43] fixes celery import issue --- Makefile | 13 +++++++++++-- src/etools_datamart/celery.py | 7 ++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f8a668d29..72f6fadda 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,6 @@ fullclean: rm -fr .tox .cache .pytest_cache .venv $(MAKE) clean - sync-etools: sh src/etools_datamart/apps/multitenant/postgresql/dump.sh ${PG_ETOOLS_PARAMS} @@ -56,6 +55,16 @@ ifdef BROWSE firefox ${BUILDDIR}/docs/index.html endif - urf: pipenv run pytest tests/urf --cov-config tests/urf/.coveragerc + + +demo: + pipenv run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid & + pipenv run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid & + pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & + +demo-stop: + - kill `cat gunicorn.pid` + - kill `cat beat.pid` + - kill `cat celery.pid` diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 6779a8489..46930d1c1 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -4,9 +4,6 @@ from celery import Celery from celery.signals import task_postrun, task_prerun from celery.task import Task -from django.utils import timezone - -from etools_datamart.apps.etl.lock import only_one os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etools_datamart.config.settings') @@ -21,6 +18,7 @@ class DatamartCelery(Celery): _mapping = {} def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): + from etools_datamart.apps.etl.lock import only_one linked_model = options.get('linked_model', None) name = name or self.gen_task_name(fun.__name__, fun.__module__) options['lock_key'] = f"{name}-lock" @@ -72,6 +70,7 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): app.timers[task_id] = time() from django.contrib.contenttypes.models import ContentType from etools_datamart.apps.etl.models import EtlTask + from django.utils import timezone defs = {'result': 'RUNNING', 'timestamp': timezone.now()} @@ -83,6 +82,8 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): @task_postrun.connect def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): + from django.utils import timezone + if not hasattr(sender, 'linked_model'): return try: From 9378662fbbb08229727fdd5fb14729a56bbf9e5c Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 21 Nov 2018 11:10:04 +0100 Subject: [PATCH 02/43] updates version number --- CHANGES | 4 ++++ src/etools_datamart/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 29824c8ee..644b09057 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +1.7 (dev) +--------- + + 1.6 --- * add ability to invalidate cache directly from admin endpoint diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index b925ae7f5..775f70f33 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = 'etools-datamart' -VERSION = __version__ = '1.6' +VERSION = __version__ = '1.7a0' __author__ = '' From a1ebcbb9f998a0959f2f5d9b719bb7de6045c38d Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 21 Nov 2018 11:24:05 +0100 Subject: [PATCH 03/43] fixes Makefile --- Makefile | 4 +++- docker/Makefile | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 72f6fadda..71c9b7b61 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,10 @@ demo: pipenv run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid & pipenv run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid & pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & + pipenv run docker run -d -p 5555:5555 --name datamart-flower --rm saxix/flower -demo-stop: +stop-demo: - kill `cat gunicorn.pid` - kill `cat beat.pid` - kill `cat celery.pid` + - docker stop datamart-flower diff --git a/docker/Makefile b/docker/Makefile index f8e32a82a..df8fa0fed 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,8 +2,8 @@ DATABASE_URL?= DATABASE_URL_ETOOLS?= DEVELOP?=0 -DOCKER_PASS?=${DOCKER_PASS} -DOCKER_USER?=${DOCKER_USER} +DOCKER_PASS?= +DOCKER_USER?= TARGET?=dev # below var are used internally BUILD_OPTIONS?= From 413457cb7167573ed7a4e9f3773deeb84e596815 Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 21 Nov 2018 14:21:46 +0100 Subject: [PATCH 04/43] adopt python 3.7 --- .gitignore | 1 + CHANGES | 1 + Pipfile | 6 ++- Pipfile.lock | 122 +++++++++++++++++++++++++++------------------- docker/Dockerfile | 2 +- docker/Makefile | 1 + tox.ini | 5 +- 7 files changed, 83 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 65c3d93b0..bd9c5a080 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +docker/.envrc .DS_Store *.egg-info *.pyo diff --git a/CHANGES b/CHANGES index 644b09057..85566c6d0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,6 @@ 1.7 (dev) --------- +* python 3.7 1.6 diff --git a/Pipfile b/Pipfile index d08c7890a..7427c7f41 100644 --- a/Pipfile +++ b/Pipfile @@ -46,6 +46,7 @@ drf-renderer-xlsx = "*" django-redisboard = "*" djangorestframework-xml = "*" redis = "==2.10.6" +importlib-resources = "==1.0.1" [dev-packages] "flake8" = ">=3.6.0" @@ -58,7 +59,6 @@ factory-boy = "*" ipython = "*" isort = "*" pdbpp = "*" -pre-commit = "*" pytest = "*" pytest-coverage = "*" pytest-django = "*" @@ -66,9 +66,11 @@ pytest-echo = "*" pytest-pythonpath = "*" yapf = "*" vcrpy = "*" +pre-commit = "*" +importlib-resources = "==1.0.1" [requires] -python_version = "3.6" +python_version = "3.7" [scripts] test = "pytest tests" diff --git a/Pipfile.lock b/Pipfile.lock index b549503fc..09435125e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "248d5d3c8b7361358b8f71a0951e19f52b3abd72335b3252070dab04b3e5165a" + "sha256": "7c3846fee4ad797ce98ede0c5c3701dfafe141aade9172e34c0fd25cdbfdb71d" }, "pipfile-spec": 6, "requires": { - "python_version": "3.6" + "python_version": "3.7" }, "sources": [ { @@ -126,28 +126,28 @@ }, "cryptography": { "hashes": [ - "sha256:02915ee546b42ce513e8167140e9937fc4c81a06a82216e086ccce51f347948a", - "sha256:03cc8bc5a69ae3d44acf1a03facdb7c10a94c67907862c563e10efe72b737977", - "sha256:07f76bde6815c55195f3b3812d35769cc7c765144c0bb71ae45e02535d078591", - "sha256:13eac1c477b9af7e9a9024369468d08aead6ad78ed599d163ad046684474364b", - "sha256:179bfb585c5efc87ae0e665770e4896727b92dbc1f810c761b1ebf8363e2fec8", - "sha256:414af0ba308e74c1f8bc5b11befc86cb66b10be8959547786f64258830d2096f", - "sha256:41a1ca14f255df8c44dd22c6006441d631d1589104045ec7263cc47e9772f41a", - "sha256:54947eb98bc4eef99ddf49f45d2694ea5a3929ab3edc9806ad01967368594d82", - "sha256:5bac7a2abda07d0c3c8429210349bb54149ad8940dc7bcffedcd56519b410a3c", - "sha256:7f41af8c586bed9f59cfe8832d818b3b75c860d7025da9cd2db76875a72ff785", - "sha256:8004fae1b3cb2dbd90a011ad972e49a7e78a871b89c70cc7213cf4ebd2532bcb", - "sha256:8e0eccadc3b465e12c50a5b8fb4d39cf401b44d7bb9936c70fddb5e5aaf740d5", - "sha256:95b4741722269cfdc134fec23b7ae6503ee2aea83d0924cfee6d6ec54cd42d8e", - "sha256:a06f5aa6d7a94531dfe82eb2972e669258c452fe9cf88f76116610de4c789785", - "sha256:b0833d27c7eb536bc27323a1e8e22cb39ebac78c4ef3be0167ba40f447344808", - "sha256:b72dec675bc59a01edc96616cd48ec465b714481caa0938c8bbca5d18f17d5df", - "sha256:c800ddc23b5206ce025f23225fdde89cdc0e64016ad914d5be32d1f602ce9495", - "sha256:c980c8c313a5e014ae12e2245e89e7b30427e5a98cbb88afe478ecae85f3abaa", - "sha256:e85b410885addaeb31a867eabcefc9ef4a7e904ad45eac9e60a763a54b244626" + "sha256:05a6052c6a9f17ff78ba78f8e6eb1d777d25db3b763343a1ae89a7a8670386dd", + "sha256:0eb83a24c650a36f68e31a6d0a70f7ad9c358fa2506dc7b683398b92e354a038", + "sha256:0ff4a3d6ea86aa0c9e06e92a9f986de7ee8231f36c4da1b31c61a7e692ef3378", + "sha256:1699f3e916981df32afdd014fb3164db28cdb61c757029f502cb0a8c29b2fdb3", + "sha256:1b1f136d74f411f587b07c076149c4436a169dc19532e587460d9ced24adcc13", + "sha256:21e63dd20f5e5455e8b34179ac43d95b3fb1ffa54d071fd2ed5d67da82cfe6dc", + "sha256:2454ada8209bbde97065453a6ca488884bbb263e623d35ba183821317a58b46f", + "sha256:3cdc5f7ca057b2214ce4569e01b0f368b3de9d8ee01887557755ccd1c15d9427", + "sha256:418e7a5ec02a7056d3a4f0c0e7ea81df374205f25f4720bb0e84189aa5fd2515", + "sha256:471a097076a7c4ab85561d7fa9a1239bd2ae1f9fd0047520f13d8b340bf3210b", + "sha256:5ecaf9e7db3ca582c6de6229525d35db8a4e59dc3e8a40a331674ed90e658cbf", + "sha256:63b064a074f8dc61be81449796e2c3f4e308b6eba04a241a5c9f2d05e882c681", + "sha256:6afe324dfe6074822ccd56d80420df750e19ac30a4e56c925746c735cf22ae8b", + "sha256:70596e90398574b77929cd87e1ac6e43edd0e29ba01e1365fed9c26bde295aa5", + "sha256:70c2b04e905d3f72e2ba12c58a590817128dfca08949173faa19a42c824efa0b", + "sha256:8908f1db90be48b060888e9c96a0dee9d842765ce9594ff6a23da61086116bb6", + "sha256:af12dfc9874ac27ebe57fc28c8df0e8afa11f2a1025566476b0d50cdb8884f70", + "sha256:b4fc04326b2d259ddd59ed8ea20405d2e695486ab4c5e1e49b025c484845206e", + "sha256:da5b5dda4aa0d5e2b758cc8dfc67f8d4212e88ea9caad5f61ba132f948bab859" ], "index": "pypi", - "version": "==2.4.1" + "version": "==2.4.2" }, "defusedxml": { "hashes": [ @@ -421,6 +421,14 @@ ], "version": "==2.7" }, + "importlib-resources": { + "hashes": [ + "sha256:73f454e062ac149bafd262b18c1f9ebc91f53bd6474e028d1bf1c59ebd152efb", + "sha256:87684c76eca1c1b76012a6771be451b2aadc549dbe2725d17ab6398d39878a33" + ], + "index": "pypi", + "version": "==1.0.1" + }, "inflection": { "hashes": [ "sha256:18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca" @@ -664,30 +672,30 @@ }, "ruamel.yaml": { "hashes": [ - "sha256:0dad3f56197e28c04ab251aeb739377a5da74ea83bcac00b79232d7cef66dc7a", - "sha256:19a2757b13c2461572c7d4be61ec25caabb0fc34af48b4565c88a085f570cb76", - "sha256:39cfb38a8590df4ab940b99d512df8a61f871a25f5912727fab1f36b20a12c1d", - "sha256:475762beae3ceedc7e4a007ed12a90f97a262e7ce7239029dc3d0efcdffb2e7b", - "sha256:4d282afb28a7a09dee7df7ddffe3611411e33d05bafd6ddfd6ef40869479252c", - "sha256:54137fa40c62fd72d2646f2635e97e5173dd144a48a3bd25c0c7d743c2966162", - "sha256:54359de3ee1c8955607a050f1dbdfddd0c043f0df1e99d5b82128e3fc6b5c966", - "sha256:584a8513bc7cac176112093c8bc55e457fd4f694b267ef581d8eb1e983e77e7f", - "sha256:5953840b9852e1e2735c9d413d01ddfb768334e475cf130d70064ad8831c30ce", - "sha256:659767cb717052f98fd358ae21bbe192957535693e2c68fba27b2f3c6a14917c", - "sha256:85793c5fe321e9202eba521b0bb3e6303bcb61f6e56378f59e874ca36a7e9d5f", - "sha256:8609f1263a73da187f7febebca24b347aee7da777edc83b8625cb7af3e16b345", - "sha256:883e42e30fcd485b8195522121960cbcc4bba2cb5161cec29b3cfd6e1072eaf4", - "sha256:928146b4a6ef2ce4a5b26663ceddb4961bda00fa1c4e255e6b3e3ef9b284df04", - "sha256:a91a0f5b1e18b4335a0febc757c4a465da0ab6a4f9c672fcc917bcf738efcd66", - "sha256:c0522d1d02856a00e139e2a915c74b20ad1d38f2748ea3f2d4e087ac30322ad2", - "sha256:e3238c0b169f104fa20b217d741632228cbf9011690edabdd265deff09efb299", - "sha256:e9c0b17da17ee512f49de3683c97118877d4ff706ea2edac82f187cd4a9cca4d", - "sha256:eabf69011659182a1044e7581f8ea69e4f4511933955e71fc2f007b28e214791", - "sha256:f00a2ad9b03a8cad5ff620dc0bb7afbd2eaf0ebbf1e5477b632680a2c1d7c656", - "sha256:f6f5921a67211c6a98940d5f03474158c8e3744b26fd0bd5d60ab951aa36c002", - "sha256:fcb63dc194609d1f49e309b976976f8ddd8416e0b8942b3091d3d0a525bfd18a" - ], - "version": "==0.15.78" + "sha256:03d7398452c45bee100f3030bba91c84ef2eb2d8a1607791a170ceb083f81624", + "sha256:05ba73f85dc4fe17964903dd90e1074efaf3299b8db91d66cf0970581f92d9fb", + "sha256:13432f257b626923a389ceb9f13c64c8453f3b58c2a611066ca9cc5582d6d078", + "sha256:161bfaf2379c3fb9d351be3dc24b74801985fe44cc1f031cee97b78d4a33a843", + "sha256:343d4f537e0330dad00fe853d8894fd87c46604024c76c3815b0168eb12062dc", + "sha256:39480f2182b164453405e8dd2bae05108b0da5cbba494f16dffa43e64e7d60dd", + "sha256:3c67edfd639e73e15736e1f92cbf66e0afa7386628388113ea9db023274e15b4", + "sha256:4102b7f97cb6f589c669cd52a033d2fb8f3944a925439baa75fc107d1460499b", + "sha256:45efad27ca82d055a0b2916f63b5a277e6b1b8b955390723cbc4b0edec8d0fb7", + "sha256:4642c7425446bfd3dee66f333050c8ed3cd6c0e0acfa7d63e27c6f871b29d991", + "sha256:4bc5007df709bed082652acc4268c3453927a1223ff9ef42646c1a9d8b4a1416", + "sha256:6daff7cf84a4ec6dc13bf67cce793916eb0f3cbae2648a6bdcc5acf424cffabb", + "sha256:72c6083a1deab4bec019460714d35ab5eb4c0d89fc4fc806306da541f166d062", + "sha256:7c499e1764bbf45b0e9de664e54984ea829506566a03fc96aff47ac58a3f30cb", + "sha256:85cdfff4f2d01758083b07957d3e72fcfdb9f740ea44c5e6f4b815bb41ed087e", + "sha256:953373ab6bce8bb48e7a201eea00c3e989cacc13fe9a5dd10fd9b6ebe1e74fba", + "sha256:a5fdaf4c57d9b2e2ef3e911c6aef61de503a292b47df06a072e2da0a93213f30", + "sha256:abd7c4277b7ccd13e2eafb0094322c9ad22803d86ffa7aaabcddb30253fdf309", + "sha256:c97d7d30a83baf84fbeb3adafec173d05e831bf2281c24e211847aa88775ce07", + "sha256:d0a51f929fa8cffc845e58c5c1cca59d53d1dd5f356b24f55a010eff1f13434f", + "sha256:e4c8857b4883c12f5dcfa6b9d86d4f495a001d93e071cb6e855bb272594e714f", + "sha256:ff89559d2088688ee67b1e9ed96ace44110fa85b47821213f859b65d46310bda" + ], + "version": "==0.15.79" }, "six": { "hashes": [ @@ -769,6 +777,13 @@ ], "version": "==1.1.4" }, + "wheel": { + "hashes": [ + "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", + "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + ], + "version": "==0.32.3" + }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", @@ -999,11 +1014,11 @@ }, "importlib-resources": { "hashes": [ - "sha256:6e2783b2538bd5a14678284a3962b0660c715e5a0f10243fd5e00a4b5974f50b", - "sha256:d3279fd0f6f847cced9f7acc19bd3e5df54d34f93a2e7bb5f238f81545787078" + "sha256:73f454e062ac149bafd262b18c1f9ebc91f53bd6474e028d1bf1c59ebd152efb", + "sha256:87684c76eca1c1b76012a6771be451b2aadc549dbe2725d17ab6398d39878a33" ], - "markers": "python_version < '3.7'", - "version": "==1.0.2" + "index": "pypi", + "version": "==1.0.1" }, "ipython": { "hashes": [ @@ -1352,6 +1367,13 @@ ], "version": "==2.0.32" }, + "wheel": { + "hashes": [ + "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", + "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + ], + "version": "==0.32.3" + }, "wmctrl": { "hashes": [ "sha256:d806f65ac1554366b6e31d29d7be2e8893996c0acbb2824bbf2b1f49cf628a13" diff --git a/docker/Dockerfile b/docker/Dockerfile index 85e7d9744..38b80b2d3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,7 +13,7 @@ RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ fi -FROM python:3.6.7-alpine as base +FROM python:3.7-alpine as base COPY --from=builder /code /code RUN apk add --no-cache --virtual .build-deps \ gcc \ diff --git a/docker/Makefile b/docker/Makefile index df8fa0fed..865d3c846 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -32,6 +32,7 @@ build: --build-arg VERSION=${TARGET} \ -t ${DOCKER_IMAGE} \ -f docker/${DOCKERFILE} . + docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:dev docker images | grep ${DOCKER_IMAGE_NAME} .run: diff --git a/tox.ini b/tox.ini index 2f5da5d0a..c1f006fcf 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ filterwarnings = ignore::django.utils.deprecation.RemovedInDjango30Warning [tox] -envlist = py{36}-d{21} +envlist = py{37}-d{21} minversion = 3.5.2 [testenv] @@ -36,7 +36,8 @@ setenv = deps = pipenv==2018.10.13 - +;PIPSI_HOME +;PIPSI_BIN_DIR commands = pipenv install -d --deploy --ignore-pipfile pipenv run pre-commit run --all-files From ff7aefa1c29d1890b068822708371eab29c1e8f3 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 23 Nov 2018 12:57:51 +0100 Subject: [PATCH 05/43] ad yaml support --- CHANGES | 4 +- Pipfile | 5 +- Pipfile.lock | 114 +++++++------- src/etools_datamart/apps/data/models/fam.py | 1 + src/etools_datamart/apps/data/models/hact.py | 1 + .../apps/data/models/intervention.py | 1 + src/etools_datamart/apps/data/models/pmp.py | 1 + src/etools_datamart/apps/data/models/user.py | 1 + src/etools_datamart/apps/etl/models.py | 7 + src/etools_datamart/apps/web/static/style.css | 141 ------------------ .../apps/web/static/style.css.map | 7 - .../apps/web/static/style.scss | 55 ++++++- .../apps/web/templates/base.html | 2 +- .../apps/web/templates/index.html | 4 +- .../apps/web/templates/monitor.html | 28 ++++ src/etools_datamart/apps/web/urls.py | 3 +- src/etools_datamart/apps/web/views.py | 9 +- src/unicef_rest_framework/views.py | 2 + 18 files changed, 160 insertions(+), 226 deletions(-) delete mode 100644 src/etools_datamart/apps/web/static/style.css delete mode 100644 src/etools_datamart/apps/web/static/style.css.map create mode 100644 src/etools_datamart/apps/web/templates/monitor.html diff --git a/CHANGES b/CHANGES index 85566c6d0..5065c4027 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ 1.7 (dev) --------- -* python 3.7 +* add YAML format +* new 'Monitor' page to check dataset last update date +* 1.6 diff --git a/Pipfile b/Pipfile index 7427c7f41..65981d26b 100644 --- a/Pipfile +++ b/Pipfile @@ -46,7 +46,7 @@ drf-renderer-xlsx = "*" django-redisboard = "*" djangorestframework-xml = "*" redis = "==2.10.6" -importlib-resources = "==1.0.1" +djangorestframework-yaml = "*" [dev-packages] "flake8" = ">=3.6.0" @@ -67,10 +67,9 @@ pytest-pythonpath = "*" yapf = "*" vcrpy = "*" pre-commit = "*" -importlib-resources = "==1.0.1" [requires] -python_version = "3.7" +python_version = "3.6" [scripts] test = "pytest tests" diff --git a/Pipfile.lock b/Pipfile.lock index 09435125e..246fa448d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "7c3846fee4ad797ce98ede0c5c3701dfafe141aade9172e34c0fd25cdbfdb71d" + "sha256": "ce4baecf816efa2f9103b90ab9950ba1e8e544561d45561972ffa087befe1afb" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "3.6" }, "sources": [ { @@ -351,6 +351,14 @@ "index": "pypi", "version": "==1.3.0" }, + "djangorestframework-yaml": { + "hashes": [ + "sha256:58e44b2ba2b1484fe793adcdc5d04910ab9b385a2991603b049b96eed6a76d32", + "sha256:b2277cb0459cf14b473e8cb6e0055725582afe862049c32b840b261ca8fbce3e" + ], + "index": "pypi", + "version": "==1.0.3" + }, "drf-dynamic-serializer": { "hashes": [ "sha256:058ae34570c1dfce4e8e97ac4a5c4ad543279b77f97e2ca254caaff9407c149a" @@ -421,14 +429,6 @@ ], "version": "==2.7" }, - "importlib-resources": { - "hashes": [ - "sha256:73f454e062ac149bafd262b18c1f9ebc91f53bd6474e028d1bf1c59ebd152efb", - "sha256:87684c76eca1c1b76012a6771be451b2aadc549dbe2725d17ab6398d39878a33" - ], - "index": "pypi", - "version": "==1.0.1" - }, "inflection": { "hashes": [ "sha256:18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca" @@ -518,9 +518,9 @@ }, "openpyxl": { "hashes": [ - "sha256:41eb21a5620343d715b38081536c4ed3c37249afb72e569fd2af93852ed4ddde" + "sha256:8b0c2a44f394a7a913a2e7cdcc1dc601d5f45c59b85a356e591e2ac5463e21e7" ], - "version": "==2.5.10" + "version": "==2.5.11" }, "psutil": { "hashes": [ @@ -777,13 +777,6 @@ ], "version": "==1.1.4" }, - "wheel": { - "hashes": [ - "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", - "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" - ], - "version": "==0.32.3" - }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", @@ -919,11 +912,11 @@ }, "django-extensions": { "hashes": [ - "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07", - "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62" + "sha256:8317a3fe479b1ba3e3a04ecf33fb8d6ccf09bb18f30eab64e34c40a593741d26", + "sha256:a76a61566f1c8d96acc7bcf765080b8e91367a25a2c6f8c5bddd574493839180" ], "index": "pypi", - "version": "==2.1.3" + "version": "==2.1.4" }, "django-webtest": { "hashes": [ @@ -1014,11 +1007,11 @@ }, "importlib-resources": { "hashes": [ - "sha256:73f454e062ac149bafd262b18c1f9ebc91f53bd6474e028d1bf1c59ebd152efb", - "sha256:87684c76eca1c1b76012a6771be451b2aadc549dbe2725d17ab6398d39878a33" + "sha256:6e2783b2538bd5a14678284a3962b0660c715e5a0f10243fd5e00a4b5974f50b", + "sha256:d3279fd0f6f847cced9f7acc19bd3e5df54d34f93a2e7bb5f238f81545787078" ], - "index": "pypi", - "version": "==1.0.1" + "markers": "python_version < '3.7'", + "version": "==1.0.2" }, "ipython": { "hashes": [ @@ -1068,37 +1061,37 @@ }, "multidict": { "hashes": [ - "sha256:0573239b5241a075b944b39bdf87fb6600e3a56ad5ca6d2ba9699d62de872309", - "sha256:085b1f55327b4c8c425ce96a7fdfd6a6a1e864444a90d2107f47de4c53b6edea", - "sha256:1ee14a2e7bef872ddac61450e6128aae21304b5165d21fd04681faa3261a7b2e", - "sha256:2c1791371a973d93facccc38adf9e9c14656bf85c2beddd48329134d139b6e7f", - "sha256:2cda0064cab0e2d46b02aeee9e218066993b40d4900b07d8360f54c80eed4c5c", - "sha256:3574eef3ceb983658c3c8bef0c1b3771a2dea338b3822a0c2bec03363f1dc8bb", - "sha256:3864b26cdb1c7454809ec12fb0998989b8832ebb8423aa69ff3a1ad82b9756f3", - "sha256:3e6f7161ea60795f33b21e91b5c9fa66a3dd416f949684ade8ba8a9b193f7e50", - "sha256:3fa7944194cc96319cbbd53a1e0fb6dfe1e437efb75117828c35ce5b30d9d0c9", - "sha256:470ddec7a3ae052862af73dd39a1b1c582a1ec397f8643f09658de56a0a84ac9", - "sha256:5cb1a5926fe898451688036b5e95863c6e75110c98810584695b2403cb04522f", - "sha256:754ed617ecb736261ee3595b4a5dc035bcb5e897ce0a0148252aa8cbc2e59e60", - "sha256:79879c5c0434840d6ac5246e53d22e18c7f5b87419abb968e6357ba65386993a", - "sha256:7db4a72fa35bbe9ccaf3c856b14d89e26e8de5ca0c31604ac5970a3746182f5a", - "sha256:839676a86dbadf4a0be32ca580292c764245044eae324ebfc55362c13886d5d6", - "sha256:8805d8eec8437f9d2b3fd5c09822ef259af08ece0a19f41d2ad8d322a7a67054", - "sha256:8cbb4725aad6dc38cca571dab803f53ed76fc5cc468088636b42b539719aa5f9", - "sha256:9137d7e3db47641aa86526b60dd3d50d2066617668f8617f0c16adf92dfbaa9e", - "sha256:a6d985c3ccc1fca18cfd415406047f0984e3b07f533d50aa91c36eadb46681a9", - "sha256:af381425a02e0a235b23a685cda2d94cd0679ed8257d8a54c5f03ff2eee1fbb7", - "sha256:b7399dfd7f977c419d6e2b08d1099afe00f51454eb2ecc6b067c9eba6efb0a34", - "sha256:b8204fe2cb7199ecd568e67268a49d87f031c294e46d6fdf17bd1e544bcb81ba", - "sha256:ba973fd954f3de8e47e4be43f530729dd7e894615d3734a1b23f4de14f883142", - "sha256:c00b1423d09a73c94553f80ea52dc8c4341beae448bd4468a603263040debb17", - "sha256:ca4fab7f9590b7fa6c5dcde16356726f254456a2bb33d98828d896ba57a5eda1", - "sha256:dcb97bf0efa02a067f2a42c457dfa1548d8bc8913c12f96e26460709bc8a2ae5", - "sha256:e7d1f2671bd62064da2c7d6318c4f9307889cb85c59e00b2d1a66c2ed3bae3eb", - "sha256:ed8a1c22cbf6b0840e8b8a436bc378164a0474580968f38a0eeec8ed7cb78b75", - "sha256:f3826e28328455f62e8de193fb4ab5349ad78da693f1e002fd90d249a0cfaa8b" - ], - "version": "==4.5.0" + "sha256:013eb6591ab95173fd3deb7667d80951abac80100335b3e97b5fa778c1bb4b91", + "sha256:0bffbbbb48db35f57dfb4733e943ac8178efb31aab5601cb7b303ee228ce96af", + "sha256:1a34aab1dfba492407c757532f665ba3282ec4a40b0d2f678bda828ef422ebb7", + "sha256:1b4b46a33f459a2951b0fd26c2d80639810631eb99b3d846d298b02d28a3e31d", + "sha256:1d616d80c37a388891bf760d64bc50cac7c61dbb7d7013f2373aa4b44936e9f0", + "sha256:225aefa7befbe05bd0116ef87e8cd76cbf4ac39457a66faf7fb5f3c2d7bea19a", + "sha256:2c9b28985ef7c830d5c7ea344d068bcdee22f8b6c251369dea98c3a814713d44", + "sha256:39e0600f8dd72acb011d09960da560ba3451b1eca8de5557c15705afc9d35f0e", + "sha256:3c642c40ea1ca074397698446893a45cd6059d5d071fc3ba3915c430c125320f", + "sha256:42357c90b488fac38852bcd7b31dcd36b1e2325413960304c28b8d98e6ff5fd4", + "sha256:6ac668f27dbdf8a69c31252f501e128a69a60b43a44e43d712fb58ce3e5dfcca", + "sha256:713683da2e3f1dd81a920c995df5dda51f1fff2b3995f5864c3ee782fcdcb96c", + "sha256:73b6e7853b6d3bc0eac795044e700467631dff37a5a33d3230122b03076ac2f9", + "sha256:77534c1b9f4a5d0962392cad3f668d1a04036b807618e3357eb2c50d8b05f7f7", + "sha256:77b579ef57e27457064bb6bb4c8e5ede866af071af60fe3576226136048c6dfa", + "sha256:82cf28f18c935d66c15a6f82fda766a4138d21e78532a1946b8ec603019ba0b8", + "sha256:937e8f12f9edc0d2e351c09fc3e7335a65eefb75406339d488ee46ef241f75d8", + "sha256:985dbf59e92f475573a04598f9a00f92b4fdb64fc41f1df2ea6f33b689319537", + "sha256:9c4fab7599ba8c0dbf829272c48c519625c2b7f5630b49925802f1af3a77f1f4", + "sha256:9e8772be8455b49a85ad6dbf6ce433da7856ba481d6db36f53507ae540823b15", + "sha256:a06d6d88ce3be4b54deabd078810e3c077a8b2e20f0ce541c979b5dd49337031", + "sha256:a1da0cdc3bc45315d313af976dab900888dbb477d812997ee0e6e4ea43d325e5", + "sha256:a6652466a4800e9fde04bf0252e914fff5f05e2a40ee1453db898149624dfe04", + "sha256:a7f23523ea6a01f77e0c6da8aae37ab7943e35630a8d2eda7e49502f36b51b46", + "sha256:a87429da49f4c9fb37a6a171fa38b59a99efdeabffb34b4255a7a849ffd74a20", + "sha256:c26bb81d0d19619367a96593a097baec2d5a7b3a0cfd1e3a9470277505a465c2", + "sha256:d4f4545edb4987f00fde44241cef436bf6471aaac7d21c6bbd497cca6049f613", + "sha256:daabc2766a2b76b3bec2086954c48d5f215f75a335eaee1e89c8357922a3c4d5", + "sha256:f08c1dcac70b558183b3b755b92f1135a76fd1caa04009b89ddea57a815599aa" + ], + "version": "==4.5.1" }, "nodeenv": { "hashes": [ @@ -1367,13 +1360,6 @@ ], "version": "==2.0.32" }, - "wheel": { - "hashes": [ - "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", - "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" - ], - "version": "==0.32.3" - }, "wmctrl": { "hashes": [ "sha256:d806f65ac1554366b6e31d29d7be2e8893996c0acbb2824bbf2b1f49cf628a13" diff --git a/src/etools_datamart/apps/data/models/fam.py b/src/etools_datamart/apps/data/models/fam.py index 11084fc49..c4445a04b 100644 --- a/src/etools_datamart/apps/data/models/fam.py +++ b/src/etools_datamart/apps/data/models/fam.py @@ -27,3 +27,4 @@ class FAMIndicator(DataMartModel): class Meta: ordering = ('month', 'country_name') unique_together = ('month', 'country_name') + verbose_name = "FAM Indicator" diff --git a/src/etools_datamart/apps/data/models/hact.py b/src/etools_datamart/apps/data/models/hact.py index bf43fb2d6..b0615b667 100644 --- a/src/etools_datamart/apps/data/models/hact.py +++ b/src/etools_datamart/apps/data/models/hact.py @@ -21,3 +21,4 @@ class HACT(DataMartModel): class Meta: ordering = ('year', 'country_name') unique_together = ('year', 'country_name') + verbose_name = "HACT" diff --git a/src/etools_datamart/apps/data/models/intervention.py b/src/etools_datamart/apps/data/models/intervention.py index ddda23684..ccd9bcb3d 100644 --- a/src/etools_datamart/apps/data/models/intervention.py +++ b/src/etools_datamart/apps/data/models/intervention.py @@ -58,3 +58,4 @@ class Intervention(DataMartModel): class Meta: ordering = ('country_name', 'title') + verbose_name = "Intervention" diff --git a/src/etools_datamart/apps/data/models/pmp.py b/src/etools_datamart/apps/data/models/pmp.py index 135a71980..fa6783a36 100644 --- a/src/etools_datamart/apps/data/models/pmp.py +++ b/src/etools_datamart/apps/data/models/pmp.py @@ -48,3 +48,4 @@ class PMPIndicators(DataMartModel): class Meta: ordering = ('country_name', 'partner_name') + verbose_name = "PMP Indicator" diff --git a/src/etools_datamart/apps/data/models/user.py b/src/etools_datamart/apps/data/models/user.py index 7f94c8b5d..c9a239918 100644 --- a/src/etools_datamart/apps/data/models/user.py +++ b/src/etools_datamart/apps/data/models/user.py @@ -14,3 +14,4 @@ class UserStats(DataMartModel): class Meta: ordering = ('-month', 'country_name') unique_together = ('country_name', 'month') + verbose_name = "User Access Statistics" diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index 501f3e250..87e820339 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -52,6 +52,13 @@ class Meta: def __str__(self): return f"{self.task} {self.result}" + @cached_property + def verbose_name(self): + return self.content_type.model_class()._meta.verbose_name + + def model(self): + return "" + @cached_property def periodic_task(self): try: diff --git a/src/etools_datamart/apps/web/static/style.css b/src/etools_datamart/apps/web/static/style.css deleted file mode 100644 index a01d77625..000000000 --- a/src/etools_datamart/apps/web/static/style.css +++ /dev/null @@ -1,141 +0,0 @@ -* { - margin: 0px; - padding: 0px; - box-sizing: border-box; } - -body, html { - height: 50%; - font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif; } - body a, html a { - color: black; } - body a:hover, html a:hover { - color: black; } - body span.str, html span.str { - color: black; } - -.limiter { - width: 100%; - margin: 0 auto; } - -.master-container { - width: 100%; - min-height: 94vh; - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - flex-wrap: wrap; - justify-content: center; - align-items: center; - padding: 15px; } - -.box { - width: 1170px; - background: #fff; - overflow: hidden; - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - flex-wrap: wrap; - align-items: stretch; - flex-direction: row-reverse; } - -.right { - width: 50%; - flex-wrap: wrap; } - .right form { - padding: 100px 0 0 65px; } - .right .center { - width: 100%; - text-align: center; } - .right h1 { - width: 100%; - font-size: 24pt; - text-align: center; } - .right h2 { - font-size: 14pt; } - .right ul { - padding: 20px 0 0 65px; - list-style: none; - font-size: 18pt; } - .right a { - color: #2090F8; } - -/* [ login more ]*/ -.left { - width: 50%; - background-repeat: no-repeat; - background-size: cover; - background-position: center; - position: relative; - z-index: 1; } - -.m-b-20 { - margin-bottom: 20px; } - -.wrap-input100 { - width: 100%; - position: relative; - border: 1px solid #e6e6e6; } - -.input100 { - display: block; - width: 100%; - background: transparent; - font-size: 18px; - color: #666666; - line-height: 1.2; - padding: 0 25px; } - -input.input100 { - height: 55px; } - -/*[ Button ]*/ -.container-login100-form-btn { - width: 100%; - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - flex-wrap: wrap; - justify-content: center; } - -.login100-form-btn { - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - justify-content: center; - align-items: center; - padding: 0 20px; - width: 100%; - height: 50px; - border-radius: 3px; - background: #2090F8; - font-size: 12px; - color: #fff; - line-height: 1.2; - text-transform: uppercase; - letter-spacing: 1px; - -webkit-transition: all 0.4s; - -o-transition: all 0.4s; - -moz-transition: all 0.4s; - transition: all 0.4s; } - -.login100-form-btn:hover { - background: #333333; } - -/* [ Responsive ]*/ -.navbar { - border-top: none; - background: #2090F8; - color: black; } - .navbar a.navbar-brand { - color: white; } - -/*# sourceMappingURL=style.css.map */ diff --git a/src/etools_datamart/apps/web/static/style.css.map b/src/etools_datamart/apps/web/static/style.css.map deleted file mode 100644 index 5b430f3e7..000000000 --- a/src/etools_datamart/apps/web/static/style.css.map +++ /dev/null @@ -1,7 +0,0 @@ -{ -"version": 3, -"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAO;IACL,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEnB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKhB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK", -"sources": ["style.scss"], -"names": [], -"file": "style.css" -} diff --git a/src/etools_datamart/apps/web/static/style.scss b/src/etools_datamart/apps/web/static/style.scss index 4229471c8..070fc5546 100644 --- a/src/etools_datamart/apps/web/static/style.scss +++ b/src/etools_datamart/apps/web/static/style.scss @@ -64,7 +64,7 @@ body, html { form { padding: 100px 0 0 65px; } - .center{ + .center { width: 100%; text-align: center; } @@ -87,9 +87,9 @@ body, html { list-style: none; font-size: 18pt; } -a { - color: #2090F8; -} + a { + color: #2090F8; + } } @@ -230,3 +230,50 @@ input.input100 { color: white; } } + +.monitor { + .menubar { + padding: 10px; + a { + &:visited { + text-transform: none; + } + text-transform: none; + color: #2090F8; + } + } + #monitor { + width: 100%; + padding: 30px; + .SUCCESS { + color: #00b200; + font-weight: bold; + } + .row { + width: 100%; + padding: 5px; + &.header{ + font-weight: bold; + } + &.odd{ + background-color: #eeeeee; + } + &.even{ + + } + } + + .col { + display: inline-block; + &.task { + width: 30%; + } + &.timestamp { + width: 300px; + } + &.result { + width: 100px; + } + } + } +} diff --git a/src/etools_datamart/apps/web/templates/base.html b/src/etools_datamart/apps/web/templates/base.html index cc2b9a4e7..0b5e894bb 100644 --- a/src/etools_datamart/apps/web/templates/base.html +++ b/src/etools_datamart/apps/web/templates/base.html @@ -9,7 +9,7 @@ {% endblock head %} - + {% block content %} diff --git a/src/etools_datamart/apps/web/templates/index.html b/src/etools_datamart/apps/web/templates/index.html index 065f9d192..9d09c1960 100644 --- a/src/etools_datamart/apps/web/templates/index.html +++ b/src/etools_datamart/apps/web/templates/index.html @@ -9,14 +9,12 @@

eTools Datamart

  • API
  • Documentation
  • Swagger
    • +
  • Monitor
    • {% if request.user.is_staff %}
  • Admin
    • {% endif %} {% if request.user.is_superuser %}
  • System Info
    • - -{#
  • Supervisor
    • #} -{#
  • Flower
    • #} {% endif %}
     
  • Logout
    • diff --git a/src/etools_datamart/apps/web/templates/monitor.html b/src/etools_datamart/apps/web/templates/monitor.html new file mode 100644 index 000000000..13496f22c --- /dev/null +++ b/src/etools_datamart/apps/web/templates/monitor.html @@ -0,0 +1,28 @@ +{% extends 'base.html' %}{% load static %} + +{% block title %}Monitor{% endblock %} + +{% block content %} + +
    + Home + API + Documentation +
    +
    +
    + Data + Last update + Result +
    + {% for t in tasks %} +
    + {{ t.verbose_name }} + {{ t.timestamp|default_if_none:"" }} + {{ t.result }} +
    + {% endfor %} +
    +{% endblock %} diff --git a/src/etools_datamart/apps/web/urls.py b/src/etools_datamart/apps/web/urls.py index 4bd2a39b0..cc066783c 100644 --- a/src/etools_datamart/apps/web/urls.py +++ b/src/etools_datamart/apps/web/urls.py @@ -1,10 +1,11 @@ from django.contrib.auth.views import LoginView, LogoutView from django.urls import path -from .views import DisconnectView, index +from .views import DisconnectView, index, monitor urlpatterns = [ path(r'', index, name='home'), + path(r'monitor/', monitor, name='monitor'), path(r'login/', LoginView.as_view(template_name='login.html'), name='login'), path(r'logout/', LogoutView.as_view(next_page='/'), name='logout'), path(r'disconnect/', DisconnectView.as_view(next_page='/'), name='disconnect'), diff --git a/src/etools_datamart/apps/web/views.py b/src/etools_datamart/apps/web/views.py index d019fab59..c07b1c7cf 100644 --- a/src/etools_datamart/apps/web/views.py +++ b/src/etools_datamart/apps/web/views.py @@ -1,11 +1,18 @@ from django.contrib.auth.views import LogoutView from django.template.response import TemplateResponse +from etools_datamart.apps.etl.models import EtlTask from etools_datamart.config.settings import env def index(request): - return TemplateResponse(request, 'index.html') + context = {'page': 'index'} + return TemplateResponse(request, 'index.html', context) + + +def monitor(request): + context = {'tasks': EtlTask.objects.all(), 'page': 'monitor'} + return TemplateResponse(request, 'monitor.html', context) class DisconnectView(LogoutView): diff --git a/src/unicef_rest_framework/views.py b/src/unicef_rest_framework/views.py index 5cead7c55..ba45e83cb 100644 --- a/src/unicef_rest_framework/views.py +++ b/src/unicef_rest_framework/views.py @@ -10,6 +10,7 @@ from rest_framework.filters import OrderingFilter from rest_framework.renderers import JSONRenderer from rest_framework_xml.renderers import XMLRenderer +from rest_framework_yaml.renderers import YAMLRenderer from strategy_field.utils import fqn from unicef_rest_framework import acl from unicef_rest_framework.auth import JWTAuthentication @@ -46,6 +47,7 @@ class ReadOnlyModelViewSet(DynamicSerializerMixin, viewsets.ReadOnlyModelViewSet renderer_classes = [JSONRenderer, APIBrowsableAPIRenderer, CSVRenderer, + YAMLRenderer, XLSXRenderer, MSJSONRenderer, XMLRenderer, From b3d8c280c0d3fa7effc033a3ed2009b978b9248e Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 23 Nov 2018 13:11:11 +0100 Subject: [PATCH 06/43] ad yaml support --- src/etools_datamart/apps/web/static/style.css | 170 ++++++++++++++++++ .../apps/web/static/style.css.map | 7 + 2 files changed, 177 insertions(+) create mode 100644 src/etools_datamart/apps/web/static/style.css create mode 100644 src/etools_datamart/apps/web/static/style.css.map diff --git a/src/etools_datamart/apps/web/static/style.css b/src/etools_datamart/apps/web/static/style.css new file mode 100644 index 000000000..d0f8b91d6 --- /dev/null +++ b/src/etools_datamart/apps/web/static/style.css @@ -0,0 +1,170 @@ +* { + margin: 0px; + padding: 0px; + box-sizing: border-box; } + +body, html { + height: 50%; + font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif; } + body a, html a { + color: black; } + body a:hover, html a:hover { + color: black; } + body span.str, html span.str { + color: black; } + +.limiter { + width: 100%; + margin: 0 auto; } + +.master-container { + width: 100%; + min-height: 94vh; + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + flex-wrap: wrap; + justify-content: center; + align-items: center; + padding: 15px; } + +.box { + width: 1170px; + background: #fff; + overflow: hidden; + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + flex-wrap: wrap; + align-items: stretch; + flex-direction: row-reverse; } + +.right { + width: 50%; + flex-wrap: wrap; } + .right form { + padding: 100px 0 0 65px; } + .right .center { + width: 100%; + text-align: center; } + .right h1 { + width: 100%; + font-size: 24pt; + text-align: center; } + .right h2 { + font-size: 14pt; } + .right ul { + padding: 20px 0 0 65px; + list-style: none; + font-size: 18pt; } + .right a { + color: #2090F8; } + +/* [ login more ]*/ +.left { + width: 50%; + background-repeat: no-repeat; + background-size: cover; + background-position: center; + position: relative; + z-index: 1; } + +.m-b-20 { + margin-bottom: 20px; } + +.wrap-input100 { + width: 100%; + position: relative; + border: 1px solid #e6e6e6; } + +.input100 { + display: block; + width: 100%; + background: transparent; + font-size: 18px; + color: #666666; + line-height: 1.2; + padding: 0 25px; } + +input.input100 { + height: 55px; } + +/*[ Button ]*/ +.container-login100-form-btn { + width: 100%; + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + flex-wrap: wrap; + justify-content: center; } + +.login100-form-btn { + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + justify-content: center; + align-items: center; + padding: 0 20px; + width: 100%; + height: 50px; + border-radius: 3px; + background: #2090F8; + font-size: 12px; + color: #fff; + line-height: 1.2; + text-transform: uppercase; + letter-spacing: 1px; + -webkit-transition: all 0.4s; + -o-transition: all 0.4s; + -moz-transition: all 0.4s; + transition: all 0.4s; } + +.login100-form-btn:hover { + background: #333333; } + +/* [ Responsive ]*/ +.navbar { + border-top: none; + background: #2090F8; + color: black; } + .navbar a.navbar-brand { + color: white; } + +.monitor .menubar { + padding: 10px; } + .monitor .menubar a { + text-transform: none; + color: #2090F8; } + .monitor .menubar a:visited { + text-transform: none; } +.monitor #monitor { + width: 100%; + padding: 30px; } + .monitor #monitor .SUCCESS { + color: #00b200; + font-weight: bold; } + .monitor #monitor .row { + width: 100%; + padding: 5px; } + .monitor #monitor .row.header { + font-weight: bold; } + .monitor #monitor .row.odd { + background-color: #eeeeee; } + .monitor #monitor .col { + display: inline-block; } + .monitor #monitor .col.task { + width: 30%; } + .monitor #monitor .col.timestamp { + width: 300px; } + .monitor #monitor .col.result { + width: 100px; } + +/*# sourceMappingURL=style.css.map */ diff --git a/src/etools_datamart/apps/web/static/style.css.map b/src/etools_datamart/apps/web/static/style.css.map new file mode 100644 index 000000000..49796ab0e --- /dev/null +++ b/src/etools_datamart/apps/web/static/style.css.map @@ -0,0 +1,7 @@ +{ +"version": 3, +"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEjB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKlB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAKd,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAQ;MACN,WAAW,EAAE,IAAI;IAEnB,0BAAK;MACH,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,gCAAY;MACV,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK", +"sources": ["style.scss"], +"names": [], +"file": "style.css" +} \ No newline at end of file From 718cb53fe28926d14b232396368958fcf0a1cdc0 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 23 Nov 2018 13:35:46 +0100 Subject: [PATCH 07/43] update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c1f006fcf..1bb0cc45b 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ filterwarnings = ignore::django.utils.deprecation.RemovedInDjango30Warning [tox] -envlist = py{37}-d{21} +envlist = py{36}-d{21} minversion = 3.5.2 [testenv] From a1c7272b11cd4a0ccb01a93f65a155a5b9192e1e Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 23 Nov 2018 19:43:59 +0100 Subject: [PATCH 08/43] support azure blob storage --- Pipfile | 2 + Pipfile.lock | 634 ++++++++++++++++++++++++- docker/Dockerfile | 2 +- src/etools_datamart/config/settings.py | 12 + tests/exporters/test_exporter_data.py | 28 ++ 5 files changed, 676 insertions(+), 2 deletions(-) create mode 100644 tests/exporters/test_exporter_data.py diff --git a/Pipfile b/Pipfile index 65981d26b..93e47d253 100644 --- a/Pipfile +++ b/Pipfile @@ -47,6 +47,8 @@ django-redisboard = "*" djangorestframework-xml = "*" redis = "==2.10.6" djangorestframework-yaml = "*" +django-storages = {extras = ["azure"], version = "*"} +onedrivesdk = "*" [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index 246fa448d..9fd3a545f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ce4baecf816efa2f9103b90ab9950ba1e8e544561d45561972ffa087befe1afb" + "sha256": "dcfe11f0359c368a78b964dc582a591500ee26e438ac5c3ce0a52a0730cde154" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,13 @@ ] }, "default": { + "adal": { + "hashes": [ + "sha256:ba52913c38d76b4a4d88eaab41a5763d056ab6d073f106e0605b051ab930f5c1", + "sha256:bf79392b8e9e5e82aa6acac3835ba58bbac0ccf7e15befa215863f83d5f6a007" + ], + "version": "==1.2.0" + }, "admin-extra-urls": { "hashes": [ "sha256:cd5c77c3fcde240472bc9a6c79f8c2358ad287c243e6336902c4996c38212b92" @@ -37,6 +44,585 @@ ], "version": "==0.24.0" }, + "azure": { + "hashes": [ + "sha256:7d6afa332fccffe1a9390bcfac5122317eec657c6029f144d794603a81cd0e50", + "sha256:8bee1f569f700519e8cdfe210259c5c51fa96cee69ccae6b06f60d7224e334e6" + ], + "markers": "extra == 'azure'", + "version": "==4.0.0" + }, + "azure-applicationinsights": { + "hashes": [ + "sha256:6e1839169bb6ffd2d2c21ee3f4afbdd068ea428ad47cf884ea3167ecf7fd0859", + "sha256:c007a9cb7cee4852673e915220fca267e78f97f275f1a8f09b2266a55702462b" + ], + "version": "==0.1.0" + }, + "azure-batch": { + "hashes": [ + "sha256:017be21a9e6db92473d2e33170d5dd445596fc70d706f73552ac9c6b57a6ef1c", + "sha256:cd71c7ebb5beab174b6225bbf79ae18d6db0c8d63227a7e514da0a75f138364c" + ], + "version": "==4.1.3" + }, + "azure-common": { + "hashes": [ + "sha256:2606ae77ff81c0036965b92ec2efe03eaec02a66714140ca0f7aa401b8b9bbb0", + "sha256:c908621a71eb4ee9fab0962e35d3c27a18f09a854d8359c2f32c15b3f4fc576e" + ], + "version": "==1.1.16" + }, + "azure-cosmosdb-nspkg": { + "hashes": [ + "sha256:5d83961922812ffb6e23f7321550b2916abbd7480ec5b4a798417c4a682ff5e9", + "sha256:acf691e692818d9a65c653c7a3485eb8e35c0bdc496bba652e5ea3905ba09cd8" + ], + "version": "==2.0.2" + }, + "azure-cosmosdb-table": { + "hashes": [ + "sha256:4a34c2c792036afc2a3811f4440ab967351e9ceee6542cc96453b63c678c0145", + "sha256:a9c3d2a75b376a45f4bda84af28e698b7578544bb0e9eb7b53fedef3369634a0" + ], + "version": "==1.0.5" + }, + "azure-datalake-store": { + "hashes": [ + "sha256:995703113db6840aa02abab71b2d0699dd283a12130cd843fff8c7a1acde9661", + "sha256:fd1ca3384808ac806470c26c98bc2346c1784d5b281fac4ea468ba018269ee3a" + ], + "version": "==0.0.39" + }, + "azure-eventgrid": { + "hashes": [ + "sha256:297fa9619622338691078112e479361311507e28ce54af17d2c16ace71ffa4e4", + "sha256:7ebbe1c4266ba176aa4969d9755c08f10b89848ad50fb0bfd16fa82e29234f95" + ], + "version": "==1.2.0" + }, + "azure-graphrbac": { + "hashes": [ + "sha256:825b397665f478fab511e521f3f3f4b64189cb9f6c2e7e873b3b7333dc533974", + "sha256:f94b97bdcf774878fe2f8b8c46a5d6550a4ed891350ed0730c1561a24d488ee2" + ], + "version": "==0.40.0" + }, + "azure-keyvault": { + "hashes": [ + "sha256:37a8e5f376eb5a304fcd066d414b5d93b987e68f9212b0c41efa37d429aadd49", + "sha256:dec5334cde846849dfe7896f2e98f17b4f4d75c316a4d30e7171ce71ca20713d" + ], + "version": "==1.1.0" + }, + "azure-loganalytics": { + "hashes": [ + "sha256:3ceb350def677a351f34b0a0d1637df6be0c6fe87ff32a5270b17f540f6da06e", + "sha256:5a1bdb33e1fd3dfb275d9eec45ed8e1126eda51e9072ccf08a19922ee5e0ad98" + ], + "version": "==0.1.0" + }, + "azure-mgmt": { + "hashes": [ + "sha256:8dcbee7b323c3898ae92f5e2d88c3e6201f197ae48a712970929c4646cc2580a", + "sha256:fdca6b5ecf17e5583f6fd3a20cd4e4e0335a3d91131d62b50cfef40d657234b2" + ], + "version": "==4.0.0" + }, + "azure-mgmt-advisor": { + "hashes": [ + "sha256:8fdcb41f760a216e6b835eaec11dba61822777b386139d83eee31f0ff63b05da", + "sha256:9d166cc9868971d03ec852e900fa1f1b95d8829233eff2b1baf702bd34b9887b" + ], + "version": "==1.0.1" + }, + "azure-mgmt-applicationinsights": { + "hashes": [ + "sha256:929c30559692c77d424ca36f11e98f066c98e7eb7b742c44beadc082715f19df", + "sha256:f10229eb9e3e9d0ad20188b8d14d67055e86f3815b43b75eedf96b654bee2a9b" + ], + "version": "==0.1.1" + }, + "azure-mgmt-authorization": { + "hashes": [ + "sha256:2b8504763ea8b1b475f2c3533b171bedb91ffae459f48f1f885ec8536df91093", + "sha256:535de12ff4f628b62b939ae17cc6186226d7783bf02f242cdd3512ee03a6a40e" + ], + "version": "==0.50.0" + }, + "azure-mgmt-batch": { + "hashes": [ + "sha256:6e375ecdd5966ee9ee45b29c90a806388c27ceacc2cbd6dd406ff311b5d7da72", + "sha256:d942e225180d07aae07a43e489ae0acbacd92651be3a019f4eb330a587028161" + ], + "version": "==5.0.1" + }, + "azure-mgmt-batchai": { + "hashes": [ + "sha256:b5f7df6a77fde0bd6b486762eb2c81750b6f1730ee1116689d2dfbd3e03dba95", + "sha256:f1870b0f97d5001cdb66208e5a236c9717a0ed18b34dbfdb238a828f3ca2a683" + ], + "version": "==2.0.0" + }, + "azure-mgmt-billing": { + "hashes": [ + "sha256:3810cdda69ec1409191b292628fe6ba86ce5e0444723b960d91af4f401846ac3", + "sha256:85f73bb3808a7d0d2543307e8f41e5b90a170ad6eeedd54fe7fcaac61b5b22d2" + ], + "version": "==0.2.0" + }, + "azure-mgmt-cdn": { + "hashes": [ + "sha256:069774eb4b59b76ff9bd01708be0c8f9254ed40237b48368c3bb173f298755dd", + "sha256:5d5fa3e3f37632e5787e73e78af37a28d7dd0206c854cc0a4d39d75b953f0c31" + ], + "version": "==3.0.0" + }, + "azure-mgmt-cognitiveservices": { + "hashes": [ + "sha256:36b406ee4b6652cd144a99309cd823ac1c726b0160120c14e4c35cb668f3f8ff", + "sha256:c3247f2786b996a5f328ebdaf65d31507571979e004de7a5ed0ff280f95d80b4" + ], + "version": "==3.0.0" + }, + "azure-mgmt-commerce": { + "hashes": [ + "sha256:c48e84ed322fa9ddbc2d7fcca754c5e97171919be94f510bd2579cf5666684c3", + "sha256:ddcd403bcaf6b7de2cbf1bc249b7db452b35dc1f0503f940368efc722dc0bc90" + ], + "version": "==1.0.1" + }, + "azure-mgmt-compute": { + "hashes": [ + "sha256:5b0c2390af3e29d910e3d6e7a72b0be59d6e15933740dd193129217c000e4fed", + "sha256:8982944ae9022e4999d152356c236c9ff3df0c6024b97f4ef3a4cdef6c01eb62" + ], + "version": "==4.3.1" + }, + "azure-mgmt-consumption": { + "hashes": [ + "sha256:36ea28bb2ed4bec7e4d643444085ba4debed20a01fbd87f599896a4bda3318bd", + "sha256:9a85a89f30f224d261749be20b4616a0eb8948586f7f0f20573b8ea32f265189" + ], + "version": "==2.0.0" + }, + "azure-mgmt-containerinstance": { + "hashes": [ + "sha256:274a9def808407fafe123aa8e9bc1c838a48af2de56419598db7a8b8901086e3", + "sha256:f1ea7d150447f0d8d670b7db13bd2f47320385526021053445d15c427cba6713" + ], + "version": "==1.4.0" + }, + "azure-mgmt-containerregistry": { + "hashes": [ + "sha256:7db871a74dfe6b8f54e208f6e2f43e0f6a034625a735d62013071c5e44409f8d", + "sha256:d5419db4543aaf5d83f73e087df0c0193f6b987f5c6161ac0fdd8eeabbfd23b0" + ], + "version": "==2.4.0" + }, + "azure-mgmt-containerservice": { + "hashes": [ + "sha256:2ff15bc2de14dbcee93d25d7cbe379c88aa751bc13cb199076b127fc4e221acb", + "sha256:99df430a03aada02625e35ef13d7de6c667e9bef56b5e2f60b2c284514223bff" + ], + "version": "==4.2.2" + }, + "azure-mgmt-cosmosdb": { + "hashes": [ + "sha256:a6e70527994d8ce7f4eeca80c7691bc9555adf90819848a9a30284a33b0cffe2", + "sha256:d5f448e9e6733b83e2e6bde28267bb30620c5a3f1399e66dcfe04d0b80960094" + ], + "version": "==0.4.1" + }, + "azure-mgmt-datafactory": { + "hashes": [ + "sha256:63d6cad02ef4d9da788b148b69c708227085b2d948d805bbf54150c7a692393c", + "sha256:6ee02286e9950b9f5b76589459f6d060a962faaab1f49c263a55d011e98b30bf" + ], + "version": "==0.6.0" + }, + "azure-mgmt-datalake-analytics": { + "hashes": [ + "sha256:0d64c4689a67d6138eb9ffbaff2eda2bace7d30b846401673183dcb42714de8f", + "sha256:ac96c9777314831db37461f0602e75298bc25277ba7f4d0d3e7966a926669b7e" + ], + "version": "==0.6.0" + }, + "azure-mgmt-datalake-nspkg": { + "hashes": [ + "sha256:2ac6fa13c55b87112199c5fb03a3098cefebed5f44ac34ab3d39b399951b22c4", + "sha256:3b9e2843f5d0fd6015bba13040dfc2f5fe9bc7b02c9d91dd578e8fe852d1b2dd", + "sha256:deb192ba422f8b3ec272ce4e88736796f216f28ea5b03f28331d784b7a3f4880" + ], + "version": "==3.0.1" + }, + "azure-mgmt-datalake-store": { + "hashes": [ + "sha256:2af98236cd7eaa439b239bf761338c866996ce82e9c129b204e8851e5dc095dd", + "sha256:9376d35495661d19f8acc5604f67b0bc59493b1835bbc480f9a1952f90017a4c" + ], + "version": "==0.5.0" + }, + "azure-mgmt-datamigration": { + "hashes": [ + "sha256:bb654d9f96166a5cf2638f821af5cee8b1c331257c69b053592375a4d2bb4d5e", + "sha256:ea2920475f9e56e660003a06397228243042157d46674f8a09abaf2d0a933aed" + ], + "version": "==1.0.0" + }, + "azure-mgmt-devspaces": { + "hashes": [ + "sha256:220f1610c2cda584e4212611679868d8cc5bdd789d3f0dfa1259b64fc968f580", + "sha256:4710dd59fc219ebfa4272dbbad58bf62093b52ce22bfd32a5c0279d2149471b5" + ], + "version": "==0.1.0" + }, + "azure-mgmt-devtestlabs": { + "hashes": [ + "sha256:7e91bb139b59cfaf1c1b2b0e3e21f091768c658c1879797757dedc6312f00c8c", + "sha256:d416a6d0883b0d33a63c524db6455ee90a01a72a9d8757653e446bf4d3f69796" + ], + "version": "==2.2.0" + }, + "azure-mgmt-dns": { + "hashes": [ + "sha256:3730b1b3f545a5aa43c0fff07418b362a789eb7d81286e2bed90ffef88bfa5d0", + "sha256:5b80546b0f182d7abe90c43025cd5ca7e6605224b4d5b872cca2456667f172ef" + ], + "version": "==2.1.0" + }, + "azure-mgmt-eventgrid": { + "hashes": [ + "sha256:824503b668137affa5b3782c6348c0bb6ab012c72fe47a3be9942c5639f82f8a", + "sha256:9518e9d7e60ab90a7d18ae6a3f0049ca57588f8f9583ab40d442c2a5387cf94e" + ], + "version": "==1.0.0" + }, + "azure-mgmt-eventhub": { + "hashes": [ + "sha256:675804761cb146fd4eef9b4fb2aecf257da9837780bd03147e74906d7473c4c1", + "sha256:b5407e529b9daeefbb9393c8f7401f44a21ecfeede6e03cf08456149c1d3533e" + ], + "version": "==2.2.0" + }, + "azure-mgmt-hanaonazure": { + "hashes": [ + "sha256:9fe4dc0adeb772d13918e1d6126d83c7770b762f358487504c5f082f542d0189", + "sha256:aec953c54809d0cc2f61f24d4d62a97f02c466bdc7906fd66f30120becf0c3df" + ], + "version": "==0.1.1" + }, + "azure-mgmt-iotcentral": { + "hashes": [ + "sha256:0d2101f3ea8a21ec3b29ee72d83e6ca606a241efec3b042cda8c656ad99b8fd2", + "sha256:59f7c653ac7d6475d5d7900902a5d0e0fe7aad03224c47d70f2cf1e20d43a81d" + ], + "version": "==0.1.0" + }, + "azure-mgmt-iothub": { + "hashes": [ + "sha256:08388142ed6844f0a0e97d2740decf80ffc94f22adca174c15f60b9e2c2d14be", + "sha256:77221c5b6ff7feabc2e6d44156e29fbee9098b34de044165d3532b64b678bdb1" + ], + "version": "==0.5.0" + }, + "azure-mgmt-iothubprovisioningservices": { + "hashes": [ + "sha256:2b3480a8ad2e535928da55de92b6127d02171768fed375b112274eb1e55268c1", + "sha256:8c37acfd1c33aba845f2e0302ef7266cad31cba503cc990a48684659acb7b91d" + ], + "version": "==0.2.0" + }, + "azure-mgmt-keyvault": { + "hashes": [ + "sha256:05a15327a922441d2ba32add50a35c7f1b9225727cbdd3eeb98bc656e4684099", + "sha256:406298b6236abf9e3bae3218df2fc89c25aee471b917eca7769ff5696fc8ec01" + ], + "version": "==1.1.0" + }, + "azure-mgmt-loganalytics": { + "hashes": [ + "sha256:c7315ff0ee4d618fb38dca68548ef4023a7a20ce00efe27eb2105a5426237d86", + "sha256:f224b7d52f4369ce057c7f83e80da1d00a8887ad5c15606529e9c930e601088f" + ], + "version": "==0.2.0" + }, + "azure-mgmt-logic": { + "hashes": [ + "sha256:232c175e45582f7c547d3b50d93bd64aec37b400426962e4fd0cd235980ea110", + "sha256:d163dfc32e3cfa84f3f8131a75d9e94f5c4595907332cc001e45bf7e4efd5add" + ], + "version": "==3.0.0" + }, + "azure-mgmt-machinelearningcompute": { + "hashes": [ + "sha256:4995ba9ee392eb4f5579e93dba9187b67007187e7fd022d6b417ec56e6761a6b", + "sha256:7a52f85591114ef33a599dabbef840d872b7f599b7823e596af9490ec51b873f" + ], + "version": "==0.4.1" + }, + "azure-mgmt-managementgroups": { + "hashes": [ + "sha256:005e8289c2e1d8a8368c96790edf6a34e5c37b4096bce2eb8a923c6d5dc11fb2", + "sha256:ff62d982edda634a36160cb1d15a367a9572a5acb419e5e7ad371e8c83bd47c7" + ], + "version": "==0.1.0" + }, + "azure-mgmt-managementpartner": { + "hashes": [ + "sha256:1b0ec9b9d084e331b863cef77f002ede8cbc6214bb56c3c8dd7945d10c7ffc77", + "sha256:2d7d5f346cc3d6ad621a39357637565199c51a65c7e451b73aced1c097a78165" + ], + "version": "==0.1.0" + }, + "azure-mgmt-maps": { + "hashes": [ + "sha256:a779b1ddbbcd95393e53f11b586dd26c42a709aaa226412a2df64d0da6807a80", + "sha256:c120e210bb61768da29de24d28b82f8d42ae24e52396eb6569b499709e22f006" + ], + "version": "==0.1.0" + }, + "azure-mgmt-marketplaceordering": { + "hashes": [ + "sha256:6da12425cbab0cc62f246e7266b4d67aff6bdd031ecbe50c7542c2f2b2440ad4", + "sha256:fb7a21f4a4a4b8d32bae600614f047a17993111374c9567ac11f241ada61d69f" + ], + "version": "==0.1.0" + }, + "azure-mgmt-media": { + "hashes": [ + "sha256:688b56daad16e84afc7ad788a1d0d2969e365317db0d6034dd428a08030d21aa", + "sha256:6d68668b14c00b4c68f695f2bb69ff77af29034d440371c2bdd9c80187b1a08c" + ], + "version": "==1.0.0" + }, + "azure-mgmt-monitor": { + "hashes": [ + "sha256:838867a150694837e9c6141760ff0f20fe9e5b7ab88f9ba868fde1810855895e", + "sha256:f1a58d483e3292ba4f7bbf3104573130c9265d6c9262e26b60cbfa950b5601e4" + ], + "version": "==0.5.2" + }, + "azure-mgmt-msi": { + "hashes": [ + "sha256:8622bc9a164169a0113728ebe7fd43a88189708ce6e10d4507247d6907987167", + "sha256:e989e61753bf4eca0e688526b7c31c9a88082080acfb038fad17dda7f084a026" + ], + "version": "==0.2.0" + }, + "azure-mgmt-network": { + "hashes": [ + "sha256:bf2b459d055083eb5c8883b66828c1e7d917f2d3eef98bf65417022450c6ea7b", + "sha256:ef8593a3a911417959f70df25f15344eadcdf1bfc7c0706a62d82ede2d1fab31" + ], + "version": "==2.3.0" + }, + "azure-mgmt-notificationhubs": { + "hashes": [ + "sha256:481aaf1c5c4b951c3114f9d256913a443892a18d3c2130f63149edf160b6b70d", + "sha256:7c4c7755c28c8301cfa90d6ded9509c30444e5dfc5001b132dca57836930602b" + ], + "version": "==2.0.0" + }, + "azure-mgmt-nspkg": { + "hashes": [ + "sha256:1c6f5134de78c8907e8b73a8ceaaf1f336a24193a543039994fe002bb5f7f39f", + "sha256:8b2287f671529505b296005e6de9150b074344c2c7d1c805b3f053d081d58c52", + "sha256:d638ea5fda3ed323db943feb29acaa200f5d8ff092078bf8d29d4a2f8ed16999" + ], + "version": "==3.0.2" + }, + "azure-mgmt-policyinsights": { + "hashes": [ + "sha256:49b88331bf823a030182ff492b728828c35758c7e49c8898a403bce5b210ba49", + "sha256:ff94cb12d6e01bf1470c2a6af4ce6960669ab4209106153879ff97addc569ce1" + ], + "version": "==0.1.0" + }, + "azure-mgmt-powerbiembedded": { + "hashes": [ + "sha256:2f05be73f2a086c579a78fc900e3b2ae14ccde5bcec54e29dfc73e626b377476", + "sha256:6f75fef7ff576383c8c6692ba5a1efa634e6eded99d0ff6e76fdc8327325fe2f" + ], + "version": "==2.0.0" + }, + "azure-mgmt-rdbms": { + "hashes": [ + "sha256:3b6a194e6b82aa9fa187d1060ff3f19ad7218317b9ae30d9b64c3113ac8dfd7c", + "sha256:f9b77f0ead387f48c3a81914e2b976967300b3e31be10a101abd00057af7bddd" + ], + "version": "==1.5.0" + }, + "azure-mgmt-recoveryservices": { + "hashes": [ + "sha256:29df3e58890492efdd80d608b7a0fd2006c8a908687c35b9b1f70af068c0f6e4", + "sha256:e48f7769fb10a85ad857710c2cba47880166f69fe7da6b331771f129b21de95c" + ], + "version": "==0.3.0" + }, + "azure-mgmt-recoveryservicesbackup": { + "hashes": [ + "sha256:1e55b6cbb808df83576cef352ba0065f4878fe505299c0a4c5a97f4f1e5793df", + "sha256:5c44bd73df6eb55382335f16a21aee62ddc8e271cd913b9147768f42aba59c3d" + ], + "version": "==0.3.0" + }, + "azure-mgmt-redis": { + "hashes": [ + "sha256:374a267b83ec4e71077b8afad537863fb93816c96407595cdd02973235356ded", + "sha256:41d12cea5673b2e277ea298d85bab7d1fdbd0636ff6be08b15ee30a312c07d6d" + ], + "version": "==5.0.0" + }, + "azure-mgmt-relay": { + "hashes": [ + "sha256:1411e734573ce6166ac7a75fbfc0afb7d6b3f47a94d0b4999b6adf2709eba87c", + "sha256:d9f987cf2998b8a354f331b2a71082c049193f1e1cd345812e14b9b821365acb" + ], + "version": "==0.1.0" + }, + "azure-mgmt-reservations": { + "hashes": [ + "sha256:40618a3700c47a788182649f238d985edf15b08b6577ea27557e70e2866ac171", + "sha256:612acfa18f005c2ee5dda5c473b8bf6540d232db331b27be47de2608c5855adb" + ], + "version": "==0.2.1" + }, + "azure-mgmt-resource": { + "hashes": [ + "sha256:2e83289369be88d0f06792118db5a7d4ed7150f956aaae64c528808da5518d7f", + "sha256:8dcd62521482f04fb0927ee4800ef6ab3ac99a9158005d552a7c10f9c756cd8c" + ], + "version": "==2.0.0" + }, + "azure-mgmt-scheduler": { + "hashes": [ + "sha256:59e7cced3ee9b93016efb8cf5f965ca11a463bb8e55f96a2f200b013426dd751", + "sha256:c6e6edd386ddc4c21d54b1497c3397b970bc127b71809b51bd2391cb1f3d1a14" + ], + "version": "==2.0.0" + }, + "azure-mgmt-search": { + "hashes": [ + "sha256:0ec5de861bd786bcb8691322feed6e6caa8d2f0806a50dc0ca5d640591926893", + "sha256:fdbaa1721b045a4ea4a21c84c6bc1f9636b39e93dff09ffd68f22e5da88bd3ea" + ], + "version": "==2.0.0" + }, + "azure-mgmt-servicebus": { + "hashes": [ + "sha256:7d1e8c3dc05ffdfe496ae643290ce4de93a3bf814ffda69121223e3d7da12408", + "sha256:cbc3fc8d8b8930452cf7d499856313da208eb66f62ade0091aa9bae0db2b1674" + ], + "version": "==0.5.3" + }, + "azure-mgmt-servicefabric": { + "hashes": [ + "sha256:0c1434e789d0c036c613855b898a385a4533656f45eafae3ef7af3ecf4d6a3e8", + "sha256:b2bf2279b8ff8450c35e78e226231655021482fdbda27db09975ebfc983398ad" + ], + "version": "==0.2.0" + }, + "azure-mgmt-signalr": { + "hashes": [ + "sha256:37a79dfe21af4addbd9cdf248a260387caabdfdd60d2ba6f3174d28e96660655", + "sha256:8a6266a59a5c69102e274806ccad3ac74b06fd2c226e16426bbe248fc2174903" + ], + "version": "==0.1.1" + }, + "azure-mgmt-sql": { + "hashes": [ + "sha256:43668705f17bd3532e2e489d368937eb19e7d0515638146c156982ace76e0743", + "sha256:5da488a56d5265757b45747cf5fd22413eb089e606658d6e6d84fe3e9b07e4fa" + ], + "version": "==0.9.1" + }, + "azure-mgmt-storage": { + "hashes": [ + "sha256:512a29798833453f8c32a5b6d038a459649bbb5b9970ac23c982b5787057fa2b", + "sha256:9577cea1f7a86ca1db6f14539bd05ce27f43ebe590cc7f23c943961a2c5c1cdc" + ], + "version": "==2.0.0" + }, + "azure-mgmt-subscription": { + "hashes": [ + "sha256:309b23f0de65f26da80c801e913b0c3b2aea8b90ba583d919f81fe6f329d3f1b", + "sha256:a37925fb820cb86dfb57559846cc97c7e066fe0e64da7594175f4a4f5e50783c" + ], + "version": "==0.2.0" + }, + "azure-mgmt-trafficmanager": { + "hashes": [ + "sha256:126167eaa82b443b5b71394050ec292f45074701232bdbdda71f636e9b46516b", + "sha256:65796588ffbeac45bf73668977131c317b64d6a3f32faecdc5cbf9683d48132c" + ], + "version": "==0.50.0" + }, + "azure-mgmt-web": { + "hashes": [ + "sha256:8ea0794eef22a257773c13269b94855ab79d36c342ad15a98135403c9785cc0a", + "sha256:f4ddb4850314325db688241caa323fa80d811dc4590454d87f5c5b558557ea51" + ], + "version": "==0.35.0" + }, + "azure-nspkg": { + "hashes": [ + "sha256:1d0bbb2157cf57b1bef6c8c8e5b41133957364456c43b0a43599890023cca0a8", + "sha256:31a060caca00ed1ebd369fc7fe01a56768c927e404ebc92268f4d9d636435e28", + "sha256:e7d3cea6af63e667d87ba1ca4f8cd7cb4dfca678e4c55fc1cedb320760e39dd0" + ], + "version": "==3.0.2" + }, + "azure-servicebus": { + "hashes": [ + "sha256:30d5beaf73eaf40aba52fdd7f7f26dd8c3e639051dc19a5f2ab5f8e7832d68f7", + "sha256:bb6a27afc8f1ea9ab46ff2371069243d45000d351d9b64e450b63d52409b934d" + ], + "version": "==0.21.1" + }, + "azure-servicefabric": { + "hashes": [ + "sha256:8724718ef48c2810dbd8609c193225fa1e037a2c9ce70c7263f5b031e2e8f208", + "sha256:c82575cbdf95cc897c3230ea889d4e751d8760a2223857fe6fbeeea5b802e5e2" + ], + "version": "==6.3.0.0" + }, + "azure-servicemanagement-legacy": { + "hashes": [ + "sha256:282d48aae6aa002c59db6f651b68777a8f93692bb8e9b443113e6a8d5ce5e875", + "sha256:c883ff8fa3d4f4cb7b9344e8cb7d92a9feca2aa5efd596237aeea89e5c10981d" + ], + "version": "==0.20.6" + }, + "azure-storage-blob": { + "hashes": [ + "sha256:65ebe2e54460566c2077c6b3773a2a0623eabc7b95602010cb51b84077087fda", + "sha256:baa828607e21e5c7b6ceb2ede9894d465adf586373c2f7c988fe55eca8e9048c" + ], + "markers": "extra == 'azure'", + "version": "==1.4.0" + }, + "azure-storage-common": { + "hashes": [ + "sha256:69bba6aad1e8a717eeee0f95c2feeeed72ef802001e66d6d15bf8446c4f53e6a", + "sha256:7ab607f9b8fd27b817482194b1e7d43484c65dcf2605aae21ad8706c6891934d" + ], + "version": "==1.4.0" + }, + "azure-storage-file": { + "hashes": [ + "sha256:5217b0441b671246a8d5f506a459fa3af084eeb9297c5be3bbe95d75d23bac2f", + "sha256:65831e66594cdda36e02f5566ea9d8a6ad35eca6691c28f1fbb49f23987752ff" + ], + "version": "==1.4.0" + }, + "azure-storage-queue": { + "hashes": [ + "sha256:0bafe9e61c0ce7b3f3ecadea21e931dab3248bd4989dc327a8666c5deae7f7ed", + "sha256:d28e6f854ed5d719d62637c1b5c2b74d9c67584bc326de5ce41ba0af73e3a3f0" + ], + "version": "==1.4.0" + }, "billiard": { "hashes": [ "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84" @@ -300,6 +886,17 @@ "index": "pypi", "version": "==1.1.0" }, + "django-storages": { + "extras": [ + "azure" + ], + "hashes": [ + "sha256:8e35d2c7baeda5dc6f0b4f9a0fc142d25f9a1bf72b8cebfcbc5db4863abc552d", + "sha256:b1a63cd5ea286ee5a9fb45de6c3c5c0ae132d58308d06f1ce9865cfcd5e470a7" + ], + "index": "pypi", + "version": "==1.7.1" + }, "django-strategy-field": { "hashes": [ "sha256:659ce7d1fec5dc7770291f64d530e345e155d4d24f843f225d0b5a451f7b3706" @@ -435,6 +1032,13 @@ ], "version": "==0.3.1" }, + "isodate": { + "hashes": [ + "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", + "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" + ], + "version": "==0.6.0" + }, "itypes": { "hashes": [ "sha256:c6e77bb9fd68a4bfeb9d958fea421802282451a25bac4913ec94db82a899c073" @@ -509,6 +1113,20 @@ ], "version": "==1.1.0" }, + "msrest": { + "hashes": [ + "sha256:1b8daa01341fb77b0797c5fbc28e7e957388eb562721cc6392603ea5a4a39345", + "sha256:75adf8a044a6ff3a93dee977b8e8efb44da3e339b07aada87dcf53d6d51ca67c" + ], + "version": "==0.6.2" + }, + "msrestazure": { + "hashes": [ + "sha256:1118d52fb60fd71732a51bcb669189af4f72f40ea460e656465ee83c4738f2a0", + "sha256:fd1bcb9652cf04b711e21dcbef377a7e43f9492afeb0a59622b5623ce8715825" + ], + "version": "==0.5.1" + }, "oauthlib": { "hashes": [ "sha256:ac35665a61c1685c56336bda97d5eefa246f1202618a1d6f34fccb1bdd404162", @@ -516,6 +1134,13 @@ ], "version": "==2.1.0" }, + "onedrivesdk": { + "hashes": [ + "sha256:922037d226dd374ab5a28379c45322cbc15f4d77f5117e9904599fdba05f6054" + ], + "index": "pypi", + "version": "==1.1.8" + }, "openpyxl": { "hashes": [ "sha256:8b0c2a44f394a7a913a2e7cdcc1dc601d5f45c59b85a356e591e2ac5463e21e7" @@ -593,6 +1218,13 @@ "index": "pypi", "version": "==2.3.0" }, + "python-dateutil": { + "hashes": [ + "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", + "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + ], + "version": "==2.7.5" + }, "python-social-auth": { "hashes": [ "sha256:6986220df76934aee15c54938a13ebe370a1976e043af01fa3bea417f6722e74", diff --git a/docker/Dockerfile b/docker/Dockerfile index 38b80b2d3..1e00b610e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,7 +13,7 @@ RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ fi -FROM python:3.7-alpine as base +FROM python:3.6-alpine as base COPY --from=builder /code /code RUN apk add --no-cache --virtual .build-deps \ gcc \ diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 09989622e..4b522b983 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -44,6 +44,12 @@ AZURE_CLIENT_SECRET=(str, ''), AZURE_TENANT=(str, ''), + AZURE_ACCOUNT_NAME=(str, ''), + AZURE_ACCOUNT_KEY=(str, ''), + AZURE_CONTAINER=(str, ''), + AZURE_OVERWRITE_FILES=(bool, True), + AZURE_LOCATION=(str, ''), + ) DEBUG = env.bool('DEBUG') @@ -335,6 +341,12 @@ }], } +AZURE_ACCOUNT_NAME = env('AZURE_ACCOUNT_NAME') +AZURE_ACCOUNT_KEY = env('AZURE_ACCOUNT_KEY') +AZURE_CONTAINER = env('AZURE_CONTAINER') +AZURE_OVERWRITE_FILES = env.bool('AZURE_OVERWRITE_FILES') +AZURE_LOCATION = env('AZURE_LOCATION') + CONSTANCE_CONFIG = { 'AZURE_USE_GRAPH': (True, 'Use MS Graph API to fetch user data', bool), 'DEFAULT_GROUP': ('Guests', 'Default group new users belong to', 'select_group'), diff --git a/tests/exporters/test_exporter_data.py b/tests/exporters/test_exporter_data.py new file mode 100644 index 000000000..887239739 --- /dev/null +++ b/tests/exporters/test_exporter_data.py @@ -0,0 +1,28 @@ +import io + +import pytest +from django.urls import reverse +from rest_framework.test import APIClient + +from etools_datamart.apps.data.models import UserStats +from etools_datamart.apps.etl.tasks.etl import load_user_report + + +@pytest.fixture() +def client(admin_user): + client = APIClient() + assert client.login(username='admin', password='password') + return client + + +def test_1(db, client, settings): + load_user_report.unlock() + load_user_report() + assert UserStats.objects.count() + + url = reverse("api:userstats-list") + res = client.get(f"{url}?format=xlsx") + + from storages.backends.azure_storage import AzureStorage + storage = AzureStorage() + storage.save('test1.xlsx', io.BytesIO(res.content)) From eebb3d3c049f2ca8eff286bec6184d9a2625f2db Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 23 Nov 2018 23:02:06 +0100 Subject: [PATCH 09/43] updates EtlTask --- CHANGES | 4 +- src/etools_datamart/apps/etl/admin.py | 4 +- .../etl/migrations/0003_auto_20181123_1903.py | 28 ++++++++++++++ src/etools_datamart/apps/etl/models.py | 10 ++--- src/etools_datamart/apps/etl/tasks/etl.py | 37 +++++++++++++------ src/etools_datamart/celery.py | 7 +++- tests/api/test_api_data.py | 4 +- tests/api/test_api_web.py | 2 +- tests/api/test_datamart_security.py | 3 +- tests/etl/test_etl_loaders.py | 2 + tests/etl/test_etl_tasklog.py | 6 +-- tests/test_views.py | 8 +++- 12 files changed, 85 insertions(+), 30 deletions(-) create mode 100644 src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py diff --git a/CHANGES b/CHANGES index 5065c4027..4fbc8c7f6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,8 +1,10 @@ 1.7 (dev) --------- +* WARNINGS: migration reset * add YAML format * new 'Monitor' page to check dataset last update date -* +* add ability to intercept changed/unchanged datamart records +* Azure storage support 1.6 diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index 7be70cd58..b40f54ae4 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -18,10 +18,10 @@ @register(models.EtlTask) class EtlTaskAdmin(TruncateTableMixin, admin.ModelAdmin): - list_display = ('task', 'timestamp', 'result', 'time', + list_display = ('task', 'timestamp', 'status', 'time', 'last_success', 'last_failure', 'lock', 'scheduling', 'queue_task') - readonly_fields = ('task', 'timestamp', 'result', 'elapsed', 'time', + readonly_fields = ('task', 'timestamp', 'results', 'elapsed', 'time', 'status', 'last_success', 'last_failure', 'table_name', 'content_type') date_hierarchy = 'timestamp' actions = None diff --git a/src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py b/src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py new file mode 100644 index 000000000..fc9613abc --- /dev/null +++ b/src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py @@ -0,0 +1,28 @@ +# Generated by Django 2.1.3 on 2018-11-23 19:03 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0002_auto_20181119_2028'), + ] + + operations = [ + migrations.AlterModelOptions( + name='etltask', + options={'get_latest_by': 'timestamp'}, + ), + migrations.RenameField( + model_name='etltask', + old_name='result', + new_name='status', + ), + migrations.AddField( + model_name='etltask', + name='results', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + ] diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index 87e820339..adafac96c 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from django.contrib.contenttypes.models import ContentType +from django.contrib.postgres.fields import JSONField from django.db import models from django.db.models import Model from django.utils.functional import cached_property @@ -37,28 +38,27 @@ def inspect(self): class EtlTask(models.Model): task = models.CharField(max_length=200, unique=True) timestamp = models.DateTimeField(null=True) - result = models.CharField(max_length=200) + status = models.CharField(max_length=200) elapsed = models.IntegerField(null=True) last_success = models.DateTimeField(null=True) last_failure = models.DateTimeField(null=True) table_name = models.CharField(max_length=200, null=True) content_type = models.ForeignKey(ContentType, models.CASCADE, null=True) + results = JSONField(blank=True, null=True) + objects = TaskLogManager() class Meta: get_latest_by = 'timestamp' def __str__(self): - return f"{self.task} {self.result}" + return f"{self.task} {self.status}" @cached_property def verbose_name(self): return self.content_type.model_class()._meta.verbose_name - def model(self): - return "" - @cached_property def periodic_task(self): try: diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index cfa440876..c9638fa4a 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -23,14 +23,21 @@ "load_pmp_indicator", "load_intervention"] +def equal(record, values): + other = type(record)(**values) + for field_name, field_value in values.items(): + if getattr(record, field_name) != getattr(other, field_name): + return False + return True + + @app.etl(HACT) def load_hact(): connection = connections['etools'] countries = connection.get_tenants() today = timezone.now() - created = {} + results = {'created': 0, 'updated': 0, 'unchanged': 0} for country in countries: - created[country.name] = 0 connection.set_schemas([country.schema_name]) logger.info(u'Running on %s' % country.name) @@ -67,12 +74,23 @@ def load_hact(): completed_hact_audits=data['assurance_activities']['scheduled_audit'], completed_special_audits=data['assurance_activities']['special_audit'], ) - HACT.objects.update_or_create(year=today.year, - country_name=country.name, - schema_name=country.schema_name, - defaults=values) - - return created + existing, created = HACT.objects.get_or_create(year=today.year, + country_name=country.name, + schema_name=country.schema_name, + defaults=values) + if created: + results['created'] += 1 + else: + if equal(existing, values): + results['unchanged'] += 1 + else: + results['updated'] += 1 + HACT.objects.update_or_create(year=today.year, + country_name=country.name, + schema_name=country.schema_name, + defaults=values) + + return results @app.etl(PMPIndicators) @@ -89,9 +107,6 @@ def load_pmp_indicator(): logger.info(u'Running on %s' % country.name) for partner in PartnersPartnerorganization.objects.all(): - # .prefetch_related( - # 'partnerspartnerorganization_partners_corevaluesassessment_partner_id'): - # .select_related('partnersintervention_partners_interventionbudget_intervention_id') for intervention in PartnersIntervention.objects.filter(agreement__partner=partner): planned_budget = getattr(intervention, 'partnersintervention_partners_interventionbudget_intervention_id', None) diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 46930d1c1..66e8186df 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -72,7 +72,7 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): from etools_datamart.apps.etl.models import EtlTask from django.utils import timezone - defs = {'result': 'RUNNING', + defs = {'status': 'RUNNING', 'timestamp': timezone.now()} EtlTask.objects.update_or_create(task=task.name, content_type=ContentType.objects.get_for_model(task.linked_model), @@ -90,8 +90,11 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st cost = time() - app.timers.pop(task_id) except KeyError: # pragma: no cover cost = -1 + if not isinstance(retval, dict): + retval = {'error': str(retval)} defs = {'elapsed': cost, - 'result': state, + 'status': state, + 'results': retval, 'timestamp': timezone.now()} if state == 'SUCCESS': defs['last_success'] = timezone.now() diff --git a/tests/api/test_api_data.py b/tests/api/test_api_data.py index 5c452dd1a..355b9b3db 100644 --- a/tests/api/test_api_data.py +++ b/tests/api/test_api_data.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import pytest -from tests._test_lib.test_utilities.factories import (FAMIndicatoFactory, HACTFactory, InterventionFactory, - PMPIndicatorFactory, UserStatsFactory,) +from test_utilities.factories import (FAMIndicatoFactory, HACTFactory, InterventionFactory, + PMPIndicatorFactory, UserStatsFactory,) from etools_datamart.api.endpoints import (FAMIndicatorViewSet, InterventionViewSet, PMPIndicatorsViewSet, UserStatsViewSet,) diff --git a/tests/api/test_api_web.py b/tests/api/test_api_web.py index dc77b459f..5a0a5568a 100644 --- a/tests/api/test_api_web.py +++ b/tests/api/test_api_web.py @@ -1,7 +1,7 @@ # test BrowsableAPI import pytest from rest_framework.test import APIClient -from tests._test_lib.test_utilities.factories import UserFactory +from test_utilities.factories import UserFactory from unicef_rest_framework.test_utils import user_allow_service from unicef_security.models import User diff --git a/tests/api/test_datamart_security.py b/tests/api/test_datamart_security.py index 5ff28e1d0..db5fb186c 100644 --- a/tests/api/test_datamart_security.py +++ b/tests/api/test_datamart_security.py @@ -2,8 +2,7 @@ import pytest from rest_framework.test import APIClient -from test_utilities.factories import UserFactory -from tests._test_lib.test_utilities.factories import UserStatsFactory +from test_utilities.factories import UserFactory, UserStatsFactory from unicef_rest_framework.test_utils import user_allow_service from etools_datamart.api.endpoints import PartnerViewSet, UserStatsViewSet diff --git a/tests/etl/test_etl_loaders.py b/tests/etl/test_etl_loaders.py index 82bfd48bc..29b18b771 100644 --- a/tests/etl/test_etl_loaders.py +++ b/tests/etl/test_etl_loaders.py @@ -41,3 +41,5 @@ def test_load_hact(db, settings, monkeypatch): assert bolivia.completed_spotcheck == 0 assert bolivia.completed_hact_audits == 0 assert bolivia.completed_special_audits == 0 + res = load_hact() + assert res == {'created': 0, 'updated': 0, 'unchanged': 3} diff --git a/tests/etl/test_etl_tasklog.py b/tests/etl/test_etl_tasklog.py index 1ddbc8986..1db437319 100644 --- a/tests/etl/test_etl_tasklog.py +++ b/tests/etl/test_etl_tasklog.py @@ -25,14 +25,14 @@ def test_load_pmp_indicator(db): with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run'): assert load_pmp_indicator.apply() assert EtlTask.objects.filter(task='etools_datamart.apps.etl.tasks.etl.load_pmp_indicator', - result='SUCCESS').exists() + status='SUCCESS').exists() def test_load_pmp_indicator_fail(db): with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run', side_effect=Exception): assert load_pmp_indicator.apply() assert EtlTask.objects.filter(task='etools_datamart.apps.etl.tasks.etl.load_pmp_indicator', - result='FAILURE') + status='FAILURE') @pytest.fixture() @@ -46,4 +46,4 @@ def test_load_pmp_indicator_running(db, disable_post_run): with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run'): assert load_pmp_indicator.apply() assert EtlTask.objects.filter(task='etools_datamart.apps.etl.tasks.etl.load_pmp_indicator', - result='RUNNING') + status='RUNNING') diff --git a/tests/test_views.py b/tests/test_views.py index f93a21b5e..cc69ef4be 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,6 +1,12 @@ # -*- coding: utf-8 -*- +from django.urls import reverse def test_home(django_app, admin_user): - res = django_app.get('/') + res = django_app.get(reverse('home')) + assert res.status_code == 200 + + +def test_monitor(django_app, admin_user): + res = django_app.get(reverse('monitor')) assert res.status_code == 200 From e887796f8fb72e74158d0a9ecc10b8a927e3685c Mon Sep 17 00:00:00 2001 From: sax Date: Sat, 24 Nov 2018 11:42:11 +0100 Subject: [PATCH 10/43] register updated/created/unchanged records durong ETL --- Pipfile | 2 + Pipfile.lock | 23 +- src/etools_datamart/apps/data/models/base.py | 5 +- src/etools_datamart/apps/etl/tasks/etl.py | 362 +++++++++++-------- tests/etl/test_etl_loaders.py | 32 +- tests/exporters/test_exporter_data.py | 5 +- tests/test_views.py | 3 + 7 files changed, 276 insertions(+), 156 deletions(-) diff --git a/Pipfile b/Pipfile index 93e47d253..31941dccb 100644 --- a/Pipfile +++ b/Pipfile @@ -69,6 +69,8 @@ pytest-pythonpath = "*" yapf = "*" vcrpy = "*" pre-commit = "*" +freezegun = "*" +pytest-ignore-flaky = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 9fd3a545f..3f2990600 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "dcfe11f0359c368a78b964dc582a591500ee26e438ac5c3ce0a52a0730cde154" + "sha256": "7d626fa15c7a7f09cdf0a17442e74c6f763c9f72000c26cd055bd7381d8def21" }, "pipfile-spec": 6, "requires": { @@ -1616,6 +1616,14 @@ "index": "pypi", "version": "==3.6.0" }, + "freezegun": { + "hashes": [ + "sha256:6cb82b276f83f2acce67f121dc2656f4df26c71e32238334eb071170b892a278", + "sha256:e839b43bfbe8158b4d62bb97e6313d39f3586daf48e1314fb1083d2ef17700da" + ], + "index": "pypi", + "version": "==0.3.11" + }, "identify": { "hashes": [ "sha256:5e956558a9a1e3b3891d7c6609fc9709657a11878af288ace484d1a46a93922b", @@ -1820,11 +1828,11 @@ }, "pytest": { "hashes": [ - "sha256:488c842647bbeb350029da10325cb40af0a9c7a2fdda45aeb1dda75b60048ffb", - "sha256:c055690dfefa744992f563e8c3a654089a6aa5b8092dded9b6fafbd70b2e45a7" + "sha256:1d131cc532be0023ef8ae265e2a779938d0619bb6c2510f52987ffcba7fa1ee4", + "sha256:ca4761407f1acc85ffd1609f464ca20bb71a767803505bd4127d0e45c5a50e23" ], "index": "pypi", - "version": "==4.0.0" + "version": "==4.0.1" }, "pytest-cov": { "hashes": [ @@ -1863,6 +1871,13 @@ "index": "pypi", "version": "==1.6.0" }, + "pytest-ignore-flaky": { + "hashes": [ + "sha256:78f8ddf9b405c09ce852a4ceac58258e4b8d98d2bc236a03d4a075cdc51c56c6" + ], + "index": "pypi", + "version": "==0.1.1" + }, "pytest-pythonpath": { "hashes": [ "sha256:63fc546ace7d2c845c1ee289e8f7a6362c2b6bae497d10c716e58e253e801d62" diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index fd40300dc..b89f83bf6 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -14,8 +14,9 @@ def filter_schemas(self, *schemas): class DataMartManager(BaseManager.from_queryset(DataMartQuerySet)): - def truncate(self): - self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) + pass + # def truncate(self): + # self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) class DataMartModel(models.Model, TimeStampedModel): diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index c9638fa4a..abc826f27 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -23,12 +23,37 @@ "load_pmp_indicator", "load_intervention"] -def equal(record, values): +class EtlResult: + def __init__(self, updated=0, created=0, unchanged=0): + self.created = created + self.updated = updated + self.unchanged = unchanged + + def __repr__(self): + return repr(self.as_dict()) + + def as_dict(self): + return {'created': self.created, + 'updated': self.updated, + 'unchanged': self.unchanged} + + def __eq__(self, other): + if isinstance(other, EtlResult): + return (self.created == other.created and + self.updated == other.updated and + self.unchanged == other.unchanged) + elif isinstance(other, dict): + return (self.created == other['created'] and + self.updated == other['updated'] and + self.unchanged == other['unchanged']) + + +def is_record_changed(record, values): other = type(record)(**values) for field_name, field_value in values.items(): if getattr(record, field_name) != getattr(other, field_name): - return False - return True + return True + return False @app.etl(HACT) @@ -36,7 +61,7 @@ def load_hact(): connection = connections['etools'] countries = connection.get_tenants() today = timezone.now() - results = {'created': 0, 'updated': 0, 'unchanged': 0} + results = EtlResult() for country in countries: connection.set_schemas([country.schema_name]) @@ -79,30 +104,28 @@ def load_hact(): schema_name=country.schema_name, defaults=values) if created: - results['created'] += 1 + results.created += 1 else: - if equal(existing, values): - results['unchanged'] += 1 - else: - results['updated'] += 1 + if is_record_changed(existing, values): + results.updated += 1 HACT.objects.update_or_create(year=today.year, country_name=country.name, schema_name=country.schema_name, defaults=values) + else: + results.unchanged += 1 - return results + return results.as_dict() @app.etl(PMPIndicators) def load_pmp_indicator(): connection = connections['etools'] countries = connection.get_tenants() - PMPIndicators.objects.truncate() base_url = 'https://etools.unicef.org' - created = {} + results = EtlResult() for country in countries: - created[country.name] = 0 connection.set_schemas([country.schema_name]) logger.info(u'Running on %s' % country.name) @@ -112,52 +135,73 @@ def load_pmp_indicator(): 'partnersintervention_partners_interventionbudget_intervention_id', None) fr_currencies = intervention.frs.all().values_list('currency', flat=True).distinct() has_assessment = bool(getattr(partner.current_core_value_assessment, 'assessment', False)) - PMPIndicators.objects.create( - country_id=country.pk, - partner_id=partner.pk, - intervention_id=intervention.pk, - **{ - 'country_name': country.name, - 'schema_name': country.schema_name, - 'business_area_code': country.business_area_code, - 'partner_name': partner.name, - 'partner_type': partner.cso_type, - 'vendor_number': partner.vendor_number, - - 'pd_ssfa_ref': intervention.number.replace(',', '-'), - 'pd_ssfa_status': intervention.status.title(), - 'pd_ssfa_start_date': intervention.start, - 'pd_ssfa_creation_date': intervention.created, - 'pd_ssfa_end_date': intervention.end, - - 'cash_contribution': intervention.total_unicef_cash or 0, - 'supply_contribution': intervention.total_in_kind_amount or 0, - 'total_budget': intervention.total_budget or 0, - 'unicef_budget': intervention.total_unicef_budget or 0, - - 'currency': intervention.planned_budget.currency if planned_budget else '-', - 'partner_contribution': intervention.planned_budget.partner_contribution if planned_budget else '-', - 'unicef_cash': intervention.planned_budget.unicef_cash if planned_budget else '-', - 'in_kind_amount': intervention.planned_budget.in_kind_amount if planned_budget else '-', - 'total': intervention.planned_budget.total if planned_budget else '-', - 'fr_numbers_against_pd_ssfa': ' - '.join([fh.fr_number for fh in intervention.frs.all()]), - 'fr_currencies': ', '.join(fr for fr in fr_currencies), - 'sum_of_all_fr_planned_amount': intervention.frs.aggregate( - total=Coalesce(Sum('intervention_amt'), 0))['total'] if fr_currencies.count() <= 1 else '-', - 'core_value_attached': has_assessment, - 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), - 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), - }) - created[country.name] += 1 - return created + values = {'country_name': country.name, + 'schema_name': country.schema_name, + 'business_area_code': country.business_area_code, + 'partner_name': partner.name, + 'partner_type': partner.cso_type, + 'vendor_number': partner.vendor_number, + + 'pd_ssfa_ref': intervention.number.replace(',', '-'), + 'pd_ssfa_status': intervention.status.title(), + 'pd_ssfa_start_date': intervention.start, + 'pd_ssfa_creation_date': intervention.created, + 'pd_ssfa_end_date': intervention.end, + + 'cash_contribution': intervention.total_unicef_cash or 0, + 'supply_contribution': intervention.total_in_kind_amount or 0, + 'total_budget': intervention.total_budget or 0, + 'unicef_budget': intervention.total_unicef_budget or 0, + + 'currency': intervention.planned_budget.currency if planned_budget else '-', + 'partner_contribution': intervention.planned_budget.partner_contribution if planned_budget else '-', + 'unicef_cash': intervention.planned_budget.unicef_cash if planned_budget else '-', + 'in_kind_amount': intervention.planned_budget.in_kind_amount if planned_budget else '-', + 'total': intervention.planned_budget.total if planned_budget else '-', + 'fr_numbers_against_pd_ssfa': ' - '.join([fh.fr_number for fh in intervention.frs.all()]), + 'fr_currencies': ', '.join(fr for fr in fr_currencies), + 'sum_of_all_fr_planned_amount': intervention.frs.aggregate( + total=Coalesce(Sum('intervention_amt'), 0))[ + 'total'] if fr_currencies.count() <= 1 else '-', + 'core_value_attached': has_assessment, + 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), + 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), + } + existing, created = PMPIndicators.objects.get_or_create(country_name=country.name, + schema_name=country.schema_name, + country_id=partner.id, + partner_id=partner.pk, + intervention_id=intervention.pk, + defaults=values) + if created: + results.created += 1 + else: + if is_record_changed(existing, values): + results.updated += 1 + PMPIndicators.objects.update_or_create(country_name=country.name, + schema_name=country.schema_name, + country_id=partner.id, + partner_id=partner.pk, + intervention_id=intervention.pk, + defaults=values) + else: + results.unchanged += 1 + + return results.as_dict() + # PMPIndicators.objects.create( + # country_id=country.pk, + # partner_id=partner.pk, + # intervention_id=intervention.pk) + # created[country.name] += 1 + # + # return created @app.etl(Intervention) def load_intervention(): connection = connections['etools'] countries = connection.get_tenants() - Intervention.objects.truncate() - created = {} + results = EtlResult() for country in countries: connection.set_schemas([country.schema_name]) qs = PartnersIntervention.objects.all().select_related('agreement', @@ -167,68 +211,79 @@ def load_intervention(): ) num = 0 for num, record in enumerate(qs, 1): - Intervention.objects.create(country_name=country.name, - schema_name=country.schema_name, - number=record.number, - title=record.title, - status=record.status, - start_date=record.start, - end_date=record.end, - review_date_prc=record.review_date_prc, - prc_review_document=record.prc_review_document, - partner_name=record.agreement.partner.name, - agreement_id=record.agreement.pk, - partner_authorized_officer_signatory_id=get_attr(record, - 'partner_authorized_officer_signatory.pk'), - country_programme_id=get_attr(record, 'country_programme.pk'), - intervention_id=record.pk, - unicef_signatory_id=get_attr(record, 'unicef_signatory.pk'), - - signed_by_unicef_date=record.signed_by_unicef_date, - signed_by_partner_date=record.signed_by_partner_date, - population_focus=record.population_focus, - signed_pd_document=record.signed_pd_document, - - submission_date=record.submission_date, - submission_date_prc=record.submission_date_prc, - - unicef_signatory_first_name=get_attr(record, - 'unicef_signatory.first_name'), - unicef_signatory_last_name=get_attr(record, - 'unicef_signatory.last_name'), - unicef_signatory_email=get_attr(record, 'unicef_signatory.email'), - - partner_signatory_title=get_attr(record, - 'partner_authorized_officer_signatory.title'), - partner_signatory_first_name=get_attr(record, - 'partner_authorized_officer_signatory.first_name'), - partner_signatory_last_name=get_attr(record, - 'partner_authorized_officer_signatory.last_name'), - partner_signatory_email=get_attr(record, - 'partner_authorized_officer_signatory.email'), - partner_signatory_phone=get_attr(record, - 'partner_authorized_officer_signatory.phone'), - - partner_focal_point_title=get_attr(record, - 'partner_focal_point.title'), - partner_focal_point_first_name=get_attr(record, - 'partner_focal_point.first_name'), - partner_focal_point_last_name=get_attr(record, - 'partner_focal_point.last_name'), - partner_focal_point_email=get_attr(record, - 'partner_focal_point.email'), - partner_focal_point_phone=get_attr(record, - 'partner_focal_point.phone'), - - metadata=record.metadata, - document_type=record.document_type, - updated=record.modified, - created=record.created, - - ) - created[country.name] = num - - return created + values = dict(number=record.number, + title=record.title, + status=record.status, + start_date=record.start, + end_date=record.end, + review_date_prc=record.review_date_prc, + prc_review_document=record.prc_review_document, + partner_name=record.agreement.partner.name, + agreement_id=record.agreement.pk, + partner_authorized_officer_signatory_id=get_attr(record, + 'partner_authorized_officer_signatory.pk'), + country_programme_id=get_attr(record, 'country_programme.pk'), + intervention_id=record.pk, + unicef_signatory_id=get_attr(record, 'unicef_signatory.pk'), + + signed_by_unicef_date=record.signed_by_unicef_date, + signed_by_partner_date=record.signed_by_partner_date, + population_focus=record.population_focus, + signed_pd_document=record.signed_pd_document, + + submission_date=record.submission_date, + submission_date_prc=record.submission_date_prc, + + unicef_signatory_first_name=get_attr(record, + 'unicef_signatory.first_name'), + unicef_signatory_last_name=get_attr(record, + 'unicef_signatory.last_name'), + unicef_signatory_email=get_attr(record, 'unicef_signatory.email'), + + partner_signatory_title=get_attr(record, + 'partner_authorized_officer_signatory.title'), + partner_signatory_first_name=get_attr(record, + 'partner_authorized_officer_signatory.first_name'), + partner_signatory_last_name=get_attr(record, + 'partner_authorized_officer_signatory.last_name'), + partner_signatory_email=get_attr(record, + 'partner_authorized_officer_signatory.email'), + partner_signatory_phone=get_attr(record, + 'partner_authorized_officer_signatory.phone'), + + partner_focal_point_title=get_attr(record, + 'partner_focal_point.title'), + partner_focal_point_first_name=get_attr(record, + 'partner_focal_point.first_name'), + partner_focal_point_last_name=get_attr(record, + 'partner_focal_point.last_name'), + partner_focal_point_email=get_attr(record, + 'partner_focal_point.email'), + partner_focal_point_phone=get_attr(record, + 'partner_focal_point.phone'), + + metadata=record.metadata, + document_type=record.document_type, + updated=record.modified, + created=record.created, + ) + existing, created = Intervention.objects.get_or_create(country_name=country.name, + schema_name=country.schema_name, + intervention_id=record.pk, + defaults=values) + if created: + results.created += 1 + else: + if is_record_changed(existing, values): + results.updated += 1 + Intervention.objects.update_or_create(country_name=country.name, + schema_name=country.schema_name, + intervention_id=record.pk, + defaults=values) + else: + results.unchanged += 1 + + return results.as_dict() @app.etl(FAMIndicator) @@ -238,16 +293,16 @@ def load_fam_indicator(): engagements = (AuditSpotcheck, AuditAudit, AuditSpecialaudit, AuditMicroassessment) start_date = date.today() # + relativedelta(months=-1) - created = {} + results = EtlResult() for country in countries: - created[country.name] = 0 - connection.set_schemas([country.schema_name]) for model in engagements: - indicator, __ = FAMIndicator.objects.get_or_create(month=start_date, - country_name=country.name, - schema_name=country.schema_name) - + indicator, created = FAMIndicator.objects.get_or_create(month=start_date, + country_name=country.name, + schema_name=country.schema_name) + if created: + results.created += 1 + changed = created realname = "_".join(model._meta.db_table.split('_')[1:]) for status, status_display in AuditEngagement.STATUSES: filter_dict = { @@ -261,14 +316,18 @@ def load_fam_indicator(): # just a safety check if not hasattr(indicator, field_name): # pragma: no cover raise ValueError(field_name) - setattr(indicator, field_name, value) + if getattr(indicator, field_name) == value: + changed = False + else: + changed = changed and True + setattr(indicator, field_name, value) except Exception as e: # pragma: no cover logger.error(e) raise - indicator.save() - created[country.name] += 1 + if changed: + indicator.save() - return created + return results.as_dict() @app.etl(UserStats) @@ -277,23 +336,40 @@ def load_user_report(): countries = connection.get_tenants() today = date.today() first_of_month = datetime(today.year, today.month, 1) - created = {} + results = EtlResult() for country in countries: - created[country.name] = 0 connection.set_schemas([country.schema_name]) base = AuthUser.objects.filter(profile__country=country) - UserStats.objects.update_or_create(month=first_of_month, - country_name=country.name, - schema_name=country.schema_name, - defaults={ - 'total': base.count(), - 'unicef': base.filter(email__endswith='@unicef.org').count(), - 'logins': base.filter( - last_login__month=first_of_month.month).count(), - 'unicef_logins': base.filter( - last_login__month=first_of_month.month, - email__endswith='@unicef.org').count(), - }) - created[country.name] += 1 - - return created + values = { + 'total': base.count(), + 'unicef': base.filter(email__endswith='@unicef.org').count(), + 'logins': base.filter( + last_login__month=first_of_month.month).count(), + 'unicef_logins': base.filter( + last_login__month=first_of_month.month, + email__endswith='@unicef.org').count(), + } + existing, created = UserStats.objects.get_or_create(month=first_of_month, + country_name=country.name, + schema_name=country.schema_name, + defaults=values) + if created: + results.created += 1 + else: + if is_record_changed(existing, values): + results.updated += 1 + UserStats.objects.update_or_create(month=first_of_month, + country_name=country.name, + schema_name=country.schema_name, + defaults=values) + else: + results.unchanged += 1 + + return results.as_dict() + # UserStats.objects.update_or_create(month=first_of_month, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # created[country.name] += 1 + # + # return created diff --git a/tests/etl/test_etl_loaders.py b/tests/etl/test_etl_loaders.py index 29b18b771..78b11933b 100644 --- a/tests/etl/test_etl_loaders.py +++ b/tests/etl/test_etl_loaders.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- +from freezegun import freeze_time + from etools_datamart.apps.data.models import FAMIndicator, HACT, Intervention, PMPIndicators, UserStats -from etools_datamart.apps.etl.tasks.etl import (load_fam_indicator, load_hact, load_intervention, - load_pmp_indicator, load_user_report,) +from etools_datamart.apps.etl.tasks.etl import (EtlResult, load_fam_indicator, load_hact, + load_intervention, load_pmp_indicator, load_user_report,) def test_load_pmp_indicator(number_of_intervention): load_pmp_indicator.unlock() - assert load_pmp_indicator() == {'Bolivia': number_of_intervention, - 'Chad': number_of_intervention, - 'Lebanon': number_of_intervention} + assert load_pmp_indicator() == EtlResult(created=153) assert PMPIndicators.objects.count() == number_of_intervention * 3 @@ -42,4 +42,24 @@ def test_load_hact(db, settings, monkeypatch): assert bolivia.completed_hact_audits == 0 assert bolivia.completed_special_audits == 0 res = load_hact() - assert res == {'created': 0, 'updated': 0, 'unchanged': 3} + assert res == EtlResult(unchanged=3) + + +@freeze_time("2018-11-10") +def test_dataset_increased(db, settings, monkeypatch): + load_user_report.unlock() + load_user_report() + UserStats.objects.first().delete() + ret = load_user_report() + assert ret == EtlResult(created=1, unchanged=2) + + +@freeze_time("2018-11-10") +def test_dataset_changed(db, settings, monkeypatch): + load_user_report.unlock() + ret = load_user_report() + assert ret == EtlResult(created=3) + UserStats.objects.update(total=999, unicef=999) + + ret = load_user_report() + assert ret == EtlResult(updated=3) diff --git a/tests/exporters/test_exporter_data.py b/tests/exporters/test_exporter_data.py index 887239739..36d112628 100644 --- a/tests/exporters/test_exporter_data.py +++ b/tests/exporters/test_exporter_data.py @@ -1,4 +1,5 @@ import io +import os import pytest from django.urls import reverse @@ -15,7 +16,9 @@ def client(admin_user): return client -def test_1(db, client, settings): +@pytest.mark.skipif("CIRCLECI" in os.environ, + reason="Skip in CirlceCI") +def test_export_azure_data(db, client, settings): load_user_report.unlock() load_user_report() assert UserStats.objects.count() diff --git a/tests/test_views.py b/tests/test_views.py index cc69ef4be..970e4507e 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- from django.urls import reverse +from etools_datamart.apps.etl.models import EtlTask + def test_home(django_app, admin_user): res = django_app.get(reverse('home')) @@ -8,5 +10,6 @@ def test_home(django_app, admin_user): def test_monitor(django_app, admin_user): + EtlTask.objects.inspect() res = django_app.get(reverse('monitor')) assert res.status_code == 200 From 07aba94cf3fd7c438d040e64834bf86b225e056d Mon Sep 17 00:00:00 2001 From: sax Date: Sat, 24 Nov 2018 18:50:16 +0100 Subject: [PATCH 11/43] minor ETL refactoring --- src/etools_datamart/apps/etl/tasks/etl.py | 238 +++++++++++++--------- tests/etl/test_etl_loaders.py | 2 +- tests/etl/test_etl_result.py | 21 ++ 3 files changed, 168 insertions(+), 93 deletions(-) create mode 100644 tests/etl/test_etl_result.py diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index abc826f27..08a90143b 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -22,8 +22,14 @@ __all__ = ["load_hact", "load_user_report", "load_fam_indicator", "load_pmp_indicator", "load_intervention"] +CREATED = 'created' +UPDATED = 'updated' +UNCHANGED = 'unchanged' + class EtlResult: + __slots__ = [CREATED, UPDATED, UNCHANGED] + def __init__(self, updated=0, created=0, unchanged=0): self.created = created self.updated = updated @@ -32,6 +38,9 @@ def __init__(self, updated=0, created=0, unchanged=0): def __repr__(self): return repr(self.as_dict()) + def incr(self, counter): + setattr(self, counter, getattr(self, counter) + 1) + def as_dict(self): return {'created': self.created, 'updated': self.updated, @@ -39,13 +48,13 @@ def as_dict(self): def __eq__(self, other): if isinstance(other, EtlResult): - return (self.created == other.created and - self.updated == other.updated and - self.unchanged == other.unchanged) - elif isinstance(other, dict): + other = other.as_dict() + + if isinstance(other, dict): return (self.created == other['created'] and self.updated == other['updated'] and self.unchanged == other['unchanged']) + return False def is_record_changed(record, values): @@ -56,6 +65,21 @@ def is_record_changed(record, values): return False +def process(Model, filters, values): + existing, created = Model.objects.get_or_create(**filters, + defaults=values) + if created: + op = CREATED + else: + if is_record_changed(existing, values): + op = UPDATED + Model.objects.update_or_create(**filters, + defaults=values) + else: + op = UNCHANGED + return op + + @app.etl(HACT) def load_hact(): connection = connections['etools'] @@ -99,21 +123,26 @@ def load_hact(): completed_hact_audits=data['assurance_activities']['scheduled_audit'], completed_special_audits=data['assurance_activities']['special_audit'], ) - existing, created = HACT.objects.get_or_create(year=today.year, - country_name=country.name, - schema_name=country.schema_name, - defaults=values) - if created: - results.created += 1 - else: - if is_record_changed(existing, values): - results.updated += 1 - HACT.objects.update_or_create(year=today.year, - country_name=country.name, - schema_name=country.schema_name, - defaults=values) - else: - results.unchanged += 1 + op = process(HACT, filters=dict(year=today.year, + country_name=country.name, + schema_name=country.schema_name), + values=values) + results.incr(op) + # existing, created = HACT.objects.get_or_create(year=today.year, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # HACT.objects.update_or_create(year=today.year, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # else: + # results.unchanged += 1 return results.as_dict() @@ -167,25 +196,32 @@ def load_pmp_indicator(): 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), } - existing, created = PMPIndicators.objects.get_or_create(country_name=country.name, - schema_name=country.schema_name, - country_id=partner.id, - partner_id=partner.pk, - intervention_id=intervention.pk, - defaults=values) - if created: - results.created += 1 - else: - if is_record_changed(existing, values): - results.updated += 1 - PMPIndicators.objects.update_or_create(country_name=country.name, - schema_name=country.schema_name, - country_id=partner.id, - partner_id=partner.pk, - intervention_id=intervention.pk, - defaults=values) - else: - results.unchanged += 1 + op = process(PMPIndicators, filters=dict(country_name=country.name, + schema_name=country.schema_name, + country_id=partner.id, + partner_id=partner.pk, + intervention_id=intervention.pk), + values=values) + results.incr(op) + # existing, created = PMPIndicators.objects.get_or_create(country_name=country.name, + # schema_name=country.schema_name, + # country_id=partner.id, + # partner_id=partner.pk, + # intervention_id=intervention.pk, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # PMPIndicators.objects.update_or_create(country_name=country.name, + # schema_name=country.schema_name, + # country_id=partner.id, + # partner_id=partner.pk, + # intervention_id=intervention.pk, + # defaults=values) + # else: + # results.unchanged += 1 return results.as_dict() # PMPIndicators.objects.create( @@ -267,21 +303,27 @@ def load_intervention(): updated=record.modified, created=record.created, ) - existing, created = Intervention.objects.get_or_create(country_name=country.name, - schema_name=country.schema_name, - intervention_id=record.pk, - defaults=values) - if created: - results.created += 1 - else: - if is_record_changed(existing, values): - results.updated += 1 - Intervention.objects.update_or_create(country_name=country.name, - schema_name=country.schema_name, - intervention_id=record.pk, - defaults=values) - else: - results.unchanged += 1 + op = process(Intervention, filters=dict(country_name=country.name, + schema_name=country.schema_name, + intervention_id=record.pk), + values=values) + results.incr(op) + + # existing, created = Intervention.objects.get_or_create(country_name=country.name, + # schema_name=country.schema_name, + # intervention_id=record.pk, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # Intervention.objects.update_or_create(country_name=country.name, + # schema_name=country.schema_name, + # intervention_id=record.pk, + # defaults=values) + # else: + # results.unchanged += 1 return results.as_dict() @@ -297,36 +339,42 @@ def load_fam_indicator(): for country in countries: connection.set_schemas([country.schema_name]) for model in engagements: - indicator, created = FAMIndicator.objects.get_or_create(month=start_date, - country_name=country.name, - schema_name=country.schema_name) - if created: - results.created += 1 - changed = created + # indicator, created = FAMIndicator.objects.get_or_create(month=start_date, + # country_name=country.name, + # schema_name=country.schema_name) + # if created: + # results.created += 1 + # changed = created realname = "_".join(model._meta.db_table.split('_')[1:]) + values = {} for status, status_display in AuditEngagement.STATUSES: filter_dict = { 'engagement_ptr__status': status, 'engagement_ptr__start_date__month': start_date.month, 'engagement_ptr__start_date__year': start_date.year, } - try: - field_name = f"{realname}_{status_display}".replace(" ", "_").lower() - value = model.objects.filter(**filter_dict).count() - # just a safety check - if not hasattr(indicator, field_name): # pragma: no cover - raise ValueError(field_name) - if getattr(indicator, field_name) == value: - changed = False - else: - changed = changed and True - setattr(indicator, field_name, value) - except Exception as e: # pragma: no cover - logger.error(e) - raise - if changed: - indicator.save() - + field_name = f"{realname}_{status_display}".replace(" ", "_").lower() + value = model.objects.filter(**filter_dict).count() + values[field_name] = value + # try: + # field_name = f"{realname}_{status_display}".replace(" ", "_").lower() + # value = model.objects.filter(**filter_dict).count() + # # just a safety check + # if not hasattr(indicator, field_name): # pragma: no cover + # raise ValueError(field_name) + # if getattr(indicator, field_name) == value: + # changed = False + # else: + # changed = changed and True + # setattr(indicator, field_name, value) + # except Exception as e: # pragma: no cover + # logger.error(e) + # raise + op = process(FAMIndicator, filters=dict(month=start_date, + country_name=country.name, + schema_name=country.schema_name), + values=values) + results.incr(op) return results.as_dict() @@ -349,22 +397,28 @@ def load_user_report(): last_login__month=first_of_month.month, email__endswith='@unicef.org').count(), } - existing, created = UserStats.objects.get_or_create(month=first_of_month, - country_name=country.name, - schema_name=country.schema_name, - defaults=values) - if created: - results.created += 1 - else: - if is_record_changed(existing, values): - results.updated += 1 - UserStats.objects.update_or_create(month=first_of_month, - country_name=country.name, - schema_name=country.schema_name, - defaults=values) - else: - results.unchanged += 1 - + op = process(UserStats, filters=dict(month=first_of_month, + country_name=country.name, + schema_name=country.schema_name, ), + values=values) + results.incr(op) + + # existing, created = UserStats.objects.get_or_create(month=first_of_month, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # UserStats.objects.update_or_create(month=first_of_month, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # else: + # results.unchanged += 1 + # return results.as_dict() # UserStats.objects.update_or_create(month=first_of_month, # country_name=country.name, diff --git a/tests/etl/test_etl_loaders.py b/tests/etl/test_etl_loaders.py index 78b11933b..00582c886 100644 --- a/tests/etl/test_etl_loaders.py +++ b/tests/etl/test_etl_loaders.py @@ -14,7 +14,7 @@ def test_load_pmp_indicator(number_of_intervention): def test_load_intervention(number_of_intervention, settings, monkeypatch): load_intervention.unlock() - load_intervention() + assert load_intervention() == EtlResult(created=number_of_intervention*3) assert Intervention.objects.count() == number_of_intervention * 3 diff --git a/tests/etl/test_etl_result.py b/tests/etl/test_etl_result.py new file mode 100644 index 000000000..e9ebac924 --- /dev/null +++ b/tests/etl/test_etl_result.py @@ -0,0 +1,21 @@ +from etools_datamart.apps.etl.tasks.etl import EtlResult + + +def test_result_eq(): + assert EtlResult() == EtlResult() + + +def test_result_ne(): + assert not EtlResult() == EtlResult(created=1) + + +def test_result_eq_dict(): + assert EtlResult() == {'created': 0, 'updated': 0, 'unchanged': 0} + + +def test_result_ne_dict(): + assert not EtlResult() == {'created': 1, 'updated': 1, 'unchanged': 1} + + +def test_result_ne_other(): + assert not EtlResult() == 1 From c1dcb0d549f28e5534c05ad5697172d4c2c3eaeb Mon Sep 17 00:00:00 2001 From: sax Date: Sun, 25 Nov 2018 21:34:20 +0100 Subject: [PATCH 12/43] fixes dependencies --- Pipfile | 1 + Pipfile.lock | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Pipfile b/Pipfile index 31941dccb..7edcc1229 100644 --- a/Pipfile +++ b/Pipfile @@ -49,6 +49,7 @@ redis = "==2.10.6" djangorestframework-yaml = "*" django-storages = {extras = ["azure"], version = "*"} onedrivesdk = "*" +azure-storage = "*" [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index 3f2990600..33ee8cae9 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7d626fa15c7a7f09cdf0a17442e74c6f763c9f72000c26cd055bd7381d8def21" + "sha256": "8f86868b9753fa670c7c91c6a2eecc06abeb689cd5475fe1baf332d13dbe8ebc" }, "pipfile-spec": 6, "requires": { @@ -594,6 +594,14 @@ ], "version": "==0.20.6" }, + "azure-storage": { + "hashes": [ + "sha256:4c406422e3edd41920bb1f0c3930c34fee3eb0d55258ef7ec7308ccbb9385ad5", + "sha256:fb6212dcbed91b49d9637aa5e8888eafdfcd523b7e560c8044d2d838bbd3ca5f" + ], + "index": "pypi", + "version": "==0.36.0" + }, "azure-storage-blob": { "hashes": [ "sha256:65ebe2e54460566c2077c6b3773a2a0623eabc7b95602010cb51b84077087fda", @@ -850,11 +858,11 @@ }, "django-picklefield": { "hashes": [ - "sha256:8d1de6be099044ae61e55998b35de18a57499b946fe45781077f5cec4f73f0e0", - "sha256:ce7fee5c6558fe5dc8924993d994ccde75bb75b91cd82787cbd4c92b95a69f9c" + "sha256:9052f2dcf4882c683ce87b4356f29b4d014c0dad645b6906baf9f09571f52bc8", + "sha256:f1733a8db1b6046c0d7d738e785f9875aa3c198215de11993463a9339aa4ea24" ], "index": "pypi", - "version": "==1.1.0" + "version": "==2.0" }, "django-redis": { "hashes": [ @@ -1821,10 +1829,10 @@ }, "pygments": { "hashes": [ - "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", - "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" + "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", + "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" ], - "version": "==2.2.0" + "version": "==2.3.0" }, "pytest": { "hashes": [ @@ -2021,11 +2029,11 @@ }, "yapf": { "hashes": [ - "sha256:b96815bd0bbd2ab290f2ae9e610756940b17a0523ef2f6b2d31da749fc395137", - "sha256:cebb6faf35c9027c08996c07831b8971f3d67c0eb615269f66dfd7e6815fdc2a" + "sha256:8aa7f9abdb97b4da4d3227306b88477982daafef0a96cc41639754ca31f46d55", + "sha256:f2df5891481f94ddadfbf8ae8ae499080752cfb06005a31bbb102f3012f8b944" ], "index": "pypi", - "version": "==0.24.0" + "version": "==0.25.0" }, "yarl": { "hashes": [ From 1f1bc2e9832ed21b02cf9e3634d557f3c08ca1e3 Mon Sep 17 00:00:00 2001 From: sax Date: Sun, 25 Nov 2018 21:41:20 +0100 Subject: [PATCH 13/43] pass CIRCLE_* env to tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1bb0cc45b..bbbb23a40 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ minversion = 3.5.2 [testenv] basepython = python3.6 -passenv = PYTHONDONTWRITEBYTECODE USER PYTHONPATH DATABASE_URL DATABASE_URL_ETOOLS +passenv = PYTHONDONTWRITEBYTECODE USER PYTHONPATH DATABASE_URL DATABASE_URL_ETOOLS CIRCLECI CIRCLE_* CI setenv = PYTHONDONTWRITEBYTECODE=true PYTHONPATH={toxinidir}/src From 07fd63fe225d4aa606cf970a49247e3181ea3b15 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 11:47:22 +0100 Subject: [PATCH 14/43] use custom celery decoder/encoder to better handle ETL results --- CHANGES | 2 +- src/etools_datamart/apps/etl/admin.py | 6 +- .../etl/migrations/0004_auto_20181126_0922.py | 18 ++++++ .../migrations/0005_etltask_last_changes.py | 18 ++++++ src/etools_datamart/apps/etl/models.py | 9 +-- src/etools_datamart/apps/etl/results.py | 63 +++++++++++++++++++ src/etools_datamart/apps/etl/tasks/etl.py | 53 +++------------- src/etools_datamart/celery.py | 23 +++++-- src/etools_datamart/config/settings.py | 10 ++- tests/etl/test_etl_tasklog.py | 5 +- 10 files changed, 146 insertions(+), 61 deletions(-) create mode 100644 src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py create mode 100644 src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py create mode 100644 src/etools_datamart/apps/etl/results.py diff --git a/CHANGES b/CHANGES index 4fbc8c7f6..98bdc4d32 100644 --- a/CHANGES +++ b/CHANGES @@ -5,7 +5,7 @@ * new 'Monitor' page to check dataset last update date * add ability to intercept changed/unchanged datamart records * Azure storage support - +* 1.6 --- diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index b40f54ae4..37841f2f0 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -18,12 +18,12 @@ @register(models.EtlTask) class EtlTaskAdmin(TruncateTableMixin, admin.ModelAdmin): - list_display = ('task', 'timestamp', 'status', 'time', + list_display = ('task', 'last_run', 'status', 'time', 'last_success', 'last_failure', 'lock', 'scheduling', 'queue_task') - readonly_fields = ('task', 'timestamp', 'results', 'elapsed', 'time', 'status', + readonly_fields = ('task', 'last_run', 'results', 'elapsed', 'time', 'status', 'last_success', 'last_failure', 'table_name', 'content_type') - date_hierarchy = 'timestamp' + date_hierarchy = 'last_run' actions = None def scheduling(self, obj): diff --git a/src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py b/src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py new file mode 100644 index 000000000..6fb77a828 --- /dev/null +++ b/src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.3 on 2018-11-26 09:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0003_auto_20181123_1903'), + ] + + operations = [ + migrations.RenameField( + model_name='etltask', + old_name='timestamp', + new_name='last_run', + ), + ] diff --git a/src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py b/src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py new file mode 100644 index 000000000..c9daa0931 --- /dev/null +++ b/src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.3 on 2018-11-26 09:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('etl', '0004_auto_20181126_0922'), + ] + + operations = [ + migrations.AddField( + model_name='etltask', + name='last_changes', + field=models.DateTimeField(null=True), + ), + ] diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index adafac96c..3e3286bb0 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -16,7 +16,7 @@ def get_for_model(self, model: Model): def get_for_task(self, task: ETLTask): return self.get_or_create(task=task.name, defaults=dict(content_type=ContentType.objects.get_for_model(task.linked_model), - timestamp=None, + last_run=None, table_name=task.linked_model._meta.db_table))[0] def inspect(self): @@ -27,7 +27,7 @@ def inspect(self): t, created = self.get_or_create(task=task.name, defaults=dict( content_type=ContentType.objects.get_for_model(task.linked_model), - timestamp=None, + last_run=None, table_name=task.linked_model._meta.db_table)) results[created] += 1 new.append(t.id) @@ -37,11 +37,12 @@ def inspect(self): class EtlTask(models.Model): task = models.CharField(max_length=200, unique=True) - timestamp = models.DateTimeField(null=True) + last_run = models.DateTimeField(null=True) status = models.CharField(max_length=200) elapsed = models.IntegerField(null=True) last_success = models.DateTimeField(null=True) last_failure = models.DateTimeField(null=True) + last_changes = models.DateTimeField(null=True) table_name = models.CharField(max_length=200, null=True) content_type = models.ForeignKey(ContentType, models.CASCADE, null=True) @@ -50,7 +51,7 @@ class EtlTask(models.Model): objects = TaskLogManager() class Meta: - get_latest_by = 'timestamp' + get_latest_by = 'last_run' def __str__(self): return f"{self.task} {self.status}" diff --git a/src/etools_datamart/apps/etl/results.py b/src/etools_datamart/apps/etl/results.py new file mode 100644 index 000000000..5a4c8a11f --- /dev/null +++ b/src/etools_datamart/apps/etl/results.py @@ -0,0 +1,63 @@ +import json + +CREATED = 'created' +UPDATED = 'updated' +UNCHANGED = 'unchanged' + + +class EtlResult: + __slots__ = [CREATED, UPDATED, UNCHANGED] + + def __init__(self, updated=0, created=0, unchanged=0): + self.created = created + self.updated = updated + self.unchanged = unchanged + + def __repr__(self): + return repr(self.as_dict()) + + def incr(self, counter): + setattr(self, counter, getattr(self, counter) + 1) + + def as_dict(self): + return {'created': self.created, + 'updated': self.updated, + 'unchanged': self.unchanged} + + def __eq__(self, other): + if isinstance(other, EtlResult): + other = other.as_dict() + + if isinstance(other, dict): + return (self.created == other['created'] and + self.updated == other['updated'] and + self.unchanged == other['unchanged']) + return False + + +class EtlEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, EtlResult): + return { + '__type__': '__EtlResult__', + 'data': obj.as_dict() + } + else: + return json.JSONEncoder.default(self, obj) + + +def etl_decoder(obj): + if '__type__' in obj: + if obj['__type__'] == '__EtlResult__': + return EtlResult(**obj) + return obj + + +# Encoder function +def etl_dumps(obj): + return json.dumps(obj, cls=EtlEncoder) + + +# Decoder function +def etl_loads(obj): + return json.loads(obj, object_hook=etl_decoder) diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index 08a90143b..640526f9e 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -12,6 +12,7 @@ from etools_datamart.apps.data.models import HACT, Intervention, PMPIndicators from etools_datamart.apps.data.models.fam import FAMIndicator from etools_datamart.apps.data.models.user import UserStats +from etools_datamart.apps.etl.results import CREATED, EtlResult, UNCHANGED, UPDATED from etools_datamart.apps.etools.models import (AuditAudit, AuditEngagement, AuditMicroassessment, AuditSpecialaudit, AuditSpotcheck, AuthUser, HactAggregatehact, PartnersIntervention, PartnersPartnerorganization,) @@ -22,40 +23,6 @@ __all__ = ["load_hact", "load_user_report", "load_fam_indicator", "load_pmp_indicator", "load_intervention"] -CREATED = 'created' -UPDATED = 'updated' -UNCHANGED = 'unchanged' - - -class EtlResult: - __slots__ = [CREATED, UPDATED, UNCHANGED] - - def __init__(self, updated=0, created=0, unchanged=0): - self.created = created - self.updated = updated - self.unchanged = unchanged - - def __repr__(self): - return repr(self.as_dict()) - - def incr(self, counter): - setattr(self, counter, getattr(self, counter) + 1) - - def as_dict(self): - return {'created': self.created, - 'updated': self.updated, - 'unchanged': self.unchanged} - - def __eq__(self, other): - if isinstance(other, EtlResult): - other = other.as_dict() - - if isinstance(other, dict): - return (self.created == other['created'] and - self.updated == other['updated'] and - self.unchanged == other['unchanged']) - return False - def is_record_changed(record, values): other = type(record)(**values) @@ -144,11 +111,11 @@ def load_hact(): # else: # results.unchanged += 1 - return results.as_dict() + return results @app.etl(PMPIndicators) -def load_pmp_indicator(): +def load_pmp_indicator() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() base_url = 'https://etools.unicef.org' @@ -223,7 +190,7 @@ def load_pmp_indicator(): # else: # results.unchanged += 1 - return results.as_dict() + return results # PMPIndicators.objects.create( # country_id=country.pk, # partner_id=partner.pk, @@ -234,7 +201,7 @@ def load_pmp_indicator(): @app.etl(Intervention) -def load_intervention(): +def load_intervention() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() results = EtlResult() @@ -325,11 +292,11 @@ def load_intervention(): # else: # results.unchanged += 1 - return results.as_dict() + return results @app.etl(FAMIndicator) -def load_fam_indicator(): +def load_fam_indicator() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() @@ -375,11 +342,11 @@ def load_fam_indicator(): schema_name=country.schema_name), values=values) results.incr(op) - return results.as_dict() + return results @app.etl(UserStats) -def load_user_report(): +def load_user_report() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() today = date.today() @@ -419,7 +386,7 @@ def load_user_report(): # else: # results.unchanged += 1 # - return results.as_dict() + return results # UserStats.objects.update_or_create(month=first_of_month, # country_name=country.name, # schema_name=country.schema_name, diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 66e8186df..c89fb3eb1 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -4,6 +4,9 @@ from celery import Celery from celery.signals import task_postrun, task_prerun from celery.task import Task +from kombu.serialization import register + +from etools_datamart.apps.etl.results import etl_dumps, etl_loads os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etools_datamart.config.settings') @@ -73,33 +76,41 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): from django.utils import timezone defs = {'status': 'RUNNING', - 'timestamp': timezone.now()} + 'last_run': timezone.now()} EtlTask.objects.update_or_create(task=task.name, content_type=ContentType.objects.get_for_model(task.linked_model), table_name=task.linked_model._meta.db_table, defaults=defs) +register('etljson', etl_dumps, etl_loads, + content_type='application/x-myjson', content_encoding='utf-8') + + @task_postrun.connect def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): from django.utils import timezone - if not hasattr(sender, 'linked_model'): return try: cost = time() - app.timers.pop(task_id) except KeyError: # pragma: no cover cost = -1 - if not isinstance(retval, dict): - retval = {'error': str(retval)} defs = {'elapsed': cost, 'status': state, - 'results': retval, - 'timestamp': timezone.now()} + 'last_run': timezone.now()} + if state == 'SUCCESS': + defs['results'] = retval.as_dict() + if retval.created > 0 or retval.updated > 0: + defs['last_changes'] = timezone.now() + # if defs. defs['last_success'] = timezone.now() else: + if not isinstance(retval, dict): + defs['results'] = str(retval) defs['last_failure'] = timezone.now() + from etools_datamart.apps.etl.models import EtlTask EtlTask.objects.update_or_create(task=task.name, defaults=defs) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 4b522b983..7ad9c92ea 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -356,9 +356,9 @@ CELERY_TIMEZONE = 'America/New_York' CELERY_BROKER_URL = env('CELERY_BROKER_URL') CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') -CELERY_ACCEPT_CONTENT = ['application/json'] -CELERY_RESULT_SERIALIZER = 'json' -CELERY_TASK_SERIALIZER = 'json' +# CELERY_ACCEPT_CONTENT = ['application/json'] +# CELERY_RESULT_SERIALIZER = 'json' +# CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_IMPORTS = ["etools_datamart.apps.etl.tasks.etl", "etools_datamart.apps.etl.tasks.tasks", ] CELERY_BEAT_SCHEDULE = {} @@ -369,6 +369,10 @@ 'etools_datamart.apps.etl.tasks.tasks': {'queue': 'tasks'}, } +CELERY_ACCEPT_CONTENT = ['etljson'] +CELERY_TASK_SERIALIZER = 'etljson' +CELERY_RESULT_SERIALIZER = 'etljson' + CONCURRENCY_IGNORE_DEFAULT = False REST_FRAMEWORK = { diff --git a/tests/etl/test_etl_tasklog.py b/tests/etl/test_etl_tasklog.py index 1db437319..60559d3d6 100644 --- a/tests/etl/test_etl_tasklog.py +++ b/tests/etl/test_etl_tasklog.py @@ -5,6 +5,7 @@ from etools_datamart.apps.data.models import PMPIndicators from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.etl.results import EtlResult from etools_datamart.apps.etl.tasks.etl import load_pmp_indicator from etools_datamart.celery import task_postrun_handler @@ -22,9 +23,11 @@ def test_check_extra_attributes(db): def test_load_pmp_indicator(db): - with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run'): + with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run', + return_value=EtlResult(created=11)): assert load_pmp_indicator.apply() assert EtlTask.objects.filter(task='etools_datamart.apps.etl.tasks.etl.load_pmp_indicator', + results__created=11, status='SUCCESS').exists() From dc6cdff0b678b1bc5ace31474505fd726daed5a6 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 12:47:15 +0100 Subject: [PATCH 15/43] add api versioning --- src/etools_datamart/api/urls.py | 8 ++++---- src/etools_datamart/apps/web/templates/index.html | 2 +- src/etools_datamart/apps/web/templates/monitor.html | 2 +- src/etools_datamart/config/settings.py | 2 ++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index 92cc86182..2a2a498dc 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -1,4 +1,4 @@ -from django.urls import path +from django.urls import include, path, re_path from unicef_rest_framework.routers import APIReadOnlyRouter from . import endpoints @@ -34,10 +34,10 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'system/tasks-log', endpoints.TaskLogViewSet) -urlpatterns = router.urls +# urlpatterns = router.urls -urlpatterns += [ - # url(r'^+swagger(?P\.json|\.yaml)$', endpoints.schema_view.without_ui(cache_timeout=0), name='schema-json'), +urlpatterns = [ + re_path(r'(?P(v1|latest))/', include(router.urls)), path(r'+swagger/', endpoints.schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), path(r'+redoc/', endpoints.schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), ] diff --git a/src/etools_datamart/apps/web/templates/index.html b/src/etools_datamart/apps/web/templates/index.html index 9d09c1960..39a772774 100644 --- a/src/etools_datamart/apps/web/templates/index.html +++ b/src/etools_datamart/apps/web/templates/index.html @@ -6,7 +6,7 @@

    eTools Datamart

  • API (needs token)
  • Login
    • {% else %} -
  • API
    • +
  • API
  • Documentation
  • Swagger
  • Monitor
    • diff --git a/src/etools_datamart/apps/web/templates/monitor.html b/src/etools_datamart/apps/web/templates/monitor.html index 13496f22c..c1c3ccc80 100644 --- a/src/etools_datamart/apps/web/templates/monitor.html +++ b/src/etools_datamart/apps/web/templates/monitor.html @@ -8,7 +8,7 @@
    Home - API + API Documentation
    diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 7ad9c92ea..79f82fc83 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -364,6 +364,7 @@ CELERY_BEAT_SCHEDULE = {} CELERY_TASK_ALWAYS_EAGER = env.bool('CELERY_ALWAYS_EAGER', False) CELERY_EAGER_PROPAGATES_EXCEPTIONS = CELERY_TASK_ALWAYS_EAGER + CELERY_TASK_ROUTES = { 'etools_datamart.apps.etl.tasks.etl': {'queue': 'etl'}, 'etools_datamart.apps.etl.tasks.tasks': {'queue': 'tasks'}, @@ -384,6 +385,7 @@ # "DEFAULT_PAGINATION_CLASS": 'rest_framework.pagination.CursorPagination', 'DEFAULT_PAGINATION_CLASS': 'unicef_rest_framework.pagination.APIPagination', 'DEFAULT_METADATA_CLASS': 'etools_datamart.api.metadata.SimpleMetadataWithFilters', + 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning', # 'DEFAULT_SCHEMA_CLASS': 'etools_datamart.api.swagger.APIAutoSchema', # 'EXCEPTION_HANDLER': 'my_project.my_app.utils.custom_exception_handler' 'SEARCH_PARAM': 'search', From a1e2dcf57d5af5514f162eafa67f71d00137bb5a Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 12:56:44 +0100 Subject: [PATCH 16/43] add last_changes to monitor --- src/etools_datamart/apps/etl/models.py | 8 +- src/etools_datamart/apps/web/static/style.css | 170 ------------------ .../apps/web/static/style.css.map | 7 - .../apps/web/static/style.scss | 7 +- .../apps/web/templates/monitor.html | 8 +- 5 files changed, 14 insertions(+), 186 deletions(-) delete mode 100644 src/etools_datamart/apps/web/static/style.css delete mode 100644 src/etools_datamart/apps/web/static/style.css.map diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index 3e3286bb0..a3f77c145 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -37,12 +37,12 @@ def inspect(self): class EtlTask(models.Model): task = models.CharField(max_length=200, unique=True) - last_run = models.DateTimeField(null=True) + last_run = models.DateTimeField(null=True, help_text="last execution time") status = models.CharField(max_length=200) elapsed = models.IntegerField(null=True) - last_success = models.DateTimeField(null=True) - last_failure = models.DateTimeField(null=True) - last_changes = models.DateTimeField(null=True) + last_success = models.DateTimeField(null=True, help_text="last successully execution time") + last_failure = models.DateTimeField(null=True, help_text="last failure execution time") + last_changes = models.DateTimeField(null=True, help_text="last time data have been changed") table_name = models.CharField(max_length=200, null=True) content_type = models.ForeignKey(ContentType, models.CASCADE, null=True) diff --git a/src/etools_datamart/apps/web/static/style.css b/src/etools_datamart/apps/web/static/style.css deleted file mode 100644 index d0f8b91d6..000000000 --- a/src/etools_datamart/apps/web/static/style.css +++ /dev/null @@ -1,170 +0,0 @@ -* { - margin: 0px; - padding: 0px; - box-sizing: border-box; } - -body, html { - height: 50%; - font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif; } - body a, html a { - color: black; } - body a:hover, html a:hover { - color: black; } - body span.str, html span.str { - color: black; } - -.limiter { - width: 100%; - margin: 0 auto; } - -.master-container { - width: 100%; - min-height: 94vh; - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - flex-wrap: wrap; - justify-content: center; - align-items: center; - padding: 15px; } - -.box { - width: 1170px; - background: #fff; - overflow: hidden; - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - flex-wrap: wrap; - align-items: stretch; - flex-direction: row-reverse; } - -.right { - width: 50%; - flex-wrap: wrap; } - .right form { - padding: 100px 0 0 65px; } - .right .center { - width: 100%; - text-align: center; } - .right h1 { - width: 100%; - font-size: 24pt; - text-align: center; } - .right h2 { - font-size: 14pt; } - .right ul { - padding: 20px 0 0 65px; - list-style: none; - font-size: 18pt; } - .right a { - color: #2090F8; } - -/* [ login more ]*/ -.left { - width: 50%; - background-repeat: no-repeat; - background-size: cover; - background-position: center; - position: relative; - z-index: 1; } - -.m-b-20 { - margin-bottom: 20px; } - -.wrap-input100 { - width: 100%; - position: relative; - border: 1px solid #e6e6e6; } - -.input100 { - display: block; - width: 100%; - background: transparent; - font-size: 18px; - color: #666666; - line-height: 1.2; - padding: 0 25px; } - -input.input100 { - height: 55px; } - -/*[ Button ]*/ -.container-login100-form-btn { - width: 100%; - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - flex-wrap: wrap; - justify-content: center; } - -.login100-form-btn { - display: -webkit-box; - display: -webkit-flex; - display: -moz-box; - display: -ms-flexbox; - display: flex; - justify-content: center; - align-items: center; - padding: 0 20px; - width: 100%; - height: 50px; - border-radius: 3px; - background: #2090F8; - font-size: 12px; - color: #fff; - line-height: 1.2; - text-transform: uppercase; - letter-spacing: 1px; - -webkit-transition: all 0.4s; - -o-transition: all 0.4s; - -moz-transition: all 0.4s; - transition: all 0.4s; } - -.login100-form-btn:hover { - background: #333333; } - -/* [ Responsive ]*/ -.navbar { - border-top: none; - background: #2090F8; - color: black; } - .navbar a.navbar-brand { - color: white; } - -.monitor .menubar { - padding: 10px; } - .monitor .menubar a { - text-transform: none; - color: #2090F8; } - .monitor .menubar a:visited { - text-transform: none; } -.monitor #monitor { - width: 100%; - padding: 30px; } - .monitor #monitor .SUCCESS { - color: #00b200; - font-weight: bold; } - .monitor #monitor .row { - width: 100%; - padding: 5px; } - .monitor #monitor .row.header { - font-weight: bold; } - .monitor #monitor .row.odd { - background-color: #eeeeee; } - .monitor #monitor .col { - display: inline-block; } - .monitor #monitor .col.task { - width: 30%; } - .monitor #monitor .col.timestamp { - width: 300px; } - .monitor #monitor .col.result { - width: 100px; } - -/*# sourceMappingURL=style.css.map */ diff --git a/src/etools_datamart/apps/web/static/style.css.map b/src/etools_datamart/apps/web/static/style.css.map deleted file mode 100644 index 49796ab0e..000000000 --- a/src/etools_datamart/apps/web/static/style.css.map +++ /dev/null @@ -1,7 +0,0 @@ -{ -"version": 3, -"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEjB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKlB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAKd,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAQ;MACN,WAAW,EAAE,IAAI;IAEnB,0BAAK;MACH,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,gCAAY;MACV,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK", -"sources": ["style.scss"], -"names": [], -"file": "style.css" -} \ No newline at end of file diff --git a/src/etools_datamart/apps/web/static/style.scss b/src/etools_datamart/apps/web/static/style.scss index 070fc5546..5abe87f78 100644 --- a/src/etools_datamart/apps/web/static/style.scss +++ b/src/etools_datamart/apps/web/static/style.scss @@ -268,10 +268,13 @@ input.input100 { &.task { width: 30%; } - &.timestamp { + &.last_run{ width: 300px; } - &.result { + &.last_changes { + width: 300px; + } + &.status { width: 100px; } } diff --git a/src/etools_datamart/apps/web/templates/monitor.html b/src/etools_datamart/apps/web/templates/monitor.html index c1c3ccc80..e4910f88d 100644 --- a/src/etools_datamart/apps/web/templates/monitor.html +++ b/src/etools_datamart/apps/web/templates/monitor.html @@ -14,14 +14,16 @@
    Data - Last update + Last run + Last change Result
    {% for t in tasks %}
    {{ t.verbose_name }} - {{ t.timestamp|default_if_none:"" }} - {{ t.result }} + {{ t.last_run|default_if_none:"" }} + {{ t.last_changes|default_if_none:"" }} + {{ t.status }}
    {% endfor %}
    From 59cf3cae6f792277916fee0576eeec807626f85c Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 13:15:56 +0100 Subject: [PATCH 17/43] updates index page --- CHANGES | 2 +- src/etools_datamart/api/serializers/audit.py | 3 +++ src/etools_datamart/apps/web/templates/index.html | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 98bdc4d32..4fbc8c7f6 100644 --- a/CHANGES +++ b/CHANGES @@ -5,7 +5,7 @@ * new 'Monitor' page to check dataset last update date * add ability to intercept changed/unchanged datamart records * Azure storage support -* + 1.6 --- diff --git a/src/etools_datamart/api/serializers/audit.py b/src/etools_datamart/api/serializers/audit.py index bb8f60cf4..4aac25198 100644 --- a/src/etools_datamart/api/serializers/audit.py +++ b/src/etools_datamart/api/serializers/audit.py @@ -8,3 +8,6 @@ class EngagementSerializer(serializers.ModelSerializer): class Meta: model = models.AuditEngagement exclude = () + + def fail(self, key, **kwargs): + return super().fail(key, **kwargs) diff --git a/src/etools_datamart/apps/web/templates/index.html b/src/etools_datamart/apps/web/templates/index.html index 39a772774..a0be8b868 100644 --- a/src/etools_datamart/apps/web/templates/index.html +++ b/src/etools_datamart/apps/web/templates/index.html @@ -3,7 +3,7 @@

    eTools Datamart

      {% if request.user.is_anonymous %} -
    • API (needs token)
      • +
    • API (needs token)
    • Login
      • {% else %}
    • API
      • From 68004b4cf959c0c0d7f6d1bfa91cb9c1eac07faf Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 13:27:03 +0100 Subject: [PATCH 18/43] allow supersusers to edit data --- src/etools_datamart/apps/data/admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index 7b63661b4..421a52e78 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -45,10 +45,11 @@ def has_add_permission(self, request): return False def has_delete_permission(self, request, obj=None): - return False + return request.user.is_superuser def get_readonly_fields(self, request, obj=None): - self.readonly_fields = [field.name for field in obj.__class__._meta.fields] + if not request.user.is_superuser: + self.readonly_fields = [field.name for field in obj.__class__._meta.fields] return self.readonly_fields def changeform_view(self, request, object_id=None, form_url='', extra_context=None): From cdbcfc3472787494ab6317616fabd4fefc3b5576 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 13:28:10 +0100 Subject: [PATCH 19/43] apdates CHANGES --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 4fbc8c7f6..9e02d0cf2 100644 --- a/CHANGES +++ b/CHANGES @@ -5,7 +5,7 @@ * new 'Monitor' page to check dataset last update date * add ability to intercept changed/unchanged datamart records * Azure storage support - +* use versioning in API urls 1.6 --- From d8222a21197c9708e0af772785bbd9a96823c18a Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 13:28:52 +0100 Subject: [PATCH 20/43] add missed files --- src/etools_datamart/apps/web/static/style.css | 172 ++++++++++++++++++ .../apps/web/static/style.css.map | 7 + 2 files changed, 179 insertions(+) create mode 100644 src/etools_datamart/apps/web/static/style.css create mode 100644 src/etools_datamart/apps/web/static/style.css.map diff --git a/src/etools_datamart/apps/web/static/style.css b/src/etools_datamart/apps/web/static/style.css new file mode 100644 index 000000000..df8e33bf6 --- /dev/null +++ b/src/etools_datamart/apps/web/static/style.css @@ -0,0 +1,172 @@ +* { + margin: 0px; + padding: 0px; + box-sizing: border-box; } + +body, html { + height: 50%; + font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif; } + body a, html a { + color: black; } + body a:hover, html a:hover { + color: black; } + body span.str, html span.str { + color: black; } + +.limiter { + width: 100%; + margin: 0 auto; } + +.master-container { + width: 100%; + min-height: 94vh; + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + flex-wrap: wrap; + justify-content: center; + align-items: center; + padding: 15px; } + +.box { + width: 1170px; + background: #fff; + overflow: hidden; + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + flex-wrap: wrap; + align-items: stretch; + flex-direction: row-reverse; } + +.right { + width: 50%; + flex-wrap: wrap; } + .right form { + padding: 100px 0 0 65px; } + .right .center { + width: 100%; + text-align: center; } + .right h1 { + width: 100%; + font-size: 24pt; + text-align: center; } + .right h2 { + font-size: 14pt; } + .right ul { + padding: 20px 0 0 65px; + list-style: none; + font-size: 18pt; } + .right a { + color: #2090F8; } + +/* [ login more ]*/ +.left { + width: 50%; + background-repeat: no-repeat; + background-size: cover; + background-position: center; + position: relative; + z-index: 1; } + +.m-b-20 { + margin-bottom: 20px; } + +.wrap-input100 { + width: 100%; + position: relative; + border: 1px solid #e6e6e6; } + +.input100 { + display: block; + width: 100%; + background: transparent; + font-size: 18px; + color: #666666; + line-height: 1.2; + padding: 0 25px; } + +input.input100 { + height: 55px; } + +/*[ Button ]*/ +.container-login100-form-btn { + width: 100%; + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + flex-wrap: wrap; + justify-content: center; } + +.login100-form-btn { + display: -webkit-box; + display: -webkit-flex; + display: -moz-box; + display: -ms-flexbox; + display: flex; + justify-content: center; + align-items: center; + padding: 0 20px; + width: 100%; + height: 50px; + border-radius: 3px; + background: #2090F8; + font-size: 12px; + color: #fff; + line-height: 1.2; + text-transform: uppercase; + letter-spacing: 1px; + -webkit-transition: all 0.4s; + -o-transition: all 0.4s; + -moz-transition: all 0.4s; + transition: all 0.4s; } + +.login100-form-btn:hover { + background: #333333; } + +/* [ Responsive ]*/ +.navbar { + border-top: none; + background: #2090F8; + color: black; } + .navbar a.navbar-brand { + color: white; } + +.monitor .menubar { + padding: 10px; } + .monitor .menubar a { + text-transform: none; + color: #2090F8; } + .monitor .menubar a:visited { + text-transform: none; } +.monitor #monitor { + width: 100%; + padding: 30px; } + .monitor #monitor .SUCCESS { + color: #00b200; + font-weight: bold; } + .monitor #monitor .row { + width: 100%; + padding: 5px; } + .monitor #monitor .row.header { + font-weight: bold; } + .monitor #monitor .row.odd { + background-color: #eeeeee; } + .monitor #monitor .col { + display: inline-block; } + .monitor #monitor .col.task { + width: 30%; } + .monitor #monitor .col.last_run { + width: 300px; } + .monitor #monitor .col.last_changes { + width: 300px; } + .monitor #monitor .col.status { + width: 100px; } + +/*# sourceMappingURL=style.css.map */ diff --git a/src/etools_datamart/apps/web/static/style.css.map b/src/etools_datamart/apps/web/static/style.css.map new file mode 100644 index 000000000..9993f31c6 --- /dev/null +++ b/src/etools_datamart/apps/web/static/style.css.map @@ -0,0 +1,7 @@ +{ +"version": 3, +"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEjB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKlB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAKd,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAQ;MACN,WAAW,EAAE,IAAI;IAEnB,0BAAK;MACH,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,+BAAU;MACR,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK", +"sources": ["style.scss"], +"names": [], +"file": "style.css" +} \ No newline at end of file From 86adbddd1dc39e40bf3b225d4566779acb18a6a0 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 13:41:46 +0100 Subject: [PATCH 21/43] fixes datamart admin editing --- src/etools_datamart/apps/data/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index 421a52e78..445a83e74 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -53,7 +53,7 @@ def get_readonly_fields(self, request, obj=None): return self.readonly_fields def changeform_view(self, request, object_id=None, form_url='', extra_context=None): - if request.method == 'POST': + if request.method == 'POST' and not request.user.is_superuser: redirect_url = reverse('admin:%s_%s_changelist' % (self.opts.app_label, self.opts.model_name)) From 918ce4123b02d17fb044e895e9e6061cd97037b5 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 14:13:49 +0100 Subject: [PATCH 22/43] reset migrations --- .../apps/data/migrations/0001_initial.py | 39 ++++++++++++++++++- .../0002_intervention_partner_name.py | 18 --------- .../apps/data/migrations/0003_hact.py | 33 ---------------- .../apps/etl/migrations/0001_initial.py | 18 ++++++--- .../etl/migrations/0002_auto_20181119_2028.py | 18 --------- .../etl/migrations/0003_auto_20181123_1903.py | 28 ------------- .../etl/migrations/0004_auto_20181126_0922.py | 18 --------- .../migrations/0005_etltask_last_changes.py | 18 --------- .../apps/tracking/migrations/0001_initial.py | 6 +-- ...031_1959.py => 0002_auto_20181126_1313.py} | 4 +- .../migrations/0001_initial.py | 3 +- ...031_1959.py => 0002_auto_20181126_1313.py} | 6 +-- .../migrations/0001_initial.py | 2 +- 13 files changed, 60 insertions(+), 151 deletions(-) delete mode 100644 src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py delete mode 100644 src/etools_datamart/apps/data/migrations/0003_hact.py delete mode 100644 src/etools_datamart/apps/etl/migrations/0002_auto_20181119_2028.py delete mode 100644 src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py delete mode 100644 src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py delete mode 100644 src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py rename src/etools_datamart/apps/tracking/migrations/{0002_auto_20181031_1959.py => 0002_auto_20181126_1313.py} (97%) rename src/unicef_rest_framework/migrations/{0002_auto_20181031_1959.py => 0002_auto_20181126_1313.py} (99%) diff --git a/src/etools_datamart/apps/data/migrations/0001_initial.py b/src/etools_datamart/apps/data/migrations/0001_initial.py index 0367341e6..80f81cdd6 100644 --- a/src/etools_datamart/apps/data/migrations/0001_initial.py +++ b/src/etools_datamart/apps/data/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-26 13:13 import django.contrib.postgres.fields.jsonb import month_field.models @@ -39,10 +39,31 @@ class Migration(migrations.Migration): ('microassessment_cancelled', models.IntegerField(default=0, verbose_name='Micro Assessment-Cancelled')), ], options={ + 'verbose_name': 'FAM Indicator', 'ordering': ('month', 'country_name'), }, bases=(models.Model, unicef_security.models.TimeStampedModel), ), + migrations.CreateModel( + name='HACT', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('country_name', models.CharField(db_index=True, max_length=50)), + ('schema_name', models.CharField(db_index=True, max_length=50)), + ('year', models.IntegerField()), + ('microassessments_total', models.IntegerField(default=0, help_text='Total number of completed Microassessments in the business area in the past year')), + ('programmaticvisits_total', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), + ('followup_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), + ('completed_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), + ('completed_hact_audits', models.IntegerField(default=0, help_text='Total number of completed scheduled audits for the workspace.')), + ('completed_special_audits', models.IntegerField(default=0, help_text='Total number of completed special audits for the workspace. ')), + ], + options={ + 'verbose_name': 'HACT', + 'ordering': ('year', 'country_name'), + }, + bases=(models.Model, unicef_security.models.TimeStampedModel), + ), migrations.CreateModel( name='Intervention', fields=[ @@ -71,6 +92,7 @@ class Migration(migrations.Migration): ('unicef_signatory_first_name', models.CharField(max_length=30, null=True)), ('unicef_signatory_last_name', models.CharField(max_length=30, null=True)), ('unicef_signatory_email', models.CharField(max_length=254, null=True)), + ('partner_name', models.CharField(max_length=200, null=True)), ('partner_signatory_title', models.CharField(max_length=64, null=True)), ('partner_signatory_first_name', models.CharField(max_length=64, null=True)), ('partner_signatory_last_name', models.CharField(max_length=64, null=True)), @@ -90,6 +112,7 @@ class Migration(migrations.Migration): ('unicef_signatory_id', models.IntegerField(null=True)), ], options={ + 'verbose_name': 'Intervention', 'ordering': ('country_name', 'title'), }, bases=(models.Model, unicef_security.models.TimeStampedModel), @@ -131,6 +154,7 @@ class Migration(migrations.Migration): ('updated', models.DateTimeField(null=True)), ], options={ + 'verbose_name': 'PMP Indicator', 'ordering': ('country_name', 'partner_name'), }, bases=(models.Model, unicef_security.models.TimeStampedModel), @@ -148,8 +172,21 @@ class Migration(migrations.Migration): ('unicef_logins', models.IntegerField(default=0, verbose_name='Number of UNICEF logins')), ], options={ + 'verbose_name': 'User Access Statistics', 'ordering': ('-month', 'country_name'), }, bases=(models.Model, unicef_security.models.TimeStampedModel), ), + migrations.AlterUniqueTogether( + name='userstats', + unique_together={('country_name', 'month')}, + ), + migrations.AlterUniqueTogether( + name='hact', + unique_together={('year', 'country_name')}, + ), + migrations.AlterUniqueTogether( + name='famindicator', + unique_together={('month', 'country_name')}, + ), ] diff --git a/src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py b/src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py deleted file mode 100644 index 05810c918..000000000 --- a/src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-06 13:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('data', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='intervention', - name='partner_name', - field=models.CharField(max_length=200, null=True), - ), - ] diff --git a/src/etools_datamart/apps/data/migrations/0003_hact.py b/src/etools_datamart/apps/data/migrations/0003_hact.py deleted file mode 100644 index 486edd3b9..000000000 --- a/src/etools_datamart/apps/data/migrations/0003_hact.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-09 17:08 - -import unicef_security.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('data', '0002_intervention_partner_name'), - ] - - operations = [ - migrations.CreateModel( - name='HACT', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('country_name', models.CharField(db_index=True, max_length=50)), - ('schema_name', models.CharField(db_index=True, max_length=50)), - ('year', models.IntegerField()), - ('microassessments_total', models.IntegerField(default=0, help_text='Total number of completed Microassessments in the business area in the past year')), - ('programmaticvisits_total', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), - ('followup_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), - ('completed_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), - ('completed_hact_audits', models.IntegerField(default=0, help_text='Total number of completed scheduled audits for the workspace.')), - ('completed_special_audits', models.IntegerField(default=0, help_text='Total number of completed special audits for the workspace. ')), - ], - options={ - 'abstract': False, - }, - bases=(models.Model, unicef_security.models.TimeStampedModel), - ), - ] diff --git a/src/etools_datamart/apps/etl/migrations/0001_initial.py b/src/etools_datamart/apps/etl/migrations/0001_initial.py index 9743c7ab6..0cf3dc2e0 100644 --- a/src/etools_datamart/apps/etl/migrations/0001_initial.py +++ b/src/etools_datamart/apps/etl/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-26 13:13 +import django.contrib.postgres.fields.jsonb import django.db.models.deletion from django.db import migrations, models @@ -14,17 +15,22 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='TaskLog', + name='EtlTask', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('task', models.CharField(max_length=200, unique=True)), - ('timestamp', models.DateTimeField(null=True)), - ('result', models.CharField(max_length=200)), + ('last_run', models.DateTimeField(help_text='last execution time', null=True)), + ('status', models.CharField(max_length=200)), ('elapsed', models.IntegerField(null=True)), - ('last_success', models.DateTimeField(null=True)), - ('last_failure', models.DateTimeField(null=True)), + ('last_success', models.DateTimeField(help_text='last successully execution time', null=True)), + ('last_failure', models.DateTimeField(help_text='last failure execution time', null=True)), + ('last_changes', models.DateTimeField(help_text='last time data have been changed', null=True)), ('table_name', models.CharField(max_length=200, null=True)), + ('results', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)), ('content_type', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), ], + options={ + 'get_latest_by': 'last_run', + }, ), ] diff --git a/src/etools_datamart/apps/etl/migrations/0002_auto_20181119_2028.py b/src/etools_datamart/apps/etl/migrations/0002_auto_20181119_2028.py deleted file mode 100644 index 3306e10fd..000000000 --- a/src/etools_datamart/apps/etl/migrations/0002_auto_20181119_2028.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-19 20:28 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('etl', '0001_initial'), - ] - - operations = [ - migrations.RenameModel( - old_name='TaskLog', - new_name='EtlTask', - ), - ] diff --git a/src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py b/src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py deleted file mode 100644 index fc9613abc..000000000 --- a/src/etools_datamart/apps/etl/migrations/0003_auto_20181123_1903.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-23 19:03 - -import django.contrib.postgres.fields.jsonb -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('etl', '0002_auto_20181119_2028'), - ] - - operations = [ - migrations.AlterModelOptions( - name='etltask', - options={'get_latest_by': 'timestamp'}, - ), - migrations.RenameField( - model_name='etltask', - old_name='result', - new_name='status', - ), - migrations.AddField( - model_name='etltask', - name='results', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), - ), - ] diff --git a/src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py b/src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py deleted file mode 100644 index 6fb77a828..000000000 --- a/src/etools_datamart/apps/etl/migrations/0004_auto_20181126_0922.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-26 09:22 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('etl', '0003_auto_20181123_1903'), - ] - - operations = [ - migrations.RenameField( - model_name='etltask', - old_name='timestamp', - new_name='last_run', - ), - ] diff --git a/src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py b/src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py deleted file mode 100644 index c9daa0931..000000000 --- a/src/etools_datamart/apps/etl/migrations/0005_etltask_last_changes.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-26 09:22 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('etl', '0004_auto_20181126_0922'), - ] - - operations = [ - migrations.AddField( - model_name='etltask', - name='last_changes', - field=models.DateTimeField(null=True), - ), - ] diff --git a/src/etools_datamart/apps/tracking/migrations/0001_initial.py b/src/etools_datamart/apps/tracking/migrations/0001_initial.py index 57174107b..6c01795ff 100644 --- a/src/etools_datamart/apps/tracking/migrations/0001_initial.py +++ b/src/etools_datamart/apps/tracking/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-26 13:13 import django.utils.timezone import strategy_field.fields @@ -32,8 +32,8 @@ class Migration(migrations.Migration): ('viewset', strategy_field.fields.StrategyClassField(blank=True, null=True)), ], options={ - 'verbose_name': 'Log', - 'verbose_name_plural': 'Logs', + 'verbose_name': 'Access Log', + 'verbose_name_plural': 'Access Log', 'ordering': ('-id',), 'get_latest_by': 'requested_at', }, diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181031_1959.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181126_1313.py similarity index 97% rename from src/etools_datamart/apps/tracking/migrations/0002_auto_20181031_1959.py rename to src/etools_datamart/apps/tracking/migrations/0002_auto_20181126_1313.py index 28bc34837..113821101 100644 --- a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181031_1959.py +++ b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181126_1313.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-26 13:13 import django.db.models.deletion from django.conf import settings @@ -10,9 +10,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('unicef_rest_framework', '0001_initial'), ('tracking', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ diff --git a/src/unicef_rest_framework/migrations/0001_initial.py b/src/unicef_rest_framework/migrations/0001_initial.py index 000ea3f87..86acb8f67 100644 --- a/src/unicef_rest_framework/migrations/0001_initial.py +++ b/src/unicef_rest_framework/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-26 13:13 import uuid @@ -69,7 +69,6 @@ class Migration(migrations.Migration): ], options={ 'ordering': ('name',), - 'permissions': (('do_not_scramble', 'Can read any service unscrambled'),), }, ), migrations.CreateModel( diff --git a/src/unicef_rest_framework/migrations/0002_auto_20181031_1959.py b/src/unicef_rest_framework/migrations/0002_auto_20181126_1313.py similarity index 99% rename from src/unicef_rest_framework/migrations/0002_auto_20181031_1959.py rename to src/unicef_rest_framework/migrations/0002_auto_20181126_1313.py index 1de465ebc..9e5a822f1 100644 --- a/src/unicef_rest_framework/migrations/0002_auto_20181031_1959.py +++ b/src/unicef_rest_framework/migrations/0002_auto_20181126_1313.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-26 13:13 import django.db.models.deletion from django.conf import settings @@ -10,10 +10,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('auth', '0009_alter_user_last_name_max_length'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('contenttypes', '0002_remove_content_type_name'), ('unicef_rest_framework', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('auth', '0009_alter_user_last_name_max_length'), ] operations = [ diff --git a/src/unicef_security/migrations/0001_initial.py b/src/unicef_security/migrations/0001_initial.py index fdf22ac53..7cad73d43 100644 --- a/src/unicef_security/migrations/0001_initial.py +++ b/src/unicef_security/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-26 13:13 import django.contrib.auth.models import django.contrib.auth.validators From ef979861fef35fd4d3580f73984277d113f9c166 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 17:20:07 +0100 Subject: [PATCH 23/43] fixes tests --- src/etools_datamart/api/serializers/audit.py | 3 --- src/unicef_rest_framework/models/service.py | 2 +- tests/_test_lib/test_utilities/factories.py | 7 +++++++ tests/api/test_api_auth_jwt.py | 4 ++-- tests/api/test_api_common.py | 2 +- tests/api/test_api_etools.py | 11 +++++----- tests/api/test_api_filtering.py | 6 +++--- tests/api/test_api_pages.py | 6 +++--- tests/api/test_api_system.py | 3 ++- tests/api/test_api_web.py | 2 +- tests/api/test_datamart_security.py | 6 ------ tests/api/test_etools_security.py | 6 ------ tests/conftest.py | 14 +++++++++++++ tests/datamart/test_data_admin.py | 21 ++++++++++++++++---- tests/exporters/test_exporter_data.py | 2 +- tests/multitenant/test_schema_selection.py | 2 +- tests/tracking/test_tracking_log.py | 4 ++-- 17 files changed, 61 insertions(+), 40 deletions(-) diff --git a/src/etools_datamart/api/serializers/audit.py b/src/etools_datamart/api/serializers/audit.py index 4aac25198..bb8f60cf4 100644 --- a/src/etools_datamart/api/serializers/audit.py +++ b/src/etools_datamart/api/serializers/audit.py @@ -8,6 +8,3 @@ class EngagementSerializer(serializers.ModelSerializer): class Meta: model = models.AuditEngagement exclude = () - - def fail(self, key, **kwargs): - return super().fail(key, **kwargs) diff --git a/src/unicef_rest_framework/models/service.py b/src/unicef_rest_framework/models/service.py index ea358ebd0..9ab318428 100644 --- a/src/unicef_rest_framework/models/service.py +++ b/src/unicef_rest_framework/models/service.py @@ -118,7 +118,7 @@ def get_access_level(self): def endpoint(self): for __, viewset, base_name in conf.ROUTER.registry: if viewset == self.viewset: - return reverse(f'api:{base_name}-list') + return reverse(f'api:{base_name}-list', args=['v1']) else: return None diff --git a/tests/_test_lib/test_utilities/factories.py b/tests/_test_lib/test_utilities/factories.py index 28fc572e3..fa57ff038 100644 --- a/tests/_test_lib/test_utilities/factories.py +++ b/tests/_test_lib/test_utilities/factories.py @@ -3,6 +3,7 @@ import factory import unicef_security.models +from django.contrib.auth.models import Group from django.db import connections from django.utils import timezone from unicef_rest_framework.models import Service, SystemFilter, UserAccessControl @@ -65,6 +66,12 @@ class Meta: model = Intervention +class GroupFactory(factory.DjangoModelFactory): + class Meta: + model = Group + django_get_or_create = ('name',) + + class UserFactory(factory.DjangoModelFactory): class Meta: model = unicef_security.models.User diff --git a/tests/api/test_api_auth_jwt.py b/tests/api/test_api_auth_jwt.py index 859f0315f..8e359c618 100644 --- a/tests/api/test_api_auth_jwt.py +++ b/tests/api/test_api_auth_jwt.py @@ -27,7 +27,7 @@ def user(db): def test_token(user, client): - url = reverse('api:partners-list') + url = reverse('api:partners-list', args=['v1']) client.credentials(HTTP_AUTHORIZATION='jwt ' + TOKEN) with mock.patch('unicef_security.azure.Synchronizer.get_user', return_value={'@odata.context': 'https://graph.microsoft.com/v1.0/$metadata#users/$entity', @@ -43,7 +43,7 @@ def test_token(user, client): @override_config(AZURE_USE_GRAPH=False) def test_token2(user, client): - url = reverse('api:partners-list') + url = reverse('api:partners-list', args=['v1']) client.credentials(HTTP_AUTHORIZATION='jwt ' + TOKEN) with mock.patch('rest_framework_jwt.settings.api_settings.JWT_VERIFY_EXPIRATION', False): ret = client.get(url) diff --git a/tests/api/test_api_common.py b/tests/api/test_api_common.py index 9c8fc7763..4185c6da6 100644 --- a/tests/api/test_api_common.py +++ b/tests/api/test_api_common.py @@ -14,7 +14,7 @@ def pytest_generate_tests(metafunc): if 'url' in metafunc.fixturenames: - urls = [reverse("api:%s" % url.name) for url in router.urls + urls = [reverse("api:%s" % url.name, args=['v1']) for url in router.urls if url.name.endswith('-list')] metafunc.parametrize("url", urls, ids=urls) diff --git a/tests/api/test_api_etools.py b/tests/api/test_api_etools.py index cba7704ba..c76391388 100644 --- a/tests/api/test_api_etools.py +++ b/tests/api/test_api_etools.py @@ -12,10 +12,11 @@ def pytest_generate_tests(metafunc): if 'url' in metafunc.fixturenames: if metafunc.function.__name__ == 'test_list': urls = filter(lambda url: 'etools/' in url, - [reverse("api:%s" % url.name) for url in router.urls if url.name.endswith('-list')]) + [reverse("api:%s" % url.name, args=['latest']) + for url in router.urls if url.name.endswith('-list')]) elif metafunc.function.__name__ == 'test_retrieve': urls = filter(lambda url: 'etools/' in url, - [reverse("api:%s" % url.name, args=['_lastest_']) for url in router.urls if + [reverse("api:%s" % url.name, args=['latest', '_lastest_']) for url in router.urls if url.name.endswith('-detail')]) metafunc.parametrize("url", list(urls)) @@ -31,7 +32,7 @@ def test_list(client, url, format, schema): def test_list_with_no_schema_search_all_schemas(client): - url = reverse("api:partners-list") + url = reverse("api:partners-list", args=['latest']) res = client.get(url) assert res.status_code == 200, res.content @@ -45,7 +46,7 @@ def test_retrieve(client, url, format): def test_retrieve_requires_only_one_schema(client): - url = reverse("api:partners-detail", args=['_lastest_']) + url = reverse("api:partners-detail", args=['latest', '_lastest_']) url = f"{url}?country_name=bolivia,chad" res = client.get(url) assert res.status_code == 400, res.content @@ -53,7 +54,7 @@ def test_retrieve_requires_only_one_schema(client): def test_retrieve_requires_one_schema(client): - url = reverse("api:partners-detail", args=['_lastest_']) + url = reverse("api:partners-detail", args=['latest', '_lastest_']) res = client.get(url) assert res.status_code == 400 assert res.json()['error'] == "country_name parameter is mandatory" diff --git a/tests/api/test_api_filtering.py b/tests/api/test_api_filtering.py index 8cdf75a50..715324280 100644 --- a/tests/api/test_api_filtering.py +++ b/tests/api/test_api_filtering.py @@ -3,7 +3,7 @@ @pytest.mark.parametrize('flt', ['country_name=bolivia', 'country_name=', 'country_name=bolivia,chad']) def test_filter_etools_country_name(db, client, flt): - url = f"/api/etools/audit/engagement/?%s" % flt + url = f"/api/latest/etools/audit/engagement/?%s" % flt res = client.get(url) assert res.status_code == 200 assert res.json() @@ -11,7 +11,7 @@ def test_filter_etools_country_name(db, client, flt): @pytest.mark.parametrize('flt', ['country_name=bolivia', 'country_name=', 'country_name=bolivia,chad']) def test_filter_datamart_country_name(db, client, flt): - url = f"/api/datamart/interventions/?%s" % flt + url = f"/api/latest/datamart/interventions/?%s" % flt res = client.get(url) assert res.status_code == 200 assert res.json() @@ -19,7 +19,7 @@ def test_filter_datamart_country_name(db, client, flt): @pytest.mark.parametrize('flt', ['10', 'oct', '10-2018', 'current', '']) def test_filter_datamart_month(db, client, flt): - url = f"/api/datamart/user-stats/?month=%s" % flt + url = f"/api/latest/datamart/user-stats/?month=%s" % flt res = client.get(url) assert res.status_code == 200 assert res.json() diff --git a/tests/api/test_api_pages.py b/tests/api/test_api_pages.py index 91c2251cb..59d215d9b 100644 --- a/tests/api/test_api_pages.py +++ b/tests/api/test_api_pages.py @@ -6,7 +6,7 @@ @pytest.mark.django_db() def test_api_site_root(django_app, admin_user): - url = reverse("api:api-root") + url = reverse("api:api-root", args=['latest']) res = django_app.get(url, user=admin_user, extra_environ={'HTTP_X_SCHEMA': "bolivia,chad,lebanon", @@ -16,7 +16,7 @@ def test_api_site_root(django_app, admin_user): @pytest.mark.django_db() def test_api_list(django_app, admin_user): - url = reverse("api:intervention-list") + url = reverse("api:intervention-list", args=['latest']) res = django_app.get(url, user=admin_user, extra_environ={'HTTP_X_SCHEMA': "bolivia,chad,lebanon", @@ -27,7 +27,7 @@ def test_api_list(django_app, admin_user): @pytest.mark.django_db() def test_api_detail(django_app, admin_user): i = InterventionFactory() - url = reverse("api:intervention-detail", args=[i.pk]) + url = reverse("api:intervention-detail", args=['latest', i.pk]) res = django_app.get(url, user=admin_user, extra_environ={'HTTP_X_SCHEMA': "bolivia,chad,lebanon", diff --git a/tests/api/test_api_system.py b/tests/api/test_api_system.py index 4947cb3b7..5a94b33d0 100644 --- a/tests/api/test_api_system.py +++ b/tests/api/test_api_system.py @@ -8,7 +8,8 @@ def pytest_generate_tests(metafunc): if 'url' in metafunc.fixturenames: urls = filter(lambda url: 'system/' in url, - [reverse("api:%s" % url.name) for url in router.urls if url.name.endswith('-list')]) + [reverse("api:%s" % url.name, args=['latest']) + for url in router.urls if url.name.endswith('-list')]) metafunc.parametrize("url", urls) diff --git a/tests/api/test_api_web.py b/tests/api/test_api_web.py index 5a0a5568a..bb13548a8 100644 --- a/tests/api/test_api_web.py +++ b/tests/api/test_api_web.py @@ -19,7 +19,7 @@ def users(db): def test_api_web_index(user): client = APIClient() client.force_authenticate(user) - res = client.get('/api/') + res = client.get('/api/latest/') assert res.status_code == 200 diff --git a/tests/api/test_datamart_security.py b/tests/api/test_datamart_security.py index db5fb186c..2ae628dcf 100644 --- a/tests/api/test_datamart_security.py +++ b/tests/api/test_datamart_security.py @@ -6,12 +6,6 @@ from unicef_rest_framework.test_utils import user_allow_service from etools_datamart.api.endpoints import PartnerViewSet, UserStatsViewSet -from etools_datamart.apps.etools.models import AuthUser - - -@pytest.fixture() -def etools_user(db): - return AuthUser.objects.get(id=1) @pytest.fixture() diff --git a/tests/api/test_etools_security.py b/tests/api/test_etools_security.py index 33949284e..6641c749e 100644 --- a/tests/api/test_etools_security.py +++ b/tests/api/test_etools_security.py @@ -4,12 +4,6 @@ from unicef_rest_framework.test_utils import user_allow_service from etools_datamart.api.endpoints import PartnerViewSet -from etools_datamart.apps.etools.models import AuthUser - - -@pytest.fixture() -def etools_user(db): - return AuthUser.objects.get(id=1) @pytest.fixture() diff --git a/tests/conftest.py b/tests/conftest.py index 9e417447f..76b4bb289 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,3 +127,17 @@ def number_of_partnerorganization(db): def number_of_intervention(db): # number of partners.Intervention return int((Path(__file__).parent / 'INTERVENTION').read_text()) + + +@pytest.fixture() +def etools_user(db): + from etools_datamart.apps.etools.models import AuthUser + return AuthUser.objects.get(id=1) + + +@pytest.fixture() +def staff_user(etools_user): + from test_utilities.factories import UserFactory + return UserFactory(username=etools_user.username, + email=etools_user.email, + is_staff=True) diff --git a/tests/datamart/test_data_admin.py b/tests/datamart/test_data_admin.py index 256e8d945..269a83839 100644 --- a/tests/datamart/test_data_admin.py +++ b/tests/datamart/test_data_admin.py @@ -3,6 +3,7 @@ from django.contrib import messages from django.urls import reverse from test_utilities.factories import PMPIndicatorFactory +from test_utilities.perms import user_grant_permissions @pytest.mark.django_db() @@ -28,16 +29,28 @@ def test_pmpindicators_filter(django_app, admin_user): @pytest.mark.django_db() -def test_pmpindicators_detail(django_app, admin_user, settings): +def test_pmpindicators_detail(django_app, staff_user, settings): + i = PMPIndicatorFactory() + url = reverse("admin:data_pmpindicators_change", args=[i.pk]) + assert staff_user.is_authenticated + with user_grant_permissions(staff_user, ['data.change_pmpindicators']): + res = django_app.get(url, user=staff_user) + assert res.status_code == 200 + res = res.form.submit().follow() + assert res.status_code == 200 + storage = res.context['messages'] + assert [m.message for m in storage] == ['This admin is read-only. Record not saved.'] + + +@pytest.mark.django_db() +def test_pmpindicators_detail_supersuser(django_app, admin_user, settings): i = PMPIndicatorFactory() url = reverse("admin:data_pmpindicators_change", args=[i.pk]) assert admin_user.is_authenticated res = django_app.get(url, user=admin_user) assert res.status_code == 200 res = res.form.submit().follow() - assert res.status_code == 200 - storage = res.context['messages'] - assert [m.message for m in storage] == ['This admin is read-only. Record not saved.'] + assert res.status_code == 302 def test_pmpindicators_refresh(django_app, admin_user): diff --git a/tests/exporters/test_exporter_data.py b/tests/exporters/test_exporter_data.py index 36d112628..96b63f36e 100644 --- a/tests/exporters/test_exporter_data.py +++ b/tests/exporters/test_exporter_data.py @@ -23,7 +23,7 @@ def test_export_azure_data(db, client, settings): load_user_report() assert UserStats.objects.count() - url = reverse("api:userstats-list") + url = reverse("api:userstats-list", args=['v1']) res = client.get(f"{url}?format=xlsx") from storages.backends.azure_storage import AzureStorage diff --git a/tests/multitenant/test_schema_selection.py b/tests/multitenant/test_schema_selection.py index ecf43f296..b1b11161d 100644 --- a/tests/multitenant/test_schema_selection.py +++ b/tests/multitenant/test_schema_selection.py @@ -81,7 +81,7 @@ def test_select_schema_data(django_app, admin_user): def test_api_call_queryparam(client, admin_user): client.login(username='admin', password='password') - url = f'{reverse("api:partners-list")}?country_name=bolivia,lebanon' + url = f'{reverse("api:partners-list", args=["v1"])}?country_name=bolivia,lebanon' res = client.get(url) assert res.status_code == 200 assert conn.schemas == ['bolivia', 'lebanon'] diff --git a/tests/tracking/test_tracking_log.py b/tests/tracking/test_tracking_log.py index 96aabe459..f2fb32267 100644 --- a/tests/tracking/test_tracking_log.py +++ b/tests/tracking/test_tracking_log.py @@ -28,7 +28,7 @@ def django_app(django_app_mixin, system_user): @pytest.mark.django_db def test_log(enable_stats, django_app, system_user, reset_stats): - url = reverse("api:intervention-list") + url = reverse("api:intervention-list", args=['v1']) url = f"{url}?country_name=bolivia,chad,lebanon" res = django_app.get(url) @@ -53,7 +53,7 @@ def test_log(enable_stats, django_app, system_user, reset_stats): @pytest.mark.django_db def test_threaedlog(enable_threadstats, django_app, admin_user): - url = reverse("api:intervention-list") + url = reverse("api:intervention-list", args=['v1']) url = f"{url}?country_name=bolivia,chad,lebanon" res = django_app.get(url) From 94ee309fc90ffe47f1140ba8f5a1fb9ee4ccda10 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 18:33:43 +0100 Subject: [PATCH 24/43] fixes tests --- tests/datamart/test_data_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datamart/test_data_admin.py b/tests/datamart/test_data_admin.py index 269a83839..a6b53be99 100644 --- a/tests/datamart/test_data_admin.py +++ b/tests/datamart/test_data_admin.py @@ -50,7 +50,7 @@ def test_pmpindicators_detail_supersuser(django_app, admin_user, settings): res = django_app.get(url, user=admin_user) assert res.status_code == 200 res = res.form.submit().follow() - assert res.status_code == 302 + assert res.status_code == 200 def test_pmpindicators_refresh(django_app, admin_user): From 150c27f48b51b695d85134d8c69d9544a4cb164e Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 20:35:36 +0100 Subject: [PATCH 25/43] updates social_auth config --- src/etools_datamart/config/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 79f82fc83..bdcea50f5 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -423,6 +423,7 @@ SOCIAL_AUTH_SANITIZE_REDIRECTS = False SOCIAL_AUTH_URL_NAMESPACE = 'social' SOCIAL_AUTH_WHITELISTED_DOMAINS = ['unicef.org', ] +SOCIAL_AUTH_REVOKE_TOKENS_ON_DISCONNECT = True SOCIAL_AUTH_PIPELINE = ( 'social_core.pipeline.social_auth.social_details', 'unicef_security.azure.get_unicef_user', @@ -444,6 +445,9 @@ SOCIAL_AUTH_AZUREAD_TENANT_OAUTH2_SECRET = env.str('AZURE_CLIENT_SECRET') SOCIAL_AUTH_AZUREAD_TENANT_OAUTH2_TENANT_ID = env.str('AZURE_TENANT') SOCIAL_AUTH_AZUREAD_OAUTH2_KEY = env.str('AZURE_CLIENT_ID') +SOCIAL_AUTH_AZUREAD_OAUTH2_RESOURCE = 'https://graph.microsoft.com/' +SOCIAL_AUTH_USER_MODEL = 'unicef_security.User' + # POLICY = os.getenv('AZURE_B2C_POLICY_NAME', "b2c_1A_UNICEF_PARTNERS_signup_signin") SCOPE = ['openid', 'email'] IGNORE_DEFAULT_SCOPE = True From 7729bd83acb0b2f57dea79a0e9fedd4b3e8da5e3 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 20:51:41 +0100 Subject: [PATCH 26/43] fixes GroupACL --- src/etools_datamart/celery.py | 4 ++++ src/unicef_rest_framework/permissions.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index c89fb3eb1..3a0bf44b6 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -90,6 +90,8 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): @task_postrun.connect def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): from django.utils import timezone + # from unicef_rest_framework.models import Service + if not hasattr(sender, 'linked_model'): return try: @@ -114,4 +116,6 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st from etools_datamart.apps.etl.models import EtlTask EtlTask.objects.update_or_create(task=task.name, defaults=defs) + + # Service.objects.invalidate_cache() app.timers[task.name] = cost diff --git a/src/unicef_rest_framework/permissions.py b/src/unicef_rest_framework/permissions.py index c88cac98d..37835c2f9 100644 --- a/src/unicef_rest_framework/permissions.py +++ b/src/unicef_rest_framework/permissions.py @@ -17,7 +17,7 @@ def get_acl(self, request, view): try: return UserAccessControl.objects.get(service__viewset=fqn(view), user=request.user) - except GroupAccessControl.DoesNotExist: + except UserAccessControl.DoesNotExist: return GroupAccessControl.objects.get(service__viewset=fqn(view), group__user=request.user) From 035f06e7f29a0dd0ad8bea1754204c5dc1f76a4f Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 23:14:21 +0100 Subject: [PATCH 27/43] helper to massive creation group ACL --- Pipfile | 2 +- Pipfile.lock | 94 +++++++++---------- src/etools_datamart/celery.py | 5 +- src/etools_datamart/config/settings.py | 3 +- src/unicef_rest_framework/admin/acl.py | 65 ++++++++++++- .../groupaccesscontrol/add.html | 28 ++++++ src/unicef_security/azure.py | 35 +++++++ 7 files changed, 177 insertions(+), 55 deletions(-) create mode 100644 src/unicef_rest_framework/templates/admin/unicef_rest_framework/groupaccesscontrol/add.html diff --git a/Pipfile b/Pipfile index 7edcc1229..4c9000b56 100644 --- a/Pipfile +++ b/Pipfile @@ -37,7 +37,7 @@ django-model-utils = "*" python-social-auth = "*" social-auth-app-django = "*" django-db-logging = "*" -cryptography = "*" +cryptography = "==2.4.1" #rest-social-auth = "*" "django-rest-framework-social-oauth2" = "*" django-countries = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 33ee8cae9..f972c8bac 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8f86868b9753fa670c7c91c6a2eecc06abeb689cd5475fe1baf332d13dbe8ebc" + "sha256": "a36e8c115d54312043b8b85cfd29e955db56c7cd098ef082b4d13c5aa6158079" }, "pipfile-spec": 6, "requires": { @@ -720,28 +720,28 @@ }, "cryptography": { "hashes": [ - "sha256:05a6052c6a9f17ff78ba78f8e6eb1d777d25db3b763343a1ae89a7a8670386dd", - "sha256:0eb83a24c650a36f68e31a6d0a70f7ad9c358fa2506dc7b683398b92e354a038", - "sha256:0ff4a3d6ea86aa0c9e06e92a9f986de7ee8231f36c4da1b31c61a7e692ef3378", - "sha256:1699f3e916981df32afdd014fb3164db28cdb61c757029f502cb0a8c29b2fdb3", - "sha256:1b1f136d74f411f587b07c076149c4436a169dc19532e587460d9ced24adcc13", - "sha256:21e63dd20f5e5455e8b34179ac43d95b3fb1ffa54d071fd2ed5d67da82cfe6dc", - "sha256:2454ada8209bbde97065453a6ca488884bbb263e623d35ba183821317a58b46f", - "sha256:3cdc5f7ca057b2214ce4569e01b0f368b3de9d8ee01887557755ccd1c15d9427", - "sha256:418e7a5ec02a7056d3a4f0c0e7ea81df374205f25f4720bb0e84189aa5fd2515", - "sha256:471a097076a7c4ab85561d7fa9a1239bd2ae1f9fd0047520f13d8b340bf3210b", - "sha256:5ecaf9e7db3ca582c6de6229525d35db8a4e59dc3e8a40a331674ed90e658cbf", - "sha256:63b064a074f8dc61be81449796e2c3f4e308b6eba04a241a5c9f2d05e882c681", - "sha256:6afe324dfe6074822ccd56d80420df750e19ac30a4e56c925746c735cf22ae8b", - "sha256:70596e90398574b77929cd87e1ac6e43edd0e29ba01e1365fed9c26bde295aa5", - "sha256:70c2b04e905d3f72e2ba12c58a590817128dfca08949173faa19a42c824efa0b", - "sha256:8908f1db90be48b060888e9c96a0dee9d842765ce9594ff6a23da61086116bb6", - "sha256:af12dfc9874ac27ebe57fc28c8df0e8afa11f2a1025566476b0d50cdb8884f70", - "sha256:b4fc04326b2d259ddd59ed8ea20405d2e695486ab4c5e1e49b025c484845206e", - "sha256:da5b5dda4aa0d5e2b758cc8dfc67f8d4212e88ea9caad5f61ba132f948bab859" - ], - "index": "pypi", - "version": "==2.4.2" + "sha256:02915ee546b42ce513e8167140e9937fc4c81a06a82216e086ccce51f347948a", + "sha256:03cc8bc5a69ae3d44acf1a03facdb7c10a94c67907862c563e10efe72b737977", + "sha256:07f76bde6815c55195f3b3812d35769cc7c765144c0bb71ae45e02535d078591", + "sha256:13eac1c477b9af7e9a9024369468d08aead6ad78ed599d163ad046684474364b", + "sha256:179bfb585c5efc87ae0e665770e4896727b92dbc1f810c761b1ebf8363e2fec8", + "sha256:414af0ba308e74c1f8bc5b11befc86cb66b10be8959547786f64258830d2096f", + "sha256:41a1ca14f255df8c44dd22c6006441d631d1589104045ec7263cc47e9772f41a", + "sha256:54947eb98bc4eef99ddf49f45d2694ea5a3929ab3edc9806ad01967368594d82", + "sha256:5bac7a2abda07d0c3c8429210349bb54149ad8940dc7bcffedcd56519b410a3c", + "sha256:7f41af8c586bed9f59cfe8832d818b3b75c860d7025da9cd2db76875a72ff785", + "sha256:8004fae1b3cb2dbd90a011ad972e49a7e78a871b89c70cc7213cf4ebd2532bcb", + "sha256:8e0eccadc3b465e12c50a5b8fb4d39cf401b44d7bb9936c70fddb5e5aaf740d5", + "sha256:95b4741722269cfdc134fec23b7ae6503ee2aea83d0924cfee6d6ec54cd42d8e", + "sha256:a06f5aa6d7a94531dfe82eb2972e669258c452fe9cf88f76116610de4c789785", + "sha256:b0833d27c7eb536bc27323a1e8e22cb39ebac78c4ef3be0167ba40f447344808", + "sha256:b72dec675bc59a01edc96616cd48ec465b714481caa0938c8bbca5d18f17d5df", + "sha256:c800ddc23b5206ce025f23225fdde89cdc0e64016ad914d5be32d1f602ce9495", + "sha256:c980c8c313a5e014ae12e2245e89e7b30427e5a98cbb88afe478ecae85f3abaa", + "sha256:e85b410885addaeb31a867eabcefc9ef4a7e904ad45eac9e60a763a54b244626" + ], + "index": "pypi", + "version": "==2.4.1" }, "defusedxml": { "hashes": [ @@ -1312,30 +1312,30 @@ }, "ruamel.yaml": { "hashes": [ - "sha256:03d7398452c45bee100f3030bba91c84ef2eb2d8a1607791a170ceb083f81624", - "sha256:05ba73f85dc4fe17964903dd90e1074efaf3299b8db91d66cf0970581f92d9fb", - "sha256:13432f257b626923a389ceb9f13c64c8453f3b58c2a611066ca9cc5582d6d078", - "sha256:161bfaf2379c3fb9d351be3dc24b74801985fe44cc1f031cee97b78d4a33a843", - "sha256:343d4f537e0330dad00fe853d8894fd87c46604024c76c3815b0168eb12062dc", - "sha256:39480f2182b164453405e8dd2bae05108b0da5cbba494f16dffa43e64e7d60dd", - "sha256:3c67edfd639e73e15736e1f92cbf66e0afa7386628388113ea9db023274e15b4", - "sha256:4102b7f97cb6f589c669cd52a033d2fb8f3944a925439baa75fc107d1460499b", - "sha256:45efad27ca82d055a0b2916f63b5a277e6b1b8b955390723cbc4b0edec8d0fb7", - "sha256:4642c7425446bfd3dee66f333050c8ed3cd6c0e0acfa7d63e27c6f871b29d991", - "sha256:4bc5007df709bed082652acc4268c3453927a1223ff9ef42646c1a9d8b4a1416", - "sha256:6daff7cf84a4ec6dc13bf67cce793916eb0f3cbae2648a6bdcc5acf424cffabb", - "sha256:72c6083a1deab4bec019460714d35ab5eb4c0d89fc4fc806306da541f166d062", - "sha256:7c499e1764bbf45b0e9de664e54984ea829506566a03fc96aff47ac58a3f30cb", - "sha256:85cdfff4f2d01758083b07957d3e72fcfdb9f740ea44c5e6f4b815bb41ed087e", - "sha256:953373ab6bce8bb48e7a201eea00c3e989cacc13fe9a5dd10fd9b6ebe1e74fba", - "sha256:a5fdaf4c57d9b2e2ef3e911c6aef61de503a292b47df06a072e2da0a93213f30", - "sha256:abd7c4277b7ccd13e2eafb0094322c9ad22803d86ffa7aaabcddb30253fdf309", - "sha256:c97d7d30a83baf84fbeb3adafec173d05e831bf2281c24e211847aa88775ce07", - "sha256:d0a51f929fa8cffc845e58c5c1cca59d53d1dd5f356b24f55a010eff1f13434f", - "sha256:e4c8857b4883c12f5dcfa6b9d86d4f495a001d93e071cb6e855bb272594e714f", - "sha256:ff89559d2088688ee67b1e9ed96ace44110fa85b47821213f859b65d46310bda" - ], - "version": "==0.15.79" + "sha256:10079d03b5c93d54be90e4fe23d4b1f32502d7da98077e2a746c216bedba3d75", + "sha256:1ff2289958e09fac2aa573e7a9fb9c953ddf89f67c3a42693394920f72001348", + "sha256:3b8af255839c39d3dfd0dcb82db349f38db28a2f7adbe05387bea87de15ac146", + "sha256:418ee849362ad59c19064af3ce09666d0898969eadb25964b693827fb68cfeca", + "sha256:4f203351575dba0829c7b1e5d376d08cf5f58e4a2b844e8ce552b3e41cd414e6", + "sha256:5421c3fb144d6e1de0dc00d8a1f919f558c3156c48aa7aa2acbd7754530dfeb7", + "sha256:59a17a6225d6d60150647d2fd707fcb1ee54ea31dd0048ea120c7bf2c9093451", + "sha256:6acdf9d7bea6ff8541e96e4694b9fd1e0728be88ef512afc28da0804590533f7", + "sha256:70ed366ad65780040f5bf5e34c75f450e3f0ca9a8b2534cfc0293c387ec1c32f", + "sha256:7e9dc8d095d952c352d9f63cab5283430c910b77793f323ce8d64921f65aaa53", + "sha256:8344a08555f8494ae16a2e4f445e5bfce80f82010d9e5091c870aadee5c4b14a", + "sha256:90b1f9ed3893b0713e0bc47cfa93be72ddb6d5a969c31979d36c2854dc8b87dd", + "sha256:93f262943089657675b336f804b721e6b76f67cc61f6bfc91b3f35f8e71a8a64", + "sha256:9996c6371bfe3051340a469037323f533bbdf2dea9b6914da27e62696c81712b", + "sha256:9d9a382be1c150d23cd1291091454ea801522629bd22531f0b2c41ab21a81deb", + "sha256:adb57424caeb48f8cf1c937e4c571e5720e38601a77dd87d4c1178d406a72821", + "sha256:b5147c0919c6ce29c278afe21bf64c3f099afc271364a038c5e59fcf7bb672fb", + "sha256:b71cfb6c90f7b6db26842923e5c178c3ad232577f6097ebca3651be366c84b36", + "sha256:d9a82d37a4b006b12e09550ff57f7edb5ec987f0d9128fc44f590d56683aa97d", + "sha256:dc76688fb7994bf9c2370e28238bd56f9fe7e1d02675f3b1e06fc35967375869", + "sha256:fa5ae31d1aea11b528f5987e43194abdfcb44a5a259172221097eb6f74bd0965", + "sha256:ff6046de0ed29c3f3e6ba2ce164a85781af16dd49049af1b0402b8a6d15a25ca" + ], + "version": "==0.15.80" }, "six": { "hashes": [ diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 3a0bf44b6..686625bcd 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -106,7 +106,9 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st defs['results'] = retval.as_dict() if retval.created > 0 or retval.updated > 0: defs['last_changes'] = timezone.now() - # if defs. + service = sender.linked_model.get_service() + service.invalidate_cache() + defs['last_success'] = timezone.now() else: if not isinstance(retval, dict): @@ -116,6 +118,5 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st from etools_datamart.apps.etl.models import EtlTask EtlTask.objects.update_or_create(task=task.name, defaults=defs) - # Service.objects.invalidate_cache() app.timers[task.name] = cost diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index bdcea50f5..9b4c3ee6f 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import datetime -import os from pathlib import Path import environ @@ -406,7 +405,7 @@ 'JWT_DECODE_HANDLER': 'rest_framework_jwt.utils.jwt_decode_handler', # Keys will be set in core.apps.Config.ready() - 'JWT_PUBLIC_KEY': os.environ, + 'JWT_PUBLIC_KEY': '?', # 'JWT_PRIVATE_KEY': wallet.get_private(), # 'JWT_PRIVATE_KEY': None, 'JWT_ALGORITHM': 'RS256', diff --git a/src/unicef_rest_framework/admin/acl.py b/src/unicef_rest_framework/admin/acl.py index c60548bc9..66ffc0d75 100644 --- a/src/unicef_rest_framework/admin/acl.py +++ b/src/unicef_rest_framework/admin/acl.py @@ -2,10 +2,16 @@ import logging +from admin_extra_urls.extras import ExtraUrlMixin, link from django import forms from django.contrib import admin -from unicef_rest_framework.models import UserAccessControl -from unicef_rest_framework.models.acl import GroupAccessControl +from django.contrib.admin import widgets +from django.contrib.admin.helpers import AdminForm +from django.contrib.auth.models import Group +from django.contrib.postgres.forms import SimpleArrayField +from django.template.response import TemplateResponse +from unicef_rest_framework.models import Service, UserAccessControl +from unicef_rest_framework.models.acl import AbstractAccessControl, GroupAccessControl logger = logging.getLogger(__name__) @@ -32,11 +38,64 @@ def get_queryset(self, request): return super(UserAccessControlAdmin, self).get_queryset(request).select_related(*self.raw_id_fields) -class GroupAccessControlAdmin(admin.ModelAdmin): +class GroupAccessControlForm(forms.Form): + group = forms.ModelChoiceField(queryset=Group.objects.all()) + policy = forms.ChoiceField(choices=AbstractAccessControl.POLICIES) + services = forms.ModelMultipleChoiceField(queryset=Service.objects.all(), + widget=widgets.FilteredSelectMultiple('Services', False) + ) + rate = forms.CharField(max_length=100) + serializers = SimpleArrayField(forms.CharField(), + max_length=255) + + +class GroupAccessControlAdmin(ExtraUrlMixin, admin.ModelAdmin): list_display = ('group', 'service', 'rate', 'serializers', 'policy') list_filter = ('group', 'policy', 'service') search_fields = ('group', 'service',) form = GroupACLAdminForm + autocomplete_fields = ('group',) + + # filter_horizontal = ('services',) def get_queryset(self, request): return super(GroupAccessControlAdmin, self).get_queryset(request).select_related(*self.raw_id_fields) + + @link() + def add_acl(self, request): + opts = self.model._meta + ctx = { + 'opts': opts, + 'add': False, + 'has_view_permission': True, + 'has_editable_inline_admin_formsets': True, + 'app_label': opts.app_label, + 'change': True, + 'is_popup': False, + 'save_as': False, + 'media': self.media, + 'has_delete_permission': False, + 'has_add_permission': False, + 'has_change_permission': True, + } + if request.method == 'POST': + form = GroupAccessControlForm(request.POST) + if form.is_valid(): + services = form.cleaned_data.pop('services') + for service in services: + GroupAccessControl.objects.get_or_create(service=service, + **form.cleaned_data) + self.message_user(request, 'ACLs created') + + else: + form = GroupAccessControlForm(initial={'rate': '*', + 'policy': AbstractAccessControl.POLICY_ALLOW, + 'serializers': 'std'}) + ctx['adminform'] = AdminForm(form, + [(None, {'fields': [['group', + 'policy'], + 'services', + ['rate', 'serializers']]})], + {}) + ctx['media'] = self.media + form.media + return TemplateResponse(request, 'admin/unicef_rest_framework/groupaccesscontrol/add.html', ctx) diff --git a/src/unicef_rest_framework/templates/admin/unicef_rest_framework/groupaccesscontrol/add.html b/src/unicef_rest_framework/templates/admin/unicef_rest_framework/groupaccesscontrol/add.html new file mode 100644 index 000000000..2f0cdaf9e --- /dev/null +++ b/src/unicef_rest_framework/templates/admin/unicef_rest_framework/groupaccesscontrol/add.html @@ -0,0 +1,28 @@ +{% extends "admin/change_form.html" %}{% load i18n admin_urls static admin_modify %} +{% block breadcrumbs %} +
      + {% trans 'Home' %} + › {{ opts.app_config.verbose_name }} + › {{ opts.verbose_name_plural|capfirst }} + › Enable multiple services +
      +{% endblock %} +{% block object-tools %}{% endblock %} + +{#{% block content %}#} +{#

      Grant Access multiple services

      #} +{#
      #} +{# {% csrf_token %}#} +{#
      #} +{#
      {{ form.services }}
      #} +{#
      #} +{# #} +{# {{ form }}#} +{#
      #} +{#
      #} +{#
      #} +{#{% endblock content %}#} + +{% block submit_buttons_bottom %} + +{% endblock %} diff --git a/src/unicef_security/azure.py b/src/unicef_security/azure.py index 62804459f..8e5c61730 100644 --- a/src/unicef_security/azure.py +++ b/src/unicef_security/azure.py @@ -1,4 +1,7 @@ +import base64 +import json import logging +import os import requests from constance import config as constance @@ -7,6 +10,9 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import Group from django.core.cache import cache +from jwt import decode as jwt_decode, DecodeError, ExpiredSignature +from social_core.backends.azuread_tenant import AzureADTenantOAuth2 +from social_core.exceptions import AuthTokenError from social_django.models import UserSocialAuth from . import config @@ -28,6 +34,35 @@ ADMIN_EMAILS = [i[1] for i in settings.ADMINS] +class AzureADTenantOAuth2Ext(AzureADTenantOAuth2): + def user_data(self, access_token, *args, **kwargs): + response = kwargs.get('response') + id_token = response.get('id_token') + + # decode the JWT header as JSON dict + jwt_header = json.loads( + base64.b64decode(id_token.split('.', 1)[0]).decode() + ) + + # get key id and algorithm + key_id = jwt_header['kid'] + algorithm = jwt_header['alg'] + + try: + # retrieve certificate for key_id + certificate = self.get_certificate(key_id) + + return jwt_decode( + id_token, + verify=os.environ.get('OAUTH2_VERIFY'), + key=certificate.public_key(), + algorithms=algorithm, + audience=self.setting('KEY') + ) + except (DecodeError, ExpiredSignature) as error: + raise AuthTokenError(self, error) + + def default_group(**kwargs): is_new = kwargs.get('is_new', False) user = kwargs.get('user', None) From 90efb3e80756dce75475ee49a55d683010e675e5 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 23:32:34 +0100 Subject: [PATCH 28/43] custom OAuth Backend to bypass certificate issue --- src/etools_datamart/config/settings.py | 3 ++- src/unicef_security/azure.py | 11 +++++++---- tests/_test_lib/settings_test.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 9b4c3ee6f..06c9f7dd4 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -173,7 +173,8 @@ ] AUTHENTICATION_BACKENDS = [ - 'social_core.backends.azuread_tenant.AzureADTenantOAuth2', + # 'social_core.backends.azuread_tenant.AzureADTenantOAuth2', + 'unicef_security.azure.AzureADTenantOAuth2Ext', 'django.contrib.auth.backends.ModelBackend', ] diff --git a/src/unicef_security/azure.py b/src/unicef_security/azure.py index 8e5c61730..292adeac6 100644 --- a/src/unicef_security/azure.py +++ b/src/unicef_security/azure.py @@ -47,15 +47,18 @@ def user_data(self, access_token, *args, **kwargs): # get key id and algorithm key_id = jwt_header['kid'] algorithm = jwt_header['alg'] - + verify = os.environ.get('OAUTH2_VERIFY', False) + key = '' try: # retrieve certificate for key_id - certificate = self.get_certificate(key_id) + if verify: + certificate = self.get_certificate(key_id) + key = certificate.public_key() return jwt_decode( id_token, - verify=os.environ.get('OAUTH2_VERIFY'), - key=certificate.public_key(), + verify=verify, + key=key, algorithms=algorithm, audience=self.setting('KEY') ) diff --git a/tests/_test_lib/settings_test.py b/tests/_test_lib/settings_test.py index 81d5b423a..414414610 100644 --- a/tests/_test_lib/settings_test.py +++ b/tests/_test_lib/settings_test.py @@ -1,3 +1,4 @@ +import os import random from etools_datamart.config.settings import * # noqa From de33ba82d41be14ef03da1f9be2f93ae53aab014 Mon Sep 17 00:00:00 2001 From: sax Date: Mon, 26 Nov 2018 23:58:31 +0100 Subject: [PATCH 29/43] fixes tests --- src/etools_datamart/celery.py | 4 ++-- src/unicef_rest_framework/permissions.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 686625bcd..1c0d9678a 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -106,8 +106,8 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st defs['results'] = retval.as_dict() if retval.created > 0 or retval.updated > 0: defs['last_changes'] = timezone.now() - service = sender.linked_model.get_service() - service.invalidate_cache() + for service in sender.linked_model.linked_services: + service.invalidate_cache() defs['last_success'] = timezone.now() else: diff --git a/src/unicef_rest_framework/permissions.py b/src/unicef_rest_framework/permissions.py index 37835c2f9..b26e80672 100644 --- a/src/unicef_rest_framework/permissions.py +++ b/src/unicef_rest_framework/permissions.py @@ -42,6 +42,6 @@ def has_permission(self, request, view): raise PermissionDenied(f"Forbidden serializer '{requested_serializer}'") return True - except (UserAccessControl.DoesNotExist): + except (GroupAccessControl.DoesNotExist): logger.error(f"User '{request.user}' does not have grants for '{fqn(view)}'") return False From 799e2cce862fe7bedc2edb6aade5dc04f2794c19 Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 28 Nov 2018 20:11:24 +0100 Subject: [PATCH 30/43] add email notifications --- Makefile | 8 +- Pipfile | 3 + Pipfile.lock | 52 +++++++++-- .../api/endpoints/datamart/serializers.py | 2 +- .../apps/data/migrations/0001_initial.py | 2 +- .../apps/etl/migrations/0001_initial.py | 2 +- .../init/management/commands/init-setup.py | 41 ++++++++- .../apps/subscriptions/__init__.py | 0 .../apps/subscriptions/admin.py | 12 +++ .../subscriptions/migrations/0001_initial.py | 25 +++++ .../migrations/0002_auto_20181128_1410.py | 27 ++++++ .../apps/subscriptions/migrations/__init__.py | 0 .../apps/subscriptions/models.py | 56 ++++++++++++ .../apps/subscriptions/tasks.py | 8 ++ .../templates/subscription_select.html | 6 ++ .../subscriptions/templatetags/__init__.py | 0 .../templatetags/subscriptions.py | 21 +++++ .../apps/subscriptions/urls.py | 36 ++++++++ .../apps/subscriptions/views.py | 51 +++++++++++ .../apps/tracking/migrations/0001_initial.py | 2 +- ...126_1313.py => 0002_auto_20181128_1410.py} | 4 +- .../apps/web/static/jquery-3.3.1.min.js | 2 + src/etools_datamart/apps/web/static/style.css | 6 +- .../apps/web/static/style.css.map | 2 +- .../apps/web/static/style.scss | 7 +- .../apps/web/templates/admin/+base.html | 91 ------------------- .../apps/web/templates/base.html | 5 +- .../apps/web/templates/monitor.html | 38 ++++++-- src/etools_datamart/apps/web/views.py | 6 +- src/etools_datamart/celery.py | 21 +++-- src/etools_datamart/config/settings.py | 29 +++++- src/etools_datamart/config/urls.py | 20 ++-- src/unicef_rest_framework/admin/service.py | 2 +- .../migrations/0001_initial.py | 2 +- ...126_1313.py => 0002_auto_20181128_1410.py} | 15 ++- src/unicef_rest_framework/models/service.py | 18 +++- .../migrations/0001_initial.py | 2 +- tests/test_subscription.py | 64 +++++++++++++ tests/test_views.py | 2 +- 39 files changed, 529 insertions(+), 161 deletions(-) create mode 100644 src/etools_datamart/apps/subscriptions/__init__.py create mode 100644 src/etools_datamart/apps/subscriptions/admin.py create mode 100644 src/etools_datamart/apps/subscriptions/migrations/0001_initial.py create mode 100644 src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py create mode 100644 src/etools_datamart/apps/subscriptions/migrations/__init__.py create mode 100644 src/etools_datamart/apps/subscriptions/models.py create mode 100644 src/etools_datamart/apps/subscriptions/tasks.py create mode 100644 src/etools_datamart/apps/subscriptions/templates/subscription_select.html create mode 100644 src/etools_datamart/apps/subscriptions/templatetags/__init__.py create mode 100644 src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py create mode 100644 src/etools_datamart/apps/subscriptions/urls.py create mode 100644 src/etools_datamart/apps/subscriptions/views.py rename src/etools_datamart/apps/tracking/migrations/{0002_auto_20181126_1313.py => 0002_auto_20181128_1410.py} (97%) create mode 100644 src/etools_datamart/apps/web/static/jquery-3.3.1.min.js delete mode 100644 src/etools_datamart/apps/web/templates/admin/+base.html rename src/unicef_rest_framework/migrations/{0002_auto_20181126_1313.py => 0002_auto_20181128_1410.py} (91%) create mode 100644 tests/test_subscription.py diff --git a/Makefile b/Makefile index 71c9b7b61..ddf55f8bd 100644 --- a/Makefile +++ b/Makefile @@ -60,10 +60,10 @@ urf: demo: - pipenv run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid & - pipenv run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid & - pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & - pipenv run docker run -d -p 5555:5555 --name datamart-flower --rm saxix/flower + PYTHONPATH=./src pipenv run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid & + PYTHONPATH=./src pipenv run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid & + PYTHONPATH=./src pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & + pipenv run docker run -d -p 5555:5555 -e CELERY_BROKER_URL=$CELERY_BROKER_URL --name datamart-flower --rm saxix/flower stop-demo: - kill `cat gunicorn.pid` diff --git a/Pipfile b/Pipfile index 4c9000b56..abf8adc6a 100644 --- a/Pipfile +++ b/Pipfile @@ -50,6 +50,9 @@ djangorestframework-yaml = "*" django-storages = {extras = ["azure"], version = "*"} onedrivesdk = "*" azure-storage = "*" +django-basicauth = "*" +django-post-office = "*" +django-celery-email = "*" [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index f972c8bac..a9b18c25c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a36e8c115d54312043b8b85cfd29e955db56c7cd098ef082b4d13c5aa6158079" + "sha256": "27eb754085af424f6ca84bd7961e8b161086e7a5cb591de162845faf06b98390" }, "pipfile-spec": 6, "requires": { @@ -412,10 +412,10 @@ }, "azure-mgmt-network": { "hashes": [ - "sha256:bf2b459d055083eb5c8883b66828c1e7d917f2d3eef98bf65417022450c6ea7b", - "sha256:ef8593a3a911417959f70df25f15344eadcdf1bfc7c0706a62d82ede2d1fab31" + "sha256:37c11c131ec55bf13216d62b058786491c8dd5700ffe19fec68b4557b87408a6", + "sha256:4ee99a9b1b648f31c0fb156a7cda7e680f29890465dd7863c9fcccc6aed53f71" ], - "version": "==2.3.0" + "version": "==2.4.0" }, "azure-mgmt-notificationhubs": { "hashes": [ @@ -766,6 +766,21 @@ "index": "pypi", "version": "==1.1.0" }, + "django-appconf": { + "hashes": [ + "sha256:6a4d9aea683b4c224d97ab8ee11ad2d29a37072c0c6c509896dd9857466fb261", + "sha256:ddab987d14b26731352c01ee69c090a4ebfc9141ed223bef039d79587f22acd9" + ], + "version": "==1.0.2" + }, + "django-basicauth": { + "hashes": [ + "sha256:0ceff44ebc129eb7f8bde212a2f663210796ea1ba6e00d944cba50d4ef326f79", + "sha256:740a176e0bbeed8fd267e165a4373aa51a346258fb87479670dd7f6846f118d1" + ], + "index": "pypi", + "version": "==0.5.1" + }, "django-braces": { "hashes": [ "sha256:a457d74ea29478123c0c4652272681b3cea0bf1232187fd9f9b6f1d97d32a890", @@ -781,6 +796,14 @@ "index": "pypi", "version": "==1.1.1" }, + "django-celery-email": { + "hashes": [ + "sha256:1b2e0e31c6266007463befdc23934696fc93dcf320dfc85b8bb6b063cfe9558a", + "sha256:e5f9122c02ec58d3e49653475ad1b8612fd752681ce2f006d9c0792c57046283" + ], + "index": "pypi", + "version": "==2.0.1" + }, "django-concurrency": { "hashes": [ "sha256:233d23a8751989df6db2886957a8fbcc2408a1f16bb28262aab8a538d756d9d2" @@ -864,6 +887,14 @@ "index": "pypi", "version": "==2.0" }, + "django-post-office": { + "hashes": [ + "sha256:207b663a05d5d6a62765eb30081093837272a888cf00557d89d0e6f467928871", + "sha256:827937a944fe47cea393853069cd9315d080298c8ddb0faf787955d6aa51a030" + ], + "index": "pypi", + "version": "==3.1.0" + }, "django-redis": { "hashes": [ "sha256:af0b393864e91228dd30d8c85b5c44d670b5524cb161b7f9e41acc98b6e5ace7", @@ -1067,6 +1098,13 @@ ], "version": "==2.10" }, + "jsonfield": { + "hashes": [ + "sha256:a0a7fdee736ff049059409752b045281a225610fecbda9b9bd588ba976493c12", + "sha256:beb1cd4850d6d6351c32daefcb826c01757744e9c863228a642f87a1a4acb834" + ], + "version": "==2.0.2" + }, "jsonpointer": { "hashes": [ "sha256:381b613fd1afd65376fb28948c4744f035e47ab049a9fdde0c48cc1c30b68559", @@ -1648,10 +1686,10 @@ }, "importlib-metadata": { "hashes": [ - "sha256:36b02c84f9001adf65209fefdf951be8e9014a95eab9938c0779ad5670359b1c", - "sha256:60b6481a72908c93ccb707abeb926fb5a15319b9e6f0b76639a718837ee12de0" + "sha256:28fba9f65e5415a691dd254cdb602bcc4d6f738e68407ad251651db358b63bcf", + "sha256:4a545e6125dc72b4ad98201ea3f40f92e8126e3a19667352b3a134d22b8bc74f" ], - "version": "==0.6" + "version": "==0.7" }, "importlib-resources": { "hashes": [ diff --git a/src/etools_datamart/api/endpoints/datamart/serializers.py b/src/etools_datamart/api/endpoints/datamart/serializers.py index 4b7539a54..b1fb17ba5 100644 --- a/src/etools_datamart/api/endpoints/datamart/serializers.py +++ b/src/etools_datamart/api/endpoints/datamart/serializers.py @@ -31,13 +31,13 @@ class UserStatsSerializer(serializers.ModelSerializer): class Meta: model = models.UserStats exclude = () + read_only = ['last_modify_date', ] def get_month(self, obj): return datetime.strftime(obj.month._date, '%b %Y') class HACTSerializer(serializers.ModelSerializer): - class Meta: model = models.HACT exclude = () diff --git a/src/etools_datamart/apps/data/migrations/0001_initial.py b/src/etools_datamart/apps/data/migrations/0001_initial.py index 80f81cdd6..ca15c1e2b 100644 --- a/src/etools_datamart/apps/data/migrations/0001_initial.py +++ b/src/etools_datamart/apps/data/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-26 13:13 +# Generated by Django 2.1.3 on 2018-11-28 14:10 import django.contrib.postgres.fields.jsonb import month_field.models diff --git a/src/etools_datamart/apps/etl/migrations/0001_initial.py b/src/etools_datamart/apps/etl/migrations/0001_initial.py index 0cf3dc2e0..dba214ef8 100644 --- a/src/etools_datamart/apps/etl/migrations/0001_initial.py +++ b/src/etools_datamart/apps/etl/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-26 13:13 +# Generated by Django 2.1.3 on 2018-11-28 14:10 import django.contrib.postgres.fields.jsonb import django.db.models.deletion diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index 70ab9a837..a7edb37ec 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -9,8 +9,9 @@ from django.core.management import call_command from django.core.management.base import BaseCommand from django.utils.module_loading import import_string -from django_celery_beat.models import CrontabSchedule, PeriodicTask +from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask from humanize import naturaldelta +from post_office.models import EmailTemplate from redisboard.models import RedisServer from strategy_field.utils import fqn from unicef_rest_framework.models.acl import GroupAccessControl @@ -18,6 +19,29 @@ from etools_datamart.apps.etl.models import EtlTask from etools_datamart.celery import app +MAIL = r"""Dear {{user.label}}, + +On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`. +Please visit {{base_url}}{{service.endpoint}} + +— +You are receiving this because you are subscribed to this thread. +To unsubscribe, change your preferences in {{base_url}}{% url 'monitor' %} +""" + +MAIL_HTML = r"""
      Dear {{user.label}},
      +
       
      +
      On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`.
      +
      You can view data following this link + or download the as excel
      +
       
      +
       
      +
       
      +
      -
      +
      You are receiving this because you are subscribed to this thread.
      +
      To unsubscribe, change your preferences in Datamart Monitor
      +""" + class Command(BaseCommand): help = "My shiny new management command." @@ -135,6 +159,7 @@ def handle(self, *args, **options): if options['tasks'] or _all or options['refresh']: midnight, __ = CrontabSchedule.objects.get_or_create(minute=0, hour=0) + every_minute, __ = IntervalSchedule.objects.get_or_create(every=1, period=IntervalSchedule.MINUTES) tasks = app.get_all_etls() counters = {True: 0, False: 0} @@ -153,10 +178,22 @@ def handle(self, *args, **options): self.stdout.write( f"{PeriodicTask.objects.count()} tasks found. {counters[True]} new. {counters[False]} deleted") + PeriodicTask.objects.get_or_create(task='send_queued_mail', + defaults={'name': 'process mail queue', + 'interval': every_minute}) + + EmailTemplate.objects.get_or_create(name='dataset_changed', + defaults=dict(subject='Dataset changed', + content=MAIL, + html_content=MAIL_HTML)) + if options['refresh']: self.stdout.write("Refreshing datamart...") for task in PeriodicTask.objects.all()[1:]: - etl = import_string(task.task) + try: + etl = import_string(task.task) + except ImportError: + continue self.stdout.write(f"Running {task.name}...", ending='\r') self.stdout.flush() diff --git a/src/etools_datamart/apps/subscriptions/__init__.py b/src/etools_datamart/apps/subscriptions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/subscriptions/admin.py b/src/etools_datamart/apps/subscriptions/admin.py new file mode 100644 index 000000000..854a0c404 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/admin.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +from admin_extra_urls.extras import ExtraUrlMixin +from django.contrib import admin +from django.contrib.admin import register + +from . import models + + +@register(models.Subscription) +class SubscriptionAdmin(ExtraUrlMixin, admin.ModelAdmin): + list_display = ('user', 'content_type', 'type') + list_filter = ('user', 'content_type', 'type') diff --git a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py new file mode 100644 index 000000000..c783b380b --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py @@ -0,0 +1,25 @@ +# Generated by Django 2.1.3 on 2018-11-28 14:10 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='Subscription', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('type', models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel')])), + ('kwargs', models.CharField(blank=True, default='', max_length=500)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + ), + ] diff --git a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py new file mode 100644 index 000000000..8276d0f3a --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py @@ -0,0 +1,27 @@ +# Generated by Django 2.1.3 on 2018-11-28 14:10 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('subscriptions', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='subscription', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='subscriptions', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterUniqueTogether( + name='subscription', + unique_together={('user', 'content_type', 'kwargs')}, + ), + ] diff --git a/src/etools_datamart/apps/subscriptions/migrations/__init__.py b/src/etools_datamart/apps/subscriptions/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/subscriptions/models.py b/src/etools_datamart/apps/subscriptions/models.py new file mode 100644 index 000000000..9a603216e --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/models.py @@ -0,0 +1,56 @@ +from crashlog.middleware import process_exception +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.db import models +from post_office import mail +from unicef_rest_framework.models import Service + +from etools_datamart.apps.etl.models import EtlTask + + +class SubscriptionManager(models.Manager): + def notify(self, model): + ct = ContentType.objects.get_for_model(model) + etl = EtlTask.objects.filter(content_type=ct).first() + + for subscription in self.filter(content_type=ct).exclude(type=Subscription.NONE): + try: + mail.send( + subscription.user.email, # List of email addresses also accepted + 'notification@datamart.unicef.io', + template='dataset_changed', # Could be an EmailTemplate instance or name + context={'subscription': subscription, + 'user': subscription.user, + 'base_url': settings.ABSOLUTE_BASE_URL, + 'verbose_name': model._meta.verbose_name, + 'etl': etl, + 'model': ct.model, + 'service': Service.objects.get(source_model=ct) + }, + ) + except Exception as e: + process_exception(e) + + +class Subscription(models.Model): + NONE = 0 + MESSAGE = 1 + EXCEL = 2 + + TYPES = ((NONE, 'None'), + (MESSAGE, 'Email'), + # (EXCEL, 'Email+Excel'), + ) + user = models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE, + related_name='subscriptions') + type = models.IntegerField(choices=TYPES) + content_type = models.ForeignKey(ContentType, models.CASCADE) + kwargs = models.CharField(max_length=500, blank=True, null=False, default='') + + objects = SubscriptionManager() + + class Meta: + unique_together = ('user', 'content_type', 'kwargs') + + def __str__(self): + return f"{self.user} {self.get_type_display()}" diff --git a/src/etools_datamart/apps/subscriptions/tasks.py b/src/etools_datamart/apps/subscriptions/tasks.py new file mode 100644 index 000000000..f0c831ad8 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/tasks.py @@ -0,0 +1,8 @@ +from django.core.management import call_command + +from etools_datamart.celery import app + + +@app.task(name='send_queued_mail') +def send_queued_mail(): + call_command('send_queued_mail') diff --git a/src/etools_datamart/apps/subscriptions/templates/subscription_select.html b/src/etools_datamart/apps/subscriptions/templates/subscription_select.html new file mode 100644 index 000000000..74608fa21 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/templates/subscription_select.html @@ -0,0 +1,6 @@ + diff --git a/src/etools_datamart/apps/subscriptions/templatetags/__init__.py b/src/etools_datamart/apps/subscriptions/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py b/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py new file mode 100644 index 000000000..39fc2a4de --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py @@ -0,0 +1,21 @@ +from django import template + +from etools_datamart.apps.subscriptions.models import Subscription + +register = template.Library() + + +@register.filter +def get_for_task(subscriptions): + return 1 + + +@register.inclusion_tag('subscription_select.html', takes_context=True) +def subscription_select(context, task): + user = context['user'] + s = Subscription.objects.filter(content_type=task.content_type, + kwargs='', + user=user).first() + return {'options': Subscription.TYPES, + 'task': task, + 'subscription': s} diff --git a/src/etools_datamart/apps/subscriptions/urls.py b/src/etools_datamart/apps/subscriptions/urls.py new file mode 100644 index 000000000..1df0179d1 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/urls.py @@ -0,0 +1,36 @@ +import codecs +from functools import wraps + +from django.contrib.auth.decorators import login_required +from django.http import HttpResponse +from django.urls import path + +from etools_datamart.apps.subscriptions.views import subscribe + + +def http_basic_auth(func): + @wraps(func) + def _decorator(request, *args, **kwargs): + from django.contrib.auth import authenticate, login + if "HTTP_AUTHORIZATION" in request.META: + authmeth, auth = request.META["HTTP_AUTHORIZATION"].split(" ", 1) + if authmeth.lower() == "basic": + auth = codecs.decode(auth.encode("utf8").strip(), "base64").decode() + username, password = auth.split(":", 1) + user = authenticate(username=username, password=password) + if user: + login(request, user, backend='django.contrib.auth.backends.RemoteUserBackend') + else: + return HttpResponse(status=401) + return func(request, *args, **kwargs) + + return _decorator + + +def http_basic_login(func): + return http_basic_auth(login_required(func)) + + +urlpatterns = [ + path(r'subscribe//', http_basic_login(subscribe), name='subscribe'), +] diff --git a/src/etools_datamart/apps/subscriptions/views.py b/src/etools_datamart/apps/subscriptions/views.py new file mode 100644 index 000000000..af8d864e4 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/views.py @@ -0,0 +1,51 @@ +import json + +from django.forms import ModelForm +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt + +from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.subscriptions.models import Subscription + + +class SubscriptionForm(ModelForm): + class Meta: + model = Subscription + fields = ['type', 'kwargs'] + + +@csrf_exempt +def subscribe(request, etl_id): + code = 200 + values = {'status': "", 'detail': ""} + try: + user = request.user + payload = json.loads(request.body) + form = SubscriptionForm(data=payload) + if form.is_valid(): + etl = EtlTask.objects.get(id=etl_id) + s, created = Subscription.objects.update_or_create(user=user, + content_type=etl.content_type, + kwargs=payload.get("kwargs", ''), + defaults={ + 'type': payload["type"] + }) + values['status'] = {True: "created", False: "updated"}[created] + values['detail'] = {"id": s.id, + "type": s.type, + "type_label": s.get_type_display(), + } + else: + values['detail'] = "Invalid request" + values['error'] = form.errors + code = 400 + + except EtlTask.DoesNotExist as e: + values['detail'] = f"Invalid task id `{etl_id}`" + values['error'] = str(e) + code = 404 + except Exception as e: + values['error'] = type(e).__name__ + values['detail'] = str(e) + code = 500 + return JsonResponse(values, status=code) diff --git a/src/etools_datamart/apps/tracking/migrations/0001_initial.py b/src/etools_datamart/apps/tracking/migrations/0001_initial.py index 6c01795ff..f60153acf 100644 --- a/src/etools_datamart/apps/tracking/migrations/0001_initial.py +++ b/src/etools_datamart/apps/tracking/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-26 13:13 +# Generated by Django 2.1.3 on 2018-11-28 14:10 import django.utils.timezone import strategy_field.fields diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181126_1313.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181128_1410.py similarity index 97% rename from src/etools_datamart/apps/tracking/migrations/0002_auto_20181126_1313.py rename to src/etools_datamart/apps/tracking/migrations/0002_auto_20181128_1410.py index 113821101..88428e3fc 100644 --- a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181126_1313.py +++ b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181128_1410.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-26 13:13 +# Generated by Django 2.1.3 on 2018-11-28 14:10 import django.db.models.deletion from django.conf import settings @@ -10,8 +10,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('unicef_rest_framework', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('tracking', '0001_initial'), ] diff --git a/src/etools_datamart/apps/web/static/jquery-3.3.1.min.js b/src/etools_datamart/apps/web/static/jquery-3.3.1.min.js new file mode 100644 index 000000000..4d9b3a258 --- /dev/null +++ b/src/etools_datamart/apps/web/static/jquery-3.3.1.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.3.1 | (c) JS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(e,t){"use strict";var n=[],r=e.document,i=Object.getPrototypeOf,o=n.slice,a=n.concat,s=n.push,u=n.indexOf,l={},c=l.toString,f=l.hasOwnProperty,p=f.toString,d=p.call(Object),h={},g=function e(t){return"function"==typeof t&&"number"!=typeof t.nodeType},y=function e(t){return null!=t&&t===t.window},v={type:!0,src:!0,noModule:!0};function m(e,t,n){var i,o=(t=t||r).createElement("script");if(o.text=e,n)for(i in v)n[i]&&(o[i]=n[i]);t.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?l[c.call(e)]||"object":typeof e}var b="3.3.1",w=function(e,t){return new w.fn.init(e,t)},T=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;w.fn=w.prototype={jquery:"3.3.1",constructor:w,length:0,toArray:function(){return o.call(this)},get:function(e){return null==e?o.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=w.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return w.each(this,e)},map:function(e){return this.pushStack(w.map(this,function(t,n){return e.call(t,n,t)}))},slice:function(){return this.pushStack(o.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(n>=0&&n0&&t-1 in e)}var E=function(e){var t,n,r,i,o,a,s,u,l,c,f,p,d,h,g,y,v,m,x,b="sizzle"+1*new Date,w=e.document,T=0,C=0,E=ae(),k=ae(),S=ae(),D=function(e,t){return e===t&&(f=!0),0},N={}.hasOwnProperty,A=[],j=A.pop,q=A.push,L=A.push,H=A.slice,O=function(e,t){for(var n=0,r=e.length;n+~]|"+M+")"+M+"*"),z=new RegExp("="+M+"*([^\\]'\"]*?)"+M+"*\\]","g"),X=new RegExp(W),U=new RegExp("^"+R+"$"),V={ID:new RegExp("^#("+R+")"),CLASS:new RegExp("^\\.("+R+")"),TAG:new RegExp("^("+R+"|[*])"),ATTR:new RegExp("^"+I),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+P+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},G=/^(?:input|select|textarea|button)$/i,Y=/^h\d$/i,Q=/^[^{]+\{\s*\[native \w/,J=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,K=/[+~]/,Z=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ee=function(e,t,n){var r="0x"+t-65536;return r!==r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},te=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ne=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},re=function(){p()},ie=me(function(e){return!0===e.disabled&&("form"in e||"label"in e)},{dir:"parentNode",next:"legend"});try{L.apply(A=H.call(w.childNodes),w.childNodes),A[w.childNodes.length].nodeType}catch(e){L={apply:A.length?function(e,t){q.apply(e,H.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function oe(e,t,r,i){var o,s,l,c,f,h,v,m=t&&t.ownerDocument,T=t?t.nodeType:9;if(r=r||[],"string"!=typeof e||!e||1!==T&&9!==T&&11!==T)return r;if(!i&&((t?t.ownerDocument||t:w)!==d&&p(t),t=t||d,g)){if(11!==T&&(f=J.exec(e)))if(o=f[1]){if(9===T){if(!(l=t.getElementById(o)))return r;if(l.id===o)return r.push(l),r}else if(m&&(l=m.getElementById(o))&&x(t,l)&&l.id===o)return r.push(l),r}else{if(f[2])return L.apply(r,t.getElementsByTagName(e)),r;if((o=f[3])&&n.getElementsByClassName&&t.getElementsByClassName)return L.apply(r,t.getElementsByClassName(o)),r}if(n.qsa&&!S[e+" "]&&(!y||!y.test(e))){if(1!==T)m=t,v=e;else if("object"!==t.nodeName.toLowerCase()){(c=t.getAttribute("id"))?c=c.replace(te,ne):t.setAttribute("id",c=b),s=(h=a(e)).length;while(s--)h[s]="#"+c+" "+ve(h[s]);v=h.join(","),m=K.test(e)&&ge(t.parentNode)||t}if(v)try{return L.apply(r,m.querySelectorAll(v)),r}catch(e){}finally{c===b&&t.removeAttribute("id")}}}return u(e.replace(B,"$1"),t,r,i)}function ae(){var e=[];function t(n,i){return e.push(n+" ")>r.cacheLength&&delete t[e.shift()],t[n+" "]=i}return t}function se(e){return e[b]=!0,e}function ue(e){var t=d.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function le(e,t){var n=e.split("|"),i=n.length;while(i--)r.attrHandle[n[i]]=t}function ce(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function fe(e){return function(t){return"input"===t.nodeName.toLowerCase()&&t.type===e}}function pe(e){return function(t){var n=t.nodeName.toLowerCase();return("input"===n||"button"===n)&&t.type===e}}function de(e){return function(t){return"form"in t?t.parentNode&&!1===t.disabled?"label"in t?"label"in t.parentNode?t.parentNode.disabled===e:t.disabled===e:t.isDisabled===e||t.isDisabled!==!e&&ie(t)===e:t.disabled===e:"label"in t&&t.disabled===e}}function he(e){return se(function(t){return t=+t,se(function(n,r){var i,o=e([],n.length,t),a=o.length;while(a--)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}function ge(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}n=oe.support={},o=oe.isXML=function(e){var t=e&&(e.ownerDocument||e).documentElement;return!!t&&"HTML"!==t.nodeName},p=oe.setDocument=function(e){var t,i,a=e?e.ownerDocument||e:w;return a!==d&&9===a.nodeType&&a.documentElement?(d=a,h=d.documentElement,g=!o(d),w!==d&&(i=d.defaultView)&&i.top!==i&&(i.addEventListener?i.addEventListener("unload",re,!1):i.attachEvent&&i.attachEvent("onunload",re)),n.attributes=ue(function(e){return e.className="i",!e.getAttribute("className")}),n.getElementsByTagName=ue(function(e){return e.appendChild(d.createComment("")),!e.getElementsByTagName("*").length}),n.getElementsByClassName=Q.test(d.getElementsByClassName),n.getById=ue(function(e){return h.appendChild(e).id=b,!d.getElementsByName||!d.getElementsByName(b).length}),n.getById?(r.filter.ID=function(e){var t=e.replace(Z,ee);return function(e){return e.getAttribute("id")===t}},r.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&g){var n=t.getElementById(e);return n?[n]:[]}}):(r.filter.ID=function(e){var t=e.replace(Z,ee);return function(e){var n="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return n&&n.value===t}},r.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&g){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),r.find.TAG=n.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):n.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},r.find.CLASS=n.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&g)return t.getElementsByClassName(e)},v=[],y=[],(n.qsa=Q.test(d.querySelectorAll))&&(ue(function(e){h.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&y.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||y.push("\\["+M+"*(?:value|"+P+")"),e.querySelectorAll("[id~="+b+"-]").length||y.push("~="),e.querySelectorAll(":checked").length||y.push(":checked"),e.querySelectorAll("a#"+b+"+*").length||y.push(".#.+[+~]")}),ue(function(e){e.innerHTML="";var t=d.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&y.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&y.push(":enabled",":disabled"),h.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&y.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),y.push(",.*:")})),(n.matchesSelector=Q.test(m=h.matches||h.webkitMatchesSelector||h.mozMatchesSelector||h.oMatchesSelector||h.msMatchesSelector))&&ue(function(e){n.disconnectedMatch=m.call(e,"*"),m.call(e,"[s!='']:x"),v.push("!=",W)}),y=y.length&&new RegExp(y.join("|")),v=v.length&&new RegExp(v.join("|")),t=Q.test(h.compareDocumentPosition),x=t||Q.test(h.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return f=!0,0;var r=!e.compareDocumentPosition-!t.compareDocumentPosition;return r||(1&(r=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!n.sortDetached&&t.compareDocumentPosition(e)===r?e===d||e.ownerDocument===w&&x(w,e)?-1:t===d||t.ownerDocument===w&&x(w,t)?1:c?O(c,e)-O(c,t):0:4&r?-1:1)}:function(e,t){if(e===t)return f=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===d?-1:t===d?1:i?-1:o?1:c?O(c,e)-O(c,t):0;if(i===o)return ce(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?ce(a[r],s[r]):a[r]===w?-1:s[r]===w?1:0},d):d},oe.matches=function(e,t){return oe(e,null,null,t)},oe.matchesSelector=function(e,t){if((e.ownerDocument||e)!==d&&p(e),t=t.replace(z,"='$1']"),n.matchesSelector&&g&&!S[t+" "]&&(!v||!v.test(t))&&(!y||!y.test(t)))try{var r=m.call(e,t);if(r||n.disconnectedMatch||e.document&&11!==e.document.nodeType)return r}catch(e){}return oe(t,d,null,[e]).length>0},oe.contains=function(e,t){return(e.ownerDocument||e)!==d&&p(e),x(e,t)},oe.attr=function(e,t){(e.ownerDocument||e)!==d&&p(e);var i=r.attrHandle[t.toLowerCase()],o=i&&N.call(r.attrHandle,t.toLowerCase())?i(e,t,!g):void 0;return void 0!==o?o:n.attributes||!g?e.getAttribute(t):(o=e.getAttributeNode(t))&&o.specified?o.value:null},oe.escape=function(e){return(e+"").replace(te,ne)},oe.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},oe.uniqueSort=function(e){var t,r=[],i=0,o=0;if(f=!n.detectDuplicates,c=!n.sortStable&&e.slice(0),e.sort(D),f){while(t=e[o++])t===e[o]&&(i=r.push(o));while(i--)e.splice(r[i],1)}return c=null,e},i=oe.getText=function(e){var t,n="",r=0,o=e.nodeType;if(o){if(1===o||9===o||11===o){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=i(e)}else if(3===o||4===o)return e.nodeValue}else while(t=e[r++])n+=i(t);return n},(r=oe.selectors={cacheLength:50,createPseudo:se,match:V,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(Z,ee),e[3]=(e[3]||e[4]||e[5]||"").replace(Z,ee),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||oe.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&oe.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return V.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=a(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(Z,ee).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=E[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&E(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=oe.attr(r,e);return null==i?"!="===t:!t||(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i.replace($," ")+" ").indexOf(n)>-1:"|="===t&&(i===n||i.slice(0,n.length+1)===n+"-"))}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,u){var l,c,f,p,d,h,g=o!==a?"nextSibling":"previousSibling",y=t.parentNode,v=s&&t.nodeName.toLowerCase(),m=!u&&!s,x=!1;if(y){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===v:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?y.firstChild:y.lastChild],a&&m){x=(d=(l=(c=(f=(p=y)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1])&&l[2],p=d&&y.childNodes[d];while(p=++d&&p&&p[g]||(x=d=0)||h.pop())if(1===p.nodeType&&++x&&p===t){c[e]=[T,d,x];break}}else if(m&&(x=d=(l=(c=(f=(p=t)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1]),!1===x)while(p=++d&&p&&p[g]||(x=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===v:1===p.nodeType)&&++x&&(m&&((c=(f=p[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]=[T,x]),p===t))break;return(x-=i)===r||x%r==0&&x/r>=0}}},PSEUDO:function(e,t){var n,i=r.pseudos[e]||r.setFilters[e.toLowerCase()]||oe.error("unsupported pseudo: "+e);return i[b]?i(t):i.length>1?(n=[e,e,"",t],r.setFilters.hasOwnProperty(e.toLowerCase())?se(function(e,n){var r,o=i(e,t),a=o.length;while(a--)e[r=O(e,o[a])]=!(n[r]=o[a])}):function(e){return i(e,0,n)}):i}},pseudos:{not:se(function(e){var t=[],n=[],r=s(e.replace(B,"$1"));return r[b]?se(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),t[0]=null,!n.pop()}}),has:se(function(e){return function(t){return oe(e,t).length>0}}),contains:se(function(e){return e=e.replace(Z,ee),function(t){return(t.textContent||t.innerText||i(t)).indexOf(e)>-1}}),lang:se(function(e){return U.test(e||"")||oe.error("unsupported lang: "+e),e=e.replace(Z,ee).toLowerCase(),function(t){var n;do{if(n=g?t.lang:t.getAttribute("xml:lang")||t.getAttribute("lang"))return(n=n.toLowerCase())===e||0===n.indexOf(e+"-")}while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===h},focus:function(e){return e===d.activeElement&&(!d.hasFocus||d.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:de(!1),disabled:de(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!r.pseudos.empty(e)},header:function(e){return Y.test(e.nodeName)},input:function(e){return G.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:he(function(){return[0]}),last:he(function(e,t){return[t-1]}),eq:he(function(e,t,n){return[n<0?n+t:n]}),even:he(function(e,t){for(var n=0;n=0;)e.push(r);return e}),gt:he(function(e,t,n){for(var r=n<0?n+t:n;++r1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function be(e,t,n){for(var r=0,i=t.length;r-1&&(o[l]=!(a[l]=f))}}else v=we(v===a?v.splice(h,v.length):v),i?i(null,a,v,u):L.apply(a,v)})}function Ce(e){for(var t,n,i,o=e.length,a=r.relative[e[0].type],s=a||r.relative[" "],u=a?1:0,c=me(function(e){return e===t},s,!0),f=me(function(e){return O(t,e)>-1},s,!0),p=[function(e,n,r){var i=!a&&(r||n!==l)||((t=n).nodeType?c(e,n,r):f(e,n,r));return t=null,i}];u1&&xe(p),u>1&&ve(e.slice(0,u-1).concat({value:" "===e[u-2].type?"*":""})).replace(B,"$1"),n,u0,i=e.length>0,o=function(o,a,s,u,c){var f,h,y,v=0,m="0",x=o&&[],b=[],w=l,C=o||i&&r.find.TAG("*",c),E=T+=null==w?1:Math.random()||.1,k=C.length;for(c&&(l=a===d||a||c);m!==k&&null!=(f=C[m]);m++){if(i&&f){h=0,a||f.ownerDocument===d||(p(f),s=!g);while(y=e[h++])if(y(f,a||d,s)){u.push(f);break}c&&(T=E)}n&&((f=!y&&f)&&v--,o&&x.push(f))}if(v+=m,n&&m!==v){h=0;while(y=t[h++])y(x,b,a,s);if(o){if(v>0)while(m--)x[m]||b[m]||(b[m]=j.call(u));b=we(b)}L.apply(u,b),c&&!o&&b.length>0&&v+t.length>1&&oe.uniqueSort(u)}return c&&(T=E,l=w),x};return n?se(o):o}return s=oe.compile=function(e,t){var n,r=[],i=[],o=S[e+" "];if(!o){t||(t=a(e)),n=t.length;while(n--)(o=Ce(t[n]))[b]?r.push(o):i.push(o);(o=S(e,Ee(i,r))).selector=e}return o},u=oe.select=function(e,t,n,i){var o,u,l,c,f,p="function"==typeof e&&e,d=!i&&a(e=p.selector||e);if(n=n||[],1===d.length){if((u=d[0]=d[0].slice(0)).length>2&&"ID"===(l=u[0]).type&&9===t.nodeType&&g&&r.relative[u[1].type]){if(!(t=(r.find.ID(l.matches[0].replace(Z,ee),t)||[])[0]))return n;p&&(t=t.parentNode),e=e.slice(u.shift().value.length)}o=V.needsContext.test(e)?0:u.length;while(o--){if(l=u[o],r.relative[c=l.type])break;if((f=r.find[c])&&(i=f(l.matches[0].replace(Z,ee),K.test(u[0].type)&&ge(t.parentNode)||t))){if(u.splice(o,1),!(e=i.length&&ve(u)))return L.apply(n,i),n;break}}}return(p||s(e,d))(i,t,!g,n,!t||K.test(e)&&ge(t.parentNode)||t),n},n.sortStable=b.split("").sort(D).join("")===b,n.detectDuplicates=!!f,p(),n.sortDetached=ue(function(e){return 1&e.compareDocumentPosition(d.createElement("fieldset"))}),ue(function(e){return e.innerHTML="","#"===e.firstChild.getAttribute("href")})||le("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),n.attributes&&ue(function(e){return e.innerHTML="",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||le("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ue(function(e){return null==e.getAttribute("disabled")})||le(P,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),oe}(e);w.find=E,w.expr=E.selectors,w.expr[":"]=w.expr.pseudos,w.uniqueSort=w.unique=E.uniqueSort,w.text=E.getText,w.isXMLDoc=E.isXML,w.contains=E.contains,w.escapeSelector=E.escape;var k=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&w(e).is(n))break;r.push(e)}return r},S=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},D=w.expr.match.needsContext;function N(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var A=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,t,n){return g(t)?w.grep(e,function(e,r){return!!t.call(e,r,e)!==n}):t.nodeType?w.grep(e,function(e){return e===t!==n}):"string"!=typeof t?w.grep(e,function(e){return u.call(t,e)>-1!==n}):w.filter(t,e,n)}w.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?w.find.matchesSelector(r,e)?[r]:[]:w.find.matches(e,w.grep(t,function(e){return 1===e.nodeType}))},w.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(w(e).filter(function(){for(t=0;t1?w.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&D.test(e)?w(e):e||[],!1).length}});var q,L=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(w.fn.init=function(e,t,n){var i,o;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(i="<"===e[0]&&">"===e[e.length-1]&&e.length>=3?[null,e,null]:L.exec(e))||!i[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(i[1]){if(t=t instanceof w?t[0]:t,w.merge(this,w.parseHTML(i[1],t&&t.nodeType?t.ownerDocument||t:r,!0)),A.test(i[1])&&w.isPlainObject(t))for(i in t)g(this[i])?this[i](t[i]):this.attr(i,t[i]);return this}return(o=r.getElementById(i[2]))&&(this[0]=o,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):g(e)?void 0!==n.ready?n.ready(e):e(w):w.makeArray(e,this)}).prototype=w.fn,q=w(r);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};w.fn.extend({has:function(e){var t=w(e,this),n=t.length;return this.filter(function(){for(var e=0;e-1:1===n.nodeType&&w.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(o.length>1?w.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?u.call(w(e),this[0]):u.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(w.uniqueSort(w.merge(this.get(),w(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}});function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}w.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return k(e,"parentNode")},parentsUntil:function(e,t,n){return k(e,"parentNode",n)},next:function(e){return P(e,"nextSibling")},prev:function(e){return P(e,"previousSibling")},nextAll:function(e){return k(e,"nextSibling")},prevAll:function(e){return k(e,"previousSibling")},nextUntil:function(e,t,n){return k(e,"nextSibling",n)},prevUntil:function(e,t,n){return k(e,"previousSibling",n)},siblings:function(e){return S((e.parentNode||{}).firstChild,e)},children:function(e){return S(e.firstChild)},contents:function(e){return N(e,"iframe")?e.contentDocument:(N(e,"template")&&(e=e.content||e),w.merge([],e.childNodes))}},function(e,t){w.fn[e]=function(n,r){var i=w.map(this,t,n);return"Until"!==e.slice(-5)&&(r=n),r&&"string"==typeof r&&(i=w.filter(r,i)),this.length>1&&(O[e]||w.uniqueSort(i),H.test(e)&&i.reverse()),this.pushStack(i)}});var M=/[^\x20\t\r\n\f]+/g;function R(e){var t={};return w.each(e.match(M)||[],function(e,n){t[n]=!0}),t}w.Callbacks=function(e){e="string"==typeof e?R(e):w.extend({},e);var t,n,r,i,o=[],a=[],s=-1,u=function(){for(i=i||e.once,r=t=!0;a.length;s=-1){n=a.shift();while(++s-1)o.splice(n,1),n<=s&&s--}),this},has:function(e){return e?w.inArray(e,o)>-1:o.length>0},empty:function(){return o&&(o=[]),this},disable:function(){return i=a=[],o=n="",this},disabled:function(){return!o},lock:function(){return i=a=[],n||t||(o=n=""),this},locked:function(){return!!i},fireWith:function(e,n){return i||(n=[e,(n=n||[]).slice?n.slice():n],a.push(n),t||u()),this},fire:function(){return l.fireWith(this,arguments),this},fired:function(){return!!r}};return l};function I(e){return e}function W(e){throw e}function $(e,t,n,r){var i;try{e&&g(i=e.promise)?i.call(e).done(t).fail(n):e&&g(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}w.extend({Deferred:function(t){var n=[["notify","progress",w.Callbacks("memory"),w.Callbacks("memory"),2],["resolve","done",w.Callbacks("once memory"),w.Callbacks("once memory"),0,"resolved"],["reject","fail",w.Callbacks("once memory"),w.Callbacks("once memory"),1,"rejected"]],r="pending",i={state:function(){return r},always:function(){return o.done(arguments).fail(arguments),this},"catch":function(e){return i.then(null,e)},pipe:function(){var e=arguments;return w.Deferred(function(t){w.each(n,function(n,r){var i=g(e[r[4]])&&e[r[4]];o[r[1]](function(){var e=i&&i.apply(this,arguments);e&&g(e.promise)?e.promise().progress(t.notify).done(t.resolve).fail(t.reject):t[r[0]+"With"](this,i?[e]:arguments)})}),e=null}).promise()},then:function(t,r,i){var o=0;function a(t,n,r,i){return function(){var s=this,u=arguments,l=function(){var e,l;if(!(t=o&&(r!==W&&(s=void 0,u=[e]),n.rejectWith(s,u))}};t?c():(w.Deferred.getStackHook&&(c.stackTrace=w.Deferred.getStackHook()),e.setTimeout(c))}}return w.Deferred(function(e){n[0][3].add(a(0,e,g(i)?i:I,e.notifyWith)),n[1][3].add(a(0,e,g(t)?t:I)),n[2][3].add(a(0,e,g(r)?r:W))}).promise()},promise:function(e){return null!=e?w.extend(e,i):i}},o={};return w.each(n,function(e,t){var a=t[2],s=t[5];i[t[1]]=a.add,s&&a.add(function(){r=s},n[3-e][2].disable,n[3-e][3].disable,n[0][2].lock,n[0][3].lock),a.add(t[3].fire),o[t[0]]=function(){return o[t[0]+"With"](this===o?void 0:this,arguments),this},o[t[0]+"With"]=a.fireWith}),i.promise(o),t&&t.call(o,o),o},when:function(e){var t=arguments.length,n=t,r=Array(n),i=o.call(arguments),a=w.Deferred(),s=function(e){return function(n){r[e]=this,i[e]=arguments.length>1?o.call(arguments):n,--t||a.resolveWith(r,i)}};if(t<=1&&($(e,a.done(s(n)).resolve,a.reject,!t),"pending"===a.state()||g(i[n]&&i[n].then)))return a.then();while(n--)$(i[n],s(n),a.reject);return a.promise()}});var B=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;w.Deferred.exceptionHook=function(t,n){e.console&&e.console.warn&&t&&B.test(t.name)&&e.console.warn("jQuery.Deferred exception: "+t.message,t.stack,n)},w.readyException=function(t){e.setTimeout(function(){throw t})};var F=w.Deferred();w.fn.ready=function(e){return F.then(e)["catch"](function(e){w.readyException(e)}),this},w.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--w.readyWait:w.isReady)||(w.isReady=!0,!0!==e&&--w.readyWait>0||F.resolveWith(r,[w]))}}),w.ready.then=F.then;function _(){r.removeEventListener("DOMContentLoaded",_),e.removeEventListener("load",_),w.ready()}"complete"===r.readyState||"loading"!==r.readyState&&!r.documentElement.doScroll?e.setTimeout(w.ready):(r.addEventListener("DOMContentLoaded",_),e.addEventListener("load",_));var z=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===x(n)){i=!0;for(s in n)z(e,t,s,n[s],!0,o,a)}else if(void 0!==r&&(i=!0,g(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(w(e),n)})),t))for(;s1,null,!0)},removeData:function(e){return this.each(function(){K.remove(this,e)})}}),w.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=J.get(e,t),n&&(!r||Array.isArray(n)?r=J.access(e,t,w.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=w.queue(e,t),r=n.length,i=n.shift(),o=w._queueHooks(e,t),a=function(){w.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return J.get(e,n)||J.access(e,n,{empty:w.Callbacks("once memory").add(function(){J.remove(e,[t+"queue",n])})})}}),w.fn.extend({queue:function(e,t){var n=2;return"string"!=typeof e&&(t=e,e="fx",n--),arguments.length\x20\t\r\n\f]+)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,""],thead:[1,"","
      "],col:[2,"","
      "],tr:[2,"","
      "],td:[3,"","
      "],_default:[0,"",""]};ge.optgroup=ge.option,ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td;function ye(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&N(e,t)?w.merge([e],n):n}function ve(e,t){for(var n=0,r=e.length;n-1)i&&i.push(o);else if(l=w.contains(o.ownerDocument,o),a=ye(f.appendChild(o),"script"),l&&ve(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}!function(){var e=r.createDocumentFragment().appendChild(r.createElement("div")),t=r.createElement("input");t.setAttribute("type","radio"),t.setAttribute("checked","checked"),t.setAttribute("name","t"),e.appendChild(t),h.checkClone=e.cloneNode(!0).cloneNode(!0).lastChild.checked,e.innerHTML="",h.noCloneChecked=!!e.cloneNode(!0).lastChild.defaultValue}();var be=r.documentElement,we=/^key/,Te=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ce=/^([^.]*)(?:\.(.+)|)/;function Ee(){return!0}function ke(){return!1}function Se(){try{return r.activeElement}catch(e){}}function De(e,t,n,r,i,o){var a,s;if("object"==typeof t){"string"!=typeof n&&(r=r||n,n=void 0);for(s in t)De(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=ke;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return w().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=w.guid++)),e.each(function(){w.event.add(this,t,i,r,n)})}w.event={global:{},add:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,y=J.get(e);if(y){n.handler&&(n=(o=n).handler,i=o.selector),i&&w.find.matchesSelector(be,i),n.guid||(n.guid=w.guid++),(u=y.events)||(u=y.events={}),(a=y.handle)||(a=y.handle=function(t){return"undefined"!=typeof w&&w.event.triggered!==t.type?w.event.dispatch.apply(e,arguments):void 0}),l=(t=(t||"").match(M)||[""]).length;while(l--)d=g=(s=Ce.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=w.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=w.event.special[d]||{},c=w.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&w.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(e,r,h,a)||e.addEventListener&&e.addEventListener(d,a)),f.add&&(f.add.call(e,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),w.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,y=J.hasData(e)&&J.get(e);if(y&&(u=y.events)){l=(t=(t||"").match(M)||[""]).length;while(l--)if(s=Ce.exec(t[l])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){f=w.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,y.handle)||w.removeEvent(e,d,y.handle),delete u[d])}else for(d in u)w.event.remove(e,d+t[l],n,r,!0);w.isEmptyObject(u)&&J.remove(e,"handle events")}},dispatch:function(e){var t=w.event.fix(e),n,r,i,o,a,s,u=new Array(arguments.length),l=(J.get(this,"events")||{})[t.type]||[],c=w.event.special[t.type]||{};for(u[0]=t,n=1;n=1))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n-1:w.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u\x20\t\r\n\f]*)[^>]*)\/>/gi,Ae=/\s*$/g;function Le(e,t){return N(e,"table")&&N(11!==t.nodeType?t:t.firstChild,"tr")?w(e).children("tbody")[0]||e:e}function He(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Oe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Pe(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(J.hasData(e)&&(o=J.access(e),a=J.set(t,o),l=o.events)){delete a.handle,a.events={};for(i in l)for(n=0,r=l[i].length;n1&&"string"==typeof y&&!h.checkClone&&je.test(y))return e.each(function(i){var o=e.eq(i);v&&(t[0]=y.call(this,i,o.html())),Re(o,t,n,r)});if(p&&(i=xe(t,e[0].ownerDocument,!1,e,r),o=i.firstChild,1===i.childNodes.length&&(i=o),o||r)){for(u=(s=w.map(ye(i,"script"),He)).length;f")},clone:function(e,t,n){var r,i,o,a,s=e.cloneNode(!0),u=w.contains(e.ownerDocument,e);if(!(h.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||w.isXMLDoc(e)))for(a=ye(s),r=0,i=(o=ye(e)).length;r0&&ve(a,!u&&ye(e,"script")),s},cleanData:function(e){for(var t,n,r,i=w.event.special,o=0;void 0!==(n=e[o]);o++)if(Y(n)){if(t=n[J.expando]){if(t.events)for(r in t.events)i[r]?w.event.remove(n,r):w.removeEvent(n,r,t.handle);n[J.expando]=void 0}n[K.expando]&&(n[K.expando]=void 0)}}}),w.fn.extend({detach:function(e){return Ie(this,e,!0)},remove:function(e){return Ie(this,e)},text:function(e){return z(this,function(e){return void 0===e?w.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Re(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Le(this,e).appendChild(e)})},prepend:function(){return Re(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Le(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Re(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Re(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(w.cleanData(ye(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return w.clone(this,e,t)})},html:function(e){return z(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Ae.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=w.htmlPrefilter(e);try{for(;n=0&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))),u}function et(e,t,n){var r=$e(e),i=Fe(e,t,r),o="border-box"===w.css(e,"boxSizing",!1,r),a=o;if(We.test(i)){if(!n)return i;i="auto"}return a=a&&(h.boxSizingReliable()||i===e.style[t]),("auto"===i||!parseFloat(i)&&"inline"===w.css(e,"display",!1,r))&&(i=e["offset"+t[0].toUpperCase()+t.slice(1)],a=!0),(i=parseFloat(i)||0)+Ze(e,t,n||(o?"border":"content"),a,r,i)+"px"}w.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Fe(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=G(t),u=Xe.test(t),l=e.style;if(u||(t=Je(s)),a=w.cssHooks[t]||w.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"==(o=typeof n)&&(i=ie.exec(n))&&i[1]&&(n=ue(e,t,i),o="number"),null!=n&&n===n&&("number"===o&&(n+=i&&i[3]||(w.cssNumber[s]?"":"px")),h.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=G(t);return Xe.test(t)||(t=Je(s)),(a=w.cssHooks[t]||w.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Fe(e,t,r)),"normal"===i&&t in Ve&&(i=Ve[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),w.each(["height","width"],function(e,t){w.cssHooks[t]={get:function(e,n,r){if(n)return!ze.test(w.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?et(e,t,r):se(e,Ue,function(){return et(e,t,r)})},set:function(e,n,r){var i,o=$e(e),a="border-box"===w.css(e,"boxSizing",!1,o),s=r&&Ze(e,t,r,a,o);return a&&h.scrollboxSize()===o.position&&(s-=Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-parseFloat(o[t])-Ze(e,t,"border",!1,o)-.5)),s&&(i=ie.exec(n))&&"px"!==(i[3]||"px")&&(e.style[t]=n,n=w.css(e,t)),Ke(e,n,s)}}}),w.cssHooks.marginLeft=_e(h.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Fe(e,"marginLeft"))||e.getBoundingClientRect().left-se(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),w.each({margin:"",padding:"",border:"Width"},function(e,t){w.cssHooks[e+t]={expand:function(n){for(var r=0,i={},o="string"==typeof n?n.split(" "):[n];r<4;r++)i[e+oe[r]+t]=o[r]||o[r-2]||o[0];return i}},"margin"!==e&&(w.cssHooks[e+t].set=Ke)}),w.fn.extend({css:function(e,t){return z(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=$e(e),i=t.length;a1)}});function tt(e,t,n,r,i){return new tt.prototype.init(e,t,n,r,i)}w.Tween=tt,tt.prototype={constructor:tt,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||w.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(w.cssNumber[n]?"":"px")},cur:function(){var e=tt.propHooks[this.prop];return e&&e.get?e.get(this):tt.propHooks._default.get(this)},run:function(e){var t,n=tt.propHooks[this.prop];return this.options.duration?this.pos=t=w.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):tt.propHooks._default.set(this),this}},tt.prototype.init.prototype=tt.prototype,tt.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=w.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){w.fx.step[e.prop]?w.fx.step[e.prop](e):1!==e.elem.nodeType||null==e.elem.style[w.cssProps[e.prop]]&&!w.cssHooks[e.prop]?e.elem[e.prop]=e.now:w.style(e.elem,e.prop,e.now+e.unit)}}},tt.propHooks.scrollTop=tt.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},w.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},w.fx=tt.prototype.init,w.fx.step={};var nt,rt,it=/^(?:toggle|show|hide)$/,ot=/queueHooks$/;function at(){rt&&(!1===r.hidden&&e.requestAnimationFrame?e.requestAnimationFrame(at):e.setTimeout(at,w.fx.interval),w.fx.tick())}function st(){return e.setTimeout(function(){nt=void 0}),nt=Date.now()}function ut(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=oe[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function lt(e,t,n){for(var r,i=(pt.tweeners[t]||[]).concat(pt.tweeners["*"]),o=0,a=i.length;o1)},removeAttr:function(e){return this.each(function(){w.removeAttr(this,e)})}}),w.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?w.prop(e,t,n):(1===o&&w.isXMLDoc(e)||(i=w.attrHooks[t.toLowerCase()]||(w.expr.match.bool.test(t)?dt:void 0)),void 0!==n?null===n?void w.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=w.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!h.radioValue&&"radio"===t&&N(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(M);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),dt={set:function(e,t,n){return!1===t?w.removeAttr(e,n):e.setAttribute(n,n),n}},w.each(w.expr.match.bool.source.match(/\w+/g),function(e,t){var n=ht[t]||w.find.attr;ht[t]=function(e,t,r){var i,o,a=t.toLowerCase();return r||(o=ht[a],ht[a]=i,i=null!=n(e,t,r)?a:null,ht[a]=o),i}});var gt=/^(?:input|select|textarea|button)$/i,yt=/^(?:a|area)$/i;w.fn.extend({prop:function(e,t){return z(this,w.prop,e,t,arguments.length>1)},removeProp:function(e){return this.each(function(){delete this[w.propFix[e]||e]})}}),w.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&w.isXMLDoc(e)||(t=w.propFix[t]||t,i=w.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=w.find.attr(e,"tabindex");return t?parseInt(t,10):gt.test(e.nodeName)||yt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),h.optSelected||(w.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),w.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){w.propFix[this.toLowerCase()]=this});function vt(e){return(e.match(M)||[]).join(" ")}function mt(e){return e.getAttribute&&e.getAttribute("class")||""}function xt(e){return Array.isArray(e)?e:"string"==typeof e?e.match(M)||[]:[]}w.fn.extend({addClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){w(this).addClass(e.call(this,t,mt(this)))});if((t=xt(e)).length)while(n=this[u++])if(i=mt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=t[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},removeClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){w(this).removeClass(e.call(this,t,mt(this)))});if(!arguments.length)return this.attr("class","");if((t=xt(e)).length)while(n=this[u++])if(i=mt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=t[a++])while(r.indexOf(" "+o+" ")>-1)r=r.replace(" "+o+" "," ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(e,t){var n=typeof e,r="string"===n||Array.isArray(e);return"boolean"==typeof t&&r?t?this.addClass(e):this.removeClass(e):g(e)?this.each(function(n){w(this).toggleClass(e.call(this,n,mt(this),t),t)}):this.each(function(){var t,i,o,a;if(r){i=0,o=w(this),a=xt(e);while(t=a[i++])o.hasClass(t)?o.removeClass(t):o.addClass(t)}else void 0!==e&&"boolean"!==n||((t=mt(this))&&J.set(this,"__className__",t),this.setAttribute&&this.setAttribute("class",t||!1===e?"":J.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&(" "+vt(mt(n))+" ").indexOf(t)>-1)return!0;return!1}});var bt=/\r/g;w.fn.extend({val:function(e){var t,n,r,i=this[0];{if(arguments.length)return r=g(e),this.each(function(n){var i;1===this.nodeType&&(null==(i=r?e.call(this,n,w(this).val()):e)?i="":"number"==typeof i?i+="":Array.isArray(i)&&(i=w.map(i,function(e){return null==e?"":e+""})),(t=w.valHooks[this.type]||w.valHooks[this.nodeName.toLowerCase()])&&"set"in t&&void 0!==t.set(this,i,"value")||(this.value=i))});if(i)return(t=w.valHooks[i.type]||w.valHooks[i.nodeName.toLowerCase()])&&"get"in t&&void 0!==(n=t.get(i,"value"))?n:"string"==typeof(n=i.value)?n.replace(bt,""):null==n?"":n}}}),w.extend({valHooks:{option:{get:function(e){var t=w.find.attr(e,"value");return null!=t?t:vt(w.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r-1)&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),w.each(["radio","checkbox"],function(){w.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=w.inArray(w(e).val(),t)>-1}},h.checkOn||(w.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),h.focusin="onfocusin"in e;var wt=/^(?:focusinfocus|focusoutblur)$/,Tt=function(e){e.stopPropagation()};w.extend(w.event,{trigger:function(t,n,i,o){var a,s,u,l,c,p,d,h,v=[i||r],m=f.call(t,"type")?t.type:t,x=f.call(t,"namespace")?t.namespace.split("."):[];if(s=h=u=i=i||r,3!==i.nodeType&&8!==i.nodeType&&!wt.test(m+w.event.triggered)&&(m.indexOf(".")>-1&&(m=(x=m.split(".")).shift(),x.sort()),c=m.indexOf(":")<0&&"on"+m,t=t[w.expando]?t:new w.Event(m,"object"==typeof t&&t),t.isTrigger=o?2:3,t.namespace=x.join("."),t.rnamespace=t.namespace?new RegExp("(^|\\.)"+x.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,t.result=void 0,t.target||(t.target=i),n=null==n?[t]:w.makeArray(n,[t]),d=w.event.special[m]||{},o||!d.trigger||!1!==d.trigger.apply(i,n))){if(!o&&!d.noBubble&&!y(i)){for(l=d.delegateType||m,wt.test(l+m)||(s=s.parentNode);s;s=s.parentNode)v.push(s),u=s;u===(i.ownerDocument||r)&&v.push(u.defaultView||u.parentWindow||e)}a=0;while((s=v[a++])&&!t.isPropagationStopped())h=s,t.type=a>1?l:d.bindType||m,(p=(J.get(s,"events")||{})[t.type]&&J.get(s,"handle"))&&p.apply(s,n),(p=c&&s[c])&&p.apply&&Y(s)&&(t.result=p.apply(s,n),!1===t.result&&t.preventDefault());return t.type=m,o||t.isDefaultPrevented()||d._default&&!1!==d._default.apply(v.pop(),n)||!Y(i)||c&&g(i[m])&&!y(i)&&((u=i[c])&&(i[c]=null),w.event.triggered=m,t.isPropagationStopped()&&h.addEventListener(m,Tt),i[m](),t.isPropagationStopped()&&h.removeEventListener(m,Tt),w.event.triggered=void 0,u&&(i[c]=u)),t.result}},simulate:function(e,t,n){var r=w.extend(new w.Event,n,{type:e,isSimulated:!0});w.event.trigger(r,null,t)}}),w.fn.extend({trigger:function(e,t){return this.each(function(){w.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return w.event.trigger(e,t,n,!0)}}),h.focusin||w.each({focus:"focusin",blur:"focusout"},function(e,t){var n=function(e){w.event.simulate(t,e.target,w.event.fix(e))};w.event.special[t]={setup:function(){var r=this.ownerDocument||this,i=J.access(r,t);i||r.addEventListener(e,n,!0),J.access(r,t,(i||0)+1)},teardown:function(){var r=this.ownerDocument||this,i=J.access(r,t)-1;i?J.access(r,t,i):(r.removeEventListener(e,n,!0),J.remove(r,t))}}});var Ct=e.location,Et=Date.now(),kt=/\?/;w.parseXML=function(t){var n;if(!t||"string"!=typeof t)return null;try{n=(new e.DOMParser).parseFromString(t,"text/xml")}catch(e){n=void 0}return n&&!n.getElementsByTagName("parsererror").length||w.error("Invalid XML: "+t),n};var St=/\[\]$/,Dt=/\r?\n/g,Nt=/^(?:submit|button|image|reset|file)$/i,At=/^(?:input|select|textarea|keygen)/i;function jt(e,t,n,r){var i;if(Array.isArray(t))w.each(t,function(t,i){n||St.test(e)?r(e,i):jt(e+"["+("object"==typeof i&&null!=i?t:"")+"]",i,n,r)});else if(n||"object"!==x(t))r(e,t);else for(i in t)jt(e+"["+i+"]",t[i],n,r)}w.param=function(e,t){var n,r=[],i=function(e,t){var n=g(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(Array.isArray(e)||e.jquery&&!w.isPlainObject(e))w.each(e,function(){i(this.name,this.value)});else for(n in e)jt(n,e[n],t,i);return r.join("&")},w.fn.extend({serialize:function(){return w.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=w.prop(this,"elements");return e?w.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!w(this).is(":disabled")&&At.test(this.nodeName)&&!Nt.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=w(this).val();return null==n?null:Array.isArray(n)?w.map(n,function(e){return{name:t.name,value:e.replace(Dt,"\r\n")}}):{name:t.name,value:n.replace(Dt,"\r\n")}}).get()}});var qt=/%20/g,Lt=/#.*$/,Ht=/([?&])_=[^&]*/,Ot=/^(.*?):[ \t]*([^\r\n]*)$/gm,Pt=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Mt=/^(?:GET|HEAD)$/,Rt=/^\/\//,It={},Wt={},$t="*/".concat("*"),Bt=r.createElement("a");Bt.href=Ct.href;function Ft(e){return function(t,n){"string"!=typeof t&&(n=t,t="*");var r,i=0,o=t.toLowerCase().match(M)||[];if(g(n))while(r=o[i++])"+"===r[0]?(r=r.slice(1)||"*",(e[r]=e[r]||[]).unshift(n)):(e[r]=e[r]||[]).push(n)}}function _t(e,t,n,r){var i={},o=e===Wt;function a(s){var u;return i[s]=!0,w.each(e[s]||[],function(e,s){var l=s(t,n,r);return"string"!=typeof l||o||i[l]?o?!(u=l):void 0:(t.dataTypes.unshift(l),a(l),!1)}),u}return a(t.dataTypes[0])||!i["*"]&&a("*")}function zt(e,t){var n,r,i=w.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&w.extend(!0,e,r),e}function Xt(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}function Ut(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}w.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Ct.href,type:"GET",isLocal:Pt.test(Ct.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":$t,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":w.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?zt(zt(e,w.ajaxSettings),t):zt(w.ajaxSettings,e)},ajaxPrefilter:Ft(It),ajaxTransport:Ft(Wt),ajax:function(t,n){"object"==typeof t&&(n=t,t=void 0),n=n||{};var i,o,a,s,u,l,c,f,p,d,h=w.ajaxSetup({},n),g=h.context||h,y=h.context&&(g.nodeType||g.jquery)?w(g):w.event,v=w.Deferred(),m=w.Callbacks("once memory"),x=h.statusCode||{},b={},T={},C="canceled",E={readyState:0,getResponseHeader:function(e){var t;if(c){if(!s){s={};while(t=Ot.exec(a))s[t[1].toLowerCase()]=t[2]}t=s[e.toLowerCase()]}return null==t?null:t},getAllResponseHeaders:function(){return c?a:null},setRequestHeader:function(e,t){return null==c&&(e=T[e.toLowerCase()]=T[e.toLowerCase()]||e,b[e]=t),this},overrideMimeType:function(e){return null==c&&(h.mimeType=e),this},statusCode:function(e){var t;if(e)if(c)E.always(e[E.status]);else for(t in e)x[t]=[x[t],e[t]];return this},abort:function(e){var t=e||C;return i&&i.abort(t),k(0,t),this}};if(v.promise(E),h.url=((t||h.url||Ct.href)+"").replace(Rt,Ct.protocol+"//"),h.type=n.method||n.type||h.method||h.type,h.dataTypes=(h.dataType||"*").toLowerCase().match(M)||[""],null==h.crossDomain){l=r.createElement("a");try{l.href=h.url,l.href=l.href,h.crossDomain=Bt.protocol+"//"+Bt.host!=l.protocol+"//"+l.host}catch(e){h.crossDomain=!0}}if(h.data&&h.processData&&"string"!=typeof h.data&&(h.data=w.param(h.data,h.traditional)),_t(It,h,n,E),c)return E;(f=w.event&&h.global)&&0==w.active++&&w.event.trigger("ajaxStart"),h.type=h.type.toUpperCase(),h.hasContent=!Mt.test(h.type),o=h.url.replace(Lt,""),h.hasContent?h.data&&h.processData&&0===(h.contentType||"").indexOf("application/x-www-form-urlencoded")&&(h.data=h.data.replace(qt,"+")):(d=h.url.slice(o.length),h.data&&(h.processData||"string"==typeof h.data)&&(o+=(kt.test(o)?"&":"?")+h.data,delete h.data),!1===h.cache&&(o=o.replace(Ht,"$1"),d=(kt.test(o)?"&":"?")+"_="+Et+++d),h.url=o+d),h.ifModified&&(w.lastModified[o]&&E.setRequestHeader("If-Modified-Since",w.lastModified[o]),w.etag[o]&&E.setRequestHeader("If-None-Match",w.etag[o])),(h.data&&h.hasContent&&!1!==h.contentType||n.contentType)&&E.setRequestHeader("Content-Type",h.contentType),E.setRequestHeader("Accept",h.dataTypes[0]&&h.accepts[h.dataTypes[0]]?h.accepts[h.dataTypes[0]]+("*"!==h.dataTypes[0]?", "+$t+"; q=0.01":""):h.accepts["*"]);for(p in h.headers)E.setRequestHeader(p,h.headers[p]);if(h.beforeSend&&(!1===h.beforeSend.call(g,E,h)||c))return E.abort();if(C="abort",m.add(h.complete),E.done(h.success),E.fail(h.error),i=_t(Wt,h,n,E)){if(E.readyState=1,f&&y.trigger("ajaxSend",[E,h]),c)return E;h.async&&h.timeout>0&&(u=e.setTimeout(function(){E.abort("timeout")},h.timeout));try{c=!1,i.send(b,k)}catch(e){if(c)throw e;k(-1,e)}}else k(-1,"No Transport");function k(t,n,r,s){var l,p,d,b,T,C=n;c||(c=!0,u&&e.clearTimeout(u),i=void 0,a=s||"",E.readyState=t>0?4:0,l=t>=200&&t<300||304===t,r&&(b=Xt(h,E,r)),b=Ut(h,b,E,l),l?(h.ifModified&&((T=E.getResponseHeader("Last-Modified"))&&(w.lastModified[o]=T),(T=E.getResponseHeader("etag"))&&(w.etag[o]=T)),204===t||"HEAD"===h.type?C="nocontent":304===t?C="notmodified":(C=b.state,p=b.data,l=!(d=b.error))):(d=C,!t&&C||(C="error",t<0&&(t=0))),E.status=t,E.statusText=(n||C)+"",l?v.resolveWith(g,[p,C,E]):v.rejectWith(g,[E,C,d]),E.statusCode(x),x=void 0,f&&y.trigger(l?"ajaxSuccess":"ajaxError",[E,h,l?p:d]),m.fireWith(g,[E,C]),f&&(y.trigger("ajaxComplete",[E,h]),--w.active||w.event.trigger("ajaxStop")))}return E},getJSON:function(e,t,n){return w.get(e,t,n,"json")},getScript:function(e,t){return w.get(e,void 0,t,"script")}}),w.each(["get","post"],function(e,t){w[t]=function(e,n,r,i){return g(n)&&(i=i||r,r=n,n=void 0),w.ajax(w.extend({url:e,type:t,dataType:i,data:n,success:r},w.isPlainObject(e)&&e))}}),w._evalUrl=function(e){return w.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,"throws":!0})},w.fn.extend({wrapAll:function(e){var t;return this[0]&&(g(e)&&(e=e.call(this[0])),t=w(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(e){return g(e)?this.each(function(t){w(this).wrapInner(e.call(this,t))}):this.each(function(){var t=w(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=g(e);return this.each(function(n){w(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(e){return this.parent(e).not("body").each(function(){w(this).replaceWith(this.childNodes)}),this}}),w.expr.pseudos.hidden=function(e){return!w.expr.pseudos.visible(e)},w.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},w.ajaxSettings.xhr=function(){try{return new e.XMLHttpRequest}catch(e){}};var Vt={0:200,1223:204},Gt=w.ajaxSettings.xhr();h.cors=!!Gt&&"withCredentials"in Gt,h.ajax=Gt=!!Gt,w.ajaxTransport(function(t){var n,r;if(h.cors||Gt&&!t.crossDomain)return{send:function(i,o){var a,s=t.xhr();if(s.open(t.type,t.url,t.async,t.username,t.password),t.xhrFields)for(a in t.xhrFields)s[a]=t.xhrFields[a];t.mimeType&&s.overrideMimeType&&s.overrideMimeType(t.mimeType),t.crossDomain||i["X-Requested-With"]||(i["X-Requested-With"]="XMLHttpRequest");for(a in i)s.setRequestHeader(a,i[a]);n=function(e){return function(){n&&(n=r=s.onload=s.onerror=s.onabort=s.ontimeout=s.onreadystatechange=null,"abort"===e?s.abort():"error"===e?"number"!=typeof s.status?o(0,"error"):o(s.status,s.statusText):o(Vt[s.status]||s.status,s.statusText,"text"!==(s.responseType||"text")||"string"!=typeof s.responseText?{binary:s.response}:{text:s.responseText},s.getAllResponseHeaders()))}},s.onload=n(),r=s.onerror=s.ontimeout=n("error"),void 0!==s.onabort?s.onabort=r:s.onreadystatechange=function(){4===s.readyState&&e.setTimeout(function(){n&&r()})},n=n("abort");try{s.send(t.hasContent&&t.data||null)}catch(e){if(n)throw e}},abort:function(){n&&n()}}}),w.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),w.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return w.globalEval(e),e}}}),w.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),w.ajaxTransport("script",function(e){if(e.crossDomain){var t,n;return{send:function(i,o){t=w(" + +{% endblock %} diff --git a/src/etools_datamart/apps/web/views.py b/src/etools_datamart/apps/web/views.py index c07b1c7cf..8fe858b6d 100644 --- a/src/etools_datamart/apps/web/views.py +++ b/src/etools_datamart/apps/web/views.py @@ -1,3 +1,4 @@ +from django.contrib.auth.decorators import login_required from django.contrib.auth.views import LogoutView from django.template.response import TemplateResponse @@ -10,8 +11,11 @@ def index(request): return TemplateResponse(request, 'index.html', context) +@login_required def monitor(request): - context = {'tasks': EtlTask.objects.all(), 'page': 'monitor'} + context = {'tasks': EtlTask.objects.all(), + 'subscriptions': request.user.subscriptions, + 'page': 'monitor'} return TemplateResponse(request, 'monitor.html', context) diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 1c0d9678a..9cb4eb74a 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -4,6 +4,7 @@ from celery import Celery from celery.signals import task_postrun, task_prerun from celery.task import Task +from django.apps import apps from kombu.serialization import register from etools_datamart.apps.etl.results import etl_dumps, etl_loads @@ -23,16 +24,16 @@ class DatamartCelery(Celery): def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): from etools_datamart.apps.etl.lock import only_one linked_model = options.get('linked_model', None) - name = name or self.gen_task_name(fun.__name__, fun.__module__) - options['lock_key'] = f"{name}-lock" - fun = only_one(fun, options['lock_key']) - options['unlock'] = fun.unlock - - task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) if linked_model: + name = name or self.gen_task_name(fun.__name__, fun.__module__) + options['lock_key'] = f"{name}-lock" + fun = only_one(fun, options['lock_key']) + options['unlock'] = fun.unlock + task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) linked_model._etl_task = task linked_model._etl_loader = fun - + else: + task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) return task def etl(self, model, *args, **opts): @@ -56,7 +57,8 @@ def get_all_etls(self): app = DatamartCelery('datamart') app.config_from_object('django.conf:settings', namespace='CELERY') -# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) +# app.autodiscover_tasks() +app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) # app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], # related_name='tasks') # app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], @@ -90,6 +92,8 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): @task_postrun.connect def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): from django.utils import timezone + from etools_datamart.apps.subscriptions.models import Subscription + # from unicef_rest_framework.models import Service if not hasattr(sender, 'linked_model'): @@ -108,6 +112,7 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st defs['last_changes'] = timezone.now() for service in sender.linked_model.linked_services: service.invalidate_cache() + Subscription.objects.notify(sender.linked_model) defs['last_success'] = timezone.now() else: diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 06c9f7dd4..25f19dc7e 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -18,6 +18,7 @@ API_CACHE_URL=(str, "locmemcache://"), # CACHE_URL=(str, "dummycache://"), # API_CACHE_URL=(str, "dummycache://"), + ABSOLUTE_BASE_URL=(str, 'http://localhost:8000'), DISCONNECT_URL=(str, 'https://login.microsoftonline.com/unicef.org/oauth2/logout'), ENABLE_LIVE_STATS=(bool, True), CELERY_BROKER_URL=(str, 'redis://127.0.0.1:6379/2'), @@ -49,6 +50,12 @@ AZURE_OVERWRITE_FILES=(bool, True), AZURE_LOCATION=(str, ''), + EMAIL_USE_TLS=(bool, True), + EMAIL_HOST=(str, ''), + EMAIL_HOST_USER=(str, ''), + EMAIL_HOST_PASSWORD=(str, ''), + EMAIL_PORT=(int, 587), + ) DEBUG = env.bool('DEBUG') @@ -61,6 +68,7 @@ SECRET_KEY = env('SECRET_KEY') ALLOWED_HOSTS = tuple(env.list('ALLOWED_HOSTS', default=[])) +ABSOLUTE_BASE_URL = env('ABSOLUTE_BASE_URL') ADMINS = ( ('Stefano', 'saxix@saxix.onmicrosoft.com'), @@ -176,6 +184,7 @@ # 'social_core.backends.azuread_tenant.AzureADTenantOAuth2', 'unicef_security.azure.AzureADTenantOAuth2Ext', 'django.contrib.auth.backends.ModelBackend', + 'django.contrib.auth.backends.RemoteUserBackend', ] CACHES = { @@ -271,6 +280,8 @@ 'django_db_logging', 'django_sysinfo', 'crashlog', + 'post_office', + 'djcelery_email', 'django_celery_beat', @@ -279,6 +290,7 @@ 'etools_datamart.apps.data', 'etools_datamart.apps.etl.apps.Config', 'etools_datamart.apps.tracking.apps.Config', + 'etools_datamart.apps.subscriptions', 'etools_datamart.api', ] @@ -304,7 +316,21 @@ '%m/%d/%y %H:%M', # '10/25/06 14:30' '%m/%d/%y', # '10/25/06' ] - +EMAIL_BACKEND = 'post_office.EmailBackend' +EMAIL_POST_OFFICE_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST_USER = env('EMAIL_HOST_USER') +EMAIL_HOST_PASSWORD = env('EMAIL_HOST_PASSWORD') +EMAIL_HOST = env('EMAIL_HOST') +EMAIL_PORT = env.int('EMAIL_PORT') +EMAIL_USE_TLS = env.bool('EMAIL_USE_TLS') +EMAIL_SUBJECT_PREFIX = "[ETOOLS-DATAMART]" +POST_OFFICE = { + 'DEFAULT_PRIORITY': 'now', + 'BACKENDS': { + 'default': 'djcelery_email.backends.CeleryEmailBackend' + } +} +CELERY_EMAIL_CHUNK_SIZE = 10 # django-secure CSRF_COOKIE_SECURE = env.bool('CSRF_COOKIE_SECURE') SECURE_BROWSER_XSS_FILTER = True @@ -318,7 +344,6 @@ X_FRAME_OPTIONS = env('X_FRAME_OPTIONS') NOTIFICATION_SENDER = "etools_datamart@unicef.org" -EMAIL_SUBJECT_PREFIX = "[ETOOLS-DATAMART]" # django-constance CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' diff --git a/src/etools_datamart/config/urls.py b/src/etools_datamart/config/urls.py index 73df8dcce..9cbc0cf6e 100644 --- a/src/etools_datamart/config/urls.py +++ b/src/etools_datamart/config/urls.py @@ -7,26 +7,18 @@ from etools_datamart.apps.multitenant.views import SelectSchema urlpatterns = [ + path(r's/', include('etools_datamart.apps.subscriptions.urls')), path(r'', include('etools_datamart.apps.web.urls')), - path('', include('social_django.urls', namespace='social')), - - # path(r'api/auth/', include('rest_framework_social_oauth2.urls')), + path(r'', include('social_django.urls', namespace='social')), re_path(r'^authorize/?$', AuthorizationView.as_view(), name="authorize"), - # url(r'^token/?$', TokenView.as_view(), name="token"), - # url('', include('social_django.urls', namespace="social")), - # url(r'^convert-token/?$', ConvertTokenView.as_view(), name="convert_token"), - # url(r'^revoke-token/?$', RevokeTokenView.as_view(), name="revoke_token"), - # url(r'^invalidate-sessions/?$', invalidate_sessions, name="invalidate_sessions") - - path(r'api/', include(etools_datamart.api.urls), name='api'), - path('admin/', site.urls), + path(r'admin/', site.urls), path(r'admin/schemas/', SelectSchema.as_view(), name='select-schema'), + path(r'admin/sysinfo/', admin_sysinfo, name="sys-admin-info"), - path('sys/info/', http_basic_login(sysinfo), name='sys-info'), - re_path('sys/version/(?P.*)/', http_basic_login(version), name='sys-version'), - path("admin/sysinfo/", admin_sysinfo, name="sys-admin-info"), + path(r'sys/info/', http_basic_login(sysinfo), name='sys-info'), + path(r'sys/version//', http_basic_login(version), name='sys-version'), ] diff --git a/src/unicef_rest_framework/admin/service.py b/src/unicef_rest_framework/admin/service.py index 7d7027990..397d2a082 100644 --- a/src/unicef_rest_framework/admin/service.py +++ b/src/unicef_rest_framework/admin/service.py @@ -39,7 +39,7 @@ def get_stash_url(obj, label=None, **kwargs): class ServiceAdmin(ExtraUrlMixin, admin.ModelAdmin): - list_display = ('name', 'visible', 'security', 'cache_version', + list_display = ('name', 'visible', 'security', 'cache_version', 'source_model', 'json', 'admin') list_filter = ('hidden', 'access') diff --git a/src/unicef_rest_framework/migrations/0001_initial.py b/src/unicef_rest_framework/migrations/0001_initial.py index 86acb8f67..8c266ce44 100644 --- a/src/unicef_rest_framework/migrations/0001_initial.py +++ b/src/unicef_rest_framework/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-26 13:13 +# Generated by Django 2.1.3 on 2018-11-28 14:10 import uuid diff --git a/src/unicef_rest_framework/migrations/0002_auto_20181126_1313.py b/src/unicef_rest_framework/migrations/0002_auto_20181128_1410.py similarity index 91% rename from src/unicef_rest_framework/migrations/0002_auto_20181126_1313.py rename to src/unicef_rest_framework/migrations/0002_auto_20181128_1410.py index 9e5a822f1..c4e54356d 100644 --- a/src/unicef_rest_framework/migrations/0002_auto_20181126_1313.py +++ b/src/unicef_rest_framework/migrations/0002_auto_20181128_1410.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-26 13:13 +# Generated by Django 2.1.3 on 2018-11-28 14:10 import django.db.models.deletion from django.conf import settings @@ -10,9 +10,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('contenttypes', '0002_remove_content_type_name'), ('unicef_rest_framework', '0001_initial'), + ('contenttypes', '0002_remove_content_type_name'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('auth', '0009_alter_user_last_name_max_length'), ] @@ -70,7 +70,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='service', name='linked_models', - field=models.ManyToManyField(blank=True, help_text='models that the service depends on', to='contenttypes.ContentType'), + field=models.ManyToManyField(blank=True, help_text='models that the service depends on', related_name='_service_linked_models_+', to='contenttypes.ContentType'), + ), + migrations.AddField( + model_name='service', + name='source_model', + field=models.ForeignKey(blank=True, help_text='model used as primary datasource', on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType'), ), migrations.AddField( model_name='groupaccesscontrol', @@ -126,7 +131,7 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='systemfilter', - unique_together={('service', 'user'), ('service', 'group')}, + unique_together={('service', 'group'), ('service', 'user')}, ), migrations.AlterUniqueTogether( name='groupaccesscontrol', diff --git a/src/unicef_rest_framework/models/service.py b/src/unicef_rest_framework/models/service.py index 9ab318428..ee4b0fdde 100644 --- a/src/unicef_rest_framework/models/service.py +++ b/src/unicef_rest_framework/models/service.py @@ -25,16 +25,20 @@ def invalidate_cache(self, **kwargs): def get_for_viewset(self, viewset): name = getattr(viewset, 'label', viewset.__name__) - + source_model = ContentType.objects.get_for_model(viewset().get_queryset().model) service, isnew = self.model.objects.get_or_create(viewset=viewset, defaults={ 'name': name, 'cache_ttl': '1y', 'access': getattr(viewset, 'default_access', conf.DEFAULT_ACCESS), - 'description': getattr(viewset, '__doc__', "")}) - - viewset.get_service.cache_clear() + 'description': getattr(viewset, '__doc__', ""), + 'source_model': source_model + }) + if not isnew: + service.source_model = source_model + service.save() + viewset.get_service.cache_clear() return service, isnew def load_services(self): @@ -93,7 +97,13 @@ class Service(MasterDataModel): null=True, blank=True, help_text='Key used to invalidate service cache') + source_model = models.ForeignKey(ContentType, + models.CASCADE, + blank=True, + help_text="model used as primary datasource") + linked_models = models.ManyToManyField(ContentType, + related_name='+', blank=True, help_text="models that the service depends on") diff --git a/src/unicef_security/migrations/0001_initial.py b/src/unicef_security/migrations/0001_initial.py index 7cad73d43..572419c36 100644 --- a/src/unicef_security/migrations/0001_initial.py +++ b/src/unicef_security/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-26 13:13 +# Generated by Django 2.1.3 on 2018-11-28 14:10 import django.contrib.auth.models import django.contrib.auth.validators diff --git a/tests/test_subscription.py b/tests/test_subscription.py new file mode 100644 index 000000000..0bba5f296 --- /dev/null +++ b/tests/test_subscription.py @@ -0,0 +1,64 @@ +import json + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.urls import reverse + +from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.subscriptions.views import subscribe + + +@pytest.fixture() +def etltask(db): + EtlTask.objects.inspect() + return EtlTask.objects.first() + + +@pytest.mark.django_db +def test_subscribe_create(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = admin_user + res = subscribe(request, etltask.pk) + + data = json.loads(res.content) + assert data["status"] == "created" + + +@pytest.mark.django_db +def test_subscribe_update(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = admin_user + res = subscribe(request, etltask.pk) + res = subscribe(request, etltask.pk) + + data = json.loads(res.content) + assert data["status"] == "updated" + + +@pytest.mark.django_db +def test_subscribe_404(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = admin_user + res = subscribe(request, 21) + assert res.status_code == 404 + + +@pytest.mark.django_db +def test_subscribe_invalid(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 2}, content_type='application/json') + request.user = admin_user + res = subscribe(request, etltask.pk) + assert res.status_code == 400 + + +@pytest.mark.django_db +def test_subscribe_error(rf, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = AnonymousUser() + res = subscribe(request, etltask.pk) + assert res.status_code == 500 diff --git a/tests/test_views.py b/tests/test_views.py index 970e4507e..404c905e9 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -11,5 +11,5 @@ def test_home(django_app, admin_user): def test_monitor(django_app, admin_user): EtlTask.objects.inspect() - res = django_app.get(reverse('monitor')) + res = django_app.get(reverse('monitor'), user=admin_user) assert res.status_code == 200 From ac97148ca370cdee7d6f7f244915d7966d22850e Mon Sep 17 00:00:00 2001 From: sax Date: Wed, 28 Nov 2018 20:12:14 +0100 Subject: [PATCH 31/43] updates CHANGES --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 9e02d0cf2..8ca582fbf 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,7 @@ * add ability to intercept changed/unchanged datamart records * Azure storage support * use versioning in API urls +* add email notifications on dataset changes 1.6 --- From 99a978003b6a383fdd1b9d506332b1843c1ef6ae Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 29 Nov 2018 09:22:53 +0100 Subject: [PATCH 32/43] receive email notification with Excel attachment --- src/etools_datamart/apps/data/models/base.py | 6 +++ src/etools_datamart/apps/etl/models.py | 4 +- .../init/management/commands/init-setup.py | 27 ++++++++++ .../apps/subscriptions/models.py | 54 ++++++++++++++++--- .../apps/subscriptions/tasks.py | 2 +- .../templatetags/subscriptions.py | 5 -- .../renderers/__init__.py | 1 + src/unicef_rest_framework/renderers/xls.py | 9 ++++ src/unicef_rest_framework/views.py | 8 +-- tests/_test_lib/test_utilities/factories.py | 23 ++++++++ tests/test_subscription.py | 40 +++++++++++++- 11 files changed, 159 insertions(+), 20 deletions(-) create mode 100644 src/unicef_rest_framework/renderers/xls.py diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index b89f83bf6..cfdd96b01 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -1,4 +1,5 @@ from celery.local import class_property +from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models import QuerySet from django.db.models.manager import BaseManager @@ -28,6 +29,11 @@ class Meta: objects = DataMartManager() + @class_property + def service(self): + from unicef_rest_framework.models import Service + return Service.objects.get(source_model=ContentType.objects.get_for_model(self)) + @class_property def linked_services(self): from unicef_rest_framework.models import Service diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index a3f77c145..c950d129e 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -2,15 +2,15 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import JSONField from django.db import models -from django.db.models import Model from django.utils.functional import cached_property from django_celery_beat.models import PeriodicTask +from etools_datamart.apps.data.models.base import DataMartModel from etools_datamart.celery import app, ETLTask class TaskLogManager(models.Manager): - def get_for_model(self, model: Model): + def get_for_model(self, model: DataMartModel): return self.get(content_type=ContentType.objects.get_for_model(model)) def get_for_task(self, task: ETLTask): diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index a7edb37ec..82cb068ba 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -42,6 +42,28 @@
      To unsubscribe, change your preferences in Datamart Monitor
      """ +MAIL_ATTACHMENT = r"""Dear {{user.label}}, + +On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`. +You can find here in attachment a excel file with new data + +— +You are receiving this because you are subscribed to this thread. +To unsubscribe, change your preferences in {{base_url}}{% url 'monitor' %} +""" + +MAIL_ATTACHMENT_HTML = r"""
      Dear {{user.label}},
      +
       
      +
      On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`.
      +
      Attached to this email you can find excel file with new data
      +
       
      +
       
      +
       
      +
      -
      +
      You are receiving this because you are subscribed to this thread.
      +
      To unsubscribe, change your preferences in Datamart Monitor
      +""" + class Command(BaseCommand): help = "My shiny new management command." @@ -182,6 +204,11 @@ def handle(self, *args, **options): defaults={'name': 'process mail queue', 'interval': every_minute}) + EmailTemplate.objects.get_or_create(name='dataset_changed_attachment', + defaults=dict(subject='Dataset changed', + content=MAIL_ATTACHMENT, + html_content=MAIL_ATTACHMENT_HTML)) + EmailTemplate.objects.get_or_create(name='dataset_changed', defaults=dict(subject='Dataset changed', content=MAIL, diff --git a/src/etools_datamart/apps/subscriptions/models.py b/src/etools_datamart/apps/subscriptions/models.py index 9a603216e..7d465f885 100644 --- a/src/etools_datamart/apps/subscriptions/models.py +++ b/src/etools_datamart/apps/subscriptions/models.py @@ -1,21 +1,46 @@ +import logging +from io import BytesIO + from crashlog.middleware import process_exception from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.db import models +from django.utils.functional import cached_property from post_office import mail +from rest_framework.test import APIRequestFactory from unicef_rest_framework.models import Service from etools_datamart.apps.etl.models import EtlTask +logger = logging.getLogger(__name__) + class SubscriptionManager(models.Manager): def notify(self, model): ct = ContentType.objects.get_for_model(model) etl = EtlTask.objects.filter(content_type=ct).first() - + service = Service.objects.get(source_model=ct) + ret = [] for subscription in self.filter(content_type=ct).exclude(type=Subscription.NONE): + logger.info(f"Process subscription {subscription}") try: - mail.send( + if subscription.type == Subscription.EXCEL: + rf = APIRequestFactory() + request = rf.get(f"{service.endpoint}?format=xlsx") + request.user = subscription.user + request.api_info = {} # this is set my the middleware, so we must set manually here + response = service.viewset.as_view({'get': 'list'})(request) + response.render() + + # check headers set in ApiMiddleware in request.api_info + request.api_info.update(dict(response.items())) + + attachments = { + f'{model._meta.verbose_name}.xlsx': BytesIO(response.content), + } + else: + attachments = None + ret.append(mail.send( subscription.user.email, # List of email addresses also accepted 'notification@datamart.unicef.io', template='dataset_changed', # Could be an EmailTemplate instance or name @@ -25,11 +50,14 @@ def notify(self, model): 'verbose_name': model._meta.verbose_name, 'etl': etl, 'model': ct.model, - 'service': Service.objects.get(source_model=ct) + 'service': service }, - ) - except Exception as e: + attachments=attachments + )) + except Exception as e: # pragma: no cover + logger.exception(e) process_exception(e) + return ret class Subscription(models.Model): @@ -39,7 +67,7 @@ class Subscription(models.Model): TYPES = ((NONE, 'None'), (MESSAGE, 'Email'), - # (EXCEL, 'Email+Excel'), + (EXCEL, 'Email+Excel'), ) user = models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE, related_name='subscriptions') @@ -53,4 +81,16 @@ class Meta: unique_together = ('user', 'content_type', 'kwargs') def __str__(self): - return f"{self.user} {self.get_type_display()}" + return f"#{self.pk} {self.user} {self.get_type_display()} {self.content_type}" + + @cached_property + def endpoint(self): + return self.content_type.model_class().service.endpoint + + @cached_property + def service(self): + return self.content_type.model_class().service + + @cached_property + def viewset(self): + return self.content_type.model_class().service.viewset diff --git a/src/etools_datamart/apps/subscriptions/tasks.py b/src/etools_datamart/apps/subscriptions/tasks.py index f0c831ad8..846561377 100644 --- a/src/etools_datamart/apps/subscriptions/tasks.py +++ b/src/etools_datamart/apps/subscriptions/tasks.py @@ -4,5 +4,5 @@ @app.task(name='send_queued_mail') -def send_queued_mail(): +def send_queued_mail(): # pragma: no cover call_command('send_queued_mail') diff --git a/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py b/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py index 39fc2a4de..68ca19cbe 100644 --- a/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py +++ b/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py @@ -5,11 +5,6 @@ register = template.Library() -@register.filter -def get_for_task(subscriptions): - return 1 - - @register.inclusion_tag('subscription_select.html', takes_context=True) def subscription_select(context, task): user = context['user'] diff --git a/src/unicef_rest_framework/renderers/__init__.py b/src/unicef_rest_framework/renderers/__init__.py index 9651defec..14ec17a16 100644 --- a/src/unicef_rest_framework/renderers/__init__.py +++ b/src/unicef_rest_framework/renderers/__init__.py @@ -1,3 +1,4 @@ from .api import APIBrowsableAPIRenderer # noqa from .microsoft.json import MSJSONRenderer # noqa from .microsoft.xml import MSXmlRenderer # noqa +from .xls import XLSXRenderer # noqa diff --git a/src/unicef_rest_framework/renderers/xls.py b/src/unicef_rest_framework/renderers/xls.py new file mode 100644 index 000000000..1d89c11c2 --- /dev/null +++ b/src/unicef_rest_framework/renderers/xls.py @@ -0,0 +1,9 @@ +from drf_renderer_xlsx.renderers import XLSXRenderer as _XLSXRenderer + + +class XLSXRenderer(_XLSXRenderer): + + def render(self, data, accepted_media_type=None, renderer_context=None): + if not data['results']: + return '' + return super().render(data, accepted_media_type, renderer_context) diff --git a/src/unicef_rest_framework/views.py b/src/unicef_rest_framework/views.py index ba45e83cb..62abd2ef7 100644 --- a/src/unicef_rest_framework/views.py +++ b/src/unicef_rest_framework/views.py @@ -3,7 +3,6 @@ import rest_framework_extensions.utils from drf_querystringfilter.backend import QueryStringFilterBackend -from drf_renderer_xlsx.renderers import XLSXRenderer from dynamic_serializer.core import DynamicSerializerMixin from rest_framework import viewsets from rest_framework.authentication import BasicAuthentication, SessionAuthentication, TokenAuthentication @@ -17,7 +16,7 @@ from unicef_rest_framework.cache import cache_response, etag, ListKeyConstructor from unicef_rest_framework.filtering import SystemFilterBackend from unicef_rest_framework.permissions import ServicePermission -from unicef_rest_framework.renderers import APIBrowsableAPIRenderer, MSJSONRenderer, MSXmlRenderer +from unicef_rest_framework.renderers import APIBrowsableAPIRenderer, MSJSONRenderer, MSXmlRenderer, XLSXRenderer from unicef_rest_framework.renderers.csv import CSVRenderer @@ -65,8 +64,9 @@ def store(self, key, value): self.request._request.api_info[key] = value def dispatch(self, request, *args, **kwargs): - request.api_info["view"] = fqn(self) - request.api_info["service"] = self.get_service() + if hasattr(request, 'api_info'): + request.api_info["view"] = fqn(self) + request.api_info["service"] = self.get_service() return super().dispatch(request, *args, **kwargs) diff --git a/tests/_test_lib/test_utilities/factories.py b/tests/_test_lib/test_utilities/factories.py index fa57ff038..ab1823d1a 100644 --- a/tests/_test_lib/test_utilities/factories.py +++ b/tests/_test_lib/test_utilities/factories.py @@ -4,12 +4,16 @@ import factory import unicef_security.models from django.contrib.auth.models import Group +from django.contrib.contenttypes.models import ContentType from django.db import connections from django.utils import timezone +from factory import SubFactory +from post_office.models import EmailTemplate from unicef_rest_framework.models import Service, SystemFilter, UserAccessControl from etools_datamart.apps.data.models import FAMIndicator, HACT, Intervention, PMPIndicators, UserStats from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.subscriptions.models import Subscription from etools_datamart.apps.tracking.models import APIRequestLog today = timezone.now() @@ -137,3 +141,22 @@ def rules(self, create, extracted, **kwargs): for field, value in extracted.items(): rule = self.rules.create(field=field, value=value) rule.save() + + +class SubscriptionFactory(factory.DjangoModelFactory): + kwargs = '' + user = SubFactory(UserFactory) + type = Subscription.MESSAGE + content_type = lambda x: ContentType.objects.get_for_model(HACT) # noqa: E731 + + class Meta: + model = Subscription + django_get_or_create = ('user', 'content_type') + + +class EmailTemplateFactory(factory.DjangoModelFactory): + name = 'dataset_changed' + + class Meta: + model = EmailTemplate + django_get_or_create = ('name',) diff --git a/tests/test_subscription.py b/tests/test_subscription.py index 0bba5f296..3320c0420 100644 --- a/tests/test_subscription.py +++ b/tests/test_subscription.py @@ -3,8 +3,11 @@ import pytest from django.contrib.auth.models import AnonymousUser from django.urls import reverse +from test_utilities.factories import EmailTemplateFactory, SubscriptionFactory +from unicef_rest_framework.test_utils import user_allow_service from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.subscriptions.models import Subscription from etools_datamart.apps.subscriptions.views import subscribe @@ -14,6 +17,23 @@ def etltask(db): return EtlTask.objects.first() +@pytest.fixture() +def subscription(etltask): + return SubscriptionFactory(content_type=etltask.content_type, + type=Subscription.MESSAGE) + + +@pytest.fixture() +def subscription_attachment(etltask): + return SubscriptionFactory(content_type=etltask.content_type, + type=Subscription.EXCEL) + + +@pytest.fixture() +def email_template(): + return EmailTemplateFactory() + + @pytest.mark.django_db def test_subscribe_create(rf, admin_user, etltask): request = rf.post(reverse("subscribe", args=[etltask.pk]), @@ -49,7 +69,7 @@ def test_subscribe_404(rf, admin_user, etltask): @pytest.mark.django_db def test_subscribe_invalid(rf, admin_user, etltask): request = rf.post(reverse("subscribe", args=[etltask.pk]), - {"type": 2}, content_type='application/json') + {"type": 3}, content_type='application/json') request.user = admin_user res = subscribe(request, etltask.pk) assert res.status_code == 400 @@ -62,3 +82,21 @@ def test_subscribe_error(rf, etltask): request.user = AnonymousUser() res = subscribe(request, etltask.pk) assert res.status_code == 500 + + +@pytest.mark.django_db +def test_notification_email(subscription: Subscription, email_template): + with user_allow_service(subscription.user, subscription.viewset): + emails = Subscription.objects.notify(subscription.content_type.model_class()) + assert len(emails) == 1 + assert emails[0].to == [subscription.user.email] + assert emails[0].attachments.count() == 0 + + +@pytest.mark.django_db +def test_notification_email_attachment(subscription_attachment: Subscription, email_template): + with user_allow_service(subscription_attachment.user, subscription_attachment.viewset): + emails = Subscription.objects.notify(subscription_attachment.content_type.model_class()) + assert len(emails) == 1 + assert emails[0].to == [subscription_attachment.user.email] + assert emails[0].attachments.count() == 1 From ff9634054f9e9c983d95ca49cfd3d05964ae5477 Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 29 Nov 2018 11:09:22 +0100 Subject: [PATCH 33/43] add last_modify_date filter --- CHANGES | 2 + src/drf_querystringfilter/backend.py | 29 ++++++----- src/etools_datamart/api/endpoints/common.py | 23 ++++++++- .../api/endpoints/datamart/famindicator.py | 4 +- .../api/endpoints/datamart/hact.py | 4 +- .../api/endpoints/datamart/intervention.py | 4 +- .../api/endpoints/datamart/pmpindicators.py | 4 +- .../api/endpoints/datamart/user.py | 4 +- .../apps/data/migrations/0001_initial.py | 13 +++-- src/etools_datamart/apps/data/models/base.py | 4 +- .../apps/etl/migrations/0001_initial.py | 2 +- .../subscriptions/migrations/0001_initial.py | 2 +- .../migrations/0002_auto_20181128_1410.py | 27 ---------- .../apps/tracking/migrations/0001_initial.py | 2 +- .../migrations/0002_auto_20181128_1410.py | 51 ------------------- .../migrations/0001_initial.py | 2 +- ...128_1410.py => 0002_auto_20181129_0824.py} | 8 +-- .../migrations/0001_initial.py | 2 +- 18 files changed, 67 insertions(+), 120 deletions(-) delete mode 100644 src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py delete mode 100644 src/etools_datamart/apps/tracking/migrations/0002_auto_20181128_1410.py rename src/unicef_rest_framework/migrations/{0002_auto_20181128_1410.py => 0002_auto_20181129_0824.py} (99%) diff --git a/CHANGES b/CHANGES index 8ca582fbf..083a8e1a1 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,8 @@ * Azure storage support * use versioning in API urls * add email notifications on dataset changes +* add filter by 'lat_modify_date' + 1.6 --- diff --git a/src/drf_querystringfilter/backend.py b/src/drf_querystringfilter/backend.py index 8e374f6a5..cf5629c6e 100644 --- a/src/drf_querystringfilter/backend.py +++ b/src/drf_querystringfilter/backend.py @@ -8,6 +8,7 @@ import coreapi import coreschema from django import forms +from django.core.exceptions import FieldError from django.db import models from django.db.models import BooleanField, FieldDoesNotExist from django.template import loader @@ -127,8 +128,8 @@ def _get_filters(self, request, queryset, view): # noqa """ self.opts = queryset.model._meta filter_fields = getattr(view, 'filter_fields', None) - exclude = {} - filters = {} + self.exclude = {} + self.filters = {} if filter_fields: blacklist = RexList(getattr(view, 'filter_blacklist', [])) @@ -140,9 +141,9 @@ def _get_filters(self, request, queryset, view): # noqa if negate: filter_field_name = fieldname_arg[:-1] - TARGET = exclude + TARGET = self.exclude else: - TARGET = filters + TARGET = self.filters filter_field_name = fieldname_arg if filter_field_name in self.excluded_query_params: @@ -163,8 +164,8 @@ def _get_filters(self, request, queryset, view): # noqa # parts = [field_name] processor = getattr(self, 'process_{}'.format(filter_field_name), None) - # if (field_name not in filter_fields) and (not processor): - # raise InvalidQueryArgumentError(fieldname_arg) + if (filter_field_name not in filter_fields) and (not processor): + raise InvalidQueryArgumentError(filter_field_name) # field is configured in Serializer # so we use 'source' attribute if filter_field_name in mapping: @@ -185,9 +186,9 @@ def _get_filters(self, request, queryset, view): # noqa 'parts': parts, 'value': raw_value, 'real_field_name': real_field_name} - _f, _e = processor(dict(filters), dict(exclude), **payload) - filters.update(**_f) - exclude.update(**_e) + _f, _e = processor(dict(self.filters), dict(self.exclude), **payload) + self.filters.update(**_f) + self.exclude.update(**_e) else: # field_object = opts.get_field(real_field_name) value_type = self.field_type(real_field_name) @@ -211,20 +212,22 @@ def _get_filters(self, request, queryset, view): # noqa except Exception as e: logger.exception(e) raise - return filters, exclude + return self.filters, self.exclude def filter_queryset(self, request, queryset, view): self.request = request try: - self.filters, self.exclude = self._get_filters(request, queryset, view) - qs = queryset.filter(**self.filters).exclude(**self.exclude) + filters, exclude = self._get_filters(request, queryset, view) + qs = queryset.filter(**filters).exclude(**exclude) logger.debug("""Filtering using: {} -{}""".format(self.filters, self.exclude)) +{}""".format(filters, exclude)) # if '_distinct' in self.query_params: # f = self.get_param_value('_distinct') # qs = qs.order_by(*f).distinct(*f) return qs + except FieldError as e: + raise QueryFilterException(str(e)) except (InvalidFilterError, QueryFilterException) as e: logger.exception(e) raise diff --git a/src/etools_datamart/api/endpoints/common.py b/src/etools_datamart/api/endpoints/common.py index 664aafba6..408056e1f 100644 --- a/src/etools_datamart/api/endpoints/common.py +++ b/src/etools_datamart/api/endpoints/common.py @@ -7,6 +7,7 @@ from django.http import Http404 from drf_querystringfilter.exceptions import QueryFilterException from dynamic_serializer.core import DynamicSerializerMixin +from rest_framework.decorators import action from rest_framework.exceptions import NotAuthenticated, PermissionDenied from rest_framework.filters import OrderingFilter from rest_framework.response import Response @@ -14,6 +15,7 @@ from unicef_rest_framework.views import ReadOnlyModelViewSet from etools_datamart.api.filtering import DatamartQueryStringFilterBackend, TenantQueryStringFilterBackend +from etools_datamart.apps.etl.models import EtlTask from etools_datamart.apps.multitenant.exceptions import InvalidSchema, NotAuthorizedSchema __all__ = ['APIMultiTenantReadOnlyModelViewSet'] @@ -43,6 +45,20 @@ def build_description(self): return description +class UpdatesMixin: + pass + # @action(methods=['get'], detail=False) + # def updates(self, request, version): + # """ Returns only records changed from last ETL task""" + # task = EtlTask.objects.get_for_model(self.queryset.model) + # queryset = self.queryset.filter(last_modify_date__gte=task.last_changes.date()) + # + # serializer = self.get_serializer(queryset, many=True) + # return Response(serializer.data, + # headers={'update-date': task.last_changes.strftime('%Y %B %d')}) + # + + class APIReadOnlyModelViewSet(ReadOnlyModelViewSet): filter_backends = [SystemFilterBackend, DatamartQueryStringFilterBackend, @@ -68,7 +84,8 @@ def drf_ignore_filter(self, request, field): def handle_exception(self, exc): conn = connections['etools'] - if isinstance(exc, QueryFilterException): + if isinstance(exc, (QueryFilterException,)): + # FieldError can happen due cache attempt to create return Response({"error": str(exc)}, status=400) elif isinstance(exc, NotAuthenticated): return Response({"error": "Authentication credentials were not provided."}, status=401) @@ -149,3 +166,7 @@ def get_schema_fields(self): schema=coreschema.String(description="comma separated list of schemas") )) return ret + + +class DataMartViewSet(APIReadOnlyModelViewSet, UpdatesMixin): + pass diff --git a/src/etools_datamart/api/endpoints/datamart/famindicator.py b/src/etools_datamart/api/endpoints/datamart/famindicator.py index 9c4d656c1..03e9c15e0 100644 --- a/src/etools_datamart/api/endpoints/datamart/famindicator.py +++ b/src/etools_datamart/api/endpoints/datamart/famindicator.py @@ -7,8 +7,8 @@ from .. import common -class FAMIndicatorViewSet(common.APIReadOnlyModelViewSet): +class FAMIndicatorViewSet(common.DataMartViewSet): serializer_class = serializers.FAMIndicatorSerializer queryset = models.FAMIndicator.objects.all() - filter_fields = ('country_name', ) + filter_fields = ('country_name', 'last_modify_date') filter_backends = [MonthFilterBackend] + common.APIReadOnlyModelViewSet.filter_backends diff --git a/src/etools_datamart/api/endpoints/datamart/hact.py b/src/etools_datamart/api/endpoints/datamart/hact.py index 8cbebff2a..3f86d31e8 100644 --- a/src/etools_datamart/api/endpoints/datamart/hact.py +++ b/src/etools_datamart/api/endpoints/datamart/hact.py @@ -5,7 +5,7 @@ from .. import common -class HACTViewSet(common.APIReadOnlyModelViewSet): +class HACTViewSet(common.DataMartViewSet): serializer_class = serializers.HACTSerializer queryset = models.HACT.objects.all() - filter_fields = ('country_name', 'month') + filter_fields = ('country_name', 'month', 'last_modify_date') diff --git a/src/etools_datamart/api/endpoints/datamart/intervention.py b/src/etools_datamart/api/endpoints/datamart/intervention.py index 627cba00d..6b344552b 100644 --- a/src/etools_datamart/api/endpoints/datamart/intervention.py +++ b/src/etools_datamart/api/endpoints/datamart/intervention.py @@ -35,13 +35,13 @@ class Meta: # } -class InterventionViewSet(common.APIReadOnlyModelViewSet): +class InterventionViewSet(common.DataMartViewSet): """ """ serializer_class = serializers.InterventionSerializer queryset = models.Intervention.objects.all() - filter_fields = ('country_name', 'title', 'status', + filter_fields = ('country_name', 'title', 'status', 'last_modify_date', 'start_date', 'submission_date', 'document_type') serializers_fieldsets = {'std': None, 'short': ["title", "number"]} diff --git a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py index 4ba01e7e5..3289b3ec4 100644 --- a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py +++ b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py @@ -5,12 +5,12 @@ from .. import common -class PMPIndicatorsViewSet(common.APIReadOnlyModelViewSet): +class PMPIndicatorsViewSet(common.DataMartViewSet): """ """ serializer_class = serializers.PMPIndicatorsSerializer queryset = models.PMPIndicators.objects.all() filter_fields = ('country_name', 'business_area_code', 'vendor_number', - 'partner_name', 'partner_type', + 'partner_name', 'partner_type', 'last_modify_date', 'pd_ssfa_ref', ) diff --git a/src/etools_datamart/api/endpoints/datamart/user.py b/src/etools_datamart/api/endpoints/datamart/user.py index 569d037c1..b442c66d2 100644 --- a/src/etools_datamart/api/endpoints/datamart/user.py +++ b/src/etools_datamart/api/endpoints/datamart/user.py @@ -5,7 +5,7 @@ from .. import common -class UserStatsViewSet(common.APIReadOnlyModelViewSet): +class UserStatsViewSet(common.DataMartViewSet): serializer_class = serializers.UserStatsSerializer queryset = models.UserStats.objects.all() - filter_fields = ('country_name', 'month') + filter_fields = ('country_name', 'month', 'last_modify_date') diff --git a/src/etools_datamart/apps/data/migrations/0001_initial.py b/src/etools_datamart/apps/data/migrations/0001_initial.py index ca15c1e2b..9bbc51de0 100644 --- a/src/etools_datamart/apps/data/migrations/0001_initial.py +++ b/src/etools_datamart/apps/data/migrations/0001_initial.py @@ -1,8 +1,7 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.contrib.postgres.fields.jsonb import month_field.models -import unicef_security.models from django.db import migrations, models @@ -20,6 +19,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('spotcheck_ip_contacted', models.IntegerField(default=0, verbose_name='Spot Check-IP Contacted')), ('spotcheck_report_submitted', models.IntegerField(default=0, verbose_name='Spot Check-Report Submitted')), @@ -42,7 +42,6 @@ class Migration(migrations.Migration): 'verbose_name': 'FAM Indicator', 'ordering': ('month', 'country_name'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), ), migrations.CreateModel( name='HACT', @@ -50,6 +49,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('year', models.IntegerField()), ('microassessments_total', models.IntegerField(default=0, help_text='Total number of completed Microassessments in the business area in the past year')), ('programmaticvisits_total', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), @@ -62,7 +62,6 @@ class Migration(migrations.Migration): 'verbose_name': 'HACT', 'ordering': ('year', 'country_name'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), ), migrations.CreateModel( name='Intervention', @@ -70,6 +69,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('created', models.DateTimeField(auto_now=True)), ('updated', models.DateTimeField(null=True)), ('document_type', models.CharField(max_length=255, null=True)), @@ -115,7 +115,6 @@ class Migration(migrations.Migration): 'verbose_name': 'Intervention', 'ordering': ('country_name', 'title'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), ), migrations.CreateModel( name='PMPIndicators', @@ -123,6 +122,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('vendor_number', models.CharField(db_index=True, max_length=255, null=True)), ('business_area_code', models.CharField(db_index=True, max_length=100, null=True)), ('partner_name', models.CharField(db_index=True, max_length=255, null=True)), @@ -157,7 +157,6 @@ class Migration(migrations.Migration): 'verbose_name': 'PMP Indicator', 'ordering': ('country_name', 'partner_name'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), ), migrations.CreateModel( name='UserStats', @@ -165,6 +164,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('total', models.IntegerField(default=0, verbose_name='Total users')), ('unicef', models.IntegerField(default=0, verbose_name='UNICEF uswers')), @@ -175,7 +175,6 @@ class Migration(migrations.Migration): 'verbose_name': 'User Access Statistics', 'ordering': ('-month', 'country_name'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), ), migrations.AlterUniqueTogether( name='userstats', diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index cfdd96b01..3891c199b 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -3,7 +3,6 @@ from django.db import models from django.db.models import QuerySet from django.db.models.manager import BaseManager -from unicef_security.models import TimeStampedModel class DataMartQuerySet(QuerySet): @@ -20,9 +19,10 @@ class DataMartManager(BaseManager.from_queryset(DataMartQuerySet)): # self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) -class DataMartModel(models.Model, TimeStampedModel): +class DataMartModel(models.Model): country_name = models.CharField(max_length=50, db_index=True) schema_name = models.CharField(max_length=50, db_index=True) + last_modify_date = models.DateTimeField(blank=True, auto_now=True) class Meta: abstract = True diff --git a/src/etools_datamart/apps/etl/migrations/0001_initial.py b/src/etools_datamart/apps/etl/migrations/0001_initial.py index dba214ef8..034d0d20f 100644 --- a/src/etools_datamart/apps/etl/migrations/0001_initial.py +++ b/src/etools_datamart/apps/etl/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.contrib.postgres.fields.jsonb import django.db.models.deletion diff --git a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py index c783b380b..8fee9683c 100644 --- a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py +++ b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.db.models.deletion from django.db import migrations, models diff --git a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py deleted file mode 100644 index 8276d0f3a..000000000 --- a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181128_1410.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('subscriptions', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='subscription', - name='user', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='subscriptions', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterUniqueTogether( - name='subscription', - unique_together={('user', 'content_type', 'kwargs')}, - ), - ] diff --git a/src/etools_datamart/apps/tracking/migrations/0001_initial.py b/src/etools_datamart/apps/tracking/migrations/0001_initial.py index f60153acf..425d9911c 100644 --- a/src/etools_datamart/apps/tracking/migrations/0001_initial.py +++ b/src/etools_datamart/apps/tracking/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.utils.timezone import strategy_field.fields diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181128_1410.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181128_1410.py deleted file mode 100644 index 88428e3fc..000000000 --- a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181128_1410.py +++ /dev/null @@ -1,51 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('unicef_rest_framework', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('tracking', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='usercounter', - name='user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='pathcounter', - name='service', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='unicef_rest_framework.Service'), - ), - migrations.AddField( - model_name='monthlycounter', - name='user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='apirequestlog', - name='user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), - ), - migrations.AlterUniqueTogether( - name='usercounter', - unique_together={('day', 'user')}, - ), - migrations.AlterUniqueTogether( - name='pathcounter', - unique_together={('day', 'path')}, - ), - migrations.AlterUniqueTogether( - name='monthlycounter', - unique_together={('day', 'user')}, - ), - ] diff --git a/src/unicef_rest_framework/migrations/0001_initial.py b/src/unicef_rest_framework/migrations/0001_initial.py index 8c266ce44..06eaedc84 100644 --- a/src/unicef_rest_framework/migrations/0001_initial.py +++ b/src/unicef_rest_framework/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import uuid diff --git a/src/unicef_rest_framework/migrations/0002_auto_20181128_1410.py b/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py similarity index 99% rename from src/unicef_rest_framework/migrations/0002_auto_20181128_1410.py rename to src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py index c4e54356d..bf25f9213 100644 --- a/src/unicef_rest_framework/migrations/0002_auto_20181128_1410.py +++ b/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.db.models.deletion from django.conf import settings @@ -10,10 +10,10 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('unicef_rest_framework', '0001_initial'), - ('contenttypes', '0002_remove_content_type_name'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('auth', '0009_alter_user_last_name_max_length'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('contenttypes', '0002_remove_content_type_name'), + ('unicef_rest_framework', '0001_initial'), ] operations = [ diff --git a/src/unicef_security/migrations/0001_initial.py b/src/unicef_security/migrations/0001_initial.py index 572419c36..b87338433 100644 --- a/src/unicef_security/migrations/0001_initial.py +++ b/src/unicef_security/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-28 14:10 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.contrib.auth.models import django.contrib.auth.validators From d716285f4130d25e9c0a5709c5fea8d94abad7f8 Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 29 Nov 2018 11:31:04 +0100 Subject: [PATCH 34/43] add 'updates/' endpoint --- CHANGES | 3 +- src/etools_datamart/api/endpoints/common.py | 22 ++++---- .../migrations/0002_auto_20181129_0824.py | 27 ++++++++++ .../apps/subscriptions/models.py | 5 +- .../migrations/0002_auto_20181129_0824.py | 51 +++++++++++++++++++ src/etools_datamart/celery.py | 3 +- tests/test_subscription.py | 9 ++-- 7 files changed, 101 insertions(+), 19 deletions(-) create mode 100644 src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py create mode 100644 src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py diff --git a/CHANGES b/CHANGES index 083a8e1a1..5b8ce1fdd 100644 --- a/CHANGES +++ b/CHANGES @@ -7,7 +7,8 @@ * Azure storage support * use versioning in API urls * add email notifications on dataset changes -* add filter by 'lat_modify_date' +* add filter by 'last_modify_date' +* add 'updates/' endpoint to fetch only changes from last ETL 1.6 diff --git a/src/etools_datamart/api/endpoints/common.py b/src/etools_datamart/api/endpoints/common.py index 408056e1f..253e2e8e4 100644 --- a/src/etools_datamart/api/endpoints/common.py +++ b/src/etools_datamart/api/endpoints/common.py @@ -46,17 +46,17 @@ def build_description(self): class UpdatesMixin: - pass - # @action(methods=['get'], detail=False) - # def updates(self, request, version): - # """ Returns only records changed from last ETL task""" - # task = EtlTask.objects.get_for_model(self.queryset.model) - # queryset = self.queryset.filter(last_modify_date__gte=task.last_changes.date()) - # - # serializer = self.get_serializer(queryset, many=True) - # return Response(serializer.data, - # headers={'update-date': task.last_changes.strftime('%Y %B %d')}) - # + + @action(methods=['get'], detail=False) + def updates(self, request, version): + """ Returns only records changed from last ETL task""" + task = EtlTask.objects.get_for_model(self.queryset.model) + offset = task.last_changes.strftime('%Y-%m-%d %H:%M') + queryset = self.queryset.filter(last_modify_date__gte=offset) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data, + headers={'update-date': offset}) class APIReadOnlyModelViewSet(ReadOnlyModelViewSet): diff --git a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py new file mode 100644 index 000000000..da9944c95 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py @@ -0,0 +1,27 @@ +# Generated by Django 2.1.3 on 2018-11-29 08:24 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('subscriptions', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='subscription', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='subscriptions', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterUniqueTogether( + name='subscription', + unique_together={('user', 'content_type', 'kwargs')}, + ), + ] diff --git a/src/etools_datamart/apps/subscriptions/models.py b/src/etools_datamart/apps/subscriptions/models.py index 7d465f885..e8f289f86 100644 --- a/src/etools_datamart/apps/subscriptions/models.py +++ b/src/etools_datamart/apps/subscriptions/models.py @@ -38,12 +38,15 @@ def notify(self, model): attachments = { f'{model._meta.verbose_name}.xlsx': BytesIO(response.content), } + template = 'dataset_changed_attachment' else: attachments = None + template = 'dataset_changed' + ret.append(mail.send( subscription.user.email, # List of email addresses also accepted 'notification@datamart.unicef.io', - template='dataset_changed', # Could be an EmailTemplate instance or name + template=template, context={'subscription': subscription, 'user': subscription.user, 'base_url': settings.ABSOLUTE_BASE_URL, diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py new file mode 100644 index 000000000..f98434cb8 --- /dev/null +++ b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py @@ -0,0 +1,51 @@ +# Generated by Django 2.1.3 on 2018-11-29 08:24 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('unicef_rest_framework', '0001_initial'), + ('tracking', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='usercounter', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='pathcounter', + name='service', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='unicef_rest_framework.Service'), + ), + migrations.AddField( + model_name='monthlycounter', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='apirequestlog', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterUniqueTogether( + name='usercounter', + unique_together={('day', 'user')}, + ), + migrations.AlterUniqueTogether( + name='pathcounter', + unique_together={('day', 'path')}, + ), + migrations.AlterUniqueTogether( + name='monthlycounter', + unique_together={('day', 'user')}, + ), + ] diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 9cb4eb74a..5bb98c317 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -103,8 +103,7 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st except KeyError: # pragma: no cover cost = -1 defs = {'elapsed': cost, - 'status': state, - 'last_run': timezone.now()} + 'status': state} if state == 'SUCCESS': defs['results'] = retval.as_dict() diff --git a/tests/test_subscription.py b/tests/test_subscription.py index 3320c0420..e32c0f3ef 100644 --- a/tests/test_subscription.py +++ b/tests/test_subscription.py @@ -30,8 +30,9 @@ def subscription_attachment(etltask): @pytest.fixture() -def email_template(): - return EmailTemplateFactory() +def email_templates(): + return (EmailTemplateFactory(name='dataset_changed'), + EmailTemplateFactory(name='dataset_changed_attachment')) @pytest.mark.django_db @@ -85,7 +86,7 @@ def test_subscribe_error(rf, etltask): @pytest.mark.django_db -def test_notification_email(subscription: Subscription, email_template): +def test_notification_email(subscription: Subscription, email_templates): with user_allow_service(subscription.user, subscription.viewset): emails = Subscription.objects.notify(subscription.content_type.model_class()) assert len(emails) == 1 @@ -94,7 +95,7 @@ def test_notification_email(subscription: Subscription, email_template): @pytest.mark.django_db -def test_notification_email_attachment(subscription_attachment: Subscription, email_template): +def test_notification_email_attachment(subscription_attachment: Subscription, email_templates): with user_allow_service(subscription_attachment.user, subscription_attachment.viewset): emails = Subscription.objects.notify(subscription_attachment.content_type.model_class()) assert len(emails) == 1 From f084a280bf91f90a4a865b48dea7758030448236 Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 29 Nov 2018 12:14:40 +0100 Subject: [PATCH 35/43] add xhtml renderer --- .../renderers/__init__.py | 1 + src/unicef_rest_framework/renderers/html.py | 21 ++++++++++++++ .../templates/renderers/html.html | 29 +++++++++++++++++++ src/unicef_rest_framework/views.py | 4 ++- 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 src/unicef_rest_framework/renderers/html.py create mode 100644 src/unicef_rest_framework/templates/renderers/html.html diff --git a/src/unicef_rest_framework/renderers/__init__.py b/src/unicef_rest_framework/renderers/__init__.py index 14ec17a16..cb69181cc 100644 --- a/src/unicef_rest_framework/renderers/__init__.py +++ b/src/unicef_rest_framework/renderers/__init__.py @@ -2,3 +2,4 @@ from .microsoft.json import MSJSONRenderer # noqa from .microsoft.xml import MSXmlRenderer # noqa from .xls import XLSXRenderer # noqa +from .html import HTMLRenderer # noqa diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py new file mode 100644 index 000000000..b23fefd92 --- /dev/null +++ b/src/unicef_rest_framework/renderers/html.py @@ -0,0 +1,21 @@ +from django.template import loader +from rest_framework.renderers import BaseRenderer + + +class HTMLRenderer(BaseRenderer): + media_type = 'text/html' + format = 'xhtml' + charset = 'utf-8' + render_style = 'text' + + def render(self, data, accepted_media_type=None, renderer_context=None): + model = renderer_context['view'].queryset.model + opts = model._meta + template = loader.select_template([ + f'renderers/{opts.app_label}/{opts.model_name}.html', + 'renderers/html.html']) + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': data['results'][0].keys()} + return template.render(c) diff --git a/src/unicef_rest_framework/templates/renderers/html.html b/src/unicef_rest_framework/templates/renderers/html.html new file mode 100644 index 000000000..b370db53f --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/html.html @@ -0,0 +1,29 @@ +{% load static datamart %} + + + + {{ opts.verbose_name }} + + + +

      {{ opts.verbose_name }}

      + + + {% for v in headers %} + + {% endfor %} + + {% for row in data.results %} + + {% for k,v in row.items %} + + {% endfor %} + + {% endfor %} +
      {{ v }}
      {{ v }}
      + + diff --git a/src/unicef_rest_framework/views.py b/src/unicef_rest_framework/views.py index 62abd2ef7..959e4414c 100644 --- a/src/unicef_rest_framework/views.py +++ b/src/unicef_rest_framework/views.py @@ -16,7 +16,8 @@ from unicef_rest_framework.cache import cache_response, etag, ListKeyConstructor from unicef_rest_framework.filtering import SystemFilterBackend from unicef_rest_framework.permissions import ServicePermission -from unicef_rest_framework.renderers import APIBrowsableAPIRenderer, MSJSONRenderer, MSXmlRenderer, XLSXRenderer +from unicef_rest_framework.renderers import (APIBrowsableAPIRenderer, HTMLRenderer, + MSJSONRenderer, MSXmlRenderer, XLSXRenderer,) from unicef_rest_framework.renderers.csv import CSVRenderer @@ -48,6 +49,7 @@ class ReadOnlyModelViewSet(DynamicSerializerMixin, viewsets.ReadOnlyModelViewSet CSVRenderer, YAMLRenderer, XLSXRenderer, + HTMLRenderer, MSJSONRenderer, XMLRenderer, MSXmlRenderer, From f28858d74eef81fb572623408fbe2ebdacad5160 Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 29 Nov 2018 12:48:05 +0100 Subject: [PATCH 36/43] add pdf renderer --- CHANGES | 2 + Pipfile | 2 + Pipfile.lock | 195 +++++++++++++----- .../renderers/__init__.py | 1 + src/unicef_rest_framework/renderers/pdf.py | 52 +++++ src/unicef_rest_framework/views.py | 5 +- 6 files changed, 209 insertions(+), 48 deletions(-) create mode 100644 src/unicef_rest_framework/renderers/pdf.py diff --git a/CHANGES b/CHANGES index 5b8ce1fdd..79c706c30 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,8 @@ * add email notifications on dataset changes * add filter by 'last_modify_date' * add 'updates/' endpoint to fetch only changes from last ETL +* add XHTML renderer +* add PDF renderer 1.6 diff --git a/Pipfile b/Pipfile index abf8adc6a..5c09fe079 100644 --- a/Pipfile +++ b/Pipfile @@ -53,6 +53,8 @@ azure-storage = "*" django-basicauth = "*" django-post-office = "*" django-celery-email = "*" +"xhtml2pdf" = "*" +pisa = "*" [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index a9b18c25c..355c31e12 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "27eb754085af424f6ca84bd7961e8b161086e7a5cb591de162845faf06b98390" + "sha256": "19e2894907faa1249e7a55f7edfda1a976c93470fe669c30a64901f031efae41" }, "pipfile-spec": 6, "requires": { @@ -1023,11 +1023,11 @@ "validation" ], "hashes": [ - "sha256:24a02bbf56361ae0e304744f9c4aa96544270decd9d79b37d10dfd6dd04e5f22", - "sha256:b07192a6697ced6da49c5b016f1805960b0a7a1682fb21e86045a6bb572ffa99" + "sha256:9ee2072fb84ec60d951fa105e6926cf16e332973ba20ab2e3962fd9445cfd102", + "sha256:b0d5304cd2180699980fc8336edb5bfb774bbdfb79760376ed69538bf49cdcb2" ], "index": "pypi", - "version": "==1.11.0" + "version": "==1.11.1" }, "et-xmlfile": { "hashes": [ @@ -1051,6 +1051,13 @@ "index": "pypi", "version": "==19.9.0" }, + "html5lib": { + "hashes": [ + "sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3", + "sha256:66cb0dcfdbbc4f9c3ba1a63fdb511ffdbd4f513b2b6d81b80cd26ce6b3fb3736" + ], + "version": "==1.0.1" + }, "humanize": { "hashes": [ "sha256:a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19" @@ -1189,9 +1196,52 @@ }, "openpyxl": { "hashes": [ - "sha256:8b0c2a44f394a7a913a2e7cdcc1dc601d5f45c59b85a356e591e2ac5463e21e7" - ], - "version": "==2.5.11" + "sha256:7bcf019a0be528673a8aec1e60b5c863342c3231962dbf7922fd4da42a49a91a" + ], + "version": "==2.5.12" + }, + "pillow": { + "hashes": [ + "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360", + "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7", + "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e", + "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93", + "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c", + "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8", + "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b", + "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0", + "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd", + "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb", + "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581", + "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64", + "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace", + "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956", + "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60", + "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16", + "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37", + "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae", + "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a", + "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073", + "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8", + "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb", + "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409", + "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f", + "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2", + "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4", + "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74", + "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e", + "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1", + "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888" + ], + "version": "==5.3.0" + }, + "pisa": { + "hashes": [ + "sha256:94c4ae0995c84bb0588ece4480486464612ed1526f0987fb1016b9c50e5d3327", + "sha256:a7164ac81ab5ea01fbae4f29d2c00183a31142ca30ad527f6ac96635819cbd12" + ], + "index": "pypi", + "version": "==3.0.33" }, "psutil": { "hashes": [ @@ -1264,6 +1314,12 @@ "index": "pypi", "version": "==2.3.0" }, + "pypdf2": { + "hashes": [ + "sha256:e28f902f2f0a1603ea95ebe21dff311ef09be3d0f0ef29a3e44a932729564385" + ], + "version": "==1.26.0" + }, "python-dateutil": { "hashes": [ "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", @@ -1327,6 +1383,39 @@ "index": "pypi", "version": "==2.10.6" }, + "reportlab": { + "hashes": [ + "sha256:0fcc951899c1bb6e0fa90996d5ed9d24265c5a8eafabf2303570e079d0290c28", + "sha256:234e1790858f1608d2fc8f820fa5660b7838cb14f12dec81e3c1156a4891ee1a", + "sha256:23df47e5a4225728a6d0a16b1dbc2674fe8a535603c191ad9d33f8755d1ab08b", + "sha256:277e920f1f5aa48531d34a25ba83e4e5a201e15a6a75263d529c8996a84233ae", + "sha256:2c52b314f29a40b8ed863190947e556b75a719566d4e2e7d25b8aeb18f60b58c", + "sha256:2e38e61d8e32b003a4c50f4bf1593bc5784395a62d180edd7b206f7dfd77836c", + "sha256:2f42e2e6177756a8265126fbeb7be58fbc40affd471253c829ab165c13253f2e", + "sha256:463f446a96bd0ddaba8ca46747f251868334c119f28bffd42fc0add2b3bd986f", + "sha256:4be2e78b900a601028b4ff42c547096cc204a21ac2a9ce7d16768195106ffd12", + "sha256:4db5daa83aa5c74fba41c3212549ff62abc84921b9ce848bc04cda728e00ecf0", + "sha256:4e4669e49a4cbb323c6a3cb45297535e910d4627f796dba90362e456fb013e44", + "sha256:551bed92c44e302ac7d663242cda10bb8e459ccff0178f51d5693f846336961d", + "sha256:60995c16fa961a6576a67aaeb7a6657a081dabdea298d4c15ec68423171dcdc6", + "sha256:6228323d6a40355c20b4159bc927cb93b9c39537bd39c25b9071eedcf80b4a36", + "sha256:62aafd7769fcf36c9910a332ccee002ce4ff850c9b3ff89bbea8d7b78be979c1", + "sha256:64bf9f0778477a69563db3ca53db3312b7a65756c60e674de54e7527a0852fa0", + "sha256:678667a846ad4181b5a392c59e1ec37e03abc684854d66c7a05df32113a4d337", + "sha256:722bc6e77526cbe927f9a65f37b06840e6d99d6938462892558c6cbb90445d4c", + "sha256:8c967cf193ebeca8231e3c0c28d94c9d54dd1ee38284785c3a1cbf3aca637fd9", + "sha256:95bc72a157559b5963967cf8be17d9185ed64b5c408504b9efbd51ddcf4c145e", + "sha256:9b22737cbaff0e7d875bcb61cb593df908513f1b807551c71afc3a9d1fe83aeb", + "sha256:a2f26770e8af0d383d586fd95446ee246d0532395c9392463c07047cb89862bb", + "sha256:a417205a64cbec93219a8d7e268fe4ba4b7f3e037f7d1ca42f432b4388fb93dd", + "sha256:a6856b4118abbc783ce419e50185e69bceb4d49de39d8439f28571d5949fc34c", + "sha256:b04381abcb3c14afcef7faababbd620b2a352a2f2765f3bf18deabb2dfc7d617", + "sha256:b2348ec5f615d50913c1eaa8c3fc71d4478e4ea0e627950f6fabd70d2a38ff1d", + "sha256:f86f13f4e6d8c815c35ec75e3d3a94cf7de12d544b4d038269c8b7c73de6b906", + "sha256:fa7761cca303d4dbc2f5c5989b2be4baa4068fa4137a5fdc03eb47bfc1af0a5d" + ], + "version": "==3.5.11" + }, "requests": { "hashes": [ "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", @@ -1455,6 +1544,13 @@ ], "version": "==1.1.4" }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" + }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", @@ -1462,6 +1558,13 @@ ], "index": "pypi", "version": "==4.1.2" + }, + "xhtml2pdf": { + "hashes": [ + "sha256:86a37e78d7a8d8bb2761746c3d559e12284d92c4d531b3a8a0f8fd632b436f82" + ], + "index": "pypi", + "version": "==0.2.3" } }, "develop": { @@ -1613,18 +1716,18 @@ }, "docker": { "hashes": [ - "sha256:31421f16c01ffbd1ea7353c7e7cd7540bf2e5906d6173eb51c8fea4e0ea38b19", - "sha256:fbe82af9b94ccced752527c8de07fa20267f9634b48674ba478a0bb4000a0b1e" + "sha256:145c673f531df772a957bd1ebc49fc5a366bcd55efa0e64bbd029f5cc7a1fd8e", + "sha256:666611862edded75f6049893f779bff629fdcd4cd21ccf01d648626e709adb13" ], "index": "pypi", - "version": "==3.5.1" + "version": "==3.6.0" }, "docker-pycreds": { "hashes": [ - "sha256:0a941b290764ea7286bd77f54c0ace43b86a8acd6eb9ead3de9840af52384079", - "sha256:8b0e956c8d206f832b06aa93a710ba2c3bcbacb5a314449c040b0b814355bbff" + "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", + "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49" ], - "version": "==0.3.0" + "version": "==0.4.0" }, "drf-api-checker": { "hashes": [ @@ -1747,37 +1850,37 @@ }, "multidict": { "hashes": [ - "sha256:013eb6591ab95173fd3deb7667d80951abac80100335b3e97b5fa778c1bb4b91", - "sha256:0bffbbbb48db35f57dfb4733e943ac8178efb31aab5601cb7b303ee228ce96af", - "sha256:1a34aab1dfba492407c757532f665ba3282ec4a40b0d2f678bda828ef422ebb7", - "sha256:1b4b46a33f459a2951b0fd26c2d80639810631eb99b3d846d298b02d28a3e31d", - "sha256:1d616d80c37a388891bf760d64bc50cac7c61dbb7d7013f2373aa4b44936e9f0", - "sha256:225aefa7befbe05bd0116ef87e8cd76cbf4ac39457a66faf7fb5f3c2d7bea19a", - "sha256:2c9b28985ef7c830d5c7ea344d068bcdee22f8b6c251369dea98c3a814713d44", - "sha256:39e0600f8dd72acb011d09960da560ba3451b1eca8de5557c15705afc9d35f0e", - "sha256:3c642c40ea1ca074397698446893a45cd6059d5d071fc3ba3915c430c125320f", - "sha256:42357c90b488fac38852bcd7b31dcd36b1e2325413960304c28b8d98e6ff5fd4", - "sha256:6ac668f27dbdf8a69c31252f501e128a69a60b43a44e43d712fb58ce3e5dfcca", - "sha256:713683da2e3f1dd81a920c995df5dda51f1fff2b3995f5864c3ee782fcdcb96c", - "sha256:73b6e7853b6d3bc0eac795044e700467631dff37a5a33d3230122b03076ac2f9", - "sha256:77534c1b9f4a5d0962392cad3f668d1a04036b807618e3357eb2c50d8b05f7f7", - "sha256:77b579ef57e27457064bb6bb4c8e5ede866af071af60fe3576226136048c6dfa", - "sha256:82cf28f18c935d66c15a6f82fda766a4138d21e78532a1946b8ec603019ba0b8", - "sha256:937e8f12f9edc0d2e351c09fc3e7335a65eefb75406339d488ee46ef241f75d8", - "sha256:985dbf59e92f475573a04598f9a00f92b4fdb64fc41f1df2ea6f33b689319537", - "sha256:9c4fab7599ba8c0dbf829272c48c519625c2b7f5630b49925802f1af3a77f1f4", - "sha256:9e8772be8455b49a85ad6dbf6ce433da7856ba481d6db36f53507ae540823b15", - "sha256:a06d6d88ce3be4b54deabd078810e3c077a8b2e20f0ce541c979b5dd49337031", - "sha256:a1da0cdc3bc45315d313af976dab900888dbb477d812997ee0e6e4ea43d325e5", - "sha256:a6652466a4800e9fde04bf0252e914fff5f05e2a40ee1453db898149624dfe04", - "sha256:a7f23523ea6a01f77e0c6da8aae37ab7943e35630a8d2eda7e49502f36b51b46", - "sha256:a87429da49f4c9fb37a6a171fa38b59a99efdeabffb34b4255a7a849ffd74a20", - "sha256:c26bb81d0d19619367a96593a097baec2d5a7b3a0cfd1e3a9470277505a465c2", - "sha256:d4f4545edb4987f00fde44241cef436bf6471aaac7d21c6bbd497cca6049f613", - "sha256:daabc2766a2b76b3bec2086954c48d5f215f75a335eaee1e89c8357922a3c4d5", - "sha256:f08c1dcac70b558183b3b755b92f1135a76fd1caa04009b89ddea57a815599aa" - ], - "version": "==4.5.1" + "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f", + "sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3", + "sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef", + "sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b", + "sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73", + "sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc", + "sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3", + "sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd", + "sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351", + "sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941", + "sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d", + "sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1", + "sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b", + "sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a", + "sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3", + "sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7", + "sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0", + "sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0", + "sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014", + "sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5", + "sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036", + "sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d", + "sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a", + "sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce", + "sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1", + "sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a", + "sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9", + "sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7", + "sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b" + ], + "version": "==4.5.2" }, "nodeenv": { "hashes": [ @@ -1794,10 +1897,10 @@ }, "pdbpp": { "hashes": [ - "sha256:dde77326e4ea41439c243ed065826d53539530eeabd1b6615aae15cfbb9fda05" + "sha256:535085916fcfb768690ba0aeab2967c2a2163a0a60e5b703776846873e171399" ], "index": "pypi", - "version": "==0.9.2" + "version": "==0.9.3" }, "pexpect": { "hashes": [ diff --git a/src/unicef_rest_framework/renderers/__init__.py b/src/unicef_rest_framework/renderers/__init__.py index cb69181cc..8cc3dadc9 100644 --- a/src/unicef_rest_framework/renderers/__init__.py +++ b/src/unicef_rest_framework/renderers/__init__.py @@ -3,3 +3,4 @@ from .microsoft.xml import MSXmlRenderer # noqa from .xls import XLSXRenderer # noqa from .html import HTMLRenderer # noqa +from .pdf import PDFRenderer # noqa diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py new file mode 100644 index 000000000..a15195874 --- /dev/null +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -0,0 +1,52 @@ +import io +import os + +from django.conf import settings +from xhtml2pdf import pisa + +from .html import HTMLRenderer + + +def link_callback(uri, rel): + """ + Convert HTML URIs to absolute system paths so xhtml2pdf can access those + resources + """ + # use short variable names + sUrl = settings.STATIC_URL # Typically /static/ + sRoot = settings.STATIC_ROOT # Typically /home/userX/project_static/ + mUrl = settings.MEDIA_URL # Typically /static/media/ + mRoot = settings.MEDIA_ROOT # Typically /home/userX/project_static/media/ + + # convert URIs to absolute system paths + if uri.startswith(mUrl): + path = os.path.join(mRoot, uri.replace(mUrl, "")) + elif uri.startswith(sUrl): + path = os.path.join(sRoot, uri.replace(sUrl, "")) + else: + return uri # handle absolute uri (ie: http://some.tld/foo.png) + + # make sure that file exists + if not os.path.isfile(path): + raise Exception( + 'media URI must start with %s or %s' % (sUrl, mUrl) + ) + return path + + +class PDFRenderer(HTMLRenderer): + media_type = 'application/pdf' + format = 'pdf' + charset = 'utf-8' + render_style = 'text' + + def render(self, data, accepted_media_type=None, renderer_context=None): + html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) + # create a pdf + buffer = io.BytesIO() + pisaStatus = pisa.CreatePDF(html, dest=buffer, link_callback=link_callback) + # if error then show some funy view + if pisaStatus.err: + raise Exception('We had some errors 
      ' + html + '
      ') + buffer.seek(0) + return buffer.read() diff --git a/src/unicef_rest_framework/views.py b/src/unicef_rest_framework/views.py index 959e4414c..1d287d0a4 100644 --- a/src/unicef_rest_framework/views.py +++ b/src/unicef_rest_framework/views.py @@ -16,8 +16,8 @@ from unicef_rest_framework.cache import cache_response, etag, ListKeyConstructor from unicef_rest_framework.filtering import SystemFilterBackend from unicef_rest_framework.permissions import ServicePermission -from unicef_rest_framework.renderers import (APIBrowsableAPIRenderer, HTMLRenderer, - MSJSONRenderer, MSXmlRenderer, XLSXRenderer,) +from unicef_rest_framework.renderers import (APIBrowsableAPIRenderer, HTMLRenderer, MSJSONRenderer, + MSXmlRenderer, PDFRenderer, XLSXRenderer,) from unicef_rest_framework.renderers.csv import CSVRenderer @@ -50,6 +50,7 @@ class ReadOnlyModelViewSet(DynamicSerializerMixin, viewsets.ReadOnlyModelViewSet YAMLRenderer, XLSXRenderer, HTMLRenderer, + PDFRenderer, MSJSONRenderer, XMLRenderer, MSXmlRenderer, From 3176faddd3b3e1550ac71997040f6d7051d23111 Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 29 Nov 2018 13:03:42 +0100 Subject: [PATCH 37/43] add pdf format to available notification attachment --- Pipfile | 1 + Pipfile.lock | 10 +++++++++- docker/Dockerfile | 1 + src/etools_datamart/apps/subscriptions/models.py | 11 ++++++++--- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Pipfile b/Pipfile index 5c09fe079..ce634ef2c 100644 --- a/Pipfile +++ b/Pipfile @@ -77,6 +77,7 @@ vcrpy = "*" pre-commit = "*" freezegun = "*" pytest-ignore-flaky = "*" +bumpversion = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 355c31e12..dede2401d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "19e2894907faa1249e7a55f7edfda1a976c93470fe669c30a64901f031efae41" + "sha256": "7da456c863ca47d1dd7f8bc8dc91b8bf676d9764913c81a321e6290804c76362" }, "pipfile-spec": 6, "requires": { @@ -1612,6 +1612,14 @@ ], "version": "==4.6.3" }, + "bumpversion": { + "hashes": [ + "sha256:6744c873dd7aafc24453d8b6a1a0d6d109faf63cd0cd19cb78fd46e74932c77e", + "sha256:6753d9ff3552013e2130f7bc03c1007e24473b4835952679653fb132367bdd57" + ], + "index": "pypi", + "version": "==0.5.3" + }, "cached-property": { "hashes": [ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", diff --git a/docker/Dockerfile b/docker/Dockerfile index 1e00b610e..f7360e7d8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,6 +21,7 @@ RUN apk add --no-cache --virtual .build-deps \ linux-headers \ musl-dev \ postgresql-dev \ + libjpeg-dev \ python3-dev diff --git a/src/etools_datamart/apps/subscriptions/models.py b/src/etools_datamart/apps/subscriptions/models.py index e8f289f86..5fba886f3 100644 --- a/src/etools_datamart/apps/subscriptions/models.py +++ b/src/etools_datamart/apps/subscriptions/models.py @@ -24,9 +24,12 @@ def notify(self, model): for subscription in self.filter(content_type=ct).exclude(type=Subscription.NONE): logger.info(f"Process subscription {subscription}") try: - if subscription.type == Subscription.EXCEL: + if subscription.type in (Subscription.EXCEL, Subscription.PDF): + format = {Subscription.EXCEL: 'xlsx', + Subscription.PDF: 'pdf', + }[subscription.type] rf = APIRequestFactory() - request = rf.get(f"{service.endpoint}?format=xlsx") + request = rf.get(f"{service.endpoint}?format={format}") request.user = subscription.user request.api_info = {} # this is set my the middleware, so we must set manually here response = service.viewset.as_view({'get': 'list'})(request) @@ -36,7 +39,7 @@ def notify(self, model): request.api_info.update(dict(response.items())) attachments = { - f'{model._meta.verbose_name}.xlsx': BytesIO(response.content), + f'{model._meta.verbose_name}.{format}': BytesIO(response.content), } template = 'dataset_changed_attachment' else: @@ -67,10 +70,12 @@ class Subscription(models.Model): NONE = 0 MESSAGE = 1 EXCEL = 2 + PDF = 3 TYPES = ((NONE, 'None'), (MESSAGE, 'Email'), (EXCEL, 'Email+Excel'), + (PDF, 'Email+Pdf'), ) user = models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE, related_name='subscriptions') From fec982252662ac8fdf8f670c8af0893a1c698cbc Mon Sep 17 00:00:00 2001 From: sax Date: Thu, 29 Nov 2018 14:39:18 +0100 Subject: [PATCH 38/43] bug fixing --- docker/Dockerfile | 11 +++++-- src/etools_datamart/apps/etl/tasks/etl.py | 31 +++++++++++-------- .../init/management/commands/init-setup.py | 4 +-- src/etools_datamart/celery.py | 5 +-- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f7360e7d8..c76a9f046 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,13 +16,20 @@ RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ FROM python:3.6-alpine as base COPY --from=builder /code /code RUN apk add --no-cache --virtual .build-deps \ + freetype-dev \ gcc \ + jpeg-dev \ + lcms2-dev \ libffi-dev \ linux-headers \ musl-dev \ + openjpeg-dev \ postgresql-dev \ - libjpeg-dev \ - python3-dev + python3-dev \ + tcl-dev \ + tiff-dev \ + tk-dev \ + zlib-dev RUN apk add --no-cache --virtual .deps \ diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index 640526f9e..d45bdc0fc 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -3,6 +3,7 @@ import logging from datetime import date, datetime +from crashlog.middleware import process_exception from django.db import connections from django.db.models import Sum from django.db.models.functions import Coalesce @@ -33,18 +34,23 @@ def is_record_changed(record, values): def process(Model, filters, values): - existing, created = Model.objects.get_or_create(**filters, - defaults=values) - if created: - op = CREATED - else: - if is_record_changed(existing, values): - op = UPDATED - Model.objects.update_or_create(**filters, - defaults=values) + try: + existing, created = Model.objects.get_or_create(**filters, + defaults=values) + if created: + op = CREATED else: - op = UNCHANGED - return op + if is_record_changed(existing, values): + op = UPDATED + Model.objects.update_or_create(**filters, + defaults=values) + else: + op = UNCHANGED + return op + except Exception as e: + logging.exception(e) + process_exception(e) + raise @app.etl(HACT) @@ -165,7 +171,6 @@ def load_pmp_indicator() -> EtlResult: } op = process(PMPIndicators, filters=dict(country_name=country.name, schema_name=country.schema_name, - country_id=partner.id, partner_id=partner.pk, intervention_id=intervention.pk), values=values) @@ -222,7 +227,6 @@ def load_intervention() -> EtlResult: review_date_prc=record.review_date_prc, prc_review_document=record.prc_review_document, partner_name=record.agreement.partner.name, - agreement_id=record.agreement.pk, partner_authorized_officer_signatory_id=get_attr(record, 'partner_authorized_officer_signatory.pk'), country_programme_id=get_attr(record, 'country_programme.pk'), @@ -272,6 +276,7 @@ def load_intervention() -> EtlResult: ) op = process(Intervention, filters=dict(country_name=country.name, schema_name=country.schema_name, + agreement_id=record.agreement.pk, intervention_id=record.pk), values=values) results.incr(op) diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index 82cb068ba..953465f85 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -45,7 +45,7 @@ MAIL_ATTACHMENT = r"""Dear {{user.label}}, On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`. -You can find here in attachment a excel file with new data +You can find here in attachment a excel file with updated data — You are receiving this because you are subscribed to this thread. @@ -55,7 +55,7 @@ MAIL_ATTACHMENT_HTML = r"""
      Dear {{user.label}},
       
      On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`.
      -
      Attached to this email you can find excel file with new data
      +
      Attached to this email you can find excel file with updated data
       
       
       
      diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 5bb98c317..d512a4eb4 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -93,8 +93,11 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): from django.utils import timezone from etools_datamart.apps.subscriptions.models import Subscription + from etools_datamart.apps.etl.models import EtlTask # from unicef_rest_framework.models import Service + if state != 'SUCCESS': + EtlTask.objects.filter(task=task.name).update(status=state) if not hasattr(sender, 'linked_model'): return @@ -119,8 +122,6 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st defs['results'] = str(retval) defs['last_failure'] = timezone.now() - from etools_datamart.apps.etl.models import EtlTask - EtlTask.objects.update_or_create(task=task.name, defaults=defs) # Service.objects.invalidate_cache() app.timers[task.name] = cost From fa2fc7b031f19d17cb75c85ae55e4411dfb4829c Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 00:21:46 +0100 Subject: [PATCH 39/43] minor css/style improvements --- Pipfile | 1 + Pipfile.lock | 22 +++-- docker/Dockerfile.2 | 97 +++++++++++++++++++ docker/Dockerfile.2.base | 46 +++++++++ docker/Makefile | 20 +++- .../api/endpoints/datamart/hact.py | 2 +- .../api/endpoints/datamart/serializers.py | 29 +++--- src/etools_datamart/apps/etl/admin.py | 9 +- src/etools_datamart/apps/web/static/style.css | 7 +- .../apps/web/static/style.css.map | 2 +- .../apps/web/static/style.scss | 7 +- .../apps/web/templates/monitor.html | 21 ++++ src/etools_datamart/config/settings.py | 6 +- src/unicef_rest_framework/renderers/html.py | 15 ++- src/unicef_rest_framework/renderers/pdf.py | 6 ++ .../templates/renderers/html/html.html | 46 +++++++++ .../renderers/{html.html => pdf/pdf.html} | 19 +++- tests/test_subscription.py | 2 +- 18 files changed, 320 insertions(+), 37 deletions(-) create mode 100644 docker/Dockerfile.2 create mode 100644 docker/Dockerfile.2.base create mode 100644 src/unicef_rest_framework/templates/renderers/html/html.html rename src/unicef_rest_framework/templates/renderers/{html.html => pdf/pdf.html} (58%) diff --git a/Pipfile b/Pipfile index ce634ef2c..b84c8130c 100644 --- a/Pipfile +++ b/Pipfile @@ -55,6 +55,7 @@ django-post-office = "*" django-celery-email = "*" "xhtml2pdf" = "*" pisa = "*" +django-crispy-forms = "*" [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index dede2401d..2b97db8c8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7da456c863ca47d1dd7f8bc8dc91b8bf676d9764913c81a321e6290804c76362" + "sha256": "2c79b29b6f0397ba9724ed2ec50a286a3178b5b335f0a3b70ff3594a60dc9a9d" }, "pipfile-spec": 6, "requires": { @@ -647,10 +647,10 @@ }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" ], - "version": "==2018.10.15" + "version": "==2018.11.29" }, "cffi": { "hashes": [ @@ -842,6 +842,14 @@ "index": "pypi", "version": "==1.0.0" }, + "django-crispy-forms": { + "hashes": [ + "sha256:5952bab971110d0b86c278132dae0aa095beee8f723e625c3d3fa28888f1675f", + "sha256:705ededc554ad8736157c666681165fe22ead2dec0d5446d65fc9dd976a5a876" + ], + "index": "pypi", + "version": "==1.7.2" + }, "django-db-logging": { "hashes": [ "sha256:4787de15c1a933e016766f023730f9551e8bf341e99549d13a918f27bf403f30" @@ -1629,10 +1637,10 @@ }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" ], - "version": "==2018.10.15" + "version": "==2018.11.29" }, "cfgv": { "hashes": [ diff --git a/docker/Dockerfile.2 b/docker/Dockerfile.2 new file mode 100644 index 000000000..8874efa14 --- /dev/null +++ b/docker/Dockerfile.2 @@ -0,0 +1,97 @@ +FROM pstauffer/curl:latest as builder +ARG DEVELOP +ARG GITHUB_CREDENTIALS +ARG VERSION + +RUN mkdir /code +ADD . /code + +RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ + echo "${VERSION}-develop"; \ + else \ + echo "Download package: https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" \ + && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ + fi + +FROM unicef/datamart:base as base +COPY --from=builder /code /code + +LABEL org.label.name="eTools Datamart" \ + org.label.maintainer="sapostolico@unicef.org" \ + org.label.description="" \ + org.label.url="https://datamart.unicef.io/" \ + org.label.vcs-url="https://github.com/unicef/etools-datamart" \ + org.label.version=$VERSION + + +ARG BUILD_DATE +ARG PIPENV_PYPI_MIRROR +ARG PIPENV_ARGS +ARG VERSION + +ENV VERSION ${VERSION} + +ENV PIPENV_PYPI_MIRROR ${PIPENV_PYPI_MIRROR} +ENV PIPENV_ARGS ${PIPENV_ARGS} +ENV HOME /root/ +ENV PIPSI_HOME=/usr/local/pipsi/environments +ENV PIPSI_BIN_DIR=/usr/local/bin +ENV PYTHONUNBUFFERED 1 +ENV USE_GUNICORN 0 +ENV GUNICORN_CMD_ARGS "-b 0.0.0.0:8000 \ +--chdir /var/datamart \ +--access-logfile - \ +--access-logformat \"%(h)s %(l)s %(u)s %(t)s '%(r)s' %(s)s\" " + +ENV ALLOWED_HOSTS * +ENV CACHE_URL "redis://127.0.0.1:6379/1" +ENV CELERY_BROKER_URL "redis://127.0.0.1:6379/2" +ENV CELERY_RESULT_BACKEND "redis://127.0.0.1:6379/3" +ENV CSRF_COOKIE_SECURE True +ENV DATABASE_URL "postgres://postgres:@127.0.0.1:5432/etools_datamart" +ENV DATABASE_URL_ETOOLS "postgis://postgres:@127.0.0.1:5432/etools" +ENV DEBUG 0 +ENV DEVELOPMENT_MODE 0 +ENV DJANGO_SETTINGS_MODULE etools_datamart.config.settings +ENV MEDIA_ROOT /tmp/media +ENV SECRET_KEY "secret" +ENV SECURE_BROWSER_XSS_FILTER True +ENV SECURE_CONTENT_TYPE_NOSNIFF True +ENV SECURE_FRAME_DENY True +ENV SECURE_HSTS_INCLUDE_SUBDOMAINS True +ENV SECURE_HSTS_PRELOAD True +ENV SECURE_HSTS_SECONDS 1 +ENV SECURE_SSL_REDIRECT True +ENV SENTRY_DSN "" +ENV SESSION_COOKIE_HTTPONLY True +ENV SESSION_COOKIE_SECURE True +ENV STATIC_ROOT /tmp/static +#ENV SUPERVISOR_USER admin +#ENV SUPERVISOR_PWD "" +#ENV FLOWER_USER admin +#ENV FLOWER_PWD "" +#ENV X_FRAME_OPTIONS "DENY" +#ENV START_DATAMART "true" +#ENV START_REDIS "true" +#ENV START_CELERY "true" + +RUN pip install . \ + && rm -fr /code + +#RUN apt-get autoremove --yes --force-yes \ +# gcc \ +# && rm -fr /var/lib/apt/lists/* \ +# && rm -fr /var/cache/apt/* + +RUN apk del .build-deps \ + && rm -rf /var/cache/apk/* \ + && rm -fr /root/.cache/ + +WORKDIR /var/datamart + +EXPOSE 8000 + +ADD docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["datamart"] diff --git a/docker/Dockerfile.2.base b/docker/Dockerfile.2.base new file mode 100644 index 000000000..1a6d19254 --- /dev/null +++ b/docker/Dockerfile.2.base @@ -0,0 +1,46 @@ +FROM pstauffer/curl:latest as builder +ARG DEVELOP +ARG GITHUB_CREDENTIALS +ARG VERSION + +RUN mkdir /code +ADD . /code + +RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ + echo "${VERSION}-develop"; \ + else \ + echo "Download package: https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" \ + && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ + fi + +FROM python:3.6-alpine as base +COPY --from=builder /code /code +RUN apk add --no-cache --virtual .build-deps \ + freetype-dev \ + gcc \ + jpeg-dev \ + lcms2-dev \ + libffi-dev \ + linux-headers \ + musl-dev \ + openjpeg-dev \ + postgresql-dev \ + python3-dev \ + tcl-dev \ + tiff-dev \ + tk-dev \ + zlib-dev + + +RUN apk add --no-cache --virtual .deps \ + bash + +RUN mkdir -p \ + /var/datamart/{static,log,conf,run} \ + && pip install pip==18.0 pipenv --upgrade + +WORKDIR /code + +RUN set -ex \ + ls -al /code \ + && pipenv install --system --deploy --ignore-pipfile $PIPENV_ARGS diff --git a/docker/Makefile b/docker/Makefile index 865d3c846..0107a9857 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -5,7 +5,7 @@ DEVELOP?=0 DOCKER_PASS?= DOCKER_USER?= TARGET?=dev -# below var are used internally +# below vars are used internally BUILD_OPTIONS?= CMD?=datamart CONTAINER_NAME?=datamart-${TARGET} @@ -13,7 +13,7 @@ ORGANIZATION=unicef IMAGE_NAME=datamart DOCKER_IMAGE_NAME=${ORGANIZATION}/${IMAGE_NAME} DOCKER_IMAGE=${DOCKER_IMAGE_NAME}:${TARGET} -DOCKERFILE?=Dockerfile +DOCKERFILE?=Dockerfile.2 RUN_OPTIONS?= PIPENV_ARGS?= @@ -24,12 +24,26 @@ help: @echo "run run ${DOCKER_IMAGE} locally" -build: +build-base: + if [[ "`docker images -q ${DOCKER_IMAGE_NAME}:base 2> /dev/null`" == "" ]]; then \ + cd .. && docker build \ + --build-arg DEVELOP=${DEVELOP} \ + --build-arg GITHUB_CREDENTIALS=${GITHUB_CREDENTIALS} \ + --build-arg VERSION=${TARGET} \ + --squash \ + -t ${DOCKER_IMAGE_NAME}:base \ + -f docker/${DOCKERFILE}.base . ; \ + docker push ${DOCKER_IMAGE_NAME}:base + fi + + +build: build-base cd .. && docker build \ ${BUILD_OPTIONS} \ --build-arg DEVELOP=${DEVELOP} \ --build-arg GITHUB_CREDENTIALS=${GITHUB_CREDENTIALS} \ --build-arg VERSION=${TARGET} \ + --squash \ -t ${DOCKER_IMAGE} \ -f docker/${DOCKERFILE} . docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:dev diff --git a/src/etools_datamart/api/endpoints/datamart/hact.py b/src/etools_datamart/api/endpoints/datamart/hact.py index 3f86d31e8..2783992ec 100644 --- a/src/etools_datamart/api/endpoints/datamart/hact.py +++ b/src/etools_datamart/api/endpoints/datamart/hact.py @@ -8,4 +8,4 @@ class HACTViewSet(common.DataMartViewSet): serializer_class = serializers.HACTSerializer queryset = models.HACT.objects.all() - filter_fields = ('country_name', 'month', 'last_modify_date') + filter_fields = ('country_name', 'year', 'last_modify_date') diff --git a/src/etools_datamart/api/endpoints/datamart/serializers.py b/src/etools_datamart/api/endpoints/datamart/serializers.py index b1fb17ba5..4861985ac 100644 --- a/src/etools_datamart/api/endpoints/datamart/serializers.py +++ b/src/etools_datamart/api/endpoints/datamart/serializers.py @@ -7,37 +7,38 @@ from etools_datamart.apps.data import models -class PMPIndicatorsSerializer(serializers.ModelSerializer): +class DataMartSerializer(serializers.ModelSerializer): class Meta: + exclude = ('schema_name',) + + +class PMPIndicatorsSerializer(DataMartSerializer): + class Meta(DataMartSerializer.Meta): model = models.PMPIndicators - exclude = () -class InterventionSerializer(serializers.ModelSerializer): - class Meta: +class InterventionSerializer(DataMartSerializer): + class Meta(DataMartSerializer.Meta): model = models.Intervention - exclude = () -class FAMIndicatorSerializer(serializers.ModelSerializer): - class Meta: +class FAMIndicatorSerializer(DataMartSerializer): + class Meta(DataMartSerializer.Meta): model = models.FAMIndicator - exclude = () -class UserStatsSerializer(serializers.ModelSerializer): +class UserStatsSerializer(DataMartSerializer): month = SerializerMethodField(help_text="---") - class Meta: + class Meta(DataMartSerializer.Meta): model = models.UserStats - exclude = () read_only = ['last_modify_date', ] def get_month(self, obj): return datetime.strftime(obj.month._date, '%b %Y') -class HACTSerializer(serializers.ModelSerializer): - class Meta: +class HACTSerializer(DataMartSerializer): + # last_modify_date = serializers.DateTimeField(format=settings.DATETIME_FORMAT) + class Meta(DataMartSerializer.Meta): model = models.HACT - exclude = () diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index 37841f2f0..2130f383d 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -21,8 +21,11 @@ class EtlTaskAdmin(TruncateTableMixin, admin.ModelAdmin): list_display = ('task', 'last_run', 'status', 'time', 'last_success', 'last_failure', 'lock', 'scheduling', 'queue_task') - readonly_fields = ('task', 'last_run', 'results', 'elapsed', 'time', 'status', - 'last_success', 'last_failure', 'table_name', 'content_type') + readonly_fields = ('task', 'last_run', + 'last_success', 'last_failure', 'last_changes', + 'results', 'elapsed', 'time', 'status', + 'table_name', 'content_type', + ) date_hierarchy = 'last_run' actions = None @@ -45,6 +48,7 @@ def queue_task(self, obj): url = reverse('admin:%s_%s_queue' % (opts.app_label, opts.model_name), args=[obj.id]) return format_html(f'queue') + queue_task.verbse_name = 'queue' def has_add_permission(self, request): @@ -77,6 +81,7 @@ def queue(self, request, pk): self.message_user(request, f"Task '{obj.task}' queued", messages.SUCCESS) except Exception as e: # pragma: no cover self.message_user(request, f"Cannot queue '{obj.task}': {e}", messages.ERROR) + return HttpResponseRedirect(reverse("admin:etl_etltask_changelist")) @action() def unlock(self, request, pk): diff --git a/src/etools_datamart/apps/web/static/style.css b/src/etools_datamart/apps/web/static/style.css index edf3bd04d..feeacaa12 100644 --- a/src/etools_datamart/apps/web/static/style.css +++ b/src/etools_datamart/apps/web/static/style.css @@ -138,6 +138,11 @@ input.input100 { .navbar a.navbar-brand { color: white; } +.rounded { + -webkit-border-radius: 20px; + -moz-border-radius: 20px; + border-radius: 20px; } + .monitor .menubar { padding: 10px; } .monitor .menubar a { @@ -169,6 +174,6 @@ input.input100 { .monitor #monitor .col.status { width: 120px; } .monitor #monitor .col.subscription { - width: 100px; } + width: 180px; } /*# sourceMappingURL=style.css.map */ diff --git a/src/etools_datamart/apps/web/static/style.css.map b/src/etools_datamart/apps/web/static/style.css.map index a12c161b1..16c09e0aa 100644 --- a/src/etools_datamart/apps/web/static/style.css.map +++ b/src/etools_datamart/apps/web/static/style.css.map @@ -1,6 +1,6 @@ { "version": 3, -"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEjB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKlB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAKd,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAQ;MACN,WAAW,EAAE,IAAI;IAEnB,0BAAK;MACH,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,+BAAU;MACR,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK", +"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEjB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKlB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAGhB,QAAS;EACN,qBAAqB,EAAE,IAAI;EAC3B,kBAAkB,EAAE,IAAI;EACxB,aAAa,EAAE,IAAI;;AAIpB,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAQ;MACN,WAAW,EAAE,IAAI;IAEnB,0BAAK;MACH,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,+BAAU;MACR,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK", "sources": ["style.scss"], "names": [], "file": "style.css" diff --git a/src/etools_datamart/apps/web/static/style.scss b/src/etools_datamart/apps/web/static/style.scss index 66eae1e5c..3ac8fdc1e 100644 --- a/src/etools_datamart/apps/web/static/style.scss +++ b/src/etools_datamart/apps/web/static/style.scss @@ -230,6 +230,11 @@ input.input100 { color: white; } } +.rounded { + -webkit-border-radius: 20px; + -moz-border-radius: 20px; + border-radius: 20px; +} .monitor { .menubar { @@ -278,7 +283,7 @@ input.input100 { width: 120px; } &.subscription { - width: 100px; + width: 180px; } } } diff --git a/src/etools_datamart/apps/web/templates/monitor.html b/src/etools_datamart/apps/web/templates/monitor.html index e4e5077a8..dca68f9d4 100644 --- a/src/etools_datamart/apps/web/templates/monitor.html +++ b/src/etools_datamart/apps/web/templates/monitor.html @@ -3,6 +3,20 @@ {% block title %}Monitor{% endblock %} {% block content %} + @@ -10,6 +24,9 @@ Home API Documentation + {% if user.is_staff %} + Admin + {% endif %}
    @@ -28,6 +45,10 @@ {% subscription_select t %} + {% if user.is_staff %} + Admin + {% endif %} +
    {% endfor %}
    diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 25f19dc7e..25e761071 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -274,6 +274,7 @@ 'django_filters', 'month_field', 'drf_querystringfilter', + 'crispy_forms', 'drf_yasg', 'adminfilters', @@ -293,7 +294,7 @@ 'etools_datamart.apps.subscriptions', 'etools_datamart.api', ] - +DATE_FORMAT = '%d %b %Y' DATE_INPUT_FORMATS = [ '%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y', # '2006-10-25', '10/25/2006', '10/25/06' '%b %d %Y', '%b %d, %Y', # 'Oct 25 2006', 'Oct 25, 2006' @@ -302,6 +303,8 @@ '%d %B %Y', '%d %B, %Y', # '25 October 2006', '25 October, 2006' ] +DATETIME_FORMAT = '%d %b %Y %H:%M:%S' + DATETIME_INPUT_FORMATS = [ '%Y-%m-%d %H:%M:%S', # '2006-10-25 14:30:59' '%Y-%m-%d %H:%M:%S.%f', # '2006-10-25 14:30:59.000200' @@ -415,6 +418,7 @@ # 'EXCEPTION_HANDLER': 'my_project.my_app.utils.custom_exception_handler' 'SEARCH_PARAM': 'search', 'ORDERING_PARAM': 'ordering', + 'DATETIME_FORMAT': DATETIME_FORMAT } JWT_AUTH = { diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index b23fefd92..b276f67bd 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -2,20 +2,27 @@ from rest_framework.renderers import BaseRenderer +def labelize(v): + return v.replace("_", " ").title() + + class HTMLRenderer(BaseRenderer): media_type = 'text/html' format = 'xhtml' charset = 'utf-8' render_style = 'text' + def get_template(self, meta): + return loader.select_template([ + f'renderers/html/{meta.app_label}/{meta.model_name}.html', + 'renderers/html/html.html']) + def render(self, data, accepted_media_type=None, renderer_context=None): model = renderer_context['view'].queryset.model opts = model._meta - template = loader.select_template([ - f'renderers/{opts.app_label}/{opts.model_name}.html', - 'renderers/html.html']) + template = self.get_template(opts) c = {'data': data, 'model': model, 'opts': opts, - 'headers': data['results'][0].keys()} + 'headers': [labelize(v) for v in data['results'][0].keys()]} return template.render(c) diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index a15195874..4f3702245 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -2,6 +2,7 @@ import os from django.conf import settings +from django.template import loader from xhtml2pdf import pisa from .html import HTMLRenderer @@ -40,6 +41,11 @@ class PDFRenderer(HTMLRenderer): charset = 'utf-8' render_style = 'text' + def get_template(self, meta): + return loader.select_template([ + f'renderers/pdf/{meta.app_label}/{meta.model_name}.html', + 'renderers/pdf/pdf.html']) + def render(self, data, accepted_media_type=None, renderer_context=None): html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) # create a pdf diff --git a/src/unicef_rest_framework/templates/renderers/html/html.html b/src/unicef_rest_framework/templates/renderers/html/html.html new file mode 100644 index 000000000..a807290c9 --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/html/html.html @@ -0,0 +1,46 @@ +{% load static datamart %} + + + + {{ opts.verbose_name }} + + + +

    {{ opts.verbose_name }}

    +
    + + + {% for v in headers %} + + {% endfor %} + + {% for row in data.results %} + + {% for k,v in row.items %} + + {% endfor %} + + {% endfor %} +
    {{ v }}
    {{ v }}
    +
    + + diff --git a/src/unicef_rest_framework/templates/renderers/html.html b/src/unicef_rest_framework/templates/renderers/pdf/pdf.html similarity index 58% rename from src/unicef_rest_framework/templates/renderers/html.html rename to src/unicef_rest_framework/templates/renderers/pdf/pdf.html index b370db53f..bfd81328c 100644 --- a/src/unicef_rest_framework/templates/renderers/html.html +++ b/src/unicef_rest_framework/templates/renderers/pdf/pdf.html @@ -4,13 +4,29 @@ {{ opts.verbose_name }}

    {{ opts.verbose_name }}

    +
    {% for v in headers %} @@ -18,12 +34,13 @@

    {{ opts.verbose_name }}

    {% endfor %} {% for row in data.results %} - + {% for k,v in row.items %} {% endfor %} {% endfor %}
    {{ v }}
    +
    diff --git a/tests/test_subscription.py b/tests/test_subscription.py index e32c0f3ef..f5dc622d3 100644 --- a/tests/test_subscription.py +++ b/tests/test_subscription.py @@ -70,7 +70,7 @@ def test_subscribe_404(rf, admin_user, etltask): @pytest.mark.django_db def test_subscribe_invalid(rf, admin_user, etltask): request = rf.post(reverse("subscribe", args=[etltask.pk]), - {"type": 3}, content_type='application/json') + {"type": 99}, content_type='application/json') request.user = admin_user res = subscribe(request, etltask.pk) assert res.status_code == 400 From 467fc0a6dd025447320b600ff5b8f2fd4e9f6bee Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 00:36:06 +0100 Subject: [PATCH 40/43] handle empty recordset im html/pdf --- src/unicef_rest_framework/renderers/html.py | 14 ++++++++++---- src/unicef_rest_framework/renderers/pdf.py | 1 + .../templates/renderers/pdf/pdf.html | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index b276f67bd..13cc70e23 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -21,8 +21,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None): model = renderer_context['view'].queryset.model opts = model._meta template = self.get_template(opts) - c = {'data': data, - 'model': model, - 'opts': opts, - 'headers': [labelize(v) for v in data['results'][0].keys()]} + if data['results']: + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': [labelize(v) for v in data['results'][0].keys()]} + else: + c = {'data': {}, + 'model': model, + 'opts': opts, + 'headers': []} return template.render(c) diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index 4f3702245..71c076b24 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -48,6 +48,7 @@ def get_template(self, meta): def render(self, data, accepted_media_type=None, renderer_context=None): html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) + # create a pdf buffer = io.BytesIO() pisaStatus = pisa.CreatePDF(html, dest=buffer, link_callback=link_callback) diff --git a/src/unicef_rest_framework/templates/renderers/pdf/pdf.html b/src/unicef_rest_framework/templates/renderers/pdf/pdf.html index bfd81328c..f989b316b 100644 --- a/src/unicef_rest_framework/templates/renderers/pdf/pdf.html +++ b/src/unicef_rest_framework/templates/renderers/pdf/pdf.html @@ -27,6 +27,7 @@

    {{ opts.verbose_name }}

    +{% if headers %} {% for v in headers %} @@ -41,6 +42,7 @@

    {{ opts.verbose_name }}

    {% endfor %}
    +{% endif %}
    From 6c0f7d4d4cfe37f2e436e2088c26aa04bd19ec5a Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 14:54:12 +0100 Subject: [PATCH 41/43] docker refactoring --- .gitignore | 28 +++--- docker/Dockerfile | 38 ++++---- docker/Dockerfile.2 | 97 --------------------- docker/Dockerfile.2.base | 46 ---------- docker/Makefile | 27 +++--- docker/entrypoint.sh | 6 +- src/unicef_rest_framework/renderers/csv.py | 14 ++- src/unicef_rest_framework/renderers/html.py | 38 +++++--- src/unicef_rest_framework/renderers/pdf.py | 29 +++--- src/unicef_rest_framework/renderers/xls.py | 16 +++- 10 files changed, 119 insertions(+), 220 deletions(-) delete mode 100644 docker/Dockerfile.2 delete mode 100644 docker/Dockerfile.2.base diff --git a/.gitignore b/.gitignore index bd9c5a080..74390e2d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -docker/.envrc -.DS_Store *.egg-info *.pyo *.sqlite @@ -7,8 +5,12 @@ docker/.envrc *credentials.* .cache .coverage +.DS_Store +.env .idea +.pydevproject .tox +.venv /contacts.json /dist /media @@ -17,23 +19,21 @@ __pycache__ CACHE celerybeat-schedule coverage. -xml +db/clean.sql +db/db1.bz2 +db/etools.dump +db/public.sqldump +db/tenant.sql djcelery.schedulers.DatabaseScheduler +docker/.envrc +docker/cache +docker/superset.db LOCAL local.yaml pytest.xml settings_local.py -~* -.pydevproject -db/db1.bz2 -.env -.venv -docker/cache src/etools_datamart/apps/core/static/api-doc.css.map -docker/superset.db -db/clean.sql -db/etools.dump -db/public.sqldump -db/tenant.sql src/etools_datamart/apps/etools/models/public_old.py src/etools_datamart/apps/etools/models/tenant_old.py +xml +~* diff --git a/docker/Dockerfile b/docker/Dockerfile index c76a9f046..b99c917ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,8 +14,18 @@ RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ fi FROM python:3.6-alpine as base -COPY --from=builder /code /code -RUN apk add --no-cache --virtual .build-deps \ + +RUN apk add --no-cache \ + bash \ + freetype \ + lcms2 \ + libjpeg-turbo \ + libpng \ + libpq \ + openjpeg \ + postgresql-libs \ + tiff \ + && apk add --no-cache --virtual .build-deps \ freetype-dev \ gcc \ jpeg-dev \ @@ -29,13 +39,12 @@ RUN apk add --no-cache --virtual .build-deps \ tcl-dev \ tiff-dev \ tk-dev \ - zlib-dev - - -RUN apk add --no-cache --virtual .deps \ - bash + zlib-dev \ + && pip install pip==18.0 pipenv --upgrade \ + && adduser -S datamart FROM base +COPY --from=builder /code /code LABEL org.label.name="eTools Datamart" \ org.label.maintainer="sapostolico@unicef.org" \ @@ -100,7 +109,8 @@ ENV STATIC_ROOT /tmp/static # gcc RUN mkdir -p \ - /var/datamart/{static,log,conf,run} \ + /var/datamart/ \ + && chown datamart /var/datamart/ \ && pip install pip==18.0 pipenv --upgrade WORKDIR /code @@ -112,19 +122,17 @@ RUN set -ex \ RUN pip install . \ && rm -fr /code -#RUN apt-get autoremove --yes --force-yes \ -# gcc \ -# && rm -fr /var/lib/apt/lists/* \ -# && rm -fr /var/cache/apt/* -#RUN apk del .build-deps \ -# && rm -rf /var/cache/apk/* \ -# && rm -fr /root/.cache/ +RUN apk del .build-deps \ + && rm -rf /var/cache/apk/* \ + && rm -fr /root/.cache/ WORKDIR /var/datamart EXPOSE 8000 +USER datamart + ADD docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/Dockerfile.2 b/docker/Dockerfile.2 deleted file mode 100644 index 8874efa14..000000000 --- a/docker/Dockerfile.2 +++ /dev/null @@ -1,97 +0,0 @@ -FROM pstauffer/curl:latest as builder -ARG DEVELOP -ARG GITHUB_CREDENTIALS -ARG VERSION - -RUN mkdir /code -ADD . /code - -RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ - echo "${VERSION}-develop"; \ - else \ - echo "Download package: https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" \ - && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ - fi - -FROM unicef/datamart:base as base -COPY --from=builder /code /code - -LABEL org.label.name="eTools Datamart" \ - org.label.maintainer="sapostolico@unicef.org" \ - org.label.description="" \ - org.label.url="https://datamart.unicef.io/" \ - org.label.vcs-url="https://github.com/unicef/etools-datamart" \ - org.label.version=$VERSION - - -ARG BUILD_DATE -ARG PIPENV_PYPI_MIRROR -ARG PIPENV_ARGS -ARG VERSION - -ENV VERSION ${VERSION} - -ENV PIPENV_PYPI_MIRROR ${PIPENV_PYPI_MIRROR} -ENV PIPENV_ARGS ${PIPENV_ARGS} -ENV HOME /root/ -ENV PIPSI_HOME=/usr/local/pipsi/environments -ENV PIPSI_BIN_DIR=/usr/local/bin -ENV PYTHONUNBUFFERED 1 -ENV USE_GUNICORN 0 -ENV GUNICORN_CMD_ARGS "-b 0.0.0.0:8000 \ ---chdir /var/datamart \ ---access-logfile - \ ---access-logformat \"%(h)s %(l)s %(u)s %(t)s '%(r)s' %(s)s\" " - -ENV ALLOWED_HOSTS * -ENV CACHE_URL "redis://127.0.0.1:6379/1" -ENV CELERY_BROKER_URL "redis://127.0.0.1:6379/2" -ENV CELERY_RESULT_BACKEND "redis://127.0.0.1:6379/3" -ENV CSRF_COOKIE_SECURE True -ENV DATABASE_URL "postgres://postgres:@127.0.0.1:5432/etools_datamart" -ENV DATABASE_URL_ETOOLS "postgis://postgres:@127.0.0.1:5432/etools" -ENV DEBUG 0 -ENV DEVELOPMENT_MODE 0 -ENV DJANGO_SETTINGS_MODULE etools_datamart.config.settings -ENV MEDIA_ROOT /tmp/media -ENV SECRET_KEY "secret" -ENV SECURE_BROWSER_XSS_FILTER True -ENV SECURE_CONTENT_TYPE_NOSNIFF True -ENV SECURE_FRAME_DENY True -ENV SECURE_HSTS_INCLUDE_SUBDOMAINS True -ENV SECURE_HSTS_PRELOAD True -ENV SECURE_HSTS_SECONDS 1 -ENV SECURE_SSL_REDIRECT True -ENV SENTRY_DSN "" -ENV SESSION_COOKIE_HTTPONLY True -ENV SESSION_COOKIE_SECURE True -ENV STATIC_ROOT /tmp/static -#ENV SUPERVISOR_USER admin -#ENV SUPERVISOR_PWD "" -#ENV FLOWER_USER admin -#ENV FLOWER_PWD "" -#ENV X_FRAME_OPTIONS "DENY" -#ENV START_DATAMART "true" -#ENV START_REDIS "true" -#ENV START_CELERY "true" - -RUN pip install . \ - && rm -fr /code - -#RUN apt-get autoremove --yes --force-yes \ -# gcc \ -# && rm -fr /var/lib/apt/lists/* \ -# && rm -fr /var/cache/apt/* - -RUN apk del .build-deps \ - && rm -rf /var/cache/apk/* \ - && rm -fr /root/.cache/ - -WORKDIR /var/datamart - -EXPOSE 8000 - -ADD docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh -ENTRYPOINT ["docker-entrypoint.sh"] - -CMD ["datamart"] diff --git a/docker/Dockerfile.2.base b/docker/Dockerfile.2.base deleted file mode 100644 index 1a6d19254..000000000 --- a/docker/Dockerfile.2.base +++ /dev/null @@ -1,46 +0,0 @@ -FROM pstauffer/curl:latest as builder -ARG DEVELOP -ARG GITHUB_CREDENTIALS -ARG VERSION - -RUN mkdir /code -ADD . /code - -RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ - echo "${VERSION}-develop"; \ - else \ - echo "Download package: https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" \ - && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ - fi - -FROM python:3.6-alpine as base -COPY --from=builder /code /code -RUN apk add --no-cache --virtual .build-deps \ - freetype-dev \ - gcc \ - jpeg-dev \ - lcms2-dev \ - libffi-dev \ - linux-headers \ - musl-dev \ - openjpeg-dev \ - postgresql-dev \ - python3-dev \ - tcl-dev \ - tiff-dev \ - tk-dev \ - zlib-dev - - -RUN apk add --no-cache --virtual .deps \ - bash - -RUN mkdir -p \ - /var/datamart/{static,log,conf,run} \ - && pip install pip==18.0 pipenv --upgrade - -WORKDIR /code - -RUN set -ex \ - ls -al /code \ - && pipenv install --system --deploy --ignore-pipfile $PIPENV_ARGS diff --git a/docker/Makefile b/docker/Makefile index 0107a9857..4a60ccef2 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -6,14 +6,14 @@ DOCKER_PASS?= DOCKER_USER?= TARGET?=dev # below vars are used internally -BUILD_OPTIONS?= +BUILD_OPTIONS?=--squash CMD?=datamart CONTAINER_NAME?=datamart-${TARGET} ORGANIZATION=unicef IMAGE_NAME=datamart DOCKER_IMAGE_NAME=${ORGANIZATION}/${IMAGE_NAME} DOCKER_IMAGE=${DOCKER_IMAGE_NAME}:${TARGET} -DOCKERFILE?=Dockerfile.2 +DOCKERFILE?=Dockerfile RUN_OPTIONS?= PIPENV_ARGS?= @@ -23,31 +23,26 @@ help: @echo "release release tag ${TARGET} on docker hub" @echo "run run ${DOCKER_IMAGE} locally" - build-base: - if [[ "`docker images -q ${DOCKER_IMAGE_NAME}:base 2> /dev/null`" == "" ]]; then \ - cd .. && docker build \ - --build-arg DEVELOP=${DEVELOP} \ - --build-arg GITHUB_CREDENTIALS=${GITHUB_CREDENTIALS} \ - --build-arg VERSION=${TARGET} \ - --squash \ + docker build --target base \ -t ${DOCKER_IMAGE_NAME}:base \ - -f docker/${DOCKERFILE}.base . ; \ - docker push ${DOCKER_IMAGE_NAME}:base - fi - + -f docker/${DOCKERFILE} . -build: build-base +build: cd .. && docker build \ ${BUILD_OPTIONS} \ --build-arg DEVELOP=${DEVELOP} \ --build-arg GITHUB_CREDENTIALS=${GITHUB_CREDENTIALS} \ --build-arg VERSION=${TARGET} \ - --squash \ -t ${DOCKER_IMAGE} \ -f docker/${DOCKERFILE} . docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:dev docker images | grep ${DOCKER_IMAGE_NAME} +# flatten image + docker create ${DOCKER_IMAGE_NAME}:dev foo + docker export foo | docker import - unicef/datamart:flat + docker rm foo + .run: cd .. && docker run \ @@ -100,7 +95,7 @@ test: -e CSRF_COOKIE_SECURE=1 \ -e SECURE_HSTS_PRELOAD=1 \ -e SECURE_SSL_REDIRECT=1" \ - CMD='bash -c "django-admin check --deploy"' \ + CMD='bash -c "touch /var/datamart/.touch && django-admin check --deploy "' \ $(MAKE) .run scratch: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8c16ad6ca..adc012523 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -5,13 +5,13 @@ mkdir -p /var/datamart/{static,log,conf,run} rm -f /var/datamart/run/* -if [ "$*" == "workers" ];then +if [[ "$*" == "workers" ]];then django-admin db-isready --wait --timeout 60 --sleep 5 django-admin db-isready --wait --timeout 300 --sleep 5 --connection etools celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile run/celery.pid -elif [ "$*" == "beat" ];then +elif [[ "$*" == "beat" ]];then celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile run/celerybeat.pid -elif [ "$*" == "datamart" ];then +elif [[ "$*" == "datamart" ]];then django-admin db-isready --wait --timeout 60 django-admin check --deploy django-admin init-setup --all --verbosity 1 diff --git a/src/unicef_rest_framework/renderers/csv.py b/src/unicef_rest_framework/renderers/csv.py index e8fb81431..73b03479b 100644 --- a/src/unicef_rest_framework/renderers/csv.py +++ b/src/unicef_rest_framework/renderers/csv.py @@ -1,8 +1,18 @@ +import logging + +from crashlog.middleware import process_exception from rest_framework_csv import renderers as r +logger = logging.getLogger(__name__) + class CSVRenderer(r.CSVRenderer): def render(self, data, media_type=None, renderer_context=None, writer_opts=None): - data = dict(data)['results'] - return super().render(data, media_type, renderer_context or {}, writer_opts) + try: + data = dict(data)['results'] + return super().render(data, media_type, renderer_context or {}, writer_opts) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index 13cc70e23..59627a407 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -1,6 +1,11 @@ +import logging + +from crashlog.middleware import process_exception from django.template import loader from rest_framework.renderers import BaseRenderer +logger = logging.getLogger(__name__) + def labelize(v): return v.replace("_", " ").title() @@ -18,17 +23,22 @@ def get_template(self, meta): 'renderers/html/html.html']) def render(self, data, accepted_media_type=None, renderer_context=None): - model = renderer_context['view'].queryset.model - opts = model._meta - template = self.get_template(opts) - if data['results']: - c = {'data': data, - 'model': model, - 'opts': opts, - 'headers': [labelize(v) for v in data['results'][0].keys()]} - else: - c = {'data': {}, - 'model': model, - 'opts': opts, - 'headers': []} - return template.render(c) + try: + model = renderer_context['view'].queryset.model + opts = model._meta + template = self.get_template(opts) + if data['results']: + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': [labelize(v) for v in data['results'][0].keys()]} + else: + c = {'data': {}, + 'model': model, + 'opts': opts, + 'headers': []} + return template.render(c) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index 71c076b24..416e61531 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -1,12 +1,16 @@ import io +import logging import os +from crashlog.middleware import process_exception from django.conf import settings from django.template import loader from xhtml2pdf import pisa from .html import HTMLRenderer +logger = logging.getLogger(__name__) + def link_callback(uri, rel): """ @@ -47,13 +51,18 @@ def get_template(self, meta): 'renderers/pdf/pdf.html']) def render(self, data, accepted_media_type=None, renderer_context=None): - html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) - - # create a pdf - buffer = io.BytesIO() - pisaStatus = pisa.CreatePDF(html, dest=buffer, link_callback=link_callback) - # if error then show some funy view - if pisaStatus.err: - raise Exception('We had some errors 
    ' + html + '
    ') - buffer.seek(0) - return buffer.read() + try: + html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) + + # create a pdf + buffer = io.BytesIO() + pisaStatus = pisa.CreatePDF(html, dest=buffer, link_callback=link_callback) + # if error then show some funy view + if pisaStatus.err: + raise Exception('We had some errors 
    ' + html + '
    ') + buffer.seek(0) + return buffer.read() + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/renderers/xls.py b/src/unicef_rest_framework/renderers/xls.py index 1d89c11c2..a70a53918 100644 --- a/src/unicef_rest_framework/renderers/xls.py +++ b/src/unicef_rest_framework/renderers/xls.py @@ -1,9 +1,19 @@ +import logging + +from crashlog.middleware import process_exception from drf_renderer_xlsx.renderers import XLSXRenderer as _XLSXRenderer +logger = logging.getLogger(__name__) + class XLSXRenderer(_XLSXRenderer): def render(self, data, accepted_media_type=None, renderer_context=None): - if not data['results']: - return '' - return super().render(data, accepted_media_type, renderer_context) + try: + if not data['results']: + return '' + return super().render(data, accepted_media_type, renderer_context) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') From 7b9717ac4e49205292bc9003c70687a5a2ad73ce Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 15:03:06 +0100 Subject: [PATCH 42/43] flatten docker image --- docker/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 4a60ccef2..408ac0103 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -37,11 +37,11 @@ build: -t ${DOCKER_IMAGE} \ -f docker/${DOCKERFILE} . docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:dev - docker images | grep ${DOCKER_IMAGE_NAME} # flatten image - docker create ${DOCKER_IMAGE_NAME}:dev foo - docker export foo | docker import - unicef/datamart:flat + docker create ${DOCKER_IMAGE_NAME}:${TARGET} foo + docker export foo | docker import - unicef/datamart:${TARGET} docker rm foo + docker images | grep ${DOCKER_IMAGE_NAME} .run: From 1b6f575692c52be119fd91a319082f2c9044b5fa Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 30 Nov 2018 19:18:14 +0100 Subject: [PATCH 43/43] bump version --- CHANGES | 4 ++-- src/etools_datamart/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 79c706c30..25efa5264 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -1.7 (dev) ---------- +1.7 +--- * WARNINGS: migration reset * add YAML format * new 'Monitor' page to check dataset last update date diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index 775f70f33..d8a3c921b 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = 'etools-datamart' -VERSION = __version__ = '1.7a0' +VERSION = __version__ = '1.7' __author__ = ''