diff --git a/.circleci/config.yml b/.circleci/config.yml index 63b8c2436..8c042d5b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,15 +3,15 @@ jobs: build: working_directory: ~/code docker: - - image: circleci/python:3.6 + - image: python:3.6.7-slim environment: PGHOST: 127.0.0.1 PIPENV_VENV_IN_PROJECT: 1 - DATABASE_URL: "postgres://postgres:postgres@127.0.0.1:5432/etools_datamart" + DATABASE_URL: "postgis://postgres:postgres@127.0.0.1:5432/etools_datamart" DATABASE_URL_ETOOLS: "postgis://postgres:postgres@127.0.0.1:5432/etools" RELEASE_MATCH: "release/*" - image: redis:alpine - - image: circleci/postgres:9.6-alpine-postgis + - image: mdillon/postgis:9.6 environment: POSTGRES_USER: postgres PGUSER: postgres @@ -21,7 +21,21 @@ jobs: - run: name: Install dependencies command: | - sudo apt-get update && sudo apt-get -y install postgresql + mkdir -p /usr/share/man/man1 + mkdir -p /usr/share/man/man7 + apt-get update + apt-get install -y \ + libc-bin \ + gcc \ + curl \ + gdal-bin \ + python-dev \ + postgresql-client-9.6 + + - run: + name: Create databases + command: | + createdb --user postgres etools - restore_cache: keys: @@ -36,8 +50,8 @@ jobs: command: | export PATH=/home/circleci/.local/bin:$PATH export PYTHONHASHSEED=${RANDOM} - pip install tox --user - tox -e py36-d21 deps + pip install tox + tox -e py36-d21 - run: name: codecov command: | @@ -64,14 +78,14 @@ jobs: - deploy: name: tag and release if release candidate command: | - if [[ $CIRCLE_BRANCH == $RELEASE_MATCH ]]; then - curl --user ${CIRCLE_TOKEN}: \ - --data build_parameters[CIRCLE_JOB]=tag \ - --data revision=$CIRCLE_SHA1 \ - https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/$CIRCLE_BRANCH - else - echo "Skipped as '$CIRCLE_BRANCH' does not match '$RELEASE_MATCH' branch" - fi + if [[ $CIRCLE_BRANCH == $RELEASE_MATCH ]]; then + curl --user ${CIRCLE_TOKEN}: \ + --data build_parameters[CIRCLE_JOB]=tag \ + --data revision=$CIRCLE_SHA1 \ + https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/$CIRCLE_BRANCH + else + echo "Skipped as '$CIRCLE_BRANCH' does not match '$RELEASE_MATCH' branch" + fi tag: docker: - image: circleci/python:3.6 @@ -151,11 +165,11 @@ jobs: - deploy: name: dockerize command: | - export TAG=${TAG:=${CIRCLE_BRANCH#*/}} - curl --user ${CIRCLE_TOKEN}: \ - --data build_parameters[TAG]=$TAG \ - --data build_parameters[JOB]=dockerize \ - https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/develop + export TAG=${TAG:=${CIRCLE_BRANCH#*/}} + curl --user ${CIRCLE_TOKEN}: \ + --data build_parameters[TAG]=$TAG \ + --data build_parameters[JOB]=dockerize \ + https://circleci.com/api/v1.1/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/tree/develop dockerize: working_directory: ~/code @@ -201,25 +215,25 @@ jobs: - run: name: Test Backend docker image command: | - echo "Testing ${DOCKER_IMAGE}:${TAG}" - docker run -p 8000:8000 \ - --rm \ - -e DATABASE_URL=${DATABASE_URL} \ - -e DEBUG=0 \ - -e SECURE_SSL_REDIRECT=0 \ - -e SESSION_COOKIE_SECURE=0 \ - -e SESSION_COOKIE_HTTPONLY=9 \ - -e SESSION_COOKIE_HTTPONLY=0 \ - -e CSRF_COOKIE_SECURE=0 \ - -it ${DOCKER_IMAGE}:${TAG} \ - django-admin check --deploy + echo "Testing ${DOCKER_IMAGE}:${TAG}" + docker run -p 8000:8000 \ + --rm \ + -e DATABASE_URL=${DATABASE_URL} \ + -e DEBUG=0 \ + -e SECURE_SSL_REDIRECT=0 \ + -e SESSION_COOKIE_SECURE=0 \ + -e SESSION_COOKIE_HTTPONLY=9 \ + -e SESSION_COOKIE_HTTPONLY=0 \ + -e CSRF_COOKIE_SECURE=0 \ + -it ${DOCKER_IMAGE}:${TAG} \ + django-admin check --deploy - deploy: name: Push Backend docker image command: | - echo "Pushing ${DOCKER_IMAGE}:${TAG} to Docker Hub" - export TODAY=`date '+%d %B %Y at %H:%M'` + echo "Pushing ${DOCKER_IMAGE}:${TAG} to Docker Hub" + export TODAY=`date '+%d %B %Y at %H:%M'` - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker tag ${DOCKER_IMAGE}:${TAG} ${DOCKER_IMAGE}:latest - docker push ${DOCKER_IMAGE}:latest - docker push ${DOCKER_IMAGE}:${TAG} + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker tag ${DOCKER_IMAGE}:${TAG} ${DOCKER_IMAGE}:latest + docker push ${DOCKER_IMAGE}:latest + docker push ${DOCKER_IMAGE}:${TAG} diff --git a/.gitignore b/.gitignore index 74390e2d4..e866167db 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ src/etools_datamart/apps/etools/models/public_old.py src/etools_datamart/apps/etools/models/tenant_old.py xml ~* +**/dual-listbox-master/ +#src/etools_datamart/apps/security/static/security/dual-listbox-master diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index bb30eb930..000000000 --- a/.style.yapf +++ /dev/null @@ -1,4 +0,0 @@ -[style] -based_on_style = pep8 -spaces_before_comment = 4 -split_before_logical_operator = true diff --git a/CHANGES b/CHANGES index 25efa5264..b0d89854f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,18 @@ +1.8 +--- +* WARNINGS: migration reset +* add ability to create 'per model' custom templates for html/pdf renderers + new `TEMPLATE_CACHE_URL` enironment variable +* fixes error in some renderers with cached response +* Adopting of RabbitMQ to prevent message loss +* new admin index page +* Excel IQY support (beta) +* Allow endpoints to be consumend bny anonymous users +* Countries can be selected using name, schema_name,country_short_code or business_area_code +* Improved Browseable API +* `page_size` now accept `-1` to disable pagination + + 1.7 --- * WARNINGS: migration reset diff --git a/Makefile b/Makefile index ddf55f8bd..b7528c32b 100644 --- a/Makefile +++ b/Makefile @@ -30,9 +30,6 @@ lint: pipenv run pre-commit run --all-files pipenv run pre-commit run --all-files --hook-stage push pipenv run pre-commit run --all-files --hook-stage manual -# pipenv run flake8 src/ tests/ -# pipenv run isort -rc src/ --check-only -# pipenv run check-manifest clean: rm -fr ${BUILDDIR} dist *.egg-info .coverage coverage.xml .eggs @@ -59,14 +56,17 @@ urf: pipenv run pytest tests/urf --cov-config tests/urf/.coveragerc -demo: +stack: PYTHONPATH=./src pipenv run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid & PYTHONPATH=./src pipenv run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid & +# PYTHONPATH=./src pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & + pipenv run docker run -d -p 5555:5555 -e CELERY_BROKER_URL=${CELERY_BROKER_URL} --name datamart-flower --rm saxix/flower + +demo: stack PYTHONPATH=./src pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & - pipenv run docker run -d -p 5555:5555 -e CELERY_BROKER_URL=$CELERY_BROKER_URL --name datamart-flower --rm saxix/flower -stop-demo: - - kill `cat gunicorn.pid` - - kill `cat beat.pid` - - kill `cat celery.pid` - - docker stop datamart-flower +demo-stop: + -kill `cat gunicorn.pid` + -kill `cat beat.pid` + -kill `cat celery.pid` + -docker stop datamart-flower diff --git a/Pipfile b/Pipfile index b84c8130c..5cd7ffc74 100644 --- a/Pipfile +++ b/Pipfile @@ -8,7 +8,7 @@ psycopg2 = "*" admin-extra-urls = ">=2.1" celery = "*" coreapi = "*" -django = ">=2.1" +django = ">=2.1.4" django-adminfilters = ">=1.1" django-celery-beat = "==1.1.1" django-concurrency = "*" @@ -21,7 +21,6 @@ django-redis = "*" django-regex = "*" django-strategy-field = "*" django-sysinfo = "*" -django-tenant-schemas = "*" djangorestframework-csv = "*" djangorestframework-jwt = "*" drf-dynamic-serializer = ">=1.2.0" @@ -50,12 +49,13 @@ djangorestframework-yaml = "*" django-storages = {extras = ["azure"], version = "*"} onedrivesdk = "*" azure-storage = "*" -django-basicauth = "*" django-post-office = "*" django-celery-email = "*" "xhtml2pdf" = "*" -pisa = "*" django-crispy-forms = "*" +django-adminactions = "*" +django-dbtemplates = {file = "https://github.com/jazzband/django-dbtemplates/archive/2.0.1.tar.gz"} +django-mptt = "*" [dev-packages] "flake8" = ">=3.6.0" diff --git a/Pipfile.lock b/Pipfile.lock index 2b97db8c8..76e5b6155 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2c79b29b6f0397ba9724ed2ec50a286a3178b5b335f0a3b70ff3594a60dc9a9d" + "sha256": "8832190e1faa25c54328fd0f59bc55c31188ce6291b45d07206e16041494d72a" }, "pipfile-spec": 6, "requires": { @@ -215,17 +215,17 @@ }, "azure-mgmt-containerregistry": { "hashes": [ - "sha256:7db871a74dfe6b8f54e208f6e2f43e0f6a034625a735d62013071c5e44409f8d", - "sha256:d5419db4543aaf5d83f73e087df0c0193f6b987f5c6161ac0fdd8eeabbfd23b0" + "sha256:09f1554da5a86f645fe816620223f93000bcbc5cbfb92e210077e3c5cdaf0b95", + "sha256:75cd215bbf2e3254b2e03eb0b4efc047aac85c79d3527aeb2aa6059e784b7ed0" ], - "version": "==2.4.0" + "version": "==2.5.0" }, "azure-mgmt-containerservice": { "hashes": [ - "sha256:2ff15bc2de14dbcee93d25d7cbe379c88aa751bc13cb199076b127fc4e221acb", - "sha256:99df430a03aada02625e35ef13d7de6c667e9bef56b5e2f60b2c284514223bff" + "sha256:219ace1349dd95198de4f4c8e63b67d4780a8e93a0151f3423db3bafadde912d", + "sha256:c8b8dfe8a2bfeac5d19dd6e32251a306741c50b4b167ca765dc83ea13a06dd48" ], - "version": "==4.2.2" + "version": "==4.3.0" }, "azure-mgmt-cosmosdb": { "hashes": [ @@ -633,9 +633,9 @@ }, "billiard": { "hashes": [ - "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84" + "sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e" ], - "version": "==3.5.0.4" + "version": "==3.5.0.5" }, "celery": { "hashes": [ @@ -753,11 +753,18 @@ }, "django": { "hashes": [ - "sha256:1ffab268ada3d5684c05ba7ce776eaeedef360712358d6a6b340ae9f16486916", - "sha256:dd46d87af4c1bf54f4c926c3cfa41dc2b5c15782f15e4329752ce65f5dad1c37" + "sha256:068d51054083d06ceb32ce02b7203f1854256047a0d58682677dd4f81bceabd7", + "sha256:55409a056b27e6d1246f19ede41c6c610e4cab549c005b62cbeefabc6433356b" ], "index": "pypi", - "version": "==2.1.3" + "version": "==2.1.4" + }, + "django-adminactions": { + "hashes": [ + "sha256:cc8d236797dcebdfa229dc0f938dc8c5ab648b488318c9f584a13df8a03bd066" + ], + "index": "pypi", + "version": "==1.6.0" }, "django-adminfilters": { "hashes": [ @@ -773,14 +780,6 @@ ], "version": "==1.0.2" }, - "django-basicauth": { - "hashes": [ - "sha256:0ceff44ebc129eb7f8bde212a2f663210796ea1ba6e00d944cba50d4ef326f79", - "sha256:740a176e0bbeed8fd267e165a4373aa51a346258fb87479670dd7f6846f118d1" - ], - "index": "pypi", - "version": "==0.5.1" - }, "django-braces": { "hashes": [ "sha256:a457d74ea29478123c0c4652272681b3cea0bf1232187fd9f9b6f1d97d32a890", @@ -857,6 +856,9 @@ "index": "pypi", "version": "==0.5.0" }, + "django-dbtemplates": { + "file": "https://github.com/jazzband/django-dbtemplates/archive/2.0.1.tar.gz" + }, "django-environ": { "hashes": [ "sha256:6c9d87660142608f63ec7d5ce5564c49b603ea8ff25da595fd6098f6dc82afde", @@ -873,6 +875,13 @@ "index": "pypi", "version": "==2.0.0" }, + "django-js-asset": { + "hashes": [ + "sha256:30149158206f693a5d027fe590096fc84495486bd11cd77d395b4f2ec27fc1d0", + "sha256:a395d8d19eb201ea8d2bd4f145b38f1717cd74c0f609f040141d8724c5a27f36" + ], + "version": "==1.1.0" + }, "django-model-utils": { "hashes": [ "sha256:2c057f3bf0859aba27f04389f0cedd2d48f8c9b3848acb86fd9970794e58f477", @@ -881,6 +890,14 @@ "index": "pypi", "version": "==3.1.2" }, + "django-mptt": { + "hashes": [ + "sha256:18a41d1b56ca7c02a5b04d246e33ee2d18f6ee5459c02ed1d945f5abdef23a2e", + "sha256:689a04cce0981671d6061a9928c33a16b47abb0d4cd43cf7dec31ae284fdae9d" + ], + "index": "pypi", + "version": "==0.9.1" + }, "django-oauth-toolkit": { "hashes": [ "sha256:ad1b76275950ebbff708222cec57bbdb879f89bac7df6b9dee0f4b9db485c264" @@ -958,13 +975,6 @@ "index": "pypi", "version": "==1.3.2" }, - "django-tenant-schemas": { - "hashes": [ - "sha256:7d3f96f6e1969ade79b22a7600553fa8843c8aff77dbbad8f1e1e7523b9a0cd3" - ], - "index": "pypi", - "version": "==1.9.0" - }, "djangorestframework": { "hashes": [ "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136", @@ -989,11 +999,11 @@ }, "djangorestframework-xml": { "hashes": [ - "sha256:caea8e446298b7fe1eb9a79306f35554db7531c2e637734d32de3cf99afbdc5a", - "sha256:f7d5efc26eabbca73db0ff0f0c15b59ca08e36660c02f96563a0d937321f519f" + "sha256:d8118580b6c0e94a6b908a78c8d842e9f349901dfff43d91adc2d73a54f4ba59", + "sha256:d85d5744e75fe01ea2af667b15f6aa7df97c710516477ba493558da8432f6b0f" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.4.0" }, "djangorestframework-yaml": { "hashes": [ @@ -1020,11 +1030,11 @@ }, "drf-renderer-xlsx": { "hashes": [ - "sha256:1fad2c299f444a68b2ff963a28df11697427afc582485bab26e8efacf1596bfb", - "sha256:b08c55b4a0c75578457fbfcaf75fce081cce0f46c84ddce0d86abf01dbce8c27" + "sha256:21e9975f6ac6a45c91e8109f5c53ae6ae0569b9a8d4406e74590efafc56ef448", + "sha256:9bbdcd210abdf66e8c22b2c2cdb3fac67b017c151d496a944a52ff33a777c3b6" ], "index": "pypi", - "version": "==0.3.0" + "version": "==0.3.1" }, "drf-yasg": { "extras": [ @@ -1075,10 +1085,10 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "inflection": { "hashes": [ @@ -1136,10 +1146,10 @@ }, "kombu": { "hashes": [ - "sha256:86adec6c60f63124e2082ea8481bbe4ebe04fde8ebed32c177c7f0cd2c1c9082", - "sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af" + "sha256:52763f41077e25fe7e2f17b8319d8a7b7ab953a888c49d9e4e0464fceb716896", + "sha256:9bf7d37b93249b76a03afb7bbcf7149a358b6079ca2431e725414b1caa10922c" ], - "version": "==4.2.1" + "version": "==4.2.2" }, "markupsafe": { "hashes": [ @@ -1183,10 +1193,10 @@ }, "msrestazure": { "hashes": [ - "sha256:1118d52fb60fd71732a51bcb669189af4f72f40ea460e656465ee83c4738f2a0", - "sha256:fd1bcb9652cf04b711e21dcbef377a7e43f9492afeb0a59622b5623ce8715825" + "sha256:86695c9728e91ee6f4cbb3b6e82ebd92233b72b6e145f7dcafa6b2da17412b88", + "sha256:99b185dcfc94f4ac2f7792691b865958c5a9871281cd618c50963c6b8d23401b" ], - "version": "==0.5.1" + "version": "==0.6.0" }, "oauthlib": { "hashes": [ @@ -1243,14 +1253,6 @@ ], "version": "==5.3.0" }, - "pisa": { - "hashes": [ - "sha256:94c4ae0995c84bb0588ece4480486464612ed1526f0987fb1016b9c50e5d3327", - "sha256:a7164ac81ab5ea01fbae4f29d2c00183a31142ca30ad527f6ac96635819cbd12" - ], - "index": "pypi", - "version": "==3.0.33" - }, "psutil": { "hashes": [ "sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736", @@ -1309,10 +1311,10 @@ }, "pyjwt": { "hashes": [ - "sha256:30b1380ff43b55441283cc2b2676b755cca45693ae3097325dea01f3d110628c", - "sha256:4ee413b357d53fd3fb44704577afac88e72e878716116270d722723d65b42176" + "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", + "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96" ], - "version": "==1.6.4" + "version": "==1.7.1" }, "pyparsing": { "hashes": [ @@ -1377,11 +1379,11 @@ }, "raven": { "hashes": [ - "sha256:3fd787d19ebb49919268f06f19310e8112d619ef364f7989246fc8753d469888", - "sha256:95f44f3ea2c1b176d5450df4becdb96c15bf2632888f9ab193e9dd22300ce46a" + "sha256:3fa6de6efa2493a7c827472e984ce9b020797d0da16f1db67197bcc23c8fae54", + "sha256:44a13f87670836e153951af9a3c80405d36b43097db869a36e92809673692ce4" ], "index": "pypi", - "version": "==6.9.0" + "version": "==6.10.0" }, "redis": { "hashes": [ @@ -1393,43 +1395,43 @@ }, "reportlab": { "hashes": [ - "sha256:0fcc951899c1bb6e0fa90996d5ed9d24265c5a8eafabf2303570e079d0290c28", - "sha256:234e1790858f1608d2fc8f820fa5660b7838cb14f12dec81e3c1156a4891ee1a", - "sha256:23df47e5a4225728a6d0a16b1dbc2674fe8a535603c191ad9d33f8755d1ab08b", - "sha256:277e920f1f5aa48531d34a25ba83e4e5a201e15a6a75263d529c8996a84233ae", - "sha256:2c52b314f29a40b8ed863190947e556b75a719566d4e2e7d25b8aeb18f60b58c", - "sha256:2e38e61d8e32b003a4c50f4bf1593bc5784395a62d180edd7b206f7dfd77836c", - "sha256:2f42e2e6177756a8265126fbeb7be58fbc40affd471253c829ab165c13253f2e", - "sha256:463f446a96bd0ddaba8ca46747f251868334c119f28bffd42fc0add2b3bd986f", - "sha256:4be2e78b900a601028b4ff42c547096cc204a21ac2a9ce7d16768195106ffd12", - "sha256:4db5daa83aa5c74fba41c3212549ff62abc84921b9ce848bc04cda728e00ecf0", - "sha256:4e4669e49a4cbb323c6a3cb45297535e910d4627f796dba90362e456fb013e44", - "sha256:551bed92c44e302ac7d663242cda10bb8e459ccff0178f51d5693f846336961d", - "sha256:60995c16fa961a6576a67aaeb7a6657a081dabdea298d4c15ec68423171dcdc6", - "sha256:6228323d6a40355c20b4159bc927cb93b9c39537bd39c25b9071eedcf80b4a36", - "sha256:62aafd7769fcf36c9910a332ccee002ce4ff850c9b3ff89bbea8d7b78be979c1", - "sha256:64bf9f0778477a69563db3ca53db3312b7a65756c60e674de54e7527a0852fa0", - "sha256:678667a846ad4181b5a392c59e1ec37e03abc684854d66c7a05df32113a4d337", - "sha256:722bc6e77526cbe927f9a65f37b06840e6d99d6938462892558c6cbb90445d4c", - "sha256:8c967cf193ebeca8231e3c0c28d94c9d54dd1ee38284785c3a1cbf3aca637fd9", - "sha256:95bc72a157559b5963967cf8be17d9185ed64b5c408504b9efbd51ddcf4c145e", - "sha256:9b22737cbaff0e7d875bcb61cb593df908513f1b807551c71afc3a9d1fe83aeb", - "sha256:a2f26770e8af0d383d586fd95446ee246d0532395c9392463c07047cb89862bb", - "sha256:a417205a64cbec93219a8d7e268fe4ba4b7f3e037f7d1ca42f432b4388fb93dd", - "sha256:a6856b4118abbc783ce419e50185e69bceb4d49de39d8439f28571d5949fc34c", - "sha256:b04381abcb3c14afcef7faababbd620b2a352a2f2765f3bf18deabb2dfc7d617", - "sha256:b2348ec5f615d50913c1eaa8c3fc71d4478e4ea0e627950f6fabd70d2a38ff1d", - "sha256:f86f13f4e6d8c815c35ec75e3d3a94cf7de12d544b4d038269c8b7c73de6b906", - "sha256:fa7761cca303d4dbc2f5c5989b2be4baa4068fa4137a5fdc03eb47bfc1af0a5d" - ], - "version": "==3.5.11" + "sha256:00c4d275b14ccd77316c0e6b5ad12881c70edc97556b1177d90fe366163df786", + "sha256:03f968b21c41ff3364665b3f08426233e8255313c158a50f71745e66accd6d90", + "sha256:0a6bb96cc4ca7faaa5bcfb599eadd4afe6b90929cdc31e4913be4899f9ae4ac6", + "sha256:1857bb48c67b9278d5ada84a9c584f6bd61bb95d4f67d55799e8af9c8a5ef6e2", + "sha256:1b52d4d5ba9f5a1e939eed9441ca62a2ef7063a2610d001c70b39760b9d3314c", + "sha256:2637ea9a2ca093f10f0fc27ea02a8bdde0faadbeab9205ef1f5bf560510c6d52", + "sha256:2d63984f9593e5408cfc3da043cae871ab20006d494d5029a04ec5fa0d3044d6", + "sha256:55945d811655c52515067826f4f2a258e472fc34ad995acdc9a5f53910b8ebf6", + "sha256:63f1928c47acd9aa81ef75dd29ce74d292e50d77eaf2629ad0c99bbc1d2fa125", + "sha256:65a70673bc7d673ca52ee8b1b14e37959b01d45ef43729ed1507b6718351ec47", + "sha256:6dd39a260fd8e315f55e5f61c0a4b07994c3cd06d4893aacc9575a0e8804ba12", + "sha256:786e0d5b166fcfa1d0044ee411f820c3314b0bbc513985b0adc227118c2db009", + "sha256:7e10261065d0f926d9d83fd1f2edb8bec466f3c60b3e927ef40e2262805c069d", + "sha256:8474ed2a5d89a1546435294c91ae42fc18a29c3172af4b5f97d43bbf4d0a9018", + "sha256:8ce32d0878a38bc1f8b6ee1656f638a7629b2067f9b048787d4ad5fbd409093d", + "sha256:8fd604e791367038b673082c6e1d1801ef0998a68b0d436bcc0befc592f99541", + "sha256:9c25661f5089863c6976c0525593cdd3f3afb59092fbaa4ea9eb83c4e95a8c3c", + "sha256:a5c58f0ca4a4dd74fc8816a5240891614355d909b8c606febfb24375330b2a3b", + "sha256:a6b9a411ff87bca1ce33c902c25d25f11ea53d4941ad5d41b070a3eb95843b09", + "sha256:c1f0104d0a85d0db9bc98d5fa3679391ef052219b0f3ff0b741d22b0b78deb5f", + "sha256:c8f59aaa6989e111b0bfb46daae91c02cc4f5c23ce053fbd82ea6dac1c4c087d", + "sha256:cdca0b57cb5efccd946f79470598706fff661190627cb213e6eb7749aaeb02eb", + "sha256:d37854d9bce188b336dbf6de598924656d2ff0555dc720d347970ea8df097895", + "sha256:d6ab0ff7f2c3cf9ef33c163f5c85e3ebb1ced512d6ef8cdd022617f5951b1385", + "sha256:e7a145f376fcf56d0697b028fbd228dd6defe78bdf671a6ec6a8a87a930d8b6c", + "sha256:e94bd2457a5030df103d2f1b892d322076cd8c4b8db6c4331382f35431eb4ece", + "sha256:ecd7026ec27e6b33513fe53338ca33bd44dce6d8b00f6ec0ab4bfa10614a4503", + "sha256:fd697bc9afdad1e05d245da169a8c98f576a5c7b3d835fee76922f3e8b2d0388" + ], + "version": "==3.5.12" }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" ], - "version": "==2.20.1" + "version": "==2.21.0" }, "requests-oauthlib": { "hashes": [ @@ -1447,37 +1449,37 @@ }, "ruamel.yaml": { "hashes": [ - "sha256:10079d03b5c93d54be90e4fe23d4b1f32502d7da98077e2a746c216bedba3d75", - "sha256:1ff2289958e09fac2aa573e7a9fb9c953ddf89f67c3a42693394920f72001348", - "sha256:3b8af255839c39d3dfd0dcb82db349f38db28a2f7adbe05387bea87de15ac146", - "sha256:418ee849362ad59c19064af3ce09666d0898969eadb25964b693827fb68cfeca", - "sha256:4f203351575dba0829c7b1e5d376d08cf5f58e4a2b844e8ce552b3e41cd414e6", - "sha256:5421c3fb144d6e1de0dc00d8a1f919f558c3156c48aa7aa2acbd7754530dfeb7", - "sha256:59a17a6225d6d60150647d2fd707fcb1ee54ea31dd0048ea120c7bf2c9093451", - "sha256:6acdf9d7bea6ff8541e96e4694b9fd1e0728be88ef512afc28da0804590533f7", - "sha256:70ed366ad65780040f5bf5e34c75f450e3f0ca9a8b2534cfc0293c387ec1c32f", - "sha256:7e9dc8d095d952c352d9f63cab5283430c910b77793f323ce8d64921f65aaa53", - "sha256:8344a08555f8494ae16a2e4f445e5bfce80f82010d9e5091c870aadee5c4b14a", - "sha256:90b1f9ed3893b0713e0bc47cfa93be72ddb6d5a969c31979d36c2854dc8b87dd", - "sha256:93f262943089657675b336f804b721e6b76f67cc61f6bfc91b3f35f8e71a8a64", - "sha256:9996c6371bfe3051340a469037323f533bbdf2dea9b6914da27e62696c81712b", - "sha256:9d9a382be1c150d23cd1291091454ea801522629bd22531f0b2c41ab21a81deb", - "sha256:adb57424caeb48f8cf1c937e4c571e5720e38601a77dd87d4c1178d406a72821", - "sha256:b5147c0919c6ce29c278afe21bf64c3f099afc271364a038c5e59fcf7bb672fb", - "sha256:b71cfb6c90f7b6db26842923e5c178c3ad232577f6097ebca3651be366c84b36", - "sha256:d9a82d37a4b006b12e09550ff57f7edb5ec987f0d9128fc44f590d56683aa97d", - "sha256:dc76688fb7994bf9c2370e28238bd56f9fe7e1d02675f3b1e06fc35967375869", - "sha256:fa5ae31d1aea11b528f5987e43194abdfcb44a5a259172221097eb6f74bd0965", - "sha256:ff6046de0ed29c3f3e6ba2ce164a85781af16dd49049af1b0402b8a6d15a25ca" - ], - "version": "==0.15.80" + "sha256:013ae9f361cc491f491bf42070a855304a2af08d8bae30286da3c0df9a534d48", + "sha256:1890548aeeb0486892a316292bfa4041a7322c0f8047b0b30efa3c1aa2041d10", + "sha256:203d03410fa1bc5e16e587597919c3b5b25134f77a3480504ecc37ac5f80bc0c", + "sha256:22de3967f810236f0a7012d8f61465eb4f8775c9ea2c26352126d12d53dc171b", + "sha256:2a8e57c9ada267713eea270c3e7027372fe3dbcdad896447bd040381ab4194a7", + "sha256:3ef17bf29cc5b0c261e73a8f414e8cc097568095caca4e81a33d85d01f09a255", + "sha256:54ed540886773f23213b9af3b96a5d56289bec49a20cafa967aee4f2d4a5bac6", + "sha256:667b9cd5722b8f5e67d7b8858ed0cf0e91341a022805133416017a32a9b0c731", + "sha256:6cbe7273a2e7667cd2ca7b12bec1c715a8259ad80f09c6f12c378f664d29fa5e", + "sha256:7615753d902daa884efbabcfa794e6452f7a2da70eb614bf4b6c1bdf294a351e", + "sha256:95a82a2818b7e9c288dd5b4f18cea8e3e265cbcc3f6f9a59aa1bb8c7148f7554", + "sha256:9642484f4d669337894e2b5503df6b01b64ae063fe2d335b5352a1baefb16245", + "sha256:a6bedf2118add5143c9a5b36add2ed6d761831494923a0fca7a0d71b3abdaea5", + "sha256:a798ab6e524fe378f9c671e47cb6ba8c8448a05b98f3191cd513a420f0ca72cc", + "sha256:bc063dec1ab6701498a71e6e8c66851d0cdd2ed4037f96e0d162b649a891c3a2", + "sha256:c32261177f6f78a706fc850b3c7537ec9d3c56bb951801c7da2a1743388a8f22", + "sha256:c4d08d4dbb433917ca5a9cdfba79df4fb80bfc88292ca4c8ef68091ee2b86798", + "sha256:db0e164fced130cd379b9610e3eb20f9d4caca8ceee69c6860ea6d9c3ab6fed4", + "sha256:dbce257cd2c7e15c8e55609955788533091e2895af027a7753100aaab8530601", + "sha256:ec74a3893fb4078fc86b85814db980a1fa2ed43141095ac9faab76828ffe7326", + "sha256:ecc9d9e81567452244c3e3b2a1032de4626ae79743426aa6372c7ec3f88917f8", + "sha256:f29fecbe81a6e1648ae60fe5c0662805de05c205b2d0f5662e8da72aee1a40dd" + ], + "version": "==0.15.81" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" }, "social-auth-app-django": { "hashes": [ @@ -1573,6 +1575,20 @@ ], "index": "pypi", "version": "==0.2.3" + }, + "xlrd": { + "hashes": [ + "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2", + "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde" + ], + "version": "==1.2.0" + }, + "xlwt": { + "hashes": [ + "sha256:a082260524678ba48a297d922cc385f58278b8aa68741596a87de01a9c628b2e", + "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88" + ], + "version": "==1.3.0" } }, "develop": { @@ -1762,10 +1778,10 @@ }, "faker": { "hashes": [ - "sha256:c61a41d0dab8865b850bd00454fb11e90f3fd2a092d8bc90120d1e1c01cff906", - "sha256:f909ff9133ce0625ca388b6838190630ad7a593f87eaf058d872338a76241d5d" + "sha256:228419b0a788a7ac867ebfafdd438461559ab1a0975edb607300852d9acaa78d", + "sha256:52a3dcc6a565b15fe1c95090321756d5a8a7c1caf5ab3df2f573ed70936ff518" ], - "version": "==1.0.0" + "version": "==1.0.1" }, "fancycompleter": { "hashes": [ @@ -1798,10 +1814,10 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "importlib-metadata": { "hashes": [ @@ -1820,11 +1836,11 @@ }, "ipython": { "hashes": [ - "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435", - "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd" + "sha256:6a9496209b76463f1dec126ab928919aaf1f55b38beb9219af3fe202f6bbdd12", + "sha256:f69932b1e806b38a7818d9a1e918e5821b685715040b48e59c657b3c7961b742" ], "index": "pypi", - "version": "==7.1.1" + "version": "==7.2.0" }, "ipython-genutils": { "hashes": [ @@ -1844,10 +1860,10 @@ }, "jedi": { "hashes": [ - "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7", - "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148" + "sha256:571702b5bd167911fe9036e5039ba67f820d6502832285cde8c881ab2b2149fd", + "sha256:c8481b5e59d34a5c7c42e98f6625e633f6ef59353abea6437472c7ec2093f191" ], - "version": "==0.13.1" + "version": "==0.13.2" }, "mccabe": { "hashes": [ @@ -1942,11 +1958,11 @@ }, "pre-commit": { "hashes": [ - "sha256:7542bd8ae1c58745175ea0a9295964ee82a10f7e18c4344f5e4c02bd85d02561", - "sha256:87f687da6a2651d5067cfec95b854b004e95b70143cbf2369604bb3acbce25ec" + "sha256:33bb9bf599c334d458fa9e311bde54e0c306a651473b6a36fdb36a61c8605c89", + "sha256:e233f5cf3230ae9ed9ada132e9cf6890e18cc937adc669353fb64394f6e80c17" ], "index": "pypi", - "version": "==1.12.0" + "version": "==1.13.0" }, "prompt-toolkit": { "hashes": [ @@ -1986,18 +2002,18 @@ }, "pygments": { "hashes": [ - "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", - "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" + "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", + "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" ], - "version": "==2.3.0" + "version": "==2.3.1" }, "pytest": { "hashes": [ - "sha256:1d131cc532be0023ef8ae265e2a779938d0619bb6c2510f52987ffcba7fa1ee4", - "sha256:ca4761407f1acc85ffd1609f464ca20bb71a767803505bd4127d0e45c5a50e23" + "sha256:f689bf2fc18c4585403348dd56f47d87780bf217c53ed9ae7a3e2d7faa45f8e9", + "sha256:f812ea39a0153566be53d88f8de94839db1e8a05352ed8a49525d7d7f37861e9" ], "index": "pypi", - "version": "==4.0.1" + "version": "==4.0.2" }, "pytest-cov": { "hashes": [ @@ -2082,17 +2098,17 @@ }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" ], - "version": "==2.20.1" + "version": "==2.21.0" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" }, "text-unidecode": { "hashes": [ @@ -2194,18 +2210,20 @@ }, "yarl": { "hashes": [ - "sha256:2556b779125621b311844a072e0ed367e8409a18fa12cbd68eb1258d187820f9", - "sha256:4aec0769f1799a9d4496827292c02a7b1f75c0bab56ab2b60dd94ebb57cbd5ee", - "sha256:55369d95afaacf2fa6b49c84d18b51f1704a6560c432a0f9a1aeb23f7b971308", - "sha256:6c098b85442c8fe3303e708bbb775afd0f6b29f77612e8892627bcab4b939357", - "sha256:9182cd6f93412d32e009020a44d6d170d2093646464a88aeec2aef50592f8c78", - "sha256:c8cbc21bbfa1dd7d5386d48cc814fe3d35b80f60299cdde9279046f399c3b0d8", - "sha256:db6f70a4b09cde813a4807843abaaa60f3b15fb4a2a06f9ae9c311472662daa1", - "sha256:f17495e6fe3d377e3faac68121caef6f974fcb9e046bc075bcff40d8e5cc69a4", - "sha256:f85900b9cca0c67767bb61b2b9bd53208aaa7373dae633dbe25d179b4bf38aa7" + "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9", + "sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f", + "sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb", + "sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320", + "sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842", + "sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0", + "sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829", + "sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310", + "sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4", + "sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8", + "sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1" ], "markers": "python_version >= '3.4'", - "version": "==1.2.6" + "version": "==1.3.0" } } } diff --git a/db/update_etools_schema.sh b/db/update_etools_schema.sh index 7d1497f7b..7a830f219 100755 --- a/db/update_etools_schema.sh +++ b/db/update_etools_schema.sh @@ -138,7 +138,7 @@ echo "Running..." # 1 - restore from database dump if [ "$RESTORE" == "1" ]; then - echo "1.1 Dropping ans recreating database ${DATABASE_NAME}" + echo "1.1 Dropping and recreating database ${DATABASE_NAME}" dropdb -h ${PGHOST} -p ${PGPORT} --if-exists ${DATABASE_NAME} || exit 1 createdb -h ${PGHOST} -p ${PGPORT} ${DATABASE_NAME} || exit 1 diff --git a/docker/.bumpversion.cfg b/docker/.bumpversion.cfg new file mode 100644 index 000000000..794d6d68d --- /dev/null +++ b/docker/.bumpversion.cfg @@ -0,0 +1,13 @@ +[bumpversion] +current_version = 1.8 +commit = False +tag = False +allow_dirty = True +parse = (?P\d+)\.(?P.*)a(?P.*) +serialize = + {major}.{minor}a{release} + +[bumpversion:file:Makefile] + +[bumpversion:file:../src/etools_datamart/__init__.py] + diff --git a/docker/Dockerfile b/docker/Dockerfile index b99c917ca..60da2db93 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,35 +13,49 @@ RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ fi -FROM python:3.6-alpine as base - -RUN apk add --no-cache \ - bash \ - freetype \ - lcms2 \ - libjpeg-turbo \ - libpng \ - libpq \ - openjpeg \ - postgresql-libs \ - tiff \ - && apk add --no-cache --virtual .build-deps \ - freetype-dev \ - gcc \ - jpeg-dev \ - lcms2-dev \ - libffi-dev \ - linux-headers \ - musl-dev \ - openjpeg-dev \ - postgresql-dev \ - python3-dev \ - tcl-dev \ - tiff-dev \ - tk-dev \ - zlib-dev \ - && pip install pip==18.0 pipenv --upgrade \ - && adduser -S datamart +FROM python:3.6.7-slim as base +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + gcc \ + gdal-bin \ + python-dev + +#RUN apk add --no-cache \ +# --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \ +# --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \ +# bash \ +# freetype \ +# geos \ +# gdal \ +# lcms2 \ +# libjpeg-turbo \ +# libpng \ +# libpq \ +# openjpeg \ +# postgresql-libs \ +# tiff \ +# && apk add --no-cache --virtual .build-deps \ +# --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \ +# --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \ +# freetype-dev \ +# gcc \ +# gdal-dev \ +# geos-dev \ +# jpeg-dev \ +# lcms2-dev \ +# libffi-dev \ +# linux-headers \ +# musl-dev \ +# openjpeg-dev \ +# postgresql-dev \ +# python3-dev \ +# tcl-dev \ +# tiff-dev \ +# tk-dev \ +# zlib-dev \ + +RUN pip install pip==18.0 pipenv --upgrade \ + && adduser --system datamart FROM base COPY --from=builder /code /code @@ -55,7 +69,6 @@ LABEL org.label.name="eTools Datamart" \ ARG BUILD_DATE -ARG PIPENV_PYPI_MIRROR ARG PIPENV_ARGS ARG VERSION @@ -117,15 +130,15 @@ WORKDIR /code RUN set -ex \ ls -al /code \ - && pipenv install --system --deploy --ignore-pipfile $PIPENV_ARGS + && pipenv install --verbose --system --deploy --ignore-pipfile $PIPENV_ARGS RUN pip install . \ && rm -fr /code -RUN apk del .build-deps \ - && rm -rf /var/cache/apk/* \ - && rm -fr /root/.cache/ +#RUN apk del .build-deps \ +# && rm -rf /var/cache/apk/* \ +# && rm -fr /root/.cache/ WORKDIR /var/datamart diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine new file mode 100644 index 000000000..454208dc9 --- /dev/null +++ b/docker/Dockerfile.alpine @@ -0,0 +1,146 @@ +FROM pstauffer/curl:latest as builder +ARG DEVELOP +ARG GITHUB_CREDENTIALS +ARG VERSION + +RUN mkdir /code +ADD . /code + +RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ + echo "${VERSION}-develop"; \ + else \ + echo "Download package: https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" \ + && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ + fi + +FROM python:3.6-alpine as base + +RUN apk add --no-cache \ + --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \ + --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + bash \ + freetype \ + geos \ + gdal \ + lcms2 \ + libjpeg-turbo \ + libpng \ + libpq \ + openjpeg \ + postgresql-libs \ + tiff \ + && apk add --no-cache --virtual .build-deps \ + --repository http://dl-cdn.alpinelinux.org/alpine/edge/main \ + --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + freetype-dev \ + gcc \ + gdal-dev \ + geos-dev \ + jpeg-dev \ + lcms2-dev \ + libffi-dev \ + linux-headers \ + musl-dev \ + openjpeg-dev \ + postgresql-dev \ + python3-dev \ + tcl-dev \ + tiff-dev \ + tk-dev \ + zlib-dev \ + && pip install pip==18.0 pipenv --upgrade \ + && adduser -S datamart + +FROM base +COPY --from=builder /code /code + +LABEL org.label.name="eTools Datamart" \ + org.label.maintainer="sapostolico@unicef.org" \ + org.label.description="" \ + org.label.url="https://datamart.unicef.io/" \ + org.label.vcs-url="https://github.com/unicef/etools-datamart" \ + org.label.version=$VERSION + + +ARG BUILD_DATE +ARG PIPENV_ARGS +ARG VERSION + +ENV VERSION ${VERSION} + +ENV PIPENV_PYPI_MIRROR ${PIPENV_PYPI_MIRROR} +ENV PIPENV_ARGS ${PIPENV_ARGS} +ENV HOME /root/ +ENV PIPSI_HOME=/usr/local/pipsi/environments +ENV PIPSI_BIN_DIR=/usr/local/bin +ENV PYTHONUNBUFFERED 1 +ENV USE_GUNICORN 0 +ENV GUNICORN_CMD_ARGS "-b 0.0.0.0:8000 \ +--chdir /var/datamart \ +--access-logfile - \ +--access-logformat \"%(h)s %(l)s %(u)s %(t)s '%(r)s' %(s)s\" " + +ENV ALLOWED_HOSTS * +ENV CACHE_URL "redis://127.0.0.1:6379/1" +ENV CELERY_BROKER_URL "redis://127.0.0.1:6379/2" +ENV CELERY_RESULT_BACKEND "redis://127.0.0.1:6379/3" +ENV CSRF_COOKIE_SECURE True +ENV DATABASE_URL "postgres://postgres:@127.0.0.1:5432/etools_datamart" +ENV DATABASE_URL_ETOOLS "postgis://postgres:@127.0.0.1:5432/etools" +ENV DEBUG 0 +ENV DEVELOPMENT_MODE 0 +ENV DJANGO_SETTINGS_MODULE etools_datamart.config.settings +ENV MEDIA_ROOT /tmp/media +ENV SECRET_KEY "secret" +ENV SECURE_BROWSER_XSS_FILTER True +ENV SECURE_CONTENT_TYPE_NOSNIFF True +ENV SECURE_FRAME_DENY True +ENV SECURE_HSTS_INCLUDE_SUBDOMAINS True +ENV SECURE_HSTS_PRELOAD True +ENV SECURE_HSTS_SECONDS 1 +ENV SECURE_SSL_REDIRECT True +ENV SENTRY_DSN "" +ENV SESSION_COOKIE_HTTPONLY True +ENV SESSION_COOKIE_SECURE True +ENV STATIC_ROOT /tmp/static +#ENV SUPERVISOR_USER admin +#ENV SUPERVISOR_PWD "" +#ENV FLOWER_USER admin +#ENV FLOWER_PWD "" +#ENV X_FRAME_OPTIONS "DENY" +#ENV START_DATAMART "true" +#ENV START_REDIS "true" +#ENV START_CELERY "true" + +#RUN apt-get update && apt-get install -y --force-yes \ +# gcc + +RUN mkdir -p \ + /var/datamart/ \ + && chown datamart /var/datamart/ \ + && pip install pip==18.0 pipenv --upgrade + +WORKDIR /code + +RUN set -ex \ + ls -al /code \ + && pipenv install --verbose --system --deploy --ignore-pipfile $PIPENV_ARGS + +RUN pip install . \ + && rm -fr /code + + +RUN apk del .build-deps \ + && rm -rf /var/cache/apk/* \ + && rm -fr /root/.cache/ + +WORKDIR /var/datamart + +EXPOSE 8000 + +USER datamart + +ADD docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["datamart"] diff --git a/docker/Makefile b/docker/Makefile index 408ac0103..fae59589b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=0 DOCKER_PASS?= DOCKER_USER?= -TARGET?=dev +TARGET?=1.8 # below vars are used internally BUILD_OPTIONS?=--squash CMD?=datamart @@ -38,9 +38,9 @@ build: -f docker/${DOCKERFILE} . docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:dev # flatten image - docker create ${DOCKER_IMAGE_NAME}:${TARGET} foo - docker export foo | docker import - unicef/datamart:${TARGET} - docker rm foo +# docker create --name foo ${DOCKER_IMAGE_NAME}:${TARGET} +# docker export foo | docker import - unicef/datamart:${TARGET} +# docker rm foo docker images | grep ${DOCKER_IMAGE_NAME} @@ -49,13 +49,16 @@ build: --rm \ --name=${CONTAINER_NAME} \ -p 8000:8000 \ - -e CACHE_URL=redis://127.0.0.1:6379/1 \ - -e CELERY_BROKER_URL=redis://127.0.0.1:6379/2 \ - -e CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/3 \ + -e CACHE_URL=redis://192.168.66.66:6379/1 \ + -e CACHE_URL_API=redis://192.168.66.66:6379/2 \ + -e CACHE_URL_LOCK=redis://192.168.66.66:6379/3 \ + -e CACHE_URL_TEMPLATE=redis://192.168.66.66:6379/4 \ + -e CELERY_BROKER_URL=redis://192.168.66.66:6379/2 \ + -e CELERY_RESULT_BACKEND=redis://192.168.66.66:6379/3 \ -e CSRF_COOKIE_SECURE=false \ -e DATABASE_URL=${DATABASE_URL} \ -e DATABASE_URL_ETOOLS=${DATABASE_URL_ETOOLS} \ - -e DEBUG=true \ + -e DEBUG=false \ -e DJANGO_SETTINGS_MODULE=etools_datamart.config.settings \ -e SECURE_BROWSER_XSS_FILTER=0 \ -e SECURE_CONTENT_TYPE_NOSNIFF=0 \ @@ -80,10 +83,11 @@ local: $(MAKE) .run release: - docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} + @echo ${DOCKER_PASS} | docker login -u ${DOCKER_USER} --password-stdin docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:latest docker push ${DOCKER_IMAGE_NAME}:latest docker push ${DOCKER_IMAGE_NAME}:${TARGET} + bumpversion release run: $(MAKE) .run diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index adc012523..3953f6eb0 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,6 +2,7 @@ set -e mkdir -p /var/datamart/{static,log,conf,run} +mkdir -p ${STATIC_ROOT} rm -f /var/datamart/run/* diff --git a/setup.cfg b/setup.cfg index 4d37930c9..b86839397 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,13 @@ default_section = THIRDPARTY include_trailing_comma = true known_first_party = etools_datamart multi_line_output = 0 -line_length=120 +line_length = 120 balanced_wrapping = true -order_by_type=false -known_standard_library= +order_by_type = false +known_third_party = drf_querystringfilter,month_field +known_unicef = unicef_security,unicef_rest_framework +known_django = django +sections = FUTURE,STDLIB,DJANGO,THIRDPARTY,UNICEF,FIRSTPARTY,LOCALFOLDER [wheel] universal = 1 @@ -27,7 +30,7 @@ tag_svn_revision = 0 max-line-length = 160 ;exclude = .tox,migrations,.git,docs,diff_match_patch.py,deploy/**,settings,**/etools/models/__init__.py exclude = **/migrations/**, -;ignore = E501,E401,W391,E128,E261,E731 - +ignore = E501,E401,E731,W504 +;W391,E128,E261, [pycodestyle] -ignore = E501,E401,W391,E128,E261 +ignore = E501,E401,W391,E128,E261,E731 diff --git a/src/drf_querystringfilter/backend.py b/src/drf_querystringfilter/backend.py index cf5629c6e..fbf55b802 100644 --- a/src/drf_querystringfilter/backend.py +++ b/src/drf_querystringfilter/backend.py @@ -5,14 +5,15 @@ from collections import OrderedDict from functools import lru_cache -import coreapi -import coreschema from django import forms from django.core.exceptions import FieldError from django.db import models from django.db.models import BooleanField, FieldDoesNotExist from django.template import loader from django.utils.encoding import force_text + +import coreapi +import coreschema from rest_framework.filters import BaseFilterBackend from rest_framework.settings import api_settings @@ -35,6 +36,9 @@ class QueryStringFilterBackend(BaseFilterBackend): allowed_joins = -1 field_casting = {} + def __init__(self) -> None: + self.unknown_arguments = [] + def get_form_class(self, request, view): fields = OrderedDict([ (name, forms.CharField(required=False)) @@ -44,7 +48,9 @@ def get_form_class(self, request, view): (forms.Form,), fields) def get_form(self, request, view): - # if not hasattr(self, '_form'): + if hasattr(view, 'get_querystringfilter_form'): + return view.get_querystringfilter_form(request.GET, prefix=self.form_prefix) + Form = self.get_form_class(request, view) self._form = Form(request.GET, prefix=self.form_prefix) return self._form @@ -123,7 +129,7 @@ def _get_filters(self, request, queryset, view): # noqa - exclude null values: country__not=>< - only values in list: &country__id__in=176,20 - - exclude values in list: &country__id__not_in=176,20 + - exclude values in list: &country__id=176,20 """ self.opts = queryset.model._meta @@ -137,6 +143,9 @@ def _get_filters(self, request, queryset, view): # noqa for fieldname_arg in self.query_params: raw_value = self.query_params.get(fieldname_arg) + if raw_value in ["''", '""']: + raw_value = "" + negate = fieldname_arg[-1] == "!" if negate: @@ -161,11 +170,13 @@ def _get_filters(self, request, queryset, view): # noqa else: op = '' - # parts = [field_name] - processor = getattr(self, 'process_{}'.format(filter_field_name), None) if (filter_field_name not in filter_fields) and (not processor): - raise InvalidQueryArgumentError(filter_field_name) + self.unknown_arguments.append((fieldname_arg, filter_field_name)) + continue + # raise InvalidQueryArgumentError(filter_field_name) + if raw_value is None and not processor: + continue # field is configured in Serializer # so we use 'source' attribute if filter_field_name in mapping: @@ -190,6 +201,8 @@ def _get_filters(self, request, queryset, view): # noqa self.filters.update(**_f) self.exclude.update(**_e) else: + if not raw_value: + continue # field_object = opts.get_field(real_field_name) value_type = self.field_type(real_field_name) if parts: diff --git a/src/drf_querystringfilter/exceptions.py b/src/drf_querystringfilter/exceptions.py index d854eac8b..255225804 100644 --- a/src/drf_querystringfilter/exceptions.py +++ b/src/drf_querystringfilter/exceptions.py @@ -21,7 +21,7 @@ def __init__(self, field, *args, **kwargs): class InvalidQueryValueError(QueryFilterException): def __init__(self, field, argument='', *args, **kwargs): - msg = "Invalid value '{}' for parameter {}".format(field, argument) + msg = "Invalid value '{}' for parameter {}".format(argument, field) super(InvalidQueryValueError, self).__init__(msg) diff --git a/src/drf_querystringfilter/templates/querystringfilter/filter.html b/src/drf_querystringfilter/templates/querystringfilter/filter.html index 60c7086e0..6f7906050 100644 --- a/src/drf_querystringfilter/templates/querystringfilter/filter.html +++ b/src/drf_querystringfilter/templates/querystringfilter/filter.html @@ -1,8 +1,9 @@ -{% load rest_framework i18n %} -
- - {{ form.as_table }} +{% load rest_framework i18n crispy_forms_tags %} +{#

{% trans "Fields" %}

#} +{##} -
- -
+{# {{ form|crispy }}#} + {{ form|crispy }} + +{# #} +{##} diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index d8a3c921b..6097faa35 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = 'etools-datamart' -VERSION = __version__ = '1.7' +VERSION = __version__ = '1.8' __author__ = '' diff --git a/src/etools_datamart/api/endpoints/__init__.py b/src/etools_datamart/api/endpoints/__init__.py index b0965583a..4430b3d65 100644 --- a/src/etools_datamart/api/endpoints/__init__.py +++ b/src/etools_datamart/api/endpoints/__init__.py @@ -2,4 +2,5 @@ from .datamart import * # noqa from .system import * # noqa from .etools import * # noqa +from .unicef import * # noqa from .openapi import schema_view # noqa diff --git a/src/etools_datamart/api/endpoints/common.py b/src/etools_datamart/api/endpoints/common.py index 253e2e8e4..48920ffb9 100644 --- a/src/etools_datamart/api/endpoints/common.py +++ b/src/etools_datamart/api/endpoints/common.py @@ -1,86 +1,67 @@ from functools import wraps -import coreapi -import coreschema from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db import connections from django.http import Http404 + +import coreapi +import coreschema from drf_querystringfilter.exceptions import QueryFilterException -from dynamic_serializer.core import DynamicSerializerMixin +from dynamic_serializer.core import InvalidSerializerError from rest_framework.decorators import action from rest_framework.exceptions import NotAuthenticated, PermissionDenied -from rest_framework.filters import OrderingFilter from rest_framework.response import Response + +from unicef_rest_framework.ds import DynamicSerializerFilter from unicef_rest_framework.filtering import SystemFilterBackend -from unicef_rest_framework.views import ReadOnlyModelViewSet +from unicef_rest_framework.ordering import OrderingFilter +from unicef_rest_framework.views import URFReadOnlyModelViewSet +from unicef_rest_framework.views_mixins import IQYConnectionMixin -from etools_datamart.api.filtering import DatamartQueryStringFilterBackend, TenantQueryStringFilterBackend +from etools_datamart.api.filtering import CountryFilter, DatamartQueryStringFilterBackend, TenantCountryFilter from etools_datamart.apps.etl.models import EtlTask from etools_datamart.apps.multitenant.exceptions import InvalidSchema, NotAuthorizedSchema __all__ = ['APIMultiTenantReadOnlyModelViewSet'] -class SchemaSerializerField(coreschema.Enum): - - def __init__(self, view: DynamicSerializerMixin, **kwargs): - self.view = view - kwargs.setdefault('title', 'serializers') - kwargs.setdefault('description', self.build_description()) - super().__init__(list(view.serializers_fieldsets.keys()), **kwargs) - - def build_description(self): - defs = [] - names = [] - for k, v in self.view.serializers_fieldsets.items(): - names.append(k) - defs.append(f"""- **{k}**: {self.view.get_serializer_fields(k)} -""") - - description = f"""Define the set of fields to return. Allowed values are: - [{'*, *'.join(names)}*] - -{''.join(defs)} - """ - return description - - class UpdatesMixin: @action(methods=['get'], detail=False) def updates(self, request, version): """ Returns only records changed from last ETL task""" task = EtlTask.objects.get_for_model(self.queryset.model) - offset = task.last_changes.strftime('%Y-%m-%d %H:%M') - queryset = self.queryset.filter(last_modify_date__gte=offset) + if task.last_changes: + offset = task.last_changes.strftime('%Y-%m-%d %H:%M') + queryset = self.queryset.filter(last_modify_date__gte=offset) + else: + offset = 'none' + queryset = self.queryset.all() serializer = self.get_serializer(queryset, many=True) return Response(serializer.data, headers={'update-date': offset}) -class APIReadOnlyModelViewSet(ReadOnlyModelViewSet): - filter_backends = [SystemFilterBackend, +class APIReadOnlyModelViewSet(URFReadOnlyModelViewSet, IQYConnectionMixin): + filter_backends = [CountryFilter, DatamartQueryStringFilterBackend, - OrderingFilter] - filter_fields = ['country_name'] + OrderingFilter, + DynamicSerializerFilter, + ] + # filter_fields = ['country_name'] ordering_fields = ('id',) ordering = 'id' def get_schema_fields(self): ret = [] - if self.serializers_fieldsets: - ret.append(coreapi.Field( - name=self.serializer_field_param, - required=False, - location='query', - schema=SchemaSerializerField(self) - )) return ret def drf_ignore_filter(self, request, field): - return field in ['+serializer', 'cursor', '+fields', - 'ordering', 'page_size', 'format', ] + return field in [self.serializer_field_param, + self.dynamic_fields_param, + 'cursor', CountryFilter.query_param, 'month', + 'ordering', 'page_size', 'format', 'page'] def handle_exception(self, exc): conn = connections['etools'] @@ -95,6 +76,8 @@ def handle_exception(self, exc): return Response({"error": str(exc)}, status=403) elif isinstance(exc, PermissionDenied): return Response({"error": str(exc)}, status=403) + elif isinstance(exc, InvalidSerializerError): + return Response({"error": str(exc)}, status=400) elif isinstance(exc, InvalidSchema): return Response({"error": str(exc), "hint": "Removes wrong schema from selection", @@ -143,9 +126,12 @@ def _inner(self, request, *args, **kwargs): class APIMultiTenantReadOnlyModelViewSet(APIReadOnlyModelViewSet): - filter_backends = [SystemFilterBackend, - TenantQueryStringFilterBackend, - OrderingFilter] + filter_backends = [TenantCountryFilter, + SystemFilterBackend, + DatamartQueryStringFilterBackend, + OrderingFilter, + DynamicSerializerFilter, + ] ordering_fields = ('id',) ordering = 'id' diff --git a/src/etools_datamart/api/endpoints/datamart/famindicator.py b/src/etools_datamart/api/endpoints/datamart/famindicator.py index 03e9c15e0..e4d45b6eb 100644 --- a/src/etools_datamart/api/endpoints/datamart/famindicator.py +++ b/src/etools_datamart/api/endpoints/datamart/famindicator.py @@ -10,5 +10,25 @@ class FAMIndicatorViewSet(common.DataMartViewSet): serializer_class = serializers.FAMIndicatorSerializer queryset = models.FAMIndicator.objects.all() - filter_fields = ('country_name', 'last_modify_date') + filter_fields = ('last_modify_date', ) filter_backends = [MonthFilterBackend] + common.APIReadOnlyModelViewSet.filter_backends + serializers_fieldsets = {"std": None, + "spotcheck": ["country_name", "month", "spotcheck_ip_contacted", + "spotcheck_report_submitted", + "spotcheck_final_report", + "spotcheck_cancelled"], + "audit": ["country_name", "month", + "audit_ip_contacted", "audit_report_submitted", + "audit_final_report", + "audit_cancelled"], + "specialaudit": ["country_name", "month", + "specialaudit_ip_contacted", + "specialaudit_report_submitted", + "specialaudit_final_report", + "specialaudit_cancelled"], + "microassessment": ["country_name", "month", + "microassessment_ip_contacted", + "microassessment_report_submitted", + "microassessment_final_report", + "microassessment_cancelled"] + } diff --git a/src/etools_datamart/api/endpoints/datamart/hact.py b/src/etools_datamart/api/endpoints/datamart/hact.py index 2783992ec..106414159 100644 --- a/src/etools_datamart/api/endpoints/datamart/hact.py +++ b/src/etools_datamart/api/endpoints/datamart/hact.py @@ -8,4 +8,4 @@ class HACTViewSet(common.DataMartViewSet): serializer_class = serializers.HACTSerializer queryset = models.HACT.objects.all() - filter_fields = ('country_name', 'year', 'last_modify_date') + filter_fields = ('year', 'last_modify_date') diff --git a/src/etools_datamart/api/endpoints/datamart/intervention.py b/src/etools_datamart/api/endpoints/datamart/intervention.py index 6b344552b..f0fbf7826 100644 --- a/src/etools_datamart/api/endpoints/datamart/intervention.py +++ b/src/etools_datamart/api/endpoints/datamart/intervention.py @@ -2,6 +2,7 @@ # import django_filters from django_filters import rest_framework as filters +from etools_datamart.api.endpoints.datamart.serializers import InterventionSerializerFull from etools_datamart.apps.data import models from . import serializers @@ -44,7 +45,8 @@ class InterventionViewSet(common.DataMartViewSet): filter_fields = ('country_name', 'title', 'status', 'last_modify_date', 'start_date', 'submission_date', 'document_type') serializers_fieldsets = {'std': None, - 'short': ["title", "number"]} + 'full': InterventionSerializerFull, + 'short': ["title", "number", "country_name", "start_date"]} # filter_backends = [DjangoFilterBackend, OrderingFilter] # filterset_fields = ('category', 'in_stock') # filterset_class = InterventionFilter diff --git a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py index 3289b3ec4..cb27ebaf8 100644 --- a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py +++ b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py @@ -6,11 +6,16 @@ class PMPIndicatorsViewSet(common.DataMartViewSet): - """ - - """ serializer_class = serializers.PMPIndicatorsSerializer queryset = models.PMPIndicators.objects.all() - filter_fields = ('country_name', 'business_area_code', 'vendor_number', - 'partner_name', 'partner_type', 'last_modify_date', - 'pd_ssfa_ref', ) + filter_fields = ('partner_type', 'last_modify_date', 'pd_ssfa_status') + serializers_fieldsets = {"std": None, + "brief": ["id", "country_name", + "partner_name", "partner_type", "total_budget", + "unicef_budget", "currency", + ], + "ssfa": ["pd_ssfa_ref", + "pd_ssfa_status", + "pd_ssfa_start_date", + "pd_ssfa_creation_date", + "pd_ssfa_end_date"]} diff --git a/src/etools_datamart/api/endpoints/datamart/serializers.py b/src/etools_datamart/api/endpoints/datamart/serializers.py index 4861985ac..b4990a923 100644 --- a/src/etools_datamart/api/endpoints/datamart/serializers.py +++ b/src/etools_datamart/api/endpoints/datamart/serializers.py @@ -17,11 +17,17 @@ class Meta(DataMartSerializer.Meta): model = models.PMPIndicators -class InterventionSerializer(DataMartSerializer): +class InterventionSerializerFull(DataMartSerializer): class Meta(DataMartSerializer.Meta): model = models.Intervention +class InterventionSerializer(InterventionSerializerFull): + class Meta(DataMartSerializer.Meta): + model = models.Intervention + exclude = ('metadata',) + + class FAMIndicatorSerializer(DataMartSerializer): class Meta(DataMartSerializer.Meta): model = models.FAMIndicator diff --git a/src/etools_datamart/api/endpoints/datamart/user.py b/src/etools_datamart/api/endpoints/datamart/user.py index b442c66d2..d7ac0d4f1 100644 --- a/src/etools_datamart/api/endpoints/datamart/user.py +++ b/src/etools_datamart/api/endpoints/datamart/user.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from month_field.rest_framework import MonthFilterBackend + from etools_datamart.apps.data import models from . import serializers @@ -7,5 +9,7 @@ class UserStatsViewSet(common.DataMartViewSet): serializer_class = serializers.UserStatsSerializer + filter_backends = [MonthFilterBackend] + common.DataMartViewSet.filter_backends queryset = models.UserStats.objects.all() - filter_fields = ('country_name', 'month', 'last_modify_date') + filter_fields = ('last_modify_date', ) + serializers_fieldsets = {"std": None} diff --git a/src/etools_datamart/api/endpoints/etools/audit.py b/src/etools_datamart/api/endpoints/etools/audit.py index 730c19254..f6bb03772 100644 --- a/src/etools_datamart/api/endpoints/etools/audit.py +++ b/src/etools_datamart/api/endpoints/etools/audit.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- -from etools_datamart.api import serializers from etools_datamart.api.endpoints import common +from etools_datamart.api.endpoints.etools import serializers from etools_datamart.apps.etools import models class EngagementViewSet(common.APIMultiTenantReadOnlyModelViewSet): serializer_class = serializers.EngagementSerializer queryset = models.AuditEngagement.objects.all() + filter_fields = ['joint_audit', 'status', 'engagement_type', + 'cancel_comment'] diff --git a/src/etools_datamart/api/endpoints/etools/funds.py b/src/etools_datamart/api/endpoints/etools/funds.py index 3231aa863..85db9bf74 100644 --- a/src/etools_datamart/api/endpoints/etools/funds.py +++ b/src/etools_datamart/api/endpoints/etools/funds.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from etools_datamart.api import serializers from etools_datamart.api.endpoints import common +from etools_datamart.api.endpoints.etools import serializers from etools_datamart.apps.etools import models diff --git a/src/etools_datamart/api/endpoints/etools/partners.py b/src/etools_datamart/api/endpoints/etools/partners.py index 357cb7ad7..3d56b6476 100644 --- a/src/etools_datamart/api/endpoints/etools/partners.py +++ b/src/etools_datamart/api/endpoints/etools/partners.py @@ -1,5 +1,5 @@ -from etools_datamart.api import serializers from etools_datamart.api.endpoints import common +from etools_datamart.api.endpoints.etools import serializers from etools_datamart.apps.etools import models diff --git a/src/etools_datamart/api/endpoints/etools/reports.py b/src/etools_datamart/api/endpoints/etools/reports.py index 6b5f0ded0..f55ac6e2a 100644 --- a/src/etools_datamart/api/endpoints/etools/reports.py +++ b/src/etools_datamart/api/endpoints/etools/reports.py @@ -1,5 +1,5 @@ -from etools_datamart.api import serializers from etools_datamart.api.endpoints import common +from etools_datamart.api.endpoints.etools import serializers from etools_datamart.apps.etools import models diff --git a/src/etools_datamart/api/serializers/__init__.py b/src/etools_datamart/api/endpoints/etools/serializers/__init__.py similarity index 100% rename from src/etools_datamart/api/serializers/__init__.py rename to src/etools_datamart/api/endpoints/etools/serializers/__init__.py diff --git a/src/etools_datamart/api/serializers/audit.py b/src/etools_datamart/api/endpoints/etools/serializers/audit.py similarity index 60% rename from src/etools_datamart/api/serializers/audit.py rename to src/etools_datamart/api/endpoints/etools/serializers/audit.py index bb8f60cf4..46e564542 100644 --- a/src/etools_datamart/api/serializers/audit.py +++ b/src/etools_datamart/api/endpoints/etools/serializers/audit.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- -from rest_framework import serializers from etools_datamart.apps.etools import models +from .base import EToolsSerializer -class EngagementSerializer(serializers.ModelSerializer): + +class EngagementSerializer(EToolsSerializer): class Meta: model = models.AuditEngagement exclude = () diff --git a/src/etools_datamart/api/endpoints/etools/serializers/base.py b/src/etools_datamart/api/endpoints/etools/serializers/base.py new file mode 100644 index 000000000..c17aed45d --- /dev/null +++ b/src/etools_datamart/api/endpoints/etools/serializers/base.py @@ -0,0 +1,5 @@ +from rest_framework import serializers + + +class EToolsSerializer(serializers.ModelSerializer): + country_nane = serializers.ReadOnlyField(source='schema') diff --git a/src/etools_datamart/api/serializers/funds.py b/src/etools_datamart/api/endpoints/etools/serializers/funds.py similarity index 100% rename from src/etools_datamart/api/serializers/funds.py rename to src/etools_datamart/api/endpoints/etools/serializers/funds.py diff --git a/src/etools_datamart/api/serializers/partners.py b/src/etools_datamart/api/endpoints/etools/serializers/partners.py similarity index 65% rename from src/etools_datamart/api/serializers/partners.py rename to src/etools_datamart/api/endpoints/etools/serializers/partners.py index 58a376a52..7b578f2be 100644 --- a/src/etools_datamart/api/serializers/partners.py +++ b/src/etools_datamart/api/endpoints/etools/serializers/partners.py @@ -1,27 +1,28 @@ -from rest_framework import serializers from etools_datamart.apps.etools import models +from .base import EToolsSerializer -class PartnerSerializer(serializers.ModelSerializer): + +class PartnerSerializer(EToolsSerializer): class Meta: model = models.PartnersPartnerorganization exclude = () -class ReportsResultSerializer(serializers.ModelSerializer): +class ReportsResultSerializer(EToolsSerializer): class Meta: model = models.ReportsResult exclude = () -class AssessmentSerializer(serializers.ModelSerializer): +class AssessmentSerializer(EToolsSerializer): class Meta: model = models.PartnersAssessment exclude = () -class AgreementSerializer(serializers.ModelSerializer): +class AgreementSerializer(EToolsSerializer): class Meta: model = models.PartnersAgreement exclude = () @@ -33,7 +34,7 @@ class Meta: # exclude = () -class PlannedengagementSerializer(serializers.ModelSerializer): +class PlannedengagementSerializer(EToolsSerializer): class Meta: model = models.PartnersPlannedengagement exclude = () diff --git a/src/etools_datamart/api/serializers/reports.py b/src/etools_datamart/api/endpoints/etools/serializers/reports.py similarity index 58% rename from src/etools_datamart/api/serializers/reports.py rename to src/etools_datamart/api/endpoints/etools/serializers/reports.py index a3bc91bd5..a729f0bab 100644 --- a/src/etools_datamart/api/serializers/reports.py +++ b/src/etools_datamart/api/endpoints/etools/serializers/reports.py @@ -1,21 +1,22 @@ -from rest_framework import serializers from etools_datamart.apps.etools import models +from .base import EToolsSerializer -class PartnerSerializer(serializers.ModelSerializer): + +class PartnerSerializer(EToolsSerializer): class Meta: model = models.PartnersPartnerorganization exclude = () -class ReportsResultSerializer(serializers.ModelSerializer): +class ReportsResultSerializer(EToolsSerializer): class Meta: model = models.ReportsResult exclude = () -class AppliedindicatorSerializer(serializers.ModelSerializer): +class AppliedindicatorSerializer(EToolsSerializer): class Meta: model = models.ReportsAppliedindicator exclude = () diff --git a/src/etools_datamart/api/serializers/t2.py b/src/etools_datamart/api/endpoints/etools/serializers/t2.py similarity index 100% rename from src/etools_datamart/api/serializers/t2.py rename to src/etools_datamart/api/endpoints/etools/serializers/t2.py diff --git a/src/etools_datamart/api/endpoints/etools/t2.py b/src/etools_datamart/api/endpoints/etools/t2.py index 0e678e6ff..3118ba3f4 100644 --- a/src/etools_datamart/api/endpoints/etools/t2.py +++ b/src/etools_datamart/api/endpoints/etools/t2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from etools_datamart.api import serializers from etools_datamart.api.endpoints import common +from etools_datamart.api.endpoints.etools import serializers from etools_datamart.apps.etools import models diff --git a/src/etools_datamart/api/endpoints/openapi.py b/src/etools_datamart/api/endpoints/openapi.py index 04c19d5a8..86838c8cf 100644 --- a/src/etools_datamart/api/endpoints/openapi.py +++ b/src/etools_datamart/api/endpoints/openapi.py @@ -7,15 +7,12 @@ from rest_framework import permissions description = """ -# Welcome to eTools Datamart API - Each API endpoint allows filtering and/or ordering results. +Different formats can be requested using `format` argument. +Pagination can be disabled with `page_size=-1` -Fiel ## Query lookups -Any field where query functions are enabled allow to.... - ### Generic lookups - exact: Case-sensitive exact match. @@ -53,27 +50,27 @@ Select all interventions in Bolivia and Chad -- {HOST}datamart/interventions/?country_name__in=Bolivia,Chad +- {HOST}latest/datamart/interventions/?country_name__in=Bolivia,Chad Select all interventions in all countries except Bolivia and Zimbabwe -- {HOST}datamart/interventions/?country_name__in!=Bolivia,Zimbabwe (note `!=` instead of `=`) +- {HOST}latest/datamart/interventions/?country_name__in!=Bolivia,Zimbabwe (note `!=` instead of `=`) Select all interventions submitted in 2017 -- {HOST}datamart/interventions/?submission_date__year=2017 +- {HOST}latest/datamart/interventions/?submission_date__year=2017 Select all interventions submitted after 2017 (ie starting 2018) -- {HOST}datamart/interventions/?submission_date__year__gt=2017 +- {HOST}latest/datamart/interventions/?submission_date__year__gt=2017 Retrieve entries in the second quarter (April 1 to June 30): -- {HOST}datamart/interventions/?submission_date__quarter=2 +- {HOST}latest/datamart/interventions/?submission_date__quarter=2 Retrieve entries in the second/third/fourth quarter (April 1 to June 30): -- {HOST}datamart/interventions/?submission_date__quarter__gte=2 +- {HOST}latest/datamart/interventions/?submission_date__quarter__gte=2 """.format(HOST=swagger_settings.DEFAULT_API_URL) diff --git a/src/etools_datamart/api/endpoints/system/__init__.py b/src/etools_datamart/api/endpoints/system/__init__.py index f4b1a17ab..cfa7965eb 100644 --- a/src/etools_datamart/api/endpoints/system/__init__.py +++ b/src/etools_datamart/api/endpoints/system/__init__.py @@ -1,2 +1,2 @@ # -*- coding: utf-8 -*- -from .task_log import TaskLogViewSet # noqa +from .task_log import MonitorViewSet # noqa diff --git a/src/etools_datamart/api/endpoints/system/task_log.py b/src/etools_datamart/api/endpoints/system/task_log.py index 25306b949..87baac819 100644 --- a/src/etools_datamart/api/endpoints/system/task_log.py +++ b/src/etools_datamart/api/endpoints/system/task_log.py @@ -5,7 +5,7 @@ from .. import common -class TaskLogViewSet(common.APIReadOnlyModelViewSet): +class MonitorViewSet(common.APIReadOnlyModelViewSet): """ """ diff --git a/src/etools_datamart/api/endpoints/unicef/__init__.py b/src/etools_datamart/api/endpoints/unicef/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/api/endpoints/unicef/business_area.py b/src/etools_datamart/api/endpoints/unicef/business_area.py new file mode 100644 index 000000000..b2cf8ec3e --- /dev/null +++ b/src/etools_datamart/api/endpoints/unicef/business_area.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from unicef_security import models + +from . import serializers +from .. import common + + +class BusinessAreaViewSet(common.APIReadOnlyModelViewSet): + pagination_class = None + serializer_class = serializers.BusinessAreaSerializer + queryset = models.BusinessArea.objects.all() + filter_fields = ('code', 'name', 'long_name', + 'region', 'country', 'last_modify_date') diff --git a/src/etools_datamart/api/endpoints/unicef/region.py b/src/etools_datamart/api/endpoints/unicef/region.py new file mode 100644 index 000000000..f285c095c --- /dev/null +++ b/src/etools_datamart/api/endpoints/unicef/region.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from unicef_security import models + +from . import serializers +from .. import common + + +class RegionViewSet(common.APIReadOnlyModelViewSet): + pagination_class = None + serializer_class = serializers.RegionSerializer + queryset = models.Region.objects.all() + filter_fields = ('task', 'table_name', 'result', + 'last_success', 'last_failure') diff --git a/src/etools_datamart/api/endpoints/unicef/serializers.py b/src/etools_datamart/api/endpoints/unicef/serializers.py new file mode 100644 index 000000000..c1dfadb6f --- /dev/null +++ b/src/etools_datamart/api/endpoints/unicef/serializers.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +from rest_framework import serializers + +from unicef_security import models + + +class RegionSerializer(serializers.ModelSerializer): + class Meta: + model = models.Region + exclude = () diff --git a/src/etools_datamart/api/filtering.py b/src/etools_datamart/api/filtering.py index 51c9164de..9d5c71597 100644 --- a/src/etools_datamart/api/filtering.py +++ b/src/etools_datamart/api/filtering.py @@ -1,14 +1,19 @@ -from datetime import datetime - +from django.core.cache import caches from django.db import connections -from drf_querystringfilter.exceptions import InvalidQueryValueError +from django.db.models import Q +from django.template import loader + from rest_framework.exceptions import PermissionDenied +from rest_framework.filters import BaseFilterBackend + from unicef_rest_framework.filtering import CoreAPIQueryStringFilterBackend -from etools_datamart.apps.etools.utils import get_etools_allowed_schemas, validate_schemas -from etools_datamart.apps.multitenant.exceptions import NotAuthorizedSchema +from etools_datamart.apps.etools.models import UsersCountry +from etools_datamart.apps.multitenant.exceptions import InvalidSchema, NotAuthorizedSchema +from etools_datamart.apps.security.utils import conn, get_allowed_schemas # from unicef_rest_framework.state import state +cache = caches['api'] months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', @@ -16,60 +21,175 @@ 'oct', 'nov', 'dec'] -class CountryNameProcessor: - def process_country_name(self, efilters, eexclude, field, value, request, - op, param, negate, **payload): - filters = {} +def get_schema_names(query_args): + """ accept a string with a comma separated list of names,codes,bussines aread codes + and returns a set of schema names + + >>> get_schema_names("Bolivia,4020,SYR") + set('bolivia', 'sudan', 'syria') + + """ + values = sorted(set(query_args.split(","))) + key = hash(str(values)) + schemas = cache.get(key) + if not schemas: + _s = conn.schemas + conn.set_schemas([]) + schemas = [] + errors = [] + for entry in values: + try: + if entry.isdigit(): + country = UsersCountry.objects.get(business_area_code=entry) + else: + f = Q(name=entry) | Q(schema_name=entry) | Q(country_short_code=entry) + country = UsersCountry.objects.get(f) + schemas.append(country.schema_name) + # except UsersCountry.MultipleObjectsReturned: + # this is an issue only during tests + # country = UsersCountry.objects.filter(f).first() + # schemas.append(country.schema_name) + except UsersCountry.DoesNotExist: + errors.append(entry) + conn.set_schemas(_s) + if errors: + raise InvalidSchema(*errors) + cache.set(key, schemas) + return set(filter(None, schemas)) + + +class CountryFilter(BaseFilterBackend): + query_param = 'country_name' + template = 'api/country_filter.html' + + # @cached_property + # def valid_schemas(self): + # conn = connections['etools'] + # return conn.all_schemas + + def get_query(self, request): + if f"{self.query_param}!" in request.GET: + return True, ",".join(request.GET.getlist(f"{self.query_param}!", "")) + elif self.query_param in request.GET: + return False, ",".join(request.GET.getlist(self.query_param, "")) + + return "", "" + + def get_query_args(self, request): + negate, value = self.get_query(request) if not value: - if not request.user.is_superuser: - allowed = get_etools_allowed_schemas(request.user) - if not allowed: # pragma: no cover - raise PermissionDenied("You don't have enabled schemas") - filters['country_name__iregex'] = r'(' + '|'.join(allowed) + ')' + if request.user.is_superuser: + self.query_args = [] + else: + self.query_args = get_allowed_schemas(request.user) else: - value = set(value.lower().split(",")) - validate_schemas(*value) - if not request.user.is_superuser: - user_schemas = get_etools_allowed_schemas(request.user) - if not user_schemas.issuperset(value): - raise NotAuthorizedSchema(",".join(sorted(value - user_schemas))) - filters['country_name__iregex'] = r'(' + '|'.join(value) + ')' - - if negate: - return {}, filters + self.query_args = get_schema_names(value) + self.negate = negate + + def get_filters(self, request): + self.get_query_args(request) + if self.negate: + exclude = self.query_args + selection = get_allowed_schemas(request.user) - exclude else: - return filters, {} + selection = self.query_args + self.check_permission(request, selection) + return selection + def check_permission(self, request, selection): + if not request.user.is_superuser: + user_schemas = get_allowed_schemas(request.user) + if not user_schemas.issuperset(selection): + raise NotAuthorizedSchema(",".join(sorted(selection - user_schemas))) -class CountryNameProcessorTenantModel(CountryNameProcessor): - pass + def filter_queryset(self, request, queryset, view): + selection = self.get_filters(request) + if not selection: + if request.user.is_superuser: + pass + else: + raise PermissionDenied("You don't have enabled schemas") + else: + queryset = queryset.filter(schema_name__in=selection) + return queryset + def to_html(self, request, queryset, view): + self.get_query_args(request) + template = loader.get_template(self.template) + context = {'countries': sorted(conn.all_schemas), + 'selection': self.query_args, + 'header': 'aaaaaa'} + return template.render(context, request) + + +class TenantCountryFilter(CountryFilter): + def filter_queryset(self, request, queryset, view): + assert queryset.model._meta.app_label == 'etools' + conn = connections['etools'] + selection = self.get_filters(request) + if not selection: + if request.user.is_superuser: + conn.set_all_schemas() + else: + raise PermissionDenied("You don't have enabled schemas") + else: + conn.set_schemas(selection) + return queryset -class MonthProcessor: - def process_month(self, filters, exclude, field, value, **payload): - if value: - try: - if '-' in value: - m, y = value.split('-') - else: - m = value - y = datetime.now().year - - if m in months: - m = months.index(m) + 1 - elif m in list(map(str, range(12))): - m = m - elif value == 'current': - m = datetime.now().month - y = datetime.now().year - else: # pragma: no cover - raise InvalidQueryValueError('month', value) - - filters['month__month'] = int(m) - filters['month__year'] = int(y) - except ValueError: # pragma: no cover - raise InvalidQueryValueError('month', value) - return filters, exclude + +# class CountryNameProcessor: +# def ssprocess_country_name(self, efilters, eexclude, field, value, request, +# op, param, negate, **payload): +# filters = {} +# if value: +# value = process_country_value(value) +# if not request.user.is_superuser: +# user_schemas = get_etools_allowed_schemas(request.user) +# if not user_schemas.issuperset(value): +# raise NotAuthorizedSchema(",".join(sorted(value - user_schemas))) +# filters['country_name__iregex'] = r'(' + '|'.join(value) + ')' +# else: +# if not request.user.is_superuser: +# allowed = get_etools_allowed_schemas(request.user) +# if not allowed: # pragma: no cover +# raise PermissionDenied("You don't have enabled schemas") +# filters['country_name__iregex'] = r'(' + '|'.join(allowed) + ')' +# +# if negate: +# return {}, filters +# else: +# return filters, {} + + +# class CountryNameProcessorTenantModel(CountryNameProcessor): +# pass + +# +# class MonthProcessor: +# def process_month(self, filters, exclude, field, value, **payload): +# if value: +# try: +# if '-' in value: +# m, y = value.split('-') +# else: +# m = value +# y = datetime.now().year +# +# if m in months: +# m = months.index(m) + 1 +# elif m in list(map(str, range(1,13))): +# m = m +# elif value == 'current': +# m = datetime.now().month +# y = datetime.now().year +# else: # pragma: no cover +# raise InvalidQueryValueError('month', value) +# +# filters['month__month'] = int(m) +# filters['month__year'] = int(y) +# except ValueError: # pragma: no cover +# raise InvalidQueryValueError('month', value) +# return filters, exclude class SetHeaderMixin: @@ -78,42 +198,40 @@ def filter_queryset(self, request, queryset, view): ret = super().filter_queryset(request, queryset, view) request._filters = self.filters request._exclude = self.exclude - # state.set('filters', self.filters) - # state.set('excludes', self.exclude) return ret class DatamartQueryStringFilterBackend(SetHeaderMixin, CoreAPIQueryStringFilterBackend, - MonthProcessor, - CountryNameProcessor, + # MonthProcessor, + # CountryNameProcessor, ): pass - -class TenantQueryStringFilterBackend(SetHeaderMixin, - CoreAPIQueryStringFilterBackend, - MonthProcessor, - CountryNameProcessorTenantModel, - ): - def filter_queryset(self, request, queryset, view): - value = request.GET.get('country_name', None) - assert queryset.model._meta.app_label == 'etools' - conn = connections['etools'] - if not value: - if request.user.is_superuser: - conn.set_all_schemas() - else: - allowed = get_etools_allowed_schemas(request.user) - if not allowed: - raise PermissionDenied("You don't have enabled schemas") - conn.set_schemas(get_etools_allowed_schemas(request.user)) - else: - value = set(value.split(",")) - validate_schemas(*value) - if not request.user.is_superuser: - user_schemas = get_etools_allowed_schemas(request.user) - if not user_schemas.issuperset(value): - raise NotAuthorizedSchema(",".join(sorted(value - user_schemas))) - conn.set_schemas(value) - return queryset +# class TenantQueryStringFilterBackend(SetHeaderMixin, +# CoreAPIQueryStringFilterBackend, +# MonthProcessor, +# CountryNameProcessorTenantModel, +# ): +# def filter_queryset(self, request, queryset, view): +# value = request.GET.get('country_name', None) +# assert queryset.model._meta.app_label == 'etools' +# conn = connections['etools'] +# if not value: +# if request.user.is_superuser: +# conn.set_all_schemas() +# else: +# allowed = get_etools_allowed_schemas(request.user) +# if not allowed: +# raise PermissionDenied("You don't have enabled schemas") +# conn.set_schemas(get_etools_allowed_schemas(request.user)) +# else: +# # value = set(value.split(",")) +# # validate_schemas(*value) +# schemas = process_country_value(value) +# if not request.user.is_superuser: +# user_schemas = get_etools_allowed_schemas(request.user) +# if not user_schemas.issuperset(schemas): +# raise NotAuthorizedSchema(",".join(sorted(schemas - user_schemas))) +# conn.set_schemas(schemas) +# return queryset diff --git a/src/etools_datamart/api/metadata.py b/src/etools_datamart/api/metadata.py index ef1e46caa..42367a506 100644 --- a/src/etools_datamart/api/metadata.py +++ b/src/etools_datamart/api/metadata.py @@ -67,11 +67,15 @@ class SimpleMetadataWithFilters(SimpleMetadata): def determine_metadata(self, request, view): metadata = super(SimpleMetadataWithFilters, self).determine_metadata(request, view) - metadata['filters'] = getattr(view, 'filter_fields') - metadata['filter_blacklist'] = getattr(view, 'filter_blacklist') - metadata['ordering'] = getattr(view, 'ordering_fields') - metadata['serializers'] = getattr(view, 'serializers_fieldsets') - # from django.db import connection + metadata['filters'] = getattr(view, 'filter_fields', '') + metadata['filter_blacklist'] = getattr(view, 'filter_blacklist', '') + metadata['ordering'] = getattr(view, 'ordering_fields', '') + if hasattr(view, 'serializers_fieldsets'): + metadata['serializers'] = ", ".join(view.serializers_fieldsets.keys()) + else: # pragma: no cover + metadata['serializers'] = 'std' + + # from django.db import connection # with connection.schema_editor() as editor: # sql = get_create_model(editor, view.queryset.model) # metadata['sql'] = sql diff --git a/src/etools_datamart/api/renderers.py b/src/etools_datamart/api/renderers.py deleted file mode 100644 index f866af2e8..000000000 --- a/src/etools_datamart/api/renderers.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -import logging - -from unicef_rest_framework.renderers import APIBrowsableAPIRenderer as _BrowsableAPIRenderer - -logger = logging.getLogger(__name__) - - -class APIBrowsableAPIRenderer(_BrowsableAPIRenderer): - pass diff --git a/src/etools_datamart/api/templates/api/country_filter.html b/src/etools_datamart/api/templates/api/country_filter.html new file mode 100644 index 000000000..2472bc9db --- /dev/null +++ b/src/etools_datamart/api/templates/api/country_filter.html @@ -0,0 +1,11 @@ +{% load rest_framework i18n query %} +{#

{% trans "Country" %}

#} +
+ + +
+ diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index 2a2a498dc..cd533a209 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -1,4 +1,5 @@ from django.urls import include, path, re_path + from unicef_rest_framework.routers import APIReadOnlyRouter from . import endpoints @@ -32,7 +33,7 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'datamart/user-stats', endpoints.UserStatsViewSet) router.register(r'datamart/hact', endpoints.HACTViewSet) -router.register(r'system/tasks-log', endpoints.TaskLogViewSet) +router.register(r'system/monitor', endpoints.MonitorViewSet) # urlpatterns = router.urls diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index 445a83e74..adf0d3918 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -2,16 +2,19 @@ import logging from time import time -from admin_extra_urls.extras import ExtraUrlMixin, link -from adminfilters.filters import AllValuesComboFilter -from crashlog.middleware import process_exception from django.contrib import messages from django.contrib.admin import ModelAdmin, register from django.contrib.admin.templatetags.admin_urls import admin_urlname from django.contrib.admin.views.main import ChangeList from django.http import HttpResponseRedirect from django.urls import reverse + +from admin_extra_urls.extras import ExtraUrlMixin, link +from adminactions.mass_update import mass_update +from adminfilters.filters import AllValuesComboFilter +from crashlog.middleware import process_exception from humanize import naturaldelta + from unicef_rest_framework.models import Service from etools_datamart.apps.multitenant.admin import SchemaFilter @@ -28,9 +31,7 @@ class DatamartChangeList(ChangeList): class DataModelAdmin(ExtraUrlMixin, ModelAdmin): - actions = None - # load_handler = None - list_filter = (SchemaFilter,) + actions = [mass_update, ] def __init__(self, model, admin_site): import etools_datamart.apps.etl.tasks.etl as mod @@ -38,6 +39,14 @@ def __init__(self, model, admin_site): self.loaders = [v for v in mod.__dict__.values() if hasattr(v, 'apply_async')] super().__init__(model, admin_site) + def get_list_filter(self, request): + if SchemaFilter not in self.list_filter: + self.list_filter = (SchemaFilter,) + self.list_filter + + if 'last_modify_date' not in self.list_filter: + self.list_filter = self.list_filter + ('last_modify_date',) + return self.list_filter + def get_changelist(self, request, **kwargs): return DatamartChangeList @@ -78,7 +87,7 @@ def api(self, request): def queue(self, request): try: start = time() - self.model._etl_task.delay() + self.model.loader.task.delay() if settings.CELERY_TASK_ALWAYS_EAGER: # pragma: no cover stop = time() duration = stop - start @@ -86,6 +95,7 @@ def queue(self, request): else: self.message_user(request, "ETL task scheduled", messages.SUCCESS) except Exception as e: # pragma: no cover + process_exception(e) self.message_user(request, str(e), messages.ERROR) finally: return HttpResponseRedirect(reverse(admin_urlname(self.model._meta, @@ -95,7 +105,7 @@ def queue(self, request): def refresh(self, request): try: start = time() - self.model._etl_task.apply() + self.model.loader.load() stop = time() duration = stop - start self.message_user(request, "Data loaded in %s" % naturaldelta(duration), messages.SUCCESS) @@ -109,13 +119,12 @@ def refresh(self, request): @register(models.PMPIndicators) class PMPIndicatorsAdmin(DataModelAdmin, TruncateTableMixin): - list_display = ('country_name', 'partner_name', 'partner_type', 'business_area_code') - list_filter = (SchemaFilter, - ('partner_type', AllValuesComboFilter), + list_display = ('country_name', 'partner_name', 'partner_type', 'area_code') + list_filter = (('partner_type', AllValuesComboFilter), + ('pd_ssfa_status', AllValuesComboFilter), ) search_fields = ('partner_name',) date_hierarchy = 'pd_ssfa_creation_date' - # load_handler = load_pmp_indicator @register(models.Intervention) @@ -128,22 +137,19 @@ class InterventionAdmin(DataModelAdmin, TruncateTableMixin): ) search_fields = ('number', 'title') date_hierarchy = 'start_date' - # load_handler = load_intervention @register(models.FAMIndicator) class FAMIndicatorAdmin(DataModelAdmin): list_display = ('country_name', 'schema_name', 'month',) list_filter = (SchemaFilter, 'month',) - # load_handler = load_fam_indicator date_hierarchy = 'month' @register(models.UserStats) class UserStatsAdmin(DataModelAdmin): list_display = ('country_name', 'schema_name', 'month', 'total', 'unicef', 'logins', 'unicef_logins') - list_filter = (SchemaFilter, 'month') - # load_handler = load_user_report + list_filter = (SchemaFilter, 'month',) date_hierarchy = 'month' @@ -154,5 +160,15 @@ class HACTAdmin(DataModelAdmin): 'programmaticvisits_total', 'followup_spotcheck', 'completed_spotcheck', 'completed_hact_audits', 'completed_special_audits') - list_filter = (SchemaFilter, 'year') - # load_handler = load_hact + list_filter = (SchemaFilter, 'year', 'last_modify_date') + + +@register(models.GatewayType) +class GatewayTypeAdmin(DataModelAdmin): + list_display = ('country_name', 'schema_name', 'name', 'admin_level', 'source_id') + + +@register(models.Location) +class LocationAdmin(DataModelAdmin): + list_display = ('country_name', 'schema_name', 'name', 'latitude', 'longitude') + readonly_fields = ('parent', 'gateway') diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py new file mode 100644 index 000000000..f3e18b1b7 --- /dev/null +++ b/src/etools_datamart/apps/data/loader.py @@ -0,0 +1,281 @@ +import logging +from inspect import isclass + +from django.core.cache import caches +from django.db import connections, models +from django.utils import timezone + +import celery +from crashlog.middleware import process_exception +from redis.exceptions import LockError +from strategy_field.utils import fqn, get_attr + +from etools_datamart.celery import app + +loadeables = set() +locks = caches['lock'] + +logger = logging.getLogger(__name__) + +CREATED = 'created' +UPDATED = 'updated' +UNCHANGED = 'unchanged' + + +class EtlResult: + __slots__ = [CREATED, UPDATED, UNCHANGED] + + def __init__(self, updated=0, created=0, unchanged=0, **kwargs): + self.created = created + self.updated = updated + self.unchanged = unchanged + + def __repr__(self): + return repr(self.as_dict()) + + def incr(self, counter): + setattr(self, counter, getattr(self, counter) + 1) + + def as_dict(self): + return {'created': self.created, + 'updated': self.updated, + 'unchanged': self.unchanged} + + def __eq__(self, other): + if isinstance(other, EtlResult): + other = other.as_dict() + + if isinstance(other, dict): + return (self.created == other['created'] and + self.updated == other['updated'] and + self.unchanged == other['unchanged']) + return False + + +def is_record_changed(record, values): + other = type(record)(**values) + for field_name, field_value in values.items(): + if getattr(record, field_name) != getattr(other, field_name): + return True + return False + + +DEFAULT_KEY = lambda country, record: dict(country_name=country.name, + schema_name=country.schema_name, + source_id=record.pk) + + +class LoaderOptions: + __attrs__ = ['mapping', 'celery', 'source', + 'queryset', 'key', 'locks', + 'depends', 'timeout', 'lock_key'] + + def __init__(self, base=None): + self.mapping = {} + self.celery = app + self.queryset = None + self.source = None + self.lock_key = None + self.key = DEFAULT_KEY + self.timeout = None + self.depends = () + if base: + for attr in self.__attrs__: + if hasattr(base, attr): + if isinstance(getattr(self, attr), (list, tuple)): + n = getattr(self, attr) + getattr(base, attr) + setattr(self, attr, n) + else: + setattr(self, attr, getattr(base, attr, getattr(self, attr))) + + if not self.queryset and self.source: + self.queryset = lambda: self.source.objects + + def contribute_to_class(self, model, name): + self.model = model + setattr(model, name, self) + if not self.lock_key: + self.lock_key = f"{fqn(model)}-lock" + + +class LoaderTask(celery.Task): + + def __init__(self, loader) -> None: + self.loader = loader + self.linked_model = loader.model + self.name = "load_{0.app_label}_{0.model_name}".format(loader.model._meta) + + def run(self, *args, **kwargs): + return self.loader.load() + + +# def get_or_fail(Model, **kwargs): +# try: +# Model.objects.get(**kwargs) +# except Model.DoesNotExist: +# raise Model.DoesNotExist(f"Unable to get {Model.__name__} using {kwargs}") + + +class Loader: + # __slots__ = ['model', 'config', 'mapping', 'task', 'tree_parents', 'always_update'] + + def __init__(self) -> None: + self.config = None + self.tree_parents = [] + self.always_update = False + + def __repr__(self): + return "<%sLoader>" % self.model._meta.object_name + + def contribute_to_class(self, model, name): + self.model = model + self.config = model._etl_config + del model._etl_config + if not model._meta.abstract: + loadeables.add("{0._meta.app_label}.{0._meta.model_name}".format(model)) + if self.config.celery: + self.task = LoaderTask(self) + self.config.celery.tasks.register(self.task) + + setattr(model, name, self) + + # def deconstruct(self): + # return [] + # + # def check(self, **kwargs): + # return [] + + def process(self, filters, values): + try: + existing, created = self.model.objects.get_or_create(**filters, + defaults=values) + if created: + op = CREATED + else: + if self.always_update or is_record_changed(existing, values): + op = UPDATED + self.model.objects.update_or_create(**filters, + defaults=values) + else: + op = UNCHANGED + return op + except Exception as e: # pragma: no cover + logger.exception(e) + process_exception(e) + raise Exception(f"Error in {self}: {e}") from e + + def get_values(self, country, record): + ret = {} + for k, v in self.mapping.items(): + if v == '__self__': + try: + ret[k] = self.model.objects.get(schema_name=country.schema_name, + source_id=getattr(record, k).id) + except AttributeError: + ret[k] = None + except self.model.DoesNotExist: + ret[k] = None + self.tree_parents.append((record.id, getattr(record, k).id)) + + elif isclass(v) and issubclass(v, models.Model): + try: + ret[k] = v.objects.get(schema_name=country.schema_name, + source_id=getattr(record, k).id) + except AttributeError: + pass + elif callable(v): + ret[k] = v(country, record) + else: + ret[k] = get_attr(record, v) + return ret + + def process_post_country(self, country, context): + for mart, etools in self.tree_parents: + kk = self.model.objects.get(schema_name=country.schema_name, + source_id=mart) + kk.parent = self.model.objects.get(schema_name=country.schema_name, + source_id=etools) + kk.save() + self.tree_parents = [] + + def process_country(self, results: EtlResult, country, context) -> EtlResult: + qs = self.config.queryset() + stdout = context['stdout'] + max_records = context['max_records'] + for record in qs.all(): + filters = self.config.key(country, record) + values = self.get_values(country, record) + op = self.process(filters, values) + results.incr(op) + context['records'] += 1 + if stdout: # pragma: no cover + stdout.write('.') + stdout.flush() + if max_records and context['records'] >= max_records: + break + return results + + def get_context(self, **kwargs): + context = {} + context.update(kwargs) + return context + + @property + def is_locked(self): + return self.config.lock_key in locks + + def unlock(self): + try: + lock = locks.lock(self.config.lock_key, timeout=self.config.timeout) + locks.delete(self.config.lock_key) + lock.release() + except LockError: + pass + + def load(self, verbosity=0, always_update=False, stdout=None, + ignore_dependencies=False, max_records=None, countries=None): + have_lock = False + results = EtlResult() + lock = locks.lock(self.config.lock_key, timeout=self.config.timeout) + try: + have_lock = lock.acquire(blocking=False) + if have_lock: # pragma: no branch + if not ignore_dependencies: + for dependency in self.config.depends: + dependency.loader.load(stdout=stdout) + self.always_update = always_update + connection = connections['etools'] + if not countries: # pragma: no branch + countries = connection.get_tenants() + if self.config.mapping: + self.mapping = {} + mart_fields = self.model._meta.concrete_fields + for field in mart_fields: + if field.name not in ['country_name', 'schema_name', 'area_code', + 'id', 'last_modify_date']: + self.mapping[field.name] = field.name + self.mapping.update(self.config.mapping) + + context = self.get_context(today=timezone.now(), + countries=countries, + max_records=max_records, + records=0, + stdout=stdout) + + for country in countries: + if stdout: # pragma: no cover + stdout.write(f"{country}\n") + connection.set_schemas([country.schema_name]) + self.process_country(results, country, context) + self.process_post_country(country, context) + if max_records and context['records'] >= max_records: + break + if stdout: # pragma: no cover + stdout.write("\n") + finally: + if have_lock: # pragma: no branch + try: + lock.release() + except LockError as e: # pragma: no cover + logger.warning(e) + return results diff --git a/src/etools_datamart/apps/data/management/__init__.py b/src/etools_datamart/apps/data/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/data/management/commands/__init__.py b/src/etools_datamart/apps/data/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/data/management/commands/load.py b/src/etools_datamart/apps/data/management/commands/load.py new file mode 100644 index 000000000..559b47618 --- /dev/null +++ b/src/etools_datamart/apps/data/management/commands/load.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +import logging +import sys + +from django.apps import apps +from django.core.management import BaseCommand + +from etools_datamart.apps.data.loader import loadeables + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + args = '' + help = '' + requires_system_checks = False + requires_migrations_checks = False + output_transaction = False # Whether to wrap the output in a "BEGIN; COMMIT;" + + def add_arguments(self, parser): + parser.add_argument('args', + metavar='models', nargs='*', help='One or more application label.') + parser.add_argument( + '--all', action='store_true', + help="Run all loaders.", + ) + parser.add_argument( + '--ignore-changes', action='store_true', + help="Run all loaders.", + ) + parser.add_argument( + '--unlock', action='store_true', + help="Unlock all loaders.", + ) + + def notify(self, model, created, name, tpl=" {op} {model} `{name}`"): + if self.verbosity > 2: + op = {True: "Created", False: "Updated"}[created] + self.stdout.write(tpl.format(op=op, model=model, name=name)) + elif self.verbosity > 1: + self.stdout.write('.', ending='') + + def handle(self, *model_names, **options): + self.verbosity = options['verbosity'] + _all = options['all'] + unlock = options['unlock'] + if _all: + model_names = sorted(list(loadeables)) + + if not model_names: + for model_name in sorted(list(loadeables)): + self.stdout.write(model_name) + else: + for model_name in model_names: + model = apps.get_model(model_name) + if unlock: + model.loader.load.unlock() + res = model.loader.load(always_update=options['ignore_changes'], + stdout=sys.stdout) + self.stdout.write(f"{model_name:20}: " + f" created: {res.created:<3}" + f" updated: {res.updated:<3}" + f" unchanged: {res.unchanged:<3}\n" + ) diff --git a/src/etools_datamart/apps/data/migrations/0001_initial.py b/src/etools_datamart/apps/data/migrations/0001_initial.py index 9bbc51de0..ecb6b6f2e 100644 --- a/src/etools_datamart/apps/data/migrations/0001_initial.py +++ b/src/etools_datamart/apps/data/migrations/0001_initial.py @@ -1,9 +1,12 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 +import django.contrib.gis.db.models.fields import django.contrib.postgres.fields.jsonb -import month_field.models +import django.db.models.deletion from django.db import migrations, models +import month_field.models + class Migration(migrations.Migration): @@ -17,9 +20,10 @@ class Migration(migrations.Migration): name='FAMIndicator', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('country_name', models.CharField(db_index=True, max_length=50)), - ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('country_name', models.CharField(db_index=True, max_length=100)), + ('schema_name', models.CharField(db_index=True, max_length=63)), + ('area_code', models.CharField(db_index=True, max_length=10)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('spotcheck_ip_contacted', models.IntegerField(default=0, verbose_name='Spot Check-IP Contacted')), ('spotcheck_report_submitted', models.IntegerField(default=0, verbose_name='Spot Check-Report Submitted')), @@ -43,13 +47,27 @@ class Migration(migrations.Migration): 'ordering': ('month', 'country_name'), }, ), + migrations.CreateModel( + name='GatewayType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('country_name', models.CharField(db_index=True, max_length=100)), + ('schema_name', models.CharField(db_index=True, max_length=63)), + ('area_code', models.CharField(db_index=True, max_length=10)), + ('last_modify_date', models.DateTimeField(auto_now=True)), + ('name', models.CharField(db_index=True, max_length=64)), + ('admin_level', models.SmallIntegerField(blank=True, null=True)), + ('source_id', models.IntegerField(blank=True, null=True)), + ], + ), migrations.CreateModel( name='HACT', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('country_name', models.CharField(db_index=True, max_length=50)), - ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('country_name', models.CharField(db_index=True, max_length=100)), + ('schema_name', models.CharField(db_index=True, max_length=63)), + ('area_code', models.CharField(db_index=True, max_length=10)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('year', models.IntegerField()), ('microassessments_total', models.IntegerField(default=0, help_text='Total number of completed Microassessments in the business area in the past year')), ('programmaticvisits_total', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), @@ -67,9 +85,10 @@ class Migration(migrations.Migration): name='Intervention', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('country_name', models.CharField(db_index=True, max_length=50)), - ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('country_name', models.CharField(db_index=True, max_length=100)), + ('schema_name', models.CharField(db_index=True, max_length=63)), + ('area_code', models.CharField(db_index=True, max_length=10)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('created', models.DateTimeField(auto_now=True)), ('updated', models.DateTimeField(null=True)), ('document_type', models.CharField(max_length=255, null=True)), @@ -116,15 +135,44 @@ class Migration(migrations.Migration): 'ordering': ('country_name', 'title'), }, ), + migrations.CreateModel( + name='Location', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('country_name', models.CharField(db_index=True, max_length=100)), + ('schema_name', models.CharField(db_index=True, max_length=63)), + ('area_code', models.CharField(db_index=True, max_length=10)), + ('last_modify_date', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=254)), + ('latitude', models.FloatField(blank=True, null=True)), + ('longitude', models.FloatField(blank=True, null=True)), + ('p_code', models.CharField(max_length=32)), + ('point', django.contrib.gis.db.models.fields.PointField(blank=True, null=True, srid=4326)), + ('geom', django.contrib.gis.db.models.fields.MultiPolygonField(blank=True, null=True, srid=4326)), + ('level', models.IntegerField()), + ('lft', models.IntegerField()), + ('rght', models.IntegerField()), + ('tree_id', models.IntegerField()), + ('created', models.DateTimeField()), + ('modified', models.DateTimeField()), + ('is_active', models.BooleanField()), + ('source_id', models.IntegerField(blank=True, null=True)), + ('gateway', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='data.GatewayType')), + ('parent', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='data.Location')), + ], + options={ + 'abstract': False, + }, + ), migrations.CreateModel( name='PMPIndicators', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('country_name', models.CharField(db_index=True, max_length=50)), - ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('country_name', models.CharField(db_index=True, max_length=100)), + ('schema_name', models.CharField(db_index=True, max_length=63)), + ('area_code', models.CharField(db_index=True, max_length=10)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('vendor_number', models.CharField(db_index=True, max_length=255, null=True)), - ('business_area_code', models.CharField(db_index=True, max_length=100, null=True)), ('partner_name', models.CharField(db_index=True, max_length=255, null=True)), ('partner_type', models.CharField(db_index=True, max_length=255, null=True)), ('pd_ssfa_ref', models.CharField(max_length=255, null=True)), @@ -162,9 +210,10 @@ class Migration(migrations.Migration): name='UserStats', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('country_name', models.CharField(db_index=True, max_length=50)), - ('schema_name', models.CharField(db_index=True, max_length=50)), - ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('country_name', models.CharField(db_index=True, max_length=100)), + ('schema_name', models.CharField(db_index=True, max_length=63)), + ('area_code', models.CharField(db_index=True, max_length=10)), + ('last_modify_date', models.DateTimeField(auto_now=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('total', models.IntegerField(default=0, verbose_name='Total users')), ('unicef', models.IntegerField(default=0, verbose_name='UNICEF uswers')), @@ -184,6 +233,10 @@ class Migration(migrations.Migration): name='hact', unique_together={('year', 'country_name')}, ), + migrations.AlterUniqueTogether( + name='gatewaytype', + unique_together={('schema_name', 'name'), ('schema_name', 'admin_level')}, + ), migrations.AlterUniqueTogether( name='famindicator', unique_together={('month', 'country_name')}, diff --git a/src/etools_datamart/apps/data/models/__init__.py b/src/etools_datamart/apps/data/models/__init__.py index 1c4c1109f..727ca8012 100644 --- a/src/etools_datamart/apps/data/models/__init__.py +++ b/src/etools_datamart/apps/data/models/__init__.py @@ -3,3 +3,4 @@ from .fam import FAMIndicator # noqa from .user import UserStats # noqa from .hact import HACT # noqa +from .location import GatewayType, Location # noqa diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index 3891c199b..d162be924 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -1,9 +1,13 @@ -from celery.local import class_property from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models import QuerySet +from django.db.models.base import ModelBase from django.db.models.manager import BaseManager +from celery.local import class_property + +from etools_datamart.apps.data.loader import Loader, LoaderOptions + class DataMartQuerySet(QuerySet): @@ -14,14 +18,45 @@ def filter_schemas(self, *schemas): class DataMartManager(BaseManager.from_queryset(DataMartQuerySet)): - pass - # def truncate(self): - # self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) + + def truncate(self): + self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) + + +class DataMartModelBase(ModelBase): + def __new__(cls, name, bases, attrs, **kwargs): + super_new = super().__new__ + parents = [b for b in bases if isinstance(b, DataMartModelBase)] + if not parents: + return super_new(cls, name, bases, attrs) + loader = attrs.pop('loader', None) + config = attrs.pop('Options', None) + + new_class = super_new(cls, name, bases, attrs, **kwargs) + if not loader: # no custom loader use default + loader = Loader() + base_config = getattr(new_class, '_etl_config', None) + + if not config: + config = LoaderOptions(base_config) + else: + config = LoaderOptions(config) + + new_class.add_to_class('_etl_config', config) + new_class.add_to_class('loader', loader) + # + # attr_meta = attrs.get('Meta', None) + # attr_loader = attrs.get('Loader', None) + # loader = attr_meta or getattr(new_class, 'Meta', None) + # base_meta = getattr(new_class, '_meta', None) + + return new_class -class DataMartModel(models.Model): - country_name = models.CharField(max_length=50, db_index=True) - schema_name = models.CharField(max_length=50, db_index=True) +class DataMartModel(models.Model, metaclass=DataMartModelBase): + country_name = models.CharField(max_length=100, db_index=True) + schema_name = models.CharField(max_length=63, db_index=True) + area_code = models.CharField(max_length=10, db_index=True) last_modify_date = models.DateTimeField(blank=True, auto_now=True) class Meta: diff --git a/src/etools_datamart/apps/data/models/fam.py b/src/etools_datamart/apps/data/models/fam.py index c4445a04b..1143a9b6f 100644 --- a/src/etools_datamart/apps/data/models/fam.py +++ b/src/etools_datamart/apps/data/models/fam.py @@ -1,7 +1,38 @@ from django.db import models + from month_field.models import MonthField +from etools_datamart.apps.data.loader import EtlResult, Loader from etools_datamart.apps.data.models.base import DataMartModel +from etools_datamart.apps.etools.models import (AuditAudit, AuditEngagement, AuditMicroassessment, + AuditSpecialaudit, AuditSpotcheck,) + + +class FAMIndicatorLoader(Loader): + + def process_country(self, results: EtlResult, country, context): + engagements = (AuditSpotcheck, AuditAudit, AuditSpecialaudit, AuditMicroassessment) + start_date = context['today'].date() + for model in engagements: + realname = "_".join(model._meta.db_table.split('_')[1:]) + values = {} + for status, status_display in AuditEngagement.STATUSES: + filter_dict = { + 'engagement_ptr__status': status, + 'engagement_ptr__start_date__month': start_date.month, + 'engagement_ptr__start_date__year': start_date.year, + } + field_name = f"{realname}_{status_display}".replace(" ", "_").lower() + value = model.objects.filter(**filter_dict).count() + values[field_name] = value + op = self.process(filters=dict(month=start_date, + country_name=country.name, + area_code=country.business_area_code, + schema_name=country.schema_name), + values=values) + results.incr(op) + + return results class FAMIndicator(DataMartModel): @@ -28,3 +59,5 @@ class Meta: ordering = ('month', 'country_name') unique_together = ('month', 'country_name') verbose_name = "FAM Indicator" + + loader = FAMIndicatorLoader() diff --git a/src/etools_datamart/apps/data/models/hact.py b/src/etools_datamart/apps/data/models/hact.py index b0615b667..c5dde2627 100644 --- a/src/etools_datamart/apps/data/models/hact.py +++ b/src/etools_datamart/apps/data/models/hact.py @@ -1,6 +1,35 @@ +import json + from django.db import models +from django.utils import timezone +from etools_datamart.apps.data.loader import EtlResult, Loader from etools_datamart.apps.data.models.base import DataMartModel +from etools_datamart.apps.etools.models import HactAggregatehact + + +class HACTLoader(Loader): + def process_country(self, results: EtlResult, country, context): + today = timezone.now() + aggregate = HactAggregatehact.objects.get(year=today.year) + data = json.loads(aggregate.partner_values) + + # # Total number of completed Microassessments in the business area in the past year + values = dict(microassessments_total=data['assurance_activities']['micro_assessment'], + programmaticvisits_total=data['assurance_activities']['programmatic_visits']['completed'], + followup_spotcheck=data['assurance_activities']['spot_checks']['follow_up'], + completed_spotcheck=data['assurance_activities']['spot_checks']['completed'], + completed_hact_audits=data['assurance_activities']['scheduled_audit'], + completed_special_audits=data['assurance_activities']['special_audit'], + ) + op = self.process(filters=dict(year=today.year, + area_code=country.business_area_code, + country_name=country.name, + schema_name=country.schema_name), + values=values) + results.incr(op) + + return results class HACT(DataMartModel): @@ -18,6 +47,8 @@ class HACT(DataMartModel): completed_special_audits = models.IntegerField(default=0, help_text="Total number of completed special audits for the workspace. ") + loader = HACTLoader() + class Meta: ordering = ('year', 'country_name') unique_together = ('year', 'country_name') diff --git a/src/etools_datamart/apps/data/models/intervention.py b/src/etools_datamart/apps/data/models/intervention.py index ccd9bcb3d..bed13f50c 100644 --- a/src/etools_datamart/apps/data/models/intervention.py +++ b/src/etools_datamart/apps/data/models/intervention.py @@ -5,6 +5,7 @@ from django.db import models from etools_datamart.apps.data.models.base import DataMartModel +from etools_datamart.apps.etools.models import PartnersIntervention logger = logging.getLogger(__name__) @@ -59,3 +60,37 @@ class Intervention(DataMartModel): class Meta: ordering = ('country_name', 'title') verbose_name = "Intervention" + + class Options: + source = PartnersIntervention + queryset = lambda: PartnersIntervention.objects.select_related('agreement', + 'partner_authorized_officer_signatory', + 'unicef_signatory', + 'country_programme', + ) + key = lambda country, record: dict(country_name=country.name, + schema_name=country.schema_name, + area_code=country.business_area_code, + intervention_id=record.pk) + mapping = dict(start_date='start', + end_date='end', + partner_name='agreement.partner.name', + partner_authorized_officer_signatory_id='partner_authorized_officer_signatory.pk', + country_programme_id='country_programme.pk', + intervention_id='pk', + unicef_signatory_id='unicef_signatory.pk', + unicef_signatory_first_name='unicef_signatory.first_name', + unicef_signatory_last_name='unicef_signatory.last_name', + unicef_signatory_email='unicef_signatory.email', + + partner_signatory_title='partner_authorized_officer_signatory.title', + partner_signatory_first_name='partner_authorized_officer_signatory.first_name', + partner_signatory_last_name='partner_authorized_officer_signatory.last_name', + partner_signatory_email='partner_authorized_officer_signatory.email', + partner_signatory_phone='partner_authorized_officer_signatory.phone', + partner_focal_point_title='partner_focal_point.title', + partner_focal_point_first_name='partner_focal_point.first_name', + partner_focal_point_last_name='partner_focal_point.last_name', + partner_focal_point_email='partner_focal_point.email', + partner_focal_point_phone='partner_focal_point.phone', + updated='modified') diff --git a/src/etools_datamart/apps/data/models/location.py b/src/etools_datamart/apps/data/models/location.py new file mode 100644 index 000000000..a560e6400 --- /dev/null +++ b/src/etools_datamart/apps/data/models/location.py @@ -0,0 +1,51 @@ +from django.contrib.gis.db.models import MultiPolygonField, PointField +from django.db import models + +from etools_datamart.apps.data.models.base import DataMartModel +from etools_datamart.apps.etools.models import LocationsGatewaytype, LocationsLocation + + +class GatewayType(DataMartModel): + name = models.CharField(db_index=True, max_length=64) + admin_level = models.SmallIntegerField(blank=True, null=True) + source_id = models.IntegerField(blank=True, null=True) + + class Meta: + unique_together = ('schema_name', 'name'), ('schema_name', 'admin_level') + + class Options: + source = LocationsGatewaytype + mapping = {'source_id': 'id', + 'area_code': lambda country, record: country.business_area_code, + } + + +class Location(DataMartModel): + name = models.CharField(max_length=254) + latitude = models.FloatField(blank=True, null=True) + longitude = models.FloatField(blank=True, null=True) + p_code = models.CharField(max_length=32) + point = PointField(blank=True, null=True) + gateway = models.ForeignKey(GatewayType, models.DO_NOTHING) + geom = MultiPolygonField(blank=True, null=True) + level = models.IntegerField() + lft = models.IntegerField() + parent = models.ForeignKey('self', models.DO_NOTHING, blank=True, null=True) + rght = models.IntegerField() + tree_id = models.IntegerField() + created = models.DateTimeField() + modified = models.DateTimeField() + is_active = models.BooleanField() + + source_id = models.IntegerField(blank=True, null=True) + + class Options: + depends = (GatewayType,) + # source = LocationsLocation + queryset = lambda: LocationsLocation.objects.order_by('-parent') + + mapping = {'source_id': 'id', + 'area_code': lambda country, record: country.business_area_code, + 'parent': '__self__', + 'gateway': GatewayType + } diff --git a/src/etools_datamart/apps/data/models/pmp.py b/src/etools_datamart/apps/data/models/pmp.py index fa6783a36..0e78beb7e 100644 --- a/src/etools_datamart/apps/data/models/pmp.py +++ b/src/etools_datamart/apps/data/models/pmp.py @@ -2,15 +2,67 @@ import logging from django.db import models +from django.db.models import Sum +from django.db.models.functions import Coalesce +from etools_datamart.apps.data.loader import Loader from etools_datamart.apps.data.models.base import DataMartModel +from etools_datamart.apps.etools.models import PartnersIntervention, PartnersPartnerorganization logger = logging.getLogger(__name__) +class PMPIndicatorLoader(Loader): + def process_country(self, results, country, context): + for partner in PartnersPartnerorganization.objects.all(): + for intervention in PartnersIntervention.objects.filter(agreement__partner=partner): + planned_budget = getattr(intervention, + 'partnersintervention_partners_interventionbudget_intervention_id', None) + fr_currencies = intervention.frs.all().values_list('currency', flat=True).distinct() + has_assessment = bool(getattr(partner.current_core_value_assessment, 'assessment', False)) + values = {'country_name': country.name, + 'schema_name': country.schema_name, + 'area_code': country.business_area_code, + 'partner_name': partner.name, + 'partner_type': partner.cso_type, + 'vendor_number': partner.vendor_number, + + 'pd_ssfa_ref': intervention.number.replace(',', '-'), + 'pd_ssfa_status': intervention.status.title(), + 'pd_ssfa_start_date': intervention.start, + 'pd_ssfa_creation_date': intervention.created, + 'pd_ssfa_end_date': intervention.end, + + 'cash_contribution': intervention.total_unicef_cash or 0, + 'supply_contribution': intervention.total_in_kind_amount or 0, + 'total_budget': intervention.total_budget or 0, + 'unicef_budget': intervention.total_unicef_budget or 0, + + 'currency': intervention.planned_budget.currency if planned_budget else '-', + 'partner_contribution': intervention.planned_budget.partner_contribution if planned_budget else '-', + 'unicef_cash': intervention.planned_budget.unicef_cash if planned_budget else '-', + 'in_kind_amount': intervention.planned_budget.in_kind_amount if planned_budget else '-', + 'total': intervention.planned_budget.total if planned_budget else '-', + 'fr_numbers_against_pd_ssfa': ' - '.join([fh.fr_number for fh in intervention.frs.all()]), + 'fr_currencies': ', '.join(fr for fr in fr_currencies), + 'sum_of_all_fr_planned_amount': intervention.frs.aggregate( + total=Coalesce(Sum('intervention_amt'), 0))[ + 'total'] if fr_currencies.count() <= 1 else '-', + 'core_value_attached': has_assessment, + # 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), + # 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), + } + op = self.process(filters=dict(country_name=country.name, + schema_name=country.schema_name, + partner_id=partner.pk, + intervention_id=intervention.pk), + values=values) + results.incr(op) + return results + + class PMPIndicators(DataMartModel): vendor_number = models.CharField(max_length=255, null=True, db_index=True) - business_area_code = models.CharField(max_length=100, null=True, db_index=True) partner_name = models.CharField(max_length=255, null=True, db_index=True) partner_type = models.CharField(max_length=255, null=True, db_index=True) @@ -49,3 +101,9 @@ class PMPIndicators(DataMartModel): class Meta: ordering = ('country_name', 'partner_name') verbose_name = "PMP Indicator" + + loader = PMPIndicatorLoader() + + class Options: + source = PartnersPartnerorganization + mapping = None diff --git a/src/etools_datamart/apps/data/models/user.py b/src/etools_datamart/apps/data/models/user.py index c9a239918..6fcf7fb18 100644 --- a/src/etools_datamart/apps/data/models/user.py +++ b/src/etools_datamart/apps/data/models/user.py @@ -1,7 +1,39 @@ +from datetime import datetime + from django.db import models + from month_field.models import MonthField +from etools_datamart.apps.data.loader import EtlResult, Loader from etools_datamart.apps.data.models.base import DataMartModel +from etools_datamart.apps.etools.models import AuthUser + + +class UserStatsLoader(Loader): + def get_context(self, **kwargs): + today = kwargs['today'] + context = {'first_of_month': datetime(today.year, today.month, 1)} + context.update(kwargs) + return context + + def process_country(self, results: EtlResult, country, context): + first_of_month = context['first_of_month'] + base = AuthUser.objects.filter(profile__country=country) + values = { + 'total': base.count(), + 'unicef': base.filter(email__endswith='@unicef.org').count(), + 'logins': base.filter( + last_login__month=first_of_month.month).count(), + 'unicef_logins': base.filter( + last_login__month=first_of_month.month, + email__endswith='@unicef.org').count(), + } + op = self.process(filters=dict(month=first_of_month, + country_name=country.name, + schema_name=country.schema_name, ), + values=values) + results.incr(op) + return results class UserStats(DataMartModel): @@ -15,3 +47,5 @@ class Meta: ordering = ('-month', 'country_name') unique_together = ('country_name', 'month') verbose_name = "User Access Statistics" + + loader = UserStatsLoader() diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index 2130f383d..0dfeb5602 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -1,33 +1,30 @@ # -*- coding: utf-8 -*- -from admin_extra_urls.extras import action, link -from admin_extra_urls.mixins import _confirm_action from django.contrib import admin, messages from django.contrib.admin import register from django.http import HttpResponseRedirect from django.urls import reverse from django.utils.html import format_html + +from admin_extra_urls.extras import action, ExtraUrlMixin, link +from admin_extra_urls.mixins import _confirm_action +from adminactions.mass_update import mass_update +from crashlog.middleware import process_exception from django_celery_beat.models import PeriodicTask from humanize import naturaldelta -from etools_datamart.apps.etl.lock import cache from etools_datamart.celery import app -from etools_datamart.libs.truncate import TruncateTableMixin from . import models @register(models.EtlTask) -class EtlTaskAdmin(TruncateTableMixin, admin.ModelAdmin): +class EtlTaskAdmin(ExtraUrlMixin, admin.ModelAdmin): list_display = ('task', 'last_run', 'status', 'time', - 'last_success', 'last_failure', 'lock', 'scheduling', 'queue_task') + 'last_success', 'last_failure', 'locked', + 'data', 'scheduling', 'unlock_task', 'queue_task') - readonly_fields = ('task', 'last_run', - 'last_success', 'last_failure', 'last_changes', - 'results', 'elapsed', 'time', 'status', - 'table_name', 'content_type', - ) date_hierarchy = 'last_run' - actions = None + actions = [mass_update, ] def scheduling(self, obj): opts = PeriodicTask._meta @@ -43,6 +40,14 @@ def scheduling(self, obj): return format_html(f'{label}') + def data(self, obj): + model = obj.content_type.model_class() + opts = model._meta + url = reverse('admin:%s_%s_changelist' % (opts.app_label, opts.model_name)) + return format_html(f'data') + + data.verbse_name = 'data' + def queue_task(self, obj): opts = self.model._meta url = reverse('admin:%s_%s_queue' % (opts.app_label, @@ -51,6 +56,14 @@ def queue_task(self, obj): queue_task.verbse_name = 'queue' + def unlock_task(self, obj): + opts = self.model._meta + url = reverse('admin:%s_%s_unlock' % (opts.app_label, + opts.model_name), args=[obj.id]) + return format_html(f'unlock') + + queue_task.verbse_name = 'unlock' + def has_add_permission(self, request): return False @@ -60,10 +73,10 @@ def has_delete_permission(self, request, obj=None): def time(self, obj): return naturaldelta(obj.elapsed) - def lock(self, obj): - return f"{obj.task}-lock" in cache + def locked(self, obj): + return obj.content_type.model_class().loader.is_locked - lock.boolean = True + locked.boolean = True def changeform_view(self, request, object_id=None, form_url='', extra_context=None): if request.method == 'POST': @@ -76,22 +89,25 @@ def changeform_view(self, request, object_id=None, form_url='', extra_context=No def queue(self, request, pk): obj = self.get_object(request, pk) try: - task = app.tasks[obj.task] + task = app.tasks.get(obj.task) task.delay() self.message_user(request, f"Task '{obj.task}' queued", messages.SUCCESS) except Exception as e: # pragma: no cover + process_exception(e) self.message_user(request, f"Cannot queue '{obj.task}': {e}", messages.ERROR) return HttpResponseRedirect(reverse("admin:etl_etltask_changelist")) @action() def unlock(self, request, pk): obj = self.get_object(request, pk) - key = f"{obj.task}-lock" def _action(request): - cache.delete(key) + obj.loader.unlock() - return _confirm_action(self, request, _action, f"Continuing will unlock selected task. ({key})", + return _confirm_action(self, request, _action, + f"""Continuing will unlock selected task. ({obj.task}). +{obj.loader.task.name} - {obj.loader.config.lock_key} +""", "Successfully executed", ) @link() diff --git a/src/etools_datamart/apps/etl/apps.py b/src/etools_datamart/apps/etl/apps.py index a8a63ec7d..b310a0012 100644 --- a/src/etools_datamart/apps/etl/apps.py +++ b/src/etools_datamart/apps/etl/apps.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- -from celery.signals import task_postrun from django.apps import AppConfig +from celery.signals import task_postrun + from etools_datamart.apps.etl.signals import data_refreshed diff --git a/src/etools_datamart/apps/etl/lock.py b/src/etools_datamart/apps/etl/lock.py index 9f09dbafd..bf7bb47e1 100644 --- a/src/etools_datamart/apps/etl/lock.py +++ b/src/etools_datamart/apps/etl/lock.py @@ -1,56 +1,56 @@ -# -*- coding: utf-8 -*- -import logging -from functools import partial, wraps - -from django.core.cache import caches -from redis.exceptions import LockError - -cache = caches['default'] - -logger = logging.getLogger(__name__) - -LOCK_EXPIRE = 60 * 60 # Lock expires in 1 hour - - -class TaskExecutionOverlap(Exception): - pass - - -def only_one(function=None, key="", timeout=None): - """Enforce only one celery task at a time.""" - - def _unlock(key): - try: - lock = cache.lock(key, timeout=timeout) - cache.delete(key) - lock.release() - except LockError: - pass - - def _dec(run_func): - """Decorator.""" - - @wraps(run_func) - def _caller(*args, **kwargs): - """Caller.""" - ret_value = None - have_lock = False - lock = cache.lock(key, timeout=timeout) - try: - have_lock = lock.acquire(blocking=False) - if have_lock: - ret_value = run_func(*args, **kwargs) - # else: - # raise TaskExecutionOverlap(key) - finally: - if have_lock: - try: - lock.release() - except LockError as e: # pragma: no cover - logger.warning(e) - - return ret_value - - return _caller - function.unlock = partial(_unlock, key) - return _dec(function) if function is not None else _dec +# # -*- coding: utf-8 -*- +# import logging +# from functools import wraps +# +# from django.core.cache import caches +# from redis.exceptions import LockError +# +# cache = caches['lock'] +# +# logger = logging.getLogger(__name__) +# +# LOCK_EXPIRE = 60 * 60 # Lock expires in 1 hour +# +# +# class TaskExecutionOverlap(Exception): +# pass +# +# +# def only_one(function, key, timeout=None): +# """Enforce only one celery task at a time.""" +# +# def _unlock(key): +# try: +# lock = cache.lock(key, timeout=timeout) +# cache.delete(key) +# lock.release() +# except LockError: +# pass +# +# def _dec(run_func): +# """Decorator.""" +# +# @wraps(run_func) +# def _caller(*args, **kwargs): +# """Caller.""" +# ret_value = None +# have_lock = False +# lock = cache.lock(key, timeout=timeout) +# try: +# have_lock = lock.acquire(blocking=False) +# if have_lock: +# ret_value = run_func(*args, **kwargs) +# +# finally: +# if have_lock: +# try: +# lock.release() +# except LockError as e: # pragma: no cover +# logger.warning(e) +# +# return ret_value +# +# return _caller +# +# # function.unlock = partial(_unlock, key) +# return _dec(function) if function is not None else _dec diff --git a/src/etools_datamart/apps/etl/management/commands/queue.py b/src/etools_datamart/apps/etl/management/commands/queue.py index 6868eb906..546d5a741 100644 --- a/src/etools_datamart/apps/etl/management/commands/queue.py +++ b/src/etools_datamart/apps/etl/management/commands/queue.py @@ -1,5 +1,6 @@ from django.core.management.base import CommandError, LabelCommand from django.utils.module_loading import import_string + from django_extensions.management.utils import signalcommand diff --git a/src/etools_datamart/apps/etl/migrations/0001_initial.py b/src/etools_datamart/apps/etl/migrations/0001_initial.py index 034d0d20f..da78f87ab 100644 --- a/src/etools_datamart/apps/etl/migrations/0001_initial.py +++ b/src/etools_datamart/apps/etl/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 import django.contrib.postgres.fields.jsonb import django.db.models.deletion diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index c950d129e..707fe036d 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -3,21 +3,28 @@ from django.contrib.postgres.fields import JSONField from django.db import models from django.utils.functional import cached_property + from django_celery_beat.models import PeriodicTask from etools_datamart.apps.data.models.base import DataMartModel -from etools_datamart.celery import app, ETLTask +from etools_datamart.celery import app class TaskLogManager(models.Manager): + def filter_for_models(self, *models): + return self.filter(content_type__in=ContentType.objects.get_for_models(*models).values()) + def get_for_model(self, model: DataMartModel): - return self.get(content_type=ContentType.objects.get_for_model(model)) + try: + return self.get(content_type=ContentType.objects.get_for_model(model)) + except EtlTask.DoesNotExist: + raise EtlTask.DoesNotExist(f"EtlTask for model '{model.__name__}' does not exists") - def get_for_task(self, task: ETLTask): - return self.get_or_create(task=task.name, - defaults=dict(content_type=ContentType.objects.get_for_model(task.linked_model), - last_run=None, - table_name=task.linked_model._meta.db_table))[0] + # def get_for_task(self, task: ETLTask): + # return self.get_or_create(task=task.name, + # defaults=dict(content_type=ContentType.objects.get_for_model(task.linked_model), + # last_run=None, + # table_name=task.linked_model._meta.db_table))[0] def inspect(self): tasks = app.get_all_etls() @@ -56,6 +63,17 @@ class Meta: def __str__(self): return f"{self.task} {self.status}" + # @cached_property + # def lock_key(self): + # return f"{self.task}-lock" + + @cached_property + def loader(self): + try: + return self.content_type.model_class().loader + except AttributeError: + return None + @cached_property def verbose_name(self): return self.content_type.model_class()._meta.verbose_name diff --git a/src/etools_datamart/apps/etl/results.py b/src/etools_datamart/apps/etl/results.py index 5a4c8a11f..e97a4d8d2 100644 --- a/src/etools_datamart/apps/etl/results.py +++ b/src/etools_datamart/apps/etl/results.py @@ -1,63 +1,81 @@ import json +# +from json.decoder import WHITESPACE + +from rest_framework.utils import encoders + + +# CREATED = 'created' +# UPDATED = 'updated' +# UNCHANGED = 'unchanged' +# +# +# class EtlResult: +# __slots__ = [CREATED, UPDATED, UNCHANGED] +# +# def __init__(self, updated=0, created=0, unchanged=0, **kwargs): +# self.created = created +# self.updated = updated +# self.unchanged = unchanged +# +# def __repr__(self): +# return repr(self.as_dict()) +# +# def incr(self, counter): +# setattr(self, counter, getattr(self, counter) + 1) +# +# def as_dict(self): +# return {'created': self.created, +# 'updated': self.updated, +# 'unchanged': self.unchanged} +# +# def __eq__(self, other): +# # FIXME: pdb +# import pdb; pdb.set_trace() +# if isinstance(other, EtlResult): +# other = other.as_dict() +# +# if isinstance(other, dict): +# return (self.created == other['created'] and +# self.updated == other['updated'] and +# self.unchanged == other['unchanged']) +# return False +# +# +class EtlDecoder(json.JSONDecoder): + + def decode(self, s, _w=WHITESPACE.match): + return super().decode(s, _w) + + +class EtlEncoder(encoders.JSONEncoder): -CREATED = 'created' -UPDATED = 'updated' -UNCHANGED = 'unchanged' - - -class EtlResult: - __slots__ = [CREATED, UPDATED, UNCHANGED] - - def __init__(self, updated=0, created=0, unchanged=0): - self.created = created - self.updated = updated - self.unchanged = unchanged - - def __repr__(self): - return repr(self.as_dict()) - - def incr(self, counter): - setattr(self, counter, getattr(self, counter) + 1) - - def as_dict(self): - return {'created': self.created, - 'updated': self.updated, - 'unchanged': self.unchanged} - - def __eq__(self, other): - if isinstance(other, EtlResult): - other = other.as_dict() - - if isinstance(other, dict): - return (self.created == other['created'] and - self.updated == other['updated'] and - self.unchanged == other['unchanged']) - return False - - -class EtlEncoder(json.JSONEncoder): def default(self, obj): + from etools_datamart.apps.data.loader import EtlResult if isinstance(obj, EtlResult): return { '__type__': '__EtlResult__', 'data': obj.as_dict() } - else: - return json.JSONEncoder.default(self, obj) + return super(EtlEncoder, self).default(obj) def etl_decoder(obj): if '__type__' in obj: - if obj['__type__'] == '__EtlResult__': - return EtlResult(**obj) + if obj['__type__'] == '__EtlResult__': # pragma: no cover + from etools_datamart.apps.data.loader import EtlResult + return EtlResult(**obj['data']) return obj +# # Encoder function def etl_dumps(obj): return json.dumps(obj, cls=EtlEncoder) -# Decoder function +# +# +# # Decoder function def etl_loads(obj): - return json.loads(obj, object_hook=etl_decoder) + return json.loads(obj, cls=EtlDecoder, object_hook=etl_decoder) diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index d45bdc0fc..2f15e264c 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -1,401 +1,404 @@ -# -*- coding: utf-8 -*- -import json -import logging -from datetime import date, datetime - -from crashlog.middleware import process_exception -from django.db import connections -from django.db.models import Sum -from django.db.models.functions import Coalesce -from django.utils import timezone -from strategy_field.utils import get_attr - -from etools_datamart.apps.data.models import HACT, Intervention, PMPIndicators -from etools_datamart.apps.data.models.fam import FAMIndicator -from etools_datamart.apps.data.models.user import UserStats -from etools_datamart.apps.etl.results import CREATED, EtlResult, UNCHANGED, UPDATED -from etools_datamart.apps.etools.models import (AuditAudit, AuditEngagement, AuditMicroassessment, - AuditSpecialaudit, AuditSpotcheck, AuthUser, HactAggregatehact, - PartnersIntervention, PartnersPartnerorganization,) -from etools_datamart.celery import app - -logger = logging.getLogger(__name__) - -__all__ = ["load_hact", "load_user_report", "load_fam_indicator", - "load_pmp_indicator", "load_intervention"] - - -def is_record_changed(record, values): - other = type(record)(**values) - for field_name, field_value in values.items(): - if getattr(record, field_name) != getattr(other, field_name): - return True - return False - - -def process(Model, filters, values): - try: - existing, created = Model.objects.get_or_create(**filters, - defaults=values) - if created: - op = CREATED - else: - if is_record_changed(existing, values): - op = UPDATED - Model.objects.update_or_create(**filters, - defaults=values) - else: - op = UNCHANGED - return op - except Exception as e: - logging.exception(e) - process_exception(e) - raise - - -@app.etl(HACT) -def load_hact(): - connection = connections['etools'] - countries = connection.get_tenants() - today = timezone.now() - results = EtlResult() - for country in countries: - connection.set_schemas([country.schema_name]) - - logger.info(u'Running on %s' % country.name) - aggregate = HactAggregatehact.objects.get(year=today.year) - data = json.loads(aggregate.partner_values) - - # PartnersPartnerorganization.objects.hact_active() - # qs = PartnersPartnerorganization.objects.filter(Q(reported_cy__gt=0) | Q(total_ct_cy__gt=0), hidden=False) - # values = dict(microassessments_total=0, - # programmaticvisits_total=0, - # followup_spotcheck=0, - # completed_hact_audits=0, - # completed_special_audits=0, - # ) - # for partner in qs.all(): - # values['microassessments_total'] += AuditEngagement.objects.filter( - # engagement_type=AuditEngagement.TYPE_MICRO_ASSESSMENT, - # status=AuditEngagement.FINAL, date_of_draft_report_to_unicef__year=datetime.now().year).count() - # - # values['programmaticvisits_total'] += partner.hact_values['programmatic_visits']['completed']['total'] - # values['followup_spotcheck'] = qs.aggregate(total=Coalesce(Sum( - # 'planned_engagement__spot_check_follow_up'), 0))['total'] - # - # # completed_hact_audits = ? - # values['completed_special_audits'] += AuditEngagement.objects.filter( - # engagement_type=AuditEngagement.TYPE_SPECIAL_AUDIT, - # status=AuditEngagement.FINAL, date_of_draft_report_to_unicef__year=datetime.now().year).count() - - # # Total number of completed Microassessments in the business area in the past year - values = dict(microassessments_total=data['assurance_activities']['micro_assessment'], - programmaticvisits_total=data['assurance_activities']['programmatic_visits']['completed'], - followup_spotcheck=data['assurance_activities']['spot_checks']['follow_up'], - completed_spotcheck=data['assurance_activities']['spot_checks']['completed'], - completed_hact_audits=data['assurance_activities']['scheduled_audit'], - completed_special_audits=data['assurance_activities']['special_audit'], - ) - op = process(HACT, filters=dict(year=today.year, - country_name=country.name, - schema_name=country.schema_name), - values=values) - results.incr(op) - # existing, created = HACT.objects.get_or_create(year=today.year, - # country_name=country.name, - # schema_name=country.schema_name, - # defaults=values) - # if created: - # results.created += 1 - # else: - # if is_record_changed(existing, values): - # results.updated += 1 - # HACT.objects.update_or_create(year=today.year, - # country_name=country.name, - # schema_name=country.schema_name, - # defaults=values) - # else: - # results.unchanged += 1 - - return results - - -@app.etl(PMPIndicators) -def load_pmp_indicator() -> EtlResult: - connection = connections['etools'] - countries = connection.get_tenants() - base_url = 'https://etools.unicef.org' - results = EtlResult() - - for country in countries: - connection.set_schemas([country.schema_name]) - - logger.info(u'Running on %s' % country.name) - for partner in PartnersPartnerorganization.objects.all(): - for intervention in PartnersIntervention.objects.filter(agreement__partner=partner): - planned_budget = getattr(intervention, - 'partnersintervention_partners_interventionbudget_intervention_id', None) - fr_currencies = intervention.frs.all().values_list('currency', flat=True).distinct() - has_assessment = bool(getattr(partner.current_core_value_assessment, 'assessment', False)) - values = {'country_name': country.name, - 'schema_name': country.schema_name, - 'business_area_code': country.business_area_code, - 'partner_name': partner.name, - 'partner_type': partner.cso_type, - 'vendor_number': partner.vendor_number, - - 'pd_ssfa_ref': intervention.number.replace(',', '-'), - 'pd_ssfa_status': intervention.status.title(), - 'pd_ssfa_start_date': intervention.start, - 'pd_ssfa_creation_date': intervention.created, - 'pd_ssfa_end_date': intervention.end, - - 'cash_contribution': intervention.total_unicef_cash or 0, - 'supply_contribution': intervention.total_in_kind_amount or 0, - 'total_budget': intervention.total_budget or 0, - 'unicef_budget': intervention.total_unicef_budget or 0, - - 'currency': intervention.planned_budget.currency if planned_budget else '-', - 'partner_contribution': intervention.planned_budget.partner_contribution if planned_budget else '-', - 'unicef_cash': intervention.planned_budget.unicef_cash if planned_budget else '-', - 'in_kind_amount': intervention.planned_budget.in_kind_amount if planned_budget else '-', - 'total': intervention.planned_budget.total if planned_budget else '-', - 'fr_numbers_against_pd_ssfa': ' - '.join([fh.fr_number for fh in intervention.frs.all()]), - 'fr_currencies': ', '.join(fr for fr in fr_currencies), - 'sum_of_all_fr_planned_amount': intervention.frs.aggregate( - total=Coalesce(Sum('intervention_amt'), 0))[ - 'total'] if fr_currencies.count() <= 1 else '-', - 'core_value_attached': has_assessment, - 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), - 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), - } - op = process(PMPIndicators, filters=dict(country_name=country.name, - schema_name=country.schema_name, - partner_id=partner.pk, - intervention_id=intervention.pk), - values=values) - results.incr(op) - # existing, created = PMPIndicators.objects.get_or_create(country_name=country.name, - # schema_name=country.schema_name, - # country_id=partner.id, - # partner_id=partner.pk, - # intervention_id=intervention.pk, - # defaults=values) - # if created: - # results.created += 1 - # else: - # if is_record_changed(existing, values): - # results.updated += 1 - # PMPIndicators.objects.update_or_create(country_name=country.name, - # schema_name=country.schema_name, - # country_id=partner.id, - # partner_id=partner.pk, - # intervention_id=intervention.pk, - # defaults=values) - # else: - # results.unchanged += 1 - - return results - # PMPIndicators.objects.create( - # country_id=country.pk, - # partner_id=partner.pk, - # intervention_id=intervention.pk) - # created[country.name] += 1 - # - # return created - - -@app.etl(Intervention) -def load_intervention() -> EtlResult: - connection = connections['etools'] - countries = connection.get_tenants() - results = EtlResult() - for country in countries: - connection.set_schemas([country.schema_name]) - qs = PartnersIntervention.objects.all().select_related('agreement', - 'partner_authorized_officer_signatory', - 'unicef_signatory', - 'country_programme', - ) - num = 0 - for num, record in enumerate(qs, 1): - values = dict(number=record.number, - title=record.title, - status=record.status, - start_date=record.start, - end_date=record.end, - review_date_prc=record.review_date_prc, - prc_review_document=record.prc_review_document, - partner_name=record.agreement.partner.name, - partner_authorized_officer_signatory_id=get_attr(record, - 'partner_authorized_officer_signatory.pk'), - country_programme_id=get_attr(record, 'country_programme.pk'), - intervention_id=record.pk, - unicef_signatory_id=get_attr(record, 'unicef_signatory.pk'), - - signed_by_unicef_date=record.signed_by_unicef_date, - signed_by_partner_date=record.signed_by_partner_date, - population_focus=record.population_focus, - signed_pd_document=record.signed_pd_document, - - submission_date=record.submission_date, - submission_date_prc=record.submission_date_prc, - - unicef_signatory_first_name=get_attr(record, - 'unicef_signatory.first_name'), - unicef_signatory_last_name=get_attr(record, - 'unicef_signatory.last_name'), - unicef_signatory_email=get_attr(record, 'unicef_signatory.email'), - - partner_signatory_title=get_attr(record, - 'partner_authorized_officer_signatory.title'), - partner_signatory_first_name=get_attr(record, - 'partner_authorized_officer_signatory.first_name'), - partner_signatory_last_name=get_attr(record, - 'partner_authorized_officer_signatory.last_name'), - partner_signatory_email=get_attr(record, - 'partner_authorized_officer_signatory.email'), - partner_signatory_phone=get_attr(record, - 'partner_authorized_officer_signatory.phone'), - - partner_focal_point_title=get_attr(record, - 'partner_focal_point.title'), - partner_focal_point_first_name=get_attr(record, - 'partner_focal_point.first_name'), - partner_focal_point_last_name=get_attr(record, - 'partner_focal_point.last_name'), - partner_focal_point_email=get_attr(record, - 'partner_focal_point.email'), - partner_focal_point_phone=get_attr(record, - 'partner_focal_point.phone'), - - metadata=record.metadata, - document_type=record.document_type, - updated=record.modified, - created=record.created, - ) - op = process(Intervention, filters=dict(country_name=country.name, - schema_name=country.schema_name, - agreement_id=record.agreement.pk, - intervention_id=record.pk), - values=values) - results.incr(op) - - # existing, created = Intervention.objects.get_or_create(country_name=country.name, - # schema_name=country.schema_name, - # intervention_id=record.pk, - # defaults=values) - # if created: - # results.created += 1 - # else: - # if is_record_changed(existing, values): - # results.updated += 1 - # Intervention.objects.update_or_create(country_name=country.name, - # schema_name=country.schema_name, - # intervention_id=record.pk, - # defaults=values) - # else: - # results.unchanged += 1 - - return results - - -@app.etl(FAMIndicator) -def load_fam_indicator() -> EtlResult: - connection = connections['etools'] - countries = connection.get_tenants() - - engagements = (AuditSpotcheck, AuditAudit, AuditSpecialaudit, AuditMicroassessment) - start_date = date.today() # + relativedelta(months=-1) - results = EtlResult() - for country in countries: - connection.set_schemas([country.schema_name]) - for model in engagements: - # indicator, created = FAMIndicator.objects.get_or_create(month=start_date, - # country_name=country.name, - # schema_name=country.schema_name) - # if created: - # results.created += 1 - # changed = created - realname = "_".join(model._meta.db_table.split('_')[1:]) - values = {} - for status, status_display in AuditEngagement.STATUSES: - filter_dict = { - 'engagement_ptr__status': status, - 'engagement_ptr__start_date__month': start_date.month, - 'engagement_ptr__start_date__year': start_date.year, - } - field_name = f"{realname}_{status_display}".replace(" ", "_").lower() - value = model.objects.filter(**filter_dict).count() - values[field_name] = value - # try: - # field_name = f"{realname}_{status_display}".replace(" ", "_").lower() - # value = model.objects.filter(**filter_dict).count() - # # just a safety check - # if not hasattr(indicator, field_name): # pragma: no cover - # raise ValueError(field_name) - # if getattr(indicator, field_name) == value: - # changed = False - # else: - # changed = changed and True - # setattr(indicator, field_name, value) - # except Exception as e: # pragma: no cover - # logger.error(e) - # raise - op = process(FAMIndicator, filters=dict(month=start_date, - country_name=country.name, - schema_name=country.schema_name), - values=values) - results.incr(op) - return results - - -@app.etl(UserStats) -def load_user_report() -> EtlResult: - connection = connections['etools'] - countries = connection.get_tenants() - today = date.today() - first_of_month = datetime(today.year, today.month, 1) - results = EtlResult() - for country in countries: - connection.set_schemas([country.schema_name]) - base = AuthUser.objects.filter(profile__country=country) - values = { - 'total': base.count(), - 'unicef': base.filter(email__endswith='@unicef.org').count(), - 'logins': base.filter( - last_login__month=first_of_month.month).count(), - 'unicef_logins': base.filter( - last_login__month=first_of_month.month, - email__endswith='@unicef.org').count(), - } - op = process(UserStats, filters=dict(month=first_of_month, - country_name=country.name, - schema_name=country.schema_name, ), - values=values) - results.incr(op) - - # existing, created = UserStats.objects.get_or_create(month=first_of_month, - # country_name=country.name, - # schema_name=country.schema_name, - # defaults=values) - # if created: - # results.created += 1 - # else: - # if is_record_changed(existing, values): - # results.updated += 1 - # UserStats.objects.update_or_create(month=first_of_month, - # country_name=country.name, - # schema_name=country.schema_name, - # defaults=values) - # else: - # results.unchanged += 1 - # - return results - # UserStats.objects.update_or_create(month=first_of_month, - # country_name=country.name, - # schema_name=country.schema_name, - # defaults=values) - # created[country.name] += 1 - # - # return created +# # -*- coding: utf-8 -*- +# import json +# import logging +# from datetime import date, datetime +# +# from crashlog.middleware import process_exception +# from django.db import connections +# from django.db.models import Sum +# from django.db.models.functions import Coalesce +# from django.utils import timezone +# from strategy_field.utils import get_attr +# +# from etools_datamart.apps.data.models import HACT, Intervention, PMPIndicators +# from etools_datamart.apps.data.models.fam import FAMIndicator +# from etools_datamart.apps.data.models.user import UserStats +# # from etools_datamart.apps.etl.results import CREATED, EtlResult, UNCHANGED, UPDATED +# from etools_datamart.apps.etools.models import (AuditAudit, AuditEngagement, AuditMicroassessment, +# AuditSpecialaudit, AuditSpotcheck, AuthUser, HactAggregatehact, +# PartnersIntervention, PartnersPartnerorganization,) +# from etools_datamart.celery import app +# +# logger = logging.getLogger(__name__) +# +# __all__ = ["load_hact", "load_user_report", "load_fam_indicator", +# "load_pmp_indicator", "load_intervention"] +# +# +# def is_record_changed(record, values): +# other = type(record)(**values) +# for field_name, field_value in values.items(): +# if getattr(record, field_name) != getattr(other, field_name): +# return True +# return False +# +# +# def process(Model, filters, values): +# try: +# existing, created = Model.objects.get_or_create(**filters, +# defaults=values) +# if created: +# op = CREATED +# else: +# if is_record_changed(existing, values): +# op = UPDATED +# Model.objects.update_or_create(**filters, +# defaults=values) +# else: +# op = UNCHANGED +# return op +# except Exception as e: # pragma: no cover +# logging.exception(e) +# process_exception(e) +# raise +# +# +# # @app.etl(HACT) +# def load_hact(): +# connection = connections['etools'] +# countries = connection.get_tenants() +# today = timezone.now() +# results = EtlResult() +# for country in countries: +# connection.set_schemas([country.schema_name]) +# +# logger.info(u'Running on %s' % country.name) +# aggregate = HactAggregatehact.objects.get(year=today.year) +# data = json.loads(aggregate.partner_values) +# +# # PartnersPartnerorganization.objects.hact_active() +# # qs = PartnersPartnerorganization.objects.filter(Q(reported_cy__gt=0) | Q(total_ct_cy__gt=0), hidden=False) +# # values = dict(microassessments_total=0, +# # programmaticvisits_total=0, +# # followup_spotcheck=0, +# # completed_hact_audits=0, +# # completed_special_audits=0, +# # ) +# # for partner in qs.all(): +# # values['microassessments_total'] += AuditEngagement.objects.filter( +# # engagement_type=AuditEngagement.TYPE_MICRO_ASSESSMENT, +# # status=AuditEngagement.FINAL, date_of_draft_report_to_unicef__year=datetime.now().year).count() +# # +# # values['programmaticvisits_total'] += partner.hact_values['programmatic_visits']['completed']['total'] +# # values['followup_spotcheck'] = qs.aggregate(total=Coalesce(Sum( +# # 'planned_engagement__spot_check_follow_up'), 0))['total'] +# # +# # # completed_hact_audits = ? +# # values['completed_special_audits'] += AuditEngagement.objects.filter( +# # engagement_type=AuditEngagement.TYPE_SPECIAL_AUDIT, +# # status=AuditEngagement.FINAL, date_of_draft_report_to_unicef__year=datetime.now().year).count() +# +# # # Total number of completed Microassessments in the business area in the past year +# values = dict(microassessments_total=data['assurance_activities']['micro_assessment'], +# programmaticvisits_total=data['assurance_activities']['programmatic_visits']['completed'], +# followup_spotcheck=data['assurance_activities']['spot_checks']['follow_up'], +# completed_spotcheck=data['assurance_activities']['spot_checks']['completed'], +# completed_hact_audits=data['assurance_activities']['scheduled_audit'], +# completed_special_audits=data['assurance_activities']['special_audit'], +# ) +# op = process(HACT, filters=dict(year=today.year, +# area_code=country.business_area_code, +# country_name=country.name, +# schema_name=country.schema_name), +# values=values) +# results.incr(op) +# # existing, created = HACT.objects.get_or_create(year=today.year, +# # country_name=country.name, +# # schema_name=country.schema_name, +# # defaults=values) +# # if created: +# # results.created += 1 +# # else: +# # if is_record_changed(existing, values): +# # results.updated += 1 +# # HACT.objects.update_or_create(year=today.year, +# # country_name=country.name, +# # schema_name=country.schema_name, +# # defaults=values) +# # else: +# # results.unchanged += 1 +# +# return results +# +# +# # @app.etl(PMPIndicators) +# def load_pmp_indicator() -> EtlResult: +# connection = connections['etools'] +# countries = connection.get_tenants() +# base_url = 'https://etools.unicef.org' +# results = EtlResult() +# +# for country in countries: +# connection.set_schemas([country.schema_name]) +# +# logger.info(u'Running on %s' % country.name) +# for partner in PartnersPartnerorganization.objects.all(): +# for intervention in PartnersIntervention.objects.filter(agreement__partner=partner): +# planned_budget = getattr(intervention, +# 'partnersintervention_partners_interventionbudget_intervention_id', None) +# fr_currencies = intervention.frs.all().values_list('currency', flat=True).distinct() +# has_assessment = bool(getattr(partner.current_core_value_assessment, 'assessment', False)) +# values = {'country_name': country.name, +# 'schema_name': country.schema_name, +# 'area_code': country.business_area_code, +# 'partner_name': partner.name, +# 'partner_type': partner.cso_type, +# 'vendor_number': partner.vendor_number, +# +# 'pd_ssfa_ref': intervention.number.replace(',', '-'), +# 'pd_ssfa_status': intervention.status.title(), +# 'pd_ssfa_start_date': intervention.start, +# 'pd_ssfa_creation_date': intervention.created, +# 'pd_ssfa_end_date': intervention.end, +# +# 'cash_contribution': intervention.total_unicef_cash or 0, +# 'supply_contribution': intervention.total_in_kind_amount or 0, +# 'total_budget': intervention.total_budget or 0, +# 'unicef_budget': intervention.total_unicef_budget or 0, +# +# 'currency': intervention.planned_budget.currency if planned_budget else '-', +# 'partner_contribution': intervention.planned_budget.partner_contribution if planned_budget else '-', +# 'unicef_cash': intervention.planned_budget.unicef_cash if planned_budget else '-', +# 'in_kind_amount': intervention.planned_budget.in_kind_amount if planned_budget else '-', +# 'total': intervention.planned_budget.total if planned_budget else '-', +# 'fr_numbers_against_pd_ssfa': ' - '.join([fh.fr_number for fh in intervention.frs.all()]), +# 'fr_currencies': ', '.join(fr for fr in fr_currencies), +# 'sum_of_all_fr_planned_amount': intervention.frs.aggregate( +# total=Coalesce(Sum('intervention_amt'), 0))[ +# 'total'] if fr_currencies.count() <= 1 else '-', +# 'core_value_attached': has_assessment, +# 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), +# 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), +# } +# op = process(PMPIndicators, filters=dict(country_name=country.name, +# schema_name=country.schema_name, +# partner_id=partner.pk, +# intervention_id=intervention.pk), +# values=values) +# results.incr(op) +# # existing, created = PMPIndicators.objects.get_or_create(country_name=country.name, +# # schema_name=country.schema_name, +# # country_id=partner.id, +# # partner_id=partner.pk, +# # intervention_id=intervention.pk, +# # defaults=values) +# # if created: +# # results.created += 1 +# # else: +# # if is_record_changed(existing, values): +# # results.updated += 1 +# # PMPIndicators.objects.update_or_create(country_name=country.name, +# # schema_name=country.schema_name, +# # country_id=partner.id, +# # partner_id=partner.pk, +# # intervention_id=intervention.pk, +# # defaults=values) +# # else: +# # results.unchanged += 1 +# +# return results +# # PMPIndicators.objects.create( +# # country_id=country.pk, +# # partner_id=partner.pk, +# # intervention_id=intervention.pk) +# # created[country.name] += 1 +# # +# # return created +# +# +# # @app.etl(Intervention) +# def load_intervention() -> EtlResult: +# connection = connections['etools'] +# countries = connection.get_tenants() +# results = EtlResult() +# for country in countries: +# connection.set_schemas([country.schema_name]) +# qs = PartnersIntervention.objects.all().select_related('agreement', +# 'partner_authorized_officer_signatory', +# 'unicef_signatory', +# 'country_programme', +# ) +# num = 0 +# for num, record in enumerate(qs, 1): +# values = dict(number=record.number, +# title=record.title, +# status=record.status, +# start_date=record.start, +# end_date=record.end, +# review_date_prc=record.review_date_prc, +# prc_review_document=record.prc_review_document, +# partner_name=record.agreement.partner.name, +# partner_authorized_officer_signatory_id=get_attr(record, +# 'partner_authorized_officer_signatory.pk'), +# country_programme_id=get_attr(record, 'country_programme.pk'), +# intervention_id=record.pk, +# unicef_signatory_id=get_attr(record, 'unicef_signatory.pk'), +# +# signed_by_unicef_date=record.signed_by_unicef_date, +# signed_by_partner_date=record.signed_by_partner_date, +# population_focus=record.population_focus, +# signed_pd_document=record.signed_pd_document, +# +# submission_date=record.submission_date, +# submission_date_prc=record.submission_date_prc, +# +# unicef_signatory_first_name=get_attr(record, +# 'unicef_signatory.first_name'), +# unicef_signatory_last_name=get_attr(record, +# 'unicef_signatory.last_name'), +# unicef_signatory_email=get_attr(record, 'unicef_signatory.email'), +# +# partner_signatory_title=get_attr(record, +# 'partner_authorized_officer_signatory.title'), +# partner_signatory_first_name=get_attr(record, +# 'partner_authorized_officer_signatory.first_name'), +# partner_signatory_last_name=get_attr(record, +# 'partner_authorized_officer_signatory.last_name'), +# partner_signatory_email=get_attr(record, +# 'partner_authorized_officer_signatory.email'), +# partner_signatory_phone=get_attr(record, +# 'partner_authorized_officer_signatory.phone'), +# +# partner_focal_point_title=get_attr(record, +# 'partner_focal_point.title'), +# partner_focal_point_first_name=get_attr(record, +# 'partner_focal_point.first_name'), +# partner_focal_point_last_name=get_attr(record, +# 'partner_focal_point.last_name'), +# partner_focal_point_email=get_attr(record, +# 'partner_focal_point.email'), +# partner_focal_point_phone=get_attr(record, +# 'partner_focal_point.phone'), +# +# metadata=record.metadata, +# document_type=record.document_type, +# updated=record.modified, +# created=record.created, +# ) +# op = process(Intervention, filters=dict(country_name=country.name, +# schema_name=country.schema_name, +# area_code=country.business_area_code, +# agreement_id=record.agreement.pk, +# intervention_id=record.pk), +# values=values) +# results.incr(op) +# +# # existing, created = Intervention.objects.get_or_create(country_name=country.name, +# # schema_name=country.schema_name, +# # intervention_id=record.pk, +# # defaults=values) +# # if created: +# # results.created += 1 +# # else: +# # if is_record_changed(existing, values): +# # results.updated += 1 +# # Intervention.objects.update_or_create(country_name=country.name, +# # schema_name=country.schema_name, +# # intervention_id=record.pk, +# # defaults=values) +# # else: +# # results.unchanged += 1 +# +# return results +# +# +# @app.etl(FAMIndicator) +# def load_fam_indicator() -> EtlResult: +# connection = connections['etools'] +# countries = connection.get_tenants() +# +# engagements = (AuditSpotcheck, AuditAudit, AuditSpecialaudit, AuditMicroassessment) +# start_date = date.today() # + relativedelta(months=-1) +# results = EtlResult() +# for country in countries: +# connection.set_schemas([country.schema_name]) +# for model in engagements: +# # indicator, created = FAMIndicator.objects.get_or_create(month=start_date, +# # country_name=country.name, +# # schema_name=country.schema_name) +# # if created: +# # results.created += 1 +# # changed = created +# realname = "_".join(model._meta.db_table.split('_')[1:]) +# values = {} +# for status, status_display in AuditEngagement.STATUSES: +# filter_dict = { +# 'engagement_ptr__status': status, +# 'engagement_ptr__start_date__month': start_date.month, +# 'engagement_ptr__start_date__year': start_date.year, +# } +# field_name = f"{realname}_{status_display}".replace(" ", "_").lower() +# value = model.objects.filter(**filter_dict).count() +# values[field_name] = value +# # try: +# # field_name = f"{realname}_{status_display}".replace(" ", "_").lower() +# # value = model.objects.filter(**filter_dict).count() +# # # just a safety check +# # if not hasattr(indicator, field_name): # pragma: no cover +# # raise ValueError(field_name) +# # if getattr(indicator, field_name) == value: +# # changed = False +# # else: +# # changed = changed and True +# # setattr(indicator, field_name, value) +# # except Exception as e: # pragma: no cover +# # logger.error(e) +# # raise +# op = process(FAMIndicator, filters=dict(month=start_date, +# country_name=country.name, +# area_code=country.business_area_code, +# schema_name=country.schema_name), +# values=values) +# results.incr(op) +# return results +# +# +# @app.etl(UserStats, bind=True) +# def load_user_report(): +# connection = connections['etools'] +# countries = connection.get_tenants() +# today = date.today() +# first_of_month = datetime(today.year, today.month, 1) +# results = EtlResult() +# for country in countries: +# connection.set_schemas([country.schema_name]) +# base = AuthUser.objects.filter(profile__country=country) +# values = { +# 'total': base.count(), +# 'unicef': base.filter(email__endswith='@unicef.org').count(), +# 'logins': base.filter( +# last_login__month=first_of_month.month).count(), +# 'unicef_logins': base.filter( +# last_login__month=first_of_month.month, +# email__endswith='@unicef.org').count(), +# } +# op = process(UserStats, filters=dict(month=first_of_month, +# country_name=country.name, +# schema_name=country.schema_name, ), +# values=values) +# results.incr(op) +# +# # existing, created = UserStats.objects.get_or_create(month=first_of_month, +# # country_name=country.name, +# # schema_name=country.schema_name, +# # defaults=values) +# # if created: +# # results.created += 1 +# # else: +# # if is_record_changed(existing, values): +# # results.updated += 1 +# # UserStats.objects.update_or_create(month=first_of_month, +# # country_name=country.name, +# # schema_name=country.schema_name, +# # defaults=values) +# # else: +# # results.unchanged += 1 +# # +# return results +# # UserStats.objects.update_or_create(month=first_of_month, +# # country_name=country.name, +# # schema_name=country.schema_name, +# # defaults=values) +# # created[country.name] += 1 +# # +# # return created diff --git a/src/etools_datamart/apps/etools/admin.py b/src/etools_datamart/apps/etools/admin.py index 8969fe5de..13ebb1611 100644 --- a/src/etools_datamart/apps/etools/admin.py +++ b/src/etools_datamart/apps/etools/admin.py @@ -60,12 +60,12 @@ class AuditEngagementAdmin(TenantModelAdmin): @register(models.PartnersIntervention) class PartnersInterventionAdmin(TenantModelAdmin): list_display = ('number', 'title', 'document_type', 'schema') - list_filter = ('document_type',) + # list_filter = ('document_type',) @register(models.T2FTravel) class T2FTravelAdmin(TenantModelAdmin): - pass + list_display = ("id", "schema", "status", "purpose") @register(models.ReportsAppliedindicator) @@ -101,3 +101,14 @@ class FundsreservationitemAdmin(TenantModelAdmin): @register(models.HactAggregatehact) class HactAggregatehactAdmin(TenantModelAdmin): list_display = ('schema', 'year',) + + +@register(models.TpmTpmvisit) +class TpmTpmvisitAdmin(TenantModelAdmin): + pass + + +@register(models.UsersCountry) +class UsersCountryAdmin(EToolsModelAdmin): + list_display = ('name', 'schema_name', 'business_area_code', 'country_short_code') + search_fields = ('name', 'schema_name', 'business_area_code') diff --git a/src/etools_datamart/apps/etools/patch.py b/src/etools_datamart/apps/etools/patch.py index 768ce6495..5ed65f58f 100644 --- a/src/etools_datamart/apps/etools/patch.py +++ b/src/etools_datamart/apps/etools/patch.py @@ -5,9 +5,10 @@ from django.db import models from django.utils.functional import cached_property from django.utils.translation import ugettext as _ + from unicef_security.models import User -from etools_datamart.apps.etools.models import PartnersPlannedengagement +from etools_datamart.apps.etools.models import PartnersPlannedengagement, T2FTravel def label(attr, self): @@ -163,3 +164,31 @@ def patch(): 'planned_budget'], ['partnersintervention_funds_fundsreservationheader_intervention_id', 'frs']) create_alias(PartnersIntervention, aliases) + + T2FTravel.PLANNED = 'planned' + T2FTravel.SUBMITTED = 'submitted' + T2FTravel.REJECTED = 'rejected' + T2FTravel.APPROVED = 'approved' + T2FTravel.CANCELLED = 'cancelled' + T2FTravel.SENT_FOR_PAYMENT = 'sent_for_payment' + T2FTravel.CERTIFICATION_SUBMITTED = 'certification_submitted' + T2FTravel.CERTIFICATION_APPROVED = 'certification_approved' + T2FTravel.CERTIFICATION_REJECTED = 'certification_rejected' + T2FTravel.CERTIFIED = 'certified' + T2FTravel.COMPLETED = 'completed' + + T2FTravel.CHOICES = ( + (T2FTravel.PLANNED, _('Planned')), + (T2FTravel.SUBMITTED, _('Submitted')), + (T2FTravel.REJECTED, _('Rejected')), + (T2FTravel.APPROVED, _('Approved')), + (T2FTravel.COMPLETED, _('Completed')), + (T2FTravel.CANCELLED, _('Cancelled')), + (T2FTravel.SENT_FOR_PAYMENT, _('Sent for payment')), + (T2FTravel.CERTIFICATION_SUBMITTED, _('Certification submitted')), + (T2FTravel.CERTIFICATION_APPROVED, _('Certification approved')), + (T2FTravel.CERTIFICATION_REJECTED, _('Certification rejected')), + (T2FTravel.CERTIFIED, _('Certified')), + (T2FTravel.COMPLETED, _('Completed')), + ) + T2FTravel._meta.get_field('status').choices = T2FTravel.CHOICES diff --git a/src/etools_datamart/apps/etools/utils.py b/src/etools_datamart/apps/etools/utils.py index 036e6f23b..0c27d8140 100644 --- a/src/etools_datamart/apps/etools/utils.py +++ b/src/etools_datamart/apps/etools/utils.py @@ -1,29 +1,49 @@ -from django.db import connections - -from etools_datamart.apps.etools.models import UsersUserprofile -from etools_datamart.apps.multitenant.exceptions import InvalidSchema - -conn = connections['etools'] - - -def get_etools_allowed_schemas(user): - # returns all allowed schemas as per eTools configuration - # if `user` is also an eTools user. - # matching is performed per email mnatching - # TODO: manage non etools user permissions - with conn.noschema(): - etools_user = UsersUserprofile.objects.filter(user__email=user.email).first() - if etools_user: - return set(etools_user.countries_available.values_list('schema_name', flat=True)) - else: - return set() +from etools_datamart.apps.security.utils import get_allowed_schemas, get_allowed_services # noqa +# from constance import config +# from django.db import connections +# from unicef_rest_framework.models import Service +# from unicef_security.models import Role # -# def schema_is_valid(*schema): -# return schema in conn.all_schemas - - -def validate_schemas(*schemas): - invalid = set(schemas) - conn.all_schemas - if invalid: - raise InvalidSchema(",".join(invalid)) +# from etools_datamart.apps.etools.models import UsersUserprofile +# +# conn = connections['etools'] +# +# +# def get_allowed_schemas(user): +# if config.DISABLE_SCHEMA_RESTRICTIONS: +# return sorted(conn.all_schemas) +# +# if not user.is_authenticated: +# return [] +# # returns all allowed schemas +# if user.is_superuser: +# return conn.all_schemas +# with conn.noschema(): +# aa = [] +# # aa = list(Role.objects.filter(user=user).values_list('business_area__name', flat=True)) +# etools_user = UsersUserprofile.objects.filter(user__email=user.email).first() +# if etools_user: +# aa.extend(set(etools_user.countries_available.values_list('schema_name', flat=True))) +# else: +# return conn.all_schemas +# +# return set(sorted(filter(None, aa))) +# # return set(map(lambda s: s.lower(), aa)) +# +# +# def get_allowed_services(user): +# if not user.is_authenticated: +# return [] +# if user.is_superuser or config.DISABLE_SERVICE_RESTRICTIONS: +# return Service.objects.all() +# return Service.objects.filter(groupaccesscontrol__group__user=user) +# +# # def schema_is_valid(*schema): +# # return schema in conn.all_schemas +# +# +# # def validate_schemas(*schemas): +# # invalid = set(schemas) - conn.all_schemas +# # if invalid: +# # raise InvalidSchema(",".join(invalid)) diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index 953465f85..d4bfc8c76 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -1,4 +1,5 @@ import os +import uuid import warnings from urllib.parse import urlparse @@ -8,15 +9,19 @@ from django.contrib.auth.models import Group from django.core.management import call_command from django.core.management.base import BaseCommand +from django.db import connections from django.utils.module_loading import import_string + +from constance import config from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask -from humanize import naturaldelta from post_office.models import EmailTemplate from redisboard.models import RedisServer from strategy_field.utils import fqn + from unicef_rest_framework.models.acl import GroupAccessControl from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.security.models import SchemaAccessControl from etools_datamart.celery import app MAIL = r"""Dear {{user.label}}, @@ -64,9 +69,25 @@
To unsubscribe, change your preferences in Datamart Monitor
""" +RESTRICTED_AREAS = {'234R': ['mpawlowski@unicef.org', + 'jmege@unicef.org', + 'nukhan@unicef.org']} + + +def get_everybody_available_areas(): + conn = connections['etools'] + return [c.schema_name for c in conn.get_tenants() + if c.business_area_code not in RESTRICTED_AREAS.keys()] + + +def get_restricted_areas(): + conn = connections['etools'] + return [c.schema_name for c in conn.get_tenants() + if c.business_area_code in RESTRICTED_AREAS.keys()] + class Command(BaseCommand): - help = "My shiny new management command." + help = "" def add_arguments(self, parser): parser.add_argument( @@ -103,20 +124,17 @@ def add_arguments(self, parser): default=False, help='refresh datamart tables') - parser.add_argument( - '--async', - action='store_true', - dest='_async', - default=False, - help='use celery to refresh datamart') - def handle(self, *args, **options): verbosity = options['verbosity'] migrate = options['migrate'] _all = options['all'] # interactive = options['interactive'] + self.stdout.write(f"Run collectstatic") + call_command('collectstatic', verbosity=verbosity - 1, interactive=False) + if migrate or _all: + self.stdout.write(f"Run migrations") call_command('migrate', verbosity=verbosity - 1) ModelUser = get_user_model() @@ -124,21 +142,43 @@ def handle(self, *args, **options): pwd = '123' admin = os.environ.get('USER', 'admin') else: - pwd = ModelUser.objects.make_random_password() - admin = os.environ.get('USER', 'admin') + pwd = os.environ.get('ADMIN_PASSWORD', ModelUser.objects.make_random_password()) + admin = os.environ.get('ADMIN_USERNAME', 'admin') self._admin_user, created = ModelUser.objects.get_or_create(username=admin, defaults={"is_superuser": True, "is_staff": True, "password": make_password(pwd)}) - Group.objects.get_or_create(name='Guests') - all_access, __ = Group.objects.get_or_create(name='Endpoints all access') if created: # pragma: no cover self.stdout.write(f"Created superuser `{admin}` with password `{pwd}`") else: # pragma: no cover self.stdout.write(f"Superuser `{admin}` already exists`.") + self.stdout.write(f"Create anonymous") + anonymous, created = ModelUser.objects.get_or_create(username='anonymous', + defaults={"is_superuser": False, + "is_staff": False, + "password": make_password(uuid.uuid4())}) + # self.stdout.write(f"Create group `Guest`") + # Group.objects.get_or_create(name='Guests') + # self.stdout.write(f"Create group `Endpoints all access`") + # all_access, __ = Group.objects.get_or_create(name='All endpoints access') + + self.stdout.write(f"Create group `Public areas access`") + public_areas, __ = Group.objects.get_or_create(name='Public areas access') + config.DEFAULT_GROUP = 'Public areas access' + + self.stdout.write(f"Create group `Restricted areas access`") + restricted_areas, __ = Group.objects.get_or_create(name='Restricted areas access') + + self.stdout.write(f"Grants all schemas to group `Endpoints all access`") + SchemaAccessControl.objects.get_or_create(group=public_areas, + schemas=get_everybody_available_areas()) + + SchemaAccessControl.objects.get_or_create(group=restricted_areas, + schemas=get_restricted_areas()) + from unicef_rest_framework.models import Service created, deleted, total = Service.objects.load_services() self.stdout.write(f"{total} services found. {created} new. {deleted} deleted") @@ -147,17 +187,24 @@ def handle(self, *args, **options): for service in Service.objects.all(): GroupAccessControl.objects.get_or_create( - group=all_access, + group=public_areas, service=service, serializers=['*'], policy=GroupAccessControl.POLICY_ALLOW ) + for area, users in RESTRICTED_AREAS.items(): + for email in users: + u, __ = ModelUser.objects.get_or_create(username=email, + email=email) + u.groups.add(public_areas) + u.groups.add(restricted_areas) + # hostname for entry, values in settings.CACHES.items(): loc = values.get('LOCATION', '') spec = urlparse(loc) if spec.scheme == 'redis': - RedisServer.objects.get_or_create(hostname=spec.netloc, + RedisServer.objects.get_or_create(hostname=spec.hostname, port=int(spec.port)) if os.environ.get('AUTOCREATE_USERS'): @@ -165,14 +212,13 @@ def handle(self, *args, **options): self.stdout.write("Going to create new users") try: for entry in os.environ.get('AUTOCREATE_USERS').split('|'): - user, pwd = entry.split(',') - User = get_user_model() - u, created = User.objects.get_or_create(username=user) + email, pwd = entry.split(',') + u, created = ModelUser.objects.get_or_create(username=email) if created: self.stdout.write(f"Created user {u}") u.set_password(pwd) u.save() - u.groups.add(all_access) + u.groups.add(public_areas) else: # pragma: no cover self.stdout.write(f"User {u} already exists.") @@ -181,6 +227,10 @@ def handle(self, *args, **options): if options['tasks'] or _all or options['refresh']: midnight, __ = CrontabSchedule.objects.get_or_create(minute=0, hour=0) + CrontabSchedule.objects.get_or_create(hour=[0, 6, 12, 18]) + CrontabSchedule.objects.get_or_create(hour=[0, 12]) + IntervalSchedule.objects.get_or_create(every=1, period=IntervalSchedule.HOURS) + every_minute, __ = IntervalSchedule.objects.get_or_create(every=1, period=IntervalSchedule.MINUTES) tasks = app.get_all_etls() @@ -213,21 +263,18 @@ def handle(self, *args, **options): defaults=dict(subject='Dataset changed', content=MAIL, html_content=MAIL_HTML)) - - if options['refresh']: - self.stdout.write("Refreshing datamart...") - for task in PeriodicTask.objects.all()[1:]: - try: - etl = import_string(task.task) - except ImportError: - continue - self.stdout.write(f"Running {task.name}...", ending='\r') - self.stdout.flush() - - if options['_async']: - etl.delay() - self.stdout.write(f"{task.name} scheduled") - else: - etl.apply() - cost = naturaldelta(app.timers[task.name]) - self.stdout.write(f"{task.name} excuted in {cost}") + config.CACHE_VERSION = config.CACHE_VERSION + 1 + + # if options['refresh']: + # self.stdout.write("Refreshing datamart...") + # for task in PeriodicTask.objects.all()[1:]: + # try: + # etl = import_string(task.task) + # except ImportError: + # continue + # self.stdout.write(f"Running {task.name}...", ending='\r') + # self.stdout.flush() + # + # etl.apply() + # cost = naturaldelta(app.timers[task.name]) + # self.stdout.write(f"{task.name} excuted in {cost}") diff --git a/src/etools_datamart/apps/me/__init__.py b/src/etools_datamart/apps/me/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/me/apps.py b/src/etools_datamart/apps/me/apps.py new file mode 100644 index 000000000..b6966f872 --- /dev/null +++ b/src/etools_datamart/apps/me/apps.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +from django.apps import AppConfig + + +class Config(AppConfig): + name = 'etools_datamart.apps.me' diff --git a/src/etools_datamart/apps/me/forms.py b/src/etools_datamart/apps/me/forms.py new file mode 100644 index 000000000..ba693d570 --- /dev/null +++ b/src/etools_datamart/apps/me/forms.py @@ -0,0 +1,5 @@ +from django import forms + + +class ProfileForm(forms.Form): + pass diff --git a/src/etools_datamart/apps/me/templates/me/base.html b/src/etools_datamart/apps/me/templates/me/base.html new file mode 100644 index 000000000..d5c0938f8 --- /dev/null +++ b/src/etools_datamart/apps/me/templates/me/base.html @@ -0,0 +1,19 @@ +{% extends 'base.html' %}{% load static %} +{% block title %}Profile{% endblock %} +{% block body %} + +
+ Home + API + Documentation + + {% if user.is_staff %} + Admin + {% endif %} +
+ + {% block content %} + {% endblock %} +{% endblock %} diff --git a/src/etools_datamart/apps/me/templates/me/profile.html b/src/etools_datamart/apps/me/templates/me/profile.html new file mode 100644 index 000000000..14c01c346 --- /dev/null +++ b/src/etools_datamart/apps/me/templates/me/profile.html @@ -0,0 +1,88 @@ +{% extends 'me/base.html' %}{% load static %} + +{% block title %}Profile{% endblock %} +{% block content %} +
+
{{ user.display_name }} - {{ user.email }}
+
Groups:  {% for g in user.groups.all %}{{ g.name }}{% endfor %}
+
+

Areas

+
    + {% for area in business_areas %} +
  • {{ area }}
    • + {% endfor %} +
+
+
+

Services

+ +
    + {% for service in services %} +
  • {{ service.name }}
    • + {% endfor %} +
+
+
+

Excel integration

+ {% if user.azure_id %} +
+ {% csrf_token %} +
+ {% if password %} + A new password has been generated, save it in a safe place. +

Note: Datamart cannot shows it in the future.

+
If lost you must generate a new one
+ + {% else %} + {% if user.has_usable_password %} + You have enabled extended Excel integration. Datamart cannot + show you the password but can generate a new one. + + {% else %} + Extended Excel integration is disabled. + To enable it you must generate a valid password. + {% endif %} + {% endif %} +
+ + {% if password %} + + + {% else %} + {% if user.has_usable_password %} + + {% else %} + + {% endif %} + {% endif %} +
+ {% else %} +
+
+ This account cannot enable Excel integration. + If you are logged in using corporate single sign-on (Azure/Active directory credentials) + contact Datamart administrators to enable it. +
+
+ {% endif %} +
+
+ +{% endblock %} +{% block bottom_scripts %} + + + +{% endblock %} diff --git a/src/etools_datamart/apps/me/urls.py b/src/etools_datamart/apps/me/urls.py new file mode 100644 index 000000000..5c1600f65 --- /dev/null +++ b/src/etools_datamart/apps/me/urls.py @@ -0,0 +1,8 @@ +from django.urls import path + +from .views import ProfileView + +urlpatterns = [ + path(r'', ProfileView.as_view(), name='profile'), + +] diff --git a/src/etools_datamart/apps/me/views.py b/src/etools_datamart/apps/me/views.py new file mode 100644 index 000000000..cd6d4d435 --- /dev/null +++ b/src/etools_datamart/apps/me/views.py @@ -0,0 +1,67 @@ +import random +import string + +from django.contrib.auth import BACKEND_SESSION_KEY, login +from django.contrib.auth.decorators import login_required +from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator +from django.views.generic.edit import FormView + +from etools_datamart.apps.etools.utils import get_allowed_schemas, get_allowed_services +from etools_datamart.apps.me.forms import ProfileForm + + +@login_required +def profile(request): + context = {'form': ProfileForm()} + return TemplateResponse(request, 'index.html', context) + + +class ProfileView(FormView): + form_class = ProfileForm + template_name = 'me/profile.html' + + @method_decorator(login_required) + def dispatch(self, request, *args, **kwargs): + return super().dispatch(request, *args, **kwargs) + + def generate(self, length=20): + pwd = "" + count = 0 + length = max(8, length) + while count < length: + upper = [random.choice(string.ascii_uppercase)] + lower = [random.choice(string.ascii_lowercase)] + num = [random.choice(string.digits)] + symbol = [random.choice(string.punctuation)] + everything = upper + lower + num + symbol + pwd += random.choice(everything) + count += 1 + if count >= length: + break + return pwd + + def get_context_data(self, **kwargs): + kwargs.update({'page': 'profile', + 'business_areas': sorted(get_allowed_schemas(self.request.user)), + 'services': get_allowed_services(self.request.user), + 'user': self.request.user + }) + return super().get_context_data(**kwargs) + + def get_initial(self): + return { + } + + # def get_success_url(self): + # return self.request.path + + def form_valid(self, form): + ctx = self.get_context_data(form=form) + if self.request.user.is_authenticated: + pwd = self.generate() + self.request.user.set_password(pwd) + self.request.user.save() + ctx['password'] = pwd + login(self.request, self.request.user, self.request.session[BACKEND_SESSION_KEY]) + return self.render_to_response(ctx) diff --git a/src/etools_datamart/apps/multitenant/admin.py b/src/etools_datamart/apps/multitenant/admin.py index a3e29ec2d..a2b127c77 100644 --- a/src/etools_datamart/apps/multitenant/admin.py +++ b/src/etools_datamart/apps/multitenant/admin.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- -from admin_extra_urls.extras import ExtraUrlMixin from django.contrib import messages from django.contrib.admin import ListFilter, ModelAdmin from django.contrib.admin.utils import quote from django.contrib.admin.views.main import ChangeList from django.core.exceptions import MultipleObjectsReturned, ValidationError -from django.db import connections +from django.db import connections, models from django.http import HttpResponseRedirect from django.urls import reverse +from admin_extra_urls.extras import ExtraUrlMixin + class TenantChangeList(ChangeList): IGNORED_PARAMS = ['country_name', ] @@ -83,11 +84,19 @@ def has_delete_permission(self, request, obj=None): return False -class EToolsModelAdmin(ExtraUrlMixin, ReadOnlyMixin, ModelAdmin): +class DisplayAllMixin: + def get_list_display(self, request): # pragma: no cover + if self.list_display == ('__str__',): + return [field.name for field in self.model._meta.fields + if not isinstance(field, models.ForeignKey)] + return self.list_display + + +class EToolsModelAdmin(ExtraUrlMixin, DisplayAllMixin, ReadOnlyMixin, ModelAdmin): pass -class TenantModelAdmin(ExtraUrlMixin, ReadOnlyMixin, ModelAdmin): +class TenantModelAdmin(ExtraUrlMixin, DisplayAllMixin, ReadOnlyMixin, ModelAdmin): list_filter = [SchemaFilter, ] # def get_queryset(self, request): diff --git a/src/etools_datamart/apps/multitenant/exceptions.py b/src/etools_datamart/apps/multitenant/exceptions.py index f27064723..bacb75d76 100644 --- a/src/etools_datamart/apps/multitenant/exceptions.py +++ b/src/etools_datamart/apps/multitenant/exceptions.py @@ -1,17 +1,20 @@ # -*- coding: utf-8 -*- +from django.template.defaultfilters import pluralize + from rest_framework.exceptions import PermissionDenied class InvalidSchema(Exception): - def __init__(self, schema, *args, **kwargs): # real signature unknown + def __init__(self, *schema): self.schema = schema def __str__(self): - return f"Invalid schema: '{self.schema}'" + return "Invalid schema%s: %s" % (pluralize(self.schema), + ','.join(self.schema)) class NotAuthorizedSchema(PermissionDenied): - def __init__(self, schema, *args, **kwargs): # real signature unknown + def __init__(self, schema, *args, **kwargs): self.schema = schema def __str__(self): diff --git a/src/etools_datamart/apps/multitenant/management/commands/inspectschema.py b/src/etools_datamart/apps/multitenant/management/commands/inspectschema.py index 107be08d3..9dfcb1ff1 100644 --- a/src/etools_datamart/apps/multitenant/management/commands/inspectschema.py +++ b/src/etools_datamart/apps/multitenant/management/commands/inspectschema.py @@ -5,6 +5,7 @@ from django.core.management.base import BaseCommand, CommandError from django.db import connections, DEFAULT_DB_ALIAS from django.db.models.constants import LOOKUP_SEP + from django_regex.utils import RegexList # from etools_datamart import state diff --git a/src/etools_datamart/apps/multitenant/postgresql/base.py b/src/etools_datamart/apps/multitenant/postgresql/base.py index 33494a154..aa45ba8a8 100644 --- a/src/etools_datamart/apps/multitenant/postgresql/base.py +++ b/src/etools_datamart/apps/multitenant/postgresql/base.py @@ -3,16 +3,19 @@ from contextlib import contextmanager from functools import lru_cache from time import time +from typing import List import django.db.utils -import psycopg2 from django.apps import apps from django.conf import settings from django.db.backends.postgresql import base as original_backend from django.db.backends.utils import CursorWrapper from django.utils.functional import cached_property +import psycopg2 + # from etools_datamart.state import state +from etools_datamart.apps.etools.models import UsersCountry from etools_datamart.apps.multitenant.exceptions import InvalidSchema from ..sql import Parser @@ -166,20 +169,20 @@ def noschema(self): self.set_schemas(old) @lru_cache() - def get_tenants(self): + def get_tenants(self) -> List[UsersCountry]: + # should be etools.UsersCountry model = apps.get_model(settings.TENANT_MODEL) return model.objects.filter(**settings.SCHEMA_FILTER).exclude(**settings.SCHEMA_EXCLUDE).order_by('name') @cached_property def all_schemas(self): - return set([c.schema_name for c in self.get_tenants()]) + return sorted(set([c.schema_name for c in self.get_tenants()])) def set_schemas(self, schemas): """ Main API method to current database schema, but it does not actually modify the db connection. """ - def _validate(n): name = n.lower() if name not in self.all_schemas: @@ -273,7 +276,6 @@ def _cursor(self, name=None): # of `set search_path` can be quite time consuming if not self.search_path_set and self._schemas: - search_paths = ["public"] search_paths.extend(self._schemas) if name: # pragma: no cover diff --git a/src/etools_datamart/apps/multitenant/postgresql/creation.py b/src/etools_datamart/apps/multitenant/postgresql/creation.py index 528508514..de15e57d0 100644 --- a/src/etools_datamart/apps/multitenant/postgresql/creation.py +++ b/src/etools_datamart/apps/multitenant/postgresql/creation.py @@ -114,12 +114,17 @@ def create_test_db(self, verbosity=1, autoclobber=False, serialize=True, keepdb= "-h", self.connection.settings_dict['HOST'], "-d", self.connection.settings_dict['NAME'], "--no-owner", + "--clean", + "--if-exists", "--disable-triggers", "--exit-on-error", str(public_dump)] - - subprocess.check_call(cmds) - + try: + subprocess.check_call(cmds) + except Exception as e: + print(e) + print(" ".join(cmds)) + sys.exit(1) try: cur.execute(raw_sql(header.format(schema='public'))) except Exception as e: diff --git a/src/etools_datamart/apps/multitenant/postgresql/public.sqldump b/src/etools_datamart/apps/multitenant/postgresql/public.sqldump index 0a761fec4..a94c37325 100644 Binary files a/src/etools_datamart/apps/multitenant/postgresql/public.sqldump and b/src/etools_datamart/apps/multitenant/postgresql/public.sqldump differ diff --git a/src/etools_datamart/apps/multitenant/postgresql/tenant.sql b/src/etools_datamart/apps/multitenant/postgresql/tenant.sql index 5cfc76599..c3d2758b1 100644 --- a/src/etools_datamart/apps/multitenant/postgresql/tenant.sql +++ b/src/etools_datamart/apps/multitenant/postgresql/tenant.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 10.5 (Debian 10.5-2.pgdg90+1) --- Dumped by pg_dump version 10.4 +-- Dumped from database version 9.6.10 +-- Dumped by pg_dump version 11.1 SET statement_timeout = 0; SET lock_timeout = 0; diff --git a/src/etools_datamart/apps/multitenant/sql.py b/src/etools_datamart/apps/multitenant/sql.py index 863fefa34..02b1e2c35 100644 --- a/src/etools_datamart/apps/multitenant/sql.py +++ b/src/etools_datamart/apps/multitenant/sql.py @@ -2,8 +2,9 @@ import re from collections import OrderedDict -import sqlparse from django.utils.functional import cached_property + +import sqlparse from django_regex.utils import RegexList from sqlparse.sql import Function, Identifier, IdentifierList, Where from sqlparse.tokens import Keyword, Whitespace diff --git a/src/etools_datamart/apps/multitenant/views.py b/src/etools_datamart/apps/multitenant/views.py index 84602daab..2ae68371f 100644 --- a/src/etools_datamart/apps/multitenant/views.py +++ b/src/etools_datamart/apps/multitenant/views.py @@ -3,12 +3,12 @@ from django.http import HttpResponseRedirect from django.urls import reverse_lazy -from django.utils.http import urlencode from django.views.generic.edit import FormView -from etools_datamart.apps.multitenant.forms import SchemasForm - # from unicef_rest_framework.state import state +from unicef_rest_framework.utils import get_query_string + +from etools_datamart.apps.multitenant.forms import SchemasForm logger = logging.getLogger(__name__) @@ -50,19 +50,20 @@ def form_valid(self, form): return response def get_query_string(self, new_params=None, remove=None): - if new_params is None: - new_params = {} - if remove is None: - remove = [] - p = self.params.copy() - for r in remove: - for k in list(p): - if k.startswith(r): - del p[k] - for k, v in new_params.items(): - if v is None: - if k in p: - del p[k] - else: - p[k] = v - return '?%s' % urlencode(sorted(p.items())) + return get_query_string(self.params, new_params, remove) + # if new_params is None: + # new_params = {} + # if remove is None: + # remove = [] + # p = self.params.copy() + # for r in remove: + # for k in list(p): + # if k.startswith(r): + # del p[k] + # for k, v in new_params.items(): + # if v is None: + # if k in p: + # del p[k] + # else: + # p[k] = v + # return '?%s' % urlencode(sorted(p.items())) diff --git a/src/etools_datamart/apps/security/admin.py b/src/etools_datamart/apps/security/admin.py new file mode 100644 index 000000000..708ea12f6 --- /dev/null +++ b/src/etools_datamart/apps/security/admin.py @@ -0,0 +1,13 @@ +from django.contrib import admin +from django.db import connections + +from .forms import SchemaAccessControlForm +from .models import SchemaAccessControl + +conn = connections['etools'] + + +@admin.register(SchemaAccessControl) +class SchemaAccessControlAdmin(admin.ModelAdmin): + form = SchemaAccessControlForm + list_display = ('group', 'schemas') diff --git a/src/etools_datamart/apps/security/forms.py b/src/etools_datamart/apps/security/forms.py new file mode 100644 index 000000000..4747915e0 --- /dev/null +++ b/src/etools_datamart/apps/security/forms.py @@ -0,0 +1,24 @@ +from django import forms +from django.contrib.auth.models import Group +from django.db import connections +from django.forms import ModelForm + +from .models import SchemaAccessControl + +conn = connections['etools'] + + +class SchemaAccessControlForm(ModelForm): + group = forms.ModelChoiceField(queryset=Group.objects.all()) + schemas = forms.MultipleChoiceField(choices=[], required=False) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['schemas'].choices = zip(conn.all_schemas, conn.all_schemas) + + class Meta: + model = SchemaAccessControl + fields = ('group', 'schemas') + + def clean_schemas(self): + return sorted(self.cleaned_data['schemas']) diff --git a/src/etools_datamart/apps/security/migrations/0001_initial.py b/src/etools_datamart/apps/security/migrations/0001_initial.py new file mode 100644 index 000000000..0558cfd50 --- /dev/null +++ b/src/etools_datamart/apps/security/migrations/0001_initial.py @@ -0,0 +1,30 @@ +# Generated by Django 2.1.4 on 2018-12-21 17:24 + +import django.contrib.postgres.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0009_alter_user_last_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='SchemaAccessControl', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('schemas', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, null=True, size=None)), + ('group', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='schemas', to='auth.Group')), + ], + options={ + 'verbose_name': 'Schemas ACL', + 'verbose_name_plural': 'Schemas ACLs', + 'ordering': ('group',), + }, + ), + ] diff --git a/src/etools_datamart/apps/security/migrations/__init__.py b/src/etools_datamart/apps/security/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/security/models.py b/src/etools_datamart/apps/security/models.py new file mode 100644 index 000000000..77ca63b81 --- /dev/null +++ b/src/etools_datamart/apps/security/models.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from django.contrib.auth.models import Group +from django.contrib.postgres.fields import ArrayField +from django.db import models + + +class SchemaAccessControl(models.Model): + group = models.OneToOneField(Group, models.CASCADE, related_name='schemas') + schemas = ArrayField(models.CharField(max_length=200), blank=True, null=True) + + class Meta: + verbose_name = 'Schemas ACL' + verbose_name_plural = 'Schemas ACLs' + ordering = ('group',) diff --git a/src/etools_datamart/apps/security/static/security/dual-listbox.css b/src/etools_datamart/apps/security/static/security/dual-listbox.css new file mode 100755 index 000000000..ec49a675d --- /dev/null +++ b/src/etools_datamart/apps/security/static/security/dual-listbox.css @@ -0,0 +1 @@ +.dual-listbox{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column}.dual-listbox .dual-listbox__container{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:wrap;flex-wrap:wrap}.dual-listbox .dual-listbox__search{border:1px solid #ddd;padding:10px;max-width:300px}.dual-listbox .dual-listbox__available,.dual-listbox .dual-listbox__selected{border:1px solid #ddd;height:300px;overflow-y:auto;padding:0;width:300px;margin-top:0;-webkit-margin-before:0}.dual-listbox .dual-listbox__buttons{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;margin:0 10px}.dual-listbox .dual-listbox__button{margin-bottom:5px;border:0;background-color:#eee;padding:10px;color:#fff}.dual-listbox .dual-listbox__button:hover{background-color:#ddd}.dual-listbox .dual-listbox__title{padding:15px 10px;font-size:120%;font-weight:700;border-left:1px solid #efefef;border-right:1px solid #efefef;border-top:1px solid #efefef;margin-top:1rem;-webkit-margin-before:1rem}.dual-listbox .dual-listbox__item{display:block;padding:10px;cursor:pointer;user-select:none;-moz-user-select:none;-webkit-user-select:none;-ms-user-select:none;border-bottom:1px solid #efefef;transition:background .2s ease}.dual-listbox .dual-listbox__item.dual-listbox__item--selected{background-color:rgba(8,157,227,.7)} \ No newline at end of file diff --git a/src/etools_datamart/apps/security/static/security/dual-listbox.js b/src/etools_datamart/apps/security/static/security/dual-listbox.js new file mode 100755 index 000000000..a80b8b059 --- /dev/null +++ b/src/etools_datamart/apps/security/static/security/dual-listbox.js @@ -0,0 +1 @@ +!function(t,e){if("object"==typeof exports&&"object"==typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var i=e();for(var s in i)("object"==typeof exports?exports:t)[s]=i[s]}}(this,function(){return function(t){function e(s){if(i[s])return i[s].exports;var n=i[s]={exports:{},id:s,loaded:!1};return t[s].call(n.exports,n,n.exports,e),n.loaded=!0,n.exports}var i={};return e.m=t,e.c=i,e.p="",e(0)}([function(t,e){"use strict";function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},n=function(){function t(t,e){for(var i=0;i1&&void 0!==arguments[1]?arguments[1]:{};i(this,t),this.setDefaults(),this.selected=[],this.available=[],t.isDomElement(e)?this.select=e:this.select=document.querySelector(e),this._initOptions(s),this._initReusableElements(),this._splitOptions(this.select.options),void 0!==s.options&&this._splitOptions(s.options),this._buildDualListbox(this.select.parentNode),this._addActions(),this.redraw()}return n(t,[{key:"setDefaults",value:function(){this.addEvent=null,this.removeEvent=null,this.availableTitle="Available options",this.selectedTitle="Selected options",this.showAddButton=!0,this.addButtonText="add",this.showRemoveButton=!0,this.removeButtonText="remove",this.showAddAllButton=!0,this.addAllButtonText="add all",this.showRemoveAllButton=!0,this.removeAllButtonText="remove all",this.searchPlaceholder="Search"}},{key:"addEventListener",value:function(t,e){this.dualListbox.addEventListener(t,e)}},{key:"addSelected",value:function(t){var e=this,i=this.available.indexOf(t);i>-1&&(this.available.splice(i,1),this.selected.push(t),this._selectOption(t.dataset.id),this.redraw(),setTimeout(function(){var i=document.createEvent("HTMLEvents");i.initEvent("added",!1,!0),i.addedElement=t,e.dualListbox.dispatchEvent(i)},0))}},{key:"redraw",value:function(){this.updateAvailableListbox(),this.updateSelectedListbox()}},{key:"removeSelected",value:function(t){var e=this,i=this.selected.indexOf(t);i>-1&&(this.selected.splice(i,1),this.available.push(t),this._deselectOption(t.dataset.id),this.redraw(),setTimeout(function(){var i=document.createEvent("HTMLEvents");i.initEvent("removed",!1,!0),i.removedElement=t,e.dualListbox.dispatchEvent(i)},0))}},{key:"searchLists",value:function(t,e){for(var i=e.querySelectorAll(".dual-listbox__item"),s=t.toLowerCase(),n=0;n1&&void 0!==arguments[1]?arguments[1]:null;e&&(e.preventDefault(),e.stopPropagation()),this.selected.indexOf(t)>-1?this.removeSelected(t):this.addSelected(t)}},{key:"_actionItemClick",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null;i&&i.preventDefault();for(var s=e.querySelectorAll(".dual-listbox__item"),n=0;n + + + +{% endblock %} + + +{% block admin_change_form_document_ready %} + {{ block.super }} + + +{% endblock %} diff --git a/src/etools_datamart/apps/security/templates/admin/security/schemaaccesscontrol/change_form2.html b/src/etools_datamart/apps/security/templates/admin/security/schemaaccesscontrol/change_form2.html new file mode 100644 index 000000000..7616d58da --- /dev/null +++ b/src/etools_datamart/apps/security/templates/admin/security/schemaaccesscontrol/change_form2.html @@ -0,0 +1,20 @@ +{% extends "admin/change_form.html" %} +{% block extrahead %}{{ block.super }} + + + + +{% endblock %} + + +{% block admin_change_form_document_ready %} + {{ block.super }} + + +{% endblock %} diff --git a/src/etools_datamart/apps/security/templates/security/array_widget.html b/src/etools_datamart/apps/security/templates/security/array_widget.html new file mode 100644 index 000000000..ead373f86 --- /dev/null +++ b/src/etools_datamart/apps/security/templates/security/array_widget.html @@ -0,0 +1,10 @@ +{% spaceless %} +
+
    + {% for widget in widget.subwidgets %} +
  • {% include widget.template_name %}
    • + {% endfor %} +
+
+
+{% endspaceless %} diff --git a/src/etools_datamart/apps/security/utils.py b/src/etools_datamart/apps/security/utils.py new file mode 100644 index 000000000..88ca339b4 --- /dev/null +++ b/src/etools_datamart/apps/security/utils.py @@ -0,0 +1,53 @@ +from django.core.cache import caches +from django.db import connections + +from concurrency.utils import flatten +from constance import config + +from unicef_rest_framework.models import Service + +from etools_datamart.apps.etools.models import UsersUserprofile +from etools_datamart.apps.security.models import SchemaAccessControl +from etools_datamart.libs.version import get_full_version + +conn = connections['etools'] +cache = caches['default'] + + +def get_allowed_schemas(user): + key = f"allowed_schemas:{get_full_version()}:{config.CACHE_VERSION}:{user.pk}" + values = cache.get(key) + if not values: + if config.DISABLE_SCHEMA_RESTRICTIONS: + values = conn.all_schemas + elif not user.is_authenticated: + values = [] + elif user.is_superuser: + values = conn.all_schemas + else: + with conn.noschema(): + aa = flatten(list(SchemaAccessControl.objects.filter(group__user=user).values_list('schemas'))) + etools_user = UsersUserprofile.objects.filter(user__email=user.email).first() + if etools_user: + aa.extend(set(etools_user.countries_available.values_list('schema_name', flat=True))) + values = list(filter(None, aa)) + cache.set(key, list(values)) + return set(values) + # return set(map(lambda s: s.lower(), aa)) + + +def get_allowed_services(user): + if not user.is_authenticated: + return [] + if user.is_superuser or config.DISABLE_SERVICE_RESTRICTIONS: + return Service.objects.all() + return Service.objects.filter(groupaccesscontrol__group__user=user) + +# def schema_is_valid(*schema): +# return schema in conn.all_schemas + + +# def validate_schemas(*schemas): +# invalid = set(schemas) - conn.all_schemas +# if invalid: +# raise InvalidSchema(",".join(invalid)) diff --git a/src/etools_datamart/apps/subscriptions/admin.py b/src/etools_datamart/apps/subscriptions/admin.py index 854a0c404..846b9593a 100644 --- a/src/etools_datamart/apps/subscriptions/admin.py +++ b/src/etools_datamart/apps/subscriptions/admin.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- -from admin_extra_urls.extras import ExtraUrlMixin from django.contrib import admin from django.contrib.admin import register +from admin_extra_urls.extras import ExtraUrlMixin + from . import models diff --git a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py index 8fee9683c..0eadffb79 100644 --- a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py +++ b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 import django.db.models.deletion from django.db import migrations, models @@ -17,8 +17,9 @@ class Migration(migrations.Migration): name='Subscription', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel')])), + ('type', models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel'), (3, 'Email+Pdf')])), ('kwargs', models.CharField(blank=True, default='', max_length=500)), + ('last_notification', models.DateField(blank=True, null=True)), ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), ], ), diff --git a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181221_1724.py similarity index 93% rename from src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py rename to src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181221_1724.py index da9944c95..6f2c14da6 100644 --- a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py +++ b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181221_1724.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 import django.db.models.deletion from django.conf import settings @@ -10,8 +10,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('subscriptions', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('subscriptions', '0001_initial'), ] operations = [ diff --git a/src/etools_datamart/apps/subscriptions/models.py b/src/etools_datamart/apps/subscriptions/models.py index 5fba886f3..189869e54 100644 --- a/src/etools_datamart/apps/subscriptions/models.py +++ b/src/etools_datamart/apps/subscriptions/models.py @@ -1,13 +1,15 @@ import logging from io import BytesIO -from crashlog.middleware import process_exception from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.db import models from django.utils.functional import cached_property + +from crashlog.middleware import process_exception from post_office import mail from rest_framework.test import APIRequestFactory + from unicef_rest_framework.models import Service from etools_datamart.apps.etl.models import EtlTask @@ -82,6 +84,7 @@ class Subscription(models.Model): type = models.IntegerField(choices=TYPES) content_type = models.ForeignKey(ContentType, models.CASCADE) kwargs = models.CharField(max_length=500, blank=True, null=False, default='') + last_notification = models.DateField(blank=True, null=True) objects = SubscriptionManager() @@ -91,13 +94,13 @@ class Meta: def __str__(self): return f"#{self.pk} {self.user} {self.get_type_display()} {self.content_type}" - @cached_property - def endpoint(self): - return self.content_type.model_class().service.endpoint - - @cached_property - def service(self): - return self.content_type.model_class().service + # @cached_property + # def endpoint(self): + # return self.content_type.model_class().service.endpoint + # + # @cached_property + # def service(self): + # return self.content_type.model_class().service @cached_property def viewset(self): diff --git a/src/etools_datamart/apps/subscriptions/urls.py b/src/etools_datamart/apps/subscriptions/urls.py index 1df0179d1..18656fb18 100644 --- a/src/etools_datamart/apps/subscriptions/urls.py +++ b/src/etools_datamart/apps/subscriptions/urls.py @@ -21,8 +21,12 @@ def _decorator(request, *args, **kwargs): if user: login(request, user, backend='django.contrib.auth.backends.RemoteUserBackend') else: - return HttpResponse(status=401) - return func(request, *args, **kwargs) + return HttpResponse(status=403) + + if request.user.is_authenticated: + return func(request, *args, **kwargs) + else: + return HttpResponse(status=401) return _decorator diff --git a/src/etools_datamart/apps/subscriptions/views.py b/src/etools_datamart/apps/subscriptions/views.py index af8d864e4..6b5a9d5fb 100644 --- a/src/etools_datamart/apps/subscriptions/views.py +++ b/src/etools_datamart/apps/subscriptions/views.py @@ -16,13 +16,13 @@ class Meta: @csrf_exempt def subscribe(request, etl_id): - code = 200 values = {'status': "", 'detail': ""} try: user = request.user payload = json.loads(request.body) form = SubscriptionForm(data=payload) if form.is_valid(): + code = 200 etl = EtlTask.objects.get(id=etl_id) s, created = Subscription.objects.update_or_create(user=user, content_type=etl.content_type, diff --git a/src/etools_datamart/apps/tracking/admin.py b/src/etools_datamart/apps/tracking/admin.py index 8ed0f8208..62c1dc0c2 100644 --- a/src/etools_datamart/apps/tracking/admin.py +++ b/src/etools_datamart/apps/tracking/admin.py @@ -2,10 +2,12 @@ import json import logging -from admin_extra_urls.extras import link from django.contrib import admin from django.template.defaultfilters import pluralize, urlencode from django.utils.safestring import mark_safe + +from admin_extra_urls.extras import link + from unicef_rest_framework.admin import APIModelAdmin, TruncateTableMixin from unicef_rest_framework.utils import humanize_size diff --git a/src/etools_datamart/apps/tracking/management/__init__.py b/src/etools_datamart/apps/tracking/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/tracking/management/commands/__init__.py b/src/etools_datamart/apps/tracking/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/tracking/management/commands/track.py b/src/etools_datamart/apps/tracking/management/commands/track.py new file mode 100644 index 000000000..02d7d51f5 --- /dev/null +++ b/src/etools_datamart/apps/tracking/management/commands/track.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import logging +import os +from urllib.parse import urlencode + +from django.core.management import BaseCommand + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + args = '' + help = '' + requires_migrations_checks = False + requires_system_checks = False + + def add_arguments(self, parser): + + parser.add_argument( + '--tid', + action='store_true', + dest='all', + default=os.environ.get('ANALYTICS_CODE'), + help='select all options but `demo`') + parser.add_argument( + '--demo', + action='store_true', + dest='demo', + default=False, + help='create random demo .local') + + def echo(self, txt, st='SUCCESS'): + if self.verbosity == 0: + return + if self.verbosity == 1: + text = f"{txt}" + else: + text = f"\n{txt}" + self.stdout.write(getattr(self.style, st)(text)) + + def notify(self, model, created, name, tpl=" {op} {model} `{name}`"): + if self.verbosity > 2: + op = {True: "Created", False: "Updated"}[created] + self.stdout.write(tpl.format(op=op, model=model, name=name)) + elif self.verbosity > 1: + self.stdout.write('.', ending='') + + def handle(self, *args, **options): + self.verbosity = options['verbosity'] + tid = options['tid'] + import requests + # """https://www.google-analytics.com/r/collect?v=1&_v=j72&a=1254325655&t=pageview&_s=1&dl=http%3A%2F%2Flocalhost%2F&ul=en-gb&de=UTF-8&dt=Title&sd=24-bit&sr=1440x900&vp=1425x459&je=0&_u=IEBAAUAB~&jid=243210006&gjid=1351824934&cid=778822076.1544038618&tid=UA-130479575-1&_gid=909229133.1544038618&_r=1>m=2oubc0&z=118711575""" + values = { + "v": 1, + "t": "pageview", + # "t": "event", + # "ec": "video", # event category + # "ea": "play", # event action + # "el": "holiday", + "tid": tid, + "cid": 555, + "ev": 300, + "dl": "http://datamart.unicef.io/aaaa", + + } + qs = urlencode(values) + # payload = 'v=1&t=event&tid=UA-130479575-1&cid=555&ec=video&ea=play&el=holiday&ev=300&dl=http%3A%2F%2Flocalhost%2Fbbb' + requests.post('http://www.google-analytics.com/collect', data=qs) diff --git a/src/etools_datamart/apps/tracking/middleware.py b/src/etools_datamart/apps/tracking/middleware.py index fd55dc265..8fa3dd76c 100644 --- a/src/etools_datamart/apps/tracking/middleware.py +++ b/src/etools_datamart/apps/tracking/middleware.py @@ -7,9 +7,11 @@ from django.conf import settings from django.db.models import F from django.utils.timezone import now + from strategy_field.utils import fqn from etools_datamart.apps.tracking import config +from etools_datamart.apps.tracking.asyncqueue import AsyncQueue from .models import APIRequestLog, DailyCounter, MonthlyCounter, PathCounter, UserCounter @@ -21,7 +23,8 @@ def log_request(**kwargs): log = APIRequestLog.objects.create(**kwargs) - if settings.ENABLE_LIVE_STATS: # pragma: no cover + + if settings.ENABLE_LIVE_STATS: lastMonth = (log.requested_at.replace(day=1) - datetime.timedelta(days=1)).replace(day=1) def _update_stats(target, **extra): @@ -35,7 +38,7 @@ def _update_stats(target, **extra): setattr(target, k, v) try: target.save() - except Exception as e: + except Exception as e: # pragma: no cover logger.error(f"""Error updating {target.__class__.__name__}: {e} {extra} """) @@ -79,8 +82,9 @@ def _update_stats(target, **extra): def record_to_kwargs(request, response): user = None + api_info = getattr(request, 'api_info') - if request.user and request.user.is_authenticated: # pragma: no cover + if request.user and request.user.is_authenticated: user = request.user # compute response time @@ -97,11 +101,11 @@ def record_to_kwargs(request, response): media_type = response.accepted_media_type except AttributeError: # pragma: no cover media_type = response['Content-Type'].split(';')[0] - view = request.api_info.get('view', None) + view = api_info.get('view', None) if not view: # pragma: no cover return {} viewset = fqn(view) - service = request.api_info.get("service") + service = api_info.get("service") from unicef_rest_framework.utils import get_ident return dict(user=user, requested_at=request.timestamp, @@ -118,10 +122,14 @@ def record_to_kwargs(request, response): cached=request.api_info.get('cache-hit', False), # see api.common.APICacheResponse content_type=media_type) + # -# class AsyncLogger(AsyncQueue): -# def _process(self, record): -# log_request(**record_to_kwargs(**record)) +class AsyncLogger(AsyncQueue): + def _process(self, record): + # import requests + # payload = 'v=1&t=event&tid=UA-XXXXXY&cid=555&ec=video&ea=play&el=holiday&ev=300' + # r = requests.post('http://www.google-analytics.com/collect', data=payload) + log_request(**record_to_kwargs(**record)) class StatsMiddleware(object): @@ -139,15 +147,17 @@ def __call__(self, request): request.timestamp = now() response = self.get_response(request) - if response.status_code == 200 and config.TRACK_PATH.match(request.path): + if response.status_code == 200 and \ + hasattr(request, 'api_info') and \ + config.TRACK_PATH.match(request.path): self.log(request, response) return response -# -# class ThreadedStatsMiddleware(StatsMiddleware): -# def __init__(self, get_response): -# super(ThreadedStatsMiddleware, self).__init__(get_response) -# self.worker = AsyncLogger() -# -# def log(self, request, response): -# self.worker.queue({'request': request, 'response': response}) + +class ThreadedStatsMiddleware(StatsMiddleware): + def __init__(self, get_response): + super(ThreadedStatsMiddleware, self).__init__(get_response) + self.worker = AsyncLogger() + + def log(self, request, response): + self.worker.queue({'request': request, 'response': response}) diff --git a/src/etools_datamart/apps/tracking/migrations/0001_initial.py b/src/etools_datamart/apps/tracking/migrations/0001_initial.py index 425d9911c..c1e927e9e 100644 --- a/src/etools_datamart/apps/tracking/migrations/0001_initial.py +++ b/src/etools_datamart/apps/tracking/migrations/0001_initial.py @@ -1,9 +1,10 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 import django.utils.timezone -import strategy_field.fields from django.db import migrations, models +import strategy_field.fields + class Migration(migrations.Migration): diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181221_1724.py similarity index 97% rename from src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py rename to src/etools_datamart/apps/tracking/migrations/0002_auto_20181221_1724.py index f98434cb8..25f348587 100644 --- a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py +++ b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181221_1724.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 import django.db.models.deletion from django.conf import settings @@ -10,8 +10,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('unicef_rest_framework', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('tracking', '0001_initial'), ] diff --git a/src/etools_datamart/apps/tracking/models/counters.py b/src/etools_datamart/apps/tracking/models/counters.py index 4fba5ce45..0af1b1627 100644 --- a/src/etools_datamart/apps/tracking/models/counters.py +++ b/src/etools_datamart/apps/tracking/models/counters.py @@ -3,6 +3,7 @@ from django.conf import settings from django.db import connections, models + from unicef_rest_framework.models import Service logger = logging.getLogger(__name__) diff --git a/src/etools_datamart/apps/tracking/models/log.py b/src/etools_datamart/apps/tracking/models/log.py index 268800a6d..402a17275 100644 --- a/src/etools_datamart/apps/tracking/models/log.py +++ b/src/etools_datamart/apps/tracking/models/log.py @@ -4,6 +4,7 @@ from django.conf import settings from django.db import connections, models from django.utils.timezone import now + from strategy_field.fields import StrategyClassField logger = logging.getLogger(__name__) diff --git a/src/etools_datamart/apps/web/static/api-doc.css b/src/etools_datamart/apps/web/static/api-doc.css index 78a957782..32b55b319 100644 --- a/src/etools_datamart/apps/web/static/api-doc.css +++ b/src/etools_datamart/apps/web/static/api-doc.css @@ -1,15 +1,33 @@ +nav { + font-family: Roboto, sans-serif; } + nav span.title { + display: inline-block; + font-weight: normal; + font-size: 20px; } + nav img { + margin-left: 10px; + width: 50px; + vertical-align: middle; } + nav .links { + margin-right: 50px; + float: right; + font-size: 16px; + line-height: 2; + padding: 2px; } + nav .links a { + color: #32329f; + text-transform: none; + text-decoration: none; } + nav .links a::after { + content: " |"; } + nav .links a:last-child:after { + content: ''; } + .menu-content:before { - content: ''; - /* with class ModalCarrot ??*/ - background: url("unicef_logo.png"); - background-size: 100px; - background-repeat: no-repeat; margin-left: 80px; - margin-bottom: 10px; - height: 100px; + align-content: center; + min-height: 110px; overflow: hidden; - display: block; - left: -50px; - top: 10px; } + display: block; } /*# sourceMappingURL=api-doc.css.map */ diff --git a/src/etools_datamart/apps/web/static/api-doc.css.map b/src/etools_datamart/apps/web/static/api-doc.css.map index d31217370..85d80116a 100644 --- a/src/etools_datamart/apps/web/static/api-doc.css.map +++ b/src/etools_datamart/apps/web/static/api-doc.css.map @@ -1,7 +1,7 @@ { "version": 3, -"mappings": "AAAA,oBAAqB;EAEnB,OAAO,EAAE,EAAE;EAAE,8BAA8B;EAC3C,UAAU,EAAE,sBAAsB;EAClC,eAAe,EAAE,KAAK;EACtB,iBAAiB,EAAE,SAAS;EAC5B,WAAW,EAAE,IAAI;EACjB,aAAa,EAAE,IAAI;EACnB,MAAM,EAAE,KAAK;EACb,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,KAAK;EACd,IAAI,EAAE,KAAK;EACX,GAAG,EAAE,IAAI", +"mappings": "AAAA,GAAI;EACF,WAAW,EAAE,kBAAkB;EAE/B,cAAW;IACT,OAAO,EAAE,YAAY;IACrB,WAAW,EAAE,MAAM;IACnB,SAAS,EAAE,IAAI;EAGjB,OAAI;IACF,WAAW,EAAE,IAAI;IACjB,KAAK,EAAE,IAAI;IACX,cAAc,EAAE,MAAM;EAGxB,UAAO;IACL,YAAY,EAAE,IAAI;IAClB,KAAK,EAAE,KAAK;IACZ,SAAS,EAAE,IAAI;IACf,WAAW,EAAE,CAAC;IACd,OAAO,EAAE,GAAG;IAEZ,YAAE;MACA,KAAK,EAAE,OAAgB;MACvB,cAAc,EAAE,IAAI;MACpB,eAAe,EAAE,IAAI;MAErB,mBAAS;QACP,OAAO,EAAE,IAAI;MAGf,6BAAmB;QACjB,OAAO,EAAE,EAAE;;AAMnB,oBAAqB;EAMnB,WAAW,EAAE,IAAI;EAEjB,aAAa,EAAE,MAAM;EAErB,UAAU,EAAE,KAAK;EACjB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,KAAK", "sources": ["api-doc.scss"], "names": [], "file": "api-doc.css" -} +} \ No newline at end of file diff --git a/src/etools_datamart/apps/web/static/api-doc.scss b/src/etools_datamart/apps/web/static/api-doc.scss index f566e6811..34bb104fd 100644 --- a/src/etools_datamart/apps/web/static/api-doc.scss +++ b/src/etools_datamart/apps/web/static/api-doc.scss @@ -1,15 +1,54 @@ +nav { + font-family: Roboto, sans-serif; + + span.title { + display: inline-block; + font-weight: normal; + font-size: 20px; + } + + img { + margin-left: 10px; + width: 50px; + vertical-align: middle; + } + + .links { + margin-right: 50px; + float: right; + font-size: 16px; + line-height: 2; + padding: 2px; + + a { + color: rgb(50, 50, 159); + text-transform: none; + text-decoration: none; + + &::after { + content: " |"; + } + + &:last-child:after { + content: ''; + } + } + } +} + .menu-content:before { - //content:url('unicef_logo.png'); /* with class ModalCarrot ??*/ - content: ''; /* with class ModalCarrot ??*/ - background: url('unicef_logo.png'); - background-size: 100px; - background-repeat: no-repeat; + //content:url('logo_100.png'); /* with class ModalCarrot ??*/ + //content: '
'; /* with class ModalCarrot ??*/ + //background: url('logo_small.png'); + //background-size: 100px; + //background-repeat: no-repeat margin-left: 80px; - margin-bottom: 10px; - height: 100px; + //margin-bottom: 10px; + align-content: center; + //height: 100px; + min-height: 110px; overflow: hidden; display: block; - left: -50px; - top: 10px; - + //left: 50px; + //top: 10px; } diff --git a/src/etools_datamart/apps/web/static/logo_100.png b/src/etools_datamart/apps/web/static/logo_100.png new file mode 100644 index 000000000..5762cbf57 Binary files /dev/null and b/src/etools_datamart/apps/web/static/logo_100.png differ diff --git a/src/etools_datamart/apps/web/static/logo_small.png b/src/etools_datamart/apps/web/static/logo_small.png new file mode 100644 index 000000000..5762cbf57 Binary files /dev/null and b/src/etools_datamart/apps/web/static/logo_small.png differ diff --git a/src/etools_datamart/apps/web/static/style.css b/src/etools_datamart/apps/web/static/style.css index feeacaa12..3dc611649 100644 --- a/src/etools_datamart/apps/web/static/style.css +++ b/src/etools_datamart/apps/web/static/style.css @@ -57,12 +57,26 @@ body, html { text-align: center; } .right h2 { font-size: 14pt; } - .right ul { - padding: 20px 0 0 65px; - list-style: none; - font-size: 18pt; } - .right a { - color: #2090F8; } + +ul.index-menu { + padding: 20px 0 0 65px; + list-style: none; + font-size: 18pt; } + ul.index-menu a { + color: #2090F8; + text-transform: none; + text-decoration: none; } + ul.index-menu li.menu-sep { + width: 100%; + font-size: 2px; } + +.h10 { + display: block; + height: 10px; } + +.h30 { + display: block; + height: 30px; } /* [ login more ]*/ .left { @@ -95,7 +109,6 @@ input.input100 { /*[ Button ]*/ .container-login100-form-btn { - width: 100%; display: -webkit-box; display: -webkit-flex; display: -moz-box; @@ -113,7 +126,6 @@ input.input100 { justify-content: center; align-items: center; padding: 0 20px; - width: 100%; height: 50px; border-radius: 3px; background: #2090F8; @@ -143,37 +155,84 @@ input.input100 { -moz-border-radius: 20px; border-radius: 20px; } -.monitor .menubar { +.monitor .menubar, .profile .menubar { padding: 10px; } - .monitor .menubar a { + .monitor .menubar a, .profile .menubar a { text-transform: none; color: #2090F8; } - .monitor .menubar a:visited { + .monitor .menubar a:visited, .profile .menubar a:visited { text-transform: none; } -.monitor #monitor { +.monitor #monitor, .profile #monitor { width: 100%; padding: 30px; } - .monitor #monitor .SUCCESS { + .monitor #monitor .SUCCESS, .profile #monitor .SUCCESS { color: #00b200; font-weight: bold; } - .monitor #monitor .row { + .monitor #monitor .row, .profile #monitor .row { width: 100%; padding: 5px; } - .monitor #monitor .row.header { + .monitor #monitor .row.header, .profile #monitor .row.header { font-weight: bold; } - .monitor #monitor .row.odd { + .monitor #monitor .row.odd, .profile #monitor .row.odd { background-color: #eeeeee; } - .monitor #monitor .col { + .monitor #monitor .col, .profile #monitor .col { display: inline-block; } - .monitor #monitor .col.task { + .monitor #monitor .col.task, .profile #monitor .col.task { width: 30%; } - .monitor #monitor .col.last_run { + .monitor #monitor .col.last_run, .profile #monitor .col.last_run { width: 200px; } - .monitor #monitor .col.last_changes { + .monitor #monitor .col.last_changes, .profile #monitor .col.last_changes { width: 200px; } - .monitor #monitor .col.status { + .monitor #monitor .col.status, .profile #monitor .col.status { width: 120px; } - .monitor #monitor .col.subscription { + .monitor #monitor .col.subscription, .profile #monitor .col.subscription { width: 180px; } +.profile #areas, .profile #services, .profile #password { + margin-right: 20px; + padding: 10px; + position: relative; + display: inline-block; } + .profile #areas h1, .profile #services h1, .profile #password h1 { + font-size: 14pt; + margin-bottom: 5px; } + .profile #areas ul, .profile #areas form, .profile #services ul, .profile #services form, .profile #password ul, .profile #password form { + border: 1px solid #cfcfcf; + overflow-y: scroll; + width: 300px; + height: 400px; } +.profile #password .message { + margin: 20px; + text-align: left; } + .profile #password .message .password { + text-align: center; + margin: 20px; + padding: 20px; + font-family: monospace; + font-size: 20px; + border: 1px solid grey; } +.profile #password form { + width: 400px; + overflow-y: auto; + text-align: center; } +.profile #password .btn { + justify-content: center; + align-items: center; + padding: 0 20px; + height: 40px; + border-radius: 3px; + font-size: 12px; + color: #0; + line-height: 1.2; + text-transform: uppercase; + letter-spacing: 1px; + -webkit-transition: all 0.4s; + -o-transition: all 0.4s; + -moz-transition: all 0.4s; + transition: all 0.4s; + cursor: pointer; } +.profile #password .btn-submit { + background: #2090F8; + color: #fff; } + /*# sourceMappingURL=style.css.map */ diff --git a/src/etools_datamart/apps/web/static/style.css.map b/src/etools_datamart/apps/web/static/style.css.map index 16c09e0aa..be77e013e 100644 --- a/src/etools_datamart/apps/web/static/style.css.map +++ b/src/etools_datamart/apps/web/static/style.css.map @@ -1,6 +1,6 @@ { "version": 3, -"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EACnE,cAAE;IACA,KAAK,EAAE,KAAK;IACZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAIhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EACf,WAAK;IACH,OAAO,EAAE,cAAc;EAEzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAEpB,SAAG;IACD,SAAS,EAAE,IAAI;EAOjB,SAAG;IACD,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,IAAI;EAEjB,QAAE;IACA,KAAK,EAAE,OAAO;;AAKlB,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAC3B,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAGhB,QAAS;EACN,qBAAqB,EAAE,IAAI;EAC3B,kBAAkB,EAAE,IAAI;EACxB,aAAa,EAAE,IAAI;;AAIpB,iBAAS;EACP,OAAO,EAAE,IAAI;EACb,mBAAE;IAIA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IAJd,2BAAU;MACR,cAAc,EAAE,IAAI;AAM1B,iBAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EACb,0BAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAEnB,sBAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,6BAAQ;MACN,WAAW,EAAE,IAAI;IAEnB,0BAAK;MACH,gBAAgB,EAAE,OAAO;EAO7B,sBAAK;IACH,OAAO,EAAE,YAAY;IACrB,2BAAO;MACL,KAAK,EAAE,GAAG;IAEZ,+BAAU;MACR,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK;IAEd,6BAAS;MACP,KAAK,EAAE,KAAK;IAEd,mCAAe;MACb,KAAK,EAAE,KAAK", +"mappings": "AACA,CAAE;EACA,MAAM,EAAE,GAAG;EACX,OAAO,EAAE,GAAG;EACZ,UAAU,EAAE,UAAU;;AAGxB,UAAW;EACT,MAAM,EAAE,GAAG;EACX,WAAW,EAAE,sDAAsD;EAEnE,cAAE;IACA,KAAK,EAAE,KAAK;IAEZ,0BAAQ;MACN,KAAK,EAAE,KAAK;EAKhB,4BAAS;IACP,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,MAAM;;AAGhB,iBAAkB;EAChB,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,IAAI;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,IAAI;;AAGf,IAAK;EACH,KAAK,EAAE,MAAM;EACb,UAAU,EAAE,IAAI;EAChB,QAAQ,EAAE,MAAM;EAChB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,WAAW,EAAE,OAAO;EACpB,cAAc,EAAE,WAAW;;AAG7B,MAAO;EACL,KAAK,EAAE,GAAG;EAMV,SAAS,EAAE,IAAI;EAEf,WAAK;IACH,OAAO,EAAE,cAAc;EAGzB,cAAQ;IACN,KAAK,EAAE,IAAI;IACX,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;EAGpB,SAAG;IACD,SAAS,EAAE,IAAI;;AAInB,aAAc;EACZ,OAAO,EAAE,aAAa;EACtB,UAAU,EAAE,IAAI;EAChB,SAAS,EAAE,IAAI;EAEf,eAAE;IACA,KAAK,EAAE,OAAO;IACd,cAAc,EAAE,IAAI;IACpB,eAAe,EAAE,IAAI;EAGvB,yBAAY;IACV,KAAK,EAAE,IAAI;IACX,SAAS,EAAE,GAAG;;AAKlB,IAAK;EACH,OAAO,EAAE,KAAK;EACd,MAAM,EAAE,IAAI;;AAGd,IAAK;EACH,OAAO,EAAE,KAAK;EACd,MAAM,EAAE,IAAI;;AAcd,mBAAmB;AAEnB,KAAM;EACJ,KAAK,EAAE,GAAG;EACV,iBAAiB,EAAE,SAAS;EAC5B,eAAe,EAAE,KAAK;EACtB,mBAAmB,EAAE,MAAM;EAC3B,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,CAAC;;AAaZ,OAAQ;EACN,aAAa,EAAE,IAAI;;AAGrB,cAAe;EACb,KAAK,EAAE,IAAI;EACX,QAAQ,EAAE,QAAQ;EAClB,MAAM,EAAE,iBAAiB;;AAG3B,SAAU;EACR,OAAO,EAAE,KAAK;EACd,KAAK,EAAE,IAAI;EACX,UAAU,EAAE,WAAW;EACvB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,OAAO;EACd,WAAW,EAAE,GAAG;EAChB,OAAO,EAAE,MAAM;;AAGjB,cAAe;EACb,MAAM,EAAE,IAAI;;AAGd,cAAc;AACd,4BAA6B;EAE3B,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,SAAS,EAAE,IAAI;EACf,eAAe,EAAE,MAAM;;AAGzB,kBAAmB;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,YAAY;EACrB,OAAO,EAAE,QAAQ;EACjB,OAAO,EAAE,WAAW;EACpB,OAAO,EAAE,IAAI;EACb,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EAEf,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,UAAU,EAAE,OAAO;EAEnB,SAAS,EAAE,IAAI;EACf,KAAK,EAAE,IAAI;EACX,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EAEnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;;AAGtB,wBAAyB;EACvB,UAAU,EAAE,OAAO;;AAGrB,mBAAmB;AAyCnB,OAAQ;EACN,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,KAAK;EAEZ,sBAAe;IACb,KAAK,EAAE,KAAK;;AAIhB,QAAS;EACP,qBAAqB,EAAE,IAAI;EAC3B,kBAAkB,EAAE,IAAI;EACxB,aAAa,EAAE,IAAI;;AAInB,oCAAS;EACP,OAAO,EAAE,IAAI;EAEb,wCAAE;IAKA,cAAc,EAAE,IAAI;IACpB,KAAK,EAAE,OAAO;IALd,wDAAU;MACR,cAAc,EAAE,IAAI;AAQ1B,oCAAS;EACP,KAAK,EAAE,IAAI;EACX,OAAO,EAAE,IAAI;EAEb,sDAAS;IACP,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,IAAI;EAGnB,8CAAK;IACH,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IAEZ,4DAAS;MACP,WAAW,EAAE,IAAI;IAGnB,sDAAM;MACJ,gBAAgB,EAAE,OAAO;EAQ7B,8CAAK;IACH,OAAO,EAAE,YAAY;IAErB,wDAAO;MACL,KAAK,EAAE,GAAG;IAGZ,gEAAW;MACT,KAAK,EAAE,KAAK;IAGd,wEAAe;MACb,KAAK,EAAE,KAAK;IAGd,4DAAS;MACP,KAAK,EAAE,KAAK;IAGd,wEAAe;MACb,KAAK,EAAE,KAAK;;AAQlB,uDAA6B;EAC3B,YAAY,EAAE,IAAI;EAClB,OAAO,EAAE,IAAI;EACb,QAAQ,EAAE,QAAQ;EAClB,OAAO,EAAE,YAAY;EAErB,gEAAG;IACD,SAAS,EAAE,IAAI;IACf,aAAa,EAAE,GAAG;EAGpB,wIAAS;IACP,MAAM,EAAE,iBAAiB;IACzB,UAAU,EAAE,MAAM;IAClB,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;AAKf,2BAAS;EACP,MAAM,EAAE,IAAI;EACZ,UAAU,EAAE,IAAI;EAEhB,qCAAU;IACR,UAAU,EAAE,MAAM;IAElB,MAAM,EAAE,IAAI;IACZ,OAAO,EAAE,IAAI;IACb,WAAW,EAAE,SAAS;IACtB,SAAS,EAAE,IAAI;IACf,MAAM,EAAE,cAAc;AAI1B,uBAAK;EACH,KAAK,EAAE,KAAK;EACZ,UAAU,EAAE,IAAI;EAChB,UAAU,EAAE,MAAM;AAGpB,uBAAK;EACH,eAAe,EAAE,MAAM;EACvB,WAAW,EAAE,MAAM;EACnB,OAAO,EAAE,MAAM;EACf,MAAM,EAAE,IAAI;EACZ,aAAa,EAAE,GAAG;EAClB,SAAS,EAAE,IAAI;EAEf,KAAK,EAAE,EAAE;EACT,WAAW,EAAE,GAAG;EAChB,cAAc,EAAE,SAAS;EACzB,cAAc,EAAE,GAAG;EACnB,kBAAkB,EAAE,QAAQ;EAC5B,aAAa,EAAE,QAAQ;EACvB,eAAe,EAAE,QAAQ;EACzB,UAAU,EAAE,QAAQ;EACpB,MAAM,EAAE,OAAO;AAGjB,8BAAY;EACV,UAAU,EAAE,OAAO;EACnB,KAAK,EAAE,IAAI", "sources": ["style.scss"], "names": [], "file": "style.css" diff --git a/src/etools_datamart/apps/web/static/style.scss b/src/etools_datamart/apps/web/static/style.scss index 3ac8fdc1e..d13f17d40 100644 --- a/src/etools_datamart/apps/web/static/style.scss +++ b/src/etools_datamart/apps/web/static/style.scss @@ -8,13 +8,16 @@ body, html { height: 50%; font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif; + a { color: black; + &:hover { color: black; } } + span.str { color: black; } @@ -61,9 +64,11 @@ body, html { //display: -ms-flexbox; //display: flex; flex-wrap: wrap; + form { padding: 100px 0 0 65px; } + .center { width: 100%; text-align: center; @@ -74,25 +79,51 @@ body, html { font-size: 24pt; text-align: center; } + h2 { font-size: 14pt; } - //h1 { - // width: 100%; - // font-size: 24pt; - // text-align: center; - //} - ul { - padding: 20px 0 0 65px; - list-style: none; - font-size: 18pt; - } +} + +ul.index-menu { + padding: 20px 0 0 65px; + list-style: none; + font-size: 18pt; + a { color: #2090F8; + text-transform: none; + text-decoration: none; } + li.menu-sep { + width: 100%; + font-size: 2px; + //border: 1px solid black; + } } +.h10 { + display: block; + height: 10px; +} + +.h30 { + display: block; + height: 30px; +} + +//.pad4 { +// padding: 4px; +//} +// +//.pad6 { +// padding: 6px; +//} +//.pad10 { +// padding: 10px; +//} + /* [ login more ]*/ .left { @@ -140,7 +171,7 @@ input.input100 { /*[ Button ]*/ .container-login100-form-btn { - width: 100%; + //width: 100%; display: -webkit-box; display: -webkit-flex; display: -moz-box; @@ -159,7 +190,7 @@ input.input100 { justify-content: center; align-items: center; padding: 0 20px; - width: 100%; + //width: 100%; height: 50px; border-radius: 3px; background: #2090F8; @@ -230,61 +261,144 @@ input.input100 { color: white; } } + .rounded { - -webkit-border-radius: 20px; - -moz-border-radius: 20px; - border-radius: 20px; + -webkit-border-radius: 20px; + -moz-border-radius: 20px; + border-radius: 20px; } -.monitor { +.monitor, .profile { .menubar { padding: 10px; + a { &:visited { text-transform: none; } + text-transform: none; color: #2090F8; } } + #monitor { width: 100%; padding: 30px; + .SUCCESS { color: #00b200; font-weight: bold; } + .row { width: 100%; padding: 5px; - &.header{ + + &.header { font-weight: bold; } - &.odd{ + + &.odd { background-color: #eeeeee; } - &.even{ + + &.even { } } .col { display: inline-block; + &.task { width: 30%; } - &.last_run{ + + &.last_run { width: 200px; } + &.last_changes { width: 200px; } + &.status { width: 120px; } + &.subscription { width: 180px; } } } } + +.profile { + + #areas, #services, #password { + margin-right: 20px; + padding: 10px; + position: relative; + display: inline-block; + + h1 { + font-size: 14pt; + margin-bottom: 5px; + } + + ul, form { + border: 1px solid #cfcfcf; + overflow-y: scroll; + width: 300px; + height: 400px; + } + } + + #password { + .message { + margin: 20px; + text-align: left; + + .password { + text-align: center; + + margin: 20px; + padding: 20px; + font-family: monospace; + font-size: 20px; + border: 1px solid grey; + } + } + + form { + width: 400px; + overflow-y: auto; + text-align: center; + } + + .btn { + justify-content: center; + align-items: center; + padding: 0 20px; + height: 40px; + border-radius: 3px; + font-size: 12px; + //background: #2090F8; + color: #0; + line-height: 1.2; + text-transform: uppercase; + letter-spacing: 1px; + -webkit-transition: all 0.4s; + -o-transition: all 0.4s; + -moz-transition: all 0.4s; + transition: all 0.4s; + cursor: pointer; + } + + .btn-submit { + background: #2090F8; + color: #fff; + } + } +} diff --git a/src/etools_datamart/apps/web/templates/admin/base_site.html b/src/etools_datamart/apps/web/templates/admin/base_site.html index 0e505c3e8..d2b61e282 100644 --- a/src/etools_datamart/apps/web/templates/admin/base_site.html +++ b/src/etools_datamart/apps/web/templates/admin/base_site.html @@ -1,9 +1,19 @@ {% extends "admin/base.html" %} {% block title %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} - +{% block extrastyle %} + {% if config.ANALYTICS_CODE %} + + + {% endif %} +{% endblock %} {% block branding %} -

{{ site_header|default:_('Django administration') }}

+

{{ site_header|default:_('Django administration') }}

{% endblock %} {% block userlinks %} @@ -11,8 +21,16 @@

{{ site_header|default:_('D {% if user.is_superuser %} {% url 'sys-admin-info' as sysinfo %} {% if sysinfo %} - Sys Infos + / Sys Infos {% endif %} + / Switch index {% endif %} {% endblock %} {% block nav-global %}{% endblock %} +{#{% block footer %}#} +{# #} +{#{% endblock %}#} diff --git a/src/etools_datamart/apps/web/templates/admin/index_new.html b/src/etools_datamart/apps/web/templates/admin/index_new.html new file mode 100644 index 000000000..f2fbbb37f --- /dev/null +++ b/src/etools_datamart/apps/web/templates/admin/index_new.html @@ -0,0 +1,91 @@ +{% extends "admin/index.html" %}{% load i18n static %} +{% block content %} + +
+ + {% if groups %} + {% for section, apps in groups.items %} + {% if section != '_hidden_' %} +
+ + + {% for model in apps %} + + {% if model.admin_url %} + + {% else %} + + {% endif %} + + {% endfor %} +
+ {{ section }} +
{{ model.label }}{{ model.model_name }}
+
+ {% endif %} + {% endfor %} + {% else %} +

{% trans "You don't have permission to view or edit anything." %}

+ {% endif %} +
+ + + +{% endblock %} +{% block sidebar %} +
+
+
+

Links

+ +
+

{% trans 'Recent actions' %}

+

{% trans 'My actions' %}

+ {% load log %} + {% get_admin_log 10 as admin_log for_user user %} + {% if not admin_log %} +

{% trans 'None available' %}

+ {% else %} +
    + {% for entry in admin_log %} +
  • + {% if entry.is_deletion or not entry.get_admin_url %} + {{ entry.object_repr }} + {% else %} + {{ entry.object_repr }} + {% endif %} +
    + {% if entry.content_type %} + {% filter capfirst %}{{ entry.content_type }}{% endfilter %} + {% else %} + {% trans 'Unknown content' %} + {% endif %} +
    • + {% endfor %} +
+ {% endif %} +
+
{% endblock sidebar %} diff --git a/src/etools_datamart/apps/web/templates/base.html b/src/etools_datamart/apps/web/templates/base.html index cfd19a527..ca76a7932 100644 --- a/src/etools_datamart/apps/web/templates/base.html +++ b/src/etools_datamart/apps/web/templates/base.html @@ -7,10 +7,19 @@ {% endblock head %} - + {% if config.ANALYTICS_CODE %} + + + {% endif %} +{% block body %} {% block content %}
@@ -40,14 +49,17 @@ {% endblock right %}
-
- -
+ {% block left %} +
+ +
+ {% endblock left %} +{% endblock %} {% endblock %} {% block footer %} diff --git a/src/etools_datamart/apps/web/templates/drf-yasg/redoc.html b/src/etools_datamart/apps/web/templates/drf-yasg/redoc.html index db2f66abd..b015b3d70 100644 --- a/src/etools_datamart/apps/web/templates/drf-yasg/redoc.html +++ b/src/etools_datamart/apps/web/templates/drf-yasg/redoc.html @@ -8,7 +8,7 @@ {% block extra_head %} {# -- Add any extra HTML heads tags here - except scripts and styles -- #} - + {% endblock %} @@ -28,24 +28,37 @@ {% block extra_body %} {# -- Add any header/body markup here (rendered BEFORE the swagger-ui/redoc element) -- #} -{% endblock %} -
+ -{% block footer %} - {# -- Add any footer markup here (rendered AFTER the swagger-ui/redoc element) -- #} {% endblock %} +
- + {% block footer %} + {# -- Add any footer markup here (rendered AFTER the swagger-ui/redoc element) -- #} + {% endblock %} + + + + {% block main_scripts %} + + + + + {% endblock %} + {% block extra_scripts %} + {# -- Add any additional scripts here -- #} + {% endblock %} -{% block main_scripts %} - - - - -{% endblock %} -{% block extra_scripts %} - {# -- Add any additional scripts here -- #} -{% endblock %} diff --git a/src/etools_datamart/apps/web/templates/index.html b/src/etools_datamart/apps/web/templates/index.html index a0be8b868..d0b24b353 100644 --- a/src/etools_datamart/apps/web/templates/index.html +++ b/src/etools_datamart/apps/web/templates/index.html @@ -1,22 +1,30 @@ {% extends 'base.html' %}{% load static datamart %} {% block right %}

eTools Datamart

-
    +
      {% if request.user.is_anonymous %}
    • API (needs token)
      • +
    •  
      • +
    •  
    • Login
      • {% else %}
    • API
    • Documentation
    • Swagger
    • Monitor
      • +
    • Settings
      • {% if request.user.is_staff %} +
    •  
    • Admin
      • {% endif %} {% if request.user.is_superuser %} +
    •  
      • +
    • Flower
      • +
    • RabbitMQ
    • System Info
      • {% endif %} -
       
      +
    •  
      • +
    •  
    • Logout
    • Disconnect
      • diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index d512a4eb4..7adb66d3c 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -1,10 +1,12 @@ +# import json import os from time import time from celery import Celery +# from celery.contrib.abortable import AbortableTask from celery.signals import task_postrun, task_prerun -from celery.task import Task -from django.apps import apps +from celery.utils.log import get_task_logger +from kombu import Exchange, Queue from kombu.serialization import register from etools_datamart.apps.etl.results import etl_dumps, etl_loads @@ -12,59 +14,31 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etools_datamart.config.settings') -class ETLTask(Task): - abstract = True - linked_model = None - - class DatamartCelery(Celery): - etl_cls = ETLTask _mapping = {} - def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): - from etools_datamart.apps.etl.lock import only_one - linked_model = options.get('linked_model', None) - if linked_model: - name = name or self.gen_task_name(fun.__name__, fun.__module__) - options['lock_key'] = f"{name}-lock" - fun = only_one(fun, options['lock_key']) - options['unlock'] = fun.unlock - task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) - linked_model._etl_task = task - linked_model._etl_loader = fun - else: - task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) - return task - - def etl(self, model, *args, **opts): - opts['base'] = ETLTask - opts['linked_model'] = model - task = super().task(*args, **opts) - return task - def get_all_etls(self): return [cls for (name, cls) in self.tasks.items() if hasattr(cls, 'linked_model')] - # def gen_task_name(self, name, module): - # prefix = "" - # if module.endswith('.tasks.etl'): - # module = module[:-10] - # prefix = 'etl_' - # if module.endswith('.tasks'): - # module = module[:-6] - # return prefix + super(DatamartCelery, self).gen_task_name(name, module) - app = DatamartCelery('datamart') app.config_from_object('django.conf:settings', namespace='CELERY') -# app.autodiscover_tasks() -app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) -# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], -# related_name='tasks') -# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], -# related_name='etl') +app.autodiscover_tasks() +# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) # pragma app.timers = {} +app.conf.task_queues = ( + Queue('default', Exchange('default'), routing_key='default'), + Queue('etl', Exchange('etl'), routing_key='etl.#'), + Queue('mail', Exchange('mail'), routing_key='mail.#'), + Queue('subscription', Exchange('subscription'), routing_key='subscription.#'), +) +app.conf.task_default_queue = 'default' +app.conf.task_default_exchange_type = 'direct' +app.conf.task_default_routing_key = 'default' +app.conf.task_routes = {'etools_datamart.apps.etl.tasks.etl.*': {'queue': 'etl'}} +app.conf.task_routes = {'send_queued_mail': {'queue': 'mail'}} +app.conf.task_routes = {'etools_datamart.apps.etl.subscriptions.tasks.*': {'queue': 'subscription'}} @task_prerun.connect @@ -76,7 +50,6 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): from django.contrib.contenttypes.models import ContentType from etools_datamart.apps.etl.models import EtlTask from django.utils import timezone - defs = {'status': 'RUNNING', 'last_run': timezone.now()} EtlTask.objects.update_or_create(task=task.name, @@ -94,11 +67,7 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st from django.utils import timezone from etools_datamart.apps.subscriptions.models import Subscription from etools_datamart.apps.etl.models import EtlTask - - # from unicef_rest_framework.models import Service - if state != 'SUCCESS': - EtlTask.objects.filter(task=task.name).update(status=state) - + logger = get_task_logger('etl') if not hasattr(sender, 'linked_model'): return try: @@ -109,19 +78,22 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st 'status': state} if state == 'SUCCESS': - defs['results'] = retval.as_dict() - if retval.created > 0 or retval.updated > 0: - defs['last_changes'] = timezone.now() - for service in sender.linked_model.linked_services: - service.invalidate_cache() - Subscription.objects.notify(sender.linked_model) - - defs['last_success'] = timezone.now() - else: - if not isinstance(retval, dict): + try: + defs['results'] = retval.as_dict() + if retval.created > 0 or retval.updated > 0: + defs['last_changes'] = timezone.now() + for service in sender.linked_model.linked_services: + service.invalidate_cache() + Subscription.objects.notify(sender.linked_model) + defs['last_success'] = timezone.now() + + except Exception as e: # pragma: no cover + logger.error(e) defs['results'] = str(retval) + else: + # if not isinstance(retval, dict): + defs['results'] = str(retval) defs['last_failure'] = timezone.now() EtlTask.objects.update_or_create(task=task.name, defaults=defs) - # Service.objects.invalidate_cache() app.timers[task.name] = cost diff --git a/src/etools_datamart/config/admin.py b/src/etools_datamart/config/admin.py index 3c14752fa..2b7294733 100644 --- a/src/etools_datamart/config/admin.py +++ b/src/etools_datamart/config/admin.py @@ -1,7 +1,36 @@ +from collections import OrderedDict + +from django.conf import settings from django.contrib.admin import AdminSite from django.contrib.admin.apps import SimpleAdminConfig +from django.core.cache import caches from django.http import HttpResponseRedirect +from django.template.response import TemplateResponse from django.utils.translation import gettext_lazy +from django.views.decorators.cache import never_cache + +from constance import config + +from etools_datamart.libs.version import get_full_version + +cache = caches['default'] + +DEFAULT_INDEX_SECTIONS = { + 'Administration': ['unicef_rest_framework', 'constance', + 'dbtemplates', 'subscriptions', 'etl'], + 'Data': ['data', 'etools'], + 'Security': ['auth', + 'unicef_security.User', + 'security', + 'unicef_rest_framework.GroupAccessControl', + 'unicef_rest_framework.UserAccessControl', + ], + 'Logs': ['tracking', 'django_db_logging', 'crashlog', ], + 'System': ['redisboard', 'django_celery_beat', 'post_office'], + 'Other': ['unicef_rest_framework.Application', ], + '_hidden_': ['sites', 'unicef_rest_framework.Application', + 'oauth2_provider', 'social_django'] +} def reset_counters(request): @@ -37,6 +66,63 @@ def get_urls(self): ] return urls + @never_cache + def index(self, request, extra_context=None): + style = request.COOKIES.get('old_index_style', 0) + if style in [1, "1"]: + return super(DatamartAdminSite, self).index(request, {'index_style': 0}) + else: + return self.index_new(request, {'index_style': 1}) + + @never_cache + def index_new(self, request, extra_context=None): + key = f'apps_groups:{request.user.id}:{get_full_version()}:{config.CACHE_VERSION}' + app_list = self.get_app_list(request) + groups = cache.get(key) + if not groups: + sections = getattr(settings, 'INDEX_SECTIONS', DEFAULT_INDEX_SECTIONS) + groups = OrderedDict([(k, []) for k in sections.keys()]) + + def get_section(model, app): + fqn = "%s.%s" % (app['app_label'], model['object_name']) + target = 'Other' + if fqn in sections['_hidden_'] or app['app_label'] in sections['_hidden_']: + return '_hidden_' + + for sec, models in sections.items(): + if fqn in models: + return sec + elif app['app_label'] in models: + target = sec + return target + + for app in app_list: + for model in app['models']: + sec = get_section(model, app) + groups[sec].append( + {'app_label': str(app['app_label']), + 'app_name': str(app['name']), + 'app_url': app['app_url'], + 'label': "%s - %s" % (app['name'], model['object_name']), + 'model_name': str(model['name']), + 'admin_url': model['admin_url'], + 'perms': model['perms']}) + for __, models in groups.items(): + models.sort(key=lambda x: x['label']) + cache.set(key, groups, 60 * 60) + + context = { + **self.each_context(request), + # 'title': self.index_title, + 'app_list': app_list, + 'groups': dict(groups), + **(extra_context or {}), + } + + request.current_app = self.name + + return TemplateResponse(request, 'admin/index_new.html', context) + class AdminConfig(SimpleAdminConfig): """The default AppConfig for admin which does autodiscovery.""" diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 25e761071..6592c2517 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import datetime +import os from pathlib import Path import environ @@ -10,16 +11,20 @@ PACKAGE_DIR = SETTINGS_DIR.parent DEVELOPMENT_DIR = PACKAGE_DIR.parent.parent -env = environ.Env(API_URL=(str, 'http://localhost:8000/api/'), +env = environ.Env(API_PREFIX=(str, '/api/'), ETOOLS_DUMP_LOCATION=(str, str(PACKAGE_DIR / 'apps' / 'multitenant' / 'postgresql')), - + ANALYTICS_CODE=(str, ""), + REDOC_BASE=(str, '/api/+redoc/#operation/'), CACHE_URL=(str, "redis://127.0.0.1:6379/1"), - # API_CACHE_URL=(str, "redis://127.0.0.1:6379/2"), - API_CACHE_URL=(str, "locmemcache://"), + CACHE_URL_API=(str, "redis://127.0.0.1:6379/2?key_prefix=api"), + CACHE_URL_LOCK=(str, "redis://127.0.0.1:6379/2?key_prefix=lock"), + CACHE_URL_TEMPLATE=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), # CACHE_URL=(str, "dummycache://"), - # API_CACHE_URL=(str, "dummycache://"), + # CACHE_URL_API=(str, "dummycache://"), ABSOLUTE_BASE_URL=(str, 'http://localhost:8000'), DISCONNECT_URL=(str, 'https://login.microsoftonline.com/unicef.org/oauth2/logout'), + DISABLE_SCHEMA_RESTRICTIONS=(bool, False), + DISABLE_SERVICE_RESTRICTIONS=(bool, False), ENABLE_LIVE_STATS=(bool, True), CELERY_BROKER_URL=(str, 'redis://127.0.0.1:6379/2'), CELERY_RESULT_BACKEND=(str, 'redis://127.0.0.1:6379/3'), @@ -55,7 +60,7 @@ EMAIL_HOST_USER=(str, ''), EMAIL_HOST_PASSWORD=(str, ''), EMAIL_PORT=(int, 587), - + USE_X_FORWARDED_HOST=(bool, False), ) DEBUG = env.bool('DEBUG') @@ -69,6 +74,7 @@ SECRET_KEY = env('SECRET_KEY') ALLOWED_HOSTS = tuple(env.list('ALLOWED_HOSTS', default=[])) ABSOLUTE_BASE_URL = env('ABSOLUTE_BASE_URL') +API_PREFIX = env('API_PREFIX') ADMINS = ( ('Stefano', 'saxix@saxix.onmicrosoft.com'), @@ -182,14 +188,16 @@ AUTHENTICATION_BACKENDS = [ # 'social_core.backends.azuread_tenant.AzureADTenantOAuth2', - 'unicef_security.azure.AzureADTenantOAuth2Ext', + 'unicef_security.graph.AzureADTenantOAuth2Ext', 'django.contrib.auth.backends.ModelBackend', 'django.contrib.auth.backends.RemoteUserBackend', ] CACHES = { 'default': env.cache(), - 'api': env.cache('API_CACHE_URL') + 'lock': env.cache('CACHE_URL_LOCK'), + 'api': env.cache('CACHE_URL_API'), + 'dbtemplates': env.cache('CACHE_URL_TEMPLATE') } ROOT_URLCONF = 'etools_datamart.config.urls' @@ -204,10 +212,12 @@ 'APP_DIRS': False, 'OPTIONS': { 'loaders': [ + 'dbtemplates.loader.Loader', 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ], 'context_processors': [ + 'constance.context_processors.config', 'django.template.context_processors.debug', 'django.template.context_processors.request', 'etools_datamart.apps.multitenant.context_processors.schemas', @@ -250,11 +260,12 @@ 'etools_datamart.apps.web.apps.Config', 'etools_datamart.apps.init.apps.Config', 'etools_datamart.apps.multitenant', - 'etools_datamart.apps.security', + 'etools_datamart.apps.security.apps.Config', 'constance', 'constance.backends.database', 'django.contrib.auth', + 'django.contrib.gis', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', @@ -263,8 +274,18 @@ # 'django.contrib.admin', 'etools_datamart.config.admin.AdminConfig', + 'etools_datamart.apps.core.apps.Config', + 'etools_datamart.apps.etools', + 'etools_datamart.apps.data', + 'etools_datamart.apps.etl.apps.Config', + 'etools_datamart.apps.tracking.apps.Config', + 'etools_datamart.apps.subscriptions', + 'etools_datamart.apps.me', + 'etools_datamart.api', + 'admin_extra_urls', - 'unicef_rest_framework', + 'adminactions', + 'unicef_rest_framework.apps.Config', 'rest_framework', 'oauth2_provider', 'social_django', @@ -275,6 +296,7 @@ 'month_field', 'drf_querystringfilter', 'crispy_forms', + 'dbtemplates', 'drf_yasg', 'adminfilters', @@ -286,13 +308,6 @@ 'django_celery_beat', - 'etools_datamart.apps.core.apps.Config', - 'etools_datamart.apps.etools', - 'etools_datamart.apps.data', - 'etools_datamart.apps.etl.apps.Config', - 'etools_datamart.apps.tracking.apps.Config', - 'etools_datamart.apps.subscriptions', - 'etools_datamart.api', ] DATE_FORMAT = '%d %b %Y' DATE_INPUT_FORMATS = [ @@ -333,7 +348,14 @@ 'default': 'djcelery_email.backends.CeleryEmailBackend' } } +# celery-mail CELERY_EMAIL_CHUNK_SIZE = 10 + +# crispy-forms +CRISPY_FAIL_SILENTLY = not DEBUG +CRISPY_CLASS_CONVERTERS = {'textinput': "textinput inputtext"} +CRISPY_TEMPLATE_PACK = 'bootstrap3' + # django-secure CSRF_COOKIE_SECURE = env.bool('CSRF_COOKIE_SECURE') SECURE_BROWSER_XSS_FILTER = True @@ -342,27 +364,17 @@ SECURE_HSTS_INCLUDE_SUBDOMAINS = True SECURE_HSTS_SECONDS = 1 SECURE_SSL_REDIRECT = env.bool('SECURE_SSL_REDIRECT') +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') SESSION_COOKIE_HTTPONLY = True SESSION_COOKIE_SECURE = env.bool('SESSION_COOKIE_SECURE') X_FRAME_OPTIONS = env('X_FRAME_OPTIONS') - +USE_X_FORWARDED_HOST = env('USE_X_FORWARDED_HOST') +SESSION_SAVE_EVERY_REQUEST = True NOTIFICATION_SENDER = "etools_datamart@unicef.org" # django-constance CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' CONSTANCE_ADDITIONAL_FIELDS = { - # 'read_only_text': ['django.forms.fields.CharField', { - # 'required': False, - # 'widget': 'etools_datamart.libs.constance.ObfuscatedInput', - # }], - # 'write_only_text': ['django.forms.fields.CharField', { - # 'required': False, - # 'widget': 'etools_datamart.libs.constance.WriteOnlyTextarea', - # }], - # 'write_only_input': ['django.forms.fields.CharField', { - # 'required': False, - # 'widget': 'etools_datamart.libs.constance.WriteOnlyInput', - # }], 'select_group': ['etools_datamart.libs.constance.GroupChoiceField', { 'required': False, 'widget': 'etools_datamart.libs.constance.GroupChoice', @@ -376,28 +388,20 @@ AZURE_LOCATION = env('AZURE_LOCATION') CONSTANCE_CONFIG = { + 'CACHE_VERSION': (1, 'Use MS Graph API to fetch user data', int), 'AZURE_USE_GRAPH': (True, 'Use MS Graph API to fetch user data', bool), 'DEFAULT_GROUP': ('Guests', 'Default group new users belong to', 'select_group'), + 'ANALYTICS_CODE': (env('ANALYTICS_CODE'), 'Google analytics code'), + 'DISABLE_SCHEMA_RESTRICTIONS': (env('DISABLE_SCHEMA_RESTRICTIONS'), 'Disable per user schema authorizations'), + 'DISABLE_SERVICE_RESTRICTIONS': (env('DISABLE_SERVICE_RESTRICTIONS'), 'Disable per user service authorizations'), } CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler' CELERY_TIMEZONE = 'America/New_York' CELERY_BROKER_URL = env('CELERY_BROKER_URL') CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') -# CELERY_ACCEPT_CONTENT = ['application/json'] -# CELERY_RESULT_SERIALIZER = 'json' -# CELERY_TASK_SERIALIZER = 'json' -CELERY_TASK_IMPORTS = ["etools_datamart.apps.etl.tasks.etl", - "etools_datamart.apps.etl.tasks.tasks", ] -CELERY_BEAT_SCHEDULE = {} -CELERY_TASK_ALWAYS_EAGER = env.bool('CELERY_ALWAYS_EAGER', False) +CELERY_TASK_ALWAYS_EAGER = env.bool('CELERY_ALWAYS_EAGER') CELERY_EAGER_PROPAGATES_EXCEPTIONS = CELERY_TASK_ALWAYS_EAGER - -CELERY_TASK_ROUTES = { - 'etools_datamart.apps.etl.tasks.etl': {'queue': 'etl'}, - 'etools_datamart.apps.etl.tasks.tasks': {'queue': 'tasks'}, -} - CELERY_ACCEPT_CONTENT = ['etljson'] CELERY_TASK_SERIALIZER = 'etljson' CELERY_RESULT_SERIALIZER = 'etljson' @@ -410,12 +414,10 @@ "rest_framework.renderers.BrowsableAPIRenderer", ), "PAGE_SIZE": 100, - # "DEFAULT_PAGINATION_CLASS": 'rest_framework.pagination.CursorPagination', + 'DEFAULT_CONTENT_NEGOTIATION_CLASS': 'unicef_rest_framework.negotiation.CT', 'DEFAULT_PAGINATION_CLASS': 'unicef_rest_framework.pagination.APIPagination', 'DEFAULT_METADATA_CLASS': 'etools_datamart.api.metadata.SimpleMetadataWithFilters', 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning', - # 'DEFAULT_SCHEMA_CLASS': 'etools_datamart.api.swagger.APIAutoSchema', - # 'EXCEPTION_HANDLER': 'my_project.my_app.utils.custom_exception_handler' 'SEARCH_PARAM': 'search', 'ORDERING_PARAM': 'ordering', 'DATETIME_FORMAT': DATETIME_FORMAT @@ -455,7 +457,7 @@ SOCIAL_AUTH_REVOKE_TOKENS_ON_DISCONNECT = True SOCIAL_AUTH_PIPELINE = ( 'social_core.pipeline.social_auth.social_details', - 'unicef_security.azure.get_unicef_user', + 'unicef_security.graph.get_unicef_user', # 'unicef_security.azure.social_uid', # 'social_core.pipeline.social_auth.social_uid', # 'social_core.pipeline.social_auth.social_user', @@ -467,7 +469,7 @@ # 'social_core.pipeline.social_auth.load_extra_data', # 'social_core.pipeline.user.user_details', # 'social_core.pipeline.social_auth.associate_by_email', - 'unicef_security.azure.default_group', + 'unicef_security.graph.default_group', ) SOCIAL_AUTH_AZUREAD_TENANT_OAUTH2_KEY = env.str('AZURE_CLIENT_ID') @@ -478,11 +480,11 @@ SOCIAL_AUTH_USER_MODEL = 'unicef_security.User' # POLICY = os.getenv('AZURE_B2C_POLICY_NAME', "b2c_1A_UNICEF_PARTNERS_signup_signin") -SCOPE = ['openid', 'email'] +SCOPE = ['openid', ] IGNORE_DEFAULT_SCOPE = True SWAGGER_SETTINGS = { - 'DEFAULT_API_URL': env('API_URL'), + 'DEFAULT_API_URL': env('ABSOLUTE_BASE_URL') + env('API_PREFIX'), 'DEFAULT_AUTO_SCHEMA_CLASS': 'etools_datamart.api.swagger.schema.APIAutoSchema', 'DEFAULT_FILTER_INSPECTORS': ['etools_datamart.api.swagger.filters.APIFilterInspector', ], 'SECURITY_DEFINITIONS': { @@ -514,7 +516,7 @@ 'format': '%(levelname)-8s: %(asctime)s %(name)20s %(message)s' }, 'simple': { - 'format': '%(levelname)-8s: %(asctime)s %(name)20s: %(funcName)s %(message)s' + 'format': '%(levelname)-8s: %(asctime)s %(name)20s: %(funcName)s:%(lineno)s %(message)s' } }, 'filters': { @@ -601,3 +603,12 @@ BUSINESSAREA_MODEL = 'unicef_security.BusinessArea' AUTH_USER_MODEL = 'unicef_security.User' + + +def extra(r): + return {'AZURE_CLIENT_ID': os.environ['AZURE_CLIENT_ID'], + 'GRAPH_CLIENT_ID': os.environ['GRAPH_CLIENT_ID'], + 'AZURE_TENANT': os.environ['AZURE_TENANT']} + + +SYSINFO = {"extra": {'Azure': extra}} diff --git a/src/etools_datamart/config/urls.py b/src/etools_datamart/config/urls.py index 9cbc0cf6e..37dfd2783 100644 --- a/src/etools_datamart/config/urls.py +++ b/src/etools_datamart/config/urls.py @@ -1,5 +1,6 @@ from django.contrib.admin import site from django.urls import include, path, re_path + from django_sysinfo.views import admin_sysinfo, http_basic_login, sysinfo, version from oauth2_provider.views import AuthorizationView @@ -7,6 +8,7 @@ from etools_datamart.apps.multitenant.views import SelectSchema urlpatterns = [ + path(r'me/', include('etools_datamart.apps.me.urls')), path(r's/', include('etools_datamart.apps.subscriptions.urls')), path(r'', include('etools_datamart.apps.web.urls')), path(r'', include('social_django.urls', namespace='social')), diff --git a/src/etools_datamart/config/wsgi.py b/src/etools_datamart/config/wsgi.py index 587cd113b..a2a89ef4d 100644 --- a/src/etools_datamart/config/wsgi.py +++ b/src/etools_datamart/config/wsgi.py @@ -17,5 +17,10 @@ from django.core.wsgi import get_wsgi_application +from whitenoise import WhiteNoise + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etools_datamart.config.settings") -application = get_wsgi_application() + +application = WhiteNoise(get_wsgi_application()) +# application = WhiteNoise(get_wsgi_application(), root=os.environ.get('STATIC_ROOT')) +# application.add_files('/path/to/more/static/files', prefix='more-files/') diff --git a/src/etools_datamart/libs/constance.py b/src/etools_datamart/libs/constance.py index 066a5905b..2c0578954 100644 --- a/src/etools_datamart/libs/constance.py +++ b/src/etools_datamart/libs/constance.py @@ -1,15 +1,16 @@ -from constance import config from django.contrib.auth.models import Group from django.forms import ChoiceField, HiddenInput, Select, Textarea, TextInput from django.template import Context, Template from django.utils.safestring import mark_safe +from constance import config + class GroupChoiceField(ChoiceField): def __init__(self, **kwargs): names = list(Group.objects.values_list('name', flat=True)) - choices = [[i, i] for i in names] + choices = zip(names, names) super().__init__(choices=choices, **kwargs) diff --git a/src/month_field/admin.py b/src/month_field/admin.py index 306259065..da3dcf354 100644 --- a/src/month_field/admin.py +++ b/src/month_field/admin.py @@ -2,6 +2,7 @@ from django.contrib.admin import FieldListFilter from django.utils.dates import MONTHS + from month_field.models import MonthField today = datetime.today() diff --git a/src/month_field/rest_framework.py b/src/month_field/rest_framework.py index 52b80f4c1..c5f339378 100644 --- a/src/month_field/rest_framework.py +++ b/src/month_field/rest_framework.py @@ -1,12 +1,13 @@ -from datetime import datetime +from datetime import date, datetime from functools import lru_cache -import coreapi -import coreschema -from coreapi.compat import force_text from django.forms import forms from django.template import loader from django.utils.dates import MONTHS_3 + +import coreapi +import coreschema +from coreapi.compat import force_text from drf_querystringfilter.exceptions import InvalidQueryValueError from month_field.forms import MonthField from rest_framework.filters import BaseFilterBackend @@ -16,19 +17,53 @@ class MonthForm(forms.Form): month = MonthField() +def clean(value): + months = list(MONTHS_3.values()) + if '-' in value: + m, y = value.split('-') + y = y or datetime.now().year + else: + m = value + y = datetime.now().year + + if m in months: + m = months.index(m) + 1 + elif m in list(map(str, range(1, 13))): + m = m + elif value == 'current': + m = datetime.now().month + y = datetime.now().year + else: + m = 0 + y = 0 + return int(m), int(y) + + class MonthFilterBackend(BaseFilterBackend): template = 'month_field/rest_framework/month_filtering.html' month_param = 'month' - def get_value(self, request, queryset, view): - return request.query_params.get(self.month_param) + def get_value(self, request): + raw_value = request.query_params.get(self.month_param, "") + return clean(raw_value) + + def get_form(self, request, view, context): + month, year = context['current'] + Frm = type("MonthForm", (forms.Form,), + {self.month_param: MonthField(label="Month", + required=False)}) + if month: + return Frm(initial={self.month_param: date(day=1, month=month, year=year)}) + else: + return Frm() def get_template_context(self, request, queryset, view): - current = self.get_value(request, queryset, view) + current = self.get_value(request) context = { 'request': request, - 'form': MonthForm(initial={'month': current}), + 'current': current, } + context['form'] = self.get_form(request, view, context) return context def to_html(self, request, queryset, view): @@ -57,25 +92,10 @@ def get_schema_fields(self, view): def filter_queryset(self, request, queryset, view): value = request.GET.get('month', "").lower() m = y = None - months = MONTHS_3.values() + if value: try: - if '-' in value: - m, y = value.split('-') - else: - m = value - y = datetime.now().year - - if m in months: - m = months.index(m) + 1 - elif m in list(map(str, range(12))): - m = m - elif value == 'current': - m = datetime.now().month - y = datetime.now().year - # elif value == 'latest': - # m = datetime.now().month - # y = datetime.now().year + m, y = clean(value) return queryset.filter(month__month=int(m), month__year=int(y)) except ValueError: diff --git a/src/month_field/templates/month_field/rest_framework/month_filtering.html b/src/month_field/templates/month_field/rest_framework/month_filtering.html index 9dab72c0a..5bd617355 100644 --- a/src/month_field/templates/month_field/rest_framework/month_filtering.html +++ b/src/month_field/templates/month_field/rest_framework/month_filtering.html @@ -1,15 +1,5 @@ -{% load rest_framework i18n %} -

      {% trans "Month" %}

      +{% load rest_framework i18n crispy_forms_tags %} +{#

      {% trans "Month" %}

      #}
      -
      - {{ form.as_p }} - -
      - + {{ form|crispy }}
      diff --git a/src/month_field/widgets.py b/src/month_field/widgets.py index afeb921d1..42b2e4878 100644 --- a/src/month_field/widgets.py +++ b/src/month_field/widgets.py @@ -14,9 +14,11 @@ class MonthSelectorWidget(widgets.MultiWidget): def __init__(self, attrs=None): # create choices for days, months, years + months = {"": "--"} + months.update(MONTHS) _attrs = attrs or {} # default class _attrs['style'] = 'width:20%;' - _widgets = [widgets.Select(attrs=_attrs, choices=MONTHS.items()), + _widgets = [widgets.Select(attrs=_attrs, choices=months.items()), widgets.NumberInput(attrs=_attrs) ] super(MonthSelectorWidget, self).__init__(_widgets, attrs) @@ -48,7 +50,7 @@ def value_from_datadict(self, data, files, name): try: D = date(day=1, month=int(datelist[0]), year=int(datelist[1])) - except ValueError: + except (ValueError, TypeError): return '' else: return str(D) diff --git a/src/unicef_rest_framework/admin/acl.py b/src/unicef_rest_framework/admin/acl.py index 66ffc0d75..75d715c42 100644 --- a/src/unicef_rest_framework/admin/acl.py +++ b/src/unicef_rest_framework/admin/acl.py @@ -2,7 +2,6 @@ import logging -from admin_extra_urls.extras import ExtraUrlMixin, link from django import forms from django.contrib import admin from django.contrib.admin import widgets @@ -10,6 +9,9 @@ from django.contrib.auth.models import Group from django.contrib.postgres.forms import SimpleArrayField from django.template.response import TemplateResponse + +from admin_extra_urls.extras import ExtraUrlMixin, link + from unicef_rest_framework.models import Service, UserAccessControl from unicef_rest_framework.models.acl import AbstractAccessControl, GroupAccessControl @@ -39,6 +41,7 @@ def get_queryset(self, request): class GroupAccessControlForm(forms.Form): + overwrite_existing = forms.BooleanField(help_text="Overwrite existing entries", required=False) group = forms.ModelChoiceField(queryset=Group.objects.all()) policy = forms.ChoiceField(choices=AbstractAccessControl.POLICIES) services = forms.ModelMultipleChoiceField(queryset=Service.objects.all(), @@ -61,6 +64,9 @@ class GroupAccessControlAdmin(ExtraUrlMixin, admin.ModelAdmin): def get_queryset(self, request): return super(GroupAccessControlAdmin, self).get_queryset(request).select_related(*self.raw_id_fields) + def has_add_permission(self, request): + return False + @link() def add_acl(self, request): opts = self.model._meta @@ -82,9 +88,18 @@ def add_acl(self, request): form = GroupAccessControlForm(request.POST) if form.is_valid(): services = form.cleaned_data.pop('services') + group = form.cleaned_data.pop('group') + overwrite_existing = form.cleaned_data.pop('overwrite_existing') + for service in services: - GroupAccessControl.objects.get_or_create(service=service, - **form.cleaned_data) + if overwrite_existing: + GroupAccessControl.objects.update_or_create(service=service, + group=group, + defaults=form.cleaned_data) + else: + GroupAccessControl.objects.update_or_create(service=service, + group=group, + defaults=form.cleaned_data) self.message_user(request, 'ACLs created') else: @@ -92,8 +107,8 @@ def add_acl(self, request): 'policy': AbstractAccessControl.POLICY_ALLOW, 'serializers': 'std'}) ctx['adminform'] = AdminForm(form, - [(None, {'fields': [['group', - 'policy'], + [(None, {'fields': ['overwrite_existing', + ['group', 'policy'], 'services', ['rate', 'serializers']]})], {}) diff --git a/src/unicef_rest_framework/admin/application.py b/src/unicef_rest_framework/admin/application.py index 678ba3513..242bfe94c 100644 --- a/src/unicef_rest_framework/admin/application.py +++ b/src/unicef_rest_framework/admin/application.py @@ -3,9 +3,11 @@ import logging import uuid -from admin_extra_urls.extras import ExtraUrlMixin from django import forms from django.contrib import admin + +from admin_extra_urls.extras import ExtraUrlMixin + from unicef_rest_framework.models import Application logger = logging.getLogger(__name__) diff --git a/src/unicef_rest_framework/admin/base.py b/src/unicef_rest_framework/admin/base.py index ceb2ff0db..71d814eef 100644 --- a/src/unicef_rest_framework/admin/base.py +++ b/src/unicef_rest_framework/admin/base.py @@ -2,9 +2,10 @@ import logging +from django.contrib import admin + from admin_extra_urls.extras import ExtraUrlMixin, link from admin_extra_urls.mixins import _confirm_action -from django.contrib import admin logger = logging.getLogger(__name__) diff --git a/src/unicef_rest_framework/admin/cache.py b/src/unicef_rest_framework/admin/cache.py index bb3216fa6..0adb52f76 100644 --- a/src/unicef_rest_framework/admin/cache.py +++ b/src/unicef_rest_framework/admin/cache.py @@ -4,7 +4,6 @@ import re import uuid -from admin_extra_urls.extras import action, ExtraUrlMixin, link from django import forms from django.contrib import admin from django.contrib.admin.templatetags.admin_urls import admin_urlname @@ -13,6 +12,9 @@ from django.http import HttpResponseRedirect from django.template.response import TemplateResponse from django.urls import reverse + +from admin_extra_urls.extras import action, ExtraUrlMixin, link + from unicef_rest_framework.cache import humanize_ttl, parse_ttl from unicef_rest_framework.forms import CacheVersionForm from unicef_rest_framework.models import Service @@ -28,9 +30,10 @@ class CacheVersionAdmin(ExtraUrlMixin, admin.ModelAdmin): list_display = ('name', 'cache_version', 'get_cache_ttl', 'cache_key') search_fields = ('name', 'viewset') actions = ['incr_version', 'reset_version', 'generate_cache_token'] - readonly_fields = ('cache_key', ) + readonly_fields = ('cache_key', 'name') list_filter = ('hidden',) form = CacheVersionForm + fieldsets = [("", {"fields": ('name', 'cache_version', 'cache_ttl', 'cache_key')})] def get_queryset(self, request): return super(CacheVersionAdmin, self).get_queryset(request). \ @@ -40,6 +43,9 @@ def get_queryset(self, request): def has_add_permission(self, request): return False + def has_delete_permission(self, request, obj=None): + return False + def get_cache_ttl(self, obj): if re.search(r'[smhdwy]', obj.cache_ttl): return "{} ({})".format(obj.cache_ttl, parse_ttl(obj.cache_ttl)) @@ -51,7 +57,7 @@ def get_cache_ttl(self, obj): @action(label='View Service') def goto_service(self, request, pk): - url = reverse("admin:core_service_change", args=[pk]) + url = reverse("admin:unicef_rest_framework_service_change", args=[pk]) return HttpResponseRedirect(url) @link(label='Reset cache', css_class="btn btn-danger", icon="fa fa-warning icon-white") @@ -89,12 +95,12 @@ def generate_cache_token_single(self, request, pk): self.generate_cache_token(request, Service.objects.filter(id=pk)) def incr_version(self, request, queryset): - queryset.update(version=F("cache_version") + 1) + queryset.update(cache_version=F("cache_version") + 1) incr_version.short_description = "Increment version" def reset_version(self, request, queryset): - queryset.update(version=1) + queryset.update(cache_version=1) incr_version.short_description = "Increment version" diff --git a/src/unicef_rest_framework/admin/filter.py b/src/unicef_rest_framework/admin/filter.py index 8bf55906d..b42b7fb0c 100644 --- a/src/unicef_rest_framework/admin/filter.py +++ b/src/unicef_rest_framework/admin/filter.py @@ -2,14 +2,15 @@ import logging -import requests -from admin_extra_urls.extras import action, ExtraUrlMixin from django.conf import settings from django.contrib import admin, messages from django.contrib.admin import TabularInline from django.http import HttpResponseRedirect from django.urls import reverse +import requests +from admin_extra_urls.extras import action, ExtraUrlMixin + from ..models.filter import SystemFilter, SystemFilterFieldRule logger = logging.getLogger(__name__) diff --git a/src/unicef_rest_framework/admin/service.py b/src/unicef_rest_framework/admin/service.py index 397d2a082..e89e1c3c1 100644 --- a/src/unicef_rest_framework/admin/service.py +++ b/src/unicef_rest_framework/admin/service.py @@ -4,14 +4,17 @@ import logging import os -from admin_extra_urls.extras import action, ExtraUrlMixin, link -from constance import config from django.contrib import admin from django.contrib.admin.templatetags.admin_urls import admin_urlname from django.http import HttpResponse, HttpResponseRedirect from django.urls import NoReverseMatch, reverse from django.utils.safestring import mark_safe + +from admin_extra_urls.extras import action, ExtraUrlMixin, link +from adminactions.mass_update import mass_update +from constance import config from strategy_field.utils import fqn, import_by_name + from unicef_rest_framework import acl from unicef_rest_framework.forms import ServiceForm from unicef_rest_framework.models import Service @@ -39,15 +42,23 @@ def get_stash_url(obj, label=None, **kwargs): class ServiceAdmin(ExtraUrlMixin, admin.ModelAdmin): - list_display = ('name', 'visible', 'security', 'cache_version', 'source_model', - 'json', 'admin') + list_display = ('name', 'visible', 'access', 'cache_version', 'suffix', 'json', 'admin') list_filter = ('hidden', 'access') search_fields = ('name', 'viewset') readonly_fields = ('cache_version', 'cache_ttl', 'cache_key', 'viewset', 'name', 'uuid', - 'last_modify_user') + 'last_modify_user', 'source_model', 'endpoint', 'basename', 'suffix') form = ServiceForm filter_horizontal = ('linked_models',) + fieldsets = [("", {"fields": ('name', + 'description', + ('access', 'hidden',), + # 'confidentiality', + ('source_model', 'basename'), + ('suffix', 'endpoint'), + 'linked_models', + )})] + actions = [mass_update, ] # change_list_template = 'admin/unicef_rest_framework/service/change_list.html' @@ -119,6 +130,22 @@ def refresh(self, request): def invalidate_all_cache(self, request): Service.objects.invalidate_cache() + @action() + def doc(self, request, pk): + service = Service.objects.get(pk=pk) + return HttpResponseRedirect(service.doc_url) + + @action() + def api(self, request, pk): + service = Service.objects.get(pk=pk) + return HttpResponseRedirect(service.endpoint) + + @action() + def crontab_expire(self, request, pk): + base = reverse("admin:django_celery_beat_crontabschedule_add") + url = f"{base}?" + return HttpResponseRedirect(url) + @action() def invalidate_cache(self, request, pk): service = Service.objects.get(pk=pk) diff --git a/src/unicef_rest_framework/apps.py b/src/unicef_rest_framework/apps.py index a694414c5..6971b6413 100644 --- a/src/unicef_rest_framework/apps.py +++ b/src/unicef_rest_framework/apps.py @@ -4,4 +4,7 @@ class Config(AppConfig): name = 'unicef_rest_framework' - label = 'API Configuration' + verbose_name = 'API Configuration' + + def ready(self): + from . import tasks # noqa diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index 73fdd3690..68206f047 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -1,13 +1,21 @@ import logging -from constance import config -from crashlog.middleware import process_exception +from django.conf import settings from django.contrib.auth import get_user_model, login +from django.contrib.auth.backends import ModelBackend from django.utils.translation import ugettext as _ + +from constance import config +from crashlog.middleware import process_exception from rest_framework import exceptions +from rest_framework.authentication import BaseAuthentication from rest_framework.exceptions import AuthenticationFailed, PermissionDenied from rest_framework_jwt import authentication -from unicef_security.azure import default_group, Synchronizer +from strategy_field.utils import fqn + +from unicef_rest_framework import acl +from unicef_rest_framework.config import conf +from unicef_security.graph import default_group, Synchronizer logger = logging.getLogger() @@ -16,6 +24,49 @@ def jwt_get_username_from_payload(payload): return payload.get('preferred_username', payload.get('unique_name')) +def get_client_ip(environ): + """ + Naively yank the first IP address in an X-Forwarded-For header + and assume this is correct. + + Note: Don't use this in security sensitive situations since this + value may be forged from a client. + """ + try: + return environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip() + except (KeyError, IndexError): + return environ.get('REMOTE_ADDR') + + +class URLTokenAuthentication(BaseAuthentication): + def authenticate(self, request): + return None + + +class AnonymousAuthentication(BaseAuthentication): + def authenticate(self, request): + view = request._request._view + service = view.get_service() + if service.access == acl.ACL_ACCESS_OPEN: + User = get_user_model() + user = User.objects.get_or_create(username='anonymous')[0] + request.user = user + login(request, user, fqn(ModelBackend)) + return (user, None) + + +class IPBasedAuthentication(BaseAuthentication): + def authenticate(self, request): + if settings.DEBUG: # pragma: no cover + ip = get_client_ip(request.META) + if ip in conf.FREE_AUTH_IPS: + User = get_user_model() + user = User.objects.get_or_create(username=ip, email=f'noreply@{ip}.org') + request.user = user + # login(request, user, 'social_core.backends.azuread_tenant.AzureADTenantOAuth2') + return (user, None) + + class JWTAuthentication(authentication.JSONWebTokenAuthentication): def authenticate(self, request): @@ -27,7 +78,7 @@ def authenticate(self, request): try: user, jwt_value = super(JWTAuthentication, self).authenticate(request) request.user = user - login(request, user, 'social_core.backends.azuread_tenant.AzureADTenantOAuth2') + # login(request, user, 'social_core.backends.azuread_tenant.AzureADTenantOAuth2') except TypeError: # pragma: no cover raise PermissionDenied(detail='No valid authentication provided') except AuthenticationFailed as e: # pragma: no cover @@ -40,7 +91,6 @@ def authenticate_credentials(self, payload): if not username: msg = _('Invalid payload.') raise exceptions.AuthenticationFailed(msg) - created = False try: user = User.objects.get_by_natural_key(username) except User.DoesNotExist: @@ -68,6 +118,3 @@ def authenticate_credentials(self, payload): msg = _('User account is disabled.') raise exceptions.AuthenticationFailed(msg) return user - - -# AQABAAIAAAC5una0EUFgTIF8ElaxtWjTE8f9OcsbHLLnobNloaTfC--E_fRoUrtiw2jul5yBV9rN3CO2C1BJ2IB99esAhsuRrzEowH3COPLFe5hkhovi4zfceFjwu6iSXpfgAFVGuo_fmep0osVwr0WkFzhWI5QEgNNnrf7d7gFm4iVC4gFE24R_JymglPADBvJIUMGAPHYg-IEyK1GKSkzpNSjJNZz6Pad_uVlDMrssFcrRqxKOJzqIhggLq7XQpJnmfUF5dJNdriDMkUjHBhDqlNpKTJZpnJg0jfIn7843kmKH0WXbJL0ss-tfgc_d8Q0240bdYXX6YSBV20NPx7MHy5V9i1RAtmr11cHBCw3uDuRriomgOhtIxTKYLox8iKYHbELA9Opvd-zLJm9krxoxlEHVO-PKl11No1mT8ZC83Ox37yxG5vrE7U7UxaLml9PmrjRZQoD1HvJ354IxZyP2pytYq2XhvIG_NDSDfuO5hwzPKb9F7G4Hytu96plKlu_yvdZ4Gghbp7z2sryeAiCnpYNlskGVUrQwF7BSHT73XuuOWeFelp-jn3tR4LQwqEGkg3zLqswcjbsRykSvS3cY6xTdBsCb7H70nygnhOgr_WlT9oY9KS2ElBVU-Q8OE8mkJ1rDV42hRb-haC7yzyUgtofbSQdVMIUgJRpuxYCrHNJ5oRsXmrWI0EVTdWFN25kwOMYPwOI8rzVf1oHikTQiHm3AN5wz0ill40IfjLB9niMEn4kntLDGJU1rIeALxv1s4lHxMZHpc1YEgLTf_3LnGtrsca3bIAA diff --git a/src/unicef_rest_framework/cache.py b/src/unicef_rest_framework/cache.py index e4144e0a8..24442ac8f 100644 --- a/src/unicef_rest_framework/cache.py +++ b/src/unicef_rest_framework/cache.py @@ -1,8 +1,10 @@ import re +import time from django.core.cache import caches from django.utils.http import quote_etag from django.utils.translation import ugettext as _ + from humanize.i18n import ngettext from humanize.time import date_and_delta from rest_framework_extensions.cache.decorators import CacheResponse @@ -11,6 +13,9 @@ from rest_framework_extensions.key_constructor.bits import KeyBitBase from rest_framework_extensions.key_constructor.constructors import KeyConstructor from rest_framework_extensions.settings import extensions_api_settings +from strategy_field.utils import fqn + +from unicef_rest_framework.models import SystemFilter cache = caches['default'] @@ -112,17 +117,41 @@ def get_data(self, params, view_instance, view_method, request, args, kwargs): return {'cache_version': str(version)} +class SystemFilterKeyBit(KeyBitBase): + def get_data(self, params, view_instance, view_method, request, args, kwargs): + flt = SystemFilter.objects.match(request, view_instance) + request._request._system_filters = flt + qs = flt.get_querystring() if flt else '' + request._request.api_info['system-filters'] = qs + return {'systemfilter': qs} + + +class QueryPathKeyBit(KeyBitBase): + def get_data(self, params, view_instance, view_method, request, args, kwargs): + return {'path': str(request.path)} + + +class DevelopKeyBit(KeyBitBase): + def get_data(self, params, view_instance, view_method, request, args, kwargs): + if 'disable-cache' in request.GET: + return {'dev': str(time.time())} + return {} + + class ListKeyConstructor(KeyConstructor): cache_version = CacheVersionKeyBit() - # system_filter = SystemFilterKeyBit() - + system_filter = SystemFilterKeyBit() + path = QueryPathKeyBit() unique_method_id = bits.UniqueMethodIdKeyBit() format = bits.FormatKeyBit() headers = bits.HeadersKeyBit(['Accept']) + dev = DevelopKeyBit() # language = bits.LanguageKeyBit() - list_sql_query = bits.ListSqlQueryKeyBit() + # list_sql_query = bits.ListSqlQueryKeyBit() # NEVER NEVER USE THIS + querystring = bits.QueryParamsKeyBit() - pagination = bits.PaginationKeyBit() + + # pagination = bits.PaginationKeyBit() def get_key(self, view_instance, view_method, request, args, kwargs): key = super().get_key(view_instance, view_method, request, args, kwargs) @@ -176,7 +205,12 @@ def process_cache_response(self, view_instance.store('cache-ttl', view_instance.get_service().cache_ttl) view_instance.store('service', view_instance.get_service()) - view_instance.store('view', view_instance) + view_instance.store('view', fqn(view_instance)) + # view_instance.store('_filters', request._filters) + # conn = connections['etools'] + # view_instance.store('_schemas', conn.schemas) + # view_instance.store('_countries', conn.schemas) + # view_instance.request._request.api_info['cache-ttl'] = view_instance.get_service().cache_ttl # view_instance.request._request.api_info['service'] = view_instance.get_service() # view_instance.request._request.api_info['view'] = fqn(view_instance) diff --git a/src/unicef_rest_framework/config.py b/src/unicef_rest_framework/config.py index 6d2b8e47f..39b3d3053 100644 --- a/src/unicef_rest_framework/config.py +++ b/src/unicef_rest_framework/config.py @@ -2,13 +2,16 @@ from django.core.exceptions import ImproperlyConfigured from django.core.signals import setting_changed from django.urls import get_callable + from strategy_field.utils import import_by_name + from unicef_rest_framework import acl class AppSettings(object): defaults = { 'API_CACHE': 'default', + 'FREE_AUTH_IPS': [], 'ROUTER': 'unicef_rest_framework.urls.router', 'DEFAULT_ACCESS': acl.ACL_ACCESS_LOGIN, 'get_current_user': 'get_current_user', diff --git a/src/unicef_rest_framework/ds.py b/src/unicef_rest_framework/ds.py new file mode 100644 index 000000000..e1d48fc04 --- /dev/null +++ b/src/unicef_rest_framework/ds.py @@ -0,0 +1,81 @@ +from django.forms import ChoiceField, forms +from django.template import loader + +import coreapi +import coreschema +from dynamic_serializer.core import DynamicSerializerMixin as DynamicSerializerMixinBase +from rest_framework.filters import BaseFilterBackend + + +class DynamicSerializerMixin(DynamicSerializerMixinBase): + def get_selected_serializer_name(self): + return self.request.query_params.get(self.serializer_field_param, 'std') + + +class SchemaSerializerField(coreschema.Enum): + + def __init__(self, view: DynamicSerializerMixin, **kwargs): + self.view = view + kwargs.setdefault('title', 'serializers') + kwargs.setdefault('description', self.build_description()) + super().__init__(list(view.serializers_fieldsets.keys()), **kwargs) + + def build_description(self): + defs = [] + names = [] + for k, v in self.view.serializers_fieldsets.items(): + names.append(k) + defs.append(f"""- **{k}**: {self.view.get_serializer_fields(k)} +""") + + description = f"""Define the set of fields to return. Allowed values are: + [{'*, *'.join(names)}*] + +{''.join(defs)} + """ + return description + + +class DynamicSerializerFilter(BaseFilterBackend): + # template = 'rest_framework/filters/search.html' + ordering_title = 'Serializer' + template = 'dynamic_serializer/select.html' + + def get_schema_fields(self, view): + ret = [] + if view.serializers_fieldsets: + ret.append(coreapi.Field( + name=view.serializer_field_param, + required=False, + location='query', + schema=SchemaSerializerField(view) + )) + return ret + + def filter_queryset(self, request, queryset, view): + return queryset + + def get_form(self, request, view): + choices = zip(view.serializers_fieldsets.keys(), view.serializers_fieldsets.keys()) + Frm = type("SerializerForm", (forms.Form,), + {view.serializer_field_param: ChoiceField( + label="Serializer", + choices=choices, + required=False)}) + return Frm(request.GET) + + def get_template_context(self, request, queryset, view): + current = view.get_selected_serializer_name() + context = {'request': request, + 'current': current, + 'form': self.get_form(request, view), + 'param': view.serializer_field_param, + } + context['options'] = view.serializers_fieldsets.keys() + return context + + def to_html(self, request, queryset, view): + if len(view.serializers_fieldsets.keys()) > 1: + template = loader.get_template(self.template) + context = self.get_template_context(request, queryset, view) + return template.render(context) diff --git a/src/unicef_rest_framework/exceptions.py b/src/unicef_rest_framework/exceptions.py index b731b6cb1..c20cadb17 100644 --- a/src/unicef_rest_framework/exceptions.py +++ b/src/unicef_rest_framework/exceptions.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ + from rest_framework import status from rest_framework.exceptions import APIException, AuthenticationFailed from rest_framework.response import Response diff --git a/src/unicef_rest_framework/filtering.py b/src/unicef_rest_framework/filtering.py index 81ebdba72..fb9f4ae20 100644 --- a/src/unicef_rest_framework/filtering.py +++ b/src/unicef_rest_framework/filtering.py @@ -3,6 +3,7 @@ from drf_querystringfilter.backend import QueryStringFilterBackend from rest_framework.filters import BaseFilterBackend + from unicef_rest_framework.models import SystemFilter logger = logging.getLogger(__name__) @@ -11,14 +12,11 @@ class SystemFilterBackend(BaseFilterBackend): def filter_queryset(self, request, queryset, view): - filters = {} - if request.user and request.user.is_authenticated: - filters['user'] = request.user - else: - return queryset - - filters['service'] = view.get_service() - + # _system_filters has been set by cache.SystemFilterKeyBit + # if hasattr(request._request, "_system_filters"): + # if request._request._system_filters: + # queryset = request._request._system_filters.filter_queryset(queryset) + # else: filter = SystemFilter.objects.match(request, view) if filter: queryset = filter.filter_queryset(queryset) diff --git a/src/unicef_rest_framework/forms.py b/src/unicef_rest_framework/forms.py index 18554b265..1875b8339 100644 --- a/src/unicef_rest_framework/forms.py +++ b/src/unicef_rest_framework/forms.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from django.core.exceptions import ValidationError from django.forms.models import ModelForm + from strategy_field.utils import import_by_name from .cache import parse_ttl diff --git a/src/unicef_rest_framework/management/commands/api-inspect.py b/src/unicef_rest_framework/management/commands/api-inspect.py new file mode 100644 index 000000000..56e7c6b71 --- /dev/null +++ b/src/unicef_rest_framework/management/commands/api-inspect.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +import logging + +from django.core.management import BaseCommand + +from unicef_rest_framework.config import conf +from unicef_rest_framework.models import Service + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + args = '' + help = '' + + def add_arguments(self, parser): + parser.add_argument( + '--all', + action='store_true', + dest='all', + default=False, + help='select all options but `demo`') + + def handle(self, *args, **options): + router = conf.ROUTER + + list_name = router.routes[0].name + for prefix, viewset, basename in router.registry: + service, ok = Service.objects.check_or_create(prefix, + viewset, + basename, + list_name.format(basename=basename) + ) + for service in Service.objects.order_by('name'): + print("{0.id:4} {0.name:30} {0.endpoint:40}".format(service)) diff --git a/src/unicef_rest_framework/migrations/0001_initial.py b/src/unicef_rest_framework/migrations/0001_initial.py index 06eaedc84..92b4cf6f5 100644 --- a/src/unicef_rest_framework/migrations/0001_initial.py +++ b/src/unicef_rest_framework/migrations/0001_initial.py @@ -1,12 +1,15 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 import uuid -import concurrency.fields import django.contrib.postgres.fields +import django.db.models.deletion +from django.db import migrations, models + +import concurrency.fields import strategy_field.fields + import unicef_rest_framework.models.acl -from django.db import migrations, models class Migration(migrations.Migration): @@ -60,6 +63,9 @@ class Migration(migrations.Migration): ('name', models.CharField(db_index=True, help_text='unique service name', max_length=100, unique=True)), ('description', models.TextField(blank=True, null=True)), ('viewset', strategy_field.fields.StrategyClassField(db_index=True, help_text='class FQN', unique=True)), + ('basename', models.CharField(help_text='viewset basename', max_length=200)), + ('suffix', models.CharField(help_text='url suffix', max_length=200)), + ('url_name', models.CharField(help_text='url name as per drf reverse', max_length=300)), ('access', models.IntegerField(choices=[(0, 'Open'), (2, 'Login required'), (4, 'Access Restricted'), (8, 'Business authorization needed')], default=2, help_text='Required privileges')), ('confidentiality', models.IntegerField(choices=[(1, 'Strictly Confidential'), (2, 'Confidential'), (3, 'Internal'), (4, 'Internal UN'), (5, 'Public')], default=3)), ('hidden', models.BooleanField(default=False)), @@ -68,6 +74,7 @@ class Migration(migrations.Migration): ('cache_key', models.CharField(blank=True, help_text='Key used to invalidate service cache', max_length=1000, null=True)), ], options={ + 'verbose_name_plural': 'Services', 'ordering': ('name',), }, ), @@ -113,4 +120,13 @@ class Migration(migrations.Migration): 'ordering': ('user', 'service'), }, ), + migrations.CreateModel( + name='UserServiceToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('token', models.CharField(max_length=1000)), + ('expires', models.DateField(blank=True, null=True)), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='unicef_rest_framework.Service')), + ], + ), ] diff --git a/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py b/src/unicef_rest_framework/migrations/0002_auto_20181221_1724.py similarity index 94% rename from src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py rename to src/unicef_rest_framework/migrations/0002_auto_20181221_1724.py index bf25f9213..732b34a24 100644 --- a/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py +++ b/src/unicef_rest_framework/migrations/0002_auto_20181221_1724.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.3 on 2018-11-29 08:24 +# Generated by Django 2.1.4 on 2018-12-21 17:24 import django.db.models.deletion from django.conf import settings @@ -10,13 +10,18 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('auth', '0009_alter_user_last_name_max_length'), + ('unicef_rest_framework', '0001_initial'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('auth', '0009_alter_user_last_name_max_length'), ('contenttypes', '0002_remove_content_type_name'), - ('unicef_rest_framework', '0001_initial'), ] operations = [ + migrations.AddField( + model_name='userservicetoken', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), migrations.AddField( model_name='useraccesscontrol', name='last_modify_user', @@ -131,7 +136,7 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='systemfilter', - unique_together={('service', 'group'), ('service', 'user')}, + unique_together={('service', 'user'), ('service', 'group')}, ), migrations.AlterUniqueTogether( name='groupaccesscontrol', diff --git a/src/unicef_rest_framework/models/__init__.py b/src/unicef_rest_framework/models/__init__.py index 9d8732d70..899b1a388 100644 --- a/src/unicef_rest_framework/models/__init__.py +++ b/src/unicef_rest_framework/models/__init__.py @@ -2,5 +2,6 @@ from .base import MasterDataModel # noqa from .acl import UserAccessControl # noqa from .application import Application # noqa +from .token import UserServiceToken # noqa from .service import Service, CacheVersion # noqa from .filter import SystemFilter, SystemFilterFieldRule # noqa diff --git a/src/unicef_rest_framework/models/acl.py b/src/unicef_rest_framework/models/acl.py index cd728ce0c..3ef6be06f 100644 --- a/src/unicef_rest_framework/models/acl.py +++ b/src/unicef_rest_framework/models/acl.py @@ -28,7 +28,7 @@ def default_serializer(): class AbstractAccessControl(MasterDataModel): POLICY_DENY = 0 POLICY_ALLOW = 1 - POLICY_DEFAULT = 2 + POLICY_DEFAULT = 2 # not sure this is really needed. POLICIES = ((POLICY_DENY, "Forbid"), (POLICY_ALLOW, "Grant"), diff --git a/src/unicef_rest_framework/models/base.py b/src/unicef_rest_framework/models/base.py index 1578f03ad..f789b02c1 100644 --- a/src/unicef_rest_framework/models/base.py +++ b/src/unicef_rest_framework/models/base.py @@ -3,12 +3,13 @@ import logging import uuid -from concurrency.fields import IntegerVersionField from django.conf import settings from django.db import models from django.db.models import UUIDField from django.utils.translation import ugettext_lazy as _ +from concurrency.fields import IntegerVersionField + logger = logging.getLogger(__name__) diff --git a/src/unicef_rest_framework/models/filter.py b/src/unicef_rest_framework/models/filter.py index 025b7077b..dd1df7bbb 100644 --- a/src/unicef_rest_framework/models/filter.py +++ b/src/unicef_rest_framework/models/filter.py @@ -6,6 +6,7 @@ from django.contrib.auth.models import Group from django.core.exceptions import FieldError, ValidationError from django.db import models + from strategy_field.utils import fqn from .application import Application @@ -42,11 +43,12 @@ def filter_queryset(self, queryset): class SystemFilterManager(models.Manager): @lru_cache() def match(self, request, view): - try: - return SystemFilter.objects.get(service=view.get_service(), - user=request.user) - except SystemFilter.DoesNotExist: - return None + if request.user and request.user.is_authenticated: + try: + return SystemFilter.objects.get(service=view.get_service(), + user=request.user) + except SystemFilter.DoesNotExist: + return None class SystemFilter(models.Model): @@ -58,8 +60,7 @@ class SystemFilter(models.Model): group = models.ForeignKey(Group, models.CASCADE, blank=True, null=True) service = models.ForeignKey(Service, models.CASCADE) description = models.TextField(blank=True) - handler = models.CharField(max_length=500, - default=fqn(SystemFilterHandler)) + handler = models.CharField(max_length=500, default=fqn(SystemFilterHandler)) objects = SystemFilterManager() diff --git a/src/unicef_rest_framework/models/service.py b/src/unicef_rest_framework/models/service.py index ee4b0fdde..32e296237 100644 --- a/src/unicef_rest_framework/models/service.py +++ b/src/unicef_rest_framework/models/service.py @@ -3,12 +3,13 @@ import logging from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ValidationError from django.db import models from django.db.models import F -from django.urls import reverse from django.utils.functional import cached_property + +from rest_framework.reverse import reverse from strategy_field.fields import StrategyClassField + from unicef_rest_framework.config import conf from .. import acl @@ -24,6 +25,9 @@ def invalidate_cache(self, **kwargs): service.viewset.get_service.cache_clear() def get_for_viewset(self, viewset): + return self.model.objects.get(viewset=viewset) + + def check_or_create(self, prefix, viewset, basename, url_name, ): name = getattr(viewset, 'label', viewset.__name__) source_model = ContentType.objects.get_for_model(viewset().get_queryset().model) service, isnew = self.model.objects.get_or_create(viewset=viewset, @@ -35,17 +39,22 @@ def get_for_viewset(self, viewset): 'description': getattr(viewset, '__doc__', ""), 'source_model': source_model }) - if not isnew: - service.source_model = source_model - service.save() - viewset.get_service.cache_clear() + + service.url_name = url_name + service.basename = basename + service.suffix = prefix + service.source_model = source_model + service.save() + viewset.get_service.cache_clear() return service, isnew def load_services(self): router = conf.ROUTER created = deleted = 0 + list_name = router.routes[0].name for prefix, viewset, basename in router.registry: - service, isnew = self.get_for_viewset(viewset) + service, isnew = self.check_or_create(prefix, viewset, basename, + list_name.format(basename=basename)) # try: if isnew: created += 1 @@ -68,7 +77,7 @@ def load_services(self): for service in self.model.objects.all(): try: assert service.viewset - except ValidationError: + except AssertionError: service.delete() deleted += 1 @@ -81,6 +90,9 @@ class Service(MasterDataModel): description = models.TextField(blank=True, null=True) viewset = StrategyClassField(help_text='class FQN', unique=True, db_index=True) + basename = models.CharField(max_length=200, help_text='viewset basename') + suffix = models.CharField(max_length=200, help_text='url suffix') + url_name = models.CharField(max_length=300, help_text='url name as per drf reverse') access = models.IntegerField(choices=[(k, v) for k, v in acl.ACL_LABELS.items()], default=acl.ACL_ACCESS_LOGIN, help_text="Required privileges") @@ -92,7 +104,6 @@ class Service(MasterDataModel): cache_version = models.IntegerField(default=1) cache_ttl = models.CharField(default='1d', max_length=5) - # cache_expire = models.TimeField(blank=True, null=True) cache_key = models.CharField(max_length=1000, null=True, blank=True, help_text='Key used to invalidate service cache') @@ -109,12 +120,14 @@ class Service(MasterDataModel): class Meta: ordering = ('name',) + verbose_name_plural = "Services" objects = ServiceManager() def invalidate_cache(self): Service.objects.invalidate_cache(id=self.pk) self.refresh_from_db() + return self.cache_version def reset_cache(self, value=0): Service.objects.filter(id=self.pk).update(cache_version=value) @@ -126,23 +139,21 @@ def get_access_level(self): @cached_property def endpoint(self): - for __, viewset, base_name in conf.ROUTER.registry: - if viewset == self.viewset: - return reverse(f'api:{base_name}-list', args=['v1']) - else: - return None + return reverse(f'api:{self.basename}-list', args=['latest']) @cached_property def display_name(self): return "{} ({})".format(self.viewset.__name__, self.viewset.source) + def doc_url(self): + base = '/api/+redoc/#operation/' + path = self.suffix.replace('/', '_') + return "{0}api_{1}_list".format(base, path) + @cached_property def managed_model(self): try: - v = self.viewset() - m = v.get_queryset().model - del v - return m + return self.source_model.model_class() except TypeError: return None @@ -153,16 +164,12 @@ def save(self, force_insert=False, force_update=False, using=None, update_fields model = v.get_queryset().model ct = ContentType.objects.get_for_model(model) self.linked_models.add(ct) - self.viewset._service = None except Exception as e: logger.exception(e) super(Service, self).save(force_insert, force_update, using, update_fields) - # self.invalidate_cache() - def __str__(self): return self.name - # return "Service:{} ({})".format(self.name, self.viewset) class CacheVersion(Service): diff --git a/src/unicef_rest_framework/models/token.py b/src/unicef_rest_framework/models/token.py new file mode 100644 index 000000000..04bd84cb9 --- /dev/null +++ b/src/unicef_rest_framework/models/token.py @@ -0,0 +1,11 @@ +from django.conf import settings +from django.db import models + +from .service import Service + + +class UserServiceToken(models.Model): + user = models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE) + service = models.ForeignKey(Service, models.CASCADE) + token = models.CharField(max_length=1000) + expires = models.DateField(blank=True, null=True) diff --git a/src/unicef_rest_framework/negotiation.py b/src/unicef_rest_framework/negotiation.py new file mode 100644 index 000000000..4b9543e8c --- /dev/null +++ b/src/unicef_rest_framework/negotiation.py @@ -0,0 +1,14 @@ +from rest_framework.negotiation import DefaultContentNegotiation + + +class CT(DefaultContentNegotiation): + + def select_renderer(self, request, renderers, format_suffix=None): + format_query_param = self.settings.URL_FORMAT_OVERRIDE + format = format_suffix or request.query_params.get(format_query_param) + if format == 'iqy': + for renderer in renderers: + if renderer.format == format: + return renderer, renderer.media_type + + return super().select_renderer(request, renderers, format_suffix) diff --git a/src/unicef_rest_framework/ordering.py b/src/unicef_rest_framework/ordering.py new file mode 100644 index 000000000..e238cf6fa --- /dev/null +++ b/src/unicef_rest_framework/ordering.py @@ -0,0 +1,20 @@ +from django import forms + +from rest_framework.filters import OrderingFilter as OrderingFilterBase + + +class OrderingFilter(OrderingFilterBase): + template = 'rest_framework/filters/filter_form.html' + + def get_form(self, request, view, context): + Frm = type("OrderForm", (forms.Form,), + {context['param']: forms.ChoiceField( + label="Ordering", + choices=context['options'], + required=False)}) + return Frm(request.GET) + + def get_template_context(self, request, queryset, view): + context = super().get_template_context(request, queryset, view) + context['form'] = self.get_form(request, view, context) + return context diff --git a/src/unicef_rest_framework/pagination.py b/src/unicef_rest_framework/pagination.py index 0ede5b9ca..3104b4349 100644 --- a/src/unicef_rest_framework/pagination.py +++ b/src/unicef_rest_framework/pagination.py @@ -1,11 +1,52 @@ +import sys from collections import OrderedDict +from django import forms +from django.template import loader + from rest_framework import serializers +from rest_framework.filters import BaseFilterBackend from rest_framework.pagination import CursorPagination, PageNumberPagination from rest_framework.response import Response from rest_framework.utils.urls import remove_query_param, replace_query_param +class PageFilter(BaseFilterBackend): + template = 'rest_framework/filters/paging.html' + pagination_param = 'page_size' + + def get_form(self, request, view): + Frm = type("SerializerForm", (forms.Form,), + {view.pagination_class.page_size_query_param: forms.IntegerField(label="Page Size", required=False)}) + return Frm(request.GET, + initial={view.pagination_class.page_size_query_param: self.get_pagination(request, view)}) + + def filter_queryset(self, request, queryset, view): + return queryset + + def get_pagination(self, request, view): + ps = request.query_params.get(view.pagination_class.page_size_query_param) + return ps or view.pagination_class.page_size + + def get_template_context(self, request, queryset, view): + current = self.get_pagination(request, view) + context = {'request': request, + 'current': current, + 'form': self.get_form(request, view), + 'param': view.pagination_class.page_size_query_param, + } + return context + + def to_html(self, request, queryset, view): + if view.pagination_class: + template = loader.get_template(self.template) + context = self.get_template_context(request, queryset, view) + return template.render(context) + + def get_default_pagination(self, view): + return view.pagination_class.page_size + + class CurrentPageField(serializers.Field): page_field = 'page' @@ -28,31 +69,49 @@ class APIPagination(PageNumberPagination): # Client can control the page size using this query parameter. # Default is 'None'. Set to eg 'page_size' to enable usage. - page_size_query_param = None + page_size_query_param = 'page_size' # Set to an integer to limit the maximum page size the client may request. # Only relevant if 'page_size_query_param' has also been set. - max_page_size = 1000 + max_page_size = 10000 last_page_strings = ('last',) def get_paginated_response(self, data): - return Response(OrderedDict([ - ('count', self.page.paginator.count), - ('next', self.get_next_link()), - ('current_page', self.page.number), - ('total_pages', self.page.paginator.num_pages), - ('previous', self.get_previous_link()), - ('results', data) - ])) + if hasattr(self, 'page'): + return Response(OrderedDict([ + ('count', self.page.paginator.count), + ('next', self.get_next_link()), + ('current_page', self.page.number), + ('total_pages', self.page.paginator.num_pages), + ('previous', self.get_previous_link()), + ('results', data) + ])) + else: + return Response(OrderedDict([ + ('count', len(data)), + ('next', 'N/A'), + ('current_page', 1), + ('total_pages', 1), + ('previous', 'N/A'), + ('results', data) + ])) def get_page_size(self, request): - if request._request.GET.get('format') == 'csv': - return 999999999 + if request._request.GET.get('format') == ['csv', 'iqy', 'xlsx']: + return sys.maxsize + try: + desired = request.query_params[self.page_size_query_param] + if desired == "-1": + return sys.maxsize + except (KeyError, ValueError): + pass return super().get_page_size(request) def paginate_queryset(self, queryset, request, view=None): # self._handle_backwards_compat(view) + if self.get_page_size(request) == sys.maxsize: + return queryset return super(APIPagination, self).paginate_queryset(queryset, request, view) def get_next_link(self): diff --git a/src/unicef_rest_framework/permissions.py b/src/unicef_rest_framework/permissions.py index b26e80672..b4d24058c 100644 --- a/src/unicef_rest_framework/permissions.py +++ b/src/unicef_rest_framework/permissions.py @@ -4,6 +4,8 @@ from rest_framework.exceptions import PermissionDenied from rest_framework.permissions import BasePermission from strategy_field.utils import fqn + +from unicef_rest_framework.acl import ACL_ACCESS_OPEN from unicef_rest_framework.models import UserAccessControl from unicef_rest_framework.models.acl import AbstractAccessControl, GroupAccessControl @@ -11,7 +13,7 @@ class ServicePermission(BasePermission): - serializer_field = "+serializer" + # serializer_field = "-serializer" def get_acl(self, request, view): try: @@ -34,7 +36,7 @@ def has_permission(self, request, view): logger.error(f"Access denied for user '{request.user}' to '{fqn(view)}'") raise PermissionDenied - requested_serializer = request.GET.get(self.serializer_field, "std") + requested_serializer = request.GET.get(view.serializer_field_param, "std") if (requested_serializer not in acl.serializers) and ("*" not in acl.serializers): logger.error( @@ -43,5 +45,8 @@ def has_permission(self, request, view): return True except (GroupAccessControl.DoesNotExist): + service = view.get_service() + if service.access == ACL_ACCESS_OPEN: + return True logger.error(f"User '{request.user}' does not have grants for '{fqn(view)}'") return False diff --git a/src/unicef_rest_framework/renderers/__init__.py b/src/unicef_rest_framework/renderers/__init__.py index 8cc3dadc9..bc2240b38 100644 --- a/src/unicef_rest_framework/renderers/__init__.py +++ b/src/unicef_rest_framework/renderers/__init__.py @@ -1,6 +1,8 @@ -from .api import APIBrowsableAPIRenderer # noqa +from .api import URFBrowsableAPIRenderer # noqa from .microsoft.json import MSJSONRenderer # noqa from .microsoft.xml import MSXmlRenderer # noqa from .xls import XLSXRenderer # noqa from .html import HTMLRenderer # noqa from .pdf import PDFRenderer # noqa +from .txt import TextRenderer # noqa +from .iqy import IQYRenderer # noqa diff --git a/src/unicef_rest_framework/renderers/api.py b/src/unicef_rest_framework/renderers/api.py index 54b60c69c..461d15777 100644 --- a/src/unicef_rest_framework/renderers/api.py +++ b/src/unicef_rest_framework/renderers/api.py @@ -1,28 +1,83 @@ # -*- coding: utf-8 -*- import logging +from django.template import loader + from rest_framework.renderers import BrowsableAPIRenderer as _BrowsableAPIRenderer from rest_framework.reverse import reverse logger = logging.getLogger(__name__) -class APIBrowsableAPIRenderer(_BrowsableAPIRenderer): +class URFBrowsableAPIRenderer(_BrowsableAPIRenderer): template = 'rest_framework/api.html' + filter_template = 'rest_framework/filter_template.html' + + def get_filter_form(self, data, view, request): + if not hasattr(view, 'get_queryset') or not hasattr(view, 'filter_backends'): + return + + # Infer if this is a list view or not. + paginator = getattr(view, 'paginator', None) + if isinstance(data, list): + pass + elif paginator is not None and data is not None: + try: + paginator.get_results(data) + except (TypeError, KeyError): + return + elif not isinstance(data, list): + return + + queryset = view.get_queryset() + elements = [] + for backend in view.get_filter_backends(): + if hasattr(backend, 'to_html'): + html = backend().to_html(request, queryset, view) + if html: + elements.append(html) + + if not elements: + return + + template = loader.get_template(self.filter_template) + context = {'elements': elements} + return template.render(context) def get_context(self, data, accepted_media_type, renderer_context): - ctx = super(APIBrowsableAPIRenderer, self).get_context(data, accepted_media_type, renderer_context) + ctx = super(URFBrowsableAPIRenderer, self).get_context(data, accepted_media_type, renderer_context) # in the real flow, this is added by the MultiTenant Middleware # but this function is called before the middleware system is involved request = ctx['request'] - for key, value in request.api_info.items(): - ctx['response_headers'][key] = request.api_info.str(key) + view = ctx['view'] + for key, value in sorted(request.api_info.items()): + if key not in ['cache-hit']: + ctx['response_headers'][key] = request.api_info.str(key) + # ctx['response_headers']['ordering'] = getattr(view, 'ordering_fields', '') + ctx['response_headers']['serializers'] = ", ".join(getattr(view, 'serializers_fieldsets', {}).keys()) + # ctx['response_headers']['filters'] = getattr(view, 'filter_fields', '') + + ctx['extra_actions'] = view.get_extra_action_url_map() + ctx['base_action'] = reverse(f'api:{view.basename}-list', args=['latest']) if request.user.is_staff: + try: + service = view.get_service() + service_url = reverse(f'admin:unicef_rest_framework_service_change', args=[service.pk]) + ctx['service_url'] = service_url + ctx['service'] = service + except Exception: # pragma: no cover + pass try: model = ctx['view'].queryset.model + # model = service.managed_model admin_url = reverse(f'admin:{model._meta.app_label}_{model._meta.model_name}_changelist') ctx['admin_url'] = admin_url except Exception: # pragma: no cover pass + + try: + ctx['iqy_url'] = ctx['extra_actions'].pop('Iqy') + except Exception: # pragma: no cover + pass return ctx diff --git a/src/unicef_rest_framework/renderers/csv.py b/src/unicef_rest_framework/renderers/csv.py index 73b03479b..973db4126 100644 --- a/src/unicef_rest_framework/renderers/csv.py +++ b/src/unicef_rest_framework/renderers/csv.py @@ -9,9 +9,13 @@ class CSVRenderer(r.CSVRenderer): def render(self, data, media_type=None, renderer_context=None, writer_opts=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: - data = dict(data)['results'] - return super().render(data, media_type, renderer_context or {}, writer_opts) + if data and 'results' in data: + data = dict(data)['results'] + return super().render(data, media_type, renderer_context, writer_opts) except Exception as e: process_exception(e) logger.exception(e) diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index 59627a407..e15e72702 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -1,7 +1,8 @@ import logging -from crashlog.middleware import process_exception from django.template import loader + +from crashlog.middleware import process_exception from rest_framework.renderers import BaseRenderer logger = logging.getLogger(__name__) @@ -20,18 +21,24 @@ class HTMLRenderer(BaseRenderer): def get_template(self, meta): return loader.select_template([ f'renderers/html/{meta.app_label}/{meta.model_name}.html', - 'renderers/html/html.html']) + f'renderers/html/{meta.app_label}/html.html', + 'renderers/html.html']) def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: model = renderer_context['view'].queryset.model opts = model._meta template = self.get_template(opts) - if data['results']: + if data and 'results' in data: + data = data['results'] + if data: c = {'data': data, 'model': model, 'opts': opts, - 'headers': [labelize(v) for v in data['results'][0].keys()]} + 'headers': [labelize(v) for v in data[0].keys()]} else: c = {'data': {}, 'model': model, @@ -41,4 +48,4 @@ def render(self, data, accepted_media_type=None, renderer_context=None): except Exception as e: process_exception(e) logger.exception(e) - raise Exception('Error processing request') + raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/iqy.py b/src/unicef_rest_framework/renderers/iqy.py new file mode 100644 index 000000000..d347cd519 --- /dev/null +++ b/src/unicef_rest_framework/renderers/iqy.py @@ -0,0 +1,51 @@ +import logging + +from django.template import loader + +from crashlog.middleware import process_exception +from rest_framework.renderers import BaseRenderer + +logger = logging.getLogger(__name__) + + +def labelize(v): + return v.replace("_", " ").title() + + +class IQYRenderer(BaseRenderer): + media_type = 'text/plain' + format = 'iqy' + charset = 'utf-8' + render_style = 'text' + + def get_template(self, meta): + return loader.select_template([ + f'renderers/iqy/{meta.app_label}/{meta.model_name}.txt', + f'renderers/iqy/{meta.app_label}/iqy.txt', + 'renderers/iqy.txt']) + + def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' + try: + model = renderer_context['view'].queryset.model + opts = model._meta + template = self.get_template(opts) + if data and 'results' in data: + data = data['results'] + if data: + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': [labelize(v) for v in data[0].keys()]} + else: + c = {'data': {}, + 'model': model, + 'opts': opts, + 'headers': []} + return template.render(c) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/microsoft/xml.py b/src/unicef_rest_framework/renderers/microsoft/xml.py index 9e4e409ef..474e7c6ac 100644 --- a/src/unicef_rest_framework/renderers/microsoft/xml.py +++ b/src/unicef_rest_framework/renderers/microsoft/xml.py @@ -1,4 +1,5 @@ from django.utils.encoding import smart_text + from rest_framework_xml.renderers import XMLRenderer diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index 416e61531..91d277784 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -2,9 +2,10 @@ import logging import os -from crashlog.middleware import process_exception from django.conf import settings from django.template import loader + +from crashlog.middleware import process_exception from xhtml2pdf import pisa from .html import HTMLRenderer @@ -48,9 +49,13 @@ class PDFRenderer(HTMLRenderer): def get_template(self, meta): return loader.select_template([ f'renderers/pdf/{meta.app_label}/{meta.model_name}.html', - 'renderers/pdf/pdf.html']) + f'renderers/pdf/{meta.app_label}/pdf.html', + 'renderers/pdf.html']) def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) @@ -65,4 +70,4 @@ def render(self, data, accepted_media_type=None, renderer_context=None): except Exception as e: process_exception(e) logger.exception(e) - raise Exception('Error processing request') + raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/txt.py b/src/unicef_rest_framework/renderers/txt.py new file mode 100644 index 000000000..bdd386fd9 --- /dev/null +++ b/src/unicef_rest_framework/renderers/txt.py @@ -0,0 +1,51 @@ +import logging + +from django.template import loader + +from crashlog.middleware import process_exception +from rest_framework.renderers import BaseRenderer + +logger = logging.getLogger(__name__) + + +def labelize(v): + return v.replace("_", " ").title() + + +class TextRenderer(BaseRenderer): + media_type = 'text/plain' + format = 'txt' + charset = 'utf-8' + render_style = 'text' + + def get_template(self, meta): + return loader.select_template([ + f'renderers/text/{meta.app_label}/{meta.model_name}.txt', + f'renderers/text/{meta.app_label}/text.txt', + 'renderers/text.txt']) + + def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' + try: + model = renderer_context['view'].queryset.model + opts = model._meta + template = self.get_template(opts) + if data and 'results' in data: + data = data['results'] + if data: + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': [labelize(v) for v in data[0].keys()]} + else: + c = {'data': {}, + 'model': model, + 'opts': opts, + 'headers': []} + return template.render(c) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/xls.py b/src/unicef_rest_framework/renderers/xls.py index a70a53918..4e9587059 100644 --- a/src/unicef_rest_framework/renderers/xls.py +++ b/src/unicef_rest_framework/renderers/xls.py @@ -9,11 +9,16 @@ class XLSXRenderer(_XLSXRenderer): def render(self, data, accepted_media_type=None, renderer_context=None): + response = renderer_context['response'] + if response.status_code != 200: + return '' try: - if not data['results']: - return '' + # if data and 'results' in data: + # data = data['results'] + # if not data: + # return '' return super().render(data, accepted_media_type, renderer_context) except Exception as e: process_exception(e) logger.exception(e) - raise Exception('Error processing request') + raise Exception(f'Error processing request {e}') from e diff --git a/src/unicef_rest_framework/routers.py b/src/unicef_rest_framework/routers.py index 52f8c0bf5..25f7dff6c 100644 --- a/src/unicef_rest_framework/routers.py +++ b/src/unicef_rest_framework/routers.py @@ -1,14 +1,57 @@ # -*- coding: utf-8 -*- import logging +from collections import OrderedDict -from rest_framework import routers +from django.urls import NoReverseMatch + +from rest_framework import routers, views +from rest_framework.response import Response +from rest_framework.reverse import reverse from rest_framework.routers import DynamicRoute, Route +from unicef_rest_framework.models import Service + logger = logging.getLogger(__name__) +class APIRootView(views.APIView): + """ + The default basic root view for DefaultRouter + """ + _ignore_model_permissions = True + schema = None # exclude from schema + api_root_dict = None + + def get(self, request, *args, **kwargs): + # Return a plain {"name": "hyperlink"} response. + ret = OrderedDict() + namespace = request.resolver_match.namespace + for key, url_name in self.api_root_dict.items(): + hidden = Service.objects.get(suffix=key).hidden + if hidden: + if request.user.is_superuser: + key = f" * {key}" + else: + continue + if namespace: + url_name = namespace + ':' + url_name + try: + ret[key] = reverse( + url_name, + args=args, + kwargs=kwargs, + request=request, + format=kwargs.get('format', None) + ) + except NoReverseMatch: + # Don't bail out if eg. no list routes exist, only detail routes. + continue + + return Response(ret) + + class APIRouter(routers.DefaultRouter): - pass + APIRootView = APIRootView class APIReadOnlyRouter(APIRouter): @@ -51,9 +94,13 @@ class APIReadOnlyRouter(APIRouter): ), ] - # def get_urls(self): - # urls = super(ReadOnlyRouter, self).get_urls() - # view = self.get_api_root_view(api_urls=urls) - # root_url = url(r'^$', view, name=self.root_view_name) - # urls.append(root_url) - # return urls + def get_api_root_view(self, api_urls=None): + """ + Return a basic root view. + """ + api_root_dict = OrderedDict() + list_name = self.routes[0].name + for prefix, viewset, basename in self.registry: + api_root_dict[prefix] = list_name.format(basename=basename) + + return self.APIRootView.as_view(api_root_dict=api_root_dict) diff --git a/src/unicef_rest_framework/tasks.py b/src/unicef_rest_framework/tasks.py new file mode 100644 index 000000000..073963971 --- /dev/null +++ b/src/unicef_rest_framework/tasks.py @@ -0,0 +1,11 @@ +from celery.app import default_app + +from unicef_rest_framework.models import Service + + +@default_app.task() +def invalidate_cache(service_id): + service = Service.objects.get(service_id) + service.invalidate_cache() + return {"cache_versipn": service.cache_version, + "service": service.name} diff --git a/src/unicef_rest_framework/templates/dynamic_serializer/filter.html b/src/unicef_rest_framework/templates/dynamic_serializer/filter.html new file mode 100644 index 000000000..86ba4a779 --- /dev/null +++ b/src/unicef_rest_framework/templates/dynamic_serializer/filter.html @@ -0,0 +1,16 @@ +{% load rest_framework %} +{% load i18n %} +{% if options %} +

      {% trans "Serializer" %}

      +
      + {% for key in options %} + {% if key == current %} + + {{ key }} + + {% else %} + {{ key }} + {% endif %} + {% endfor %} +
      +{% endif %} diff --git a/src/unicef_rest_framework/templates/dynamic_serializer/select.html b/src/unicef_rest_framework/templates/dynamic_serializer/select.html new file mode 100644 index 000000000..5bb19afa7 --- /dev/null +++ b/src/unicef_rest_framework/templates/dynamic_serializer/select.html @@ -0,0 +1,10 @@ +{% load rest_framework crispy_forms_tags %} +{% load i18n %} +{% if title %}

      {% trans title %}

      {% endif %} +
      + {{ form|crispy }} +{# #} +
      diff --git a/src/unicef_rest_framework/templates/renderers/html/html.html b/src/unicef_rest_framework/templates/renderers/html.html similarity index 96% rename from src/unicef_rest_framework/templates/renderers/html/html.html rename to src/unicef_rest_framework/templates/renderers/html.html index a807290c9..d77cb7c80 100644 --- a/src/unicef_rest_framework/templates/renderers/html/html.html +++ b/src/unicef_rest_framework/templates/renderers/html.html @@ -33,7 +33,7 @@

      {{ opts.verbose_name }}

      {{ v }} {% endfor %} - {% for row in data.results %} + {% for row in data %} {% for k,v in row.items %} {{ v }} diff --git a/src/unicef_rest_framework/templates/renderers/iqy.txt b/src/unicef_rest_framework/templates/renderers/iqy.txt new file mode 100644 index 000000000..33648c01b --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/iqy.txt @@ -0,0 +1,2 @@ +{% for v in headers %}{% endfor %}{% for row in data %} +{% for k,v in row.items %}{% endfor %}{% endfor %}
      {{ v }}
      {{ v }}
      diff --git a/src/unicef_rest_framework/templates/renderers/pdf/pdf.html b/src/unicef_rest_framework/templates/renderers/pdf.html similarity index 96% rename from src/unicef_rest_framework/templates/renderers/pdf/pdf.html rename to src/unicef_rest_framework/templates/renderers/pdf.html index f989b316b..d49415cc9 100644 --- a/src/unicef_rest_framework/templates/renderers/pdf/pdf.html +++ b/src/unicef_rest_framework/templates/renderers/pdf.html @@ -34,7 +34,7 @@

      {{ opts.verbose_name }}

      {{ v }} {% endfor %} - {% for row in data.results %} + {% for row in data %} {% for k,v in row.items %} {{ v }} diff --git a/src/unicef_rest_framework/templates/renderers/text.txt b/src/unicef_rest_framework/templates/renderers/text.txt new file mode 100644 index 000000000..4a7bda5f3 --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/text.txt @@ -0,0 +1,2 @@ +{% for v in headers %}{{ v }} {% endfor %}{% for row in data %} +{% for k,v in row.items %}{{ v }} {% endfor %}{% endfor %} diff --git a/src/unicef_rest_framework/templates/rest_framework/base.html b/src/unicef_rest_framework/templates/rest_framework/base.html index 4751d9b04..2189f776a 100644 --- a/src/unicef_rest_framework/templates/rest_framework/base.html +++ b/src/unicef_rest_framework/templates/rest_framework/base.html @@ -1,10 +1,11 @@ {% load static %} {% load i18n %} {% load rest_framework %} +{% load query %} - + {% block head %} {% block meta %} @@ -19,14 +20,36 @@ {% endblock %} - + {% if code_style %}{% endif %} {% endblock %} - + {% if config.ANALYTICS_CODE %} + + + {% endif %} {% endblock %} + {% block body %} @@ -39,7 +62,7 @@
      {% block branding %} - + UNICEF REST Framework {% endblock %} @@ -77,6 +100,34 @@ {% block content %}
      + {% if service %} +
      + Doc +
      + {% endif %} + {% if iqy_url %} +
      +
      +
      + IQY + + +   +
      +
      +
      + {% endif %} + {% if 'GET' in allowed_methods %}
      @@ -107,6 +158,7 @@ {% endif %} + {% if delete_form %} @@ -145,13 +197,30 @@

      {{ name }}

      {{ description }} {% endblock %}
      - {% if admin_url %} Admin + class="btn btn-primary">Data + {% endif %} + {% if service_url %} + Service + {% for name,url in extra_actions.items %} + {% if url != request.build_absolute_uri %} + {{ name }} + {% endif %} + {% endfor %} + {% if request.path != base_action %} + List + {% endif %} {% endif %} + {% if paginator %}