diff --git a/.gitignore b/.gitignore index 65c3d93b0..74390e2d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.DS_Store *.egg-info *.pyo *.sqlite @@ -6,8 +5,12 @@ *credentials.* .cache .coverage +.DS_Store +.env .idea +.pydevproject .tox +.venv /contacts.json /dist /media @@ -16,23 +19,21 @@ __pycache__ CACHE celerybeat-schedule coverage. -xml +db/clean.sql +db/db1.bz2 +db/etools.dump +db/public.sqldump +db/tenant.sql djcelery.schedulers.DatabaseScheduler +docker/.envrc +docker/cache +docker/superset.db LOCAL local.yaml pytest.xml settings_local.py -~* -.pydevproject -db/db1.bz2 -.env -.venv -docker/cache src/etools_datamart/apps/core/static/api-doc.css.map -docker/superset.db -db/clean.sql -db/etools.dump -db/public.sqldump -db/tenant.sql src/etools_datamart/apps/etools/models/public_old.py src/etools_datamart/apps/etools/models/tenant_old.py +xml +~* diff --git a/CHANGES b/CHANGES index 29824c8ee..25efa5264 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,18 @@ +1.7 +--- +* WARNINGS: migration reset +* add YAML format +* new 'Monitor' page to check dataset last update date +* add ability to intercept changed/unchanged datamart records +* Azure storage support +* use versioning in API urls +* add email notifications on dataset changes +* add filter by 'last_modify_date' +* add 'updates/' endpoint to fetch only changes from last ETL +* add XHTML renderer +* add PDF renderer + + 1.6 --- * add ability to invalidate cache directly from admin endpoint diff --git a/Makefile b/Makefile index f8a668d29..ddf55f8bd 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,6 @@ fullclean: rm -fr .tox .cache .pytest_cache .venv $(MAKE) clean - sync-etools: sh src/etools_datamart/apps/multitenant/postgresql/dump.sh ${PG_ETOOLS_PARAMS} @@ -56,6 +55,18 @@ ifdef BROWSE firefox ${BUILDDIR}/docs/index.html endif - urf: pipenv run pytest tests/urf --cov-config tests/urf/.coveragerc + + +demo: + PYTHONPATH=./src pipenv run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid & + PYTHONPATH=./src pipenv run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid & + PYTHONPATH=./src pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid & + pipenv run docker run -d -p 5555:5555 -e CELERY_BROKER_URL=$CELERY_BROKER_URL --name datamart-flower --rm saxix/flower + +stop-demo: + - kill `cat gunicorn.pid` + - kill `cat beat.pid` + - kill `cat celery.pid` + - docker stop datamart-flower diff --git a/Pipfile b/Pipfile index d08c7890a..b84c8130c 100644 --- a/Pipfile +++ b/Pipfile @@ -37,7 +37,7 @@ django-model-utils = "*" python-social-auth = "*" social-auth-app-django = "*" django-db-logging = "*" -cryptography = "*" +cryptography = "==2.4.1" #rest-social-auth = "*" "django-rest-framework-social-oauth2" = "*" django-countries = "*" @@ -46,6 +46,16 @@ drf-renderer-xlsx = "*" django-redisboard = "*" djangorestframework-xml = "*" redis = "==2.10.6" +djangorestframework-yaml = "*" +django-storages = {extras = ["azure"], version = "*"} +onedrivesdk = "*" +azure-storage = "*" +django-basicauth = "*" +django-post-office = "*" +django-celery-email = "*" +"xhtml2pdf" = "*" +pisa = "*" +django-crispy-forms = "*" [dev-packages] "flake8" = ">=3.6.0" @@ -58,7 +68,6 @@ factory-boy = "*" ipython = "*" isort = "*" pdbpp = "*" -pre-commit = "*" pytest = "*" pytest-coverage = "*" pytest-django = "*" @@ -66,6 +75,10 @@ pytest-echo = "*" pytest-pythonpath = "*" yapf = "*" vcrpy = "*" +pre-commit = "*" +freezegun = "*" +pytest-ignore-flaky = "*" +bumpversion = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index b549503fc..2b97db8c8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "248d5d3c8b7361358b8f71a0951e19f52b3abd72335b3252070dab04b3e5165a" + "sha256": "2c79b29b6f0397ba9724ed2ec50a286a3178b5b335f0a3b70ff3594a60dc9a9d" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,13 @@ ] }, "default": { + "adal": { + "hashes": [ + "sha256:ba52913c38d76b4a4d88eaab41a5763d056ab6d073f106e0605b051ab930f5c1", + "sha256:bf79392b8e9e5e82aa6acac3835ba58bbac0ccf7e15befa215863f83d5f6a007" + ], + "version": "==1.2.0" + }, "admin-extra-urls": { "hashes": [ "sha256:cd5c77c3fcde240472bc9a6c79f8c2358ad287c243e6336902c4996c38212b92" @@ -37,6 +44,593 @@ ], "version": "==0.24.0" }, + "azure": { + "hashes": [ + "sha256:7d6afa332fccffe1a9390bcfac5122317eec657c6029f144d794603a81cd0e50", + "sha256:8bee1f569f700519e8cdfe210259c5c51fa96cee69ccae6b06f60d7224e334e6" + ], + "markers": "extra == 'azure'", + "version": "==4.0.0" + }, + "azure-applicationinsights": { + "hashes": [ + "sha256:6e1839169bb6ffd2d2c21ee3f4afbdd068ea428ad47cf884ea3167ecf7fd0859", + "sha256:c007a9cb7cee4852673e915220fca267e78f97f275f1a8f09b2266a55702462b" + ], + "version": "==0.1.0" + }, + "azure-batch": { + "hashes": [ + "sha256:017be21a9e6db92473d2e33170d5dd445596fc70d706f73552ac9c6b57a6ef1c", + "sha256:cd71c7ebb5beab174b6225bbf79ae18d6db0c8d63227a7e514da0a75f138364c" + ], + "version": "==4.1.3" + }, + "azure-common": { + "hashes": [ + "sha256:2606ae77ff81c0036965b92ec2efe03eaec02a66714140ca0f7aa401b8b9bbb0", + "sha256:c908621a71eb4ee9fab0962e35d3c27a18f09a854d8359c2f32c15b3f4fc576e" + ], + "version": "==1.1.16" + }, + "azure-cosmosdb-nspkg": { + "hashes": [ + "sha256:5d83961922812ffb6e23f7321550b2916abbd7480ec5b4a798417c4a682ff5e9", + "sha256:acf691e692818d9a65c653c7a3485eb8e35c0bdc496bba652e5ea3905ba09cd8" + ], + "version": "==2.0.2" + }, + "azure-cosmosdb-table": { + "hashes": [ + "sha256:4a34c2c792036afc2a3811f4440ab967351e9ceee6542cc96453b63c678c0145", + "sha256:a9c3d2a75b376a45f4bda84af28e698b7578544bb0e9eb7b53fedef3369634a0" + ], + "version": "==1.0.5" + }, + "azure-datalake-store": { + "hashes": [ + "sha256:995703113db6840aa02abab71b2d0699dd283a12130cd843fff8c7a1acde9661", + "sha256:fd1ca3384808ac806470c26c98bc2346c1784d5b281fac4ea468ba018269ee3a" + ], + "version": "==0.0.39" + }, + "azure-eventgrid": { + "hashes": [ + "sha256:297fa9619622338691078112e479361311507e28ce54af17d2c16ace71ffa4e4", + "sha256:7ebbe1c4266ba176aa4969d9755c08f10b89848ad50fb0bfd16fa82e29234f95" + ], + "version": "==1.2.0" + }, + "azure-graphrbac": { + "hashes": [ + "sha256:825b397665f478fab511e521f3f3f4b64189cb9f6c2e7e873b3b7333dc533974", + "sha256:f94b97bdcf774878fe2f8b8c46a5d6550a4ed891350ed0730c1561a24d488ee2" + ], + "version": "==0.40.0" + }, + "azure-keyvault": { + "hashes": [ + "sha256:37a8e5f376eb5a304fcd066d414b5d93b987e68f9212b0c41efa37d429aadd49", + "sha256:dec5334cde846849dfe7896f2e98f17b4f4d75c316a4d30e7171ce71ca20713d" + ], + "version": "==1.1.0" + }, + "azure-loganalytics": { + "hashes": [ + "sha256:3ceb350def677a351f34b0a0d1637df6be0c6fe87ff32a5270b17f540f6da06e", + "sha256:5a1bdb33e1fd3dfb275d9eec45ed8e1126eda51e9072ccf08a19922ee5e0ad98" + ], + "version": "==0.1.0" + }, + "azure-mgmt": { + "hashes": [ + "sha256:8dcbee7b323c3898ae92f5e2d88c3e6201f197ae48a712970929c4646cc2580a", + "sha256:fdca6b5ecf17e5583f6fd3a20cd4e4e0335a3d91131d62b50cfef40d657234b2" + ], + "version": "==4.0.0" + }, + "azure-mgmt-advisor": { + "hashes": [ + "sha256:8fdcb41f760a216e6b835eaec11dba61822777b386139d83eee31f0ff63b05da", + "sha256:9d166cc9868971d03ec852e900fa1f1b95d8829233eff2b1baf702bd34b9887b" + ], + "version": "==1.0.1" + }, + "azure-mgmt-applicationinsights": { + "hashes": [ + "sha256:929c30559692c77d424ca36f11e98f066c98e7eb7b742c44beadc082715f19df", + "sha256:f10229eb9e3e9d0ad20188b8d14d67055e86f3815b43b75eedf96b654bee2a9b" + ], + "version": "==0.1.1" + }, + "azure-mgmt-authorization": { + "hashes": [ + "sha256:2b8504763ea8b1b475f2c3533b171bedb91ffae459f48f1f885ec8536df91093", + "sha256:535de12ff4f628b62b939ae17cc6186226d7783bf02f242cdd3512ee03a6a40e" + ], + "version": "==0.50.0" + }, + "azure-mgmt-batch": { + "hashes": [ + "sha256:6e375ecdd5966ee9ee45b29c90a806388c27ceacc2cbd6dd406ff311b5d7da72", + "sha256:d942e225180d07aae07a43e489ae0acbacd92651be3a019f4eb330a587028161" + ], + "version": "==5.0.1" + }, + "azure-mgmt-batchai": { + "hashes": [ + "sha256:b5f7df6a77fde0bd6b486762eb2c81750b6f1730ee1116689d2dfbd3e03dba95", + "sha256:f1870b0f97d5001cdb66208e5a236c9717a0ed18b34dbfdb238a828f3ca2a683" + ], + "version": "==2.0.0" + }, + "azure-mgmt-billing": { + "hashes": [ + "sha256:3810cdda69ec1409191b292628fe6ba86ce5e0444723b960d91af4f401846ac3", + "sha256:85f73bb3808a7d0d2543307e8f41e5b90a170ad6eeedd54fe7fcaac61b5b22d2" + ], + "version": "==0.2.0" + }, + "azure-mgmt-cdn": { + "hashes": [ + "sha256:069774eb4b59b76ff9bd01708be0c8f9254ed40237b48368c3bb173f298755dd", + "sha256:5d5fa3e3f37632e5787e73e78af37a28d7dd0206c854cc0a4d39d75b953f0c31" + ], + "version": "==3.0.0" + }, + "azure-mgmt-cognitiveservices": { + "hashes": [ + "sha256:36b406ee4b6652cd144a99309cd823ac1c726b0160120c14e4c35cb668f3f8ff", + "sha256:c3247f2786b996a5f328ebdaf65d31507571979e004de7a5ed0ff280f95d80b4" + ], + "version": "==3.0.0" + }, + "azure-mgmt-commerce": { + "hashes": [ + "sha256:c48e84ed322fa9ddbc2d7fcca754c5e97171919be94f510bd2579cf5666684c3", + "sha256:ddcd403bcaf6b7de2cbf1bc249b7db452b35dc1f0503f940368efc722dc0bc90" + ], + "version": "==1.0.1" + }, + "azure-mgmt-compute": { + "hashes": [ + "sha256:5b0c2390af3e29d910e3d6e7a72b0be59d6e15933740dd193129217c000e4fed", + "sha256:8982944ae9022e4999d152356c236c9ff3df0c6024b97f4ef3a4cdef6c01eb62" + ], + "version": "==4.3.1" + }, + "azure-mgmt-consumption": { + "hashes": [ + "sha256:36ea28bb2ed4bec7e4d643444085ba4debed20a01fbd87f599896a4bda3318bd", + "sha256:9a85a89f30f224d261749be20b4616a0eb8948586f7f0f20573b8ea32f265189" + ], + "version": "==2.0.0" + }, + "azure-mgmt-containerinstance": { + "hashes": [ + "sha256:274a9def808407fafe123aa8e9bc1c838a48af2de56419598db7a8b8901086e3", + "sha256:f1ea7d150447f0d8d670b7db13bd2f47320385526021053445d15c427cba6713" + ], + "version": "==1.4.0" + }, + "azure-mgmt-containerregistry": { + "hashes": [ + "sha256:7db871a74dfe6b8f54e208f6e2f43e0f6a034625a735d62013071c5e44409f8d", + "sha256:d5419db4543aaf5d83f73e087df0c0193f6b987f5c6161ac0fdd8eeabbfd23b0" + ], + "version": "==2.4.0" + }, + "azure-mgmt-containerservice": { + "hashes": [ + "sha256:2ff15bc2de14dbcee93d25d7cbe379c88aa751bc13cb199076b127fc4e221acb", + "sha256:99df430a03aada02625e35ef13d7de6c667e9bef56b5e2f60b2c284514223bff" + ], + "version": "==4.2.2" + }, + "azure-mgmt-cosmosdb": { + "hashes": [ + "sha256:a6e70527994d8ce7f4eeca80c7691bc9555adf90819848a9a30284a33b0cffe2", + "sha256:d5f448e9e6733b83e2e6bde28267bb30620c5a3f1399e66dcfe04d0b80960094" + ], + "version": "==0.4.1" + }, + "azure-mgmt-datafactory": { + "hashes": [ + "sha256:63d6cad02ef4d9da788b148b69c708227085b2d948d805bbf54150c7a692393c", + "sha256:6ee02286e9950b9f5b76589459f6d060a962faaab1f49c263a55d011e98b30bf" + ], + "version": "==0.6.0" + }, + "azure-mgmt-datalake-analytics": { + "hashes": [ + "sha256:0d64c4689a67d6138eb9ffbaff2eda2bace7d30b846401673183dcb42714de8f", + "sha256:ac96c9777314831db37461f0602e75298bc25277ba7f4d0d3e7966a926669b7e" + ], + "version": "==0.6.0" + }, + "azure-mgmt-datalake-nspkg": { + "hashes": [ + "sha256:2ac6fa13c55b87112199c5fb03a3098cefebed5f44ac34ab3d39b399951b22c4", + "sha256:3b9e2843f5d0fd6015bba13040dfc2f5fe9bc7b02c9d91dd578e8fe852d1b2dd", + "sha256:deb192ba422f8b3ec272ce4e88736796f216f28ea5b03f28331d784b7a3f4880" + ], + "version": "==3.0.1" + }, + "azure-mgmt-datalake-store": { + "hashes": [ + "sha256:2af98236cd7eaa439b239bf761338c866996ce82e9c129b204e8851e5dc095dd", + "sha256:9376d35495661d19f8acc5604f67b0bc59493b1835bbc480f9a1952f90017a4c" + ], + "version": "==0.5.0" + }, + "azure-mgmt-datamigration": { + "hashes": [ + "sha256:bb654d9f96166a5cf2638f821af5cee8b1c331257c69b053592375a4d2bb4d5e", + "sha256:ea2920475f9e56e660003a06397228243042157d46674f8a09abaf2d0a933aed" + ], + "version": "==1.0.0" + }, + "azure-mgmt-devspaces": { + "hashes": [ + "sha256:220f1610c2cda584e4212611679868d8cc5bdd789d3f0dfa1259b64fc968f580", + "sha256:4710dd59fc219ebfa4272dbbad58bf62093b52ce22bfd32a5c0279d2149471b5" + ], + "version": "==0.1.0" + }, + "azure-mgmt-devtestlabs": { + "hashes": [ + "sha256:7e91bb139b59cfaf1c1b2b0e3e21f091768c658c1879797757dedc6312f00c8c", + "sha256:d416a6d0883b0d33a63c524db6455ee90a01a72a9d8757653e446bf4d3f69796" + ], + "version": "==2.2.0" + }, + "azure-mgmt-dns": { + "hashes": [ + "sha256:3730b1b3f545a5aa43c0fff07418b362a789eb7d81286e2bed90ffef88bfa5d0", + "sha256:5b80546b0f182d7abe90c43025cd5ca7e6605224b4d5b872cca2456667f172ef" + ], + "version": "==2.1.0" + }, + "azure-mgmt-eventgrid": { + "hashes": [ + "sha256:824503b668137affa5b3782c6348c0bb6ab012c72fe47a3be9942c5639f82f8a", + "sha256:9518e9d7e60ab90a7d18ae6a3f0049ca57588f8f9583ab40d442c2a5387cf94e" + ], + "version": "==1.0.0" + }, + "azure-mgmt-eventhub": { + "hashes": [ + "sha256:675804761cb146fd4eef9b4fb2aecf257da9837780bd03147e74906d7473c4c1", + "sha256:b5407e529b9daeefbb9393c8f7401f44a21ecfeede6e03cf08456149c1d3533e" + ], + "version": "==2.2.0" + }, + "azure-mgmt-hanaonazure": { + "hashes": [ + "sha256:9fe4dc0adeb772d13918e1d6126d83c7770b762f358487504c5f082f542d0189", + "sha256:aec953c54809d0cc2f61f24d4d62a97f02c466bdc7906fd66f30120becf0c3df" + ], + "version": "==0.1.1" + }, + "azure-mgmt-iotcentral": { + "hashes": [ + "sha256:0d2101f3ea8a21ec3b29ee72d83e6ca606a241efec3b042cda8c656ad99b8fd2", + "sha256:59f7c653ac7d6475d5d7900902a5d0e0fe7aad03224c47d70f2cf1e20d43a81d" + ], + "version": "==0.1.0" + }, + "azure-mgmt-iothub": { + "hashes": [ + "sha256:08388142ed6844f0a0e97d2740decf80ffc94f22adca174c15f60b9e2c2d14be", + "sha256:77221c5b6ff7feabc2e6d44156e29fbee9098b34de044165d3532b64b678bdb1" + ], + "version": "==0.5.0" + }, + "azure-mgmt-iothubprovisioningservices": { + "hashes": [ + "sha256:2b3480a8ad2e535928da55de92b6127d02171768fed375b112274eb1e55268c1", + "sha256:8c37acfd1c33aba845f2e0302ef7266cad31cba503cc990a48684659acb7b91d" + ], + "version": "==0.2.0" + }, + "azure-mgmt-keyvault": { + "hashes": [ + "sha256:05a15327a922441d2ba32add50a35c7f1b9225727cbdd3eeb98bc656e4684099", + "sha256:406298b6236abf9e3bae3218df2fc89c25aee471b917eca7769ff5696fc8ec01" + ], + "version": "==1.1.0" + }, + "azure-mgmt-loganalytics": { + "hashes": [ + "sha256:c7315ff0ee4d618fb38dca68548ef4023a7a20ce00efe27eb2105a5426237d86", + "sha256:f224b7d52f4369ce057c7f83e80da1d00a8887ad5c15606529e9c930e601088f" + ], + "version": "==0.2.0" + }, + "azure-mgmt-logic": { + "hashes": [ + "sha256:232c175e45582f7c547d3b50d93bd64aec37b400426962e4fd0cd235980ea110", + "sha256:d163dfc32e3cfa84f3f8131a75d9e94f5c4595907332cc001e45bf7e4efd5add" + ], + "version": "==3.0.0" + }, + "azure-mgmt-machinelearningcompute": { + "hashes": [ + "sha256:4995ba9ee392eb4f5579e93dba9187b67007187e7fd022d6b417ec56e6761a6b", + "sha256:7a52f85591114ef33a599dabbef840d872b7f599b7823e596af9490ec51b873f" + ], + "version": "==0.4.1" + }, + "azure-mgmt-managementgroups": { + "hashes": [ + "sha256:005e8289c2e1d8a8368c96790edf6a34e5c37b4096bce2eb8a923c6d5dc11fb2", + "sha256:ff62d982edda634a36160cb1d15a367a9572a5acb419e5e7ad371e8c83bd47c7" + ], + "version": "==0.1.0" + }, + "azure-mgmt-managementpartner": { + "hashes": [ + "sha256:1b0ec9b9d084e331b863cef77f002ede8cbc6214bb56c3c8dd7945d10c7ffc77", + "sha256:2d7d5f346cc3d6ad621a39357637565199c51a65c7e451b73aced1c097a78165" + ], + "version": "==0.1.0" + }, + "azure-mgmt-maps": { + "hashes": [ + "sha256:a779b1ddbbcd95393e53f11b586dd26c42a709aaa226412a2df64d0da6807a80", + "sha256:c120e210bb61768da29de24d28b82f8d42ae24e52396eb6569b499709e22f006" + ], + "version": "==0.1.0" + }, + "azure-mgmt-marketplaceordering": { + "hashes": [ + "sha256:6da12425cbab0cc62f246e7266b4d67aff6bdd031ecbe50c7542c2f2b2440ad4", + "sha256:fb7a21f4a4a4b8d32bae600614f047a17993111374c9567ac11f241ada61d69f" + ], + "version": "==0.1.0" + }, + "azure-mgmt-media": { + "hashes": [ + "sha256:688b56daad16e84afc7ad788a1d0d2969e365317db0d6034dd428a08030d21aa", + "sha256:6d68668b14c00b4c68f695f2bb69ff77af29034d440371c2bdd9c80187b1a08c" + ], + "version": "==1.0.0" + }, + "azure-mgmt-monitor": { + "hashes": [ + "sha256:838867a150694837e9c6141760ff0f20fe9e5b7ab88f9ba868fde1810855895e", + "sha256:f1a58d483e3292ba4f7bbf3104573130c9265d6c9262e26b60cbfa950b5601e4" + ], + "version": "==0.5.2" + }, + "azure-mgmt-msi": { + "hashes": [ + "sha256:8622bc9a164169a0113728ebe7fd43a88189708ce6e10d4507247d6907987167", + "sha256:e989e61753bf4eca0e688526b7c31c9a88082080acfb038fad17dda7f084a026" + ], + "version": "==0.2.0" + }, + "azure-mgmt-network": { + "hashes": [ + "sha256:37c11c131ec55bf13216d62b058786491c8dd5700ffe19fec68b4557b87408a6", + "sha256:4ee99a9b1b648f31c0fb156a7cda7e680f29890465dd7863c9fcccc6aed53f71" + ], + "version": "==2.4.0" + }, + "azure-mgmt-notificationhubs": { + "hashes": [ + "sha256:481aaf1c5c4b951c3114f9d256913a443892a18d3c2130f63149edf160b6b70d", + "sha256:7c4c7755c28c8301cfa90d6ded9509c30444e5dfc5001b132dca57836930602b" + ], + "version": "==2.0.0" + }, + "azure-mgmt-nspkg": { + "hashes": [ + "sha256:1c6f5134de78c8907e8b73a8ceaaf1f336a24193a543039994fe002bb5f7f39f", + "sha256:8b2287f671529505b296005e6de9150b074344c2c7d1c805b3f053d081d58c52", + "sha256:d638ea5fda3ed323db943feb29acaa200f5d8ff092078bf8d29d4a2f8ed16999" + ], + "version": "==3.0.2" + }, + "azure-mgmt-policyinsights": { + "hashes": [ + "sha256:49b88331bf823a030182ff492b728828c35758c7e49c8898a403bce5b210ba49", + "sha256:ff94cb12d6e01bf1470c2a6af4ce6960669ab4209106153879ff97addc569ce1" + ], + "version": "==0.1.0" + }, + "azure-mgmt-powerbiembedded": { + "hashes": [ + "sha256:2f05be73f2a086c579a78fc900e3b2ae14ccde5bcec54e29dfc73e626b377476", + "sha256:6f75fef7ff576383c8c6692ba5a1efa634e6eded99d0ff6e76fdc8327325fe2f" + ], + "version": "==2.0.0" + }, + "azure-mgmt-rdbms": { + "hashes": [ + "sha256:3b6a194e6b82aa9fa187d1060ff3f19ad7218317b9ae30d9b64c3113ac8dfd7c", + "sha256:f9b77f0ead387f48c3a81914e2b976967300b3e31be10a101abd00057af7bddd" + ], + "version": "==1.5.0" + }, + "azure-mgmt-recoveryservices": { + "hashes": [ + "sha256:29df3e58890492efdd80d608b7a0fd2006c8a908687c35b9b1f70af068c0f6e4", + "sha256:e48f7769fb10a85ad857710c2cba47880166f69fe7da6b331771f129b21de95c" + ], + "version": "==0.3.0" + }, + "azure-mgmt-recoveryservicesbackup": { + "hashes": [ + "sha256:1e55b6cbb808df83576cef352ba0065f4878fe505299c0a4c5a97f4f1e5793df", + "sha256:5c44bd73df6eb55382335f16a21aee62ddc8e271cd913b9147768f42aba59c3d" + ], + "version": "==0.3.0" + }, + "azure-mgmt-redis": { + "hashes": [ + "sha256:374a267b83ec4e71077b8afad537863fb93816c96407595cdd02973235356ded", + "sha256:41d12cea5673b2e277ea298d85bab7d1fdbd0636ff6be08b15ee30a312c07d6d" + ], + "version": "==5.0.0" + }, + "azure-mgmt-relay": { + "hashes": [ + "sha256:1411e734573ce6166ac7a75fbfc0afb7d6b3f47a94d0b4999b6adf2709eba87c", + "sha256:d9f987cf2998b8a354f331b2a71082c049193f1e1cd345812e14b9b821365acb" + ], + "version": "==0.1.0" + }, + "azure-mgmt-reservations": { + "hashes": [ + "sha256:40618a3700c47a788182649f238d985edf15b08b6577ea27557e70e2866ac171", + "sha256:612acfa18f005c2ee5dda5c473b8bf6540d232db331b27be47de2608c5855adb" + ], + "version": "==0.2.1" + }, + "azure-mgmt-resource": { + "hashes": [ + "sha256:2e83289369be88d0f06792118db5a7d4ed7150f956aaae64c528808da5518d7f", + "sha256:8dcd62521482f04fb0927ee4800ef6ab3ac99a9158005d552a7c10f9c756cd8c" + ], + "version": "==2.0.0" + }, + "azure-mgmt-scheduler": { + "hashes": [ + "sha256:59e7cced3ee9b93016efb8cf5f965ca11a463bb8e55f96a2f200b013426dd751", + "sha256:c6e6edd386ddc4c21d54b1497c3397b970bc127b71809b51bd2391cb1f3d1a14" + ], + "version": "==2.0.0" + }, + "azure-mgmt-search": { + "hashes": [ + "sha256:0ec5de861bd786bcb8691322feed6e6caa8d2f0806a50dc0ca5d640591926893", + "sha256:fdbaa1721b045a4ea4a21c84c6bc1f9636b39e93dff09ffd68f22e5da88bd3ea" + ], + "version": "==2.0.0" + }, + "azure-mgmt-servicebus": { + "hashes": [ + "sha256:7d1e8c3dc05ffdfe496ae643290ce4de93a3bf814ffda69121223e3d7da12408", + "sha256:cbc3fc8d8b8930452cf7d499856313da208eb66f62ade0091aa9bae0db2b1674" + ], + "version": "==0.5.3" + }, + "azure-mgmt-servicefabric": { + "hashes": [ + "sha256:0c1434e789d0c036c613855b898a385a4533656f45eafae3ef7af3ecf4d6a3e8", + "sha256:b2bf2279b8ff8450c35e78e226231655021482fdbda27db09975ebfc983398ad" + ], + "version": "==0.2.0" + }, + "azure-mgmt-signalr": { + "hashes": [ + "sha256:37a79dfe21af4addbd9cdf248a260387caabdfdd60d2ba6f3174d28e96660655", + "sha256:8a6266a59a5c69102e274806ccad3ac74b06fd2c226e16426bbe248fc2174903" + ], + "version": "==0.1.1" + }, + "azure-mgmt-sql": { + "hashes": [ + "sha256:43668705f17bd3532e2e489d368937eb19e7d0515638146c156982ace76e0743", + "sha256:5da488a56d5265757b45747cf5fd22413eb089e606658d6e6d84fe3e9b07e4fa" + ], + "version": "==0.9.1" + }, + "azure-mgmt-storage": { + "hashes": [ + "sha256:512a29798833453f8c32a5b6d038a459649bbb5b9970ac23c982b5787057fa2b", + "sha256:9577cea1f7a86ca1db6f14539bd05ce27f43ebe590cc7f23c943961a2c5c1cdc" + ], + "version": "==2.0.0" + }, + "azure-mgmt-subscription": { + "hashes": [ + "sha256:309b23f0de65f26da80c801e913b0c3b2aea8b90ba583d919f81fe6f329d3f1b", + "sha256:a37925fb820cb86dfb57559846cc97c7e066fe0e64da7594175f4a4f5e50783c" + ], + "version": "==0.2.0" + }, + "azure-mgmt-trafficmanager": { + "hashes": [ + "sha256:126167eaa82b443b5b71394050ec292f45074701232bdbdda71f636e9b46516b", + "sha256:65796588ffbeac45bf73668977131c317b64d6a3f32faecdc5cbf9683d48132c" + ], + "version": "==0.50.0" + }, + "azure-mgmt-web": { + "hashes": [ + "sha256:8ea0794eef22a257773c13269b94855ab79d36c342ad15a98135403c9785cc0a", + "sha256:f4ddb4850314325db688241caa323fa80d811dc4590454d87f5c5b558557ea51" + ], + "version": "==0.35.0" + }, + "azure-nspkg": { + "hashes": [ + "sha256:1d0bbb2157cf57b1bef6c8c8e5b41133957364456c43b0a43599890023cca0a8", + "sha256:31a060caca00ed1ebd369fc7fe01a56768c927e404ebc92268f4d9d636435e28", + "sha256:e7d3cea6af63e667d87ba1ca4f8cd7cb4dfca678e4c55fc1cedb320760e39dd0" + ], + "version": "==3.0.2" + }, + "azure-servicebus": { + "hashes": [ + "sha256:30d5beaf73eaf40aba52fdd7f7f26dd8c3e639051dc19a5f2ab5f8e7832d68f7", + "sha256:bb6a27afc8f1ea9ab46ff2371069243d45000d351d9b64e450b63d52409b934d" + ], + "version": "==0.21.1" + }, + "azure-servicefabric": { + "hashes": [ + "sha256:8724718ef48c2810dbd8609c193225fa1e037a2c9ce70c7263f5b031e2e8f208", + "sha256:c82575cbdf95cc897c3230ea889d4e751d8760a2223857fe6fbeeea5b802e5e2" + ], + "version": "==6.3.0.0" + }, + "azure-servicemanagement-legacy": { + "hashes": [ + "sha256:282d48aae6aa002c59db6f651b68777a8f93692bb8e9b443113e6a8d5ce5e875", + "sha256:c883ff8fa3d4f4cb7b9344e8cb7d92a9feca2aa5efd596237aeea89e5c10981d" + ], + "version": "==0.20.6" + }, + "azure-storage": { + "hashes": [ + "sha256:4c406422e3edd41920bb1f0c3930c34fee3eb0d55258ef7ec7308ccbb9385ad5", + "sha256:fb6212dcbed91b49d9637aa5e8888eafdfcd523b7e560c8044d2d838bbd3ca5f" + ], + "index": "pypi", + "version": "==0.36.0" + }, + "azure-storage-blob": { + "hashes": [ + "sha256:65ebe2e54460566c2077c6b3773a2a0623eabc7b95602010cb51b84077087fda", + "sha256:baa828607e21e5c7b6ceb2ede9894d465adf586373c2f7c988fe55eca8e9048c" + ], + "markers": "extra == 'azure'", + "version": "==1.4.0" + }, + "azure-storage-common": { + "hashes": [ + "sha256:69bba6aad1e8a717eeee0f95c2feeeed72ef802001e66d6d15bf8446c4f53e6a", + "sha256:7ab607f9b8fd27b817482194b1e7d43484c65dcf2605aae21ad8706c6891934d" + ], + "version": "==1.4.0" + }, + "azure-storage-file": { + "hashes": [ + "sha256:5217b0441b671246a8d5f506a459fa3af084eeb9297c5be3bbe95d75d23bac2f", + "sha256:65831e66594cdda36e02f5566ea9d8a6ad35eca6691c28f1fbb49f23987752ff" + ], + "version": "==1.4.0" + }, + "azure-storage-queue": { + "hashes": [ + "sha256:0bafe9e61c0ce7b3f3ecadea21e931dab3248bd4989dc327a8666c5deae7f7ed", + "sha256:d28e6f854ed5d719d62637c1b5c2b74d9c67584bc326de5ce41ba0af73e3a3f0" + ], + "version": "==1.4.0" + }, "billiard": { "hashes": [ "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84" @@ -53,10 +647,10 @@ }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" ], - "version": "==2018.10.15" + "version": "==2018.11.29" }, "cffi": { "hashes": [ @@ -172,6 +766,21 @@ "index": "pypi", "version": "==1.1.0" }, + "django-appconf": { + "hashes": [ + "sha256:6a4d9aea683b4c224d97ab8ee11ad2d29a37072c0c6c509896dd9857466fb261", + "sha256:ddab987d14b26731352c01ee69c090a4ebfc9141ed223bef039d79587f22acd9" + ], + "version": "==1.0.2" + }, + "django-basicauth": { + "hashes": [ + "sha256:0ceff44ebc129eb7f8bde212a2f663210796ea1ba6e00d944cba50d4ef326f79", + "sha256:740a176e0bbeed8fd267e165a4373aa51a346258fb87479670dd7f6846f118d1" + ], + "index": "pypi", + "version": "==0.5.1" + }, "django-braces": { "hashes": [ "sha256:a457d74ea29478123c0c4652272681b3cea0bf1232187fd9f9b6f1d97d32a890", @@ -187,6 +796,14 @@ "index": "pypi", "version": "==1.1.1" }, + "django-celery-email": { + "hashes": [ + "sha256:1b2e0e31c6266007463befdc23934696fc93dcf320dfc85b8bb6b063cfe9558a", + "sha256:e5f9122c02ec58d3e49653475ad1b8612fd752681ce2f006d9c0792c57046283" + ], + "index": "pypi", + "version": "==2.0.1" + }, "django-concurrency": { "hashes": [ "sha256:233d23a8751989df6db2886957a8fbcc2408a1f16bb28262aab8a538d756d9d2" @@ -225,6 +842,14 @@ "index": "pypi", "version": "==1.0.0" }, + "django-crispy-forms": { + "hashes": [ + "sha256:5952bab971110d0b86c278132dae0aa095beee8f723e625c3d3fa28888f1675f", + "sha256:705ededc554ad8736157c666681165fe22ead2dec0d5446d65fc9dd976a5a876" + ], + "index": "pypi", + "version": "==1.7.2" + }, "django-db-logging": { "hashes": [ "sha256:4787de15c1a933e016766f023730f9551e8bf341e99549d13a918f27bf403f30" @@ -264,11 +889,19 @@ }, "django-picklefield": { "hashes": [ - "sha256:8d1de6be099044ae61e55998b35de18a57499b946fe45781077f5cec4f73f0e0", - "sha256:ce7fee5c6558fe5dc8924993d994ccde75bb75b91cd82787cbd4c92b95a69f9c" + "sha256:9052f2dcf4882c683ce87b4356f29b4d014c0dad645b6906baf9f09571f52bc8", + "sha256:f1733a8db1b6046c0d7d738e785f9875aa3c198215de11993463a9339aa4ea24" ], "index": "pypi", - "version": "==1.1.0" + "version": "==2.0" + }, + "django-post-office": { + "hashes": [ + "sha256:207b663a05d5d6a62765eb30081093837272a888cf00557d89d0e6f467928871", + "sha256:827937a944fe47cea393853069cd9315d080298c8ddb0faf787955d6aa51a030" + ], + "index": "pypi", + "version": "==3.1.0" }, "django-redis": { "hashes": [ @@ -300,6 +933,17 @@ "index": "pypi", "version": "==1.1.0" }, + "django-storages": { + "extras": [ + "azure" + ], + "hashes": [ + "sha256:8e35d2c7baeda5dc6f0b4f9a0fc142d25f9a1bf72b8cebfcbc5db4863abc552d", + "sha256:b1a63cd5ea286ee5a9fb45de6c3c5c0ae132d58308d06f1ce9865cfcd5e470a7" + ], + "index": "pypi", + "version": "==1.7.1" + }, "django-strategy-field": { "hashes": [ "sha256:659ce7d1fec5dc7770291f64d530e345e155d4d24f843f225d0b5a451f7b3706" @@ -351,6 +995,14 @@ "index": "pypi", "version": "==1.3.0" }, + "djangorestframework-yaml": { + "hashes": [ + "sha256:58e44b2ba2b1484fe793adcdc5d04910ab9b385a2991603b049b96eed6a76d32", + "sha256:b2277cb0459cf14b473e8cb6e0055725582afe862049c32b840b261ca8fbce3e" + ], + "index": "pypi", + "version": "==1.0.3" + }, "drf-dynamic-serializer": { "hashes": [ "sha256:058ae34570c1dfce4e8e97ac4a5c4ad543279b77f97e2ca254caaff9407c149a" @@ -379,11 +1031,11 @@ "validation" ], "hashes": [ - "sha256:24a02bbf56361ae0e304744f9c4aa96544270decd9d79b37d10dfd6dd04e5f22", - "sha256:b07192a6697ced6da49c5b016f1805960b0a7a1682fb21e86045a6bb572ffa99" + "sha256:9ee2072fb84ec60d951fa105e6926cf16e332973ba20ab2e3962fd9445cfd102", + "sha256:b0d5304cd2180699980fc8336edb5bfb774bbdfb79760376ed69538bf49cdcb2" ], "index": "pypi", - "version": "==1.11.0" + "version": "==1.11.1" }, "et-xmlfile": { "hashes": [ @@ -407,6 +1059,13 @@ "index": "pypi", "version": "==19.9.0" }, + "html5lib": { + "hashes": [ + "sha256:20b159aa3badc9d5ee8f5c647e5efd02ed2a66ab8d354930bd9ff139fc1dc0a3", + "sha256:66cb0dcfdbbc4f9c3ba1a63fdb511ffdbd4f513b2b6d81b80cd26ce6b3fb3736" + ], + "version": "==1.0.1" + }, "humanize": { "hashes": [ "sha256:a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19" @@ -427,6 +1086,13 @@ ], "version": "==0.3.1" }, + "isodate": { + "hashes": [ + "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", + "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" + ], + "version": "==0.6.0" + }, "itypes": { "hashes": [ "sha256:c6e77bb9fd68a4bfeb9d958fea421802282451a25bac4913ec94db82a899c073" @@ -447,6 +1113,13 @@ ], "version": "==2.10" }, + "jsonfield": { + "hashes": [ + "sha256:a0a7fdee736ff049059409752b045281a225610fecbda9b9bd588ba976493c12", + "sha256:beb1cd4850d6d6351c32daefcb826c01757744e9c863228a642f87a1a4acb834" + ], + "version": "==2.0.2" + }, "jsonpointer": { "hashes": [ "sha256:381b613fd1afd65376fb28948c4744f035e47ab049a9fdde0c48cc1c30b68559", @@ -501,6 +1174,20 @@ ], "version": "==1.1.0" }, + "msrest": { + "hashes": [ + "sha256:1b8daa01341fb77b0797c5fbc28e7e957388eb562721cc6392603ea5a4a39345", + "sha256:75adf8a044a6ff3a93dee977b8e8efb44da3e339b07aada87dcf53d6d51ca67c" + ], + "version": "==0.6.2" + }, + "msrestazure": { + "hashes": [ + "sha256:1118d52fb60fd71732a51bcb669189af4f72f40ea460e656465ee83c4738f2a0", + "sha256:fd1bcb9652cf04b711e21dcbef377a7e43f9492afeb0a59622b5623ce8715825" + ], + "version": "==0.5.1" + }, "oauthlib": { "hashes": [ "sha256:ac35665a61c1685c56336bda97d5eefa246f1202618a1d6f34fccb1bdd404162", @@ -508,11 +1195,61 @@ ], "version": "==2.1.0" }, + "onedrivesdk": { + "hashes": [ + "sha256:922037d226dd374ab5a28379c45322cbc15f4d77f5117e9904599fdba05f6054" + ], + "index": "pypi", + "version": "==1.1.8" + }, "openpyxl": { "hashes": [ - "sha256:41eb21a5620343d715b38081536c4ed3c37249afb72e569fd2af93852ed4ddde" + "sha256:7bcf019a0be528673a8aec1e60b5c863342c3231962dbf7922fd4da42a49a91a" + ], + "version": "==2.5.12" + }, + "pillow": { + "hashes": [ + "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360", + "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7", + "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e", + "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93", + "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c", + "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8", + "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b", + "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0", + "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd", + "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb", + "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581", + "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64", + "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace", + "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956", + "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60", + "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16", + "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37", + "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae", + "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a", + "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073", + "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8", + "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb", + "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409", + "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f", + "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2", + "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4", + "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74", + "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e", + "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1", + "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888" + ], + "version": "==5.3.0" + }, + "pisa": { + "hashes": [ + "sha256:94c4ae0995c84bb0588ece4480486464612ed1526f0987fb1016b9c50e5d3327", + "sha256:a7164ac81ab5ea01fbae4f29d2c00183a31142ca30ad527f6ac96635819cbd12" ], - "version": "==2.5.10" + "index": "pypi", + "version": "==3.0.33" }, "psutil": { "hashes": [ @@ -585,6 +1322,19 @@ "index": "pypi", "version": "==2.3.0" }, + "pypdf2": { + "hashes": [ + "sha256:e28f902f2f0a1603ea95ebe21dff311ef09be3d0f0ef29a3e44a932729564385" + ], + "version": "==1.26.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", + "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + ], + "version": "==2.7.5" + }, "python-social-auth": { "hashes": [ "sha256:6986220df76934aee15c54938a13ebe370a1976e043af01fa3bea417f6722e74", @@ -641,6 +1391,39 @@ "index": "pypi", "version": "==2.10.6" }, + "reportlab": { + "hashes": [ + "sha256:0fcc951899c1bb6e0fa90996d5ed9d24265c5a8eafabf2303570e079d0290c28", + "sha256:234e1790858f1608d2fc8f820fa5660b7838cb14f12dec81e3c1156a4891ee1a", + "sha256:23df47e5a4225728a6d0a16b1dbc2674fe8a535603c191ad9d33f8755d1ab08b", + "sha256:277e920f1f5aa48531d34a25ba83e4e5a201e15a6a75263d529c8996a84233ae", + "sha256:2c52b314f29a40b8ed863190947e556b75a719566d4e2e7d25b8aeb18f60b58c", + "sha256:2e38e61d8e32b003a4c50f4bf1593bc5784395a62d180edd7b206f7dfd77836c", + "sha256:2f42e2e6177756a8265126fbeb7be58fbc40affd471253c829ab165c13253f2e", + "sha256:463f446a96bd0ddaba8ca46747f251868334c119f28bffd42fc0add2b3bd986f", + "sha256:4be2e78b900a601028b4ff42c547096cc204a21ac2a9ce7d16768195106ffd12", + "sha256:4db5daa83aa5c74fba41c3212549ff62abc84921b9ce848bc04cda728e00ecf0", + "sha256:4e4669e49a4cbb323c6a3cb45297535e910d4627f796dba90362e456fb013e44", + "sha256:551bed92c44e302ac7d663242cda10bb8e459ccff0178f51d5693f846336961d", + "sha256:60995c16fa961a6576a67aaeb7a6657a081dabdea298d4c15ec68423171dcdc6", + "sha256:6228323d6a40355c20b4159bc927cb93b9c39537bd39c25b9071eedcf80b4a36", + "sha256:62aafd7769fcf36c9910a332ccee002ce4ff850c9b3ff89bbea8d7b78be979c1", + "sha256:64bf9f0778477a69563db3ca53db3312b7a65756c60e674de54e7527a0852fa0", + "sha256:678667a846ad4181b5a392c59e1ec37e03abc684854d66c7a05df32113a4d337", + "sha256:722bc6e77526cbe927f9a65f37b06840e6d99d6938462892558c6cbb90445d4c", + "sha256:8c967cf193ebeca8231e3c0c28d94c9d54dd1ee38284785c3a1cbf3aca637fd9", + "sha256:95bc72a157559b5963967cf8be17d9185ed64b5c408504b9efbd51ddcf4c145e", + "sha256:9b22737cbaff0e7d875bcb61cb593df908513f1b807551c71afc3a9d1fe83aeb", + "sha256:a2f26770e8af0d383d586fd95446ee246d0532395c9392463c07047cb89862bb", + "sha256:a417205a64cbec93219a8d7e268fe4ba4b7f3e037f7d1ca42f432b4388fb93dd", + "sha256:a6856b4118abbc783ce419e50185e69bceb4d49de39d8439f28571d5949fc34c", + "sha256:b04381abcb3c14afcef7faababbd620b2a352a2f2765f3bf18deabb2dfc7d617", + "sha256:b2348ec5f615d50913c1eaa8c3fc71d4478e4ea0e627950f6fabd70d2a38ff1d", + "sha256:f86f13f4e6d8c815c35ec75e3d3a94cf7de12d544b4d038269c8b7c73de6b906", + "sha256:fa7761cca303d4dbc2f5c5989b2be4baa4068fa4137a5fdc03eb47bfc1af0a5d" + ], + "version": "==3.5.11" + }, "requests": { "hashes": [ "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", @@ -664,30 +1447,30 @@ }, "ruamel.yaml": { "hashes": [ - "sha256:0dad3f56197e28c04ab251aeb739377a5da74ea83bcac00b79232d7cef66dc7a", - "sha256:19a2757b13c2461572c7d4be61ec25caabb0fc34af48b4565c88a085f570cb76", - "sha256:39cfb38a8590df4ab940b99d512df8a61f871a25f5912727fab1f36b20a12c1d", - "sha256:475762beae3ceedc7e4a007ed12a90f97a262e7ce7239029dc3d0efcdffb2e7b", - "sha256:4d282afb28a7a09dee7df7ddffe3611411e33d05bafd6ddfd6ef40869479252c", - "sha256:54137fa40c62fd72d2646f2635e97e5173dd144a48a3bd25c0c7d743c2966162", - "sha256:54359de3ee1c8955607a050f1dbdfddd0c043f0df1e99d5b82128e3fc6b5c966", - "sha256:584a8513bc7cac176112093c8bc55e457fd4f694b267ef581d8eb1e983e77e7f", - "sha256:5953840b9852e1e2735c9d413d01ddfb768334e475cf130d70064ad8831c30ce", - "sha256:659767cb717052f98fd358ae21bbe192957535693e2c68fba27b2f3c6a14917c", - "sha256:85793c5fe321e9202eba521b0bb3e6303bcb61f6e56378f59e874ca36a7e9d5f", - "sha256:8609f1263a73da187f7febebca24b347aee7da777edc83b8625cb7af3e16b345", - "sha256:883e42e30fcd485b8195522121960cbcc4bba2cb5161cec29b3cfd6e1072eaf4", - "sha256:928146b4a6ef2ce4a5b26663ceddb4961bda00fa1c4e255e6b3e3ef9b284df04", - "sha256:a91a0f5b1e18b4335a0febc757c4a465da0ab6a4f9c672fcc917bcf738efcd66", - "sha256:c0522d1d02856a00e139e2a915c74b20ad1d38f2748ea3f2d4e087ac30322ad2", - "sha256:e3238c0b169f104fa20b217d741632228cbf9011690edabdd265deff09efb299", - "sha256:e9c0b17da17ee512f49de3683c97118877d4ff706ea2edac82f187cd4a9cca4d", - "sha256:eabf69011659182a1044e7581f8ea69e4f4511933955e71fc2f007b28e214791", - "sha256:f00a2ad9b03a8cad5ff620dc0bb7afbd2eaf0ebbf1e5477b632680a2c1d7c656", - "sha256:f6f5921a67211c6a98940d5f03474158c8e3744b26fd0bd5d60ab951aa36c002", - "sha256:fcb63dc194609d1f49e309b976976f8ddd8416e0b8942b3091d3d0a525bfd18a" - ], - "version": "==0.15.78" + "sha256:10079d03b5c93d54be90e4fe23d4b1f32502d7da98077e2a746c216bedba3d75", + "sha256:1ff2289958e09fac2aa573e7a9fb9c953ddf89f67c3a42693394920f72001348", + "sha256:3b8af255839c39d3dfd0dcb82db349f38db28a2f7adbe05387bea87de15ac146", + "sha256:418ee849362ad59c19064af3ce09666d0898969eadb25964b693827fb68cfeca", + "sha256:4f203351575dba0829c7b1e5d376d08cf5f58e4a2b844e8ce552b3e41cd414e6", + "sha256:5421c3fb144d6e1de0dc00d8a1f919f558c3156c48aa7aa2acbd7754530dfeb7", + "sha256:59a17a6225d6d60150647d2fd707fcb1ee54ea31dd0048ea120c7bf2c9093451", + "sha256:6acdf9d7bea6ff8541e96e4694b9fd1e0728be88ef512afc28da0804590533f7", + "sha256:70ed366ad65780040f5bf5e34c75f450e3f0ca9a8b2534cfc0293c387ec1c32f", + "sha256:7e9dc8d095d952c352d9f63cab5283430c910b77793f323ce8d64921f65aaa53", + "sha256:8344a08555f8494ae16a2e4f445e5bfce80f82010d9e5091c870aadee5c4b14a", + "sha256:90b1f9ed3893b0713e0bc47cfa93be72ddb6d5a969c31979d36c2854dc8b87dd", + "sha256:93f262943089657675b336f804b721e6b76f67cc61f6bfc91b3f35f8e71a8a64", + "sha256:9996c6371bfe3051340a469037323f533bbdf2dea9b6914da27e62696c81712b", + "sha256:9d9a382be1c150d23cd1291091454ea801522629bd22531f0b2c41ab21a81deb", + "sha256:adb57424caeb48f8cf1c937e4c571e5720e38601a77dd87d4c1178d406a72821", + "sha256:b5147c0919c6ce29c278afe21bf64c3f099afc271364a038c5e59fcf7bb672fb", + "sha256:b71cfb6c90f7b6db26842923e5c178c3ad232577f6097ebca3651be366c84b36", + "sha256:d9a82d37a4b006b12e09550ff57f7edb5ec987f0d9128fc44f590d56683aa97d", + "sha256:dc76688fb7994bf9c2370e28238bd56f9fe7e1d02675f3b1e06fc35967375869", + "sha256:fa5ae31d1aea11b528f5987e43194abdfcb44a5a259172221097eb6f74bd0965", + "sha256:ff6046de0ed29c3f3e6ba2ce164a85781af16dd49049af1b0402b8a6d15a25ca" + ], + "version": "==0.15.80" }, "six": { "hashes": [ @@ -769,6 +1552,13 @@ ], "version": "==1.1.4" }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" + }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", @@ -776,6 +1566,13 @@ ], "index": "pypi", "version": "==4.1.2" + }, + "xhtml2pdf": { + "hashes": [ + "sha256:86a37e78d7a8d8bb2761746c3d559e12284d92c4d531b3a8a0f8fd632b436f82" + ], + "index": "pypi", + "version": "==0.2.3" } }, "develop": { @@ -823,6 +1620,14 @@ ], "version": "==4.6.3" }, + "bumpversion": { + "hashes": [ + "sha256:6744c873dd7aafc24453d8b6a1a0d6d109faf63cd0cd19cb78fd46e74932c77e", + "sha256:6753d9ff3552013e2130f7bc03c1007e24473b4835952679653fb132367bdd57" + ], + "index": "pypi", + "version": "==0.5.3" + }, "cached-property": { "hashes": [ "sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f", @@ -832,10 +1637,10 @@ }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" ], - "version": "==2018.10.15" + "version": "==2018.11.29" }, "cfgv": { "hashes": [ @@ -904,11 +1709,11 @@ }, "django-extensions": { "hashes": [ - "sha256:30cb6a8c7d6f75a55edf0c0c4491bd98f8264ae1616ce105f9cecac4387edd07", - "sha256:4ad86a7a5e84f1c77db030761ae87a600647250c652030a2b71a16e87f3a3d62" + "sha256:8317a3fe479b1ba3e3a04ecf33fb8d6ccf09bb18f30eab64e34c40a593741d26", + "sha256:a76a61566f1c8d96acc7bcf765080b8e91367a25a2c6f8c5bddd574493839180" ], "index": "pypi", - "version": "==2.1.3" + "version": "==2.1.4" }, "django-webtest": { "hashes": [ @@ -927,18 +1732,18 @@ }, "docker": { "hashes": [ - "sha256:31421f16c01ffbd1ea7353c7e7cd7540bf2e5906d6173eb51c8fea4e0ea38b19", - "sha256:fbe82af9b94ccced752527c8de07fa20267f9634b48674ba478a0bb4000a0b1e" + "sha256:145c673f531df772a957bd1ebc49fc5a366bcd55efa0e64bbd029f5cc7a1fd8e", + "sha256:666611862edded75f6049893f779bff629fdcd4cd21ccf01d648626e709adb13" ], "index": "pypi", - "version": "==3.5.1" + "version": "==3.6.0" }, "docker-pycreds": { "hashes": [ - "sha256:0a941b290764ea7286bd77f54c0ace43b86a8acd6eb9ead3de9840af52384079", - "sha256:8b0e956c8d206f832b06aa93a710ba2c3bcbacb5a314449c040b0b814355bbff" + "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", + "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49" ], - "version": "==0.3.0" + "version": "==0.4.0" }, "drf-api-checker": { "hashes": [ @@ -976,6 +1781,14 @@ "index": "pypi", "version": "==3.6.0" }, + "freezegun": { + "hashes": [ + "sha256:6cb82b276f83f2acce67f121dc2656f4df26c71e32238334eb071170b892a278", + "sha256:e839b43bfbe8158b4d62bb97e6313d39f3586daf48e1314fb1083d2ef17700da" + ], + "index": "pypi", + "version": "==0.3.11" + }, "identify": { "hashes": [ "sha256:5e956558a9a1e3b3891d7c6609fc9709657a11878af288ace484d1a46a93922b", @@ -992,10 +1805,10 @@ }, "importlib-metadata": { "hashes": [ - "sha256:36b02c84f9001adf65209fefdf951be8e9014a95eab9938c0779ad5670359b1c", - "sha256:60b6481a72908c93ccb707abeb926fb5a15319b9e6f0b76639a718837ee12de0" + "sha256:28fba9f65e5415a691dd254cdb602bcc4d6f738e68407ad251651db358b63bcf", + "sha256:4a545e6125dc72b4ad98201ea3f40f92e8126e3a19667352b3a134d22b8bc74f" ], - "version": "==0.6" + "version": "==0.7" }, "importlib-resources": { "hashes": [ @@ -1053,37 +1866,37 @@ }, "multidict": { "hashes": [ - "sha256:0573239b5241a075b944b39bdf87fb6600e3a56ad5ca6d2ba9699d62de872309", - "sha256:085b1f55327b4c8c425ce96a7fdfd6a6a1e864444a90d2107f47de4c53b6edea", - "sha256:1ee14a2e7bef872ddac61450e6128aae21304b5165d21fd04681faa3261a7b2e", - "sha256:2c1791371a973d93facccc38adf9e9c14656bf85c2beddd48329134d139b6e7f", - "sha256:2cda0064cab0e2d46b02aeee9e218066993b40d4900b07d8360f54c80eed4c5c", - "sha256:3574eef3ceb983658c3c8bef0c1b3771a2dea338b3822a0c2bec03363f1dc8bb", - "sha256:3864b26cdb1c7454809ec12fb0998989b8832ebb8423aa69ff3a1ad82b9756f3", - "sha256:3e6f7161ea60795f33b21e91b5c9fa66a3dd416f949684ade8ba8a9b193f7e50", - "sha256:3fa7944194cc96319cbbd53a1e0fb6dfe1e437efb75117828c35ce5b30d9d0c9", - "sha256:470ddec7a3ae052862af73dd39a1b1c582a1ec397f8643f09658de56a0a84ac9", - "sha256:5cb1a5926fe898451688036b5e95863c6e75110c98810584695b2403cb04522f", - "sha256:754ed617ecb736261ee3595b4a5dc035bcb5e897ce0a0148252aa8cbc2e59e60", - "sha256:79879c5c0434840d6ac5246e53d22e18c7f5b87419abb968e6357ba65386993a", - "sha256:7db4a72fa35bbe9ccaf3c856b14d89e26e8de5ca0c31604ac5970a3746182f5a", - "sha256:839676a86dbadf4a0be32ca580292c764245044eae324ebfc55362c13886d5d6", - "sha256:8805d8eec8437f9d2b3fd5c09822ef259af08ece0a19f41d2ad8d322a7a67054", - "sha256:8cbb4725aad6dc38cca571dab803f53ed76fc5cc468088636b42b539719aa5f9", - "sha256:9137d7e3db47641aa86526b60dd3d50d2066617668f8617f0c16adf92dfbaa9e", - "sha256:a6d985c3ccc1fca18cfd415406047f0984e3b07f533d50aa91c36eadb46681a9", - "sha256:af381425a02e0a235b23a685cda2d94cd0679ed8257d8a54c5f03ff2eee1fbb7", - "sha256:b7399dfd7f977c419d6e2b08d1099afe00f51454eb2ecc6b067c9eba6efb0a34", - "sha256:b8204fe2cb7199ecd568e67268a49d87f031c294e46d6fdf17bd1e544bcb81ba", - "sha256:ba973fd954f3de8e47e4be43f530729dd7e894615d3734a1b23f4de14f883142", - "sha256:c00b1423d09a73c94553f80ea52dc8c4341beae448bd4468a603263040debb17", - "sha256:ca4fab7f9590b7fa6c5dcde16356726f254456a2bb33d98828d896ba57a5eda1", - "sha256:dcb97bf0efa02a067f2a42c457dfa1548d8bc8913c12f96e26460709bc8a2ae5", - "sha256:e7d1f2671bd62064da2c7d6318c4f9307889cb85c59e00b2d1a66c2ed3bae3eb", - "sha256:ed8a1c22cbf6b0840e8b8a436bc378164a0474580968f38a0eeec8ed7cb78b75", - "sha256:f3826e28328455f62e8de193fb4ab5349ad78da693f1e002fd90d249a0cfaa8b" - ], - "version": "==4.5.0" + "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f", + "sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3", + "sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef", + "sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b", + "sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73", + "sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc", + "sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3", + "sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd", + "sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351", + "sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941", + "sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d", + "sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1", + "sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b", + "sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a", + "sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3", + "sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7", + "sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0", + "sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0", + "sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014", + "sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5", + "sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036", + "sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d", + "sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a", + "sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce", + "sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1", + "sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a", + "sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9", + "sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7", + "sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b" + ], + "version": "==4.5.2" }, "nodeenv": { "hashes": [ @@ -1100,10 +1913,10 @@ }, "pdbpp": { "hashes": [ - "sha256:dde77326e4ea41439c243ed065826d53539530eeabd1b6615aae15cfbb9fda05" + "sha256:535085916fcfb768690ba0aeab2967c2a2163a0a60e5b703776846873e171399" ], "index": "pypi", - "version": "==0.9.2" + "version": "==0.9.3" }, "pexpect": { "hashes": [ @@ -1173,18 +1986,18 @@ }, "pygments": { "hashes": [ - "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", - "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" + "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", + "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" ], - "version": "==2.2.0" + "version": "==2.3.0" }, "pytest": { "hashes": [ - "sha256:488c842647bbeb350029da10325cb40af0a9c7a2fdda45aeb1dda75b60048ffb", - "sha256:c055690dfefa744992f563e8c3a654089a6aa5b8092dded9b6fafbd70b2e45a7" + "sha256:1d131cc532be0023ef8ae265e2a779938d0619bb6c2510f52987ffcba7fa1ee4", + "sha256:ca4761407f1acc85ffd1609f464ca20bb71a767803505bd4127d0e45c5a50e23" ], "index": "pypi", - "version": "==4.0.0" + "version": "==4.0.1" }, "pytest-cov": { "hashes": [ @@ -1223,6 +2036,13 @@ "index": "pypi", "version": "==1.6.0" }, + "pytest-ignore-flaky": { + "hashes": [ + "sha256:78f8ddf9b405c09ce852a4ceac58258e4b8d98d2bc236a03d4a075cdc51c56c6" + ], + "index": "pypi", + "version": "==0.1.1" + }, "pytest-pythonpath": { "hashes": [ "sha256:63fc546ace7d2c845c1ee289e8f7a6362c2b6bae497d10c716e58e253e801d62" @@ -1366,11 +2186,11 @@ }, "yapf": { "hashes": [ - "sha256:b96815bd0bbd2ab290f2ae9e610756940b17a0523ef2f6b2d31da749fc395137", - "sha256:cebb6faf35c9027c08996c07831b8971f3d67c0eb615269f66dfd7e6815fdc2a" + "sha256:8aa7f9abdb97b4da4d3227306b88477982daafef0a96cc41639754ca31f46d55", + "sha256:f2df5891481f94ddadfbf8ae8ae499080752cfb06005a31bbb102f3012f8b944" ], "index": "pypi", - "version": "==0.24.0" + "version": "==0.25.0" }, "yarl": { "hashes": [ diff --git a/docker/Dockerfile b/docker/Dockerfile index 85e7d9744..b99c917ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,21 +13,38 @@ RUN set -o pipefail && if [ "${DEVELOP}" = "1" ]; then \ && curl ${GITHUB_CREDENTIALS}: -L "https://github.com/unicef/etools-datamart/archive/${VERSION}.tar.gz" | tar -xzf - --strip-components=1; \ fi -FROM python:3.6.7-alpine as base -COPY --from=builder /code /code -RUN apk add --no-cache --virtual .build-deps \ +FROM python:3.6-alpine as base + +RUN apk add --no-cache \ + bash \ + freetype \ + lcms2 \ + libjpeg-turbo \ + libpng \ + libpq \ + openjpeg \ + postgresql-libs \ + tiff \ + && apk add --no-cache --virtual .build-deps \ + freetype-dev \ gcc \ + jpeg-dev \ + lcms2-dev \ libffi-dev \ linux-headers \ musl-dev \ + openjpeg-dev \ postgresql-dev \ - python3-dev - - -RUN apk add --no-cache --virtual .deps \ - bash + python3-dev \ + tcl-dev \ + tiff-dev \ + tk-dev \ + zlib-dev \ + && pip install pip==18.0 pipenv --upgrade \ + && adduser -S datamart FROM base +COPY --from=builder /code /code LABEL org.label.name="eTools Datamart" \ org.label.maintainer="sapostolico@unicef.org" \ @@ -92,7 +109,8 @@ ENV STATIC_ROOT /tmp/static # gcc RUN mkdir -p \ - /var/datamart/{static,log,conf,run} \ + /var/datamart/ \ + && chown datamart /var/datamart/ \ && pip install pip==18.0 pipenv --upgrade WORKDIR /code @@ -104,19 +122,17 @@ RUN set -ex \ RUN pip install . \ && rm -fr /code -#RUN apt-get autoremove --yes --force-yes \ -# gcc \ -# && rm -fr /var/lib/apt/lists/* \ -# && rm -fr /var/cache/apt/* -#RUN apk del .build-deps \ -# && rm -rf /var/cache/apk/* \ -# && rm -fr /root/.cache/ +RUN apk del .build-deps \ + && rm -rf /var/cache/apk/* \ + && rm -fr /root/.cache/ WORKDIR /var/datamart EXPOSE 8000 +USER datamart + ADD docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/Makefile b/docker/Makefile index f8e32a82a..408ac0103 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,11 +2,11 @@ DATABASE_URL?= DATABASE_URL_ETOOLS?= DEVELOP?=0 -DOCKER_PASS?=${DOCKER_PASS} -DOCKER_USER?=${DOCKER_USER} +DOCKER_PASS?= +DOCKER_USER?= TARGET?=dev -# below var are used internally -BUILD_OPTIONS?= +# below vars are used internally +BUILD_OPTIONS?=--squash CMD?=datamart CONTAINER_NAME?=datamart-${TARGET} ORGANIZATION=unicef @@ -23,6 +23,10 @@ help: @echo "release release tag ${TARGET} on docker hub" @echo "run run ${DOCKER_IMAGE} locally" +build-base: + docker build --target base \ + -t ${DOCKER_IMAGE_NAME}:base \ + -f docker/${DOCKERFILE} . build: cd .. && docker build \ @@ -32,8 +36,14 @@ build: --build-arg VERSION=${TARGET} \ -t ${DOCKER_IMAGE} \ -f docker/${DOCKERFILE} . + docker tag ${DOCKER_IMAGE_NAME}:${TARGET} ${DOCKER_IMAGE_NAME}:dev +# flatten image + docker create ${DOCKER_IMAGE_NAME}:${TARGET} foo + docker export foo | docker import - unicef/datamart:${TARGET} + docker rm foo docker images | grep ${DOCKER_IMAGE_NAME} + .run: cd .. && docker run \ --rm \ @@ -85,7 +95,7 @@ test: -e CSRF_COOKIE_SECURE=1 \ -e SECURE_HSTS_PRELOAD=1 \ -e SECURE_SSL_REDIRECT=1" \ - CMD='bash -c "django-admin check --deploy"' \ + CMD='bash -c "touch /var/datamart/.touch && django-admin check --deploy "' \ $(MAKE) .run scratch: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8c16ad6ca..adc012523 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -5,13 +5,13 @@ mkdir -p /var/datamart/{static,log,conf,run} rm -f /var/datamart/run/* -if [ "$*" == "workers" ];then +if [[ "$*" == "workers" ]];then django-admin db-isready --wait --timeout 60 --sleep 5 django-admin db-isready --wait --timeout 300 --sleep 5 --connection etools celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile run/celery.pid -elif [ "$*" == "beat" ];then +elif [[ "$*" == "beat" ]];then celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile run/celerybeat.pid -elif [ "$*" == "datamart" ];then +elif [[ "$*" == "datamart" ]];then django-admin db-isready --wait --timeout 60 django-admin check --deploy django-admin init-setup --all --verbosity 1 diff --git a/src/drf_querystringfilter/backend.py b/src/drf_querystringfilter/backend.py index 8e374f6a5..cf5629c6e 100644 --- a/src/drf_querystringfilter/backend.py +++ b/src/drf_querystringfilter/backend.py @@ -8,6 +8,7 @@ import coreapi import coreschema from django import forms +from django.core.exceptions import FieldError from django.db import models from django.db.models import BooleanField, FieldDoesNotExist from django.template import loader @@ -127,8 +128,8 @@ def _get_filters(self, request, queryset, view): # noqa """ self.opts = queryset.model._meta filter_fields = getattr(view, 'filter_fields', None) - exclude = {} - filters = {} + self.exclude = {} + self.filters = {} if filter_fields: blacklist = RexList(getattr(view, 'filter_blacklist', [])) @@ -140,9 +141,9 @@ def _get_filters(self, request, queryset, view): # noqa if negate: filter_field_name = fieldname_arg[:-1] - TARGET = exclude + TARGET = self.exclude else: - TARGET = filters + TARGET = self.filters filter_field_name = fieldname_arg if filter_field_name in self.excluded_query_params: @@ -163,8 +164,8 @@ def _get_filters(self, request, queryset, view): # noqa # parts = [field_name] processor = getattr(self, 'process_{}'.format(filter_field_name), None) - # if (field_name not in filter_fields) and (not processor): - # raise InvalidQueryArgumentError(fieldname_arg) + if (filter_field_name not in filter_fields) and (not processor): + raise InvalidQueryArgumentError(filter_field_name) # field is configured in Serializer # so we use 'source' attribute if filter_field_name in mapping: @@ -185,9 +186,9 @@ def _get_filters(self, request, queryset, view): # noqa 'parts': parts, 'value': raw_value, 'real_field_name': real_field_name} - _f, _e = processor(dict(filters), dict(exclude), **payload) - filters.update(**_f) - exclude.update(**_e) + _f, _e = processor(dict(self.filters), dict(self.exclude), **payload) + self.filters.update(**_f) + self.exclude.update(**_e) else: # field_object = opts.get_field(real_field_name) value_type = self.field_type(real_field_name) @@ -211,20 +212,22 @@ def _get_filters(self, request, queryset, view): # noqa except Exception as e: logger.exception(e) raise - return filters, exclude + return self.filters, self.exclude def filter_queryset(self, request, queryset, view): self.request = request try: - self.filters, self.exclude = self._get_filters(request, queryset, view) - qs = queryset.filter(**self.filters).exclude(**self.exclude) + filters, exclude = self._get_filters(request, queryset, view) + qs = queryset.filter(**filters).exclude(**exclude) logger.debug("""Filtering using: {} -{}""".format(self.filters, self.exclude)) +{}""".format(filters, exclude)) # if '_distinct' in self.query_params: # f = self.get_param_value('_distinct') # qs = qs.order_by(*f).distinct(*f) return qs + except FieldError as e: + raise QueryFilterException(str(e)) except (InvalidFilterError, QueryFilterException) as e: logger.exception(e) raise diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index b925ae7f5..d8a3c921b 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,3 +1,3 @@ NAME = 'etools-datamart' -VERSION = __version__ = '1.6' +VERSION = __version__ = '1.7' __author__ = '' diff --git a/src/etools_datamart/api/endpoints/common.py b/src/etools_datamart/api/endpoints/common.py index 664aafba6..253e2e8e4 100644 --- a/src/etools_datamart/api/endpoints/common.py +++ b/src/etools_datamart/api/endpoints/common.py @@ -7,6 +7,7 @@ from django.http import Http404 from drf_querystringfilter.exceptions import QueryFilterException from dynamic_serializer.core import DynamicSerializerMixin +from rest_framework.decorators import action from rest_framework.exceptions import NotAuthenticated, PermissionDenied from rest_framework.filters import OrderingFilter from rest_framework.response import Response @@ -14,6 +15,7 @@ from unicef_rest_framework.views import ReadOnlyModelViewSet from etools_datamart.api.filtering import DatamartQueryStringFilterBackend, TenantQueryStringFilterBackend +from etools_datamart.apps.etl.models import EtlTask from etools_datamart.apps.multitenant.exceptions import InvalidSchema, NotAuthorizedSchema __all__ = ['APIMultiTenantReadOnlyModelViewSet'] @@ -43,6 +45,20 @@ def build_description(self): return description +class UpdatesMixin: + + @action(methods=['get'], detail=False) + def updates(self, request, version): + """ Returns only records changed from last ETL task""" + task = EtlTask.objects.get_for_model(self.queryset.model) + offset = task.last_changes.strftime('%Y-%m-%d %H:%M') + queryset = self.queryset.filter(last_modify_date__gte=offset) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data, + headers={'update-date': offset}) + + class APIReadOnlyModelViewSet(ReadOnlyModelViewSet): filter_backends = [SystemFilterBackend, DatamartQueryStringFilterBackend, @@ -68,7 +84,8 @@ def drf_ignore_filter(self, request, field): def handle_exception(self, exc): conn = connections['etools'] - if isinstance(exc, QueryFilterException): + if isinstance(exc, (QueryFilterException,)): + # FieldError can happen due cache attempt to create return Response({"error": str(exc)}, status=400) elif isinstance(exc, NotAuthenticated): return Response({"error": "Authentication credentials were not provided."}, status=401) @@ -149,3 +166,7 @@ def get_schema_fields(self): schema=coreschema.String(description="comma separated list of schemas") )) return ret + + +class DataMartViewSet(APIReadOnlyModelViewSet, UpdatesMixin): + pass diff --git a/src/etools_datamart/api/endpoints/datamart/famindicator.py b/src/etools_datamart/api/endpoints/datamart/famindicator.py index 9c4d656c1..03e9c15e0 100644 --- a/src/etools_datamart/api/endpoints/datamart/famindicator.py +++ b/src/etools_datamart/api/endpoints/datamart/famindicator.py @@ -7,8 +7,8 @@ from .. import common -class FAMIndicatorViewSet(common.APIReadOnlyModelViewSet): +class FAMIndicatorViewSet(common.DataMartViewSet): serializer_class = serializers.FAMIndicatorSerializer queryset = models.FAMIndicator.objects.all() - filter_fields = ('country_name', ) + filter_fields = ('country_name', 'last_modify_date') filter_backends = [MonthFilterBackend] + common.APIReadOnlyModelViewSet.filter_backends diff --git a/src/etools_datamart/api/endpoints/datamart/hact.py b/src/etools_datamart/api/endpoints/datamart/hact.py index 8cbebff2a..2783992ec 100644 --- a/src/etools_datamart/api/endpoints/datamart/hact.py +++ b/src/etools_datamart/api/endpoints/datamart/hact.py @@ -5,7 +5,7 @@ from .. import common -class HACTViewSet(common.APIReadOnlyModelViewSet): +class HACTViewSet(common.DataMartViewSet): serializer_class = serializers.HACTSerializer queryset = models.HACT.objects.all() - filter_fields = ('country_name', 'month') + filter_fields = ('country_name', 'year', 'last_modify_date') diff --git a/src/etools_datamart/api/endpoints/datamart/intervention.py b/src/etools_datamart/api/endpoints/datamart/intervention.py index 627cba00d..6b344552b 100644 --- a/src/etools_datamart/api/endpoints/datamart/intervention.py +++ b/src/etools_datamart/api/endpoints/datamart/intervention.py @@ -35,13 +35,13 @@ class Meta: # } -class InterventionViewSet(common.APIReadOnlyModelViewSet): +class InterventionViewSet(common.DataMartViewSet): """ """ serializer_class = serializers.InterventionSerializer queryset = models.Intervention.objects.all() - filter_fields = ('country_name', 'title', 'status', + filter_fields = ('country_name', 'title', 'status', 'last_modify_date', 'start_date', 'submission_date', 'document_type') serializers_fieldsets = {'std': None, 'short': ["title", "number"]} diff --git a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py index 4ba01e7e5..3289b3ec4 100644 --- a/src/etools_datamart/api/endpoints/datamart/pmpindicators.py +++ b/src/etools_datamart/api/endpoints/datamart/pmpindicators.py @@ -5,12 +5,12 @@ from .. import common -class PMPIndicatorsViewSet(common.APIReadOnlyModelViewSet): +class PMPIndicatorsViewSet(common.DataMartViewSet): """ """ serializer_class = serializers.PMPIndicatorsSerializer queryset = models.PMPIndicators.objects.all() filter_fields = ('country_name', 'business_area_code', 'vendor_number', - 'partner_name', 'partner_type', + 'partner_name', 'partner_type', 'last_modify_date', 'pd_ssfa_ref', ) diff --git a/src/etools_datamart/api/endpoints/datamart/serializers.py b/src/etools_datamart/api/endpoints/datamart/serializers.py index 4b7539a54..4861985ac 100644 --- a/src/etools_datamart/api/endpoints/datamart/serializers.py +++ b/src/etools_datamart/api/endpoints/datamart/serializers.py @@ -7,37 +7,38 @@ from etools_datamart.apps.data import models -class PMPIndicatorsSerializer(serializers.ModelSerializer): +class DataMartSerializer(serializers.ModelSerializer): class Meta: + exclude = ('schema_name',) + + +class PMPIndicatorsSerializer(DataMartSerializer): + class Meta(DataMartSerializer.Meta): model = models.PMPIndicators - exclude = () -class InterventionSerializer(serializers.ModelSerializer): - class Meta: +class InterventionSerializer(DataMartSerializer): + class Meta(DataMartSerializer.Meta): model = models.Intervention - exclude = () -class FAMIndicatorSerializer(serializers.ModelSerializer): - class Meta: +class FAMIndicatorSerializer(DataMartSerializer): + class Meta(DataMartSerializer.Meta): model = models.FAMIndicator - exclude = () -class UserStatsSerializer(serializers.ModelSerializer): +class UserStatsSerializer(DataMartSerializer): month = SerializerMethodField(help_text="---") - class Meta: + class Meta(DataMartSerializer.Meta): model = models.UserStats - exclude = () + read_only = ['last_modify_date', ] def get_month(self, obj): return datetime.strftime(obj.month._date, '%b %Y') -class HACTSerializer(serializers.ModelSerializer): - - class Meta: +class HACTSerializer(DataMartSerializer): + # last_modify_date = serializers.DateTimeField(format=settings.DATETIME_FORMAT) + class Meta(DataMartSerializer.Meta): model = models.HACT - exclude = () diff --git a/src/etools_datamart/api/endpoints/datamart/user.py b/src/etools_datamart/api/endpoints/datamart/user.py index 569d037c1..b442c66d2 100644 --- a/src/etools_datamart/api/endpoints/datamart/user.py +++ b/src/etools_datamart/api/endpoints/datamart/user.py @@ -5,7 +5,7 @@ from .. import common -class UserStatsViewSet(common.APIReadOnlyModelViewSet): +class UserStatsViewSet(common.DataMartViewSet): serializer_class = serializers.UserStatsSerializer queryset = models.UserStats.objects.all() - filter_fields = ('country_name', 'month') + filter_fields = ('country_name', 'month', 'last_modify_date') diff --git a/src/etools_datamart/api/urls.py b/src/etools_datamart/api/urls.py index 92cc86182..2a2a498dc 100644 --- a/src/etools_datamart/api/urls.py +++ b/src/etools_datamart/api/urls.py @@ -1,4 +1,4 @@ -from django.urls import path +from django.urls import include, path, re_path from unicef_rest_framework.routers import APIReadOnlyRouter from . import endpoints @@ -34,10 +34,10 @@ class ReadOnlyRouter(APIReadOnlyRouter): router.register(r'system/tasks-log', endpoints.TaskLogViewSet) -urlpatterns = router.urls +# urlpatterns = router.urls -urlpatterns += [ - # url(r'^+swagger(?P\.json|\.yaml)$', endpoints.schema_view.without_ui(cache_timeout=0), name='schema-json'), +urlpatterns = [ + re_path(r'(?P(v1|latest))/', include(router.urls)), path(r'+swagger/', endpoints.schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), path(r'+redoc/', endpoints.schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), ] diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index 7b63661b4..445a83e74 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -45,14 +45,15 @@ def has_add_permission(self, request): return False def has_delete_permission(self, request, obj=None): - return False + return request.user.is_superuser def get_readonly_fields(self, request, obj=None): - self.readonly_fields = [field.name for field in obj.__class__._meta.fields] + if not request.user.is_superuser: + self.readonly_fields = [field.name for field in obj.__class__._meta.fields] return self.readonly_fields def changeform_view(self, request, object_id=None, form_url='', extra_context=None): - if request.method == 'POST': + if request.method == 'POST' and not request.user.is_superuser: redirect_url = reverse('admin:%s_%s_changelist' % (self.opts.app_label, self.opts.model_name)) diff --git a/src/etools_datamart/apps/data/migrations/0001_initial.py b/src/etools_datamart/apps/data/migrations/0001_initial.py index 0367341e6..9bbc51de0 100644 --- a/src/etools_datamart/apps/data/migrations/0001_initial.py +++ b/src/etools_datamart/apps/data/migrations/0001_initial.py @@ -1,8 +1,7 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.contrib.postgres.fields.jsonb import month_field.models -import unicef_security.models from django.db import migrations, models @@ -20,6 +19,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('spotcheck_ip_contacted', models.IntegerField(default=0, verbose_name='Spot Check-IP Contacted')), ('spotcheck_report_submitted', models.IntegerField(default=0, verbose_name='Spot Check-Report Submitted')), @@ -39,9 +39,29 @@ class Migration(migrations.Migration): ('microassessment_cancelled', models.IntegerField(default=0, verbose_name='Micro Assessment-Cancelled')), ], options={ + 'verbose_name': 'FAM Indicator', 'ordering': ('month', 'country_name'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), + ), + migrations.CreateModel( + name='HACT', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('country_name', models.CharField(db_index=True, max_length=50)), + ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), + ('year', models.IntegerField()), + ('microassessments_total', models.IntegerField(default=0, help_text='Total number of completed Microassessments in the business area in the past year')), + ('programmaticvisits_total', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), + ('followup_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), + ('completed_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), + ('completed_hact_audits', models.IntegerField(default=0, help_text='Total number of completed scheduled audits for the workspace.')), + ('completed_special_audits', models.IntegerField(default=0, help_text='Total number of completed special audits for the workspace. ')), + ], + options={ + 'verbose_name': 'HACT', + 'ordering': ('year', 'country_name'), + }, ), migrations.CreateModel( name='Intervention', @@ -49,6 +69,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('created', models.DateTimeField(auto_now=True)), ('updated', models.DateTimeField(null=True)), ('document_type', models.CharField(max_length=255, null=True)), @@ -71,6 +92,7 @@ class Migration(migrations.Migration): ('unicef_signatory_first_name', models.CharField(max_length=30, null=True)), ('unicef_signatory_last_name', models.CharField(max_length=30, null=True)), ('unicef_signatory_email', models.CharField(max_length=254, null=True)), + ('partner_name', models.CharField(max_length=200, null=True)), ('partner_signatory_title', models.CharField(max_length=64, null=True)), ('partner_signatory_first_name', models.CharField(max_length=64, null=True)), ('partner_signatory_last_name', models.CharField(max_length=64, null=True)), @@ -90,9 +112,9 @@ class Migration(migrations.Migration): ('unicef_signatory_id', models.IntegerField(null=True)), ], options={ + 'verbose_name': 'Intervention', 'ordering': ('country_name', 'title'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), ), migrations.CreateModel( name='PMPIndicators', @@ -100,6 +122,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('vendor_number', models.CharField(db_index=True, max_length=255, null=True)), ('business_area_code', models.CharField(db_index=True, max_length=100, null=True)), ('partner_name', models.CharField(db_index=True, max_length=255, null=True)), @@ -131,9 +154,9 @@ class Migration(migrations.Migration): ('updated', models.DateTimeField(null=True)), ], options={ + 'verbose_name': 'PMP Indicator', 'ordering': ('country_name', 'partner_name'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), ), migrations.CreateModel( name='UserStats', @@ -141,6 +164,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('country_name', models.CharField(db_index=True, max_length=50)), ('schema_name', models.CharField(db_index=True, max_length=50)), + ('last_modify_date', models.DateTimeField(auto_now=True, auto_now_add=True)), ('month', month_field.models.MonthField(verbose_name='Month Value')), ('total', models.IntegerField(default=0, verbose_name='Total users')), ('unicef', models.IntegerField(default=0, verbose_name='UNICEF uswers')), @@ -148,8 +172,20 @@ class Migration(migrations.Migration): ('unicef_logins', models.IntegerField(default=0, verbose_name='Number of UNICEF logins')), ], options={ + 'verbose_name': 'User Access Statistics', 'ordering': ('-month', 'country_name'), }, - bases=(models.Model, unicef_security.models.TimeStampedModel), + ), + migrations.AlterUniqueTogether( + name='userstats', + unique_together={('country_name', 'month')}, + ), + migrations.AlterUniqueTogether( + name='hact', + unique_together={('year', 'country_name')}, + ), + migrations.AlterUniqueTogether( + name='famindicator', + unique_together={('month', 'country_name')}, ), ] diff --git a/src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py b/src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py deleted file mode 100644 index 05810c918..000000000 --- a/src/etools_datamart/apps/data/migrations/0002_intervention_partner_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-06 13:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('data', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='intervention', - name='partner_name', - field=models.CharField(max_length=200, null=True), - ), - ] diff --git a/src/etools_datamart/apps/data/migrations/0003_hact.py b/src/etools_datamart/apps/data/migrations/0003_hact.py deleted file mode 100644 index 486edd3b9..000000000 --- a/src/etools_datamart/apps/data/migrations/0003_hact.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-09 17:08 - -import unicef_security.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('data', '0002_intervention_partner_name'), - ] - - operations = [ - migrations.CreateModel( - name='HACT', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('country_name', models.CharField(db_index=True, max_length=50)), - ('schema_name', models.CharField(db_index=True, max_length=50)), - ('year', models.IntegerField()), - ('microassessments_total', models.IntegerField(default=0, help_text='Total number of completed Microassessments in the business area in the past year')), - ('programmaticvisits_total', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), - ('followup_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), - ('completed_spotcheck', models.IntegerField(default=0, help_text='Total number of completed Programmatic visits in the business area')), - ('completed_hact_audits', models.IntegerField(default=0, help_text='Total number of completed scheduled audits for the workspace.')), - ('completed_special_audits', models.IntegerField(default=0, help_text='Total number of completed special audits for the workspace. ')), - ], - options={ - 'abstract': False, - }, - bases=(models.Model, unicef_security.models.TimeStampedModel), - ), - ] diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index fd40300dc..3891c199b 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -1,8 +1,8 @@ from celery.local import class_property +from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models import QuerySet from django.db.models.manager import BaseManager -from unicef_security.models import TimeStampedModel class DataMartQuerySet(QuerySet): @@ -14,19 +14,26 @@ def filter_schemas(self, *schemas): class DataMartManager(BaseManager.from_queryset(DataMartQuerySet)): - def truncate(self): - self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) + pass + # def truncate(self): + # self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) -class DataMartModel(models.Model, TimeStampedModel): +class DataMartModel(models.Model): country_name = models.CharField(max_length=50, db_index=True) schema_name = models.CharField(max_length=50, db_index=True) + last_modify_date = models.DateTimeField(blank=True, auto_now=True) class Meta: abstract = True objects = DataMartManager() + @class_property + def service(self): + from unicef_rest_framework.models import Service + return Service.objects.get(source_model=ContentType.objects.get_for_model(self)) + @class_property def linked_services(self): from unicef_rest_framework.models import Service diff --git a/src/etools_datamart/apps/data/models/fam.py b/src/etools_datamart/apps/data/models/fam.py index 11084fc49..c4445a04b 100644 --- a/src/etools_datamart/apps/data/models/fam.py +++ b/src/etools_datamart/apps/data/models/fam.py @@ -27,3 +27,4 @@ class FAMIndicator(DataMartModel): class Meta: ordering = ('month', 'country_name') unique_together = ('month', 'country_name') + verbose_name = "FAM Indicator" diff --git a/src/etools_datamart/apps/data/models/hact.py b/src/etools_datamart/apps/data/models/hact.py index bf43fb2d6..b0615b667 100644 --- a/src/etools_datamart/apps/data/models/hact.py +++ b/src/etools_datamart/apps/data/models/hact.py @@ -21,3 +21,4 @@ class HACT(DataMartModel): class Meta: ordering = ('year', 'country_name') unique_together = ('year', 'country_name') + verbose_name = "HACT" diff --git a/src/etools_datamart/apps/data/models/intervention.py b/src/etools_datamart/apps/data/models/intervention.py index ddda23684..ccd9bcb3d 100644 --- a/src/etools_datamart/apps/data/models/intervention.py +++ b/src/etools_datamart/apps/data/models/intervention.py @@ -58,3 +58,4 @@ class Intervention(DataMartModel): class Meta: ordering = ('country_name', 'title') + verbose_name = "Intervention" diff --git a/src/etools_datamart/apps/data/models/pmp.py b/src/etools_datamart/apps/data/models/pmp.py index 135a71980..fa6783a36 100644 --- a/src/etools_datamart/apps/data/models/pmp.py +++ b/src/etools_datamart/apps/data/models/pmp.py @@ -48,3 +48,4 @@ class PMPIndicators(DataMartModel): class Meta: ordering = ('country_name', 'partner_name') + verbose_name = "PMP Indicator" diff --git a/src/etools_datamart/apps/data/models/user.py b/src/etools_datamart/apps/data/models/user.py index 7f94c8b5d..c9a239918 100644 --- a/src/etools_datamart/apps/data/models/user.py +++ b/src/etools_datamart/apps/data/models/user.py @@ -14,3 +14,4 @@ class UserStats(DataMartModel): class Meta: ordering = ('-month', 'country_name') unique_together = ('country_name', 'month') + verbose_name = "User Access Statistics" diff --git a/src/etools_datamart/apps/etl/admin.py b/src/etools_datamart/apps/etl/admin.py index 7be70cd58..2130f383d 100644 --- a/src/etools_datamart/apps/etl/admin.py +++ b/src/etools_datamart/apps/etl/admin.py @@ -18,12 +18,15 @@ @register(models.EtlTask) class EtlTaskAdmin(TruncateTableMixin, admin.ModelAdmin): - list_display = ('task', 'timestamp', 'result', 'time', + list_display = ('task', 'last_run', 'status', 'time', 'last_success', 'last_failure', 'lock', 'scheduling', 'queue_task') - readonly_fields = ('task', 'timestamp', 'result', 'elapsed', 'time', - 'last_success', 'last_failure', 'table_name', 'content_type') - date_hierarchy = 'timestamp' + readonly_fields = ('task', 'last_run', + 'last_success', 'last_failure', 'last_changes', + 'results', 'elapsed', 'time', 'status', + 'table_name', 'content_type', + ) + date_hierarchy = 'last_run' actions = None def scheduling(self, obj): @@ -45,6 +48,7 @@ def queue_task(self, obj): url = reverse('admin:%s_%s_queue' % (opts.app_label, opts.model_name), args=[obj.id]) return format_html(f'queue') + queue_task.verbse_name = 'queue' def has_add_permission(self, request): @@ -77,6 +81,7 @@ def queue(self, request, pk): self.message_user(request, f"Task '{obj.task}' queued", messages.SUCCESS) except Exception as e: # pragma: no cover self.message_user(request, f"Cannot queue '{obj.task}': {e}", messages.ERROR) + return HttpResponseRedirect(reverse("admin:etl_etltask_changelist")) @action() def unlock(self, request, pk): diff --git a/src/etools_datamart/apps/etl/migrations/0001_initial.py b/src/etools_datamart/apps/etl/migrations/0001_initial.py index 9743c7ab6..034d0d20f 100644 --- a/src/etools_datamart/apps/etl/migrations/0001_initial.py +++ b/src/etools_datamart/apps/etl/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-29 08:24 +import django.contrib.postgres.fields.jsonb import django.db.models.deletion from django.db import migrations, models @@ -14,17 +15,22 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='TaskLog', + name='EtlTask', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('task', models.CharField(max_length=200, unique=True)), - ('timestamp', models.DateTimeField(null=True)), - ('result', models.CharField(max_length=200)), + ('last_run', models.DateTimeField(help_text='last execution time', null=True)), + ('status', models.CharField(max_length=200)), ('elapsed', models.IntegerField(null=True)), - ('last_success', models.DateTimeField(null=True)), - ('last_failure', models.DateTimeField(null=True)), + ('last_success', models.DateTimeField(help_text='last successully execution time', null=True)), + ('last_failure', models.DateTimeField(help_text='last failure execution time', null=True)), + ('last_changes', models.DateTimeField(help_text='last time data have been changed', null=True)), ('table_name', models.CharField(max_length=200, null=True)), + ('results', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)), ('content_type', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), ], + options={ + 'get_latest_by': 'last_run', + }, ), ] diff --git a/src/etools_datamart/apps/etl/migrations/0002_auto_20181119_2028.py b/src/etools_datamart/apps/etl/migrations/0002_auto_20181119_2028.py deleted file mode 100644 index 3306e10fd..000000000 --- a/src/etools_datamart/apps/etl/migrations/0002_auto_20181119_2028.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.1.3 on 2018-11-19 20:28 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('etl', '0001_initial'), - ] - - operations = [ - migrations.RenameModel( - old_name='TaskLog', - new_name='EtlTask', - ), - ] diff --git a/src/etools_datamart/apps/etl/models.py b/src/etools_datamart/apps/etl/models.py index 501f3e250..c950d129e 100644 --- a/src/etools_datamart/apps/etl/models.py +++ b/src/etools_datamart/apps/etl/models.py @@ -1,21 +1,22 @@ # -*- coding: utf-8 -*- from django.contrib.contenttypes.models import ContentType +from django.contrib.postgres.fields import JSONField from django.db import models -from django.db.models import Model from django.utils.functional import cached_property from django_celery_beat.models import PeriodicTask +from etools_datamart.apps.data.models.base import DataMartModel from etools_datamart.celery import app, ETLTask class TaskLogManager(models.Manager): - def get_for_model(self, model: Model): + def get_for_model(self, model: DataMartModel): return self.get(content_type=ContentType.objects.get_for_model(model)) def get_for_task(self, task: ETLTask): return self.get_or_create(task=task.name, defaults=dict(content_type=ContentType.objects.get_for_model(task.linked_model), - timestamp=None, + last_run=None, table_name=task.linked_model._meta.db_table))[0] def inspect(self): @@ -26,7 +27,7 @@ def inspect(self): t, created = self.get_or_create(task=task.name, defaults=dict( content_type=ContentType.objects.get_for_model(task.linked_model), - timestamp=None, + last_run=None, table_name=task.linked_model._meta.db_table)) results[created] += 1 new.append(t.id) @@ -36,21 +37,28 @@ def inspect(self): class EtlTask(models.Model): task = models.CharField(max_length=200, unique=True) - timestamp = models.DateTimeField(null=True) - result = models.CharField(max_length=200) + last_run = models.DateTimeField(null=True, help_text="last execution time") + status = models.CharField(max_length=200) elapsed = models.IntegerField(null=True) - last_success = models.DateTimeField(null=True) - last_failure = models.DateTimeField(null=True) + last_success = models.DateTimeField(null=True, help_text="last successully execution time") + last_failure = models.DateTimeField(null=True, help_text="last failure execution time") + last_changes = models.DateTimeField(null=True, help_text="last time data have been changed") table_name = models.CharField(max_length=200, null=True) content_type = models.ForeignKey(ContentType, models.CASCADE, null=True) + results = JSONField(blank=True, null=True) + objects = TaskLogManager() class Meta: - get_latest_by = 'timestamp' + get_latest_by = 'last_run' def __str__(self): - return f"{self.task} {self.result}" + return f"{self.task} {self.status}" + + @cached_property + def verbose_name(self): + return self.content_type.model_class()._meta.verbose_name @cached_property def periodic_task(self): diff --git a/src/etools_datamart/apps/etl/results.py b/src/etools_datamart/apps/etl/results.py new file mode 100644 index 000000000..5a4c8a11f --- /dev/null +++ b/src/etools_datamart/apps/etl/results.py @@ -0,0 +1,63 @@ +import json + +CREATED = 'created' +UPDATED = 'updated' +UNCHANGED = 'unchanged' + + +class EtlResult: + __slots__ = [CREATED, UPDATED, UNCHANGED] + + def __init__(self, updated=0, created=0, unchanged=0): + self.created = created + self.updated = updated + self.unchanged = unchanged + + def __repr__(self): + return repr(self.as_dict()) + + def incr(self, counter): + setattr(self, counter, getattr(self, counter) + 1) + + def as_dict(self): + return {'created': self.created, + 'updated': self.updated, + 'unchanged': self.unchanged} + + def __eq__(self, other): + if isinstance(other, EtlResult): + other = other.as_dict() + + if isinstance(other, dict): + return (self.created == other['created'] and + self.updated == other['updated'] and + self.unchanged == other['unchanged']) + return False + + +class EtlEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, EtlResult): + return { + '__type__': '__EtlResult__', + 'data': obj.as_dict() + } + else: + return json.JSONEncoder.default(self, obj) + + +def etl_decoder(obj): + if '__type__' in obj: + if obj['__type__'] == '__EtlResult__': + return EtlResult(**obj) + return obj + + +# Encoder function +def etl_dumps(obj): + return json.dumps(obj, cls=EtlEncoder) + + +# Decoder function +def etl_loads(obj): + return json.loads(obj, object_hook=etl_decoder) diff --git a/src/etools_datamart/apps/etl/tasks/etl.py b/src/etools_datamart/apps/etl/tasks/etl.py index cfa440876..d45bdc0fc 100644 --- a/src/etools_datamart/apps/etl/tasks/etl.py +++ b/src/etools_datamart/apps/etl/tasks/etl.py @@ -3,6 +3,7 @@ import logging from datetime import date, datetime +from crashlog.middleware import process_exception from django.db import connections from django.db.models import Sum from django.db.models.functions import Coalesce @@ -12,6 +13,7 @@ from etools_datamart.apps.data.models import HACT, Intervention, PMPIndicators from etools_datamart.apps.data.models.fam import FAMIndicator from etools_datamart.apps.data.models.user import UserStats +from etools_datamart.apps.etl.results import CREATED, EtlResult, UNCHANGED, UPDATED from etools_datamart.apps.etools.models import (AuditAudit, AuditEngagement, AuditMicroassessment, AuditSpecialaudit, AuditSpotcheck, AuthUser, HactAggregatehact, PartnersIntervention, PartnersPartnerorganization,) @@ -23,14 +25,41 @@ "load_pmp_indicator", "load_intervention"] +def is_record_changed(record, values): + other = type(record)(**values) + for field_name, field_value in values.items(): + if getattr(record, field_name) != getattr(other, field_name): + return True + return False + + +def process(Model, filters, values): + try: + existing, created = Model.objects.get_or_create(**filters, + defaults=values) + if created: + op = CREATED + else: + if is_record_changed(existing, values): + op = UPDATED + Model.objects.update_or_create(**filters, + defaults=values) + else: + op = UNCHANGED + return op + except Exception as e: + logging.exception(e) + process_exception(e) + raise + + @app.etl(HACT) def load_hact(): connection = connections['etools'] countries = connection.get_tenants() today = timezone.now() - created = {} + results = EtlResult() for country in countries: - created[country.name] = 0 connection.set_schemas([country.schema_name]) logger.info(u'Running on %s' % country.name) @@ -67,82 +96,120 @@ def load_hact(): completed_hact_audits=data['assurance_activities']['scheduled_audit'], completed_special_audits=data['assurance_activities']['special_audit'], ) - HACT.objects.update_or_create(year=today.year, - country_name=country.name, - schema_name=country.schema_name, - defaults=values) - - return created + op = process(HACT, filters=dict(year=today.year, + country_name=country.name, + schema_name=country.schema_name), + values=values) + results.incr(op) + # existing, created = HACT.objects.get_or_create(year=today.year, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # HACT.objects.update_or_create(year=today.year, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # else: + # results.unchanged += 1 + + return results @app.etl(PMPIndicators) -def load_pmp_indicator(): +def load_pmp_indicator() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() - PMPIndicators.objects.truncate() base_url = 'https://etools.unicef.org' - created = {} + results = EtlResult() for country in countries: - created[country.name] = 0 connection.set_schemas([country.schema_name]) logger.info(u'Running on %s' % country.name) for partner in PartnersPartnerorganization.objects.all(): - # .prefetch_related( - # 'partnerspartnerorganization_partners_corevaluesassessment_partner_id'): - # .select_related('partnersintervention_partners_interventionbudget_intervention_id') for intervention in PartnersIntervention.objects.filter(agreement__partner=partner): planned_budget = getattr(intervention, 'partnersintervention_partners_interventionbudget_intervention_id', None) fr_currencies = intervention.frs.all().values_list('currency', flat=True).distinct() has_assessment = bool(getattr(partner.current_core_value_assessment, 'assessment', False)) - PMPIndicators.objects.create( - country_id=country.pk, - partner_id=partner.pk, - intervention_id=intervention.pk, - **{ - 'country_name': country.name, - 'schema_name': country.schema_name, - 'business_area_code': country.business_area_code, - 'partner_name': partner.name, - 'partner_type': partner.cso_type, - 'vendor_number': partner.vendor_number, - - 'pd_ssfa_ref': intervention.number.replace(',', '-'), - 'pd_ssfa_status': intervention.status.title(), - 'pd_ssfa_start_date': intervention.start, - 'pd_ssfa_creation_date': intervention.created, - 'pd_ssfa_end_date': intervention.end, - - 'cash_contribution': intervention.total_unicef_cash or 0, - 'supply_contribution': intervention.total_in_kind_amount or 0, - 'total_budget': intervention.total_budget or 0, - 'unicef_budget': intervention.total_unicef_budget or 0, - - 'currency': intervention.planned_budget.currency if planned_budget else '-', - 'partner_contribution': intervention.planned_budget.partner_contribution if planned_budget else '-', - 'unicef_cash': intervention.planned_budget.unicef_cash if planned_budget else '-', - 'in_kind_amount': intervention.planned_budget.in_kind_amount if planned_budget else '-', - 'total': intervention.planned_budget.total if planned_budget else '-', - 'fr_numbers_against_pd_ssfa': ' - '.join([fh.fr_number for fh in intervention.frs.all()]), - 'fr_currencies': ', '.join(fr for fr in fr_currencies), - 'sum_of_all_fr_planned_amount': intervention.frs.aggregate( - total=Coalesce(Sum('intervention_amt'), 0))['total'] if fr_currencies.count() <= 1 else '-', - 'core_value_attached': has_assessment, - 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), - 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), - }) - created[country.name] += 1 - return created + values = {'country_name': country.name, + 'schema_name': country.schema_name, + 'business_area_code': country.business_area_code, + 'partner_name': partner.name, + 'partner_type': partner.cso_type, + 'vendor_number': partner.vendor_number, + + 'pd_ssfa_ref': intervention.number.replace(',', '-'), + 'pd_ssfa_status': intervention.status.title(), + 'pd_ssfa_start_date': intervention.start, + 'pd_ssfa_creation_date': intervention.created, + 'pd_ssfa_end_date': intervention.end, + + 'cash_contribution': intervention.total_unicef_cash or 0, + 'supply_contribution': intervention.total_in_kind_amount or 0, + 'total_budget': intervention.total_budget or 0, + 'unicef_budget': intervention.total_unicef_budget or 0, + + 'currency': intervention.planned_budget.currency if planned_budget else '-', + 'partner_contribution': intervention.planned_budget.partner_contribution if planned_budget else '-', + 'unicef_cash': intervention.planned_budget.unicef_cash if planned_budget else '-', + 'in_kind_amount': intervention.planned_budget.in_kind_amount if planned_budget else '-', + 'total': intervention.planned_budget.total if planned_budget else '-', + 'fr_numbers_against_pd_ssfa': ' - '.join([fh.fr_number for fh in intervention.frs.all()]), + 'fr_currencies': ', '.join(fr for fr in fr_currencies), + 'sum_of_all_fr_planned_amount': intervention.frs.aggregate( + total=Coalesce(Sum('intervention_amt'), 0))[ + 'total'] if fr_currencies.count() <= 1 else '-', + 'core_value_attached': has_assessment, + 'partner_link': '{}/pmp/partners/{}/details'.format(base_url, partner.pk), + 'intervention_link': '{}/pmp/interventions/{}/details'.format(base_url, intervention.pk), + } + op = process(PMPIndicators, filters=dict(country_name=country.name, + schema_name=country.schema_name, + partner_id=partner.pk, + intervention_id=intervention.pk), + values=values) + results.incr(op) + # existing, created = PMPIndicators.objects.get_or_create(country_name=country.name, + # schema_name=country.schema_name, + # country_id=partner.id, + # partner_id=partner.pk, + # intervention_id=intervention.pk, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # PMPIndicators.objects.update_or_create(country_name=country.name, + # schema_name=country.schema_name, + # country_id=partner.id, + # partner_id=partner.pk, + # intervention_id=intervention.pk, + # defaults=values) + # else: + # results.unchanged += 1 + + return results + # PMPIndicators.objects.create( + # country_id=country.pk, + # partner_id=partner.pk, + # intervention_id=intervention.pk) + # created[country.name] += 1 + # + # return created @app.etl(Intervention) -def load_intervention(): +def load_intervention() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() - Intervention.objects.truncate() - created = {} + results = EtlResult() for country in countries: connection.set_schemas([country.schema_name]) qs = PartnersIntervention.objects.all().select_related('agreement', @@ -152,133 +219,183 @@ def load_intervention(): ) num = 0 for num, record in enumerate(qs, 1): - Intervention.objects.create(country_name=country.name, - schema_name=country.schema_name, - number=record.number, - title=record.title, - status=record.status, - start_date=record.start, - end_date=record.end, - review_date_prc=record.review_date_prc, - prc_review_document=record.prc_review_document, - partner_name=record.agreement.partner.name, - agreement_id=record.agreement.pk, - partner_authorized_officer_signatory_id=get_attr(record, - 'partner_authorized_officer_signatory.pk'), - country_programme_id=get_attr(record, 'country_programme.pk'), - intervention_id=record.pk, - unicef_signatory_id=get_attr(record, 'unicef_signatory.pk'), - - signed_by_unicef_date=record.signed_by_unicef_date, - signed_by_partner_date=record.signed_by_partner_date, - population_focus=record.population_focus, - signed_pd_document=record.signed_pd_document, - - submission_date=record.submission_date, - submission_date_prc=record.submission_date_prc, - - unicef_signatory_first_name=get_attr(record, - 'unicef_signatory.first_name'), - unicef_signatory_last_name=get_attr(record, - 'unicef_signatory.last_name'), - unicef_signatory_email=get_attr(record, 'unicef_signatory.email'), - - partner_signatory_title=get_attr(record, - 'partner_authorized_officer_signatory.title'), - partner_signatory_first_name=get_attr(record, - 'partner_authorized_officer_signatory.first_name'), - partner_signatory_last_name=get_attr(record, - 'partner_authorized_officer_signatory.last_name'), - partner_signatory_email=get_attr(record, - 'partner_authorized_officer_signatory.email'), - partner_signatory_phone=get_attr(record, - 'partner_authorized_officer_signatory.phone'), - - partner_focal_point_title=get_attr(record, - 'partner_focal_point.title'), - partner_focal_point_first_name=get_attr(record, - 'partner_focal_point.first_name'), - partner_focal_point_last_name=get_attr(record, - 'partner_focal_point.last_name'), - partner_focal_point_email=get_attr(record, - 'partner_focal_point.email'), - partner_focal_point_phone=get_attr(record, - 'partner_focal_point.phone'), - - metadata=record.metadata, - document_type=record.document_type, - updated=record.modified, - created=record.created, - - ) - created[country.name] = num - - return created + values = dict(number=record.number, + title=record.title, + status=record.status, + start_date=record.start, + end_date=record.end, + review_date_prc=record.review_date_prc, + prc_review_document=record.prc_review_document, + partner_name=record.agreement.partner.name, + partner_authorized_officer_signatory_id=get_attr(record, + 'partner_authorized_officer_signatory.pk'), + country_programme_id=get_attr(record, 'country_programme.pk'), + intervention_id=record.pk, + unicef_signatory_id=get_attr(record, 'unicef_signatory.pk'), + + signed_by_unicef_date=record.signed_by_unicef_date, + signed_by_partner_date=record.signed_by_partner_date, + population_focus=record.population_focus, + signed_pd_document=record.signed_pd_document, + + submission_date=record.submission_date, + submission_date_prc=record.submission_date_prc, + + unicef_signatory_first_name=get_attr(record, + 'unicef_signatory.first_name'), + unicef_signatory_last_name=get_attr(record, + 'unicef_signatory.last_name'), + unicef_signatory_email=get_attr(record, 'unicef_signatory.email'), + + partner_signatory_title=get_attr(record, + 'partner_authorized_officer_signatory.title'), + partner_signatory_first_name=get_attr(record, + 'partner_authorized_officer_signatory.first_name'), + partner_signatory_last_name=get_attr(record, + 'partner_authorized_officer_signatory.last_name'), + partner_signatory_email=get_attr(record, + 'partner_authorized_officer_signatory.email'), + partner_signatory_phone=get_attr(record, + 'partner_authorized_officer_signatory.phone'), + + partner_focal_point_title=get_attr(record, + 'partner_focal_point.title'), + partner_focal_point_first_name=get_attr(record, + 'partner_focal_point.first_name'), + partner_focal_point_last_name=get_attr(record, + 'partner_focal_point.last_name'), + partner_focal_point_email=get_attr(record, + 'partner_focal_point.email'), + partner_focal_point_phone=get_attr(record, + 'partner_focal_point.phone'), + + metadata=record.metadata, + document_type=record.document_type, + updated=record.modified, + created=record.created, + ) + op = process(Intervention, filters=dict(country_name=country.name, + schema_name=country.schema_name, + agreement_id=record.agreement.pk, + intervention_id=record.pk), + values=values) + results.incr(op) + + # existing, created = Intervention.objects.get_or_create(country_name=country.name, + # schema_name=country.schema_name, + # intervention_id=record.pk, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # Intervention.objects.update_or_create(country_name=country.name, + # schema_name=country.schema_name, + # intervention_id=record.pk, + # defaults=values) + # else: + # results.unchanged += 1 + + return results @app.etl(FAMIndicator) -def load_fam_indicator(): +def load_fam_indicator() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() engagements = (AuditSpotcheck, AuditAudit, AuditSpecialaudit, AuditMicroassessment) start_date = date.today() # + relativedelta(months=-1) - created = {} + results = EtlResult() for country in countries: - created[country.name] = 0 - connection.set_schemas([country.schema_name]) for model in engagements: - indicator, __ = FAMIndicator.objects.get_or_create(month=start_date, - country_name=country.name, - schema_name=country.schema_name) - + # indicator, created = FAMIndicator.objects.get_or_create(month=start_date, + # country_name=country.name, + # schema_name=country.schema_name) + # if created: + # results.created += 1 + # changed = created realname = "_".join(model._meta.db_table.split('_')[1:]) + values = {} for status, status_display in AuditEngagement.STATUSES: filter_dict = { 'engagement_ptr__status': status, 'engagement_ptr__start_date__month': start_date.month, 'engagement_ptr__start_date__year': start_date.year, } - try: - field_name = f"{realname}_{status_display}".replace(" ", "_").lower() - value = model.objects.filter(**filter_dict).count() - # just a safety check - if not hasattr(indicator, field_name): # pragma: no cover - raise ValueError(field_name) - setattr(indicator, field_name, value) - except Exception as e: # pragma: no cover - logger.error(e) - raise - indicator.save() - created[country.name] += 1 - - return created + field_name = f"{realname}_{status_display}".replace(" ", "_").lower() + value = model.objects.filter(**filter_dict).count() + values[field_name] = value + # try: + # field_name = f"{realname}_{status_display}".replace(" ", "_").lower() + # value = model.objects.filter(**filter_dict).count() + # # just a safety check + # if not hasattr(indicator, field_name): # pragma: no cover + # raise ValueError(field_name) + # if getattr(indicator, field_name) == value: + # changed = False + # else: + # changed = changed and True + # setattr(indicator, field_name, value) + # except Exception as e: # pragma: no cover + # logger.error(e) + # raise + op = process(FAMIndicator, filters=dict(month=start_date, + country_name=country.name, + schema_name=country.schema_name), + values=values) + results.incr(op) + return results @app.etl(UserStats) -def load_user_report(): +def load_user_report() -> EtlResult: connection = connections['etools'] countries = connection.get_tenants() today = date.today() first_of_month = datetime(today.year, today.month, 1) - created = {} + results = EtlResult() for country in countries: - created[country.name] = 0 connection.set_schemas([country.schema_name]) base = AuthUser.objects.filter(profile__country=country) - UserStats.objects.update_or_create(month=first_of_month, - country_name=country.name, - schema_name=country.schema_name, - defaults={ - 'total': base.count(), - 'unicef': base.filter(email__endswith='@unicef.org').count(), - 'logins': base.filter( - last_login__month=first_of_month.month).count(), - 'unicef_logins': base.filter( - last_login__month=first_of_month.month, - email__endswith='@unicef.org').count(), - }) - created[country.name] += 1 - - return created + values = { + 'total': base.count(), + 'unicef': base.filter(email__endswith='@unicef.org').count(), + 'logins': base.filter( + last_login__month=first_of_month.month).count(), + 'unicef_logins': base.filter( + last_login__month=first_of_month.month, + email__endswith='@unicef.org').count(), + } + op = process(UserStats, filters=dict(month=first_of_month, + country_name=country.name, + schema_name=country.schema_name, ), + values=values) + results.incr(op) + + # existing, created = UserStats.objects.get_or_create(month=first_of_month, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # if created: + # results.created += 1 + # else: + # if is_record_changed(existing, values): + # results.updated += 1 + # UserStats.objects.update_or_create(month=first_of_month, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # else: + # results.unchanged += 1 + # + return results + # UserStats.objects.update_or_create(month=first_of_month, + # country_name=country.name, + # schema_name=country.schema_name, + # defaults=values) + # created[country.name] += 1 + # + # return created diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index 70ab9a837..953465f85 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -9,8 +9,9 @@ from django.core.management import call_command from django.core.management.base import BaseCommand from django.utils.module_loading import import_string -from django_celery_beat.models import CrontabSchedule, PeriodicTask +from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask from humanize import naturaldelta +from post_office.models import EmailTemplate from redisboard.models import RedisServer from strategy_field.utils import fqn from unicef_rest_framework.models.acl import GroupAccessControl @@ -18,6 +19,51 @@ from etools_datamart.apps.etl.models import EtlTask from etools_datamart.celery import app +MAIL = r"""Dear {{user.label}}, + +On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`. +Please visit {{base_url}}{{service.endpoint}} + +— +You are receiving this because you are subscribed to this thread. +To unsubscribe, change your preferences in {{base_url}}{% url 'monitor' %} +""" + +MAIL_HTML = r"""
Dear {{user.label}},
+
 
+
On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`.
+
You can view data following this link + or download the as excel
+
 
+
 
+
 
+
-
+
You are receiving this because you are subscribed to this thread.
+
To unsubscribe, change your preferences in Datamart Monitor
+""" + +MAIL_ATTACHMENT = r"""Dear {{user.label}}, + +On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`. +You can find here in attachment a excel file with updated data + +— +You are receiving this because you are subscribed to this thread. +To unsubscribe, change your preferences in {{base_url}}{% url 'monitor' %} +""" + +MAIL_ATTACHMENT_HTML = r"""
Dear {{user.label}},
+
 
+
On {{etl.last_changes|date:"M d, Y"}}, datamart has detected changes in dataset `{{verbose_name}}`.
+
Attached to this email you can find excel file with updated data
+
 
+
 
+
 
+
-
+
You are receiving this because you are subscribed to this thread.
+
To unsubscribe, change your preferences in Datamart Monitor
+""" + class Command(BaseCommand): help = "My shiny new management command." @@ -135,6 +181,7 @@ def handle(self, *args, **options): if options['tasks'] or _all or options['refresh']: midnight, __ = CrontabSchedule.objects.get_or_create(minute=0, hour=0) + every_minute, __ = IntervalSchedule.objects.get_or_create(every=1, period=IntervalSchedule.MINUTES) tasks = app.get_all_etls() counters = {True: 0, False: 0} @@ -153,10 +200,27 @@ def handle(self, *args, **options): self.stdout.write( f"{PeriodicTask.objects.count()} tasks found. {counters[True]} new. {counters[False]} deleted") + PeriodicTask.objects.get_or_create(task='send_queued_mail', + defaults={'name': 'process mail queue', + 'interval': every_minute}) + + EmailTemplate.objects.get_or_create(name='dataset_changed_attachment', + defaults=dict(subject='Dataset changed', + content=MAIL_ATTACHMENT, + html_content=MAIL_ATTACHMENT_HTML)) + + EmailTemplate.objects.get_or_create(name='dataset_changed', + defaults=dict(subject='Dataset changed', + content=MAIL, + html_content=MAIL_HTML)) + if options['refresh']: self.stdout.write("Refreshing datamart...") for task in PeriodicTask.objects.all()[1:]: - etl = import_string(task.task) + try: + etl = import_string(task.task) + except ImportError: + continue self.stdout.write(f"Running {task.name}...", ending='\r') self.stdout.flush() diff --git a/src/etools_datamart/apps/subscriptions/__init__.py b/src/etools_datamart/apps/subscriptions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/subscriptions/admin.py b/src/etools_datamart/apps/subscriptions/admin.py new file mode 100644 index 000000000..854a0c404 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/admin.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +from admin_extra_urls.extras import ExtraUrlMixin +from django.contrib import admin +from django.contrib.admin import register + +from . import models + + +@register(models.Subscription) +class SubscriptionAdmin(ExtraUrlMixin, admin.ModelAdmin): + list_display = ('user', 'content_type', 'type') + list_filter = ('user', 'content_type', 'type') diff --git a/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py new file mode 100644 index 000000000..8fee9683c --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/migrations/0001_initial.py @@ -0,0 +1,25 @@ +# Generated by Django 2.1.3 on 2018-11-29 08:24 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='Subscription', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('type', models.IntegerField(choices=[(0, 'None'), (1, 'Email'), (2, 'Email+Excel')])), + ('kwargs', models.CharField(blank=True, default='', max_length=500)), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), + ], + ), + ] diff --git a/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py new file mode 100644 index 000000000..da9944c95 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/migrations/0002_auto_20181129_0824.py @@ -0,0 +1,27 @@ +# Generated by Django 2.1.3 on 2018-11-29 08:24 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('subscriptions', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='subscription', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='subscriptions', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterUniqueTogether( + name='subscription', + unique_together={('user', 'content_type', 'kwargs')}, + ), + ] diff --git a/src/etools_datamart/apps/subscriptions/migrations/__init__.py b/src/etools_datamart/apps/subscriptions/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/subscriptions/models.py b/src/etools_datamart/apps/subscriptions/models.py new file mode 100644 index 000000000..5fba886f3 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/models.py @@ -0,0 +1,104 @@ +import logging +from io import BytesIO + +from crashlog.middleware import process_exception +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.utils.functional import cached_property +from post_office import mail +from rest_framework.test import APIRequestFactory +from unicef_rest_framework.models import Service + +from etools_datamart.apps.etl.models import EtlTask + +logger = logging.getLogger(__name__) + + +class SubscriptionManager(models.Manager): + def notify(self, model): + ct = ContentType.objects.get_for_model(model) + etl = EtlTask.objects.filter(content_type=ct).first() + service = Service.objects.get(source_model=ct) + ret = [] + for subscription in self.filter(content_type=ct).exclude(type=Subscription.NONE): + logger.info(f"Process subscription {subscription}") + try: + if subscription.type in (Subscription.EXCEL, Subscription.PDF): + format = {Subscription.EXCEL: 'xlsx', + Subscription.PDF: 'pdf', + }[subscription.type] + rf = APIRequestFactory() + request = rf.get(f"{service.endpoint}?format={format}") + request.user = subscription.user + request.api_info = {} # this is set my the middleware, so we must set manually here + response = service.viewset.as_view({'get': 'list'})(request) + response.render() + + # check headers set in ApiMiddleware in request.api_info + request.api_info.update(dict(response.items())) + + attachments = { + f'{model._meta.verbose_name}.{format}': BytesIO(response.content), + } + template = 'dataset_changed_attachment' + else: + attachments = None + template = 'dataset_changed' + + ret.append(mail.send( + subscription.user.email, # List of email addresses also accepted + 'notification@datamart.unicef.io', + template=template, + context={'subscription': subscription, + 'user': subscription.user, + 'base_url': settings.ABSOLUTE_BASE_URL, + 'verbose_name': model._meta.verbose_name, + 'etl': etl, + 'model': ct.model, + 'service': service + }, + attachments=attachments + )) + except Exception as e: # pragma: no cover + logger.exception(e) + process_exception(e) + return ret + + +class Subscription(models.Model): + NONE = 0 + MESSAGE = 1 + EXCEL = 2 + PDF = 3 + + TYPES = ((NONE, 'None'), + (MESSAGE, 'Email'), + (EXCEL, 'Email+Excel'), + (PDF, 'Email+Pdf'), + ) + user = models.ForeignKey(settings.AUTH_USER_MODEL, models.CASCADE, + related_name='subscriptions') + type = models.IntegerField(choices=TYPES) + content_type = models.ForeignKey(ContentType, models.CASCADE) + kwargs = models.CharField(max_length=500, blank=True, null=False, default='') + + objects = SubscriptionManager() + + class Meta: + unique_together = ('user', 'content_type', 'kwargs') + + def __str__(self): + return f"#{self.pk} {self.user} {self.get_type_display()} {self.content_type}" + + @cached_property + def endpoint(self): + return self.content_type.model_class().service.endpoint + + @cached_property + def service(self): + return self.content_type.model_class().service + + @cached_property + def viewset(self): + return self.content_type.model_class().service.viewset diff --git a/src/etools_datamart/apps/subscriptions/tasks.py b/src/etools_datamart/apps/subscriptions/tasks.py new file mode 100644 index 000000000..846561377 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/tasks.py @@ -0,0 +1,8 @@ +from django.core.management import call_command + +from etools_datamart.celery import app + + +@app.task(name='send_queued_mail') +def send_queued_mail(): # pragma: no cover + call_command('send_queued_mail') diff --git a/src/etools_datamart/apps/subscriptions/templates/subscription_select.html b/src/etools_datamart/apps/subscriptions/templates/subscription_select.html new file mode 100644 index 000000000..74608fa21 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/templates/subscription_select.html @@ -0,0 +1,6 @@ + diff --git a/src/etools_datamart/apps/subscriptions/templatetags/__init__.py b/src/etools_datamart/apps/subscriptions/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py b/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py new file mode 100644 index 000000000..68ca19cbe --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/templatetags/subscriptions.py @@ -0,0 +1,16 @@ +from django import template + +from etools_datamart.apps.subscriptions.models import Subscription + +register = template.Library() + + +@register.inclusion_tag('subscription_select.html', takes_context=True) +def subscription_select(context, task): + user = context['user'] + s = Subscription.objects.filter(content_type=task.content_type, + kwargs='', + user=user).first() + return {'options': Subscription.TYPES, + 'task': task, + 'subscription': s} diff --git a/src/etools_datamart/apps/subscriptions/urls.py b/src/etools_datamart/apps/subscriptions/urls.py new file mode 100644 index 000000000..1df0179d1 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/urls.py @@ -0,0 +1,36 @@ +import codecs +from functools import wraps + +from django.contrib.auth.decorators import login_required +from django.http import HttpResponse +from django.urls import path + +from etools_datamart.apps.subscriptions.views import subscribe + + +def http_basic_auth(func): + @wraps(func) + def _decorator(request, *args, **kwargs): + from django.contrib.auth import authenticate, login + if "HTTP_AUTHORIZATION" in request.META: + authmeth, auth = request.META["HTTP_AUTHORIZATION"].split(" ", 1) + if authmeth.lower() == "basic": + auth = codecs.decode(auth.encode("utf8").strip(), "base64").decode() + username, password = auth.split(":", 1) + user = authenticate(username=username, password=password) + if user: + login(request, user, backend='django.contrib.auth.backends.RemoteUserBackend') + else: + return HttpResponse(status=401) + return func(request, *args, **kwargs) + + return _decorator + + +def http_basic_login(func): + return http_basic_auth(login_required(func)) + + +urlpatterns = [ + path(r'subscribe//', http_basic_login(subscribe), name='subscribe'), +] diff --git a/src/etools_datamart/apps/subscriptions/views.py b/src/etools_datamart/apps/subscriptions/views.py new file mode 100644 index 000000000..af8d864e4 --- /dev/null +++ b/src/etools_datamart/apps/subscriptions/views.py @@ -0,0 +1,51 @@ +import json + +from django.forms import ModelForm +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt + +from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.subscriptions.models import Subscription + + +class SubscriptionForm(ModelForm): + class Meta: + model = Subscription + fields = ['type', 'kwargs'] + + +@csrf_exempt +def subscribe(request, etl_id): + code = 200 + values = {'status': "", 'detail': ""} + try: + user = request.user + payload = json.loads(request.body) + form = SubscriptionForm(data=payload) + if form.is_valid(): + etl = EtlTask.objects.get(id=etl_id) + s, created = Subscription.objects.update_or_create(user=user, + content_type=etl.content_type, + kwargs=payload.get("kwargs", ''), + defaults={ + 'type': payload["type"] + }) + values['status'] = {True: "created", False: "updated"}[created] + values['detail'] = {"id": s.id, + "type": s.type, + "type_label": s.get_type_display(), + } + else: + values['detail'] = "Invalid request" + values['error'] = form.errors + code = 400 + + except EtlTask.DoesNotExist as e: + values['detail'] = f"Invalid task id `{etl_id}`" + values['error'] = str(e) + code = 404 + except Exception as e: + values['error'] = type(e).__name__ + values['detail'] = str(e) + code = 500 + return JsonResponse(values, status=code) diff --git a/src/etools_datamart/apps/tracking/migrations/0001_initial.py b/src/etools_datamart/apps/tracking/migrations/0001_initial.py index 57174107b..425d9911c 100644 --- a/src/etools_datamart/apps/tracking/migrations/0001_initial.py +++ b/src/etools_datamart/apps/tracking/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.utils.timezone import strategy_field.fields @@ -32,8 +32,8 @@ class Migration(migrations.Migration): ('viewset', strategy_field.fields.StrategyClassField(blank=True, null=True)), ], options={ - 'verbose_name': 'Log', - 'verbose_name_plural': 'Logs', + 'verbose_name': 'Access Log', + 'verbose_name_plural': 'Access Log', 'ordering': ('-id',), 'get_latest_by': 'requested_at', }, diff --git a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181031_1959.py b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py similarity index 97% rename from src/etools_datamart/apps/tracking/migrations/0002_auto_20181031_1959.py rename to src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py index 28bc34837..f98434cb8 100644 --- a/src/etools_datamart/apps/tracking/migrations/0002_auto_20181031_1959.py +++ b/src/etools_datamart/apps/tracking/migrations/0002_auto_20181129_0824.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.db.models.deletion from django.conf import settings @@ -10,9 +10,9 @@ class Migration(migrations.Migration): initial = True dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('unicef_rest_framework', '0001_initial'), ('tracking', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ diff --git a/src/etools_datamart/apps/web/static/jquery-3.3.1.min.js b/src/etools_datamart/apps/web/static/jquery-3.3.1.min.js new file mode 100644 index 000000000..4d9b3a258 --- /dev/null +++ b/src/etools_datamart/apps/web/static/jquery-3.3.1.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.3.1 | (c) JS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(e,t){"use strict";var n=[],r=e.document,i=Object.getPrototypeOf,o=n.slice,a=n.concat,s=n.push,u=n.indexOf,l={},c=l.toString,f=l.hasOwnProperty,p=f.toString,d=p.call(Object),h={},g=function e(t){return"function"==typeof t&&"number"!=typeof t.nodeType},y=function e(t){return null!=t&&t===t.window},v={type:!0,src:!0,noModule:!0};function m(e,t,n){var i,o=(t=t||r).createElement("script");if(o.text=e,n)for(i in v)n[i]&&(o[i]=n[i]);t.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?l[c.call(e)]||"object":typeof e}var b="3.3.1",w=function(e,t){return new w.fn.init(e,t)},T=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;w.fn=w.prototype={jquery:"3.3.1",constructor:w,length:0,toArray:function(){return o.call(this)},get:function(e){return null==e?o.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=w.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return w.each(this,e)},map:function(e){return this.pushStack(w.map(this,function(t,n){return e.call(t,n,t)}))},slice:function(){return this.pushStack(o.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(n>=0&&n0&&t-1 in e)}var E=function(e){var t,n,r,i,o,a,s,u,l,c,f,p,d,h,g,y,v,m,x,b="sizzle"+1*new Date,w=e.document,T=0,C=0,E=ae(),k=ae(),S=ae(),D=function(e,t){return e===t&&(f=!0),0},N={}.hasOwnProperty,A=[],j=A.pop,q=A.push,L=A.push,H=A.slice,O=function(e,t){for(var n=0,r=e.length;n+~]|"+M+")"+M+"*"),z=new RegExp("="+M+"*([^\\]'\"]*?)"+M+"*\\]","g"),X=new RegExp(W),U=new RegExp("^"+R+"$"),V={ID:new RegExp("^#("+R+")"),CLASS:new RegExp("^\\.("+R+")"),TAG:new RegExp("^("+R+"|[*])"),ATTR:new RegExp("^"+I),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+P+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},G=/^(?:input|select|textarea|button)$/i,Y=/^h\d$/i,Q=/^[^{]+\{\s*\[native \w/,J=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,K=/[+~]/,Z=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ee=function(e,t,n){var r="0x"+t-65536;return r!==r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},te=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ne=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},re=function(){p()},ie=me(function(e){return!0===e.disabled&&("form"in e||"label"in e)},{dir:"parentNode",next:"legend"});try{L.apply(A=H.call(w.childNodes),w.childNodes),A[w.childNodes.length].nodeType}catch(e){L={apply:A.length?function(e,t){q.apply(e,H.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function oe(e,t,r,i){var o,s,l,c,f,h,v,m=t&&t.ownerDocument,T=t?t.nodeType:9;if(r=r||[],"string"!=typeof e||!e||1!==T&&9!==T&&11!==T)return r;if(!i&&((t?t.ownerDocument||t:w)!==d&&p(t),t=t||d,g)){if(11!==T&&(f=J.exec(e)))if(o=f[1]){if(9===T){if(!(l=t.getElementById(o)))return r;if(l.id===o)return r.push(l),r}else if(m&&(l=m.getElementById(o))&&x(t,l)&&l.id===o)return r.push(l),r}else{if(f[2])return L.apply(r,t.getElementsByTagName(e)),r;if((o=f[3])&&n.getElementsByClassName&&t.getElementsByClassName)return L.apply(r,t.getElementsByClassName(o)),r}if(n.qsa&&!S[e+" "]&&(!y||!y.test(e))){if(1!==T)m=t,v=e;else if("object"!==t.nodeName.toLowerCase()){(c=t.getAttribute("id"))?c=c.replace(te,ne):t.setAttribute("id",c=b),s=(h=a(e)).length;while(s--)h[s]="#"+c+" "+ve(h[s]);v=h.join(","),m=K.test(e)&&ge(t.parentNode)||t}if(v)try{return L.apply(r,m.querySelectorAll(v)),r}catch(e){}finally{c===b&&t.removeAttribute("id")}}}return u(e.replace(B,"$1"),t,r,i)}function ae(){var e=[];function t(n,i){return e.push(n+" ")>r.cacheLength&&delete t[e.shift()],t[n+" "]=i}return t}function se(e){return e[b]=!0,e}function ue(e){var t=d.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function le(e,t){var n=e.split("|"),i=n.length;while(i--)r.attrHandle[n[i]]=t}function ce(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function fe(e){return function(t){return"input"===t.nodeName.toLowerCase()&&t.type===e}}function pe(e){return function(t){var n=t.nodeName.toLowerCase();return("input"===n||"button"===n)&&t.type===e}}function de(e){return function(t){return"form"in t?t.parentNode&&!1===t.disabled?"label"in t?"label"in t.parentNode?t.parentNode.disabled===e:t.disabled===e:t.isDisabled===e||t.isDisabled!==!e&&ie(t)===e:t.disabled===e:"label"in t&&t.disabled===e}}function he(e){return se(function(t){return t=+t,se(function(n,r){var i,o=e([],n.length,t),a=o.length;while(a--)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}function ge(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}n=oe.support={},o=oe.isXML=function(e){var t=e&&(e.ownerDocument||e).documentElement;return!!t&&"HTML"!==t.nodeName},p=oe.setDocument=function(e){var t,i,a=e?e.ownerDocument||e:w;return a!==d&&9===a.nodeType&&a.documentElement?(d=a,h=d.documentElement,g=!o(d),w!==d&&(i=d.defaultView)&&i.top!==i&&(i.addEventListener?i.addEventListener("unload",re,!1):i.attachEvent&&i.attachEvent("onunload",re)),n.attributes=ue(function(e){return e.className="i",!e.getAttribute("className")}),n.getElementsByTagName=ue(function(e){return e.appendChild(d.createComment("")),!e.getElementsByTagName("*").length}),n.getElementsByClassName=Q.test(d.getElementsByClassName),n.getById=ue(function(e){return h.appendChild(e).id=b,!d.getElementsByName||!d.getElementsByName(b).length}),n.getById?(r.filter.ID=function(e){var t=e.replace(Z,ee);return function(e){return e.getAttribute("id")===t}},r.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&g){var n=t.getElementById(e);return n?[n]:[]}}):(r.filter.ID=function(e){var t=e.replace(Z,ee);return function(e){var n="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return n&&n.value===t}},r.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&g){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),r.find.TAG=n.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):n.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},r.find.CLASS=n.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&g)return t.getElementsByClassName(e)},v=[],y=[],(n.qsa=Q.test(d.querySelectorAll))&&(ue(function(e){h.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&y.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||y.push("\\["+M+"*(?:value|"+P+")"),e.querySelectorAll("[id~="+b+"-]").length||y.push("~="),e.querySelectorAll(":checked").length||y.push(":checked"),e.querySelectorAll("a#"+b+"+*").length||y.push(".#.+[+~]")}),ue(function(e){e.innerHTML="";var t=d.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&y.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&y.push(":enabled",":disabled"),h.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&y.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),y.push(",.*:")})),(n.matchesSelector=Q.test(m=h.matches||h.webkitMatchesSelector||h.mozMatchesSelector||h.oMatchesSelector||h.msMatchesSelector))&&ue(function(e){n.disconnectedMatch=m.call(e,"*"),m.call(e,"[s!='']:x"),v.push("!=",W)}),y=y.length&&new RegExp(y.join("|")),v=v.length&&new RegExp(v.join("|")),t=Q.test(h.compareDocumentPosition),x=t||Q.test(h.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return f=!0,0;var r=!e.compareDocumentPosition-!t.compareDocumentPosition;return r||(1&(r=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!n.sortDetached&&t.compareDocumentPosition(e)===r?e===d||e.ownerDocument===w&&x(w,e)?-1:t===d||t.ownerDocument===w&&x(w,t)?1:c?O(c,e)-O(c,t):0:4&r?-1:1)}:function(e,t){if(e===t)return f=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===d?-1:t===d?1:i?-1:o?1:c?O(c,e)-O(c,t):0;if(i===o)return ce(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?ce(a[r],s[r]):a[r]===w?-1:s[r]===w?1:0},d):d},oe.matches=function(e,t){return oe(e,null,null,t)},oe.matchesSelector=function(e,t){if((e.ownerDocument||e)!==d&&p(e),t=t.replace(z,"='$1']"),n.matchesSelector&&g&&!S[t+" "]&&(!v||!v.test(t))&&(!y||!y.test(t)))try{var r=m.call(e,t);if(r||n.disconnectedMatch||e.document&&11!==e.document.nodeType)return r}catch(e){}return oe(t,d,null,[e]).length>0},oe.contains=function(e,t){return(e.ownerDocument||e)!==d&&p(e),x(e,t)},oe.attr=function(e,t){(e.ownerDocument||e)!==d&&p(e);var i=r.attrHandle[t.toLowerCase()],o=i&&N.call(r.attrHandle,t.toLowerCase())?i(e,t,!g):void 0;return void 0!==o?o:n.attributes||!g?e.getAttribute(t):(o=e.getAttributeNode(t))&&o.specified?o.value:null},oe.escape=function(e){return(e+"").replace(te,ne)},oe.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},oe.uniqueSort=function(e){var t,r=[],i=0,o=0;if(f=!n.detectDuplicates,c=!n.sortStable&&e.slice(0),e.sort(D),f){while(t=e[o++])t===e[o]&&(i=r.push(o));while(i--)e.splice(r[i],1)}return c=null,e},i=oe.getText=function(e){var t,n="",r=0,o=e.nodeType;if(o){if(1===o||9===o||11===o){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=i(e)}else if(3===o||4===o)return e.nodeValue}else while(t=e[r++])n+=i(t);return n},(r=oe.selectors={cacheLength:50,createPseudo:se,match:V,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(Z,ee),e[3]=(e[3]||e[4]||e[5]||"").replace(Z,ee),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||oe.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&oe.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return V.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=a(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(Z,ee).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=E[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&E(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=oe.attr(r,e);return null==i?"!="===t:!t||(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i.replace($," ")+" ").indexOf(n)>-1:"|="===t&&(i===n||i.slice(0,n.length+1)===n+"-"))}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,u){var l,c,f,p,d,h,g=o!==a?"nextSibling":"previousSibling",y=t.parentNode,v=s&&t.nodeName.toLowerCase(),m=!u&&!s,x=!1;if(y){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===v:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?y.firstChild:y.lastChild],a&&m){x=(d=(l=(c=(f=(p=y)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1])&&l[2],p=d&&y.childNodes[d];while(p=++d&&p&&p[g]||(x=d=0)||h.pop())if(1===p.nodeType&&++x&&p===t){c[e]=[T,d,x];break}}else if(m&&(x=d=(l=(c=(f=(p=t)[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]||[])[0]===T&&l[1]),!1===x)while(p=++d&&p&&p[g]||(x=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===v:1===p.nodeType)&&++x&&(m&&((c=(f=p[b]||(p[b]={}))[p.uniqueID]||(f[p.uniqueID]={}))[e]=[T,x]),p===t))break;return(x-=i)===r||x%r==0&&x/r>=0}}},PSEUDO:function(e,t){var n,i=r.pseudos[e]||r.setFilters[e.toLowerCase()]||oe.error("unsupported pseudo: "+e);return i[b]?i(t):i.length>1?(n=[e,e,"",t],r.setFilters.hasOwnProperty(e.toLowerCase())?se(function(e,n){var r,o=i(e,t),a=o.length;while(a--)e[r=O(e,o[a])]=!(n[r]=o[a])}):function(e){return i(e,0,n)}):i}},pseudos:{not:se(function(e){var t=[],n=[],r=s(e.replace(B,"$1"));return r[b]?se(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),t[0]=null,!n.pop()}}),has:se(function(e){return function(t){return oe(e,t).length>0}}),contains:se(function(e){return e=e.replace(Z,ee),function(t){return(t.textContent||t.innerText||i(t)).indexOf(e)>-1}}),lang:se(function(e){return U.test(e||"")||oe.error("unsupported lang: "+e),e=e.replace(Z,ee).toLowerCase(),function(t){var n;do{if(n=g?t.lang:t.getAttribute("xml:lang")||t.getAttribute("lang"))return(n=n.toLowerCase())===e||0===n.indexOf(e+"-")}while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===h},focus:function(e){return e===d.activeElement&&(!d.hasFocus||d.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:de(!1),disabled:de(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!r.pseudos.empty(e)},header:function(e){return Y.test(e.nodeName)},input:function(e){return G.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:he(function(){return[0]}),last:he(function(e,t){return[t-1]}),eq:he(function(e,t,n){return[n<0?n+t:n]}),even:he(function(e,t){for(var n=0;n=0;)e.push(r);return e}),gt:he(function(e,t,n){for(var r=n<0?n+t:n;++r1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function be(e,t,n){for(var r=0,i=t.length;r-1&&(o[l]=!(a[l]=f))}}else v=we(v===a?v.splice(h,v.length):v),i?i(null,a,v,u):L.apply(a,v)})}function Ce(e){for(var t,n,i,o=e.length,a=r.relative[e[0].type],s=a||r.relative[" "],u=a?1:0,c=me(function(e){return e===t},s,!0),f=me(function(e){return O(t,e)>-1},s,!0),p=[function(e,n,r){var i=!a&&(r||n!==l)||((t=n).nodeType?c(e,n,r):f(e,n,r));return t=null,i}];u1&&xe(p),u>1&&ve(e.slice(0,u-1).concat({value:" "===e[u-2].type?"*":""})).replace(B,"$1"),n,u0,i=e.length>0,o=function(o,a,s,u,c){var f,h,y,v=0,m="0",x=o&&[],b=[],w=l,C=o||i&&r.find.TAG("*",c),E=T+=null==w?1:Math.random()||.1,k=C.length;for(c&&(l=a===d||a||c);m!==k&&null!=(f=C[m]);m++){if(i&&f){h=0,a||f.ownerDocument===d||(p(f),s=!g);while(y=e[h++])if(y(f,a||d,s)){u.push(f);break}c&&(T=E)}n&&((f=!y&&f)&&v--,o&&x.push(f))}if(v+=m,n&&m!==v){h=0;while(y=t[h++])y(x,b,a,s);if(o){if(v>0)while(m--)x[m]||b[m]||(b[m]=j.call(u));b=we(b)}L.apply(u,b),c&&!o&&b.length>0&&v+t.length>1&&oe.uniqueSort(u)}return c&&(T=E,l=w),x};return n?se(o):o}return s=oe.compile=function(e,t){var n,r=[],i=[],o=S[e+" "];if(!o){t||(t=a(e)),n=t.length;while(n--)(o=Ce(t[n]))[b]?r.push(o):i.push(o);(o=S(e,Ee(i,r))).selector=e}return o},u=oe.select=function(e,t,n,i){var o,u,l,c,f,p="function"==typeof e&&e,d=!i&&a(e=p.selector||e);if(n=n||[],1===d.length){if((u=d[0]=d[0].slice(0)).length>2&&"ID"===(l=u[0]).type&&9===t.nodeType&&g&&r.relative[u[1].type]){if(!(t=(r.find.ID(l.matches[0].replace(Z,ee),t)||[])[0]))return n;p&&(t=t.parentNode),e=e.slice(u.shift().value.length)}o=V.needsContext.test(e)?0:u.length;while(o--){if(l=u[o],r.relative[c=l.type])break;if((f=r.find[c])&&(i=f(l.matches[0].replace(Z,ee),K.test(u[0].type)&&ge(t.parentNode)||t))){if(u.splice(o,1),!(e=i.length&&ve(u)))return L.apply(n,i),n;break}}}return(p||s(e,d))(i,t,!g,n,!t||K.test(e)&&ge(t.parentNode)||t),n},n.sortStable=b.split("").sort(D).join("")===b,n.detectDuplicates=!!f,p(),n.sortDetached=ue(function(e){return 1&e.compareDocumentPosition(d.createElement("fieldset"))}),ue(function(e){return e.innerHTML="","#"===e.firstChild.getAttribute("href")})||le("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),n.attributes&&ue(function(e){return e.innerHTML="",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||le("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ue(function(e){return null==e.getAttribute("disabled")})||le(P,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),oe}(e);w.find=E,w.expr=E.selectors,w.expr[":"]=w.expr.pseudos,w.uniqueSort=w.unique=E.uniqueSort,w.text=E.getText,w.isXMLDoc=E.isXML,w.contains=E.contains,w.escapeSelector=E.escape;var k=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&w(e).is(n))break;r.push(e)}return r},S=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},D=w.expr.match.needsContext;function N(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var A=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,t,n){return g(t)?w.grep(e,function(e,r){return!!t.call(e,r,e)!==n}):t.nodeType?w.grep(e,function(e){return e===t!==n}):"string"!=typeof t?w.grep(e,function(e){return u.call(t,e)>-1!==n}):w.filter(t,e,n)}w.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?w.find.matchesSelector(r,e)?[r]:[]:w.find.matches(e,w.grep(t,function(e){return 1===e.nodeType}))},w.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(w(e).filter(function(){for(t=0;t1?w.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&D.test(e)?w(e):e||[],!1).length}});var q,L=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(w.fn.init=function(e,t,n){var i,o;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(i="<"===e[0]&&">"===e[e.length-1]&&e.length>=3?[null,e,null]:L.exec(e))||!i[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(i[1]){if(t=t instanceof w?t[0]:t,w.merge(this,w.parseHTML(i[1],t&&t.nodeType?t.ownerDocument||t:r,!0)),A.test(i[1])&&w.isPlainObject(t))for(i in t)g(this[i])?this[i](t[i]):this.attr(i,t[i]);return this}return(o=r.getElementById(i[2]))&&(this[0]=o,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):g(e)?void 0!==n.ready?n.ready(e):e(w):w.makeArray(e,this)}).prototype=w.fn,q=w(r);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};w.fn.extend({has:function(e){var t=w(e,this),n=t.length;return this.filter(function(){for(var e=0;e-1:1===n.nodeType&&w.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(o.length>1?w.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?u.call(w(e),this[0]):u.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(w.uniqueSort(w.merge(this.get(),w(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}});function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}w.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return k(e,"parentNode")},parentsUntil:function(e,t,n){return k(e,"parentNode",n)},next:function(e){return P(e,"nextSibling")},prev:function(e){return P(e,"previousSibling")},nextAll:function(e){return k(e,"nextSibling")},prevAll:function(e){return k(e,"previousSibling")},nextUntil:function(e,t,n){return k(e,"nextSibling",n)},prevUntil:function(e,t,n){return k(e,"previousSibling",n)},siblings:function(e){return S((e.parentNode||{}).firstChild,e)},children:function(e){return S(e.firstChild)},contents:function(e){return N(e,"iframe")?e.contentDocument:(N(e,"template")&&(e=e.content||e),w.merge([],e.childNodes))}},function(e,t){w.fn[e]=function(n,r){var i=w.map(this,t,n);return"Until"!==e.slice(-5)&&(r=n),r&&"string"==typeof r&&(i=w.filter(r,i)),this.length>1&&(O[e]||w.uniqueSort(i),H.test(e)&&i.reverse()),this.pushStack(i)}});var M=/[^\x20\t\r\n\f]+/g;function R(e){var t={};return w.each(e.match(M)||[],function(e,n){t[n]=!0}),t}w.Callbacks=function(e){e="string"==typeof e?R(e):w.extend({},e);var t,n,r,i,o=[],a=[],s=-1,u=function(){for(i=i||e.once,r=t=!0;a.length;s=-1){n=a.shift();while(++s-1)o.splice(n,1),n<=s&&s--}),this},has:function(e){return e?w.inArray(e,o)>-1:o.length>0},empty:function(){return o&&(o=[]),this},disable:function(){return i=a=[],o=n="",this},disabled:function(){return!o},lock:function(){return i=a=[],n||t||(o=n=""),this},locked:function(){return!!i},fireWith:function(e,n){return i||(n=[e,(n=n||[]).slice?n.slice():n],a.push(n),t||u()),this},fire:function(){return l.fireWith(this,arguments),this},fired:function(){return!!r}};return l};function I(e){return e}function W(e){throw e}function $(e,t,n,r){var i;try{e&&g(i=e.promise)?i.call(e).done(t).fail(n):e&&g(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}w.extend({Deferred:function(t){var n=[["notify","progress",w.Callbacks("memory"),w.Callbacks("memory"),2],["resolve","done",w.Callbacks("once memory"),w.Callbacks("once memory"),0,"resolved"],["reject","fail",w.Callbacks("once memory"),w.Callbacks("once memory"),1,"rejected"]],r="pending",i={state:function(){return r},always:function(){return o.done(arguments).fail(arguments),this},"catch":function(e){return i.then(null,e)},pipe:function(){var e=arguments;return w.Deferred(function(t){w.each(n,function(n,r){var i=g(e[r[4]])&&e[r[4]];o[r[1]](function(){var e=i&&i.apply(this,arguments);e&&g(e.promise)?e.promise().progress(t.notify).done(t.resolve).fail(t.reject):t[r[0]+"With"](this,i?[e]:arguments)})}),e=null}).promise()},then:function(t,r,i){var o=0;function a(t,n,r,i){return function(){var s=this,u=arguments,l=function(){var e,l;if(!(t=o&&(r!==W&&(s=void 0,u=[e]),n.rejectWith(s,u))}};t?c():(w.Deferred.getStackHook&&(c.stackTrace=w.Deferred.getStackHook()),e.setTimeout(c))}}return w.Deferred(function(e){n[0][3].add(a(0,e,g(i)?i:I,e.notifyWith)),n[1][3].add(a(0,e,g(t)?t:I)),n[2][3].add(a(0,e,g(r)?r:W))}).promise()},promise:function(e){return null!=e?w.extend(e,i):i}},o={};return w.each(n,function(e,t){var a=t[2],s=t[5];i[t[1]]=a.add,s&&a.add(function(){r=s},n[3-e][2].disable,n[3-e][3].disable,n[0][2].lock,n[0][3].lock),a.add(t[3].fire),o[t[0]]=function(){return o[t[0]+"With"](this===o?void 0:this,arguments),this},o[t[0]+"With"]=a.fireWith}),i.promise(o),t&&t.call(o,o),o},when:function(e){var t=arguments.length,n=t,r=Array(n),i=o.call(arguments),a=w.Deferred(),s=function(e){return function(n){r[e]=this,i[e]=arguments.length>1?o.call(arguments):n,--t||a.resolveWith(r,i)}};if(t<=1&&($(e,a.done(s(n)).resolve,a.reject,!t),"pending"===a.state()||g(i[n]&&i[n].then)))return a.then();while(n--)$(i[n],s(n),a.reject);return a.promise()}});var B=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;w.Deferred.exceptionHook=function(t,n){e.console&&e.console.warn&&t&&B.test(t.name)&&e.console.warn("jQuery.Deferred exception: "+t.message,t.stack,n)},w.readyException=function(t){e.setTimeout(function(){throw t})};var F=w.Deferred();w.fn.ready=function(e){return F.then(e)["catch"](function(e){w.readyException(e)}),this},w.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--w.readyWait:w.isReady)||(w.isReady=!0,!0!==e&&--w.readyWait>0||F.resolveWith(r,[w]))}}),w.ready.then=F.then;function _(){r.removeEventListener("DOMContentLoaded",_),e.removeEventListener("load",_),w.ready()}"complete"===r.readyState||"loading"!==r.readyState&&!r.documentElement.doScroll?e.setTimeout(w.ready):(r.addEventListener("DOMContentLoaded",_),e.addEventListener("load",_));var z=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===x(n)){i=!0;for(s in n)z(e,t,s,n[s],!0,o,a)}else if(void 0!==r&&(i=!0,g(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(w(e),n)})),t))for(;s1,null,!0)},removeData:function(e){return this.each(function(){K.remove(this,e)})}}),w.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=J.get(e,t),n&&(!r||Array.isArray(n)?r=J.access(e,t,w.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=w.queue(e,t),r=n.length,i=n.shift(),o=w._queueHooks(e,t),a=function(){w.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return J.get(e,n)||J.access(e,n,{empty:w.Callbacks("once memory").add(function(){J.remove(e,[t+"queue",n])})})}}),w.fn.extend({queue:function(e,t){var n=2;return"string"!=typeof e&&(t=e,e="fx",n--),arguments.length\x20\t\r\n\f]+)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};ge.optgroup=ge.option,ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td;function ye(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&N(e,t)?w.merge([e],n):n}function ve(e,t){for(var n=0,r=e.length;n-1)i&&i.push(o);else if(l=w.contains(o.ownerDocument,o),a=ye(f.appendChild(o),"script"),l&&ve(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}!function(){var e=r.createDocumentFragment().appendChild(r.createElement("div")),t=r.createElement("input");t.setAttribute("type","radio"),t.setAttribute("checked","checked"),t.setAttribute("name","t"),e.appendChild(t),h.checkClone=e.cloneNode(!0).cloneNode(!0).lastChild.checked,e.innerHTML="",h.noCloneChecked=!!e.cloneNode(!0).lastChild.defaultValue}();var be=r.documentElement,we=/^key/,Te=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ce=/^([^.]*)(?:\.(.+)|)/;function Ee(){return!0}function ke(){return!1}function Se(){try{return r.activeElement}catch(e){}}function De(e,t,n,r,i,o){var a,s;if("object"==typeof t){"string"!=typeof n&&(r=r||n,n=void 0);for(s in t)De(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=ke;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return w().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=w.guid++)),e.each(function(){w.event.add(this,t,i,r,n)})}w.event={global:{},add:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,y=J.get(e);if(y){n.handler&&(n=(o=n).handler,i=o.selector),i&&w.find.matchesSelector(be,i),n.guid||(n.guid=w.guid++),(u=y.events)||(u=y.events={}),(a=y.handle)||(a=y.handle=function(t){return"undefined"!=typeof w&&w.event.triggered!==t.type?w.event.dispatch.apply(e,arguments):void 0}),l=(t=(t||"").match(M)||[""]).length;while(l--)d=g=(s=Ce.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=w.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=w.event.special[d]||{},c=w.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&w.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(e,r,h,a)||e.addEventListener&&e.addEventListener(d,a)),f.add&&(f.add.call(e,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),w.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,y=J.hasData(e)&&J.get(e);if(y&&(u=y.events)){l=(t=(t||"").match(M)||[""]).length;while(l--)if(s=Ce.exec(t[l])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){f=w.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,y.handle)||w.removeEvent(e,d,y.handle),delete u[d])}else for(d in u)w.event.remove(e,d+t[l],n,r,!0);w.isEmptyObject(u)&&J.remove(e,"handle events")}},dispatch:function(e){var t=w.event.fix(e),n,r,i,o,a,s,u=new Array(arguments.length),l=(J.get(this,"events")||{})[t.type]||[],c=w.event.special[t.type]||{};for(u[0]=t,n=1;n=1))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n-1:w.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u\x20\t\r\n\f]*)[^>]*)\/>/gi,Ae=/\s*$/g;function Le(e,t){return N(e,"table")&&N(11!==t.nodeType?t:t.firstChild,"tr")?w(e).children("tbody")[0]||e:e}function He(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Oe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Pe(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(J.hasData(e)&&(o=J.access(e),a=J.set(t,o),l=o.events)){delete a.handle,a.events={};for(i in l)for(n=0,r=l[i].length;n1&&"string"==typeof y&&!h.checkClone&&je.test(y))return e.each(function(i){var o=e.eq(i);v&&(t[0]=y.call(this,i,o.html())),Re(o,t,n,r)});if(p&&(i=xe(t,e[0].ownerDocument,!1,e,r),o=i.firstChild,1===i.childNodes.length&&(i=o),o||r)){for(u=(s=w.map(ye(i,"script"),He)).length;f")},clone:function(e,t,n){var r,i,o,a,s=e.cloneNode(!0),u=w.contains(e.ownerDocument,e);if(!(h.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||w.isXMLDoc(e)))for(a=ye(s),r=0,i=(o=ye(e)).length;r0&&ve(a,!u&&ye(e,"script")),s},cleanData:function(e){for(var t,n,r,i=w.event.special,o=0;void 0!==(n=e[o]);o++)if(Y(n)){if(t=n[J.expando]){if(t.events)for(r in t.events)i[r]?w.event.remove(n,r):w.removeEvent(n,r,t.handle);n[J.expando]=void 0}n[K.expando]&&(n[K.expando]=void 0)}}}),w.fn.extend({detach:function(e){return Ie(this,e,!0)},remove:function(e){return Ie(this,e)},text:function(e){return z(this,function(e){return void 0===e?w.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Re(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Le(this,e).appendChild(e)})},prepend:function(){return Re(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Le(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Re(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Re(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(w.cleanData(ye(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return w.clone(this,e,t)})},html:function(e){return z(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Ae.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=w.htmlPrefilter(e);try{for(;n=0&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))),u}function et(e,t,n){var r=$e(e),i=Fe(e,t,r),o="border-box"===w.css(e,"boxSizing",!1,r),a=o;if(We.test(i)){if(!n)return i;i="auto"}return a=a&&(h.boxSizingReliable()||i===e.style[t]),("auto"===i||!parseFloat(i)&&"inline"===w.css(e,"display",!1,r))&&(i=e["offset"+t[0].toUpperCase()+t.slice(1)],a=!0),(i=parseFloat(i)||0)+Ze(e,t,n||(o?"border":"content"),a,r,i)+"px"}w.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Fe(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=G(t),u=Xe.test(t),l=e.style;if(u||(t=Je(s)),a=w.cssHooks[t]||w.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"==(o=typeof n)&&(i=ie.exec(n))&&i[1]&&(n=ue(e,t,i),o="number"),null!=n&&n===n&&("number"===o&&(n+=i&&i[3]||(w.cssNumber[s]?"":"px")),h.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=G(t);return Xe.test(t)||(t=Je(s)),(a=w.cssHooks[t]||w.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Fe(e,t,r)),"normal"===i&&t in Ve&&(i=Ve[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),w.each(["height","width"],function(e,t){w.cssHooks[t]={get:function(e,n,r){if(n)return!ze.test(w.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?et(e,t,r):se(e,Ue,function(){return et(e,t,r)})},set:function(e,n,r){var i,o=$e(e),a="border-box"===w.css(e,"boxSizing",!1,o),s=r&&Ze(e,t,r,a,o);return a&&h.scrollboxSize()===o.position&&(s-=Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-parseFloat(o[t])-Ze(e,t,"border",!1,o)-.5)),s&&(i=ie.exec(n))&&"px"!==(i[3]||"px")&&(e.style[t]=n,n=w.css(e,t)),Ke(e,n,s)}}}),w.cssHooks.marginLeft=_e(h.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Fe(e,"marginLeft"))||e.getBoundingClientRect().left-se(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),w.each({margin:"",padding:"",border:"Width"},function(e,t){w.cssHooks[e+t]={expand:function(n){for(var r=0,i={},o="string"==typeof n?n.split(" "):[n];r<4;r++)i[e+oe[r]+t]=o[r]||o[r-2]||o[0];return i}},"margin"!==e&&(w.cssHooks[e+t].set=Ke)}),w.fn.extend({css:function(e,t){return z(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=$e(e),i=t.length;a1)}});function tt(e,t,n,r,i){return new tt.prototype.init(e,t,n,r,i)}w.Tween=tt,tt.prototype={constructor:tt,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||w.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(w.cssNumber[n]?"":"px")},cur:function(){var e=tt.propHooks[this.prop];return e&&e.get?e.get(this):tt.propHooks._default.get(this)},run:function(e){var t,n=tt.propHooks[this.prop];return this.options.duration?this.pos=t=w.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):tt.propHooks._default.set(this),this}},tt.prototype.init.prototype=tt.prototype,tt.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=w.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){w.fx.step[e.prop]?w.fx.step[e.prop](e):1!==e.elem.nodeType||null==e.elem.style[w.cssProps[e.prop]]&&!w.cssHooks[e.prop]?e.elem[e.prop]=e.now:w.style(e.elem,e.prop,e.now+e.unit)}}},tt.propHooks.scrollTop=tt.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},w.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},w.fx=tt.prototype.init,w.fx.step={};var nt,rt,it=/^(?:toggle|show|hide)$/,ot=/queueHooks$/;function at(){rt&&(!1===r.hidden&&e.requestAnimationFrame?e.requestAnimationFrame(at):e.setTimeout(at,w.fx.interval),w.fx.tick())}function st(){return e.setTimeout(function(){nt=void 0}),nt=Date.now()}function ut(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=oe[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function lt(e,t,n){for(var r,i=(pt.tweeners[t]||[]).concat(pt.tweeners["*"]),o=0,a=i.length;o1)},removeAttr:function(e){return this.each(function(){w.removeAttr(this,e)})}}),w.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?w.prop(e,t,n):(1===o&&w.isXMLDoc(e)||(i=w.attrHooks[t.toLowerCase()]||(w.expr.match.bool.test(t)?dt:void 0)),void 0!==n?null===n?void w.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=w.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!h.radioValue&&"radio"===t&&N(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(M);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),dt={set:function(e,t,n){return!1===t?w.removeAttr(e,n):e.setAttribute(n,n),n}},w.each(w.expr.match.bool.source.match(/\w+/g),function(e,t){var n=ht[t]||w.find.attr;ht[t]=function(e,t,r){var i,o,a=t.toLowerCase();return r||(o=ht[a],ht[a]=i,i=null!=n(e,t,r)?a:null,ht[a]=o),i}});var gt=/^(?:input|select|textarea|button)$/i,yt=/^(?:a|area)$/i;w.fn.extend({prop:function(e,t){return z(this,w.prop,e,t,arguments.length>1)},removeProp:function(e){return this.each(function(){delete this[w.propFix[e]||e]})}}),w.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&w.isXMLDoc(e)||(t=w.propFix[t]||t,i=w.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=w.find.attr(e,"tabindex");return t?parseInt(t,10):gt.test(e.nodeName)||yt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),h.optSelected||(w.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),w.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){w.propFix[this.toLowerCase()]=this});function vt(e){return(e.match(M)||[]).join(" ")}function mt(e){return e.getAttribute&&e.getAttribute("class")||""}function xt(e){return Array.isArray(e)?e:"string"==typeof e?e.match(M)||[]:[]}w.fn.extend({addClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){w(this).addClass(e.call(this,t,mt(this)))});if((t=xt(e)).length)while(n=this[u++])if(i=mt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=t[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},removeClass:function(e){var t,n,r,i,o,a,s,u=0;if(g(e))return this.each(function(t){w(this).removeClass(e.call(this,t,mt(this)))});if(!arguments.length)return this.attr("class","");if((t=xt(e)).length)while(n=this[u++])if(i=mt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=t[a++])while(r.indexOf(" "+o+" ")>-1)r=r.replace(" "+o+" "," ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(e,t){var n=typeof e,r="string"===n||Array.isArray(e);return"boolean"==typeof t&&r?t?this.addClass(e):this.removeClass(e):g(e)?this.each(function(n){w(this).toggleClass(e.call(this,n,mt(this),t),t)}):this.each(function(){var t,i,o,a;if(r){i=0,o=w(this),a=xt(e);while(t=a[i++])o.hasClass(t)?o.removeClass(t):o.addClass(t)}else void 0!==e&&"boolean"!==n||((t=mt(this))&&J.set(this,"__className__",t),this.setAttribute&&this.setAttribute("class",t||!1===e?"":J.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&(" "+vt(mt(n))+" ").indexOf(t)>-1)return!0;return!1}});var bt=/\r/g;w.fn.extend({val:function(e){var t,n,r,i=this[0];{if(arguments.length)return r=g(e),this.each(function(n){var i;1===this.nodeType&&(null==(i=r?e.call(this,n,w(this).val()):e)?i="":"number"==typeof i?i+="":Array.isArray(i)&&(i=w.map(i,function(e){return null==e?"":e+""})),(t=w.valHooks[this.type]||w.valHooks[this.nodeName.toLowerCase()])&&"set"in t&&void 0!==t.set(this,i,"value")||(this.value=i))});if(i)return(t=w.valHooks[i.type]||w.valHooks[i.nodeName.toLowerCase()])&&"get"in t&&void 0!==(n=t.get(i,"value"))?n:"string"==typeof(n=i.value)?n.replace(bt,""):null==n?"":n}}}),w.extend({valHooks:{option:{get:function(e){var t=w.find.attr(e,"value");return null!=t?t:vt(w.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r-1)&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),w.each(["radio","checkbox"],function(){w.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=w.inArray(w(e).val(),t)>-1}},h.checkOn||(w.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),h.focusin="onfocusin"in e;var wt=/^(?:focusinfocus|focusoutblur)$/,Tt=function(e){e.stopPropagation()};w.extend(w.event,{trigger:function(t,n,i,o){var a,s,u,l,c,p,d,h,v=[i||r],m=f.call(t,"type")?t.type:t,x=f.call(t,"namespace")?t.namespace.split("."):[];if(s=h=u=i=i||r,3!==i.nodeType&&8!==i.nodeType&&!wt.test(m+w.event.triggered)&&(m.indexOf(".")>-1&&(m=(x=m.split(".")).shift(),x.sort()),c=m.indexOf(":")<0&&"on"+m,t=t[w.expando]?t:new w.Event(m,"object"==typeof t&&t),t.isTrigger=o?2:3,t.namespace=x.join("."),t.rnamespace=t.namespace?new RegExp("(^|\\.)"+x.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,t.result=void 0,t.target||(t.target=i),n=null==n?[t]:w.makeArray(n,[t]),d=w.event.special[m]||{},o||!d.trigger||!1!==d.trigger.apply(i,n))){if(!o&&!d.noBubble&&!y(i)){for(l=d.delegateType||m,wt.test(l+m)||(s=s.parentNode);s;s=s.parentNode)v.push(s),u=s;u===(i.ownerDocument||r)&&v.push(u.defaultView||u.parentWindow||e)}a=0;while((s=v[a++])&&!t.isPropagationStopped())h=s,t.type=a>1?l:d.bindType||m,(p=(J.get(s,"events")||{})[t.type]&&J.get(s,"handle"))&&p.apply(s,n),(p=c&&s[c])&&p.apply&&Y(s)&&(t.result=p.apply(s,n),!1===t.result&&t.preventDefault());return t.type=m,o||t.isDefaultPrevented()||d._default&&!1!==d._default.apply(v.pop(),n)||!Y(i)||c&&g(i[m])&&!y(i)&&((u=i[c])&&(i[c]=null),w.event.triggered=m,t.isPropagationStopped()&&h.addEventListener(m,Tt),i[m](),t.isPropagationStopped()&&h.removeEventListener(m,Tt),w.event.triggered=void 0,u&&(i[c]=u)),t.result}},simulate:function(e,t,n){var r=w.extend(new w.Event,n,{type:e,isSimulated:!0});w.event.trigger(r,null,t)}}),w.fn.extend({trigger:function(e,t){return this.each(function(){w.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return w.event.trigger(e,t,n,!0)}}),h.focusin||w.each({focus:"focusin",blur:"focusout"},function(e,t){var n=function(e){w.event.simulate(t,e.target,w.event.fix(e))};w.event.special[t]={setup:function(){var r=this.ownerDocument||this,i=J.access(r,t);i||r.addEventListener(e,n,!0),J.access(r,t,(i||0)+1)},teardown:function(){var r=this.ownerDocument||this,i=J.access(r,t)-1;i?J.access(r,t,i):(r.removeEventListener(e,n,!0),J.remove(r,t))}}});var Ct=e.location,Et=Date.now(),kt=/\?/;w.parseXML=function(t){var n;if(!t||"string"!=typeof t)return null;try{n=(new e.DOMParser).parseFromString(t,"text/xml")}catch(e){n=void 0}return n&&!n.getElementsByTagName("parsererror").length||w.error("Invalid XML: "+t),n};var St=/\[\]$/,Dt=/\r?\n/g,Nt=/^(?:submit|button|image|reset|file)$/i,At=/^(?:input|select|textarea|keygen)/i;function jt(e,t,n,r){var i;if(Array.isArray(t))w.each(t,function(t,i){n||St.test(e)?r(e,i):jt(e+"["+("object"==typeof i&&null!=i?t:"")+"]",i,n,r)});else if(n||"object"!==x(t))r(e,t);else for(i in t)jt(e+"["+i+"]",t[i],n,r)}w.param=function(e,t){var n,r=[],i=function(e,t){var n=g(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(Array.isArray(e)||e.jquery&&!w.isPlainObject(e))w.each(e,function(){i(this.name,this.value)});else for(n in e)jt(n,e[n],t,i);return r.join("&")},w.fn.extend({serialize:function(){return w.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=w.prop(this,"elements");return e?w.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!w(this).is(":disabled")&&At.test(this.nodeName)&&!Nt.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=w(this).val();return null==n?null:Array.isArray(n)?w.map(n,function(e){return{name:t.name,value:e.replace(Dt,"\r\n")}}):{name:t.name,value:n.replace(Dt,"\r\n")}}).get()}});var qt=/%20/g,Lt=/#.*$/,Ht=/([?&])_=[^&]*/,Ot=/^(.*?):[ \t]*([^\r\n]*)$/gm,Pt=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Mt=/^(?:GET|HEAD)$/,Rt=/^\/\//,It={},Wt={},$t="*/".concat("*"),Bt=r.createElement("a");Bt.href=Ct.href;function Ft(e){return function(t,n){"string"!=typeof t&&(n=t,t="*");var r,i=0,o=t.toLowerCase().match(M)||[];if(g(n))while(r=o[i++])"+"===r[0]?(r=r.slice(1)||"*",(e[r]=e[r]||[]).unshift(n)):(e[r]=e[r]||[]).push(n)}}function _t(e,t,n,r){var i={},o=e===Wt;function a(s){var u;return i[s]=!0,w.each(e[s]||[],function(e,s){var l=s(t,n,r);return"string"!=typeof l||o||i[l]?o?!(u=l):void 0:(t.dataTypes.unshift(l),a(l),!1)}),u}return a(t.dataTypes[0])||!i["*"]&&a("*")}function zt(e,t){var n,r,i=w.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&w.extend(!0,e,r),e}function Xt(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}function Ut(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}w.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Ct.href,type:"GET",isLocal:Pt.test(Ct.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":$t,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":w.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?zt(zt(e,w.ajaxSettings),t):zt(w.ajaxSettings,e)},ajaxPrefilter:Ft(It),ajaxTransport:Ft(Wt),ajax:function(t,n){"object"==typeof t&&(n=t,t=void 0),n=n||{};var i,o,a,s,u,l,c,f,p,d,h=w.ajaxSetup({},n),g=h.context||h,y=h.context&&(g.nodeType||g.jquery)?w(g):w.event,v=w.Deferred(),m=w.Callbacks("once memory"),x=h.statusCode||{},b={},T={},C="canceled",E={readyState:0,getResponseHeader:function(e){var t;if(c){if(!s){s={};while(t=Ot.exec(a))s[t[1].toLowerCase()]=t[2]}t=s[e.toLowerCase()]}return null==t?null:t},getAllResponseHeaders:function(){return c?a:null},setRequestHeader:function(e,t){return null==c&&(e=T[e.toLowerCase()]=T[e.toLowerCase()]||e,b[e]=t),this},overrideMimeType:function(e){return null==c&&(h.mimeType=e),this},statusCode:function(e){var t;if(e)if(c)E.always(e[E.status]);else for(t in e)x[t]=[x[t],e[t]];return this},abort:function(e){var t=e||C;return i&&i.abort(t),k(0,t),this}};if(v.promise(E),h.url=((t||h.url||Ct.href)+"").replace(Rt,Ct.protocol+"//"),h.type=n.method||n.type||h.method||h.type,h.dataTypes=(h.dataType||"*").toLowerCase().match(M)||[""],null==h.crossDomain){l=r.createElement("a");try{l.href=h.url,l.href=l.href,h.crossDomain=Bt.protocol+"//"+Bt.host!=l.protocol+"//"+l.host}catch(e){h.crossDomain=!0}}if(h.data&&h.processData&&"string"!=typeof h.data&&(h.data=w.param(h.data,h.traditional)),_t(It,h,n,E),c)return E;(f=w.event&&h.global)&&0==w.active++&&w.event.trigger("ajaxStart"),h.type=h.type.toUpperCase(),h.hasContent=!Mt.test(h.type),o=h.url.replace(Lt,""),h.hasContent?h.data&&h.processData&&0===(h.contentType||"").indexOf("application/x-www-form-urlencoded")&&(h.data=h.data.replace(qt,"+")):(d=h.url.slice(o.length),h.data&&(h.processData||"string"==typeof h.data)&&(o+=(kt.test(o)?"&":"?")+h.data,delete h.data),!1===h.cache&&(o=o.replace(Ht,"$1"),d=(kt.test(o)?"&":"?")+"_="+Et+++d),h.url=o+d),h.ifModified&&(w.lastModified[o]&&E.setRequestHeader("If-Modified-Since",w.lastModified[o]),w.etag[o]&&E.setRequestHeader("If-None-Match",w.etag[o])),(h.data&&h.hasContent&&!1!==h.contentType||n.contentType)&&E.setRequestHeader("Content-Type",h.contentType),E.setRequestHeader("Accept",h.dataTypes[0]&&h.accepts[h.dataTypes[0]]?h.accepts[h.dataTypes[0]]+("*"!==h.dataTypes[0]?", "+$t+"; q=0.01":""):h.accepts["*"]);for(p in h.headers)E.setRequestHeader(p,h.headers[p]);if(h.beforeSend&&(!1===h.beforeSend.call(g,E,h)||c))return E.abort();if(C="abort",m.add(h.complete),E.done(h.success),E.fail(h.error),i=_t(Wt,h,n,E)){if(E.readyState=1,f&&y.trigger("ajaxSend",[E,h]),c)return E;h.async&&h.timeout>0&&(u=e.setTimeout(function(){E.abort("timeout")},h.timeout));try{c=!1,i.send(b,k)}catch(e){if(c)throw e;k(-1,e)}}else k(-1,"No Transport");function k(t,n,r,s){var l,p,d,b,T,C=n;c||(c=!0,u&&e.clearTimeout(u),i=void 0,a=s||"",E.readyState=t>0?4:0,l=t>=200&&t<300||304===t,r&&(b=Xt(h,E,r)),b=Ut(h,b,E,l),l?(h.ifModified&&((T=E.getResponseHeader("Last-Modified"))&&(w.lastModified[o]=T),(T=E.getResponseHeader("etag"))&&(w.etag[o]=T)),204===t||"HEAD"===h.type?C="nocontent":304===t?C="notmodified":(C=b.state,p=b.data,l=!(d=b.error))):(d=C,!t&&C||(C="error",t<0&&(t=0))),E.status=t,E.statusText=(n||C)+"",l?v.resolveWith(g,[p,C,E]):v.rejectWith(g,[E,C,d]),E.statusCode(x),x=void 0,f&&y.trigger(l?"ajaxSuccess":"ajaxError",[E,h,l?p:d]),m.fireWith(g,[E,C]),f&&(y.trigger("ajaxComplete",[E,h]),--w.active||w.event.trigger("ajaxStop")))}return E},getJSON:function(e,t,n){return w.get(e,t,n,"json")},getScript:function(e,t){return w.get(e,void 0,t,"script")}}),w.each(["get","post"],function(e,t){w[t]=function(e,n,r,i){return g(n)&&(i=i||r,r=n,n=void 0),w.ajax(w.extend({url:e,type:t,dataType:i,data:n,success:r},w.isPlainObject(e)&&e))}}),w._evalUrl=function(e){return w.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,"throws":!0})},w.fn.extend({wrapAll:function(e){var t;return this[0]&&(g(e)&&(e=e.call(this[0])),t=w(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(e){return g(e)?this.each(function(t){w(this).wrapInner(e.call(this,t))}):this.each(function(){var t=w(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=g(e);return this.each(function(n){w(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(e){return this.parent(e).not("body").each(function(){w(this).replaceWith(this.childNodes)}),this}}),w.expr.pseudos.hidden=function(e){return!w.expr.pseudos.visible(e)},w.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},w.ajaxSettings.xhr=function(){try{return new e.XMLHttpRequest}catch(e){}};var Vt={0:200,1223:204},Gt=w.ajaxSettings.xhr();h.cors=!!Gt&&"withCredentials"in Gt,h.ajax=Gt=!!Gt,w.ajaxTransport(function(t){var n,r;if(h.cors||Gt&&!t.crossDomain)return{send:function(i,o){var a,s=t.xhr();if(s.open(t.type,t.url,t.async,t.username,t.password),t.xhrFields)for(a in t.xhrFields)s[a]=t.xhrFields[a];t.mimeType&&s.overrideMimeType&&s.overrideMimeType(t.mimeType),t.crossDomain||i["X-Requested-With"]||(i["X-Requested-With"]="XMLHttpRequest");for(a in i)s.setRequestHeader(a,i[a]);n=function(e){return function(){n&&(n=r=s.onload=s.onerror=s.onabort=s.ontimeout=s.onreadystatechange=null,"abort"===e?s.abort():"error"===e?"number"!=typeof s.status?o(0,"error"):o(s.status,s.statusText):o(Vt[s.status]||s.status,s.statusText,"text"!==(s.responseType||"text")||"string"!=typeof s.responseText?{binary:s.response}:{text:s.responseText},s.getAllResponseHeaders()))}},s.onload=n(),r=s.onerror=s.ontimeout=n("error"),void 0!==s.onabort?s.onabort=r:s.onreadystatechange=function(){4===s.readyState&&e.setTimeout(function(){n&&r()})},n=n("abort");try{s.send(t.hasContent&&t.data||null)}catch(e){if(n)throw e}},abort:function(){n&&n()}}}),w.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),w.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return w.globalEval(e),e}}}),w.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),w.ajaxTransport("script",function(e){if(e.crossDomain){var t,n;return{send:function(i,o){t=w(" + +{% endblock %} diff --git a/src/etools_datamart/apps/web/urls.py b/src/etools_datamart/apps/web/urls.py index 4bd2a39b0..cc066783c 100644 --- a/src/etools_datamart/apps/web/urls.py +++ b/src/etools_datamart/apps/web/urls.py @@ -1,10 +1,11 @@ from django.contrib.auth.views import LoginView, LogoutView from django.urls import path -from .views import DisconnectView, index +from .views import DisconnectView, index, monitor urlpatterns = [ path(r'', index, name='home'), + path(r'monitor/', monitor, name='monitor'), path(r'login/', LoginView.as_view(template_name='login.html'), name='login'), path(r'logout/', LogoutView.as_view(next_page='/'), name='logout'), path(r'disconnect/', DisconnectView.as_view(next_page='/'), name='disconnect'), diff --git a/src/etools_datamart/apps/web/views.py b/src/etools_datamart/apps/web/views.py index d019fab59..8fe858b6d 100644 --- a/src/etools_datamart/apps/web/views.py +++ b/src/etools_datamart/apps/web/views.py @@ -1,11 +1,22 @@ +from django.contrib.auth.decorators import login_required from django.contrib.auth.views import LogoutView from django.template.response import TemplateResponse +from etools_datamart.apps.etl.models import EtlTask from etools_datamart.config.settings import env def index(request): - return TemplateResponse(request, 'index.html') + context = {'page': 'index'} + return TemplateResponse(request, 'index.html', context) + + +@login_required +def monitor(request): + context = {'tasks': EtlTask.objects.all(), + 'subscriptions': request.user.subscriptions, + 'page': 'monitor'} + return TemplateResponse(request, 'monitor.html', context) class DisconnectView(LogoutView): diff --git a/src/etools_datamart/celery.py b/src/etools_datamart/celery.py index 6779a8489..d512a4eb4 100644 --- a/src/etools_datamart/celery.py +++ b/src/etools_datamart/celery.py @@ -4,9 +4,10 @@ from celery import Celery from celery.signals import task_postrun, task_prerun from celery.task import Task -from django.utils import timezone +from django.apps import apps +from kombu.serialization import register -from etools_datamart.apps.etl.lock import only_one +from etools_datamart.apps.etl.results import etl_dumps, etl_loads os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etools_datamart.config.settings') @@ -21,17 +22,18 @@ class DatamartCelery(Celery): _mapping = {} def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): + from etools_datamart.apps.etl.lock import only_one linked_model = options.get('linked_model', None) - name = name or self.gen_task_name(fun.__name__, fun.__module__) - options['lock_key'] = f"{name}-lock" - fun = only_one(fun, options['lock_key']) - options['unlock'] = fun.unlock - - task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) if linked_model: + name = name or self.gen_task_name(fun.__name__, fun.__module__) + options['lock_key'] = f"{name}-lock" + fun = only_one(fun, options['lock_key']) + options['unlock'] = fun.unlock + task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) linked_model._etl_task = task linked_model._etl_loader = fun - + else: + task = super()._task_from_fun(fun, name=name, base=None, bind=False, **options) return task def etl(self, model, *args, **opts): @@ -55,7 +57,8 @@ def get_all_etls(self): app = DatamartCelery('datamart') app.config_from_object('django.conf:settings', namespace='CELERY') -# app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) +# app.autodiscover_tasks() +app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()]) # app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], # related_name='tasks') # app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()], @@ -72,17 +75,30 @@ def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): app.timers[task_id] = time() from django.contrib.contenttypes.models import ContentType from etools_datamart.apps.etl.models import EtlTask + from django.utils import timezone - defs = {'result': 'RUNNING', - 'timestamp': timezone.now()} + defs = {'status': 'RUNNING', + 'last_run': timezone.now()} EtlTask.objects.update_or_create(task=task.name, content_type=ContentType.objects.get_for_model(task.linked_model), table_name=task.linked_model._meta.db_table, defaults=defs) +register('etljson', etl_dumps, etl_loads, + content_type='application/x-myjson', content_encoding='utf-8') + + @task_postrun.connect def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): + from django.utils import timezone + from etools_datamart.apps.subscriptions.models import Subscription + from etools_datamart.apps.etl.models import EtlTask + + # from unicef_rest_framework.models import Service + if state != 'SUCCESS': + EtlTask.objects.filter(task=task.name).update(status=state) + if not hasattr(sender, 'linked_model'): return try: @@ -90,13 +106,22 @@ def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, st except KeyError: # pragma: no cover cost = -1 defs = {'elapsed': cost, - 'result': state, - 'timestamp': timezone.now()} + 'status': state} + if state == 'SUCCESS': + defs['results'] = retval.as_dict() + if retval.created > 0 or retval.updated > 0: + defs['last_changes'] = timezone.now() + for service in sender.linked_model.linked_services: + service.invalidate_cache() + Subscription.objects.notify(sender.linked_model) + defs['last_success'] = timezone.now() else: + if not isinstance(retval, dict): + defs['results'] = str(retval) defs['last_failure'] = timezone.now() - from etools_datamart.apps.etl.models import EtlTask EtlTask.objects.update_or_create(task=task.name, defaults=defs) + # Service.objects.invalidate_cache() app.timers[task.name] = cost diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 09989622e..25e761071 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import datetime -import os from pathlib import Path import environ @@ -19,6 +18,7 @@ API_CACHE_URL=(str, "locmemcache://"), # CACHE_URL=(str, "dummycache://"), # API_CACHE_URL=(str, "dummycache://"), + ABSOLUTE_BASE_URL=(str, 'http://localhost:8000'), DISCONNECT_URL=(str, 'https://login.microsoftonline.com/unicef.org/oauth2/logout'), ENABLE_LIVE_STATS=(bool, True), CELERY_BROKER_URL=(str, 'redis://127.0.0.1:6379/2'), @@ -44,6 +44,18 @@ AZURE_CLIENT_SECRET=(str, ''), AZURE_TENANT=(str, ''), + AZURE_ACCOUNT_NAME=(str, ''), + AZURE_ACCOUNT_KEY=(str, ''), + AZURE_CONTAINER=(str, ''), + AZURE_OVERWRITE_FILES=(bool, True), + AZURE_LOCATION=(str, ''), + + EMAIL_USE_TLS=(bool, True), + EMAIL_HOST=(str, ''), + EMAIL_HOST_USER=(str, ''), + EMAIL_HOST_PASSWORD=(str, ''), + EMAIL_PORT=(int, 587), + ) DEBUG = env.bool('DEBUG') @@ -56,6 +68,7 @@ SECRET_KEY = env('SECRET_KEY') ALLOWED_HOSTS = tuple(env.list('ALLOWED_HOSTS', default=[])) +ABSOLUTE_BASE_URL = env('ABSOLUTE_BASE_URL') ADMINS = ( ('Stefano', 'saxix@saxix.onmicrosoft.com'), @@ -168,8 +181,10 @@ ] AUTHENTICATION_BACKENDS = [ - 'social_core.backends.azuread_tenant.AzureADTenantOAuth2', + # 'social_core.backends.azuread_tenant.AzureADTenantOAuth2', + 'unicef_security.azure.AzureADTenantOAuth2Ext', 'django.contrib.auth.backends.ModelBackend', + 'django.contrib.auth.backends.RemoteUserBackend', ] CACHES = { @@ -259,12 +274,15 @@ 'django_filters', 'month_field', 'drf_querystringfilter', + 'crispy_forms', 'drf_yasg', 'adminfilters', 'django_db_logging', 'django_sysinfo', 'crashlog', + 'post_office', + 'djcelery_email', 'django_celery_beat', @@ -273,9 +291,10 @@ 'etools_datamart.apps.data', 'etools_datamart.apps.etl.apps.Config', 'etools_datamart.apps.tracking.apps.Config', + 'etools_datamart.apps.subscriptions', 'etools_datamart.api', ] - +DATE_FORMAT = '%d %b %Y' DATE_INPUT_FORMATS = [ '%Y-%m-%d', '%m/%d/%Y', '%m/%d/%y', # '2006-10-25', '10/25/2006', '10/25/06' '%b %d %Y', '%b %d, %Y', # 'Oct 25 2006', 'Oct 25, 2006' @@ -284,6 +303,8 @@ '%d %B %Y', '%d %B, %Y', # '25 October 2006', '25 October, 2006' ] +DATETIME_FORMAT = '%d %b %Y %H:%M:%S' + DATETIME_INPUT_FORMATS = [ '%Y-%m-%d %H:%M:%S', # '2006-10-25 14:30:59' '%Y-%m-%d %H:%M:%S.%f', # '2006-10-25 14:30:59.000200' @@ -298,7 +319,21 @@ '%m/%d/%y %H:%M', # '10/25/06 14:30' '%m/%d/%y', # '10/25/06' ] - +EMAIL_BACKEND = 'post_office.EmailBackend' +EMAIL_POST_OFFICE_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST_USER = env('EMAIL_HOST_USER') +EMAIL_HOST_PASSWORD = env('EMAIL_HOST_PASSWORD') +EMAIL_HOST = env('EMAIL_HOST') +EMAIL_PORT = env.int('EMAIL_PORT') +EMAIL_USE_TLS = env.bool('EMAIL_USE_TLS') +EMAIL_SUBJECT_PREFIX = "[ETOOLS-DATAMART]" +POST_OFFICE = { + 'DEFAULT_PRIORITY': 'now', + 'BACKENDS': { + 'default': 'djcelery_email.backends.CeleryEmailBackend' + } +} +CELERY_EMAIL_CHUNK_SIZE = 10 # django-secure CSRF_COOKIE_SECURE = env.bool('CSRF_COOKIE_SECURE') SECURE_BROWSER_XSS_FILTER = True @@ -312,7 +347,6 @@ X_FRAME_OPTIONS = env('X_FRAME_OPTIONS') NOTIFICATION_SENDER = "etools_datamart@unicef.org" -EMAIL_SUBJECT_PREFIX = "[ETOOLS-DATAMART]" # django-constance CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' @@ -335,6 +369,12 @@ }], } +AZURE_ACCOUNT_NAME = env('AZURE_ACCOUNT_NAME') +AZURE_ACCOUNT_KEY = env('AZURE_ACCOUNT_KEY') +AZURE_CONTAINER = env('AZURE_CONTAINER') +AZURE_OVERWRITE_FILES = env.bool('AZURE_OVERWRITE_FILES') +AZURE_LOCATION = env('AZURE_LOCATION') + CONSTANCE_CONFIG = { 'AZURE_USE_GRAPH': (True, 'Use MS Graph API to fetch user data', bool), 'DEFAULT_GROUP': ('Guests', 'Default group new users belong to', 'select_group'), @@ -344,19 +384,24 @@ CELERY_TIMEZONE = 'America/New_York' CELERY_BROKER_URL = env('CELERY_BROKER_URL') CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') -CELERY_ACCEPT_CONTENT = ['application/json'] -CELERY_RESULT_SERIALIZER = 'json' -CELERY_TASK_SERIALIZER = 'json' +# CELERY_ACCEPT_CONTENT = ['application/json'] +# CELERY_RESULT_SERIALIZER = 'json' +# CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_IMPORTS = ["etools_datamart.apps.etl.tasks.etl", "etools_datamart.apps.etl.tasks.tasks", ] CELERY_BEAT_SCHEDULE = {} CELERY_TASK_ALWAYS_EAGER = env.bool('CELERY_ALWAYS_EAGER', False) CELERY_EAGER_PROPAGATES_EXCEPTIONS = CELERY_TASK_ALWAYS_EAGER + CELERY_TASK_ROUTES = { 'etools_datamart.apps.etl.tasks.etl': {'queue': 'etl'}, 'etools_datamart.apps.etl.tasks.tasks': {'queue': 'tasks'}, } +CELERY_ACCEPT_CONTENT = ['etljson'] +CELERY_TASK_SERIALIZER = 'etljson' +CELERY_RESULT_SERIALIZER = 'etljson' + CONCURRENCY_IGNORE_DEFAULT = False REST_FRAMEWORK = { @@ -368,10 +413,12 @@ # "DEFAULT_PAGINATION_CLASS": 'rest_framework.pagination.CursorPagination', 'DEFAULT_PAGINATION_CLASS': 'unicef_rest_framework.pagination.APIPagination', 'DEFAULT_METADATA_CLASS': 'etools_datamart.api.metadata.SimpleMetadataWithFilters', + 'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning', # 'DEFAULT_SCHEMA_CLASS': 'etools_datamart.api.swagger.APIAutoSchema', # 'EXCEPTION_HANDLER': 'my_project.my_app.utils.custom_exception_handler' 'SEARCH_PARAM': 'search', 'ORDERING_PARAM': 'ordering', + 'DATETIME_FORMAT': DATETIME_FORMAT } JWT_AUTH = { @@ -388,7 +435,7 @@ 'JWT_DECODE_HANDLER': 'rest_framework_jwt.utils.jwt_decode_handler', # Keys will be set in core.apps.Config.ready() - 'JWT_PUBLIC_KEY': os.environ, + 'JWT_PUBLIC_KEY': '?', # 'JWT_PRIVATE_KEY': wallet.get_private(), # 'JWT_PRIVATE_KEY': None, 'JWT_ALGORITHM': 'RS256', @@ -405,6 +452,7 @@ SOCIAL_AUTH_SANITIZE_REDIRECTS = False SOCIAL_AUTH_URL_NAMESPACE = 'social' SOCIAL_AUTH_WHITELISTED_DOMAINS = ['unicef.org', ] +SOCIAL_AUTH_REVOKE_TOKENS_ON_DISCONNECT = True SOCIAL_AUTH_PIPELINE = ( 'social_core.pipeline.social_auth.social_details', 'unicef_security.azure.get_unicef_user', @@ -426,6 +474,9 @@ SOCIAL_AUTH_AZUREAD_TENANT_OAUTH2_SECRET = env.str('AZURE_CLIENT_SECRET') SOCIAL_AUTH_AZUREAD_TENANT_OAUTH2_TENANT_ID = env.str('AZURE_TENANT') SOCIAL_AUTH_AZUREAD_OAUTH2_KEY = env.str('AZURE_CLIENT_ID') +SOCIAL_AUTH_AZUREAD_OAUTH2_RESOURCE = 'https://graph.microsoft.com/' +SOCIAL_AUTH_USER_MODEL = 'unicef_security.User' + # POLICY = os.getenv('AZURE_B2C_POLICY_NAME', "b2c_1A_UNICEF_PARTNERS_signup_signin") SCOPE = ['openid', 'email'] IGNORE_DEFAULT_SCOPE = True diff --git a/src/etools_datamart/config/urls.py b/src/etools_datamart/config/urls.py index 73df8dcce..9cbc0cf6e 100644 --- a/src/etools_datamart/config/urls.py +++ b/src/etools_datamart/config/urls.py @@ -7,26 +7,18 @@ from etools_datamart.apps.multitenant.views import SelectSchema urlpatterns = [ + path(r's/', include('etools_datamart.apps.subscriptions.urls')), path(r'', include('etools_datamart.apps.web.urls')), - path('', include('social_django.urls', namespace='social')), - - # path(r'api/auth/', include('rest_framework_social_oauth2.urls')), + path(r'', include('social_django.urls', namespace='social')), re_path(r'^authorize/?$', AuthorizationView.as_view(), name="authorize"), - # url(r'^token/?$', TokenView.as_view(), name="token"), - # url('', include('social_django.urls', namespace="social")), - # url(r'^convert-token/?$', ConvertTokenView.as_view(), name="convert_token"), - # url(r'^revoke-token/?$', RevokeTokenView.as_view(), name="revoke_token"), - # url(r'^invalidate-sessions/?$', invalidate_sessions, name="invalidate_sessions") - - path(r'api/', include(etools_datamart.api.urls), name='api'), - path('admin/', site.urls), + path(r'admin/', site.urls), path(r'admin/schemas/', SelectSchema.as_view(), name='select-schema'), + path(r'admin/sysinfo/', admin_sysinfo, name="sys-admin-info"), - path('sys/info/', http_basic_login(sysinfo), name='sys-info'), - re_path('sys/version/(?P.*)/', http_basic_login(version), name='sys-version'), - path("admin/sysinfo/", admin_sysinfo, name="sys-admin-info"), + path(r'sys/info/', http_basic_login(sysinfo), name='sys-info'), + path(r'sys/version//', http_basic_login(version), name='sys-version'), ] diff --git a/src/unicef_rest_framework/admin/acl.py b/src/unicef_rest_framework/admin/acl.py index c60548bc9..66ffc0d75 100644 --- a/src/unicef_rest_framework/admin/acl.py +++ b/src/unicef_rest_framework/admin/acl.py @@ -2,10 +2,16 @@ import logging +from admin_extra_urls.extras import ExtraUrlMixin, link from django import forms from django.contrib import admin -from unicef_rest_framework.models import UserAccessControl -from unicef_rest_framework.models.acl import GroupAccessControl +from django.contrib.admin import widgets +from django.contrib.admin.helpers import AdminForm +from django.contrib.auth.models import Group +from django.contrib.postgres.forms import SimpleArrayField +from django.template.response import TemplateResponse +from unicef_rest_framework.models import Service, UserAccessControl +from unicef_rest_framework.models.acl import AbstractAccessControl, GroupAccessControl logger = logging.getLogger(__name__) @@ -32,11 +38,64 @@ def get_queryset(self, request): return super(UserAccessControlAdmin, self).get_queryset(request).select_related(*self.raw_id_fields) -class GroupAccessControlAdmin(admin.ModelAdmin): +class GroupAccessControlForm(forms.Form): + group = forms.ModelChoiceField(queryset=Group.objects.all()) + policy = forms.ChoiceField(choices=AbstractAccessControl.POLICIES) + services = forms.ModelMultipleChoiceField(queryset=Service.objects.all(), + widget=widgets.FilteredSelectMultiple('Services', False) + ) + rate = forms.CharField(max_length=100) + serializers = SimpleArrayField(forms.CharField(), + max_length=255) + + +class GroupAccessControlAdmin(ExtraUrlMixin, admin.ModelAdmin): list_display = ('group', 'service', 'rate', 'serializers', 'policy') list_filter = ('group', 'policy', 'service') search_fields = ('group', 'service',) form = GroupACLAdminForm + autocomplete_fields = ('group',) + + # filter_horizontal = ('services',) def get_queryset(self, request): return super(GroupAccessControlAdmin, self).get_queryset(request).select_related(*self.raw_id_fields) + + @link() + def add_acl(self, request): + opts = self.model._meta + ctx = { + 'opts': opts, + 'add': False, + 'has_view_permission': True, + 'has_editable_inline_admin_formsets': True, + 'app_label': opts.app_label, + 'change': True, + 'is_popup': False, + 'save_as': False, + 'media': self.media, + 'has_delete_permission': False, + 'has_add_permission': False, + 'has_change_permission': True, + } + if request.method == 'POST': + form = GroupAccessControlForm(request.POST) + if form.is_valid(): + services = form.cleaned_data.pop('services') + for service in services: + GroupAccessControl.objects.get_or_create(service=service, + **form.cleaned_data) + self.message_user(request, 'ACLs created') + + else: + form = GroupAccessControlForm(initial={'rate': '*', + 'policy': AbstractAccessControl.POLICY_ALLOW, + 'serializers': 'std'}) + ctx['adminform'] = AdminForm(form, + [(None, {'fields': [['group', + 'policy'], + 'services', + ['rate', 'serializers']]})], + {}) + ctx['media'] = self.media + form.media + return TemplateResponse(request, 'admin/unicef_rest_framework/groupaccesscontrol/add.html', ctx) diff --git a/src/unicef_rest_framework/admin/service.py b/src/unicef_rest_framework/admin/service.py index 7d7027990..397d2a082 100644 --- a/src/unicef_rest_framework/admin/service.py +++ b/src/unicef_rest_framework/admin/service.py @@ -39,7 +39,7 @@ def get_stash_url(obj, label=None, **kwargs): class ServiceAdmin(ExtraUrlMixin, admin.ModelAdmin): - list_display = ('name', 'visible', 'security', 'cache_version', + list_display = ('name', 'visible', 'security', 'cache_version', 'source_model', 'json', 'admin') list_filter = ('hidden', 'access') diff --git a/src/unicef_rest_framework/migrations/0001_initial.py b/src/unicef_rest_framework/migrations/0001_initial.py index 000ea3f87..06eaedc84 100644 --- a/src/unicef_rest_framework/migrations/0001_initial.py +++ b/src/unicef_rest_framework/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import uuid @@ -69,7 +69,6 @@ class Migration(migrations.Migration): ], options={ 'ordering': ('name',), - 'permissions': (('do_not_scramble', 'Can read any service unscrambled'),), }, ), migrations.CreateModel( diff --git a/src/unicef_rest_framework/migrations/0002_auto_20181031_1959.py b/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py similarity index 91% rename from src/unicef_rest_framework/migrations/0002_auto_20181031_1959.py rename to src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py index 1de465ebc..bf25f9213 100644 --- a/src/unicef_rest_framework/migrations/0002_auto_20181031_1959.py +++ b/src/unicef_rest_framework/migrations/0002_auto_20181129_0824.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.db.models.deletion from django.conf import settings @@ -11,9 +11,9 @@ class Migration(migrations.Migration): dependencies = [ ('auth', '0009_alter_user_last_name_max_length'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('contenttypes', '0002_remove_content_type_name'), ('unicef_rest_framework', '0001_initial'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -70,7 +70,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='service', name='linked_models', - field=models.ManyToManyField(blank=True, help_text='models that the service depends on', to='contenttypes.ContentType'), + field=models.ManyToManyField(blank=True, help_text='models that the service depends on', related_name='_service_linked_models_+', to='contenttypes.ContentType'), + ), + migrations.AddField( + model_name='service', + name='source_model', + field=models.ForeignKey(blank=True, help_text='model used as primary datasource', on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType'), ), migrations.AddField( model_name='groupaccesscontrol', @@ -126,7 +131,7 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='systemfilter', - unique_together={('service', 'user'), ('service', 'group')}, + unique_together={('service', 'group'), ('service', 'user')}, ), migrations.AlterUniqueTogether( name='groupaccesscontrol', diff --git a/src/unicef_rest_framework/models/service.py b/src/unicef_rest_framework/models/service.py index ea358ebd0..ee4b0fdde 100644 --- a/src/unicef_rest_framework/models/service.py +++ b/src/unicef_rest_framework/models/service.py @@ -25,16 +25,20 @@ def invalidate_cache(self, **kwargs): def get_for_viewset(self, viewset): name = getattr(viewset, 'label', viewset.__name__) - + source_model = ContentType.objects.get_for_model(viewset().get_queryset().model) service, isnew = self.model.objects.get_or_create(viewset=viewset, defaults={ 'name': name, 'cache_ttl': '1y', 'access': getattr(viewset, 'default_access', conf.DEFAULT_ACCESS), - 'description': getattr(viewset, '__doc__', "")}) - - viewset.get_service.cache_clear() + 'description': getattr(viewset, '__doc__', ""), + 'source_model': source_model + }) + if not isnew: + service.source_model = source_model + service.save() + viewset.get_service.cache_clear() return service, isnew def load_services(self): @@ -93,7 +97,13 @@ class Service(MasterDataModel): null=True, blank=True, help_text='Key used to invalidate service cache') + source_model = models.ForeignKey(ContentType, + models.CASCADE, + blank=True, + help_text="model used as primary datasource") + linked_models = models.ManyToManyField(ContentType, + related_name='+', blank=True, help_text="models that the service depends on") @@ -118,7 +128,7 @@ def get_access_level(self): def endpoint(self): for __, viewset, base_name in conf.ROUTER.registry: if viewset == self.viewset: - return reverse(f'api:{base_name}-list') + return reverse(f'api:{base_name}-list', args=['v1']) else: return None diff --git a/src/unicef_rest_framework/permissions.py b/src/unicef_rest_framework/permissions.py index c88cac98d..b26e80672 100644 --- a/src/unicef_rest_framework/permissions.py +++ b/src/unicef_rest_framework/permissions.py @@ -17,7 +17,7 @@ def get_acl(self, request, view): try: return UserAccessControl.objects.get(service__viewset=fqn(view), user=request.user) - except GroupAccessControl.DoesNotExist: + except UserAccessControl.DoesNotExist: return GroupAccessControl.objects.get(service__viewset=fqn(view), group__user=request.user) @@ -42,6 +42,6 @@ def has_permission(self, request, view): raise PermissionDenied(f"Forbidden serializer '{requested_serializer}'") return True - except (UserAccessControl.DoesNotExist): + except (GroupAccessControl.DoesNotExist): logger.error(f"User '{request.user}' does not have grants for '{fqn(view)}'") return False diff --git a/src/unicef_rest_framework/renderers/__init__.py b/src/unicef_rest_framework/renderers/__init__.py index 9651defec..8cc3dadc9 100644 --- a/src/unicef_rest_framework/renderers/__init__.py +++ b/src/unicef_rest_framework/renderers/__init__.py @@ -1,3 +1,6 @@ from .api import APIBrowsableAPIRenderer # noqa from .microsoft.json import MSJSONRenderer # noqa from .microsoft.xml import MSXmlRenderer # noqa +from .xls import XLSXRenderer # noqa +from .html import HTMLRenderer # noqa +from .pdf import PDFRenderer # noqa diff --git a/src/unicef_rest_framework/renderers/csv.py b/src/unicef_rest_framework/renderers/csv.py index e8fb81431..73b03479b 100644 --- a/src/unicef_rest_framework/renderers/csv.py +++ b/src/unicef_rest_framework/renderers/csv.py @@ -1,8 +1,18 @@ +import logging + +from crashlog.middleware import process_exception from rest_framework_csv import renderers as r +logger = logging.getLogger(__name__) + class CSVRenderer(r.CSVRenderer): def render(self, data, media_type=None, renderer_context=None, writer_opts=None): - data = dict(data)['results'] - return super().render(data, media_type, renderer_context or {}, writer_opts) + try: + data = dict(data)['results'] + return super().render(data, media_type, renderer_context or {}, writer_opts) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py new file mode 100644 index 000000000..59627a407 --- /dev/null +++ b/src/unicef_rest_framework/renderers/html.py @@ -0,0 +1,44 @@ +import logging + +from crashlog.middleware import process_exception +from django.template import loader +from rest_framework.renderers import BaseRenderer + +logger = logging.getLogger(__name__) + + +def labelize(v): + return v.replace("_", " ").title() + + +class HTMLRenderer(BaseRenderer): + media_type = 'text/html' + format = 'xhtml' + charset = 'utf-8' + render_style = 'text' + + def get_template(self, meta): + return loader.select_template([ + f'renderers/html/{meta.app_label}/{meta.model_name}.html', + 'renderers/html/html.html']) + + def render(self, data, accepted_media_type=None, renderer_context=None): + try: + model = renderer_context['view'].queryset.model + opts = model._meta + template = self.get_template(opts) + if data['results']: + c = {'data': data, + 'model': model, + 'opts': opts, + 'headers': [labelize(v) for v in data['results'][0].keys()]} + else: + c = {'data': {}, + 'model': model, + 'opts': opts, + 'headers': []} + return template.render(c) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py new file mode 100644 index 000000000..416e61531 --- /dev/null +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -0,0 +1,68 @@ +import io +import logging +import os + +from crashlog.middleware import process_exception +from django.conf import settings +from django.template import loader +from xhtml2pdf import pisa + +from .html import HTMLRenderer + +logger = logging.getLogger(__name__) + + +def link_callback(uri, rel): + """ + Convert HTML URIs to absolute system paths so xhtml2pdf can access those + resources + """ + # use short variable names + sUrl = settings.STATIC_URL # Typically /static/ + sRoot = settings.STATIC_ROOT # Typically /home/userX/project_static/ + mUrl = settings.MEDIA_URL # Typically /static/media/ + mRoot = settings.MEDIA_ROOT # Typically /home/userX/project_static/media/ + + # convert URIs to absolute system paths + if uri.startswith(mUrl): + path = os.path.join(mRoot, uri.replace(mUrl, "")) + elif uri.startswith(sUrl): + path = os.path.join(sRoot, uri.replace(sUrl, "")) + else: + return uri # handle absolute uri (ie: http://some.tld/foo.png) + + # make sure that file exists + if not os.path.isfile(path): + raise Exception( + 'media URI must start with %s or %s' % (sUrl, mUrl) + ) + return path + + +class PDFRenderer(HTMLRenderer): + media_type = 'application/pdf' + format = 'pdf' + charset = 'utf-8' + render_style = 'text' + + def get_template(self, meta): + return loader.select_template([ + f'renderers/pdf/{meta.app_label}/{meta.model_name}.html', + 'renderers/pdf/pdf.html']) + + def render(self, data, accepted_media_type=None, renderer_context=None): + try: + html = super(PDFRenderer, self).render(data, accepted_media_type, renderer_context) + + # create a pdf + buffer = io.BytesIO() + pisaStatus = pisa.CreatePDF(html, dest=buffer, link_callback=link_callback) + # if error then show some funy view + if pisaStatus.err: + raise Exception('We had some errors 
' + html + '
') + buffer.seek(0) + return buffer.read() + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/renderers/xls.py b/src/unicef_rest_framework/renderers/xls.py new file mode 100644 index 000000000..a70a53918 --- /dev/null +++ b/src/unicef_rest_framework/renderers/xls.py @@ -0,0 +1,19 @@ +import logging + +from crashlog.middleware import process_exception +from drf_renderer_xlsx.renderers import XLSXRenderer as _XLSXRenderer + +logger = logging.getLogger(__name__) + + +class XLSXRenderer(_XLSXRenderer): + + def render(self, data, accepted_media_type=None, renderer_context=None): + try: + if not data['results']: + return '' + return super().render(data, accepted_media_type, renderer_context) + except Exception as e: + process_exception(e) + logger.exception(e) + raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/templates/admin/unicef_rest_framework/groupaccesscontrol/add.html b/src/unicef_rest_framework/templates/admin/unicef_rest_framework/groupaccesscontrol/add.html new file mode 100644 index 000000000..2f0cdaf9e --- /dev/null +++ b/src/unicef_rest_framework/templates/admin/unicef_rest_framework/groupaccesscontrol/add.html @@ -0,0 +1,28 @@ +{% extends "admin/change_form.html" %}{% load i18n admin_urls static admin_modify %} +{% block breadcrumbs %} +
+ {% trans 'Home' %} + › {{ opts.app_config.verbose_name }} + › {{ opts.verbose_name_plural|capfirst }} + › Enable multiple services +
+{% endblock %} +{% block object-tools %}{% endblock %} + +{#{% block content %}#} +{#

Grant Access multiple services

#} +{#
#} +{# {% csrf_token %}#} +{#
#} +{#
{{ form.services }}
#} +{#
#} +{# #} +{# {{ form }}#} +{#
#} +{#
#} +{#
#} +{#{% endblock content %}#} + +{% block submit_buttons_bottom %} + +{% endblock %} diff --git a/src/unicef_rest_framework/templates/renderers/html/html.html b/src/unicef_rest_framework/templates/renderers/html/html.html new file mode 100644 index 000000000..a807290c9 --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/html/html.html @@ -0,0 +1,46 @@ +{% load static datamart %} + + + + {{ opts.verbose_name }} + + + +

{{ opts.verbose_name }}

+
+ + + {% for v in headers %} + + {% endfor %} + + {% for row in data.results %} + + {% for k,v in row.items %} + + {% endfor %} + + {% endfor %} +
{{ v }}
{{ v }}
+
+ + diff --git a/src/unicef_rest_framework/templates/renderers/pdf/pdf.html b/src/unicef_rest_framework/templates/renderers/pdf/pdf.html new file mode 100644 index 000000000..f989b316b --- /dev/null +++ b/src/unicef_rest_framework/templates/renderers/pdf/pdf.html @@ -0,0 +1,48 @@ +{% load static datamart %} + + + + {{ opts.verbose_name }} + + + +

{{ opts.verbose_name }}

+
+{% if headers %} + + + {% for v in headers %} + + {% endfor %} + + {% for row in data.results %} + + {% for k,v in row.items %} + + {% endfor %} + + {% endfor %} +
{{ v }}
{{ v }}
+{% endif %} +
+ + diff --git a/src/unicef_rest_framework/views.py b/src/unicef_rest_framework/views.py index 5cead7c55..1d287d0a4 100644 --- a/src/unicef_rest_framework/views.py +++ b/src/unicef_rest_framework/views.py @@ -3,20 +3,21 @@ import rest_framework_extensions.utils from drf_querystringfilter.backend import QueryStringFilterBackend -from drf_renderer_xlsx.renderers import XLSXRenderer from dynamic_serializer.core import DynamicSerializerMixin from rest_framework import viewsets from rest_framework.authentication import BasicAuthentication, SessionAuthentication, TokenAuthentication from rest_framework.filters import OrderingFilter from rest_framework.renderers import JSONRenderer from rest_framework_xml.renderers import XMLRenderer +from rest_framework_yaml.renderers import YAMLRenderer from strategy_field.utils import fqn from unicef_rest_framework import acl from unicef_rest_framework.auth import JWTAuthentication from unicef_rest_framework.cache import cache_response, etag, ListKeyConstructor from unicef_rest_framework.filtering import SystemFilterBackend from unicef_rest_framework.permissions import ServicePermission -from unicef_rest_framework.renderers import APIBrowsableAPIRenderer, MSJSONRenderer, MSXmlRenderer +from unicef_rest_framework.renderers import (APIBrowsableAPIRenderer, HTMLRenderer, MSJSONRenderer, + MSXmlRenderer, PDFRenderer, XLSXRenderer,) from unicef_rest_framework.renderers.csv import CSVRenderer @@ -46,7 +47,10 @@ class ReadOnlyModelViewSet(DynamicSerializerMixin, viewsets.ReadOnlyModelViewSet renderer_classes = [JSONRenderer, APIBrowsableAPIRenderer, CSVRenderer, + YAMLRenderer, XLSXRenderer, + HTMLRenderer, + PDFRenderer, MSJSONRenderer, XMLRenderer, MSXmlRenderer, @@ -63,8 +67,9 @@ def store(self, key, value): self.request._request.api_info[key] = value def dispatch(self, request, *args, **kwargs): - request.api_info["view"] = fqn(self) - request.api_info["service"] = self.get_service() + if hasattr(request, 'api_info'): + request.api_info["view"] = fqn(self) + request.api_info["service"] = self.get_service() return super().dispatch(request, *args, **kwargs) diff --git a/src/unicef_security/azure.py b/src/unicef_security/azure.py index 62804459f..292adeac6 100644 --- a/src/unicef_security/azure.py +++ b/src/unicef_security/azure.py @@ -1,4 +1,7 @@ +import base64 +import json import logging +import os import requests from constance import config as constance @@ -7,6 +10,9 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import Group from django.core.cache import cache +from jwt import decode as jwt_decode, DecodeError, ExpiredSignature +from social_core.backends.azuread_tenant import AzureADTenantOAuth2 +from social_core.exceptions import AuthTokenError from social_django.models import UserSocialAuth from . import config @@ -28,6 +34,38 @@ ADMIN_EMAILS = [i[1] for i in settings.ADMINS] +class AzureADTenantOAuth2Ext(AzureADTenantOAuth2): + def user_data(self, access_token, *args, **kwargs): + response = kwargs.get('response') + id_token = response.get('id_token') + + # decode the JWT header as JSON dict + jwt_header = json.loads( + base64.b64decode(id_token.split('.', 1)[0]).decode() + ) + + # get key id and algorithm + key_id = jwt_header['kid'] + algorithm = jwt_header['alg'] + verify = os.environ.get('OAUTH2_VERIFY', False) + key = '' + try: + # retrieve certificate for key_id + if verify: + certificate = self.get_certificate(key_id) + key = certificate.public_key() + + return jwt_decode( + id_token, + verify=verify, + key=key, + algorithms=algorithm, + audience=self.setting('KEY') + ) + except (DecodeError, ExpiredSignature) as error: + raise AuthTokenError(self, error) + + def default_group(**kwargs): is_new = kwargs.get('is_new', False) user = kwargs.get('user', None) diff --git a/src/unicef_security/migrations/0001_initial.py b/src/unicef_security/migrations/0001_initial.py index fdf22ac53..b87338433 100644 --- a/src/unicef_security/migrations/0001_initial.py +++ b/src/unicef_security/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.2 on 2018-10-31 19:59 +# Generated by Django 2.1.3 on 2018-11-29 08:24 import django.contrib.auth.models import django.contrib.auth.validators diff --git a/tests/_test_lib/settings_test.py b/tests/_test_lib/settings_test.py index 81d5b423a..414414610 100644 --- a/tests/_test_lib/settings_test.py +++ b/tests/_test_lib/settings_test.py @@ -1,3 +1,4 @@ +import os import random from etools_datamart.config.settings import * # noqa diff --git a/tests/_test_lib/test_utilities/factories.py b/tests/_test_lib/test_utilities/factories.py index 28fc572e3..ab1823d1a 100644 --- a/tests/_test_lib/test_utilities/factories.py +++ b/tests/_test_lib/test_utilities/factories.py @@ -3,12 +3,17 @@ import factory import unicef_security.models +from django.contrib.auth.models import Group +from django.contrib.contenttypes.models import ContentType from django.db import connections from django.utils import timezone +from factory import SubFactory +from post_office.models import EmailTemplate from unicef_rest_framework.models import Service, SystemFilter, UserAccessControl from etools_datamart.apps.data.models import FAMIndicator, HACT, Intervention, PMPIndicators, UserStats from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.subscriptions.models import Subscription from etools_datamart.apps.tracking.models import APIRequestLog today = timezone.now() @@ -65,6 +70,12 @@ class Meta: model = Intervention +class GroupFactory(factory.DjangoModelFactory): + class Meta: + model = Group + django_get_or_create = ('name',) + + class UserFactory(factory.DjangoModelFactory): class Meta: model = unicef_security.models.User @@ -130,3 +141,22 @@ def rules(self, create, extracted, **kwargs): for field, value in extracted.items(): rule = self.rules.create(field=field, value=value) rule.save() + + +class SubscriptionFactory(factory.DjangoModelFactory): + kwargs = '' + user = SubFactory(UserFactory) + type = Subscription.MESSAGE + content_type = lambda x: ContentType.objects.get_for_model(HACT) # noqa: E731 + + class Meta: + model = Subscription + django_get_or_create = ('user', 'content_type') + + +class EmailTemplateFactory(factory.DjangoModelFactory): + name = 'dataset_changed' + + class Meta: + model = EmailTemplate + django_get_or_create = ('name',) diff --git a/tests/api/test_api_auth_jwt.py b/tests/api/test_api_auth_jwt.py index 859f0315f..8e359c618 100644 --- a/tests/api/test_api_auth_jwt.py +++ b/tests/api/test_api_auth_jwt.py @@ -27,7 +27,7 @@ def user(db): def test_token(user, client): - url = reverse('api:partners-list') + url = reverse('api:partners-list', args=['v1']) client.credentials(HTTP_AUTHORIZATION='jwt ' + TOKEN) with mock.patch('unicef_security.azure.Synchronizer.get_user', return_value={'@odata.context': 'https://graph.microsoft.com/v1.0/$metadata#users/$entity', @@ -43,7 +43,7 @@ def test_token(user, client): @override_config(AZURE_USE_GRAPH=False) def test_token2(user, client): - url = reverse('api:partners-list') + url = reverse('api:partners-list', args=['v1']) client.credentials(HTTP_AUTHORIZATION='jwt ' + TOKEN) with mock.patch('rest_framework_jwt.settings.api_settings.JWT_VERIFY_EXPIRATION', False): ret = client.get(url) diff --git a/tests/api/test_api_common.py b/tests/api/test_api_common.py index 9c8fc7763..4185c6da6 100644 --- a/tests/api/test_api_common.py +++ b/tests/api/test_api_common.py @@ -14,7 +14,7 @@ def pytest_generate_tests(metafunc): if 'url' in metafunc.fixturenames: - urls = [reverse("api:%s" % url.name) for url in router.urls + urls = [reverse("api:%s" % url.name, args=['v1']) for url in router.urls if url.name.endswith('-list')] metafunc.parametrize("url", urls, ids=urls) diff --git a/tests/api/test_api_data.py b/tests/api/test_api_data.py index 5c452dd1a..355b9b3db 100644 --- a/tests/api/test_api_data.py +++ b/tests/api/test_api_data.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import pytest -from tests._test_lib.test_utilities.factories import (FAMIndicatoFactory, HACTFactory, InterventionFactory, - PMPIndicatorFactory, UserStatsFactory,) +from test_utilities.factories import (FAMIndicatoFactory, HACTFactory, InterventionFactory, + PMPIndicatorFactory, UserStatsFactory,) from etools_datamart.api.endpoints import (FAMIndicatorViewSet, InterventionViewSet, PMPIndicatorsViewSet, UserStatsViewSet,) diff --git a/tests/api/test_api_etools.py b/tests/api/test_api_etools.py index cba7704ba..c76391388 100644 --- a/tests/api/test_api_etools.py +++ b/tests/api/test_api_etools.py @@ -12,10 +12,11 @@ def pytest_generate_tests(metafunc): if 'url' in metafunc.fixturenames: if metafunc.function.__name__ == 'test_list': urls = filter(lambda url: 'etools/' in url, - [reverse("api:%s" % url.name) for url in router.urls if url.name.endswith('-list')]) + [reverse("api:%s" % url.name, args=['latest']) + for url in router.urls if url.name.endswith('-list')]) elif metafunc.function.__name__ == 'test_retrieve': urls = filter(lambda url: 'etools/' in url, - [reverse("api:%s" % url.name, args=['_lastest_']) for url in router.urls if + [reverse("api:%s" % url.name, args=['latest', '_lastest_']) for url in router.urls if url.name.endswith('-detail')]) metafunc.parametrize("url", list(urls)) @@ -31,7 +32,7 @@ def test_list(client, url, format, schema): def test_list_with_no_schema_search_all_schemas(client): - url = reverse("api:partners-list") + url = reverse("api:partners-list", args=['latest']) res = client.get(url) assert res.status_code == 200, res.content @@ -45,7 +46,7 @@ def test_retrieve(client, url, format): def test_retrieve_requires_only_one_schema(client): - url = reverse("api:partners-detail", args=['_lastest_']) + url = reverse("api:partners-detail", args=['latest', '_lastest_']) url = f"{url}?country_name=bolivia,chad" res = client.get(url) assert res.status_code == 400, res.content @@ -53,7 +54,7 @@ def test_retrieve_requires_only_one_schema(client): def test_retrieve_requires_one_schema(client): - url = reverse("api:partners-detail", args=['_lastest_']) + url = reverse("api:partners-detail", args=['latest', '_lastest_']) res = client.get(url) assert res.status_code == 400 assert res.json()['error'] == "country_name parameter is mandatory" diff --git a/tests/api/test_api_filtering.py b/tests/api/test_api_filtering.py index 8cdf75a50..715324280 100644 --- a/tests/api/test_api_filtering.py +++ b/tests/api/test_api_filtering.py @@ -3,7 +3,7 @@ @pytest.mark.parametrize('flt', ['country_name=bolivia', 'country_name=', 'country_name=bolivia,chad']) def test_filter_etools_country_name(db, client, flt): - url = f"/api/etools/audit/engagement/?%s" % flt + url = f"/api/latest/etools/audit/engagement/?%s" % flt res = client.get(url) assert res.status_code == 200 assert res.json() @@ -11,7 +11,7 @@ def test_filter_etools_country_name(db, client, flt): @pytest.mark.parametrize('flt', ['country_name=bolivia', 'country_name=', 'country_name=bolivia,chad']) def test_filter_datamart_country_name(db, client, flt): - url = f"/api/datamart/interventions/?%s" % flt + url = f"/api/latest/datamart/interventions/?%s" % flt res = client.get(url) assert res.status_code == 200 assert res.json() @@ -19,7 +19,7 @@ def test_filter_datamart_country_name(db, client, flt): @pytest.mark.parametrize('flt', ['10', 'oct', '10-2018', 'current', '']) def test_filter_datamart_month(db, client, flt): - url = f"/api/datamart/user-stats/?month=%s" % flt + url = f"/api/latest/datamart/user-stats/?month=%s" % flt res = client.get(url) assert res.status_code == 200 assert res.json() diff --git a/tests/api/test_api_pages.py b/tests/api/test_api_pages.py index 91c2251cb..59d215d9b 100644 --- a/tests/api/test_api_pages.py +++ b/tests/api/test_api_pages.py @@ -6,7 +6,7 @@ @pytest.mark.django_db() def test_api_site_root(django_app, admin_user): - url = reverse("api:api-root") + url = reverse("api:api-root", args=['latest']) res = django_app.get(url, user=admin_user, extra_environ={'HTTP_X_SCHEMA': "bolivia,chad,lebanon", @@ -16,7 +16,7 @@ def test_api_site_root(django_app, admin_user): @pytest.mark.django_db() def test_api_list(django_app, admin_user): - url = reverse("api:intervention-list") + url = reverse("api:intervention-list", args=['latest']) res = django_app.get(url, user=admin_user, extra_environ={'HTTP_X_SCHEMA': "bolivia,chad,lebanon", @@ -27,7 +27,7 @@ def test_api_list(django_app, admin_user): @pytest.mark.django_db() def test_api_detail(django_app, admin_user): i = InterventionFactory() - url = reverse("api:intervention-detail", args=[i.pk]) + url = reverse("api:intervention-detail", args=['latest', i.pk]) res = django_app.get(url, user=admin_user, extra_environ={'HTTP_X_SCHEMA': "bolivia,chad,lebanon", diff --git a/tests/api/test_api_system.py b/tests/api/test_api_system.py index 4947cb3b7..5a94b33d0 100644 --- a/tests/api/test_api_system.py +++ b/tests/api/test_api_system.py @@ -8,7 +8,8 @@ def pytest_generate_tests(metafunc): if 'url' in metafunc.fixturenames: urls = filter(lambda url: 'system/' in url, - [reverse("api:%s" % url.name) for url in router.urls if url.name.endswith('-list')]) + [reverse("api:%s" % url.name, args=['latest']) + for url in router.urls if url.name.endswith('-list')]) metafunc.parametrize("url", urls) diff --git a/tests/api/test_api_web.py b/tests/api/test_api_web.py index dc77b459f..bb13548a8 100644 --- a/tests/api/test_api_web.py +++ b/tests/api/test_api_web.py @@ -1,7 +1,7 @@ # test BrowsableAPI import pytest from rest_framework.test import APIClient -from tests._test_lib.test_utilities.factories import UserFactory +from test_utilities.factories import UserFactory from unicef_rest_framework.test_utils import user_allow_service from unicef_security.models import User @@ -19,7 +19,7 @@ def users(db): def test_api_web_index(user): client = APIClient() client.force_authenticate(user) - res = client.get('/api/') + res = client.get('/api/latest/') assert res.status_code == 200 diff --git a/tests/api/test_datamart_security.py b/tests/api/test_datamart_security.py index 5ff28e1d0..2ae628dcf 100644 --- a/tests/api/test_datamart_security.py +++ b/tests/api/test_datamart_security.py @@ -2,17 +2,10 @@ import pytest from rest_framework.test import APIClient -from test_utilities.factories import UserFactory -from tests._test_lib.test_utilities.factories import UserStatsFactory +from test_utilities.factories import UserFactory, UserStatsFactory from unicef_rest_framework.test_utils import user_allow_service from etools_datamart.api.endpoints import PartnerViewSet, UserStatsViewSet -from etools_datamart.apps.etools.models import AuthUser - - -@pytest.fixture() -def etools_user(db): - return AuthUser.objects.get(id=1) @pytest.fixture() diff --git a/tests/api/test_etools_security.py b/tests/api/test_etools_security.py index 33949284e..6641c749e 100644 --- a/tests/api/test_etools_security.py +++ b/tests/api/test_etools_security.py @@ -4,12 +4,6 @@ from unicef_rest_framework.test_utils import user_allow_service from etools_datamart.api.endpoints import PartnerViewSet -from etools_datamart.apps.etools.models import AuthUser - - -@pytest.fixture() -def etools_user(db): - return AuthUser.objects.get(id=1) @pytest.fixture() diff --git a/tests/conftest.py b/tests/conftest.py index 9e417447f..76b4bb289 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,3 +127,17 @@ def number_of_partnerorganization(db): def number_of_intervention(db): # number of partners.Intervention return int((Path(__file__).parent / 'INTERVENTION').read_text()) + + +@pytest.fixture() +def etools_user(db): + from etools_datamart.apps.etools.models import AuthUser + return AuthUser.objects.get(id=1) + + +@pytest.fixture() +def staff_user(etools_user): + from test_utilities.factories import UserFactory + return UserFactory(username=etools_user.username, + email=etools_user.email, + is_staff=True) diff --git a/tests/datamart/test_data_admin.py b/tests/datamart/test_data_admin.py index 256e8d945..a6b53be99 100644 --- a/tests/datamart/test_data_admin.py +++ b/tests/datamart/test_data_admin.py @@ -3,6 +3,7 @@ from django.contrib import messages from django.urls import reverse from test_utilities.factories import PMPIndicatorFactory +from test_utilities.perms import user_grant_permissions @pytest.mark.django_db() @@ -28,7 +29,21 @@ def test_pmpindicators_filter(django_app, admin_user): @pytest.mark.django_db() -def test_pmpindicators_detail(django_app, admin_user, settings): +def test_pmpindicators_detail(django_app, staff_user, settings): + i = PMPIndicatorFactory() + url = reverse("admin:data_pmpindicators_change", args=[i.pk]) + assert staff_user.is_authenticated + with user_grant_permissions(staff_user, ['data.change_pmpindicators']): + res = django_app.get(url, user=staff_user) + assert res.status_code == 200 + res = res.form.submit().follow() + assert res.status_code == 200 + storage = res.context['messages'] + assert [m.message for m in storage] == ['This admin is read-only. Record not saved.'] + + +@pytest.mark.django_db() +def test_pmpindicators_detail_supersuser(django_app, admin_user, settings): i = PMPIndicatorFactory() url = reverse("admin:data_pmpindicators_change", args=[i.pk]) assert admin_user.is_authenticated @@ -36,8 +51,6 @@ def test_pmpindicators_detail(django_app, admin_user, settings): assert res.status_code == 200 res = res.form.submit().follow() assert res.status_code == 200 - storage = res.context['messages'] - assert [m.message for m in storage] == ['This admin is read-only. Record not saved.'] def test_pmpindicators_refresh(django_app, admin_user): diff --git a/tests/etl/test_etl_loaders.py b/tests/etl/test_etl_loaders.py index 82bfd48bc..00582c886 100644 --- a/tests/etl/test_etl_loaders.py +++ b/tests/etl/test_etl_loaders.py @@ -1,20 +1,20 @@ # -*- coding: utf-8 -*- +from freezegun import freeze_time + from etools_datamart.apps.data.models import FAMIndicator, HACT, Intervention, PMPIndicators, UserStats -from etools_datamart.apps.etl.tasks.etl import (load_fam_indicator, load_hact, load_intervention, - load_pmp_indicator, load_user_report,) +from etools_datamart.apps.etl.tasks.etl import (EtlResult, load_fam_indicator, load_hact, + load_intervention, load_pmp_indicator, load_user_report,) def test_load_pmp_indicator(number_of_intervention): load_pmp_indicator.unlock() - assert load_pmp_indicator() == {'Bolivia': number_of_intervention, - 'Chad': number_of_intervention, - 'Lebanon': number_of_intervention} + assert load_pmp_indicator() == EtlResult(created=153) assert PMPIndicators.objects.count() == number_of_intervention * 3 def test_load_intervention(number_of_intervention, settings, monkeypatch): load_intervention.unlock() - load_intervention() + assert load_intervention() == EtlResult(created=number_of_intervention*3) assert Intervention.objects.count() == number_of_intervention * 3 @@ -41,3 +41,25 @@ def test_load_hact(db, settings, monkeypatch): assert bolivia.completed_spotcheck == 0 assert bolivia.completed_hact_audits == 0 assert bolivia.completed_special_audits == 0 + res = load_hact() + assert res == EtlResult(unchanged=3) + + +@freeze_time("2018-11-10") +def test_dataset_increased(db, settings, monkeypatch): + load_user_report.unlock() + load_user_report() + UserStats.objects.first().delete() + ret = load_user_report() + assert ret == EtlResult(created=1, unchanged=2) + + +@freeze_time("2018-11-10") +def test_dataset_changed(db, settings, monkeypatch): + load_user_report.unlock() + ret = load_user_report() + assert ret == EtlResult(created=3) + UserStats.objects.update(total=999, unicef=999) + + ret = load_user_report() + assert ret == EtlResult(updated=3) diff --git a/tests/etl/test_etl_result.py b/tests/etl/test_etl_result.py new file mode 100644 index 000000000..e9ebac924 --- /dev/null +++ b/tests/etl/test_etl_result.py @@ -0,0 +1,21 @@ +from etools_datamart.apps.etl.tasks.etl import EtlResult + + +def test_result_eq(): + assert EtlResult() == EtlResult() + + +def test_result_ne(): + assert not EtlResult() == EtlResult(created=1) + + +def test_result_eq_dict(): + assert EtlResult() == {'created': 0, 'updated': 0, 'unchanged': 0} + + +def test_result_ne_dict(): + assert not EtlResult() == {'created': 1, 'updated': 1, 'unchanged': 1} + + +def test_result_ne_other(): + assert not EtlResult() == 1 diff --git a/tests/etl/test_etl_tasklog.py b/tests/etl/test_etl_tasklog.py index 1ddbc8986..60559d3d6 100644 --- a/tests/etl/test_etl_tasklog.py +++ b/tests/etl/test_etl_tasklog.py @@ -5,6 +5,7 @@ from etools_datamart.apps.data.models import PMPIndicators from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.etl.results import EtlResult from etools_datamart.apps.etl.tasks.etl import load_pmp_indicator from etools_datamart.celery import task_postrun_handler @@ -22,17 +23,19 @@ def test_check_extra_attributes(db): def test_load_pmp_indicator(db): - with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run'): + with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run', + return_value=EtlResult(created=11)): assert load_pmp_indicator.apply() assert EtlTask.objects.filter(task='etools_datamart.apps.etl.tasks.etl.load_pmp_indicator', - result='SUCCESS').exists() + results__created=11, + status='SUCCESS').exists() def test_load_pmp_indicator_fail(db): with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run', side_effect=Exception): assert load_pmp_indicator.apply() assert EtlTask.objects.filter(task='etools_datamart.apps.etl.tasks.etl.load_pmp_indicator', - result='FAILURE') + status='FAILURE') @pytest.fixture() @@ -46,4 +49,4 @@ def test_load_pmp_indicator_running(db, disable_post_run): with mock.patch('etools_datamart.apps.etl.tasks.etl.load_pmp_indicator.run'): assert load_pmp_indicator.apply() assert EtlTask.objects.filter(task='etools_datamart.apps.etl.tasks.etl.load_pmp_indicator', - result='RUNNING') + status='RUNNING') diff --git a/tests/exporters/test_exporter_data.py b/tests/exporters/test_exporter_data.py new file mode 100644 index 000000000..96b63f36e --- /dev/null +++ b/tests/exporters/test_exporter_data.py @@ -0,0 +1,31 @@ +import io +import os + +import pytest +from django.urls import reverse +from rest_framework.test import APIClient + +from etools_datamart.apps.data.models import UserStats +from etools_datamart.apps.etl.tasks.etl import load_user_report + + +@pytest.fixture() +def client(admin_user): + client = APIClient() + assert client.login(username='admin', password='password') + return client + + +@pytest.mark.skipif("CIRCLECI" in os.environ, + reason="Skip in CirlceCI") +def test_export_azure_data(db, client, settings): + load_user_report.unlock() + load_user_report() + assert UserStats.objects.count() + + url = reverse("api:userstats-list", args=['v1']) + res = client.get(f"{url}?format=xlsx") + + from storages.backends.azure_storage import AzureStorage + storage = AzureStorage() + storage.save('test1.xlsx', io.BytesIO(res.content)) diff --git a/tests/multitenant/test_schema_selection.py b/tests/multitenant/test_schema_selection.py index ecf43f296..b1b11161d 100644 --- a/tests/multitenant/test_schema_selection.py +++ b/tests/multitenant/test_schema_selection.py @@ -81,7 +81,7 @@ def test_select_schema_data(django_app, admin_user): def test_api_call_queryparam(client, admin_user): client.login(username='admin', password='password') - url = f'{reverse("api:partners-list")}?country_name=bolivia,lebanon' + url = f'{reverse("api:partners-list", args=["v1"])}?country_name=bolivia,lebanon' res = client.get(url) assert res.status_code == 200 assert conn.schemas == ['bolivia', 'lebanon'] diff --git a/tests/test_subscription.py b/tests/test_subscription.py new file mode 100644 index 000000000..f5dc622d3 --- /dev/null +++ b/tests/test_subscription.py @@ -0,0 +1,103 @@ +import json + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.urls import reverse +from test_utilities.factories import EmailTemplateFactory, SubscriptionFactory +from unicef_rest_framework.test_utils import user_allow_service + +from etools_datamart.apps.etl.models import EtlTask +from etools_datamart.apps.subscriptions.models import Subscription +from etools_datamart.apps.subscriptions.views import subscribe + + +@pytest.fixture() +def etltask(db): + EtlTask.objects.inspect() + return EtlTask.objects.first() + + +@pytest.fixture() +def subscription(etltask): + return SubscriptionFactory(content_type=etltask.content_type, + type=Subscription.MESSAGE) + + +@pytest.fixture() +def subscription_attachment(etltask): + return SubscriptionFactory(content_type=etltask.content_type, + type=Subscription.EXCEL) + + +@pytest.fixture() +def email_templates(): + return (EmailTemplateFactory(name='dataset_changed'), + EmailTemplateFactory(name='dataset_changed_attachment')) + + +@pytest.mark.django_db +def test_subscribe_create(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = admin_user + res = subscribe(request, etltask.pk) + + data = json.loads(res.content) + assert data["status"] == "created" + + +@pytest.mark.django_db +def test_subscribe_update(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = admin_user + res = subscribe(request, etltask.pk) + res = subscribe(request, etltask.pk) + + data = json.loads(res.content) + assert data["status"] == "updated" + + +@pytest.mark.django_db +def test_subscribe_404(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = admin_user + res = subscribe(request, 21) + assert res.status_code == 404 + + +@pytest.mark.django_db +def test_subscribe_invalid(rf, admin_user, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 99}, content_type='application/json') + request.user = admin_user + res = subscribe(request, etltask.pk) + assert res.status_code == 400 + + +@pytest.mark.django_db +def test_subscribe_error(rf, etltask): + request = rf.post(reverse("subscribe", args=[etltask.pk]), + {"type": 1}, content_type='application/json') + request.user = AnonymousUser() + res = subscribe(request, etltask.pk) + assert res.status_code == 500 + + +@pytest.mark.django_db +def test_notification_email(subscription: Subscription, email_templates): + with user_allow_service(subscription.user, subscription.viewset): + emails = Subscription.objects.notify(subscription.content_type.model_class()) + assert len(emails) == 1 + assert emails[0].to == [subscription.user.email] + assert emails[0].attachments.count() == 0 + + +@pytest.mark.django_db +def test_notification_email_attachment(subscription_attachment: Subscription, email_templates): + with user_allow_service(subscription_attachment.user, subscription_attachment.viewset): + emails = Subscription.objects.notify(subscription_attachment.content_type.model_class()) + assert len(emails) == 1 + assert emails[0].to == [subscription_attachment.user.email] + assert emails[0].attachments.count() == 1 diff --git a/tests/test_views.py b/tests/test_views.py index f93a21b5e..404c905e9 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,6 +1,15 @@ # -*- coding: utf-8 -*- +from django.urls import reverse + +from etools_datamart.apps.etl.models import EtlTask def test_home(django_app, admin_user): - res = django_app.get('/') + res = django_app.get(reverse('home')) + assert res.status_code == 200 + + +def test_monitor(django_app, admin_user): + EtlTask.objects.inspect() + res = django_app.get(reverse('monitor'), user=admin_user) assert res.status_code == 200 diff --git a/tests/tracking/test_tracking_log.py b/tests/tracking/test_tracking_log.py index 96aabe459..f2fb32267 100644 --- a/tests/tracking/test_tracking_log.py +++ b/tests/tracking/test_tracking_log.py @@ -28,7 +28,7 @@ def django_app(django_app_mixin, system_user): @pytest.mark.django_db def test_log(enable_stats, django_app, system_user, reset_stats): - url = reverse("api:intervention-list") + url = reverse("api:intervention-list", args=['v1']) url = f"{url}?country_name=bolivia,chad,lebanon" res = django_app.get(url) @@ -53,7 +53,7 @@ def test_log(enable_stats, django_app, system_user, reset_stats): @pytest.mark.django_db def test_threaedlog(enable_threadstats, django_app, admin_user): - url = reverse("api:intervention-list") + url = reverse("api:intervention-list", args=['v1']) url = f"{url}?country_name=bolivia,chad,lebanon" res = django_app.get(url) diff --git a/tox.ini b/tox.ini index 2f5da5d0a..bbbb23a40 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ minversion = 3.5.2 [testenv] basepython = python3.6 -passenv = PYTHONDONTWRITEBYTECODE USER PYTHONPATH DATABASE_URL DATABASE_URL_ETOOLS +passenv = PYTHONDONTWRITEBYTECODE USER PYTHONPATH DATABASE_URL DATABASE_URL_ETOOLS CIRCLECI CIRCLE_* CI setenv = PYTHONDONTWRITEBYTECODE=true PYTHONPATH={toxinidir}/src @@ -36,7 +36,8 @@ setenv = deps = pipenv==2018.10.13 - +;PIPSI_HOME +;PIPSI_BIN_DIR commands = pipenv install -d --deploy --ignore-pipfile pipenv run pre-commit run --all-files