From 61320155acdbfebad2ebb64e9a3d45458b23b98f Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 8 Jan 2021 10:45:49 -0500 Subject: [PATCH 1/4] Remove crashlog library --- Pipfile | 1 - Pipfile.lock | 99 ++++++++++++-------------- src/etools_datamart/config/settings.py | 1 - 3 files changed, 46 insertions(+), 55 deletions(-) diff --git a/Pipfile b/Pipfile index 823831528..a8d08e680 100644 --- a/Pipfile +++ b/Pipfile @@ -24,7 +24,6 @@ django-concurrency = "==2.1.1" django-constance = "==2.6.0" django-cors-headers = "==3.2.1" django-countries = "==6.1.2" -django-crashlog = "==1.0.0" django-crispy-forms = "==1.9.0" django-model-utils = "==4.0.0" django-mptt = "==0.11.0" diff --git a/Pipfile.lock b/Pipfile.lock index 0d972239a..b8d9063ba 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "436761c03ee00824836326bb5a690133034dd03bebc5b59651629319df783d30" + "sha256": "22e3d61349a478c8ffd8611759cf3e2d2420d95139b0528d9dea8a047b17ef29" }, "pipfile-spec": 6, "requires": { @@ -270,13 +270,6 @@ "index": "pypi", "version": "==6.1.2" }, - "django-crashlog": { - "hashes": [ - "sha256:147a09226c69f4f8349292680662e53f20b232dc02d0f86e4532418a1da2dd94" - ], - "index": "pypi", - "version": "==1.0.0" - }, "django-crispy-forms": { "hashes": [ "sha256:50032184708ce351e3c9f0008ac35d659d9d5973fa2db218066f2e0a76eb41d9", @@ -938,48 +931,48 @@ }, "reportlab": { "hashes": [ - "sha256:090d194d830e4a71dd98315409093d8e0dd727d7ebcb7a8c259702f689771a24", - "sha256:092868dc4d20219a25729774ab1a00a7b19a298921c54335d8400b18a5be2947", - "sha256:0bd8f893c56f7e1431474e4e2b84b0f940a383e94645c0462fdda79596a7919c", - "sha256:1460297593a23d2b7f0ac05676a5e3d199e5a60c94d474c077919cbbff2067a6", - "sha256:18236986052a1cb0258bf009ac68aba994b8c88096f7d983ef7f042cd23ee67e", - "sha256:1a8e646e34e28708295f7111ea652313c4c3e5f11bcaed7d1bf431913965ff26", - "sha256:21c11634cd29d992ee6b9483b9399b6ac3a68aac55e8695f79fac9e4748c7ec7", - "sha256:3362106e604ccf462b677062703ccc0d26f20e814d90aec7320f38462057a3a1", - "sha256:3769d938d0c600951b3e803a7a4ff91bf7e6e9648526fa5d10adff842e15650e", - "sha256:3ce2375525556a78c40213c6dc958d0d25b20bd798348c0d29d15178abce85c9", - "sha256:44881370323d74b1cbd8895d8ac6b33017e02e5b6a3ec48d48ca9464db862e86", - "sha256:4d52185ec4f76f4be1eac7ca75a034b8cf6e5349f294819bb3194bbe40a6d0ed", - "sha256:502764b41eb6e0e4c59d20ded779ab21880b6f397797c852798e5ec7c9717368", - "sha256:5ab4b3d11b834fe299d33559d4e8dced4139a4389ef07dd988c1facd5207a3c7", - "sha256:669ff6b054fdf81647da3a0c8a492f5ae5e1573b9856ae209cb5fafb432eecf8", - "sha256:6d52dfbba868658250682e8d9cfcbdd1db966a3df26daa4d7a4b113952785cd5", - "sha256:75379de38ae17ce0238d88747e0755bfa2311138f53e5d3a830e05887125b69e", - "sha256:78638a55a5ca154ca4d3a5c621a2ff319d4f0397c4cacd4297c1c2a4cdfd109c", - "sha256:7d9bdfde0528ed8df0c68ce932a0af006ba6986a11ca478b776c11136184ffe7", - "sha256:81836ff7cb3b1c70cf329a4339ade65c7abab79a95c5d09cf0c01237a6df291e", - "sha256:83b3f4394aac383b24cee9f8bd93484b109a6144ba709c8671fdac1b1ac6ce86", - "sha256:85b44b2575fa375a1786ab4cd273b7089377870f0b9d232f659082149c107210", - "sha256:896c4f2769b19910601976bd86e04e34660fd0686f71ba46b99ad058ba67b6e8", - "sha256:896ddcf720d5ec0f9ce042ce073fe7c6c5ba36ae37cf5930368fe4d9b958037b", - "sha256:8c19564962bd86fdadcd3782411a4afdf3c9e0fbc581cca61c8775df2791ee9c", - "sha256:8dfeebe5fb78b6bee7d053a3ddc1300a01c7ee6e952fdb375c9bb071e6839ef8", - "sha256:9330b55c89bc8dda66dd732cd9168f2a26ae5d43c5358a08cf4ef6763db831ee", - "sha256:954db6ae3411367f46d4900815afc01808e6cdafaf410e9beecd890f7b1783c2", - "sha256:9995b6eb3bbf235f6038d56a63b96b3883ab9c2a226f3ee692969ca56cbb24b4", - "sha256:9fd2754fb47ff13133102b7e41bbb7849c403214cc843349bb52b96b9fefeb35", - "sha256:a2d4c2795726a24514aaa281cf2ce078688dc6110f833677df0ebbe809a100ed", - "sha256:a65bf69c7a95050079c168bf8315fe213397363cc218bd6e57dabe6d2c6a5a7c", - "sha256:b4cba31a161dad4ffaf33f88b543a75b6bd808fe405b5f866b55594e78c226ab", - "sha256:c2c3ecd0af43e00a6f8104bad55f67acaf1ad57ec3cd1db6b8f4421cdbb36aab", - "sha256:c752def5b578bd89366689161a3b91fa8103be3cd8c078794bfc75fd65a756e7", - "sha256:d0b447ab2f9f14a0671232ce8f8d8e9b6b36993f71761142b5e858c77b0c5ab3", - "sha256:d808509337271ddf9022ed774e0081663b4b700e781e98b9671afc031f35f05f", - "sha256:e4592b5478e4083deda5d39f5bad657eee8ea021f55f31b2c2756a832698e910", - "sha256:ed39509a078930599c74d395f624bac3d4c341845bfbf0a22534dc65db357eb5", - "sha256:f08073a7e891525e0e406052dd81d031211bd567df729bdda450eb4b52488168" - ], - "version": "==3.5.58" + "sha256:009fa61710647cdc62eb373345248d8ebb93583a058990f7c4f9be46d90aa5b1", + "sha256:04a08d284da86882ec3a41a7c719833362ef891b09ee8e2fbb47cee352aa684a", + "sha256:07bff6742fba612da8d1b1f783c436338c6fdc6962828159827d5ca7d2b67935", + "sha256:09fb11ab1500e679fc1b01199d2fed24435499856e75043a9ac0d31dd48fd881", + "sha256:18a876449c9000c391dd3415ebc8454cd7bb9e488977b894886a2d7d018f16cd", + "sha256:18eec161411026dde49767bee4e5e8eeb8014879554811a62581dc7433628d5b", + "sha256:19353aead39fc115a4d6c598d6fb9fa26da7e69160a0443ebb49b02903e704e8", + "sha256:1b85c20e89c22ae902ca973df2afdd2d64d27dc4ffd2b29ebad8c805a213756b", + "sha256:1da3d7a35f918cee905facfa94bd00ae6091cadc06dca1b0b31b69ae02d41d1d", + "sha256:33f3cfdc492575f8af3225701301a7e62fc478358729820c9e0091aff5831378", + "sha256:3b0026c1129147befd4e5a8cf25da8dea1096fce371e7b2412e36d7254019c06", + "sha256:3d7713dddaa8081ed709a1fa2456a43f6a74b0f07d605da8441fd53fef334f69", + "sha256:3e2b4d69763103b9dc9b54c0952dc3cee05cedd06e28c0987fad7f84705b12c0", + "sha256:4ca5233a19a5ceca23546290f43addec2345789c7d65bb32f8b2668aa148351f", + "sha256:5214a289cf01ebbd65e49bae83709671dd9edb601891cf0ae8abf85f3c0b392f", + "sha256:52f8237654acbc78ea2fa6fb4a6a06e5b023b6da93f7889adfe2deba09473fad", + "sha256:5ed00894e0f8281c0b7c0494b4d3067c641fd90c8e5cf933089ec4cc9a48e491", + "sha256:6191961533d49c9d860964d42bada4d7ac3bb28502d984feb8034093f2012fa8", + "sha256:6f3ad2b1afe99c436563cd436d8693d4a12e2c4bd45f70c7705759ff7837fe53", + "sha256:739b743b7ca1ba4b4d64c321de6fccb49b562d0507ea06c817d9cc4faed5cd22", + "sha256:792efba0c0c6e4ee94f6dc95f305451733ee9230a1c7d51cb8e5301a549e0dfb", + "sha256:79d63ca40231ca3860859b39a92daa5219035ba9553da89a5e1b218550744121", + "sha256:83b28104edd58ad65748d2d0e60e0d97e3b91b3e90b4573ea6fe60de6811972c", + "sha256:85650446538cd2f606ca234634142a7ccd74cb6db7cfec250f76a4242e0f2431", + "sha256:9da445cb79e3f740756924c053edc952cde11a65ff5af8acfda3c0a1317136ef", + "sha256:9fabd5fbd24f5971085ffe53150d663f158f7d3050b25c95736e29ebf676d454", + "sha256:a0c377bc45e73c3f15f55d7de69fab270d174749d5b454ab0de502b15430ec2a", + "sha256:a1d3f7022a920d4a5e165d264581f1862e1c1b877ceeabb96fe98cec98125ae5", + "sha256:a315edef5c5610b0c75790142f49487e89ea34397fc247ae8aa890fe6d6dd057", + "sha256:a755cca2dcf023130b03bb671670301a992157d5c3151d838c0b68ef89894536", + "sha256:b1b20208ecdfffd7ca027955c4fe8972b28b30a4b3b80cf25099a08d3b20ed7c", + "sha256:b26d6f416891cef93411d6d478a25db275766081a5fb66368248293ef459f3be", + "sha256:b4ba4c30af7044ee987e61c88a5ffb76031ca0c53666bc85d823b7de55ddbc75", + "sha256:b71faf3b6e4d7058e1af1b8afedaf39a962db4a219affc8177009d8244ec10d4", + "sha256:cfa854bea525f8c913cb77e2bda724d94b965a0eb3bcfc4a645a9baa29bb86e2", + "sha256:dd9687359e466086b9f6fe6d8069034017f8b6ca3080944fae5709767ca6814e", + "sha256:de0c675fc2998a7eaa929c356ba49c84f53a892e9ab25e8ee7d8ebbbdcb2ac16", + "sha256:e2b4e33fea2ce9d3a14ea39191b169e41eb2ac995274f54ac8fd27519974bce8", + "sha256:f3d4a1a273dc141e03b72a553c11bc14dd7a27ec7654a071edcf83eb04f004bc", + "sha256:ff547cf4c1de7e104cad1a378431ff81efcb03e90e40871ee686107da5b91442" + ], + "version": "==3.5.59" }, "requests": { "hashes": [ @@ -1304,11 +1297,11 @@ }, "faker": { "hashes": [ - "sha256:7b0c4bb678be21a68640007f254259c73d18f7996a3448267716423360519732", - "sha256:7e98483fc273ec5cfe1c9efa9b99adaa2de4c6b610fbc62d3767088e4974b0ce" + "sha256:bd56ab94b9ae45df3cdb6ef3ca3c2be2617e6a640699deb74394143d220faf04", + "sha256:d70983d5e623976e9e8987e9a39e8d55378e66049f393db35ea9a37b3190085f" ], "markers": "python_version >= '3.6'", - "version": "==5.3.0" + "version": "==5.4.0" }, "flake8": { "hashes": [ diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index c7698d16f..94082b578 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -201,7 +201,6 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'impersonate.middleware.ImpersonateMiddleware', # 'django.contrib.auth.middleware.RemoteUserMiddleware', - 'crashlog.middleware.CrashLogMiddleware', 'unicef_rest_framework.middleware.ApiMiddleware', # 'etools_datamart.apps.tracking.middleware.ThreadedStatsMiddleware', 'etools_datamart.apps.tracking.middleware.StatsMiddleware', From 21e5c1798d114f428825111c761b329fadf734b6 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 8 Jan 2021 10:46:18 -0500 Subject: [PATCH 2/4] Update datamart docker-compose command Ensure EmailTemplates are created, happens in init-setup --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b24a4ee9c..1bd1a6c7c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: datamart: container_name: datamart << : *datamart_defaults - command: bash -c "python /code/manage.py migrate && python /code/manage.py runserver 0.0.0.0:8080" + command: bash -c "python /code/manage.py init-setup --migrate && python /code/manage.py runserver 0.0.0.0:8080" labels: - "traefik.http.routers.datamart.rule=PathPrefix(`/`)" - traefik.http.routers.datamart.service=datamart From 36dd8fb87bdaa0ce5eab5fc8833a0286918ddca4 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 8 Jan 2021 11:04:31 -0500 Subject: [PATCH 3/4] Update default CELERY_EMAIL_BACKEND setting --- src/etools_datamart/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 94082b578..2201d8bce 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -458,7 +458,7 @@ CELERY_TIMEZONE = 'America/New_York' CELERY_EMAIL_BACKEND = env.str( 'CELERY_EMAIL_BACKEND', - default='django.core.mail.backends.smtp.EmailBackend', + default='post_office.backends.EmailBackend', ) CELERY_EMAIL_CHUNK_SIZE = 10 From 4ffabced81623d92d9b88407f46c0a2e2104e996 Mon Sep 17 00:00:00 2001 From: Greg Reinbach Date: Fri, 8 Jan 2021 12:31:31 -0500 Subject: [PATCH 4/4] Remove crashlog exception calls --- src/unicef_rest_framework/auth.py | 2 -- src/unicef_rest_framework/models/preload.py | 2 -- src/unicef_rest_framework/renderers/csv.py | 2 -- src/unicef_rest_framework/renderers/html.py | 2 -- src/unicef_rest_framework/renderers/iqy.py | 2 -- src/unicef_rest_framework/renderers/pdf.py | 2 -- src/unicef_rest_framework/renderers/txt.py | 2 -- src/unicef_rest_framework/renderers/xls.py | 2 -- src/unicef_rest_framework/renderers/yaml.py | 2 -- src/unicef_security/graph.py | 12 ++++-------- 10 files changed, 4 insertions(+), 26 deletions(-) diff --git a/src/unicef_rest_framework/auth.py b/src/unicef_rest_framework/auth.py index 619e2b85b..82aa84614 100644 --- a/src/unicef_rest_framework/auth.py +++ b/src/unicef_rest_framework/auth.py @@ -8,7 +8,6 @@ from django.utils.translation import ugettext as _ from constance import config -from crashlog.middleware import process_exception from rest_framework import exceptions from rest_framework.authentication import BaseAuthentication from rest_framework.exceptions import AuthenticationFailed, PermissionDenied @@ -104,7 +103,6 @@ def authenticate_credentials(self, payload): user, created = User.objects.update_or_create(**pk, defaults=values) except Exception as e: - process_exception(e) raise exceptions.AuthenticationFailed("Unable to retrieve user data") else: user, created = User.objects.update_or_create(username=username, diff --git a/src/unicef_rest_framework/models/preload.py b/src/unicef_rest_framework/models/preload.py index 44ddad10d..03bc78c56 100644 --- a/src/unicef_rest_framework/models/preload.py +++ b/src/unicef_rest_framework/models/preload.py @@ -6,7 +6,6 @@ from django.db import models from django.utils import timezone -from crashlog.middleware import process_exception from rest_framework.test import APIClient, ForceAuthClientHandler from unicef_security.models import User @@ -106,7 +105,6 @@ def run(self, *, target=None, params=None, pre_save=None): pre_save(self, response) return response except Exception as e: - process_exception(e) self.status_code = 501 self.etag = "" self.response_ms = 0 diff --git a/src/unicef_rest_framework/renderers/csv.py b/src/unicef_rest_framework/renderers/csv.py index 06b1c60d5..f61f3416d 100644 --- a/src/unicef_rest_framework/renderers/csv.py +++ b/src/unicef_rest_framework/renderers/csv.py @@ -1,6 +1,5 @@ import logging -from crashlog.middleware import process_exception from rest_framework_csv import renderers as r from unicef_rest_framework.renderers.mixin import ContentDispositionMixin @@ -20,6 +19,5 @@ def render(self, data, media_type=None, renderer_context=None, writer_opts=None) data = dict(data)['results'] return super().render(data, media_type, renderer_context, writer_opts) except Exception as e: - process_exception(e) logger.exception(e) raise Exception('Error processing request') diff --git a/src/unicef_rest_framework/renderers/html.py b/src/unicef_rest_framework/renderers/html.py index e15e72702..4bfd10b3d 100644 --- a/src/unicef_rest_framework/renderers/html.py +++ b/src/unicef_rest_framework/renderers/html.py @@ -2,7 +2,6 @@ from django.template import loader -from crashlog.middleware import process_exception from rest_framework.renderers import BaseRenderer logger = logging.getLogger(__name__) @@ -46,6 +45,5 @@ def render(self, data, accepted_media_type=None, renderer_context=None): 'headers': []} return template.render(c) except Exception as e: - process_exception(e) logger.exception(e) raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/iqy.py b/src/unicef_rest_framework/renderers/iqy.py index fcc0c59e0..f4301c5de 100644 --- a/src/unicef_rest_framework/renderers/iqy.py +++ b/src/unicef_rest_framework/renderers/iqy.py @@ -3,7 +3,6 @@ from django.conf import settings from django.template import loader -from crashlog.middleware import process_exception from rest_framework.renderers import BaseRenderer from unicef_rest_framework.renderers.mixin import ContentDispositionMixin @@ -51,6 +50,5 @@ def render(self, data, accepted_media_type=None, renderer_context=None): template = self.get_template(opts) return template.render(c) except Exception as e: - process_exception(e) logger.exception(e) raise Exception('Error processing request %s' % e) from e diff --git a/src/unicef_rest_framework/renderers/pdf.py b/src/unicef_rest_framework/renderers/pdf.py index d19fcc7fb..6d4e7a6a8 100644 --- a/src/unicef_rest_framework/renderers/pdf.py +++ b/src/unicef_rest_framework/renderers/pdf.py @@ -5,7 +5,6 @@ from django.conf import settings from django.template import loader -from crashlog.middleware import process_exception from xhtml2pdf import pisa from unicef_rest_framework.renderers.mixin import ContentDispositionMixin @@ -72,6 +71,5 @@ def render(self, data, accepted_media_type=None, renderer_context=None): buffer.seek(0) return buffer.read() except Exception as e: - process_exception(e) logger.exception(e) raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/txt.py b/src/unicef_rest_framework/renderers/txt.py index bdd386fd9..8b3cbebfa 100644 --- a/src/unicef_rest_framework/renderers/txt.py +++ b/src/unicef_rest_framework/renderers/txt.py @@ -2,7 +2,6 @@ from django.template import loader -from crashlog.middleware import process_exception from rest_framework.renderers import BaseRenderer logger = logging.getLogger(__name__) @@ -46,6 +45,5 @@ def render(self, data, accepted_media_type=None, renderer_context=None): 'headers': []} return template.render(c) except Exception as e: - process_exception(e) logger.exception(e) raise Exception('Error processing request') from e diff --git a/src/unicef_rest_framework/renderers/xls.py b/src/unicef_rest_framework/renderers/xls.py index 43ce69b4d..f8e106bdf 100644 --- a/src/unicef_rest_framework/renderers/xls.py +++ b/src/unicef_rest_framework/renderers/xls.py @@ -1,7 +1,6 @@ import logging from collections import Iterable, MutableMapping -from crashlog.middleware import process_exception from drf_renderer_xlsx.renderers import XLSXRenderer as _XLSXRenderer from unicef_rest_framework.renderers.mixin import ContentDispositionMixin @@ -40,6 +39,5 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return super().render(data, accepted_media_type, renderer_context) except Exception as e: - process_exception(e) logger.exception(e) raise Exception(f'Error processing request {e}') from e diff --git a/src/unicef_rest_framework/renderers/yaml.py b/src/unicef_rest_framework/renderers/yaml.py index 71b4e9a46..2348c67c2 100644 --- a/src/unicef_rest_framework/renderers/yaml.py +++ b/src/unicef_rest_framework/renderers/yaml.py @@ -1,6 +1,5 @@ import logging -from crashlog.middleware import process_exception from rest_framework_yaml.renderers import YAMLRenderer as BaseRenderer from unicef_rest_framework.renderers.mixin import ContentDispositionMixin @@ -24,6 +23,5 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return super().render(data, accepted_media_type, renderer_context) except Exception as e: - process_exception(e) logger.exception(e) raise Exception(f'Error processing request {e}') from e diff --git a/src/unicef_security/graph.py b/src/unicef_security/graph.py index 77efdee35..a26948e59 100644 --- a/src/unicef_security/graph.py +++ b/src/unicef_security/graph.py @@ -10,7 +10,6 @@ import requests from constance import config as constance -from crashlog.middleware import process_exception from jwt import decode as jwt_decode, DecodeError, ExpiredSignature from social_core.backends.azuread_tenant import AzureADTenantOAuth2 from social_core.exceptions import AuthTokenError @@ -103,15 +102,12 @@ def get_unicef_user(backend, details, response, *args, **kwargs): for k, v in response.items(): if k in ['email', 'family_name', 'unique_name']: details[k] = v - try: - sync = Synchronizer() - data = sync.get_user(details['email']) - for k, v in data.items(): - details[k] = v + sync = Synchronizer() + data = sync.get_user(details['email']) - except Exception as e: - process_exception(e) + for k, v in data.items(): + details[k] = v user, created = User.objects.get_or_create( username=details['unique_name'],