From b660853543b691b306848ea8dbb9a650bf36388d Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Wed, 17 Dec 2025 13:31:35 -0600 Subject: [PATCH 01/13] emails: add mediawiki api backend - mediawiki mail backend for django - delay, retry, and maxlag handling adapted from pywikiapi - test_email - management command - email task and templates - task-specific mail backend switching - additional config - optional .env # @TODO: document Bug: T409420 Bug: T412427 --- TWLight/emails/backends/__init__.py | 0 TWLight/emails/backends/mediawiki.py | 284 ++++++++++++++++++ TWLight/emails/tasks.py | 20 +- .../templates/emails/test-body-html.html | 1 + .../templates/emails/test-body-text.html | 1 + .../emails/templates/emails/test-subject.html | 1 + TWLight/settings/base.py | 7 + .../users/management/commands/test_email.py | 26 ++ TWLight/users/signals.py | 4 + conf/local.twlight.env | 1 + docker-compose.override.yml | 7 +- 11 files changed, 349 insertions(+), 3 deletions(-) create mode 100644 TWLight/emails/backends/__init__.py create mode 100644 TWLight/emails/backends/mediawiki.py create mode 100644 TWLight/emails/templates/emails/test-body-html.html create mode 100644 TWLight/emails/templates/emails/test-body-text.html create mode 100644 TWLight/emails/templates/emails/test-subject.html create mode 100644 TWLight/users/management/commands/test_email.py diff --git a/TWLight/emails/backends/__init__.py b/TWLight/emails/backends/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py new file mode 100644 index 0000000000..535fa1ad34 --- /dev/null +++ b/TWLight/emails/backends/mediawiki.py @@ -0,0 +1,284 @@ +""" +Email backend that POSTs messages to the MediaWiki Emailuser endpoint. +see: https://www.mediawiki.org/wiki/API:Emailuser +""" +import logging +from requests import Session +from requests.exceptions import ConnectionError +from requests.structures import CaseInsensitiveDict +from threading import RLock +from time import sleep + +from django.conf import settings +from django.core.mail.backends.base import BaseEmailBackend + +from TWLight.users.models import Editor + +logger = logging.getLogger(__name__) + + +def retry_conn(): + """A decorator that handles connetion retries.""" + retry_delay = 0 + retry_on_connection_error = 10 + retry_after_conn = 5 + + def wrapper(func): + def conn(*args, **kwargs): + try_count = 0 + try_count_conn = 0 + while True: + try_count += 1 + try_count_conn += 1 + + if retry_delay: + sleep(retry_delay) + try: + return func(*args, **kwargs) + except ConnectionError as e: + no_retry_conn = 0 <= retry_on_connection_error < try_count_conn + if no_retry_conn: + logger.warning("ConnectionError exhausted retries") + raise e + logger.warning( + "ConnectionError, retrying in {}s".format(self.retry_after_conn) + ) + sleep(retry_after_conn) + continue + + return conn + + return wrapper + + +def json_maxlag(response): + """A helper method that handles maxlag retries.""" + data = response.json() + try: + if data["error"]["code"] != "maxlag": + return data + except KeyError: + return data + + retry_after = float(response.headers.get("Retry-After", 5)) + retry_on_lag_error = 50 + no_retry = 0 <= retry_on_lag_error < try_count + + message = "Server exceeded maxlag" + if not no_retry: + message += ", retrying in {}s".format(retry_after) + if "lag" in data["error"]: + message += ", lag={}".format(data["error"]["lag"]) + message += ", API=".format(self.url) + + log = logger.warning if no_retry else logger.info + log( + message, + { + "code": "maxlag-retry", + "retry-after": retry_after, + "lag": data["error"]["lag"] if "lag" in data["error"] else None, + "x-database-lag": response.headers.get("X-Database-Lag", 5), + }, + ) + + if no_retry: + raise Exception(message) + + sleep(retry_after) + + +class EmailBackend(BaseEmailBackend): + def __init__( + self, + url=None, + timeout=None, + delay=None, + retry_delay=None, + maxlag=None, + username=None, + password=None, + fail_silently=False, + **kwargs, + ): + super().__init__(fail_silently=fail_silently) + self.url = settings.MW_API_URL if url is None else url + self.headers = CaseInsensitiveDict() + self.headers["User-Agent"] = "{}/0.0.1".format(__name__) + self.url = settings.MW_API_URL if url is None else url + self.timeout = settings.MW_API_REQUEST_TIMEOUT if timeout is None else timeout + self.delay = settings.MW_API_REQUEST_DELAY if delay is None else delay + self.retry_delay = ( + settings.MW_API_REQUEST_RETRY_DELAY if retry_delay is None else retry_delay + ) + self.maxlag = settings.MW_API_MAXLAG if maxlag is None else maxlag + self.username = settings.MW_API_EMAIL_USER if username is None else username + self.password = settings.MW_API_EMAIL_PASSWORD if password is None else password + self.email_token = None + self.session = None + self._lock = RLock() + logger.info("Email connection constructed.") + + def open(self): + """ + Ensure an open session to the API server. Return whether or not a + new session was required (True or False) or None if an exception + passed silently. + """ + if self.session: + # Nothing to do if the session exists + return False + + try: + # GET request to fetch login token + login_token_params = { + "action": "query", + "meta": "tokens", + "type": "login", + "maxlag": self.maxlag, + "format": "json", + } + logger.info("Getting login token...") + session = Session() + response_login_token = session.get(url=self.url, params=login_token_params) + if response_login_token.status_code != 200: + raise Exception( + "There was an error in the request for obtaining the login token." + ) + login_token_data = json_maxlag(response_login_token) + login_token = login_token_data["query"]["tokens"]["logintoken"] + if not login_token: + raise Exception("There was an error obtaining the login token.") + + # POST request to log in. Use of main account for login is not + # supported. Obtain credentials via Special:BotPasswords + # (https://www.mediawiki.org/wiki/Special:BotPasswords) for lgname & lgpassword + login_params = { + "action": "login", + "lgname": self.username, + "lgpassword": self.password, + "lgtoken": login_token, + "maxlag": self.maxlag, + "format": "json", + } + logger.info("Signing in...") + login_response = session.post(url=self.url, data=login_params) + if login_response.status_code != 200: + raise Exception("There was an error in the request for the login.") + + # GET request to fetch Email token + # see: https://www.mediawiki.org/wiki/API:Emailuser#Token + email_token_params = {"action": "query", "meta": "tokens", "format": "json"} + + logger.info("Getting email token...") + email_token_response = session.get(url=self.url, params=email_token_params) + if email_token_response.status_code != 200: + raise Exception( + "There was an error in the request for the email token." + ) + + email_token_data = json_maxlag(email_token_response) + + email_token = email_token_data["query"]["tokens"]["csrftoken"] + if not email_token: + raise Exception("There was an error obtaining the email token.") + + # Assign the session and email token + self.email_token = email_token + self.session = session + logger.info("Email API session ready.") + return True + except: + if not self.fail_silently: + raise + + def close(self): + """Unset the session.""" + self.email_token = None + self.session = None + + def send_messages(self, email_messages): + """ + Send one or more EmailMessage objects and return the number of email + messages sent. + """ + if not email_messages: + return 0 + with self._lock: + new_session_created = self.open() + if not self.session or new_session_created is None: + # We failed silently on open(). + # Trying to send would be pointless. + return 0 + num_sent = 0 + for message in email_messages: + sent = self._send(message) + if sent: + num_sent += 1 + if new_session_created: + self.close() + return num_sent + + @retry_conn() + def _send(self, email_message): + """A helper method that does the actual sending.""" + if not email_message.recipients(): + return False + + try: + for recipient in email_message.recipients(): + # lookup the target editor from the email address + target = Editor.objects.values_list("wp_username", flat=True).get( + user__email=recipient + ) + + # GET request to check if user is emailable + emailable_params = { + "action": "query", + "list": "users", + "ususers": target, + "usprop": "emailable", + "maxlag": self.maxlag, + "format": "json", + } + + logger.info("Checking if user is emailable...") + emailable_response = self.session.post( + url=self.url, data=emailable_params + ) + if emailable_response.status_code != 200: + raise Exception( + "There was an error in the request to check if the user can receive emails." + ) + emailable_data = json_maxlag(emailable_response) + emailable = "emailable" in emailable_data["query"]["users"][0] + if not emailable: + logger.warning("User not emailable, email skipped.") + continue + + # POST request to send an email + email_params = { + "action": "emailuser", + "target": target, + "subject": email_message.subject, + "text": email_message.body, + "token": self.email_token, + "maxlag": self.maxlag, + "format": "json", + } + + logger.info("Sending email...") + emailuser_response = self.session.post(url=self.url, data=email_params) + if emailuser_response.status_code != 200: + raise Exception( + "There was an error in the request to send the email." + ) + emailuser_data = json_maxlag(emailuser_response) + if emailuser_data["emailuser"]["result"] != "Success": + raise Exception("There was an error when trying to send the email.") + logger.info("Email sent.") + except: + if not self.fail_silently: + raise + return False + return True diff --git a/TWLight/emails/tasks.py b/TWLight/emails/tasks.py index 7e283e95c4..2fc7219ffb 100644 --- a/TWLight/emails/tasks.py +++ b/TWLight/emails/tasks.py @@ -23,6 +23,7 @@ """ from djmail import template_mail +from djmail.models import Message from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail import logging import os @@ -31,6 +32,7 @@ from django_comments.models import Comment from django_comments.signals import comment_was_posted from django.contrib.sites.shortcuts import get_current_site +from django.core.mail import get_connection from django.urls import reverse_lazy from django.db.models.signals import pre_save from django.dispatch import receiver @@ -40,8 +42,7 @@ from TWLight.applications.signals import Reminder from TWLight.resources.models import AccessCode, Partner from TWLight.users.groups import get_restricted -from TWLight.users.signals import Notice, Survey, UserLoginRetrieval - +from TWLight.users.signals import Notice, Survey, TestEmail, UserLoginRetrieval logger = logging.getLogger(__name__) @@ -74,6 +75,10 @@ class SurveyActiveUser(template_mail.TemplateMail): name = "survey_active_user" +class Test(template_mail.TemplateMail): + name = "test" + + class CoordinatorReminderNotification(template_mail.TemplateMail): name = "coordinator_reminder_notification" @@ -203,6 +208,17 @@ def send_survey_active_user_emails(sender, **kwargs): ) +@receiver(TestEmail.test) +def send_test(sender, **kwargs): + email = kwargs["email"] + connection = get_connection( + backend="TWLight.emails.backends.mediawiki.EmailBackend" + ) + template_email = Test() + email = template_email.make_email_object(email, {}, connection=connection) + email.send() + + @receiver(comment_was_posted) def send_comment_notification_emails(sender, **kwargs): """ diff --git a/TWLight/emails/templates/emails/test-body-html.html b/TWLight/emails/templates/emails/test-body-html.html new file mode 100644 index 0000000000..27da8f6c1a --- /dev/null +++ b/TWLight/emails/templates/emails/test-body-html.html @@ -0,0 +1 @@ +

Please disregard this message; this is only a test

diff --git a/TWLight/emails/templates/emails/test-body-text.html b/TWLight/emails/templates/emails/test-body-text.html new file mode 100644 index 0000000000..c76a832c15 --- /dev/null +++ b/TWLight/emails/templates/emails/test-body-text.html @@ -0,0 +1 @@ +Please disregard this message; this is only a test diff --git a/TWLight/emails/templates/emails/test-subject.html b/TWLight/emails/templates/emails/test-subject.html new file mode 100644 index 0000000000..853a60e1c9 --- /dev/null +++ b/TWLight/emails/templates/emails/test-subject.html @@ -0,0 +1 @@ +test message; please disregard diff --git a/TWLight/settings/base.py b/TWLight/settings/base.py index ad5437b4f6..a353f7b3f6 100644 --- a/TWLight/settings/base.py +++ b/TWLight/settings/base.py @@ -419,6 +419,13 @@ def show_toolbar(request): # ------------------------------------------------------------------------------ TWLIGHT_API_PROVIDER_ENDPOINT = os.environ.get("TWLIGHT_API_PROVIDER_ENDPOINT", None) +MW_API_URL = os.environ.get("MW_API_URL", None) +MW_API_REQUEST_TIMEOUT = os.environ.get("MW_API_REQUEST_TIMEOUT", 60) +MW_API_REQUEST_DELAY = os.environ.get("MW_API_REQUEST_DELAY", 0) +MW_API_REQUEST_RETRY_DELAY = os.environ.get("MW_API_REQUEST_RETRY_DELAY", 5) +MW_API_MAXLAG = os.environ.get("MW_API_MAXLAG", 5) +MW_API_EMAIL_USER = os.environ.get("MW_API_EMAIL_USER", None) +MW_API_EMAIL_PASSWORD = os.environ.get("MW_API_EMAIL_PASSWORD", None) # COMMENTS CONFIGURATION # ------------------------------------------------------------------------------ diff --git a/TWLight/users/management/commands/test_email.py b/TWLight/users/management/commands/test_email.py new file mode 100644 index 0000000000..552a05bd07 --- /dev/null +++ b/TWLight/users/management/commands/test_email.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand + +from TWLight.users.signals import TestEmail + + +class Command(BaseCommand): + help = "Sends testmail to a wikipedia editor." + + def add_arguments(self, parser): + parser.add_argument( + "wp_username", + type=str, + help="The wikipedia editor to send the test email to", + ) + + def handle(self, *args, **options): + user = User.objects.select_related("editor").get( + editor__wp_username=options["wp_username"] + ) + TestEmail.test.send( + sender=self.__class__, + wp_username=user.editor.wp_username, + email=user.email, + ) diff --git a/TWLight/users/signals.py b/TWLight/users/signals.py index ab346d8104..477b554b2c 100644 --- a/TWLight/users/signals.py +++ b/TWLight/users/signals.py @@ -29,6 +29,10 @@ class Survey(object): survey_active_user = Signal() +class TestEmail(object): + test = Signal() + + class UserLoginRetrieval(object): user_retrieve_monthly_logins = Signal() diff --git a/conf/local.twlight.env b/conf/local.twlight.env index d61d8e4be3..3de1c323c3 100644 --- a/conf/local.twlight.env +++ b/conf/local.twlight.env @@ -19,5 +19,6 @@ DEBUG=True TWLIGHT_OAUTH_PROVIDER_URL=https://meta.wikimedia.org/w/index.php TWLIGHT_API_PROVIDER_ENDPOINT=https://meta.wikimedia.org/w/api.php TWLIGHT_EZPROXY_URL=https://ezproxy.dev.localdomain:2443 +MW_API_URL=http://host.docker.internal:8080/w/api.php # seeem to be having troubles with --workers > 1 GUNICORN_CMD_ARGS=--name twlight --workers 1 --backlog 2048 --timeout 300 --bind=0.0.0.0:80 --forwarded-allow-ips * --reload --log-level=info diff --git a/docker-compose.override.yml b/docker-compose.override.yml index c09de97e7d..a020ba52be 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -25,9 +25,14 @@ secrets: services: twlight: + extra_hosts: + - "host.docker.internal:host-gateway" image: quay.io/wikipedialibrary/twlight:local env_file: - - ./conf/local.twlight.env + - path: ./conf/local.twlight.env + required: true + - path: .env + required: false # Local environment should mount things from the code directory volumes: - type: bind From aecdf56325a17f478c22a91fde6f0af27ab84bd3 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Dec 2025 11:20:33 -0600 Subject: [PATCH 02/13] users: add survey email to admin as non-editable Bug: T409420 --- TWLight/users/admin.py | 10 ++++++++-- TWLight/users/models.py | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/TWLight/users/admin.py b/TWLight/users/admin.py index 7a41481e50..4ca3c5864f 100644 --- a/TWLight/users/admin.py +++ b/TWLight/users/admin.py @@ -39,7 +39,7 @@ class UserProfileInline(admin.StackedInline): extra = 1 can_delete = False raw_id_fields = ("user",) - readonly_fields = ("my_library_cache_key",) + readonly_fields = ("my_library_cache_key", "survey_email_sent") fieldsets = ( ( None, @@ -50,6 +50,7 @@ class UserProfileInline(admin.StackedInline): "use_wp_email", "lang", "my_library_cache_key", + "survey_email_sent", "favorites", ) }, @@ -125,7 +126,12 @@ class AuthorizationInline(admin.StackedInline): class UserAdmin(AuthUserAdmin): inlines = [EditorInline, UserProfileInline, AuthorizationInline] list_display = ["username", "get_wp_username", "email", "is_staff"] - list_filter = ["is_staff", "is_active", "is_superuser"] + list_filter = [ + "is_staff", + "is_active", + "is_superuser", + "userprofile__survey_email_sent", + ] default_filters = ["is_active__exact=1"] search_fields = ["editor__wp_username", "username", "email"] diff --git a/TWLight/users/models.py b/TWLight/users/models.py index 93bfb7e0c0..4930524e73 100644 --- a/TWLight/users/models.py +++ b/TWLight/users/models.py @@ -196,7 +196,9 @@ class Meta: help_text="The partner(s) that the user has marked as favorite.", ) survey_email_sent = models.BooleanField( - default=False, help_text="Has this user recieved the most recent survey email?" + default=False, + editable=False, + help_text="Has this user recieved the most recent survey email?", ) def delete_my_library_cache(self): From f228e59dd953bc5c63f77afb44a0441a8b7efff7 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Dec 2025 12:09:37 -0600 Subject: [PATCH 03/13] user survey email cmd: exclude blank email Bug: T409420 --- TWLight/users/management/commands/survey_active_users.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TWLight/users/management/commands/survey_active_users.py b/TWLight/users/management/commands/survey_active_users.py index f4f9fa3258..dde834205f 100644 --- a/TWLight/users/management/commands/survey_active_users.py +++ b/TWLight/users/management/commands/survey_active_users.py @@ -54,7 +54,9 @@ def handle(self, *args, **options): # meet the block criterion or have the 'ignore wp blocks' exemption Q(editor__wp_not_blocked=True) | Q(editor__ignore_wp_blocks=True), # have an non-wikimedia.org email address - Q(email__isnull=False) & ~Q(email__endswith="@wikimedia.org"), + Q(email__isnull=False) + & ~Q(email="") + & ~Q(email__endswith="@wikimedia.org"), # have not already received the email userprofile__survey_email_sent=False, # meet the 6 month criterion as of last login From b874814fdc16dd970f046d3e8729b6c28245ab8c Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Dec 2025 12:18:35 -0600 Subject: [PATCH 04/13] emails: custom model manager + cleanup_email cmd Co-authored-by: Katy - add model manager with helper methods to djmail Message model - cleanup_email cmd deletes draft emails with subject and optionally clears user flag - requires email subject - optional argument for userprofile flag field to clear sent status Bug: T409420 Bug: T412427 --- TWLight/emails/backends/mediawiki.py | 3 +- .../emails/management/commands/__init__.py | 0 .../management/commands/cleanup_email.py | 29 ++++++++++++ TWLight/emails/models.py | 46 +++++++++++++++++++ TWLight/emails/tasks.py | 5 +- 5 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 TWLight/emails/management/commands/__init__.py create mode 100644 TWLight/emails/management/commands/cleanup_email.py create mode 100644 TWLight/emails/models.py diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py index 535fa1ad34..4939d749a3 100644 --- a/TWLight/emails/backends/mediawiki.py +++ b/TWLight/emails/backends/mediawiki.py @@ -18,7 +18,7 @@ def retry_conn(): - """A decorator that handles connetion retries.""" + """A decorator that handles connection retries.""" retry_delay = 0 retry_on_connection_error = 10 retry_after_conn = 5 @@ -255,7 +255,6 @@ def _send(self, email_message): if not emailable: logger.warning("User not emailable, email skipped.") continue - # POST request to send an email email_params = { "action": "emailuser", diff --git a/TWLight/emails/management/commands/__init__.py b/TWLight/emails/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/TWLight/emails/management/commands/cleanup_email.py b/TWLight/emails/management/commands/cleanup_email.py new file mode 100644 index 0000000000..96451fe4a2 --- /dev/null +++ b/TWLight/emails/management/commands/cleanup_email.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from django.core.management.base import BaseCommand + +from TWLight.emails.models import Message + + +class Command(BaseCommand): + help = "Delete draft emails." + + def add_arguments(self, parser): + parser.add_argument( + "--subject", + type=str, + required=True, + help="Email subject", + ) + parser.add_argument( + "--userprofile_flag_field", + type=str, + help="Optionally unset userprofile boolean that tracks sent status for messages with the specified subject", + ) + + def handle(self, *args, **options): + subject = options["subject"] + userprofile_flag_field = options["userprofile_flag_field"] + + Message.objects.bulk_cleanup_drafts( + subject=subject, userprofile_flag_field=userprofile_flag_field + ) diff --git a/TWLight/emails/models.py b/TWLight/emails/models.py new file mode 100644 index 0000000000..d0ab33c8ee --- /dev/null +++ b/TWLight/emails/models.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +from django.contrib.auth.models import User +from django.db import models + +from djmail.models import Message + + +from TWLight.users.models import UserProfile + + +class MessageManager(models.Manager): + def drafts(self, subject=None): + if subject is not None: + return self.filter(status=Message.STATUS_DRAFT, subject=subject) + return self.filter(status=Message.STATUS_DRAFT) + + def users_with_drafts(self, subject=None): + email_addresses = self.drafts(subject).values_list("to_email", flat=True) + + return User.objects.select_related("userprofile").filter( + email__in=email_addresses + ) + + def userprofiles_with_drafts(self, subject=None): + email_addresses = self.drafts(subject).values_list("to_email", flat=True) + + return UserProfile.objects.select_related("user").filter( + user__email__in=email_addresses + ) + + def bulk_cleanup_drafts(self, subject=None, userprofile_flag_field=None): + if subject is not None and userprofile_flag_field is not None: + userprofiles = self.userprofiles_with_drafts(subject=subject).filter( + **{userprofile_flag_field: True} + ) + for userprofile in userprofiles: + setattr(userprofile, userprofile_flag_field, False) + UserProfile.objects.bulk_update( + userprofiles, [userprofile_flag_field], batch_size=1000 + ) + + if subject is not None: + self.drafts(subject=subject).delete() + + +Message.add_to_class("objects", MessageManager()) diff --git a/TWLight/emails/tasks.py b/TWLight/emails/tasks.py index 2fc7219ffb..6392cb638b 100644 --- a/TWLight/emails/tasks.py +++ b/TWLight/emails/tasks.py @@ -23,7 +23,6 @@ """ from djmail import template_mail -from djmail.models import Message from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail import logging import os @@ -210,12 +209,12 @@ def send_survey_active_user_emails(sender, **kwargs): @receiver(TestEmail.test) def send_test(sender, **kwargs): - email = kwargs["email"] + user_email = kwargs["email"] connection = get_connection( backend="TWLight.emails.backends.mediawiki.EmailBackend" ) template_email = Test() - email = template_email.make_email_object(email, {}, connection=connection) + email = template_email.make_email_object(user_email, {}, connection=connection) email.send() From d233bed0dc51778f5504dda24ae349a365aebb18 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Dec 2025 13:02:43 -0600 Subject: [PATCH 05/13] user survey email cmd: use mw email backend - limit emails sent per run to 1000 by default - naively creates one session + login per email email mediawiki backend: - skip accounts that share an email address Bug: T409420 Bug: T412427 --- TWLight/emails/backends/mediawiki.py | 12 ++++++++++- TWLight/emails/tasks.py | 11 ++++++++-- .../commands/survey_active_users.py | 21 +++++++++++++------ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py index 4939d749a3..631514b0fa 100644 --- a/TWLight/emails/backends/mediawiki.py +++ b/TWLight/emails/backends/mediawiki.py @@ -228,9 +228,19 @@ def _send(self, email_message): try: for recipient in email_message.recipients(): # lookup the target editor from the email address - target = Editor.objects.values_list("wp_username", flat=True).get( + target_qs = Editor.objects.values_list("wp_username", flat=True).filter( user__email=recipient ) + target_qs_count = target_qs.count() + if target_qs_count > 1: + logger.warning( + "Email address associated with {} user accounts, email skipped".format( + target_qs_count + ) + ) + continue + + target = target_qs.first() # GET request to check if user is emailable emailable_params = { diff --git a/TWLight/emails/tasks.py b/TWLight/emails/tasks.py index 6392cb638b..d75cdc8de4 100644 --- a/TWLight/emails/tasks.py +++ b/TWLight/emails/tasks.py @@ -196,16 +196,23 @@ def send_survey_active_user_emails(sender, **kwargs): base=base_url, id=survey_id, lang=survey_lang ) - email = SurveyActiveUser() + template_email = SurveyActiveUser() - email.send( + connection = get_connection( + backend="TWLight.emails.backends.mediawiki.EmailBackend" + ) + + email = template_email.make_email_object( user_email, { "lang": user_lang, "link": link, }, + connection=connection, ) + email.send() + @receiver(TestEmail.test) def send_test(sender, **kwargs): diff --git a/TWLight/users/management/commands/survey_active_users.py b/TWLight/users/management/commands/survey_active_users.py index dde834205f..c5bf7b824d 100644 --- a/TWLight/users/management/commands/survey_active_users.py +++ b/TWLight/users/management/commands/survey_active_users.py @@ -13,11 +13,6 @@ class Command(BaseCommand): help = "Sends survey invitation email to active users." def add_arguments(self, parser): - parser.add_argument( - "--staff_test", - action="store_true", - help="A flag to email only to staff users who qualify other than staff status", - ) parser.add_argument( "survey_id", type=int, help="ID number for corresponding survey" ) @@ -27,11 +22,25 @@ def add_arguments(self, parser): type=str, help="List of localized language codes for this survey", ) + parser.add_argument( + "--staff_test", + action="store_true", + required=False, + help="A flag to email only to staff users who qualify other than staff status", + ) + parser.add_argument( + "--batch_size", + type=int, + required=False, + help="number of emails to send; default is 1000", + ) def handle(self, *args, **options): # default mode excludes users who are staff or superusers role_filter = Q(is_staff=False) & Q(is_superuser=False) + batch_size = options["batch_size"] if options["batch_size"] else 1000 + # test mode excludes users who are not staff and ignores superuser status if options["staff_test"]: role_filter = Q(is_staff=True) @@ -67,7 +76,7 @@ def handle(self, *args, **options): is_active=True, ) .order_by("last_login") - ): + )[:batch_size]: # Send the email Survey.survey_active_user.send( sender=self.__class__, From 2d9e2bcb3c7f036b5d8add4e52d9104d7cb31293 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Dec 2025 13:56:16 -0600 Subject: [PATCH 06/13] user survey email cmd: 1 session per batch Bug: T409420 Bug: T412427 --- TWLight/emails/backends/mediawiki.py | 13 +++--- TWLight/emails/tasks.py | 14 ++++--- .../commands/survey_active_users.py | 40 ++++++++++++++----- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py index 631514b0fa..dd9bd2f34d 100644 --- a/TWLight/emails/backends/mediawiki.py +++ b/TWLight/emails/backends/mediawiki.py @@ -51,7 +51,7 @@ def conn(*args, **kwargs): return wrapper -def json_maxlag(response): +def _json_maxlag(response): """A helper method that handles maxlag retries.""" data = response.json() try: @@ -138,14 +138,14 @@ def open(self): "maxlag": self.maxlag, "format": "json", } - logger.info("Getting login token...") session = Session() + logger.info("Session created, getting login token...") response_login_token = session.get(url=self.url, params=login_token_params) if response_login_token.status_code != 200: raise Exception( "There was an error in the request for obtaining the login token." ) - login_token_data = json_maxlag(response_login_token) + login_token_data = _json_maxlag(response_login_token) login_token = login_token_data["query"]["tokens"]["logintoken"] if not login_token: raise Exception("There was an error obtaining the login token.") @@ -177,7 +177,7 @@ def open(self): "There was an error in the request for the email token." ) - email_token_data = json_maxlag(email_token_response) + email_token_data = _json_maxlag(email_token_response) email_token = email_token_data["query"]["tokens"]["csrftoken"] if not email_token: @@ -196,6 +196,7 @@ def close(self): """Unset the session.""" self.email_token = None self.session = None + logger.info("Session destroyed.") def send_messages(self, email_messages): """ @@ -260,7 +261,7 @@ def _send(self, email_message): raise Exception( "There was an error in the request to check if the user can receive emails." ) - emailable_data = json_maxlag(emailable_response) + emailable_data = _json_maxlag(emailable_response) emailable = "emailable" in emailable_data["query"]["users"][0] if not emailable: logger.warning("User not emailable, email skipped.") @@ -282,7 +283,7 @@ def _send(self, email_message): raise Exception( "There was an error in the request to send the email." ) - emailuser_data = json_maxlag(emailuser_response) + emailuser_data = _json_maxlag(emailuser_response) if emailuser_data["emailuser"]["result"] != "Success": raise Exception("There was an error when trying to send the email.") logger.info("Email sent.") diff --git a/TWLight/emails/tasks.py b/TWLight/emails/tasks.py index d75cdc8de4..f4cfa32017 100644 --- a/TWLight/emails/tasks.py +++ b/TWLight/emails/tasks.py @@ -174,11 +174,17 @@ def send_user_renewal_notice_emails(sender, **kwargs): @receiver(Survey.survey_active_user) -def send_survey_active_user_emails(sender, **kwargs): +def construct_survey_active_user_email(sender, **kwargs): """ Any time the related managment command is run, this sends a survey invitation to qualifying editors. """ + connection = ( + kwargs["connection"] + if "connection" in kwargs + else get_connection(backend="TWLight.emails.backends.mediawiki.EmailBackend") + ) + user_email = kwargs["user_email"] user_lang = kwargs["user_lang"] survey_id = kwargs["survey_id"] @@ -198,10 +204,6 @@ def send_survey_active_user_emails(sender, **kwargs): template_email = SurveyActiveUser() - connection = get_connection( - backend="TWLight.emails.backends.mediawiki.EmailBackend" - ) - email = template_email.make_email_object( user_email, { @@ -211,7 +213,7 @@ def send_survey_active_user_emails(sender, **kwargs): connection=connection, ) - email.send() + return email @receiver(TestEmail.test) diff --git a/TWLight/users/management/commands/survey_active_users.py b/TWLight/users/management/commands/survey_active_users.py index c5bf7b824d..f97b008cd1 100644 --- a/TWLight/users/management/commands/survey_active_users.py +++ b/TWLight/users/management/commands/survey_active_users.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from django.contrib.auth.models import User +from django.core.mail import get_connection from django.core.management.base import BaseCommand from django.db.models import DurationField, ExpressionWrapper, F, Q from django.db.models.functions import TruncDate @@ -46,7 +47,7 @@ def handle(self, *args, **options): role_filter = Q(is_staff=True) # All Wikipedia Library users who: - for user in ( + users = ( User.objects.select_related("editor", "userprofile") .annotate( # calculate account age at last login @@ -75,17 +76,38 @@ def handle(self, *args, **options): # are 'active' is_active=True, ) - .order_by("last_login") - )[:batch_size]: - # Send the email - Survey.survey_active_user.send( + .order_by("last_login")[:batch_size] + ) + + # No users qualify; exit + if not users.exists(): + return + + connection = get_connection( + backend="TWLight.emails.backends.mediawiki.EmailBackend" + ) + + email_messages = [] + + for user in users: + # Construct the email; getting a return value from a signal reciever is a quick hack + email_message = Survey.survey_active_user.send( sender=self.__class__, + connection=connection, # passing in the connection is what lets us handle these in bulk user_email=user.email, user_lang=user.userprofile.lang, survey_id=options["survey_id"], survey_langs=options["lang"], - ) + )[0][1] + # add it to the list + email_messages.append(email_message) + + # send the emails + connection.open() + for email in email_messages: + email.send() + connection.close() - # Record that we sent the email so that we only send one. - user.userprofile.survey_email_sent = True - user.userprofile.save() + # Record that we sent the email so that we only send one. + # user.userprofile.survey_email_sent = True + # user.userprofile.save() From f351494bcc294e2abb150784d99500cfc33c595f Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Dec 2025 14:46:49 -0600 Subject: [PATCH 07/13] user survey email cmd: track emails in djmail email tasks: create and save djmail message objects mediawiki backend: raise exceptions when mail can't be sent so status is tracked in djmail objects survey_active_users: clean up email sending Bug: T409420 Bug: T412427 --- TWLight/emails/backends/mediawiki.py | 34 ++++++++----------- TWLight/emails/tasks.py | 22 +++++++++--- .../commands/survey_active_users.py | 15 +++----- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py index dd9bd2f34d..84c1c331af 100644 --- a/TWLight/emails/backends/mediawiki.py +++ b/TWLight/emails/backends/mediawiki.py @@ -6,7 +6,6 @@ from requests import Session from requests.exceptions import ConnectionError from requests.structures import CaseInsensitiveDict -from threading import RLock from time import sleep from django.conf import settings @@ -116,7 +115,6 @@ def __init__( self.password = settings.MW_API_EMAIL_PASSWORD if password is None else password self.email_token = None self.session = None - self._lock = RLock() logger.info("Email connection constructed.") def open(self): @@ -205,19 +203,18 @@ def send_messages(self, email_messages): """ if not email_messages: return 0 - with self._lock: - new_session_created = self.open() - if not self.session or new_session_created is None: - # We failed silently on open(). - # Trying to send would be pointless. - return 0 - num_sent = 0 - for message in email_messages: - sent = self._send(message) - if sent: - num_sent += 1 - if new_session_created: - self.close() + new_session_created = self.open() + if not self.session or new_session_created is None: + # We failed silently on open(). + # Trying to send would be pointless. + return 0 + num_sent = 0 + for message in email_messages: + sent = self._send(message) + if sent: + num_sent += 1 + if new_session_created: + self.close() return num_sent @retry_conn() @@ -234,12 +231,11 @@ def _send(self, email_message): ) target_qs_count = target_qs.count() if target_qs_count > 1: - logger.warning( + raise Exception( "Email address associated with {} user accounts, email skipped".format( target_qs_count ) ) - continue target = target_qs.first() @@ -264,8 +260,8 @@ def _send(self, email_message): emailable_data = _json_maxlag(emailable_response) emailable = "emailable" in emailable_data["query"]["users"][0] if not emailable: - logger.warning("User not emailable, email skipped.") - continue + raise Exception("User not emailable, email skipped.") + # POST request to send an email email_params = { "action": "emailuser", diff --git a/TWLight/emails/tasks.py b/TWLight/emails/tasks.py index f4cfa32017..ff487d8ad7 100644 --- a/TWLight/emails/tasks.py +++ b/TWLight/emails/tasks.py @@ -23,9 +23,11 @@ """ from djmail import template_mail +from djmail.core import _safe_send_message from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail import logging import os +from uuid import uuid4 from reversion.models import Version from django_comments.models import Comment @@ -33,6 +35,7 @@ from django.contrib.sites.shortcuts import get_current_site from django.core.mail import get_connection from django.urls import reverse_lazy +from django.utils import timezone from django.db.models.signals import pre_save from django.dispatch import receiver from django.shortcuts import get_object_or_404 @@ -42,6 +45,7 @@ from TWLight.resources.models import AccessCode, Partner from TWLight.users.groups import get_restricted from TWLight.users.signals import Notice, Survey, TestEmail, UserLoginRetrieval +from .models import Message logger = logging.getLogger(__name__) @@ -174,7 +178,7 @@ def send_user_renewal_notice_emails(sender, **kwargs): @receiver(Survey.survey_active_user) -def construct_survey_active_user_email(sender, **kwargs): +def send_survey_active_user_email(sender, **kwargs): """ Any time the related managment command is run, this sends a survey invitation to qualifying editors. @@ -204,7 +208,7 @@ def construct_survey_active_user_email(sender, **kwargs): template_email = SurveyActiveUser() - email = template_email.make_email_object( + email_message = template_email.make_email_object( user_email, { "lang": user_lang, @@ -212,8 +216,18 @@ def construct_survey_active_user_email(sender, **kwargs): }, connection=connection, ) - - return email + # Save as a djmail instance + model_instance = Message.from_email_message(email_message) + if hasattr(email_message, "priority"): + if email_message.priority <= Message.PRIORITY_LOW: + model_instance.priority = email_message.priority + model_instance.status = Message.STATUS_PENDING + model_instance.created_at = timezone.now() + model_instance.uuid = uuid4() + model_instance.save() + + # send email and update state in djmail + _safe_send_message(model_instance, connection) @receiver(TestEmail.test) diff --git a/TWLight/users/management/commands/survey_active_users.py b/TWLight/users/management/commands/survey_active_users.py index f97b008cd1..0f4393a7ee 100644 --- a/TWLight/users/management/commands/survey_active_users.py +++ b/TWLight/users/management/commands/survey_active_users.py @@ -86,26 +86,19 @@ def handle(self, *args, **options): connection = get_connection( backend="TWLight.emails.backends.mediawiki.EmailBackend" ) + connection.open() - email_messages = [] - + # send the emails for user in users: - # Construct the email; getting a return value from a signal reciever is a quick hack - email_message = Survey.survey_active_user.send( + Survey.survey_active_user.send( sender=self.__class__, connection=connection, # passing in the connection is what lets us handle these in bulk user_email=user.email, user_lang=user.userprofile.lang, survey_id=options["survey_id"], survey_langs=options["lang"], - )[0][1] - # add it to the list - email_messages.append(email_message) + ) - # send the emails - connection.open() - for email in email_messages: - email.send() connection.close() # Record that we sent the email so that we only send one. From 371f24a8903c8a6092af924f12b01ac58b26b5b9 Mon Sep 17 00:00:00 2001 From: Kgraessle Date: Wed, 7 Jan 2026 07:28:54 -0600 Subject: [PATCH 08/13] emails: swap survey backend to default for tests Co-authored-by: jsnshrmn - Also fix send_survey_active_user_email import Bug: T409420 --- TWLight/emails/tasks.py | 9 +++++++-- TWLight/emails/tests.py | 12 ++++++----- .../commands/survey_active_users.py | 20 ++++++++++++++----- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/TWLight/emails/tasks.py b/TWLight/emails/tasks.py index ff487d8ad7..a508b70649 100644 --- a/TWLight/emails/tasks.py +++ b/TWLight/emails/tasks.py @@ -45,7 +45,7 @@ from TWLight.resources.models import AccessCode, Partner from TWLight.users.groups import get_restricted from TWLight.users.signals import Notice, Survey, TestEmail, UserLoginRetrieval -from .models import Message +from djmail.models import Message logger = logging.getLogger(__name__) @@ -183,10 +183,15 @@ def send_survey_active_user_email(sender, **kwargs): Any time the related managment command is run, this sends a survey invitation to qualifying editors. """ + backend = ( + kwargs["backend"] + if "backend" in kwargs + else "TWLight.emails.backends.mediawiki.EmailBackend" + ) connection = ( kwargs["connection"] if "connection" in kwargs - else get_connection(backend="TWLight.emails.backends.mediawiki.EmailBackend") + else get_connection(backend=backend) ) user_email = kwargs["user_email"] diff --git a/TWLight/emails/tests.py b/TWLight/emails/tests.py index b4ef0cdd21..9ccb1d6b3e 100644 --- a/TWLight/emails/tests.py +++ b/TWLight/emails/tests.py @@ -14,6 +14,7 @@ from django.urls import reverse from django.utils import timezone from django.test import TestCase, RequestFactory +from django.test.utils import override_settings from TWLight.applications.factories import ( ApplicationFactory, @@ -34,7 +35,7 @@ send_approval_notification_email, send_rejection_notification_email, send_user_renewal_notice_emails, - send_survey_active_user_emails, + send_survey_active_user_email, ) @@ -841,15 +842,16 @@ def setUpTestData(cls): superuser.user.save() superuser.save() + # Use the same override as djmail itself since the command dynamically changes the backend + @override_settings( + DJMAIL_REAL_BACKEND="django.core.mail.backends.locmem.EmailBackend" + ) def test_survey_active_users_command(self): - self.assertFalse(self.eligible.userprofile.survey_email_sent) call_command( "survey_active_users", "000001", "en", + backend="djmail.backends.default.EmailBackend", ) - - self.eligible.refresh_from_db() - self.assertTrue(self.eligible.userprofile.survey_email_sent) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].to, [self.eligible.email]) diff --git a/TWLight/users/management/commands/survey_active_users.py b/TWLight/users/management/commands/survey_active_users.py index 0f4393a7ee..e15de9cd56 100644 --- a/TWLight/users/management/commands/survey_active_users.py +++ b/TWLight/users/management/commands/survey_active_users.py @@ -27,7 +27,7 @@ def add_arguments(self, parser): "--staff_test", action="store_true", required=False, - help="A flag to email only to staff users who qualify other than staff status", + help="Email only staff users who qualify other than staff status", ) parser.add_argument( "--batch_size", @@ -35,12 +35,23 @@ def add_arguments(self, parser): required=False, help="number of emails to send; default is 1000", ) + parser.add_argument( + "--backend", + type=str, + required=False, + help="djmail backend to use; default is TWLight.emails.backends.mediawiki.EmailBackend", + ) def handle(self, *args, **options): # default mode excludes users who are staff or superusers role_filter = Q(is_staff=False) & Q(is_superuser=False) batch_size = options["batch_size"] if options["batch_size"] else 1000 + backend = ( + options["backend"] + if options["backend"] + else "TWLight.emails.backends.mediawiki.EmailBackend" + ) # test mode excludes users who are not staff and ignores superuser status if options["staff_test"]: @@ -83,16 +94,15 @@ def handle(self, *args, **options): if not users.exists(): return - connection = get_connection( - backend="TWLight.emails.backends.mediawiki.EmailBackend" - ) + connection = get_connection(backend=backend) connection.open() # send the emails for user in users: Survey.survey_active_user.send( sender=self.__class__, - connection=connection, # passing in the connection is what lets us handle these in bulk + backend=backend, # allows setting the djmail backend back to default for testing + connection=connection, # passing in the connection lets us handle these in bulk user_email=user.email, user_lang=user.userprofile.lang, survey_id=options["survey_id"], From 861f126f62c5e77be989ba5faa035fc6711e8b67 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Thu, 8 Jan 2026 16:24:24 -0600 Subject: [PATCH 09/13] user survey email cmd bugfixes - Validate language codes in command - Remove signal handling and clean up email sending code - Fix issues in custom model manger - Fix duplicate email check: - stop using survey_email_sent userprofile field - actually dropping the field should be handled later - Now searches for existing messages based on translated subject and recipient - Skips sending if message (of any status) exists - Outputs some basic statistics - Updated tests to reflect the new approach - better naming for maxlag handling method in backend Bug: T409420 --- TWLight/emails/backends/mediawiki.py | 10 +-- .../management/commands/cleanup_email.py | 17 ++-- TWLight/emails/models.py | 78 ++++++++++++------- TWLight/emails/tasks.py | 21 +---- TWLight/emails/tests.py | 19 +++-- TWLight/users/admin.py | 4 +- .../commands/survey_active_users.py | 76 ++++++++++++------ TWLight/users/models.py | 5 +- TWLight/users/signals.py | 4 - 9 files changed, 137 insertions(+), 97 deletions(-) diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py index 84c1c331af..5d46a83374 100644 --- a/TWLight/emails/backends/mediawiki.py +++ b/TWLight/emails/backends/mediawiki.py @@ -50,7 +50,7 @@ def conn(*args, **kwargs): return wrapper -def _json_maxlag(response): +def _handle_maxlag(response): """A helper method that handles maxlag retries.""" data = response.json() try: @@ -143,7 +143,7 @@ def open(self): raise Exception( "There was an error in the request for obtaining the login token." ) - login_token_data = _json_maxlag(response_login_token) + login_token_data = _handle_maxlag(response_login_token) login_token = login_token_data["query"]["tokens"]["logintoken"] if not login_token: raise Exception("There was an error obtaining the login token.") @@ -175,7 +175,7 @@ def open(self): "There was an error in the request for the email token." ) - email_token_data = _json_maxlag(email_token_response) + email_token_data = _handle_maxlag(email_token_response) email_token = email_token_data["query"]["tokens"]["csrftoken"] if not email_token: @@ -257,7 +257,7 @@ def _send(self, email_message): raise Exception( "There was an error in the request to check if the user can receive emails." ) - emailable_data = _json_maxlag(emailable_response) + emailable_data = _handle_maxlag(emailable_response) emailable = "emailable" in emailable_data["query"]["users"][0] if not emailable: raise Exception("User not emailable, email skipped.") @@ -279,7 +279,7 @@ def _send(self, email_message): raise Exception( "There was an error in the request to send the email." ) - emailuser_data = _json_maxlag(emailuser_response) + emailuser_data = _handle_maxlag(emailuser_response) if emailuser_data["emailuser"]["result"] != "Success": raise Exception("There was an error when trying to send the email.") logger.info("Email sent.") diff --git a/TWLight/emails/management/commands/cleanup_email.py b/TWLight/emails/management/commands/cleanup_email.py index 96451fe4a2..bed748b4c7 100644 --- a/TWLight/emails/management/commands/cleanup_email.py +++ b/TWLight/emails/management/commands/cleanup_email.py @@ -5,25 +5,20 @@ class Command(BaseCommand): - help = "Delete draft emails." + help = "Delete unsent emails." def add_arguments(self, parser): parser.add_argument( "--subject", type=str, - required=True, + required=False, help="Email subject", ) - parser.add_argument( - "--userprofile_flag_field", - type=str, - help="Optionally unset userprofile boolean that tracks sent status for messages with the specified subject", - ) def handle(self, *args, **options): subject = options["subject"] - userprofile_flag_field = options["userprofile_flag_field"] - Message.objects.bulk_cleanup_drafts( - subject=subject, userprofile_flag_field=userprofile_flag_field - ) + if subject is None: + Message.twl.unsent().delete() + else: + Message.twl.filter(subject=subject).unsent().delete() diff --git a/TWLight/emails/models.py b/TWLight/emails/models.py index d0ab33c8ee..c2f0db336f 100644 --- a/TWLight/emails/models.py +++ b/TWLight/emails/models.py @@ -1,46 +1,70 @@ # -*- coding: utf-8 -*- +from django.conf import settings from django.contrib.auth.models import User from django.db import models +from django.utils.translation import gettext as _, override from djmail.models import Message - from TWLight.users.models import UserProfile -class MessageManager(models.Manager): - def drafts(self, subject=None): - if subject is not None: - return self.filter(status=Message.STATUS_DRAFT, subject=subject) - return self.filter(status=Message.STATUS_DRAFT) - - def users_with_drafts(self, subject=None): - email_addresses = self.drafts(subject).values_list("to_email", flat=True) +class MessageQuerySet(models.QuerySet): + def unsent(self): + return self.exclude(status=Message.STATUS_SENT) - return User.objects.select_related("userprofile").filter( - email__in=email_addresses - ) - - def userprofiles_with_drafts(self, subject=None): - email_addresses = self.drafts(subject).values_list("to_email", flat=True) + def users_with_unsent(self): + email_addresses = self.unsent().values_list("to_email", flat=True) + return User.objects.filter(email__in=email_addresses) + def userprofiles_with_unsent(self): + email_addresses = self.unsent().values_list("to_email", flat=True) return UserProfile.objects.select_related("user").filter( user__email__in=email_addresses ) - def bulk_cleanup_drafts(self, subject=None, userprofile_flag_field=None): - if subject is not None and userprofile_flag_field is not None: - userprofiles = self.userprofiles_with_drafts(subject=subject).filter( - **{userprofile_flag_field: True} - ) - for userprofile in userprofiles: - setattr(userprofile, userprofile_flag_field, False) - UserProfile.objects.bulk_update( - userprofiles, [userprofile_flag_field], batch_size=1000 + def user_pks_with_subject_list(self, subject, users): + user_pks = [] + if users is None: + return user_pks + + subjects = [] + # Get the localized subject for each specified language + for lang_code, _lang_name in settings.LANGUAGES: + try: + with override(lang_code): + # Translators: do not translate + subjects.append(_(subject)) + except ValueError: + pass + + # Since the message object contains translated text, search for the localized email subject to check for existing messages. + for user in users.only("pk", "email"): + # search for Message objects with matching subject and recipient + existing_messages = self.filter( + to_email=user.email, + subject__in=subjects, ) + if existing_messages.exists(): + user_pks.append(user.pk) + return user_pks + + +class MessageManager(models.Manager): + def get_queryset(self): + return MessageQuerySet(self.model, using=self._db) - if subject is not None: - self.drafts(subject=subject).delete() + def unsent(self): + return self.get_queryset().unsent() + + def userprofiles_with_unsent(self): + return self.get_queryset().userprofiles_with_unsent() + + def user_pks_with_subject_list(self, subject, users): + return self.get_queryset().user_pks_with_subject_list( + subject=subject, users=users + ) -Message.add_to_class("objects", MessageManager()) +# add "twl" manager to Message +Message.add_to_class("twl", MessageManager()) diff --git a/TWLight/emails/tasks.py b/TWLight/emails/tasks.py index a508b70649..babbbd0af8 100644 --- a/TWLight/emails/tasks.py +++ b/TWLight/emails/tasks.py @@ -23,7 +23,6 @@ """ from djmail import template_mail -from djmail.core import _safe_send_message from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail import logging import os @@ -44,7 +43,7 @@ from TWLight.applications.signals import Reminder from TWLight.resources.models import AccessCode, Partner from TWLight.users.groups import get_restricted -from TWLight.users.signals import Notice, Survey, TestEmail, UserLoginRetrieval +from TWLight.users.signals import Notice, TestEmail, UserLoginRetrieval from djmail.models import Message logger = logging.getLogger(__name__) @@ -177,8 +176,7 @@ def send_user_renewal_notice_emails(sender, **kwargs): ) -@receiver(Survey.survey_active_user) -def send_survey_active_user_email(sender, **kwargs): +def send_survey_active_user_email(**kwargs): """ Any time the related managment command is run, this sends a survey invitation to qualifying editors. @@ -213,7 +211,7 @@ def send_survey_active_user_email(sender, **kwargs): template_email = SurveyActiveUser() - email_message = template_email.make_email_object( + email = template_email.make_email_object( user_email, { "lang": user_lang, @@ -221,18 +219,7 @@ def send_survey_active_user_email(sender, **kwargs): }, connection=connection, ) - # Save as a djmail instance - model_instance = Message.from_email_message(email_message) - if hasattr(email_message, "priority"): - if email_message.priority <= Message.PRIORITY_LOW: - model_instance.priority = email_message.priority - model_instance.status = Message.STATUS_PENDING - model_instance.created_at = timezone.now() - model_instance.uuid = uuid4() - model_instance.save() - - # send email and update state in djmail - _safe_send_message(model_instance, connection) + email.send() @receiver(TestEmail.test) diff --git a/TWLight/emails/tests.py b/TWLight/emails/tests.py index 9ccb1d6b3e..a049476ffd 100644 --- a/TWLight/emails/tests.py +++ b/TWLight/emails/tests.py @@ -721,7 +721,7 @@ class SurveyActiveUsersEmailTest(TestCase): @classmethod def setUpTestData(cls): """ - Creates a survey-eligible user and several eligible users. + Creates a survey-eligible user, several ineligible users, and one already sent message. Returns ------- None @@ -761,7 +761,6 @@ def setUpTestData(cls): already_sent.wp_registered = now - timedelta(days=182) already_sent.wp_enough_edits = True already_sent.user.userprofile.terms_of_use = True - already_sent.user.userprofile.survey_email_sent = True already_sent.user.userprofile.save() already_sent.user.last_login = now already_sent.user.save() @@ -844,14 +843,24 @@ def setUpTestData(cls): # Use the same override as djmail itself since the command dynamically changes the backend @override_settings( - DJMAIL_REAL_BACKEND="django.core.mail.backends.locmem.EmailBackend" + EMAIL_BACKEND="djmail.backends.default.EmailBackend", + DJMAIL_REAL_BACKEND="django.core.mail.backends.locmem.EmailBackend", ) def test_survey_active_users_command(self): + # pre-send an email to the "alreadysent" editor + already_sent_msg = mail.EmailMessage( + "The Wikipedia Library needs your help!", + "Body", + "sender@example.com", + ["alreadysent@example.com"], + ) + already_sent_msg.send() + call_command( "survey_active_users", "000001", "en", backend="djmail.backends.default.EmailBackend", ) - self.assertEqual(len(mail.outbox), 1) - self.assertEqual(mail.outbox[0].to, [self.eligible.email]) + self.assertEqual(len(mail.outbox), 2) + self.assertEqual(mail.outbox[1].to, [self.eligible.email]) diff --git a/TWLight/users/admin.py b/TWLight/users/admin.py index 4ca3c5864f..ed8509f293 100644 --- a/TWLight/users/admin.py +++ b/TWLight/users/admin.py @@ -39,7 +39,7 @@ class UserProfileInline(admin.StackedInline): extra = 1 can_delete = False raw_id_fields = ("user",) - readonly_fields = ("my_library_cache_key", "survey_email_sent") + readonly_fields = ("my_library_cache_key",) fieldsets = ( ( None, @@ -50,7 +50,6 @@ class UserProfileInline(admin.StackedInline): "use_wp_email", "lang", "my_library_cache_key", - "survey_email_sent", "favorites", ) }, @@ -130,7 +129,6 @@ class UserAdmin(AuthUserAdmin): "is_staff", "is_active", "is_superuser", - "userprofile__survey_email_sent", ] default_filters = ["is_active__exact=1"] search_fields = ["editor__wp_username", "username", "email"] diff --git a/TWLight/users/management/commands/survey_active_users.py b/TWLight/users/management/commands/survey_active_users.py index e15de9cd56..3d159c482e 100644 --- a/TWLight/users/management/commands/survey_active_users.py +++ b/TWLight/users/management/commands/survey_active_users.py @@ -1,13 +1,20 @@ # -*- coding: utf-8 -*- +import logging + +from django.conf import settings from django.contrib.auth.models import User from django.core.mail import get_connection -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.db.models import DurationField, ExpressionWrapper, F, Q from django.db.models.functions import TruncDate from django.utils.timezone import timedelta +from django.utils.translation import gettext_lazy as _ +from TWLight.emails.models import Message +from TWLight.emails.tasks import send_survey_active_user_email from TWLight.users.groups import get_restricted -from TWLight.users.signals import Survey + +logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -43,6 +50,22 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): + # Validate the lang args + survey_langs = options["lang"] + valid_langs = [] + invalid_langs = [] + for lang_code, _lang_name in settings.LANGUAGES: + valid_langs.append(lang_code) + + for survey_lang in survey_langs: + if survey_lang not in valid_langs: + invalid_langs.append(survey_lang) + + if invalid_langs: + raise CommandError( + "invalid lang argument in list: {}".format(invalid_langs) + ) + # default mode excludes users who are staff or superusers role_filter = Q(is_staff=False) & Q(is_superuser=False) @@ -78,8 +101,6 @@ def handle(self, *args, **options): Q(email__isnull=False) & ~Q(email="") & ~Q(email__endswith="@wikimedia.org"), - # have not already received the email - userprofile__survey_email_sent=False, # meet the 6 month criterion as of last login last_login_age__gte=timedelta(days=182), # meet the 500 edit criterion @@ -87,30 +108,41 @@ def handle(self, *args, **options): # are 'active' is_active=True, ) - .order_by("last_login")[:batch_size] ) + previously_sent_user_pks = Message.twl.user_pks_with_subject_list( + # Translators: email subject line + subject=_("The Wikipedia Library needs your help!"), + users=users, + ) + logger.info( + "{} users previously sent message".format(len(previously_sent_user_pks)) + ) + users = ( + users.exclude(pk__in=previously_sent_user_pks) + .distinct() + .order_by("last_login") + ) + logger.info("{} new users qualify".format(users.count())) + users = users[:batch_size] + logger.info("attempting to send to {} users".format(users.count())) - # No users qualify; exit - if not users.exists(): - return - + # Use a single connection to send all emails connection = get_connection(backend=backend) connection.open() # send the emails for user in users: - Survey.survey_active_user.send( - sender=self.__class__, - backend=backend, # allows setting the djmail backend back to default for testing - connection=connection, # passing in the connection lets us handle these in bulk - user_email=user.email, - user_lang=user.userprofile.lang, - survey_id=options["survey_id"], - survey_langs=options["lang"], - ) + try: + send_survey_active_user_email( + sender=self.__class__, + backend=backend, # allows setting the djmail backend back to default for testing + connection=connection, # passing in the connection lets us handle these in bulk + user_email=user.email, + user_lang=user.userprofile.lang, + survey_id=options["survey_id"], + survey_langs=survey_langs, + ) + except Exception as e: + logger.error(e) connection.close() - - # Record that we sent the email so that we only send one. - # user.userprofile.survey_email_sent = True - # user.userprofile.save() diff --git a/TWLight/users/models.py b/TWLight/users/models.py index 4930524e73..d252f75505 100644 --- a/TWLight/users/models.py +++ b/TWLight/users/models.py @@ -195,10 +195,9 @@ class Meta: blank=True, help_text="The partner(s) that the user has marked as favorite.", ) + # @TODO: drop this field survey_email_sent = models.BooleanField( - default=False, - editable=False, - help_text="Has this user recieved the most recent survey email?", + default=False, help_text="Has this user recieved the most recent survey email?" ) def delete_my_library_cache(self): diff --git a/TWLight/users/signals.py b/TWLight/users/signals.py index 477b554b2c..67d274b0a1 100644 --- a/TWLight/users/signals.py +++ b/TWLight/users/signals.py @@ -25,10 +25,6 @@ class Notice(object): user_renewal_notice = Signal() -class Survey(object): - survey_active_user = Signal() - - class TestEmail(object): test = Signal() From cbeb003d0c7511a5fb822a06bf9ffdc232b57aea Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 9 Jan 2026 17:20:21 -0600 Subject: [PATCH 10/13] conf: add MW_API_URL var for staging & prod Bug: T409420 --- conf/production.twlight.env | 1 + conf/staging.twlight.env | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/production.twlight.env b/conf/production.twlight.env index 26ed9cf526..6882113805 100644 --- a/conf/production.twlight.env +++ b/conf/production.twlight.env @@ -20,4 +20,5 @@ DEBUG=False TWLIGHT_OAUTH_PROVIDER_URL=https://meta.wikimedia.org/w/index.php TWLIGHT_API_PROVIDER_ENDPOINT=https://meta.wikimedia.org/w/api.php TWLIGHT_EZPROXY_URL=https://wikipedialibrary.idm.oclc.org +MW_API_URL=https://meta.wikimedia.org/w/api.php GUNICORN_CMD_ARGS=--name twlight --worker-class gthread --workers 9 --threads 1 --timeout 30 --backlog 2048 --bind=0.0.0.0:80 --forwarded-allow-ips * --reload --log-level=info diff --git a/conf/staging.twlight.env b/conf/staging.twlight.env index ef8cbf693d..20519ee589 100644 --- a/conf/staging.twlight.env +++ b/conf/staging.twlight.env @@ -20,4 +20,5 @@ DEBUG=False TWLIGHT_OAUTH_PROVIDER_URL=https://meta.wikimedia.org/w/index.php TWLIGHT_API_PROVIDER_ENDPOINT=https://meta.wikimedia.org/w/api.php TWLIGHT_EZPROXY_URL=https://wikipedialibrary.idm.oclc.org:9443 +MW_API_URL=https://meta.wikimedia.org/w/api.php GUNICORN_CMD_ARGS=--name twlight --worker-class gthread --workers 4 --threads 1 --timeout 30 --backlog 2048 --bind=0.0.0.0:80 --forwarded-allow-ips * --reload --log-level=info From 910475fcdd612ceca5e3871a20130f745dc38cbc Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Wed, 14 Jan 2026 14:27:15 -0600 Subject: [PATCH 11/13] user survey email cmd improvements - dramatic speedup - more informative log messages during execution Bug: T409420 --- TWLight/emails/models.py | 23 ++++++++----------- .../commands/survey_active_users.py | 19 +++++++++------ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/TWLight/emails/models.py b/TWLight/emails/models.py index c2f0db336f..7a2fab82fa 100644 --- a/TWLight/emails/models.py +++ b/TWLight/emails/models.py @@ -24,12 +24,11 @@ def userprofiles_with_unsent(self): ) def user_pks_with_subject_list(self, subject, users): - user_pks = [] if users is None: - return user_pks + return [] subjects = [] - # Get the localized subject for each specified language + # Get the localized subject for each available language for lang_code, _lang_name in settings.LANGUAGES: try: with override(lang_code): @@ -38,16 +37,14 @@ def user_pks_with_subject_list(self, subject, users): except ValueError: pass - # Since the message object contains translated text, search for the localized email subject to check for existing messages. - for user in users.only("pk", "email"): - # search for Message objects with matching subject and recipient - existing_messages = self.filter( - to_email=user.email, - subject__in=subjects, - ) - if existing_messages.exists(): - user_pks.append(user.pk) - return user_pks + # Search for repients of sent messages with the one of the localized email subjects. + previous_recipients = self.filter( + status=Message.STATUS_SENT, + subject__in=subjects, + ).values_list("to_email", flat=True) + + # return a list of pks for users with matching email addresses + return users.filter(email__in=previous_recipients).values_list("pk", flat=True) class MessageManager(models.Manager): diff --git a/TWLight/users/management/commands/survey_active_users.py b/TWLight/users/management/commands/survey_active_users.py index 3d159c482e..7da9d4dee4 100644 --- a/TWLight/users/management/commands/survey_active_users.py +++ b/TWLight/users/management/commands/survey_active_users.py @@ -95,34 +95,39 @@ def handle(self, *args, **options): role_filter, # have not restricted data processing ~Q(groups__name__in=[get_restricted()]), - # meet the block criterion or have the 'ignore wp blocks' exemption + # meet the block criterion or are exempt Q(editor__wp_not_blocked=True) | Q(editor__ignore_wp_blocks=True), # have an non-wikimedia.org email address Q(email__isnull=False) & ~Q(email="") & ~Q(email__endswith="@wikimedia.org"), - # meet the 6 month criterion as of last login - last_login_age__gte=timedelta(days=182), - # meet the 500 edit criterion - editor__wp_enough_edits=True, + # meet the 6 month criterion as of last login or are exempt + Q(last_login_age__gte=timedelta(days=182)) + | Q(editor__ignore_wp_account_age_requirement=True), + # meet the 500 edit criterion or are exempt + Q(editor__wp_enough_edits=True) + | Q(editor__ignore_wp_edit_requirement=True), # are 'active' is_active=True, ) ) + logger.info("{} users qualify".format(users.count())) previously_sent_user_pks = Message.twl.user_pks_with_subject_list( # Translators: email subject line subject=_("The Wikipedia Library needs your help!"), users=users, ) logger.info( - "{} users previously sent message".format(len(previously_sent_user_pks)) + "{} users previously sent message will be skipped".format( + len(previously_sent_user_pks) + ) ) users = ( users.exclude(pk__in=previously_sent_user_pks) .distinct() .order_by("last_login") ) - logger.info("{} new users qualify".format(users.count())) + logger.info("{} remaining users qualify".format(users.count())) users = users[:batch_size] logger.info("attempting to send to {} users".format(users.count())) From 44b83b56ed3bda7ad1a00e4e3e34b144c5f4f76a Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Wed, 14 Jan 2026 14:40:27 -0600 Subject: [PATCH 12/13] emails.backends.mediawiki: set user-agent Bug: T412427 --- TWLight/emails/backends/mediawiki.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py index 5d46a83374..dda6ad6ede 100644 --- a/TWLight/emails/backends/mediawiki.py +++ b/TWLight/emails/backends/mediawiki.py @@ -137,6 +137,7 @@ def open(self): "format": "json", } session = Session() + session.headers = self.headers logger.info("Session created, getting login token...") response_login_token = session.get(url=self.url, params=login_token_params) if response_login_token.status_code != 200: From 3c1a3ac14927984c76159ad17d558fd7f272fcab Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 16 Jan 2026 11:00:18 -0600 Subject: [PATCH 13/13] emails.backends.mediawiki: improve error handling - correct connection and maxlag retry bugs - bubble up api error messages - include usernames in exceptions for unsendable messages - more DRY Bug: T412427 --- TWLight/emails/backends/mediawiki.py | 172 +++++++++++++-------------- 1 file changed, 85 insertions(+), 87 deletions(-) diff --git a/TWLight/emails/backends/mediawiki.py b/TWLight/emails/backends/mediawiki.py index dda6ad6ede..dd1a358506 100644 --- a/TWLight/emails/backends/mediawiki.py +++ b/TWLight/emails/backends/mediawiki.py @@ -3,6 +3,7 @@ see: https://www.mediawiki.org/wiki/API:Emailuser """ import logging +from json import dumps from requests import Session from requests.exceptions import ConnectionError from requests.structures import CaseInsensitiveDict @@ -40,7 +41,7 @@ def conn(*args, **kwargs): logger.warning("ConnectionError exhausted retries") raise e logger.warning( - "ConnectionError, retrying in {}s".format(self.retry_after_conn) + "ConnectionError, retrying in {}s".format(retry_after_conn) ) sleep(retry_after_conn) continue @@ -50,43 +51,6 @@ def conn(*args, **kwargs): return wrapper -def _handle_maxlag(response): - """A helper method that handles maxlag retries.""" - data = response.json() - try: - if data["error"]["code"] != "maxlag": - return data - except KeyError: - return data - - retry_after = float(response.headers.get("Retry-After", 5)) - retry_on_lag_error = 50 - no_retry = 0 <= retry_on_lag_error < try_count - - message = "Server exceeded maxlag" - if not no_retry: - message += ", retrying in {}s".format(retry_after) - if "lag" in data["error"]: - message += ", lag={}".format(data["error"]["lag"]) - message += ", API=".format(self.url) - - log = logger.warning if no_retry else logger.info - log( - message, - { - "code": "maxlag-retry", - "retry-after": retry_after, - "lag": data["error"]["lag"] if "lag" in data["error"] else None, - "x-database-lag": response.headers.get("X-Database-Lag", 5), - }, - ) - - if no_retry: - raise Exception(message) - - sleep(retry_after) - - class EmailBackend(BaseEmailBackend): def __init__( self, @@ -117,6 +81,56 @@ def __init__( self.session = None logger.info("Email connection constructed.") + def _handle_request(self, response, try_count=0): + """ + A helper method that handles MW API responses + including maxlag retries. + """ + # Raise for any HTTP response errors + if response.status_code != 200: + raise Exception("HTTP {} error".format(response.status_code)) + data = response.json() + error = data.get("error", {}) + if "warnings" in data: + logger.warning(dumps(data["warnings"], indent=True)) + # raise for any api error codes besides max lag + try: + if error.get("code") != "maxlag": + raise Exception(dumps(error)) + except: + # return data if there are no errors + return data + + # handle retries with max lag + lag = error.get("lag") + request = response.request + retry_after = float(response.headers.get("Retry-After", 5)) + retry_on_lag_error = 50 + no_retry = 0 <= retry_on_lag_error < try_count + message = "Server exceeded maxlag" + if not no_retry: + message += ", retrying in {}s".format(retry_after) + if lag: + message += ", lag={}".format(lag) + message += ", url={}".format(self.url) + log = logger.warning if no_retry else logger.info + log( + message, + { + "code": "maxlag-retry", + "retry-after": retry_after, + "lag": lag, + "x-database-lag": response.headers.get("X-Database-Lag", 5), + }, + ) + if no_retry: + raise Exception(message) + + sleep(retry_after) + try_count += 1 + return self._handle_request(self.session.send(request), try_count) + + @retry_conn() def open(self): """ Ensure an open session to the API server. Return whether or not a @@ -128,6 +142,10 @@ def open(self): return False try: + self.session = Session() + self.session.headers = self.headers + logger.info("Session created, getting login token...") + # GET request to fetch login token login_token_params = { "action": "query", @@ -136,18 +154,13 @@ def open(self): "maxlag": self.maxlag, "format": "json", } - session = Session() - session.headers = self.headers - logger.info("Session created, getting login token...") - response_login_token = session.get(url=self.url, params=login_token_params) - if response_login_token.status_code != 200: - raise Exception( - "There was an error in the request for obtaining the login token." - ) - login_token_data = _handle_maxlag(response_login_token) - login_token = login_token_data["query"]["tokens"]["logintoken"] + login_token_response = self._handle_request( + self.session.get(url=self.url, params=login_token_params) + ) + login_token = login_token_response["query"]["tokens"]["logintoken"] if not login_token: - raise Exception("There was an error obtaining the login token.") + self.session = None + raise Exception(dumps(login_token_response)) # POST request to log in. Use of main account for login is not # supported. Obtain credentials via Special:BotPasswords @@ -161,30 +174,25 @@ def open(self): "format": "json", } logger.info("Signing in...") - login_response = session.post(url=self.url, data=login_params) - if login_response.status_code != 200: - raise Exception("There was an error in the request for the login.") + login_response = self._handle_request( + self.session.post(url=self.url, data=login_params) + ) # GET request to fetch Email token # see: https://www.mediawiki.org/wiki/API:Emailuser#Token email_token_params = {"action": "query", "meta": "tokens", "format": "json"} logger.info("Getting email token...") - email_token_response = session.get(url=self.url, params=email_token_params) - if email_token_response.status_code != 200: - raise Exception( - "There was an error in the request for the email token." - ) - - email_token_data = _handle_maxlag(email_token_response) - - email_token = email_token_data["query"]["tokens"]["csrftoken"] + email_token_response = self._handle_request( + self.session.get(url=self.url, params=email_token_params) + ) + email_token = email_token_response["query"]["tokens"]["csrftoken"] if not email_token: - raise Exception("There was an error obtaining the email token.") + self.session = None + raise Exception(dumps(email_token_response)) - # Assign the session and email token + # Assign the email token self.email_token = email_token - self.session = session logger.info("Email API session ready.") return True except: @@ -227,15 +235,13 @@ def _send(self, email_message): try: for recipient in email_message.recipients(): # lookup the target editor from the email address - target_qs = Editor.objects.values_list("wp_username", flat=True).filter( - user__email=recipient + target_qs = Editor.objects.filter(user__email=recipient).values_list( + "wp_username", flat=True ) target_qs_count = target_qs.count() if target_qs_count > 1: raise Exception( - "Email address associated with {} user accounts, email skipped".format( - target_qs_count - ) + "skip shared email address: {}".format(list(target_qs)) ) target = target_qs.first() @@ -250,18 +256,12 @@ def _send(self, email_message): "format": "json", } - logger.info("Checking if user is emailable...") - emailable_response = self.session.post( - url=self.url, data=emailable_params + emailable_response = self._handle_request( + self.session.post(url=self.url, data=emailable_params) ) - if emailable_response.status_code != 200: - raise Exception( - "There was an error in the request to check if the user can receive emails." - ) - emailable_data = _handle_maxlag(emailable_response) - emailable = "emailable" in emailable_data["query"]["users"][0] + emailable = "emailable" in emailable_response["query"]["users"][0] if not emailable: - raise Exception("User not emailable, email skipped.") + raise Exception("skip not emailable: {}".format(target)) # POST request to send an email email_params = { @@ -275,14 +275,12 @@ def _send(self, email_message): } logger.info("Sending email...") - emailuser_response = self.session.post(url=self.url, data=email_params) - if emailuser_response.status_code != 200: - raise Exception( - "There was an error in the request to send the email." - ) - emailuser_data = _handle_maxlag(emailuser_response) - if emailuser_data["emailuser"]["result"] != "Success": - raise Exception("There was an error when trying to send the email.") + emailuser_response = self._handle_request( + self.session.post(url=self.url, data=email_params) + ) + if emailuser_response["emailuser"]["result"] != "Success": + raise Exception(dumps(emailuser_response)) + logger.info("Email sent.") except: if not self.fail_silently: