diff --git a/app/grandchallenge/challenges/emails.py b/app/grandchallenge/challenges/emails.py index ed3f4571f..bc1be5b0d 100644 --- a/app/grandchallenge/challenges/emails.py +++ b/app/grandchallenge/challenges/emails.py @@ -1,5 +1,6 @@ from django.contrib.auth import get_user_model from django.contrib.sites.models import Site +from django.core.mail import mail_managers from django.template.loader import render_to_string from django.utils.html import format_html @@ -91,3 +92,33 @@ def send_challenge_status_update_email(challengerequest, challenge=None): recipients=[challengerequest.creator], subscription_type=EmailSubscriptionTypes.SYSTEM, ) + + +def send_email_percent_budget_consumed_alert(challenge, percent_threshold): + subject = format_html( + "[{challenge_name}] over {percent_threshold}% Budget Consumed Alert", + challenge_name=challenge.short_name, + percent_threshold=percent_threshold, + ) + message = format_html( + "We would like to inform you that more than {percent_threshold}% of the " + "compute budget for the {challenge_name} challenge has been used. " + "You can find an overview of the costs [here]({statistics_url}).", + challenge_name=challenge.short_name, + percent_threshold=percent_threshold, + statistics_url=reverse( + "pages:statistics", + kwargs={"challenge_short_name": challenge.short_name}, + ), + ) + send_standard_email_batch( + site=Site.objects.get_current(), + subject=subject, + markdown_message=message, + recipients=[*challenge.get_admins()], + subscription_type=EmailSubscriptionTypes.SYSTEM, + ) + mail_managers( + subject=subject, + message=message, + ) diff --git a/app/grandchallenge/challenges/migrations/0047_challenge_percent_budget_consumed_warning_thresholds.py b/app/grandchallenge/challenges/migrations/0047_challenge_percent_budget_consumed_warning_thresholds.py new file mode 100644 index 000000000..2bc91529e --- /dev/null +++ b/app/grandchallenge/challenges/migrations/0047_challenge_percent_budget_consumed_warning_thresholds.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.17 on 2025-01-08 11:46 + +from django.db import migrations, models + +import grandchallenge.challenges.models +import grandchallenge.core.validators + + +class Migration(migrations.Migration): + dependencies = [ + ( + "challenges", + "0046_remove_challengerequest_budget_for_hosting_challenge", + ), + ] + + operations = [ + migrations.AddField( + model_name="challenge", + name="percent_budget_consumed_warning_thresholds", + field=models.JSONField( + default=grandchallenge.challenges.models.get_default_percent_budget_consumed_warning_thresholds, + validators=[ + grandchallenge.core.validators.JSONValidator( + schema={ + "$schema": "http://json-schema.org/draft-07/schema", + "items": { + "exclusiveMinimum": 0, + "maximum": 100, + "type": "integer", + }, + "type": "array", + "uniqueItems": True, + } + ) + ], + ), + ), + ] diff --git a/app/grandchallenge/challenges/models.py b/app/grandchallenge/challenges/models.py index b70d4b16d..ec43ca5af 100644 --- a/app/grandchallenge/challenges/models.py +++ b/app/grandchallenge/challenges/models.py @@ -50,7 +50,7 @@ GPUTypeChoices, get_default_gpu_type_choices, ) -from grandchallenge.core.models import UUIDModel +from grandchallenge.core.models import FieldChangeMixin, UUIDModel from grandchallenge.core.storage import ( get_banner_path, get_logo_path, @@ -211,7 +211,11 @@ class Meta: abstract = True -class Challenge(ChallengeBase): +def get_default_percent_budget_consumed_warning_thresholds(): + return [70, 90, 100] + + +class Challenge(ChallengeBase, FieldChangeMixin): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) description = models.CharField( @@ -376,6 +380,24 @@ class Challenge(ChallengeBase): help_text="This email will be listed as the contact email for the challenge and will be visible to all users of Grand Challenge.", ) + percent_budget_consumed_warning_thresholds = models.JSONField( + default=get_default_percent_budget_consumed_warning_thresholds, + validators=[ + JSONValidator( + schema={ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "array", + "items": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 100, + }, + "uniqueItems": True, + } + ) + ], + ) + accumulated_compute_cost_in_cents = deprecate_field( models.IntegerField(default=0, blank=True) ) diff --git a/app/grandchallenge/challenges/tasks.py b/app/grandchallenge/challenges/tasks.py index 4429bd52b..1261e9486 100644 --- a/app/grandchallenge/challenges/tasks.py +++ b/app/grandchallenge/challenges/tasks.py @@ -1,10 +1,14 @@ from django.contrib.auth import get_user_model +from django.db import transaction from django.db.models import Count, Max from grandchallenge.challenges.costs import ( annotate_compute_costs_and_storage_size, annotate_job_duration_and_compute_costs, ) +from grandchallenge.challenges.emails import ( + send_email_percent_budget_consumed_alert, +) from grandchallenge.challenges.models import Challenge from grandchallenge.core.celery import acks_late_2xlarge_task from grandchallenge.evaluation.models import Evaluation, Phase @@ -55,12 +59,37 @@ def update_challenge_results_cache(): ) +def send_alert_if_budget_consumed_warning_threshold_exceeded(challenge): + if challenge.has_changed("compute_cost_euro_millicents"): + for percent_threshold in sorted( + challenge.percent_budget_consumed_warning_thresholds, reverse=True + ): + previous_cost = challenge.initial_value( + "compute_cost_euro_millicents" + ) + threshold = ( + challenge.approved_compute_costs_euro_millicents + * percent_threshold + / 100 + ) + current_cost = challenge.compute_cost_euro_millicents + if previous_cost <= threshold < current_cost: + send_email_percent_budget_consumed_alert( + challenge, percent_threshold + ) + break + + @acks_late_2xlarge_task +@transaction.atomic def update_compute_costs_and_storage_size(): - challenges = Challenge.objects.all() + challenges = Challenge.objects.all().with_available_compute() for challenge in challenges: annotate_compute_costs_and_storage_size(challenge=challenge) + send_alert_if_budget_consumed_warning_threshold_exceeded( + challenge=challenge + ) Challenge.objects.bulk_update( challenges, diff --git a/app/tests/challenges_tests/test_tasks.py b/app/tests/challenges_tests/test_tasks.py index 195ea0943..6cf6d61e3 100644 --- a/app/tests/challenges_tests/test_tasks.py +++ b/app/tests/challenges_tests/test_tasks.py @@ -1,9 +1,19 @@ import pytest +from django.core import mail from grandchallenge.challenges.models import Challenge, ChallengeRequest -from grandchallenge.challenges.tasks import update_challenge_results_cache -from tests.evaluation_tests.factories import EvaluationFactory -from tests.factories import ChallengeFactory, ChallengeRequestFactory +from grandchallenge.challenges.tasks import ( + update_challenge_results_cache, + update_compute_costs_and_storage_size, +) +from grandchallenge.invoices.models import PaymentStatusChoices +from tests.evaluation_tests.factories import EvaluationFactory, PhaseFactory +from tests.factories import ( + ChallengeFactory, + ChallengeRequestFactory, + UserFactory, +) +from tests.invoices_tests.factories import InvoiceFactory @pytest.mark.django_db @@ -121,3 +131,135 @@ def test_challenge_request_budget_calculation(settings): + challenge_request.budget["Total phase 2"] + challenge_request.budget["Docker storage cost"] ) + + +@pytest.mark.django_db +def test_challenge_budget_alert_email(settings): + challenge = ChallengeFactory(short_name="test") + challenge_admin = UserFactory() + challenge.add_admin(challenge_admin) + staff_user = UserFactory(is_staff=True) + settings.MANAGERS = [(staff_user.last_name, staff_user.email)] + InvoiceFactory( + challenge=challenge, + support_costs_euros=0, + compute_costs_euros=10, + storage_costs_euros=0, + payment_status=PaymentStatusChoices.PAID, + ) + phase = PhaseFactory(challenge=challenge) + EvaluationFactory( + submission__phase=phase, + compute_cost_euro_millicents=500000, + time_limit=60, + ) + update_compute_costs_and_storage_size() + + # Budget alert threshold not exceeded + assert len(mail.outbox) == 0 + + EvaluationFactory( + submission__phase=phase, + compute_cost_euro_millicents=300000, + time_limit=60, + ) + update_compute_costs_and_storage_size() + + # Budget alert threshold exceeded + assert len(mail.outbox) == 3 + recipients = [r for m in mail.outbox for r in m.to] + assert recipients == [ + challenge.creator.email, + challenge_admin.email, + staff_user.email, + ] + assert ( + mail.outbox[0].subject + == "[testserver] [test] over 70% Budget Consumed Alert" + ) + assert ( + "We would like to inform you that more than 70% of the compute budget for " + "the test challenge has been used. You can find an overview of the costs " + "[here](https://test.testserver/statistics/)." in mail.outbox[0].body + ) + + mail.outbox.clear() + EvaluationFactory( + submission__phase=phase, + compute_cost_euro_millicents=100000, + time_limit=60, + ) + update_compute_costs_and_storage_size() + + # Next budget alert threshold not exceeded + assert len(mail.outbox) == 0 + + EvaluationFactory( + submission__phase=phase, + compute_cost_euro_millicents=1, + time_limit=60, + ) + update_compute_costs_and_storage_size() + + # Next budget alert threshold exceeded + assert len(mail.outbox) != 0 + assert ( + mail.outbox[0].subject + == "[testserver] [test] over 90% Budget Consumed Alert" + ) + + +@pytest.mark.django_db +def test_challenge_budget_alert_two_thresholds_one_email(settings): + challenge = ChallengeFactory(short_name="test") + assert challenge.percent_budget_consumed_warning_thresholds == [ + 70, + 90, + 100, + ] + challenge_admin = UserFactory() + challenge.add_admin(challenge_admin) + staff_user = UserFactory(is_staff=True) + settings.MANAGERS = [(staff_user.last_name, staff_user.email)] + InvoiceFactory( + challenge=challenge, + support_costs_euros=0, + compute_costs_euros=10, + storage_costs_euros=0, + payment_status=PaymentStatusChoices.PAID, + ) + phase = PhaseFactory(challenge=challenge) + EvaluationFactory( + submission__phase=phase, + compute_cost_euro_millicents=950000, + time_limit=60, + ) + update_compute_costs_and_storage_size() + + # Two budget alert thresholds exceeded, alert only sent for last one. + assert len(mail.outbox) == 3 + recipients = [r for m in mail.outbox for r in m.to] + assert recipients == [ + challenge.creator.email, + challenge_admin.email, + staff_user.email, + ] + assert ( + mail.outbox[0].subject + == "[testserver] [test] over 90% Budget Consumed Alert" + ) + + +@pytest.mark.django_db +def test_challenge_budget_alert_no_budget(): + challenge = ChallengeFactory() + phase = PhaseFactory(challenge=challenge) + EvaluationFactory( + submission__phase=phase, + compute_cost_euro_millicents=1, + time_limit=60, + ) + assert len(mail.outbox) == 0 + update_compute_costs_and_storage_size() + assert len(mail.outbox) != 0 + assert "Budget Consumed Alert" in mail.outbox[0].subject