Budget alerts

app/grandchallenge/challenges/emails.py

@@ -1,5 +1,6 @@
 from django.contrib.auth import get_user_model
 from django.contrib.sites.models import Site
+from django.core.mail import mail_managers
 from django.template.loader import render_to_string
 from django.utils.html import format_html
 
@@ -91,3 +92,33 @@ def send_challenge_status_update_email(challengerequest, challenge=None):
         recipients=[challengerequest.creator],
         subscription_type=EmailSubscriptionTypes.SYSTEM,
     )
+
+
+def send_email_percent_budget_consumed_alert(challenge, percent_threshold):
+    subject = format_html(
+        "[{challenge_name}] {percent_threshold}% Budget Consumed Alert",
+        challenge_name=challenge.short_name,
+        percent_threshold=percent_threshold,
+    )
+    message = format_html(
+        "We would like to inform you that more than {percent_threshold}% of the "
+        "compute budget for the {challenge_name} challenge has been used. "
+        "You can find an overview of the costs [here]({statistics_url}).",
+        challenge_name=challenge.short_name,
+        percent_threshold=percent_threshold,
+        statistics_url=reverse(
+            "pages:statistics",
+            kwargs={"challenge_short_name": challenge.short_name},
+        ),
+    )
+    send_standard_email_batch(
+        site=Site.objects.get_current(),
+        subject=subject,
+        markdown_message=message,
+        recipients=[*challenge.get_admins()],
+        subscription_type=EmailSubscriptionTypes.SYSTEM,
+    )
+    mail_managers(
+        subject=subject,
+        message=message,
+    )

app/grandchallenge/challenges/migrations/0046_challenge_budget_warning_thresholds_percentage.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.2.17 on 2024-12-11 10:16
+
+from django.db import migrations, models
+
+import grandchallenge.challenges.models
+import grandchallenge.core.validators
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "challenges",
+            "0045_challengerequest_algorithm_maximum_settable_memory_gb_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="challenge",
+            name="percent_budget_consumed_warning_thresholds",
+            field=models.JSONField(
+                default=grandchallenge.challenges.models.get_default_percent_budget_consumed_warning_thresholds,
+                validators=[
+                    grandchallenge.core.validators.JSONValidator(
+                        schema={
+                            "$schema": "http://json-schema.org/draft-07/schema",
+                            "items": {"type": "integer"},
+                            "type": "array",
+                            "uniqueItems": True,
+                        }
+                    )
+                ],
+            ),
+        ),
+    ]

app/grandchallenge/challenges/models.py

@@ -50,7 +50,7 @@
     GPUTypeChoices,
     get_default_gpu_type_choices,
 )
-from grandchallenge.core.models import UUIDModel
+from grandchallenge.core.models import FieldChangeMixin, UUIDModel
 from grandchallenge.core.storage import (
     get_banner_path,
     get_logo_path,
@@ -211,7 +211,11 @@ class Meta:
         abstract = True
 
 
-class Challenge(ChallengeBase):
+def get_default_percent_budget_consumed_warning_thresholds():
+    return [70, 90, 100]
+
+
+class Challenge(ChallengeBase, FieldChangeMixin):
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
     description = models.CharField(
@@ -376,6 +380,20 @@ class Challenge(ChallengeBase):
         help_text="This email will be listed as the contact email for the challenge and will be visible to all users of Grand Challenge.",
     )
 
+    percent_budget_consumed_warning_thresholds = models.JSONField(
+        default=get_default_percent_budget_consumed_warning_thresholds,
+        validators=[
+            JSONValidator(
+                schema={
+                    "$schema": "http://json-schema.org/draft-07/schema",
+                    "type": "array",
+                    "items": {"type": "integer"},
+                    "uniqueItems": True,
+                }
+            )
+        ],
+    )
+
     accumulated_compute_cost_in_cents = deprecate_field(
         models.IntegerField(default=0, blank=True)
     )

app/grandchallenge/challenges/tasks.py

@@ -1,10 +1,14 @@
 from django.contrib.auth import get_user_model
+from django.db import transaction
 from django.db.models import Count, Max
 
 from grandchallenge.challenges.costs import (
     annotate_compute_costs_and_storage_size,
     annotate_job_duration_and_compute_costs,
 )
+from grandchallenge.challenges.emails import (
+    send_email_percent_budget_consumed_alert,
+)
 from grandchallenge.challenges.models import Challenge
 from grandchallenge.core.celery import acks_late_2xlarge_task
 from grandchallenge.evaluation.models import Evaluation, Phase
@@ -55,12 +59,37 @@ def update_challenge_results_cache():
     )
 
 
+def send_alert_if_budget_consumed_warning_threshold_exceeded(challenge):
+    if challenge.has_changed("compute_cost_euro_millicents"):
+        for percent_threshold in sorted(
+            challenge.percent_budget_consumed_warning_thresholds, reverse=True
+        ):
+            previous_cost = challenge.initial_value(
+                "compute_cost_euro_millicents"
+            )
+            threshold = (
+                challenge.approved_compute_costs_euro_millicents
+                * percent_threshold
+                / 100
+            )
+            current_cost = challenge.compute_cost_euro_millicents
+            if previous_cost <= threshold < current_cost:
+                send_email_percent_budget_consumed_alert(
+                    challenge, percent_threshold
+                )
+                break
+
+
 @acks_late_2xlarge_task
+@transaction.atomic
 def update_compute_costs_and_storage_size():
-    challenges = Challenge.objects.all()
+    challenges = Challenge.objects.all().with_available_compute()
 
     for challenge in challenges:
         annotate_compute_costs_and_storage_size(challenge=challenge)
+        send_alert_if_budget_consumed_warning_threshold_exceeded(
+            challenge=challenge
+        )
 
     Challenge.objects.bulk_update(
         challenges,

app/tests/challenges_tests/test_tasks.py

@@ -1,9 +1,19 @@
 import pytest
+from django.core import mail
 
 from grandchallenge.challenges.models import Challenge, ChallengeRequest
-from grandchallenge.challenges.tasks import update_challenge_results_cache
-from tests.evaluation_tests.factories import EvaluationFactory
-from tests.factories import ChallengeFactory, ChallengeRequestFactory
+from grandchallenge.challenges.tasks import (
+    update_challenge_results_cache,
+    update_compute_costs_and_storage_size,
+)
+from grandchallenge.invoices.models import PaymentStatusChoices
+from tests.evaluation_tests.factories import EvaluationFactory, PhaseFactory
+from tests.factories import (
+    ChallengeFactory,
+    ChallengeRequestFactory,
+    UserFactory,
+)
+from tests.invoices_tests.factories import InvoiceFactory
 
 
 @pytest.mark.django_db
@@ -121,3 +131,94 @@ def test_challenge_request_budget_calculation(settings):
         + challenge_request.budget["Total phase 2"]
         + challenge_request.budget["Docker storage cost"]
     )
+
+
+@pytest.mark.django_db
+def test_challenge_budget_alert_email(settings):
+    challenge = ChallengeFactory(short_name="test")
+    challenge_admin = UserFactory()
+    challenge.add_admin(challenge_admin)
+    staff_user = UserFactory(is_staff=True)
+    settings.MANAGERS = [(staff_user.last_name, staff_user.email)]
+    InvoiceFactory(
+        challenge=challenge,
+        support_costs_euros=0,
+        compute_costs_euros=10,
+        storage_costs_euros=0,
+        payment_status=PaymentStatusChoices.PAID,
+    )
+    phase = PhaseFactory(challenge=challenge)
+    EvaluationFactory(
+        submission__phase=phase,
+        compute_cost_euro_millicents=500000,
+        time_limit=60,
+    )
+    update_compute_costs_and_storage_size()
+
+    # Budget alert threshold not exceeded
+    assert len(mail.outbox) == 0
+
+    EvaluationFactory(
+        submission__phase=phase,
+        compute_cost_euro_millicents=300000,
+        time_limit=60,
+    )
+    update_compute_costs_and_storage_size()
+
+    # Budget alert threshold exceeded
+    assert len(mail.outbox) == 3
+    recipients = [r for m in mail.outbox for r in m.to]
+    assert recipients == [
+        challenge.creator.email,
+        challenge_admin.email,
+        staff_user.email,
+    ]
+    assert (
+        mail.outbox[0].subject
+        == "[testserver] [test] 70% Budget Consumed Alert"
+    )
+    assert (
+        "We would like to inform you that more than 70% of the compute budget for "
+        "the test challenge has been used. You can find an overview of the costs "
+        "[here](https://test.testserver/statistics/)." in mail.outbox[0].body
+    )
+
+    mail.outbox.clear()
+    EvaluationFactory(
+        submission__phase=phase,
+        compute_cost_euro_millicents=100000,
+        time_limit=60,
+    )
+    update_compute_costs_and_storage_size()
+
+    # Next budget alert threshold not exceeded
+    assert len(mail.outbox) == 0
+
+    EvaluationFactory(
+        submission__phase=phase,
+        compute_cost_euro_millicents=1,
+        time_limit=60,
+    )
+    update_compute_costs_and_storage_size()
+
+    # Next budget alert threshold exceeded
+    assert len(mail.outbox) != 0
+    assert (
+        mail.outbox[0].subject
+        == "[testserver] [test] 90% Budget Consumed Alert"
+    )
+
+
+@pytest.mark.django_db
+def test_challenge_budget_alert_no_budget():
+    challenge = ChallengeFactory()
+    phase = PhaseFactory(challenge=challenge)
+    EvaluationFactory(
+        submission__phase=phase,
+        compute_cost_euro_millicents=1,
+        time_limit=60,
+    )
+    assert len(mail.outbox) == 0
+    update_compute_costs_and_storage_size()
+    assert len(mail.outbox) != 0
+    assert "Budget Consumed Alert" in mail.outbox[0].subject