Adjustable gpu types (#3699)

Make the selectable gpu type and maximum memory adjustable for
evaluations.

See DIAGNijmegen/rse-roadmap#346

app/grandchallenge/evaluation/forms.py

@@ -80,6 +80,7 @@
 )
 
 scoring_options = (
+    "evaluation_requires_gpu_type",
     "evaluation_requires_memory_gb",
     "score_title",
     "score_jsonpath",
@@ -157,7 +158,13 @@ def __init__(self, *args, **kwargs):
         self.fields["parent"].queryset = self.instance.parent_phase_choices
         self.fields["evaluation_requires_memory_gb"].validators = [
             MinValueValidator(settings.ALGORITHMS_MIN_MEMORY_GB),
-            MaxValueValidator(settings.ALGORITHMS_MAX_MEMORY_GB),
+            MaxValueValidator(
+                self.instance.evaluation_maximum_settable_memory_gb
+            ),
+        ]
+        self.fields["evaluation_requires_gpu_type"].choices = [
+            (c, GPUTypeChoices(c).label)
+            for c in self.instance.evaluation_selectable_gpu_type_choices
         ]
 
         self.helper.layout = Layout(

app/grandchallenge/evaluation/migrations/0069_phase_maximum_settable_memory_gb_evaluation_and_more.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.2.16 on 2024-11-21 12:39
+
+from django.db import migrations, models
+
+import grandchallenge.core.validators
+import grandchallenge.evaluation.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("evaluation", "0068_alter_evaluation_detailed_error_message"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="phase",
+            name="evaluation_maximum_settable_memory_gb",
+            field=models.PositiveSmallIntegerField(
+                default=32,
+                help_text="Maximum amount of memory that challenge admins will be able to assign for the evaluation method.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="phase",
+            name="evaluation_selectable_gpu_type_choices",
+            field=models.JSONField(
+                default=grandchallenge.evaluation.models.get_default_gpu_type_choices,
+                help_text='The GPU type choices that challenge admins will be able to set for the evaluation method. Options are ["", "A100", "A10G", "V100", "K80", "T4"].',
+                validators=[
+                    grandchallenge.core.validators.JSONValidator(
+                        schema={
+                            "$schema": "http://json-schema.org/draft-07/schema",
+                            "items": {
+                                "enum": [
+                                    "",
+                                    "A100",
+                                    "A10G",
+                                    "V100",
+                                    "K80",
+                                    "T4",
+                                ],
+                                "type": "string",
+                            },
+                            "title": "The Selectable GPU Types Schema",
+                            "type": "array",
+                            "uniqueItems": True,
+                        }
+                    )
+                ],
+            ),
+        ),
+    ]

app/grandchallenge/evaluation/models.py

@@ -134,6 +134,21 @@
     },
 }
 
+SELECTABLE_GPU_TYPES_SCHEMA = {
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "type": "array",
+    "title": "The Selectable GPU Types Schema",
+    "items": {
+        "enum": [choice for choice in GPUTypeChoices],
+        "type": "string",
+    },
+    "uniqueItems": True,
+}
+
+
+def get_default_gpu_type_choices():
+    return [GPUTypeChoices.NO_GPU, GPUTypeChoices.T4]
+
 
 class PhaseManager(models.Manager):
     def get_queryset(self):
@@ -516,6 +531,16 @@ class Phase(FieldChangeMixin, HangingProtocolMixin, UUIDModel):
             ),
         ],
     )
+    evaluation_selectable_gpu_type_choices = models.JSONField(
+        default=get_default_gpu_type_choices,
+        help_text=(
+            "The GPU type choices that challenge admins will be able to set for the "
+            f"evaluation method. Options are {GPUTypeChoices.values}.".replace(
+                "'", '"'
+            )
+        ),
+        validators=[JSONValidator(schema=SELECTABLE_GPU_TYPES_SCHEMA)],
+    )
     evaluation_requires_gpu_type = models.CharField(
         max_length=4,
         blank=True,
@@ -527,6 +552,13 @@ class Phase(FieldChangeMixin, HangingProtocolMixin, UUIDModel):
             "is determined by the submitted algorithm."
         ),
     )
+    evaluation_maximum_settable_memory_gb = models.PositiveSmallIntegerField(
+        default=settings.ALGORITHMS_MAX_MEMORY_GB,
+        help_text=(
+            "Maximum amount of memory that challenge admins will be able to "
+            "assign for the evaluation method."
+        ),
+    )
     evaluation_requires_memory_gb = models.PositiveSmallIntegerField(
         default=8,
         help_text=(
@@ -665,6 +697,7 @@ def clean(self):
         self._clean_submission_limits()
         self._clean_parent_phase()
         self._clean_external_evaluation()
+        self._clean_evaluation_requirements()
 
     def _clean_algorithm_submission_settings(self):
         if self.submission_kind == SubmissionKindChoices.ALGORITHM:
@@ -743,6 +776,27 @@ def _clean_external_evaluation(self):
                     "An external evaluation phase must have a parent phase."
                 )
 
+    def _clean_evaluation_requirements(self):
+        if (
+            self.evaluation_requires_gpu_type
+            not in self.evaluation_selectable_gpu_type_choices
+        ):
+            raise ValidationError(
+                f"{self.evaluation_requires_gpu_type!r} is not a valid choice "
+                f"for Evaluation requires gpu type. Either change the choice or "
+                f"add it to the list of selectable gpu types."
+            )
+        if (
+            self.evaluation_requires_memory_gb
+            > self.evaluation_maximum_settable_memory_gb
+        ):
+            raise ValidationError(
+                f"Ensure the value for Evaluation requires memory gb (currently "
+                f"{self.evaluation_requires_memory_gb}) is less than or equal "
+                f"to the maximum settable (currently "
+                f"{self.evaluation_maximum_settable_memory_gb})."
+            )
+
     @property
     def scoring_method(self):
         if self.scoring_method_choice == self.ABSOLUTE:

app/tests/evaluation_tests/test_admin.py

@@ -49,3 +49,18 @@ def test_read_only_fields_disabled():
         instance=p3,
     )
     assert form.fields["submission_kind"].disabled
+
+
+@pytest.mark.django_db
+def test_selectable_gpu_type_choices_invalid():
+    phase = PhaseFactory()
+    form = PhaseAdmin.form(
+        instance=phase,
+        data={"evaluation_selectable_gpu_type_choices": '["invalid_choice"]'},
+    )
+
+    assert form.is_valid() is False
+    assert (
+        "JSON does not fulfill schema: instance 'invalid_choice' is not "
+        "one of " in str(form.errors)
+    )

app/tests/evaluation_tests/test_forms.py

@@ -6,7 +6,10 @@
 
 from grandchallenge.algorithms.forms import AlgorithmForPhaseForm
 from grandchallenge.algorithms.models import Job
-from grandchallenge.components.models import ImportStatusChoices
+from grandchallenge.components.models import (
+    GPUTypeChoices,
+    ImportStatusChoices,
+)
 from grandchallenge.evaluation.forms import (
     ConfigureAlgorithmPhasesForm,
     EvaluationGroundTruthForm,
@@ -1170,3 +1173,34 @@ def test_phase_update_form_gpu_limited_choices():
     )
     assert max_validator is not None
     assert max_validator.limit_value == 32
+
+
+@pytest.mark.django_db
+def test_phase_update_form_gpu_type_limited_choices():
+    phase = PhaseFactory()
+    form = PhaseUpdateForm(
+        instance=phase, challenge=phase.challenge, user=UserFactory.build()
+    )
+
+    choices = form.fields["evaluation_requires_gpu_type"].widget.choices
+
+    assert choices is not None
+    choice = GPUTypeChoices.NO_GPU
+    assert (choice.value, choice.label) in choices
+    choice = GPUTypeChoices.V100
+    assert (choice.value, choice.label) not in choices
+
+
+@pytest.mark.django_db
+def test_phase_update_form_gpu_type_with_additional_selectable_gpu_types():
+    phase = PhaseFactory()
+    phase.evaluation_selectable_gpu_type_choices = ["V100"]
+    form = PhaseUpdateForm(
+        instance=phase, challenge=phase.challenge, user=UserFactory.build()
+    )
+
+    choices = form.fields["evaluation_requires_gpu_type"].widget.choices
+
+    assert choices is not None
+    choice = GPUTypeChoices.V100
+    assert (choice.value, choice.label) in choices