Skip to content

Commit

Permalink
Reset start time, end time and error message on job requeue from admin (
Browse files Browse the repository at this point in the history
#3782)

closes #3435
  • Loading branch information
ammar257ammar authored Jan 13, 2025
1 parent d3d420d commit ffcde69
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 2 deletions.
16 changes: 15 additions & 1 deletion app/grandchallenge/components/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,25 @@ def requeue_jobs(modeladmin, request, queryset):
for job in queryset:
job.status = ComponentJob.RETRY
job.attempt += 1
job.started_at = None
job.completed_at = None
job.error_message = ""
job.detailed_error_message = {}
jobs.append(job)

on_commit(job.execute)

queryset.model.objects.bulk_update(jobs, fields=["status", "attempt"])
queryset.model.objects.bulk_update(
jobs,
fields=[
"status",
"attempt",
"started_at",
"completed_at",
"error_message",
"detailed_error_message",
],
)


@admin.action(
Expand Down
95 changes: 94 additions & 1 deletion app/tests/algorithms_tests/test_admin.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,20 @@
from pathlib import Path

import pytest
from django.core.files.base import ContentFile

from grandchallenge.algorithms.admin import AlgorithmAdmin
from tests.components_tests.factories import ComponentInterfaceFactory
from grandchallenge.algorithms.models import Job
from grandchallenge.algorithms.tasks import create_algorithm_jobs
from grandchallenge.components.admin import requeue_jobs
from grandchallenge.components.models import ComponentInterface
from tests.algorithms_tests.factories import AlgorithmImageFactory
from tests.components_tests.factories import (
ComponentInterfaceFactory,
ComponentInterfaceValueFactory,
)
from tests.factories import ImageFileFactory
from tests.utils import recurse_callbacks


@pytest.mark.django_db
Expand All @@ -10,3 +23,83 @@ def test_disjoint_interfaces():
form = AlgorithmAdmin.form(data={"inputs": [i.pk], "outputs": [i.pk]})
assert form.is_valid() is False
assert "The sets of Inputs and Outputs must be unique" in str(form.errors)


@pytest.mark.django_db
def test_job_updated_start_and_complete_times_after_admin_requeue(
algorithm_image, settings, django_capture_on_commit_callbacks
):
settings.task_eager_propagates = (True,)
settings.task_always_eager = (True,)

with django_capture_on_commit_callbacks() as callbacks:
ai = AlgorithmImageFactory(image=None)

with open(algorithm_image, "rb") as f:
ai.image.save(algorithm_image, ContentFile(f.read()))

recurse_callbacks(
callbacks=callbacks,
django_capture_on_commit_callbacks=django_capture_on_commit_callbacks,
)
ai.refresh_from_db()

# Make sure the job fails when trying to upload an invalid file
input_interface = ComponentInterface.objects.get(
slug="generic-medical-image"
)
detection_interface = ComponentInterfaceFactory(
store_in_database=False,
relative_path="some_text.txt",
slug="detection-json-file",
kind=ComponentInterface.Kind.ANY,
)
ai.algorithm.inputs.add(input_interface)
ai.algorithm.outputs.add(detection_interface)

image_file = ImageFileFactory(
file__from_path=Path(__file__).parent / "resources" / "input_file.tif"
)

civ = ComponentInterfaceValueFactory(
image=image_file.image, interface=input_interface, file=None
)

with django_capture_on_commit_callbacks() as callbacks:
create_algorithm_jobs(
algorithm_image=ai,
civ_sets=[{civ}],
time_limit=ai.algorithm.time_limit,
requires_gpu_type=ai.algorithm.job_requires_gpu_type,
requires_memory_gb=ai.algorithm.job_requires_memory_gb,
)
recurse_callbacks(
callbacks=callbacks,
django_capture_on_commit_callbacks=django_capture_on_commit_callbacks,
)

jobs = Job.objects.filter(
algorithm_image=ai, inputs__image=image_file.image, status=Job.FAILURE
).all()

assert len(jobs) == 1

job = jobs.first()

first_run_started_at = job.started_at
first_run_completed_at = job.completed_at

with django_capture_on_commit_callbacks() as callbacks:
requeue_jobs(None, None, jobs)

recurse_callbacks(
callbacks=callbacks,
django_capture_on_commit_callbacks=django_capture_on_commit_callbacks,
)
job.refresh_from_db()

second_run_started_at = job.started_at
second_run_completed_at = job.completed_at

assert second_run_started_at > first_run_started_at
assert second_run_completed_at > first_run_completed_at

0 comments on commit ffcde69

Please sign in to comment.