Artifacts upload

dispatcher/backend/src/routes/schedules/schedule.py

@@ -12,6 +12,7 @@
 
 import db.models as dbm
 from common.constants import REQ_TIMEOUT_GHCR
+from common.enum import Offliner
 from common.schemas.models import ScheduleConfigSchema, ScheduleSchema
 from common.schemas.orms import ScheduleFullSchema, ScheduleLightSchema
 from common.schemas.parameters import CloneSchema, SchedulesSchema, UpdateSchema
@@ -239,12 +240,21 @@ def patch(
             raise_if(not request.get_json(), ValidationError, "Update can't be empty")
 
             # ensure we test flags according to new task_name if present
-            if "task_name" in update:
+            if (
+                "task_name" in update
+                and update["task_name"] != schedule.config["task_name"]
+            ):
                 raise_if(
                     "flags" not in update,
                     ValidationError,
                     "Can't update offliner without updating flags",
                 )
+                raise_if(
+                    "image" not in update or "name" not in update["image"],
+                    ValidationError,
+                    "Image name must be updated when offliner is changed",
+                )
+
                 flags_schema = ScheduleConfigSchema.get_offliner_schema(
                     update["task_name"]
                 )
@@ -255,6 +265,17 @@ def patch(
 
             if "flags" in update:
                 flags_schema().load(update["flags"])
+
+            if "image" in update and "name" in update["image"]:
+                if "task_name" in update:
+                    future_task_name = update["task_name"]
+                else:
+                    future_task_name = schedule.config["task_name"]
+
+                if Offliner.get_image_prefix(future_task_name) + update["image"][
+                    "name"
+                ] != Offliner.get_image_name(future_task_name):
+                    raise ValidationError("Image name must match selected offliner")
         except ValidationError as e:
             raise InvalidRequestJSON(e.messages)
 

dispatcher/backend/src/tests/integration/routes/schedules/test_schedule.py

@@ -1,7 +1,9 @@
 import json
+from copy import deepcopy
 from uuid import uuid4
 
 import pytest
+from utils_for_tests import patch_dict
 
 from utils.offliners import expanded_config
 
@@ -24,76 +26,103 @@ def schedule(make_schedule):
     {"tags": ["full"]},
     {"tags": ["full", "small"]},
     {"tags": ["full", "small"], "category": "vikidia", "enabled": False},
-    {"task_name": "phet", "flags": {}},
+    {
+        "task_name": "mwoffliner",
+    },
+    {
+        "task_name": "phet",
+        "flags": {},
+        "image": {"name": "openzim/phet", "tag": "1.1.2"},
+    },
     {"flags": {"mwUrl": "https://fr.wikipedia.org", "adminEmail": "hello@test.de"}},
     {"warehouse_path": "/phet"},
-    {"image": {"name": "openzim/phet", "tag": "latest"}},
+    {"image": {"name": "openzim/mwoffliner", "tag": "1.12.2"}},
     {"resources": {"cpu": 3, "memory": MIN_RAM, "disk": ONE_GiB}},
     {
         "task_name": "gutenberg",
         "warehouse_path": "/gutenberg",
         "flags": {},
-        "image": {"name": "openzim/gutenberg", "tag": "latest"},
+        "image": {"name": "openzim/gutenberg", "tag": "1.1.2"},
         "resources": {"cpu": 3, "memory": MIN_RAM, "disk": ONE_GiB},
     },
     {"name": "new_name"},
 ]
 
 bad_patch_updates = [
+    # Empty patch not allowed
     {},
+    # wrong languages combinations
     {"language": {"name_en": "Bambara", "name_native": "Bamanankan"}},
     {"language": {"code": "bm", "name_en": "", "name_native": "Bamanankan"}},
     {"language": {"code": "bm", "name_en": "Bambara"}},
+    # enabled flag must be a boolean, not a string
     {"enabled": "False"},
+    # illegal category
     {"category": "ubuntu"},
+    # empty tags not allowed
     {"tags": ""},
+    # tags must be strings
     {"tags": ["full", 1]},
+    # tags must be a list
     {"tags": "full,small"},
+    # name cannot be empty
     {"name": ""},
+    # config cannot be empty
     {"config": ""},
+    # mwoffliner does not supports empty flags
     {"flags": {}},
+    # cannot change only offliner, image must be changed as well
+    {"task_name": "phet", "flags": {}},
+    # cannot change only image name, offliner must be changed as well
+    {"flags": {}, "image": {"name": "openzim/phet", "tag": "latest"}},
+    # illegal offliner name
     {
         "task_name": "hop",
-        "warehouse_path": "/phet",
         "flags": {},
-        "image": {"name": "openzim/phet", "tag": "latest"},
-        "resources": {"cpu": 3, "memory": MIN_RAM, "disk": ONE_GiB},
+        "image": {"name": "openzim/hop", "tag": "latest"},
     },
+    # wrong warehouse_path
     {
-        "task_name": "phet",
         "warehouse_path": "/ubuntu",
-        "flags": {},
-        "image": {"name": "openzim/phet", "tag": "latest"},
-        "resources": {"cpu": 3, "memory": MIN_RAM, "disk": ONE_GiB},
     },
+    # wrong mwUrl flag for phet
     {
         "task_name": "phet",
-        "warehouse_path": "/phet",
         "flags": {"mwUrl": "http://fr.wikipedia.org"},
         "image": {"name": "openzim/phet", "tag": "latest"},
-        "resources": {"cpu": 3, "memory": MIN_RAM, "disk": ONE_GiB},
-    },
-    {
-        "task_name": "phet",
-        "warehouse_path": "/phet",
-        "flags": {},
-        "image": {"name": "openzim/youtuba", "tag": "latest"},
-        "resources": {"cpu": 3, "memory": MIN_RAM, "disk": ONE_GiB},
     },
+    # image name not aligned with task name
     {
         "task_name": "gutenberg",
-        "warehouse_path": "/gutenberg",
         "flags": {},
         "image": {"name": "openzim/youtube", "tag": "latest"},
+    },
+    # bad cpu value
+    {
         "resources": {"cpu": -1, "memory": MIN_RAM, "disk": ONE_GiB},
     },
+    # bad mem value
+    {
+        "resources": {"cpu": 3, "memory": -1, "disk": ONE_GiB},
+    },
+    # bad disk value
+    {
+        "resources": {"cpu": 3, "memory": MIN_RAM, "disk": -1},
+    },
+    # one resource missing
+    {
+        "resources": {"cpu": 3, "memory": MIN_RAM},
+    },
+    # illegal characters in name
     {"name": "new\u0000name"},
+    # illegal characters in mwUrl
     {
         "flags": {
             "mwUrl": "https://fr.wiki\u0000pedia.org",
             "adminEmail": "hello@test.de",
         }
     },
+    # illegal characters in adminEmail
     {
         "flags": {
             "mwUrl": "https://fr.wikipedia.org",
@@ -455,6 +484,8 @@ def test_patch_schedule_via_name_with(self, client, access_token, update, schedu
         # let's reapply manually the changes that should have been done by the patch
         # so that we can confirm it has been done
         document = response.get_json()
+
+        update_patch = {}
         config_keys = [
             "task_name",
             "warehouse_path",
@@ -464,11 +495,26 @@ def test_patch_schedule_via_name_with(self, client, access_token, update, schedu
             "flags",
             "monitor",
         ]
-        # these keys must not be applied since they are somewhere else is the document
-        for key in config_keys:
-            update.pop(key, None)
-        document.update(update)
-        assert response.get_json() == document
+        for key, value in update.items():
+            if key in config_keys:
+                if "config" not in update_patch:
+                    update_patch["config"] = {}
+                update_patch["config"][key] = value
+            else:
+                update_patch[key] = value
+
+        # handle special situation for config image name
+        if (
+            "config" in update_patch
+            and "image" in update_patch["config"]
+            and "name" in update_patch["config"]["image"]
+        ):
+            update_patch["config"]["image"]["name"] = (
+                "ghcr.io/" + update_patch["config"]["image"]["name"]
+            )
+
+        patch_result = patch_dict(deepcopy(document), update_patch)
+        assert document == patch_result
 
     @pytest.mark.parametrize("update", bad_patch_updates)
     def test_patch_schedule_via_name_with_errors(

dispatcher/backend/src/tests/utils_for_tests.py

@@ -0,0 +1,21 @@
+def patch_dict(data, patch):
+    for key, patch_value in patch.items():
+        if key in data:
+            if patch_value is None:
+                # If the patch value is None, remove the key from the original
+                # dictionary
+                del data[key]
+            else:
+                original_value = data[key]
+                if isinstance(original_value, dict) and isinstance(patch_value, dict):
+                    # If both values are dictionaries, recursively patch the nested
+                    # dictionaries
+                    patch_dict(original_value, patch_value)
+                else:
+                    # Otherwise, update the value in the original dictionary
+                    data[key] = patch_value
+        else:
+            # If the key is not present in the original dictionary, set it with the
+            # patch value
+            data[key] = patch_value
+    return data