feat: add autogenerated init jobs olive (#79)

README.rst

@@ -57,6 +57,30 @@ The following configuration options are available:
 - `DRYDOCK_ENABLE_SENTRY` : Whether to enable sentry. Defaults to `true`.
 - `DRYDOCK_SENTRY_DSN` : The sentry DSN. Defaults to `""`.
 - `DRYDOCK_POD_LIFECYCLE` : Whether to enable pod lifecycle. Defaults to `true`.
+- `DRYDOCK_MIGRATE_FROM`: it allows defining the version of the OpenedX platform we are migrating from. It accepts the integer value mapping the origin release, for instance, `13`(maple) or `14`(nutmeg). When this variable is set, a group of `release-specific upgrade jobs` are added to the Kubernetes manifests. These jobs are applied to the cluster in a suitable order (thanks to the GitOps implementation with ArgoCD + sync waves) to guarantee the correct behavior of the platform in the new version. This brings the `tutor k8s upgrade <https://github.com/overhangio/tutor/blob/v15.3.7/tutor/commands/k8s.py#L484>`_ command to the GitOps pattern. The release-specific upgrade jobs are supported from release `13`(maple). Defaults to `0` (which disables release-specific upgrade jobs)
+
+.. note::
+    You also need to set `DRYDOCK_INIT_JOBS` to `true` to enable the release-specific upgrade jobs in the case of a platform migration.
+
+Job generation
+--------------
+
+Tutor doesn't generate manifest files for the initialization jobs, in consequence we can't use GitOps tools like ArgoCD to deploy the initialization jobs.
+
+We had been using a static definition of the initialization jobs, but now we are using the `Tutor filters <https://docs.tutor.edly.io/reference/api/hooks/filters.html>`_ to generate the kubernetes definition of the initialization jobs. This is a big improvement because now we can add new initialization jobs without modifying the Drydock code. The jobs are taken from `COMMANDS_PRE_INIT`, `COMMANDS_INIT` and `CLI_DO_INIT_TASKS` Filters.
+
+ArgoCD Sync Waves Support
+-----------------------
+
+`Tutor filter <https://docs.tutor.edly.io/reference/api/hooks/filters.html>`_ **SYNC_WAVES_ORDER** was added to allow define `ArgoCD Sync Waves <https://argo-cd.readthedocs.io/en/stable/user-guide/sync-waves/>`_ order and apply to the kubernetes resources through **get_sync_waves_for_resource** function.
+
+We are defined by defult the following order:
+- `All kubernetes resources` (except the ones that are defined in the next waves)
+- `Initialization Jobs`
+- `Upgrade Jobs`: When **DRYDOCK_MIGRATE_FROM** is set, over the Sync Wave 50
+- `CMS and LMS Deployments`: When **DRYDOCK_POD_LIFECYCLE** is active, over the Sync Wave 100
+- `Debug Resources`: When **DRYDOCK_DEBUG** active, over the Sync Wave 100
+- `Horizontal Pod Autoscalers`: When active, over the Sync Wave 150
 
 Rationale
 ---------

drydock/hooks.py

@@ -0,0 +1,15 @@
+"""
+These hooks are stored in a separate module. If they were included in plugin.py, then
+the drydock hooks would be created in the context of some other plugin that imports
+them.
+"""
+
+from __future__ import annotations
+
+import typing as t
+
+from tutor.core.hooks import Filter, filters
+
+SYNC_WAVES_ORDER_ATTRS_TYPE = t.Dict[str, int]
+
+SYNC_WAVES_ORDER: Filter[SYNC_WAVES_ORDER_ATTRS_TYPE, []] = filters.get("sync_waves_order")

drydock/patches/k8s-jobs

@@ -0,0 +1,15 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mongodb-job
+  labels:
+    app.kubernetes.io/component: job
+spec:
+  ttlSecondsAfterFinished: 100
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: mongodb
+        image: {{ DOCKER_IMAGE_MONGODB }}

drydock/patches/kustomization

@@ -13,21 +13,10 @@ patches:
     kind: Job
     labelSelector: app.kubernetes.io/component=job
   path: plugins/drydock/k8s/patches/tutor-jobs.yml
-# Patch the sync waves
-- target:
-    kind: Deployment
-    name:  "lms|cms|lms-worker|cms-worker|forum"
-  path: plugins/drydock/k8s/patches/sync-wave-4.yml
-{%- if DRYDOCK_DEBUG is defined and DRYDOCK_DEBUG %}
-- target:
-    kind: Deployment|Ingress|Service
-    name:  "cms-debug|lms-debug|ingress-debug"
-  path: plugins/drydock/k8s/patches/sync-wave-5.yml
-{%- endif %}
-- target:
-    kind: HorizontalPodAutoscaler
-  path: plugins/drydock/k8s/patches/sync-wave-5.yml
 {% if DRYDOCK_ENABLE_CELERY_TUNING %}
 - path: plugins/drydock/k8s/celery/cms-worker.yml
 - path: plugins/drydock/k8s/celery/lms-worker.yml
 {% endif -%}
+- target:
+    kind: HorizontalPodAutoscaler
+  path: plugins/drydock/k8s/patches/hpa-sync-wave.yml

drydock/patches/kustomization-resources

@@ -1,12 +1,6 @@
 - plugins/drydock/k8s/multipurpose-jobs.yml
 {%- if DRYDOCK_INIT_JOBS %}
-- plugins/drydock/k8s/drydock-jobs/mysql.yml
-- plugins/drydock/k8s/drydock-jobs/mongodb.yml
-- plugins/drydock/k8s/drydock-jobs/lms.yml
-- plugins/drydock/k8s/drydock-jobs/cms.yml
-- plugins/drydock/k8s/drydock-jobs/minio.yml
-- plugins/drydock/k8s/drydock-jobs/forum.yml
-- plugins/drydock/k8s/drydock-jobs/notes.yml
+- plugins/drydock/k8s/jobs.yml
 {%- endif %}
 {% if DRYDOCK_FLOWER -%}
 - plugins/drydock/k8s/flower.yml

drydock/plugin.py

@@ -1,11 +1,128 @@
 from glob import glob
 import os
+import click
 import pkg_resources
 
-from tutor import hooks
+import typing as t
+
+from .hooks import SYNC_WAVES_ORDER_ATTRS_TYPE, SYNC_WAVES_ORDER
+
+from tutor import hooks as tutor_hooks
+from tutor import env as tutor_env
+from tutor import serialize, types
+from tutor import config as tutor_config
 
 from .__about__ import __version__
 
+INIT_JOBS_SYNC_WAVE = 1
+
+# This function is taken from
+# https://github.com/overhangio/tutor/blob/v15.3.7/tutor/commands/k8s.py#L180
+def _load_jobs(tutor_conf: types.Config) -> t.Iterable[t.Any]:
+    jobs = tutor_env.render_file(tutor_conf, "k8s", "jobs.yml").strip()
+    for manifest in serialize.load_all(jobs):
+        if manifest["kind"] == "Job":
+            yield manifest
+
+
+# The definition of the init tasks is taken and adapted from
+# https://github.com/overhangio/tutor/blob/v15.3.7/tutor/commands/jobs.py#L64
+# and https://github.com/overhangio/tutor/blob/v15.3.7/tutor/commands/k8s.py#L80
+def get_init_tasks():
+    """Return the list of init tasks to run."""
+    context = click.get_current_context().obj
+    tutor_conf = tutor_config.load(context.root)
+
+    init_tasks = []
+    # Standarize deprecated COMMANDS_INIT and COMMANDS_PRE_INIT Filter
+    for service, init_path in tutor_hooks.Filters.COMMANDS_PRE_INIT.iterate():
+        init_tasks.append((service, tutor_env.read_template_file(*init_path)))
+
+    init_tasks.extend(tutor_hooks.Filters.CLI_DO_INIT_TASKS.iterate())
+
+    for service, init_path in list(tutor_hooks.Filters.COMMANDS_INIT.iterate()):
+        init_tasks.append((service, tutor_env.read_template_file(*init_path)))
+
+    for i, (service, command) in enumerate(init_tasks):
+        for template in _load_jobs(tutor_conf):
+            if template['metadata']['name'] != service + '-job':
+                continue
+
+            render_command = tutor_env.render_str(tutor_conf, command)
+
+            template['metadata']['name'] = f"drydock-{template['metadata']['name']}-{i}"
+            template['metadata']['labels'].update({
+                'app.kubernetes.io/component': 'drydock-job',
+                'drydock.io/target-service': template['metadata']['name'],
+                'drydock.io/runner-service': template['metadata']['name']
+            })
+
+            template['metadata']['annotations'] = {
+                'argocd.argoproj.io/sync-wave': INIT_JOBS_SYNC_WAVE + i * 2,
+                'argocd.argoproj.io/hook': 'Sync',
+                'argocd.argoproj.io/hook-delete-policy': 'HookSucceeded,BeforeHookCreation'
+            }
+
+            shell_command = ["sh", "-e", "-c"]
+            if template["spec"]["template"]["spec"]["containers"][0].get("command") == []:
+                # In some cases, we need to bypass the container entrypoint.
+                # Unfortunately, AFAIK, there is no way to do so in K8s manifests. So we mark
+                # some jobs with "command: []". For these jobs, the entrypoint becomes "sh -e -c".
+                # We do not do this for every job, because some (most) entrypoints are actually useful.
+                template["spec"]["template"]["spec"]["containers"][0]["command"] = shell_command
+                container_args = [render_command]
+            else:
+                container_args = shell_command + [render_command]
+
+            template["spec"]["template"]["spec"]["containers"][0]["args"] = container_args
+            template["spec"]["backoffLimit"] = 1
+            template["spec"]["ttlSecondsAfterFinished"] = 3600
+
+            yield serialize.dumps(template)
+
+
+CORE_SYNC_WAVES_ORDER: SYNC_WAVES_ORDER_ATTRS_TYPE = {
+    "drydock-upgrade-lms-job": 50,
+    "drydock-upgrade-cms-job": 51,
+    "lms-lifecycle-enabled": 100,
+    "cms-lifecycle-enabled": 100,
+    "lms-debug": 50,
+    "cms-debug": 50,
+    "ingress-debug": 200,
+    "horizontalpodautoscalers:all": 150
+}
+
+
+@SYNC_WAVES_ORDER.add()
+def _add_core_sync_waves_order(sync_waves_config: SYNC_WAVES_ORDER_ATTRS_TYPE) -> SYNC_WAVES_ORDER_ATTRS_TYPE:
+    sync_waves_config.update(CORE_SYNC_WAVES_ORDER)
+    return sync_waves_config
+
+
+def get_sync_waves_order() -> SYNC_WAVES_ORDER_ATTRS_TYPE:
+    """
+    Return the sync waves order for the plugin
+    """
+    return SYNC_WAVES_ORDER.apply({})
+
+
+def iter_sync_waves_order() -> t.Iterable[SYNC_WAVES_ORDER_ATTRS_TYPE]:
+    """
+    Yield:
+        (name, dict)
+    """
+    yield from get_sync_waves_order().items()
+
+
+def get_sync_waves_for_resource(resource_name: str) -> SYNC_WAVES_ORDER_ATTRS_TYPE:
+    """
+    Args:
+        resource_name: the name of the resource
+    Returns:
+        dict
+    """
+    return get_sync_waves_order().get(resource_name, 0)
+
 
 ################# Configuration
 config = {
@@ -15,6 +132,7 @@
         "INIT_JOBS": False,
         "CMS_SSO_USER": "cms",
         "AUTO_TLS": True,
+        "MIGRATE_FROM": 0,
         "FLOWER": False,
         "INGRESS": False,
         "INGRESS_EXTRA_HOSTS": [],
@@ -42,8 +160,8 @@
     },
 }
 
-hooks.Filters.CONFIG_DEFAULTS.add_items([("OPENEDX_DEBUG_COOKIE", "ednx_enable_debug")])
-hooks.Filters.CONFIG_OVERRIDES.add_items([
+tutor_hooks.Filters.CONFIG_DEFAULTS.add_items([("OPENEDX_DEBUG_COOKIE", "ednx_enable_debug")])
+tutor_hooks.Filters.CONFIG_OVERRIDES.add_items([
         # This values are not prefixed with DRYDOCK_
         ("MONGODB_ROOT_USERNAME", ""),
         ("MONGODB_ROOT_PASSWORD", ""),
@@ -54,16 +172,18 @@
 ################# except maybe for educational purposes :)
 
 # Plugin templates
-hooks.Filters.ENV_TEMPLATE_ROOTS.add_item(
+tutor_hooks.Filters.ENV_TEMPLATE_ROOTS.add_item(
     pkg_resources.resource_filename("drydock", "templates")
 )
-hooks.Filters.ENV_TEMPLATE_TARGETS.add_items(
+
+tutor_hooks.Filters.ENV_TEMPLATE_TARGETS.add_items(
     [
         ("drydock/build", "plugins"),
         ("drydock/apps", "plugins"),
         ("drydock/k8s", "plugins"),
     ],
 )
+
 # Load all patches from the "patches" folder
 for path in glob(
     os.path.join(
@@ -72,19 +192,34 @@
     )
 ):
     with open(path, encoding="utf-8") as patch_file:
-        hooks.Filters.ENV_PATCHES.add_item((os.path.basename(path), patch_file.read()))
+        tutor_hooks.Filters.ENV_PATCHES.add_item((os.path.basename(path), patch_file.read()))
 
 # Load all configuration entries
-hooks.Filters.CONFIG_DEFAULTS.add_items(
+tutor_hooks.Filters.CONFIG_DEFAULTS.add_items(
     [
         (f"DRYDOCK_{key}", value)
         for key, value in config["defaults"].items()
     ]
 )
-hooks.Filters.CONFIG_UNIQUE.add_items(
+tutor_hooks.Filters.CONFIG_UNIQUE.add_items(
     [
         (f"DRYDOCK_{key}", value)
         for key, value in config["unique"].items()
     ]
 )
-hooks.Filters.CONFIG_OVERRIDES.add_items(list(config["overrides"].items()))
+tutor_hooks.Filters.CONFIG_OVERRIDES.add_items(list(config["overrides"].items()))
+
+tutor_hooks.Filters.ENV_TEMPLATE_VARIABLES.add_items(
+    [
+        ('get_init_tasks', get_init_tasks),
+        ('iter_sync_waves_order', iter_sync_waves_order),
+        ('get_sync_waves_for_resource', get_sync_waves_for_resource),
+    ]
+)
+
+# # init script
+with open(
+    pkg_resources.resource_filename("drydock", "templates/drydock/task/mongodb/init"),
+    encoding="utf-8",
+) as fi:
+    tutor_hooks.Filters.CLI_DO_INIT_TASKS.add_item(("mongodb", fi.read()), priority=tutor_hooks.priorities.HIGH)

drydock/templates/drydock/k8s/debug/deployments.yml

@@ -5,6 +5,8 @@ metadata:
   name: cms-debug
   labels:
     app.kubernetes.io/name: cms-debug
+  annotations:
+    argocd.argoproj.io/sync-wave: "{{ get_sync_waves_for_resource('cms-debug') }}"
 spec:
   selector:
     matchLabels:
@@ -57,6 +59,8 @@ metadata:
   name: lms-debug
   labels:
     app.kubernetes.io/name: lms-debug
+  annotations:
+    argocd.argoproj.io/sync-wave: "{{ get_sync_waves_for_resource('lms-debug') }}"
 spec:
   selector:
     matchLabels:

drydock/templates/drydock/k8s/debug/ingress.yml

@@ -4,6 +4,7 @@ metadata:
   name: ingress-debug
   namespace: {{ K8S_NAMESPACE }}
   annotations:
+    argocd.argoproj.io/sync-wave: "{{ get_sync_waves_for_resource('ingress-debug') }}"
     kubernetes.io/ingress.class: nginx
     nginx.ingress.kubernetes.io/canary: "true"
     nginx.ingress.kubernetes.io/canary-by-cookie: {{ OPENEDX_DEBUG_COOKIE }}