Project sync

apps/odk_publish/admin.py

@@ -23,6 +23,7 @@
 class CentralServerAdmin(admin.ModelAdmin):
     list_display = ("base_url", "created_at", "modified_at")
     search_fields = ("base_url",)
+    ordering = ("base_url",)
 
 
 @admin.register(TemplateVariable)
@@ -34,8 +35,8 @@ class TemplateVariableAdmin(admin.ModelAdmin):
 
 @admin.register(Project)
 class ProjectAdmin(admin.ModelAdmin):
-    list_display = ("name", "project_id", "central_server")
-    search_fields = ("name", "project_id")
+    list_display = ("name", "central_id", "central_server")
+    search_fields = ("name", "central_id")
     list_filter = ("central_server",)
     filter_horizontal = ("template_variables",)
 

apps/odk_publish/etl/load.py

@@ -2,7 +2,7 @@
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db.models import QuerySet
 
-from ..models import AppUser, AppUserFormTemplate, FormTemplate, Project
+from ..models import AppUser, AppUserFormTemplate, CentralServer, FormTemplate, Project
 from .odk.client import ODKPublishClient
 from .odk.qrcode import create_app_user_qrcode
 
@@ -18,14 +18,14 @@ def create_or_update_app_users(form_template: FormTemplate):
     ) as client:
         app_users = client.odk_publish.get_or_create_app_users(
             display_names=[app_user_form.app_user.name for app_user_form in app_user_forms],
-            project_id=form_template.project.project_id,
+            project_id=form_template.project.central_id,
         )
         # Link form assignments to app users locally
         for app_user_form in app_user_forms:
             app_users[app_user_form.app_user.name].xml_form_ids.append(app_user_form.xml_form_id)
         # Create or update the form assignments on the server
         client.odk_publish.assign_forms(
-            app_users=app_users.values(), project_id=form_template.project.project_id
+            app_users=app_users.values(), project_id=form_template.project.central_id
         )
 
 
@@ -34,7 +34,7 @@ def generate_and_save_app_user_collect_qrcodes(project: Project):
     app_users: QuerySet[AppUser] = project.app_users.all()
     logger.info("Generating QR codes", project=project.name, app_users=len(app_users))
     with ODKPublishClient(
-        base_url=project.central_server.base_url, project_id=project.project_id
+        base_url=project.central_server.base_url, project_id=project.central_id
     ) as client:
         central_app_users = client.odk_publish.get_app_users(
             display_names=[app_user.name for app_user in app_users],
@@ -45,9 +45,74 @@ def generate_and_save_app_user_collect_qrcodes(project: Project):
             image = create_app_user_qrcode(
                 app_user=central_app_users[app_user.name],
                 base_url=client.session.base_url,
-                project_id=project.project_id,
+                central_id=project.central_id,
                 project_name_prefix=project.name,
             )
             app_user.qr_code.save(
                 f"{app_user.name}.png", SimpleUploadedFile("qrcode.png", image.getvalue())
             )
+
+
+def sync_central_project(base_url: str, project_id: int) -> Project:
+    """Sync a project from ODK Central to the local database."""
+    config = ODKPublishClient.get_config(base_url=base_url)
+    with ODKPublishClient(base_url=config.base_url, project_id=project_id) as client:
+        # CentralServer
+        server, created = CentralServer.objects.get_or_create(base_url=base_url)
+        logger.debug(
+            f"{'Created' if created else 'Retrieved'} CentralServer", base_url=server.base_url
+        )
+        # Project
+        central_project = client.projects.get()
+        project, created = Project.objects.get_or_create(
+            central_id=central_project.id,
+            central_server=server,
+            defaults={"name": central_project.name},
+        )
+        logger.info(
+            f"{'Created' if created else 'Retrieved'} Project",
+            id=project.id,
+            central_id=project.central_id,
+            project_name=project.name,
+        )
+        # AppUser
+        central_app_users = client.odk_publish.get_app_users()
+        for central_app_user in central_app_users.values():
+            if not central_app_user.deletedAt:
+                app_user, created = project.app_users.get_or_create(
+                    central_id=central_app_user.id,
+                    defaults={"name": central_app_user.displayName},
+                )
+                logger.info(
+                    f"{'Created' if created else 'Retrieved'} AppUser",
+                    id=app_user.id,
+                    central_id=app_user.central_id,
+                    app_user_name=app_user.name,
+                )
+        # FormTemplate
+        central_forms = client.odk_publish.get_forms()
+        central_form_templates = client.odk_publish.find_form_templates(
+            app_users=central_app_users, forms=central_forms
+        )
+        for form_id_base, app_users in central_form_templates.items():
+            form_name = app_users[0].forms[0].name.split("[")[0].strip()
+            form_template, created = project.form_templates.get_or_create(
+                form_id_base=form_id_base,
+                defaults={"title_base": form_name},
+            )
+            logger.info(
+                f"{'Created' if created else 'Retrieved'} FormTemplate",
+                id=form_template.id,
+                form_id_base=form_template.form_id_base,
+                form_title_base=form_template.title_base,
+            )
+            for app_user in app_users:
+                app_user_form, created = form_template.app_user_forms.get_or_create(
+                    app_user=project.app_users.get(central_id=app_user.id),
+                )
+                logger.info(
+                    f"{'Created' if created else 'Retrieved'} AppUserFormTemplate",
+                    id=app_user_form.id,
+                )
+
+    return project

apps/odk_publish/etl/odk/client.py

@@ -1,13 +1,13 @@
-import structlog
+import os
 from pathlib import Path
 
-from django.conf import settings
+import structlog
+from pydantic import BaseModel, SecretStr, field_validator
 from pyodk._utils import config
 from pyodk.client import Client, Session
 
 from .publish import PublishService
 
-
 logger = structlog.getLogger(__name__)
 
 CONFIG_TOML = """
@@ -19,6 +19,19 @@
 """
 
 
+class CentralConfig(BaseModel):
+    """Model to validate ODK Central server configuration."""
+
+    base_url: str
+    username: str
+    password: SecretStr
+
+    @field_validator("base_url")
+    @classmethod
+    def always_strip_trailing_slash(cls, value: str) -> str:
+        return value.rstrip("/")
+
+
 class ODKPublishClient(Client):
     """Extended pyODK Client for interacting with ODK Central."""
 
@@ -30,11 +43,12 @@ def __init__(self, base_url: str, project_id: int | None = None):
             config_path.write_text(CONFIG_TOML)
         # Create a session with the given authentication details and supply the
         # session to the super class, so it doesn't try and create one itself
+        server_config = self.get_config(base_url=base_url)
         session = Session(
-            base_url=base_url,
+            base_url=server_config.base_url,
             api_version="v1",
-            username=settings.ODK_CENTRAL_USERNAME,
-            password=settings.ODK_CENTRAL_PASSWORD,
+            username=server_config.username,
+            password=server_config.password.get_secret_value(),
             cache_path=None,
         )
         super().__init__(config_path=str(config_path), session=session, project_id=project_id)
@@ -44,8 +58,8 @@ def __init__(self, base_url: str, project_id: int | None = None):
             {
                 "central": {
                     "base_url": base_url,
-                    "username": settings.ODK_CENTRAL_USERNAME,
-                    "password": settings.ODK_CENTRAL_PASSWORD,
+                    "username": server_config.username,
+                    "password": server_config.password.get_secret_value(),
                 }
             }
         )
@@ -56,3 +70,38 @@ def __init__(self, base_url: str, project_id: int | None = None):
 
     def __enter__(self) -> "ODKPublishClient":
         return super().__enter__()  # type: ignore
+
+    @classmethod
+    def get_config(cls, base_url: str) -> CentralConfig:
+        """Return the CentralConfig for the matching base URL."""
+        available_configs = cls.get_configs()
+        config = available_configs[base_url.rstrip("/")]
+        logger.debug("Retrieved ODK Central config", config=config)
+        return config
+
+    @staticmethod
+    def get_configs() -> dict[str, CentralConfig]:
+        """
+        Parse the ODK_CENTRAL_CREDENTIALS environment variable and return a dictionary
+        of available server configurations.
+
+        Example environment variable:
+            export ODK_CENTRAL_CREDENTIALS="base_url=https://myserver.com;username=user1;password=pass1,base_url=https://otherserver.com;username=user2;password=pass2"
+
+            Returns:
+                {
+                    "https://myserver.com": CentralConfig(base_url="https://myserver.com", username="user1", password=SecretStr('**********')
+                    "https://otherserver.com": CentralConfig(base_url="https://otherserver.com", username="user2", password=SecretStr('**********')
+                }
+        """  # noqa
+        servers = {}
+        for server in os.getenv("ODK_CENTRAL_CREDENTIALS", "").split(","):
+            server = server.split(";")
+            server = {
+                key: value for key, value in [item.split("=") for item in server if "=" in item]
+            }
+            if server:
+                config = CentralConfig.model_validate(server)
+                servers[config.base_url] = config
+        logger.debug("Parsed ODK Central credentials", servers=list(servers.keys()))
+        return servers

apps/odk_publish/etl/odk/publish.py

@@ -1,4 +1,5 @@
 import datetime as dt
+from collections import defaultdict
 from os import PathLike
 from typing import TYPE_CHECKING
 
@@ -21,6 +22,7 @@
 class ProjectAppUserAssignment(ProjectAppUser):
     """Extended ProjectAppUser with additional form_ids attribute."""
 
+    forms: list[Form] = []
     xml_form_ids: list[str] = []
 
 
@@ -84,6 +86,24 @@ def get_forms(self, project_id: int | None = None) -> dict[str, Form]:
         forms = self.client.forms.list(project_id=project_id)
         return {form.xmlFormId: form for form in forms}
 
+    def find_form_templates(
+        self, app_users: dict[str, ProjectAppUserAssignment], forms: dict[str, Form]
+    ) -> dict[str, list[ProjectAppUserAssignment]]:
+        """Return form templates for the given app users and forms."""
+
+        form_templates = defaultdict(list)
+        for form in forms.values():
+            try:
+                xml_form_id_base, maybe_app_user = form.xmlFormId.rsplit("_", 1)
+            except ValueError:
+                continue
+            if app_user := app_users.get(maybe_app_user):
+                user = app_user.model_copy(deep=True)
+                user.forms.append(form)
+                form_templates[xml_form_id_base].append(user)
+        logger.info("Found form templates", form_templates=list(form_templates.keys()))
+        return form_templates
+
     def get_unique_version_by_form_id(self, xml_form_id_base: str, project_id: int | None = None):
         """
         Generates a new, unique version for the form whose xmlFormId starts with

apps/odk_publish/forms.py

@@ -0,0 +1,56 @@
+from django import forms
+from django.http import QueryDict
+from django.urls import reverse_lazy
+
+from apps.patterns.forms import PlatformFormMixin
+from apps.patterns.widgets import Select
+
+from .etl.odk.client import ODKPublishClient
+from .http import HttpRequest
+
+
+class ProjectSyncForm(PlatformFormMixin, forms.Form):
+    """Form for syncing projects from an ODK Central server.
+
+    In addition to processing the form normally, this form also handles
+    render logic for the project field during an HTMX request.
+    """
+
+    server = forms.ChoiceField(
+        # When a server is selected, the project field below is populated with
+        # the available projects for that server using HMTX.
+        widget=Select(
+            attrs={
+                "hx-trigger": "change",
+                "hx-get": reverse_lazy("odk_publish:server-sync-projects"),
+                "hx-target": "#id_project_container",
+                "hx-swap": "innerHTML",
+                "hx-indicator": ".loading",
+            }
+        ),
+    )
+    project = forms.ChoiceField(widget=Select(attrs={"disabled": "disabled"}))
+
+    def __init__(self, request: HttpRequest, data: QueryDict, *args, **kwargs):
+        htmx_data = data.copy() if request.htmx else {}
+        # Don't bind the form on an htmx request, otherwise we'll see "This
+        # field is required" errors
+        data = data if not request.htmx else None
+        super().__init__(data, *args, **kwargs)
+        # The server field is populated with the available ODK Central servers
+        # (from an environment variable) when the form is rendered. Loaded here to
+        # avoid fetching during the project initialization sequence.
+        self.fields["server"].choices = [("", "Select an ODK Central server...")] + [
+            (config.base_url, config.base_url) for config in ODKPublishClient.get_configs().values()
+        ]
+        # Set `project` field choices when a server is provided either via a
+        # POST or HTMX request
+        if server := htmx_data.get("server") or self.data.get("server"):
+            self.set_project_choices(base_url=server)
+            self.fields["project"].widget.attrs.pop("disabled", None)
+
+    def set_project_choices(self, base_url: str):
+        with ODKPublishClient(base_url=base_url) as client:
+            self.fields["project"].choices = [
+                (project.id, project.name) for project in client.projects.list()
+            ]

apps/odk_publish/http.py

@@ -0,0 +1,15 @@
+from django.db.models import QuerySet
+from django.http import HttpRequest as HttpRequestBase
+from django_htmx.middleware import HtmxDetails
+
+from .models import Project
+from .nav import Breadcrumbs
+
+
+class HttpRequest(HttpRequestBase):
+    """Custom HttpRequest class for type-checking purposes."""
+
+    htmx: HtmxDetails
+    odk_project = Project | None
+    odk_project_tabs = Breadcrumbs | None
+    odk_projects = QuerySet[Project] | None

apps/odk_publish/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.1.4 on 2025-01-07 18:43
+# Generated by Django 5.1.4 on 2025-01-21 18:01
 
 import django.core.validators
 import django.db.models.deletion
@@ -26,6 +26,13 @@ class Migration(migrations.Migration):
                 ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
                 ("modified_at", models.DateTimeField(auto_now=True, db_index=True)),
                 ("name", models.CharField(max_length=255)),
+                (
+                    "central_id",
+                    models.PositiveIntegerField(
+                        help_text="The ID of this app user in ODK Central.",
+                        verbose_name="app user ID",
+                    ),
+                ),
                 (
                     "qr_code",
                     models.ImageField(
@@ -64,7 +71,7 @@ class Migration(migrations.Migration):
                 ("modified_at", models.DateTimeField(auto_now=True, db_index=True)),
                 ("title_base", models.CharField(max_length=255)),
                 ("form_id_base", models.CharField(max_length=255)),
-                ("template_url", models.URLField(max_length=1024)),
+                ("template_url", models.URLField(blank=True, max_length=1024)),
             ],
             options={
                 "abstract": False,
@@ -172,7 +179,13 @@ class Migration(migrations.Migration):
                 ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
                 ("modified_at", models.DateTimeField(auto_now=True, db_index=True)),
                 ("name", models.CharField(max_length=255)),
-                ("project_id", models.PositiveIntegerField(verbose_name="project ID")),
+                (
+                    "central_id",
+                    models.PositiveIntegerField(
+                        help_text="The ID of this project in ODK Central.",
+                        verbose_name="project ID",
+                    ),
+                ),
                 (
                     "central_server",
                     models.ForeignKey(
@@ -275,6 +288,7 @@ class Migration(migrations.Migration):
             model_name="appuser",
             name="template_variables",
             field=models.ManyToManyField(
+                blank=True,
                 related_name="app_users",
                 through="odk_publish.AppUserTemplateVariable",
                 to="odk_publish.templatevariable",