create sync project page

apps/odk_publish/admin.py

@@ -23,6 +23,7 @@
 class CentralServerAdmin(admin.ModelAdmin):
     list_display = ("base_url", "created_at", "modified_at")
     search_fields = ("base_url",)
+    ordering = ("base_url",)
 
 
 @admin.register(TemplateVariable)

apps/odk_publish/etl/foo.py

@@ -0,0 +1,91 @@
+from collections import defaultdict
+
+
+def find_common_prefixes(strings):
+    prefix_dict = defaultdict(int)
+
+    for string in strings:
+        parts = string.split("_")
+        for i in range(1, len(parts) + 1):
+            prefix = "_".join(parts[:i])
+            prefix_dict[prefix] += 1
+
+    common_prefixes = [prefix for prefix, count in prefix_dict.items() if count > 1]
+    return common_prefixes
+
+
+# Example usage
+strings = [
+    "basic",
+    "card_dist_offline",
+    "clear_password",
+    "choice_name_test",
+    "RTL Alignment Test",
+    "staff_registration_center_11030",
+    "staff_registration_center_11031",
+    "staff_registration_center_11032",
+    "staff_registration_center_11033",
+    "staff_registration_center_11034",
+    "staff_registration_center_11035",
+    "staff_registration_center_11036",
+    "staff_registration_center_11038",
+    "staff_registration_center_11039",
+    "staff_registration_center_16005",
+    "staff_registration_center_32038",
+    "voter_registration",
+    "biomini_test",
+    "device_testing",
+    "device_testing_center_11030",
+    "device_testing_center_11031",
+    "device_testing_center_11032",
+    "device_testing_center_11033",
+    "device_testing_center_11034",
+    "device_testing_center_11035",
+    "device_testing_center_11036",
+    "device_testing_center_11037",
+    "device_testing_center_11038",
+    "device_testing_center_11039",
+    "device_testing_center_11046",
+    "device_testing_center_32038",
+    "device_testing_center_gabby-test",
+    "device_testing_test",
+    "device_testing_11030",
+    "device_testing_11031",
+    "device_testing_11032",
+    "device_testing_11033",
+    "device_testing_11034",
+    "device_testing_11035",
+    "device_testing_11036",
+    "device_testing_11037",
+    "device_testing_11038",
+    "device_testing_11039",
+    "device_testing_11046",
+    "device_testing_32038",
+    "device_testing_gabby-test",
+    "polling_module",
+    "polling_module_11030",
+    "polling_module_11031",
+    "polling_module_11032",
+    "polling_module_11034",
+    "polling_module_11035",
+    "polling_module_11036",
+    "polling_module_11037",
+    "polling_module_11038",
+    "polling_module_11039",
+    "polling_module_test",
+    "undist_card_recon",
+    "voter_card_dist",
+    "card_distribution_center_11030",
+    "card_distribution_center_11031",
+    "card_distribution_center_11032",
+    "card_distribution_center_11034",
+    "card_distribution_center_11035",
+    "card_distribution_center_11036",
+    "card_distribution_center_11037",
+    "card_distribution_center_11038",
+    "card_distribution_center_11039",
+    "card_distribution_center_11046",
+    "card_distribution_center_32038",
+]
+
+print(find_common_prefixes(strings))

apps/odk_publish/etl/load.py

@@ -2,9 +2,10 @@
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db.models import QuerySet
 
-from ..models import AppUser, AppUserFormTemplate, FormTemplate, Project
+from ..models import AppUser, AppUserFormTemplate, CentralServer, FormTemplate, Project
 from .odk.client import ODKPublishClient
 from .odk.qrcode import create_app_user_qrcode
+from .transform import group_by_common_prefixes
 
 logger = structlog.getLogger(__name__)
 
@@ -51,3 +52,64 @@ def generate_and_save_app_user_collect_qrcodes(project: Project):
             app_user.qr_code.save(
                 f"{app_user.name}.png", SimpleUploadedFile("qrcode.png", image.getvalue())
             )
+
+
+def sync_central_project(base_url: str, project_id: int):
+    """Sync a project from ODK Central to the local database."""
+    config = ODKPublishClient.get_config(base_url=base_url)
+    with ODKPublishClient(base_url=config.base_url, project_id=project_id) as client:
+        # CentralServer
+        server, created = CentralServer.objects.get_or_create(base_url=base_url)
+        logger.debug(
+            f"{'Created' if created else 'Retrieved'} CentralServer", base_url=server.base_url
+        )
+        # Project
+        central_project = client.projects.get()
+        project, created = Project.objects.get_or_create(
+            project_id=central_project.id,
+            central_server=server,
+            defaults={"name": central_project.name},
+        )
+        logger.info(
+            f"{'Created' if created else 'Retrieved'} Project",
+            id=project.id,
+            project_id=project.project_id,
+            project_name=project.name,
+        )
+        # AppUser
+        central_app_users = client.odk_publish.get_app_users()
+        for central_app_user in central_app_users.values():
+            if not central_app_user.deletedAt:
+                app_user, created = project.app_users.get_or_create(
+                    app_user_id=central_app_user.id,
+                    defaults={"name": central_app_user.displayName},
+                )
+                logger.info(
+                    f"{'Created' if created else 'Retrieved'} AppUser",
+                    id=app_user.id,
+                    app_user_id=app_user.app_user_id,
+                    app_user_name=app_user.name,
+                )
+        # FormTemplate
+        central_forms = client.odk_publish.get_forms()
+        possible_template_ids = group_by_common_prefixes(strings=central_forms.keys())
+        for central_form in central_forms.values():
+            for template_id, form_ids in possible_template_ids.items():
+                if central_form.xmlFormId in form_ids:
+                    break
+            form, created = project.form_templates.get_or_create(
+                form_id_base=central_form.xmlFormId,
+                defaults={
+                    "title_base": central_form.name,
+                },
+            )
+            logger.info(
+                f"{'Created' if created else 'Retrieved'} FormTemplate",
+                id=form.id,
+                form_id_base=form.form_id_base,
+                form_title_base=form.title_base,
+            )
+        for central_form in central_forms.values():
+            print(central_form.xmlFormId)
+        raise ValueError
+    return project

apps/odk_publish/etl/odk/client.py

@@ -103,5 +103,5 @@ def get_configs() -> dict[str, CentralConfig]:
             if server:
                 config = CentralConfig.model_validate(server)
                 servers[config.base_url] = config
-        logger.debug("Parsed ODK Central credentials", servers=servers.keys())
+        logger.debug("Parsed ODK Central credentials", servers=list(servers.keys()))
         return servers

apps/odk_publish/etl/transform.py

@@ -1,4 +1,5 @@
 import io
+from collections import defaultdict
 
 import openpyxl
 from django.core.files.uploadedfile import SimpleUploadedFile
@@ -44,3 +45,15 @@ def render_template_for_app_user(
         content=buffer.read(),
         content_type=ExportFormat.EXCEL,
     )
+
+
+def group_by_common_prefixes(strings) -> dict[str, list[str]]:
+    """Return strings grouped by their common prefixes. For example,
+    staff_registration_10000 and staff_registration_10001 would be grouped under
+    staff_registration.
+    """
+    prefixes = defaultdict(list)
+    for string in strings:
+        parts = string.rsplit("_", 1)
+        prefixes[parts[0]].append(string)
+    return prefixes

apps/odk_publish/forms.py

@@ -0,0 +1,56 @@
+from django import forms
+from django.http import QueryDict
+from django.urls import reverse_lazy
+
+from apps.patterns.forms import PlatformFormMixin
+from apps.patterns.widgets import Select
+
+from .etl.odk.client import ODKPublishClient
+from .http import HttpRequest
+
+
+class ProjectSyncForm(PlatformFormMixin, forms.Form):
+    """Form for syncing projects from an ODK Central server.
+
+    In addition to processing the form normally, this form also handles
+    render logic for the project field during an HTMX request.
+    """
+
+    server = forms.ChoiceField(
+        # The server field is populated with the available ODK Central servers
+        # (from an environment variable) when the form is rendered.
+        choices=[("", "Select an ODK Central server...")]
+        + [
+            (config.base_url, config.base_url) for config in ODKPublishClient.get_configs().values()
+        ],
+        # When a server is selected, the project field below is populated with
+        # the available projects for that server using HMTX.
+        widget=Select(
+            attrs={
+                "hx-trigger": "change",
+                "hx-get": reverse_lazy("odk_publish:server-sync-projects"),
+                "hx-target": "#id_project_container",
+                "hx-swap": "innerHTML",
+                "hx-indicator": ".loading",
+            }
+        ),
+    )
+    project = forms.ChoiceField(widget=Select(attrs={"disabled": "disabled"}))
+
+    def __init__(self, request: HttpRequest, data: QueryDict, *args, **kwargs):
+        htmx_data = data.copy() if request.htmx else {}
+        # Don't bind the form on an htmx request, otherwise we'll see "This
+        # field is required" errors
+        data = data if not request.htmx else None
+        super().__init__(data, *args, **kwargs)
+        # Set `project` field choices when a server is provided either via a
+        # POST or HTMX request
+        if server := htmx_data.get("server") or self.data.get("server"):
+            self.set_project_choices(base_url=server)
+            self.fields["project"].widget.attrs.pop("disabled", None)
+
+    def set_project_choices(self, base_url: str):
+        with ODKPublishClient(base_url=base_url) as client:
+            self.fields["project"].choices = [
+                (project.id, project.name) for project in client.projects.list()
+            ]

apps/odk_publish/http.py

@@ -0,0 +1,15 @@
+from django.db.models import QuerySet
+from django.http import HttpRequest as HttpRequestBase
+from django_htmx.middleware import HtmxDetails
+
+from .models import Project
+from .nav import Breadcrumbs
+
+
+class HttpRequest(HttpRequestBase):
+    """Custom HttpRequest class for type-checking purposes."""
+
+    htmx: HtmxDetails
+    odk_project = Project | None
+    odk_project_tabs = Breadcrumbs | None
+    odk_projects = QuerySet[Project] | None

apps/odk_publish/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.1.4 on 2025-01-07 18:43
+# Generated by Django 5.1.4 on 2025-01-16 22:08
 
 import django.core.validators
 import django.db.models.deletion
@@ -26,6 +26,13 @@ class Migration(migrations.Migration):
                 ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
                 ("modified_at", models.DateTimeField(auto_now=True, db_index=True)),
                 ("name", models.CharField(max_length=255)),
+                (
+                    "app_user_id",
+                    models.PositiveIntegerField(
+                        help_text="The internal ID of this app user in ODK Central.",
+                        verbose_name="app user ID",
+                    ),
+                ),
                 (
                     "qr_code",
                     models.ImageField(
@@ -275,6 +282,7 @@ class Migration(migrations.Migration):
             model_name="appuser",
             name="template_variables",
             field=models.ManyToManyField(
+                blank=True,
                 related_name="app_users",
                 through="odk_publish.AppUserTemplateVariable",
                 to="odk_publish.templatevariable",

apps/odk_publish/models.py

@@ -32,6 +32,10 @@ def __str__(self):
         parsed_url = urlparse(self.base_url)
         return parsed_url.netloc
 
+    def save(self, *args, **kwargs):
+        self.base_url = self.base_url.rstrip("/")
+        super().save(*args, **kwargs)
+
 
 class TemplateVariable(AbstractBaseModel):
     name = models.CharField(
@@ -167,12 +171,15 @@ def __str__(self):
 
 class AppUser(AbstractBaseModel):
     name = models.CharField(max_length=255)
+    app_user_id = models.PositiveIntegerField(
+        verbose_name="app user ID", help_text="The internal ID of this app user in ODK Central."
+    )
     project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="app_users")
     qr_code = models.ImageField(
         verbose_name="QR Code", upload_to="qr-codes/", blank=True, null=True
     )
     template_variables = models.ManyToManyField(
-        through=AppUserTemplateVariable, to=TemplateVariable, related_name="app_users"
+        through=AppUserTemplateVariable, to=TemplateVariable, related_name="app_users", blank=True
     )
 
     class Meta:

apps/odk_publish/urls.py

@@ -4,6 +4,16 @@
 
 app_name = "odk_publish"
 urlpatterns = [
+    path(
+        "servers/sync/",
+        views.server_sync,
+        name="server-sync",
+    ),
+    path(
+        "servers/sync/projects/",
+        views.server_sync_projects,
+        name="server-sync-projects",
+    ),
     path(
         "<int:odk_project_pk>/app-users/",
         views.app_user_list,

apps/odk_publish/views.py

@@ -7,7 +7,8 @@
 from django.shortcuts import get_object_or_404, redirect, render
 from django.views.decorators.http import require_http_methods
 
-from .etl.load import generate_and_save_app_user_collect_qrcodes
+from .etl.load import generate_and_save_app_user_collect_qrcodes, sync_central_project
+from .forms import ProjectSyncForm
 from .models import FormTemplateVersion, FormTemplate
 from .nav import Breadcrumbs
 from .tables import FormTemplateTable
@@ -16,6 +17,34 @@
 logger = logging.getLogger(__name__)
 
 
+@login_required
+@transaction.atomic
+def server_sync(request: HttpRequest):
+    form = ProjectSyncForm(request=request, data=request.POST or None)
+    if request.method == "POST" and form.is_valid():
+        sync_central_project(
+            base_url=form.cleaned_data["server"], project_id=form.cleaned_data["project"]
+        )
+        messages.add_message(request, messages.SUCCESS, "Project synced.")
+        return redirect(
+            "odk_publish:form-template-list", odk_project_pk=form.cleaned_data["project"]
+        )
+    context = {
+        "form": form,
+        "breadcrumbs": Breadcrumbs.from_items(
+            request=request,
+            items=[("Sync Project", "sync-project")],
+        ),
+    }
+    return render(request, "odk_publish/project_sync.html", context)
+
+
+@login_required
+def server_sync_projects(request: HttpRequest):
+    form = ProjectSyncForm(request=request, data=request.GET or None)
+    return render(request, "odk_publish/project_sync.html#project-select-partial", {"form": form})
+
+
 @login_required
 def app_user_list(request: HttpRequest, odk_project_pk):
     app_users = request.odk_project.app_users.prefetch_related("app_user_forms__form_template")