create sync project page

apps/odk_publish/admin.py

@@ -23,6 +23,7 @@
 class CentralServerAdmin(admin.ModelAdmin):
     list_display = ("base_url", "created_at", "modified_at")
     search_fields = ("base_url",)
+    ordering = ("base_url",)
 
 
 @admin.register(TemplateVariable)

apps/odk_publish/etl/load.py

@@ -2,9 +2,10 @@
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db.models import QuerySet
 
-from ..models import AppUser, AppUserFormTemplate, FormTemplate, Project
+from ..models import AppUser, AppUserFormTemplate, CentralServer, FormTemplate, Project
 from .odk.client import ODKPublishClient
 from .odk.qrcode import create_app_user_qrcode
+from .transform import group_by_common_prefixes
 
 logger = structlog.getLogger(__name__)
 
@@ -51,3 +52,64 @@ def generate_and_save_app_user_collect_qrcodes(project: Project):
             app_user.qr_code.save(
                 f"{app_user.name}.png", SimpleUploadedFile("qrcode.png", image.getvalue())
             )
+
+
+def sync_central_project(base_url: str, project_id: int):
+    """Sync a project from ODK Central to the local database."""
+    config = ODKPublishClient.get_config(base_url=base_url)
+    with ODKPublishClient(base_url=config.base_url, project_id=project_id) as client:
+        # CentralServer
+        server, created = CentralServer.objects.get_or_create(base_url=base_url)
+        logger.debug(
+            f"{'Created' if created else 'Retrieved'} CentralServer", base_url=server.base_url
+        )
+        # Project
+        central_project = client.projects.get()
+        project, created = Project.objects.get_or_create(
+            project_id=central_project.id,
+            central_server=server,
+            defaults={"name": central_project.name},
+        )
+        logger.info(
+            f"{'Created' if created else 'Retrieved'} Project",
+            id=project.id,
+            project_id=project.project_id,
+            project_name=project.name,
+        )
+        # AppUser
+        central_app_users = client.odk_publish.get_app_users()
+        for central_app_user in central_app_users.values():
+            if not central_app_user.deletedAt:
+                app_user, created = project.app_users.get_or_create(
+                    app_user_id=central_app_user.id,
+                    defaults={"name": central_app_user.displayName},
+                )
+                logger.info(
+                    f"{'Created' if created else 'Retrieved'} AppUser",
+                    id=app_user.id,
+                    app_user_id=app_user.app_user_id,
+                    app_user_name=app_user.name,
+                )
+        # FormTemplate
+        central_forms = client.odk_publish.get_forms()
+        possible_template_ids = group_by_common_prefixes(strings=central_forms.keys())
+        for central_form in central_forms.values():
+            for template_id, form_ids in possible_template_ids.items():
+                if central_form.xmlFormId in form_ids:
+                    break
+            form, created = project.form_templates.get_or_create(
+                form_id_base=central_form.xmlFormId,
+                defaults={
+                    "title_base": central_form.name,
+                },
+            )
+            logger.info(
+                f"{'Created' if created else 'Retrieved'} FormTemplate",
+                id=form.id,
+                form_id_base=form.form_id_base,
+                form_title_base=form.title_base,
+            )
+        for central_form in central_forms.values():
+            print(central_form.xmlFormId)
+        raise ValueError
+    return project

apps/odk_publish/etl/odk/client.py

@@ -103,5 +103,5 @@ def get_configs() -> dict[str, CentralConfig]:
             if server:
                 config = CentralConfig.model_validate(server)
                 servers[config.base_url] = config
-        logger.debug("Parsed ODK Central credentials", servers=servers.keys())
+        logger.debug("Parsed ODK Central credentials", servers=list(servers.keys()))
         return servers

apps/odk_publish/etl/transform.py

@@ -1,4 +1,5 @@
 import io
+from collections import defaultdict
 
 import openpyxl
 from django.core.files.uploadedfile import SimpleUploadedFile
@@ -44,3 +45,15 @@ def render_template_for_app_user(
         content=buffer.read(),
         content_type=ExportFormat.EXCEL,
     )
+
+
+def group_by_common_prefixes(strings) -> dict[str, list[str]]:
+    """Return strings grouped by their common prefixes. For example,
+    staff_registration_10000 and staff_registration_10001 would be grouped under
+    staff_registration.
+    """
+    prefixes = defaultdict(list)
+    for string in strings:
+        parts = string.rsplit("_", 1)
+        prefixes[parts[0]].append(string)
+    return prefixes

apps/odk_publish/forms.py

@@ -0,0 +1,56 @@
+from django import forms
+from django.http import QueryDict
+from django.urls import reverse_lazy
+
+from apps.patterns.forms import PlatformFormMixin
+from apps.patterns.widgets import Select
+
+from .etl.odk.client import ODKPublishClient
+from .http import HttpRequest
+
+
+class ProjectSyncForm(PlatformFormMixin, forms.Form):
+    """Form for syncing projects from an ODK Central server.
+
+    In addition to processing the form normally, this form also handles
+    render logic for the project field during an HTMX request.
+    """
+
+    server = forms.ChoiceField(
+        # The server field is populated with the available ODK Central servers
+        # (from an environment variable) when the form is rendered.
+        choices=[("", "Select an ODK Central server...")]
+        + [
+            (config.base_url, config.base_url) for config in ODKPublishClient.get_configs().values()
+        ],
+        # When a server is selected, the project field below is populated with
+        # the available projects for that server using HMTX.
+        widget=Select(
+            attrs={
+                "hx-trigger": "change",
+                "hx-get": reverse_lazy("odk_publish:server-sync-projects"),
+                "hx-target": "#id_project_container",
+                "hx-swap": "innerHTML",
+                "hx-indicator": ".loading",
+            }
+        ),
+    )
+    project = forms.ChoiceField(widget=Select(attrs={"disabled": "disabled"}))
+
+    def __init__(self, request: HttpRequest, data: QueryDict, *args, **kwargs):
+        htmx_data = data.copy() if request.htmx else {}
+        # Don't bind the form on an htmx request, otherwise we'll see "This
+        # field is required" errors
+        data = data if not request.htmx else None
+        super().__init__(data, *args, **kwargs)
+        # Set `project` field choices when a server is provided either via a
+        # POST or HTMX request
+        if server := htmx_data.get("server") or self.data.get("server"):
+            self.set_project_choices(base_url=server)
+            self.fields["project"].widget.attrs.pop("disabled", None)
+
+    def set_project_choices(self, base_url: str):
+        with ODKPublishClient(base_url=base_url) as client:
+            self.fields["project"].choices = [
+                (project.id, project.name) for project in client.projects.list()
+            ]

apps/odk_publish/http.py

@@ -0,0 +1,15 @@
+from django.db.models import QuerySet
+from django.http import HttpRequest as HttpRequestBase
+from django_htmx.middleware import HtmxDetails
+
+from .models import Project
+from .nav import Breadcrumbs
+
+
+class HttpRequest(HttpRequestBase):
+    """Custom HttpRequest class for type-checking purposes."""
+
+    htmx: HtmxDetails
+    odk_project = Project | None
+    odk_project_tabs = Breadcrumbs | None
+    odk_projects = QuerySet[Project] | None

apps/odk_publish/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.1.4 on 2025-01-07 18:43
+# Generated by Django 5.1.4 on 2025-01-16 22:08
 
 import django.core.validators
 import django.db.models.deletion
@@ -26,6 +26,13 @@ class Migration(migrations.Migration):
                 ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
                 ("modified_at", models.DateTimeField(auto_now=True, db_index=True)),
                 ("name", models.CharField(max_length=255)),
+                (
+                    "app_user_id",
+                    models.PositiveIntegerField(
+                        help_text="The internal ID of this app user in ODK Central.",
+                        verbose_name="app user ID",
+                    ),
+                ),
                 (
                     "qr_code",
                     models.ImageField(
@@ -275,6 +282,7 @@ class Migration(migrations.Migration):
             model_name="appuser",
             name="template_variables",
             field=models.ManyToManyField(
+                blank=True,
                 related_name="app_users",
                 through="odk_publish.AppUserTemplateVariable",
                 to="odk_publish.templatevariable",

apps/odk_publish/models.py

@@ -32,6 +32,10 @@ def __str__(self):
         parsed_url = urlparse(self.base_url)
         return parsed_url.netloc
 
+    def save(self, *args, **kwargs):
+        self.base_url = self.base_url.rstrip("/")
+        super().save(*args, **kwargs)
+
 
 class TemplateVariable(AbstractBaseModel):
     name = models.CharField(
@@ -167,12 +171,15 @@ def __str__(self):
 
 class AppUser(AbstractBaseModel):
     name = models.CharField(max_length=255)
+    app_user_id = models.PositiveIntegerField(
+        verbose_name="app user ID", help_text="The internal ID of this app user in ODK Central."
+    )
     project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="app_users")
     qr_code = models.ImageField(
         verbose_name="QR Code", upload_to="qr-codes/", blank=True, null=True
     )
     template_variables = models.ManyToManyField(
-        through=AppUserTemplateVariable, to=TemplateVariable, related_name="app_users"
+        through=AppUserTemplateVariable, to=TemplateVariable, related_name="app_users", blank=True
     )
 
     class Meta:

apps/odk_publish/urls.py

@@ -4,6 +4,16 @@
 
 app_name = "odk_publish"
 urlpatterns = [
+    path(
+        "servers/sync/",
+        views.server_sync,
+        name="server-sync",
+    ),
+    path(
+        "servers/sync/projects/",
+        views.server_sync_projects,
+        name="server-sync-projects",
+    ),
     path(
         "<int:odk_project_pk>/app-users/",
         views.app_user_list,

apps/odk_publish/views.py

@@ -7,7 +7,8 @@
 from django.shortcuts import get_object_or_404, redirect, render
 from django.views.decorators.http import require_http_methods
 
-from .etl.load import generate_and_save_app_user_collect_qrcodes
+from .etl.load import generate_and_save_app_user_collect_qrcodes, sync_central_project
+from .forms import ProjectSyncForm
 from .models import FormTemplateVersion, FormTemplate
 from .nav import Breadcrumbs
 from .tables import FormTemplateTable
@@ -16,6 +17,34 @@
 logger = logging.getLogger(__name__)
 
 
+@login_required
+@transaction.atomic
+def server_sync(request: HttpRequest):
+    form = ProjectSyncForm(request=request, data=request.POST or None)
+    if request.method == "POST" and form.is_valid():
+        sync_central_project(
+            base_url=form.cleaned_data["server"], project_id=form.cleaned_data["project"]
+        )
+        messages.add_message(request, messages.SUCCESS, "Project synced.")
+        return redirect(
+            "odk_publish:form-template-list", odk_project_pk=form.cleaned_data["project"]
+        )
+    context = {
+        "form": form,
+        "breadcrumbs": Breadcrumbs.from_items(
+            request=request,
+            items=[("Sync Project", "sync-project")],
+        ),
+    }
+    return render(request, "odk_publish/project_sync.html", context)
+
+
+@login_required
+def server_sync_projects(request: HttpRequest):
+    form = ProjectSyncForm(request=request, data=request.GET or None)
+    return render(request, "odk_publish/project_sync.html#project-select-partial", {"form": form})
+
+
 @login_required
 def app_user_list(request: HttpRequest, odk_project_pk):
     app_users = request.odk_project.app_users.prefetch_related("app_user_forms__form_template")

apps/patterns/templates/patterns/forms/div.html

@@ -0,0 +1,36 @@
+{% for error in errors %}
+    <div class="p-4 mb-4 text-sm text-brand-danger-medium rounded-lg bg-red-50 dark:bg-gray-800 dark:text-brand-danger-light"
+         role="alert">{{ error }}</div>
+{% endfor %}
+asdfasdf
+{% if errors and not fields %}
+    <div>
+        {% for field in hidden_fields %}{{ field }}{% endfor %}
+    </div>
+{% endif %}
+{% for field, errors in fields %}
+    <div {% with classes=field.css_classes %}
+         {% if classes %}classfixme="{{ classes }}"{% endif %}
+         {% endwith %}
+         class="mb-6"
+         id="id_{{ field.name }}_container">
+        {% if field.use_fieldset %}
+            <fieldset>
+                {% if field.label %}{{ field.legend_tag }}{% endif %}
+            {% else %}
+                {% if field.label %}{{ field.label_tag }}{% endif %}
+            {% endif %}
+            {% if field.help_text %}<div class="helptext">{{ field.help_text|safe }}</div>{% endif %}
+            {{ field }}
+            {% for error in erors %}
+                <p class="mt-2 text-sm text-brand-danger-medium dark:text-brand-danger-light">{{ error }}</p>
+            {% endfor %}
+            {% if field.use_fieldset %}</fieldset>{% endif %}
+        {% if forloop.last %}
+            {% for field in hidden_fields %}{{ field }}{% endfor %}
+        {% endif %}
+    </div>
+{% endfor %}
+{% if not fields and not errors %}
+    {% for field in hidden_fields %}{{ field }}{% endfor %}
+{% endif %}

apps/patterns/templates/patterns/forms/label.html

@@ -2,15 +2,15 @@
     <div class="flex flex-row items-center">
         <{{ tag }}
         {% include "patterns/forms/attrs.html" %}
-        class="block mb-2 text-sm font-medium text-blue-900
+        class="block mb-2 font-medium
         {% if not field.errors %}
-            text-blue-900 dark:text-white
+            text-gray-900 dark:text-white
         {% else %}
-            text-brand-danger-medium dark:text-brand-danger-light
+            text-red
         {% endif %}
         ">
         {{ label|title }}
-        {% if field.field.required %}<span class="text-brand-danger-medium">*</span>{% endif %}
+        {% if field.field.required %}<span class="text-red">*</span>{% endif %}
         </{{ tag }}>
         {% if field.help_text %}
             <span data-popover-target="{{ attrs.for }}_popover"
@@ -35,6 +35,6 @@
 {% else %}
     <span>
         {{ label|title }}
-        {% if field.field.required %}<span class="text-brand-danger-medium">*</span>{% endif %}
+        {% if field.field.required %}<span class="text-red">*</span>{% endif %}
     </span>
 {% endif %}

apps/patterns/templates/patterns/forms/widgets/select.html

@@ -0,0 +1,11 @@
+<select name="{{ widget.name }}"
+        {% include "django/forms/widgets/attrs.html" %}
+        class="{% if 'disabled' in widget.attrs %}cursor-not-allowed {% endif %}border rounded-md block w-full p-2.5 {% if not field.errors %}bg-gray-50 border border-gray-300 text-gray-900 focus:ring-2 focus:ring-primary-200 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500{% else %} bg-red-50  border-brand-danger-medium  text-brand-danger-medium focus:ring-red-500 focus:border-brand-danger-medium dark:bg-brand-danger-light dark:border-brand-danger-light {% endif %} ">
+    {% for group_name, group_choices, group_index in widget.optgroups %}
+        {% if group_name %}<optgroup label="{{ group_name }}">{% endif %}
+            {% for option in group_choices %}
+                {% include option.template_name with widget=option %}
+            {% endfor %}
+            {% if group_name %}</optgroup>{% endif %}
+    {% endfor %}
+</select>

apps/patterns/widgets.py

@@ -8,3 +8,7 @@ class TextInput(widgets.TextInput):
 class CheckboxSelectMultiple(widgets.CheckboxSelectMultiple):
     template_name = "patterns/forms/widgets/checkbox_select.html"
     option_template_name = "patterns/forms/widgets/checkbox_option.html"
+
+
+class Select(widgets.Select):
+    template_name = "patterns/forms/widgets/select.html"