Merge branch 'development' of https://github.com/hotosm/fmtm into fea…

…t-mobile-ui-project-details

.github/workflows/build_and_deploy.yml

@@ -68,7 +68,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Environment to .env
         env:
@@ -124,7 +124,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Frontend smoke test
         run: echo "Not implemented"
@@ -140,7 +140,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Environment to .env
         env:

.github/workflows/docs.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
       - name: Set cache key
         id: set_cache_key
-        run: echo "cache_key=docs-build-$(date --utc '+%V')" >> $GITHUB_OUTPUT
+        run: echo "cache_key=docs-build-$(date --utc +'%Y-%m-%d_%H:%M:%S')" >> $GITHUB_OUTPUT
 
   build_doxygen:
     uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@main
@@ -32,7 +32,7 @@ jobs:
     uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@main
     needs: [get_cache_key]
     with:
-      image: ghcr.io/hotosm/fmtm/backend:ci-main
+      image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}
       example_env_file_path: ".env.example"
       cache_paths: |
         docs/openapi.json
@@ -45,6 +45,7 @@ jobs:
       - build_doxygen
       - build_openapi_json
     with:
+      image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}
       cache_paths: |
         docs/apidocs
         docs/openapi.json

.github/workflows/r-extract_vars.yml

@@ -34,7 +34,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Extract api version
         id: extract_api_version

.github/workflows/r-frontend_tests.yml

@@ -10,7 +10,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Node.js
         uses: actions/setup-node@v3

.github/workflows/r-pytest.yml

@@ -34,7 +34,6 @@ jobs:
         OSM_CLIENT_SECRET: ${{ env.OSM_CLIENT_SECRET }}
         OSM_SECRET_KEY: ${{ env.OSM_SECRET_KEY }}
         FRONTEND_MAIN_URL: ${{ vars.FRONTEND_MAIN_URL }}
-      options: --user root
 
     services:
       # Start backend database
@@ -76,8 +75,8 @@ jobs:
         image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Run pytest as appuser
+      - uses: actions/checkout@v4
+      - name: Run pytest
         run: |
           cd src/backend
-          gosu appuser pytest
+          pytest

.github/workflows/wiki.yml

@@ -11,3 +11,5 @@ on:
 jobs:
   publish-docs-to-wiki:
     uses: hotosm/gh-workflows/.github/workflows/wiki.yml@main
+    with:
+      homepage_path: "index.md"

README.md

@@ -264,20 +264,21 @@ Thanks goes to these wonderful people:
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/krtonga"><img src="https://avatars.githubusercontent.com/u/7307817?v=4?s=100" width="100px;" alt="krtonga"/><br /><sub><b>krtonga</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=krtonga" title="Code">💻</a> <a href="https://github.com/hotosm/fmtm/commits?author=krtonga" title="Documentation">📖</a> <a href="#tool-krtonga" title="Tools">🔧</a> <a href="#ideas-krtonga" title="Ideas, Planning, & Feedback">🤔</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/spwoodcock"><img src="https://avatars.githubusercontent.com/u/78538841?v=4?s=100" width="100px;" alt="Sam"/><br /><sub><b>Sam</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=spwoodcock" title="Code">💻</a> <a href="https://github.com/hotosm/fmtm/pulls?q=is%3Apr+reviewed-by%3Aspwoodcock" title="Reviewed Pull Requests">👀</a> <a href="#infra-spwoodcock" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#ideas-spwoodcock" title="Ideas, Planning, & Feedback">🤔</a> <a href="#maintenance-spwoodcock" title="Maintenance">🚧</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://www.hotosm.org/people/petya-kangalova/"><img src="https://avatars.githubusercontent.com/u/98902727?v=4?s=100" width="100px;" alt="Petya "/><br /><sub><b>Petya </b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=petya-kangalova" title="Documentation">📖</a> <a href="#eventOrganizing-petya-kangalova" title="Event Organizing">📋</a> <a href="#ideas-petya-kangalova" title="Ideas, Planning, & Feedback">🤔</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="http://zanrevenue.org"><img src="https://avatars.githubusercontent.com/u/52991565?v=4?s=100" width="100px;" alt="Mohamed Bakari Mohamed"/><br /><sub><b>Mohamed Bakari Mohamed</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Mudi-business" title="Code">💻</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://www.scdhub.org"><img src="https://avatars.githubusercontent.com/u/4379874?v=4?s=100" width="100px;" alt="G. Willson"/><br /><sub><b>G. Willson</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=biomassives" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/nrjadkry"><img src="https://avatars.githubusercontent.com/u/41701707?v=4?s=100" width="100px;" alt="Niraj Adhikari"/><br /><sub><b>Niraj Adhikari</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=nrjadkry" title="Code">💻</a> <a href="#ideas-nrjadkry" title="Ideas, Planning, & Feedback">🤔</a> <a href="#maintenance-nrjadkry" title="Maintenance">🚧</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/varun2948"><img src="https://avatars.githubusercontent.com/u/37866666?v=4?s=100" width="100px;" alt="Deepak Pradhan (Varun)"/><br /><sub><b>Deepak Pradhan (Varun)</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=varun2948" title="Code">💻</a> <a href="#ideas-varun2948" title="Ideas, Planning, & Feedback">🤔</a> <a href="#maintenance-varun2948" title="Maintenance">🚧</a></td>
     </tr>
     <tr>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Ndacyayisenga-droid"><img src="https://avatars.githubusercontent.com/u/58124613?v=4?s=100" width="100px;" alt="Tayebwa Noah"/><br /><sub><b>Tayebwa Noah</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Ndacyayisenga-droid" title="Documentation">📖</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/mohammadareeb95"><img src="https://avatars.githubusercontent.com/u/77102111?v=4?s=100" width="100px;" alt="Mohammad Areeb"/><br /><sub><b>Mohammad Areeb</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=mohammadareeb95" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://zanrevenue.org"><img src="https://avatars.githubusercontent.com/u/52991565?v=4?s=100" width="100px;" alt="Mohamed Bakari Mohamed"/><br /><sub><b>Mohamed Bakari Mohamed</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Mudi-business" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sujanadh"><img src="https://avatars.githubusercontent.com/u/109404840?v=4?s=100" width="100px;" alt="Sujan Adhikari"/><br /><sub><b>Sujan Adhikari</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Sujanadh" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/AugustHottie"><img src="https://avatars.githubusercontent.com/u/96122635?v=4?s=100" width="100px;" alt="AugustHottie"/><br /><sub><b>AugustHottie</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=AugustHottie" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Balofire"><img src="https://avatars.githubusercontent.com/u/102294666?v=4?s=100" width="100px;" alt="Ahmeed Etti-Balogun"/><br /><sub><b>Ahmeed Etti-Balogun</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Balofire" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Roseford"><img src="https://avatars.githubusercontent.com/u/75838716?v=4?s=100" width="100px;" alt="Uju"/><br /><sub><b>Uju</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Roseford" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/neelimagoogly"><img src="https://avatars.githubusercontent.com/u/97789856?v=4?s=100" width="100px;" alt="Neelima Mohanty"/><br /><sub><b>Neelima Mohanty</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=neelimagoogly" title="Documentation">📖</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/varun2948"><img src="https://avatars.githubusercontent.com/u/37866666?v=4?s=100" width="100px;" alt="Deepak Pradhan (Varun)"/><br /><sub><b>Deepak Pradhan (Varun)</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=varun2948" title="Code">💻</a> <a href="#ideas-varun2948" title="Ideas, Planning, & Feedback">🤔</a> <a href="#maintenance-varun2948" title="Maintenance">🚧</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://www.scdhub.org"><img src="https://avatars.githubusercontent.com/u/4379874?v=4?s=100" width="100px;" alt="G. Willson"/><br /><sub><b>G. Willson</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=biomassives" title="Code">💻</a></td>
     </tr>
     <tr>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/nrjadkry"><img src="https://avatars.githubusercontent.com/u/41701707?v=4?s=100" width="100px;" alt="Niraj Adhikari"/><br /><sub><b>Niraj Adhikari</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=nrjadkry" title="Code">💻</a> <a href="#ideas-nrjadkry" title="Ideas, Planning, & Feedback">🤔</a> <a href="#maintenance-nrjadkry" title="Maintenance">🚧</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Ndacyayisenga-droid"><img src="https://avatars.githubusercontent.com/u/58124613?v=4?s=100" width="100px;" alt="Tayebwa Noah"/><br /><sub><b>Tayebwa Noah</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Ndacyayisenga-droid" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/mohammadareeb95"><img src="https://avatars.githubusercontent.com/u/77102111?v=4?s=100" width="100px;" alt="Mohammad Areeb"/><br /><sub><b>Mohammad Areeb</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=mohammadareeb95" title="Documentation">📖</a></td>
     </tr>
   </tbody>
   <tfoot>

docker-compose.deploy.yml

@@ -40,7 +40,7 @@ services:
       - 443:443
     networks:
       - fmtm-net
-    restart: unless-stopped
+    restart: "unless-stopped"
     command:
       - "--entrypoints.web.address=:80"
       - "--entrypoints.websecure.address=:443"
@@ -75,7 +75,7 @@ services:
       - "5433:5432"
     networks:
       - fmtm-net
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
@@ -96,7 +96,7 @@ services:
       - .env
     networks:
       - fmtm-net
-    restart: unless-stopped
+    restart: "unless-stopped"
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.api.tls=true"
@@ -105,6 +105,18 @@ services:
       - "traefik.http.services.api-svc.loadbalancer.server.port=8000"
       - "traefik.http.routers.api.service=api-svc"
 
+  migrations:
+    image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
+    container_name: fmtm_migrations
+    depends_on:
+      - api
+    env_file:
+      - .env
+    networks:
+      - fmtm-net
+    command: ["alembic", "upgrade", "head"]
+    restart: "no"
+
   ui-main:
     image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}"
     build:
@@ -124,7 +136,7 @@ services:
       - BROTLI=true
       - API_URL=${URL_SCHEME}://${API_URL}
       - FRONTEND_MAIN_URL=${URL_SCHEME}://${FRONTEND_MAIN_URL}
-    restart: unless-stopped
+    restart: "unless-stopped"
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ui-main.tls=true"

docker-compose.noodk.yml

@@ -41,7 +41,7 @@ services:
       - "5433:5432"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:debug"
@@ -65,7 +65,19 @@ services:
       - "5678:5678"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
+
+  migrations:
+    image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
+    container_name: fmtm_migrations
+    depends_on:
+      - api
+    env_file:
+      - .env
+    networks:
+      - fmtm-net
+    command: ["alembic", "upgrade", "head"]
+    restart: "no"
 
   ui-main:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"
@@ -89,4 +101,4 @@ services:
       - "8081:8081"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"

docker-compose.yml

@@ -42,7 +42,7 @@ services:
       - "5438:5432"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:debug"
@@ -72,7 +72,19 @@ services:
       - "5678:5678"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
+
+  migrations:
+    image: "ghcr.io/hotosm/fmtm/backend:debug"
+    container_name: fmtm_migrations
+    depends_on:
+      - api
+    env_file:
+      - .env
+    networks:
+      - fmtm-dev
+    command: ["alembic", "upgrade", "head"]
+    restart: "no"
 
   ui-main:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"
@@ -96,7 +108,7 @@ services:
       - "8080:8080"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   central-db:
     image: "postgis/postgis:14-3.3-alpine"
@@ -111,7 +123,7 @@ services:
       - "5434:5432"
     networks:
       - fmtm-dev
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   central:
     image: "ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1"
@@ -122,7 +134,6 @@ services:
     container_name: central_api
     depends_on:
       - central-db
-      - pyxform
     environment:
       - DOMAIN=local
       - SYSADMIN_EMAIL=${ODK_CENTRAL_USER}
@@ -154,7 +165,7 @@ services:
         "--",
         "./init-user-and-start.sh",
       ]
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   central-proxy:
     image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
@@ -165,10 +176,4 @@ services:
       - central
     networks:
       - fmtm-dev
-    restart: unless-stopped
-
-  pyxform:
-    image: "ghcr.io/getodk/pyxform-http:v1.10.1.1"
-    networks:
-      - fmtm-dev
-    restart: always
+    restart: "unless-stopped"

josm/docker-compose.yml

@@ -35,7 +35,7 @@ services:
       - x11
     ports:
       - 8111:80
-    restart: unless-stopped
+    restart: "unless-stopped"
 
   josm-novnc:
     image: "ghcr.io/hotosm/fmtm/josm-novnc:latest"
@@ -51,4 +51,4 @@ services:
       - "8112:8080"
     networks:
       - x11
-    restart: unless-stopped
+    restart: "unless-stopped"

mkdocs.yml

@@ -57,6 +57,7 @@ plugins:
             merge_init_into_class: true
 
 nav:
+  - Home: index.md
   - Get Started:
       - About: About.md
       - Installation: INSTALL.md

src/backend/.dockerignore

@@ -8,3 +8,5 @@
 !container-entrypoint.sh
 !pyproject.toml
 !pdm.lock
+!migrations
+!alembic.ini

src/backend/Dockerfile

@@ -90,7 +90,6 @@ RUN set -ex \
     -y --no-install-recommends \
         "nano" \
         "curl" \
-        "gosu" \
         "libpcre3" \
         "mime-support" \
         "postgresql-client" \
@@ -108,8 +107,10 @@ COPY --from=build \
 WORKDIR /opt
 # Add app code
 COPY app/ /opt/app/
+COPY migrations/ /opt/migrations/
+COPY alembic.ini /opt/
 # Add non-root user, permissions
-RUN useradd -r -u 900 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
+RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
     && mkdir -p /opt/logs /opt/tiles \
     && chown -R appuser:appuser /opt /home/appuser \
     && chmod +x /container-entrypoint.sh
@@ -123,7 +124,6 @@ USER appuser
 
 
 FROM runtime as debug-no-odk
-USER appuser
 COPY --from=extract-deps --chown=appuser \
     /opt/python/requirements-debug.txt /opt/python/
 RUN pip install --user --upgrade --no-warn-script-location \
@@ -147,14 +147,24 @@ USER appuser
 
 
 
-FROM debug-no-odk as ci
-COPY --from=extract-deps --chown=appuser \
+FROM runtime as ci
+# Run all ci as root
+USER root
+ENV PATH="/root/.local/bin:$PATH"
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "gosu" \
+        "git" \
+    && rm -rf /var/lib/apt/lists/*
+RUN mv /home/appuser/.local /root/.local
+COPY --from=extract-deps \
     /opt/python/requirements-ci.txt /opt/python/
 RUN pip install --user --upgrade --no-warn-script-location \
     --no-cache-dir -r \
     /opt/python/requirements-ci.txt \
     && rm -r /opt/python
-COPY --chown=appuser tests/ /opt/tests/
 # Pre-compile packages to .pyc (init speed gains)
 RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
 # Override entrypoint, as not possible in Github action