Merge branch 'staging'

.all-contributorsrc

@@ -63,7 +63,7 @@
       "name": "Sujan Adhikari",
       "avatar_url": "https://avatars.githubusercontent.com/u/109404840?v=4",
       "profile": "https://github.com/Sujanadh",
-      "contributions": ["code"]
+      "contributions": ["code", "maintenance"]
     },
     {
       "login": "Mudi-business",
@@ -77,7 +77,7 @@
       "name": "Nishit Suwal",
       "avatar_url": "https://avatars.githubusercontent.com/u/81785002?v=4",
       "profile": "https://github.com/NSUWAL123",
-      "contributions": ["code"]
+      "contributions": ["code", "maintenance"]
     },
     {
       "login": "biomassives",
@@ -154,7 +154,14 @@
       "name": "Manjita Pandey",
       "avatar_url": "https://avatars.githubusercontent.com/u/97273021?v=4",
       "profile": "https://github.com/manjitapandey",
-      "contributions": ["bug"]
+      "contributions": ["bug", "doc", "ideas"]
+    },
+    {
+      "login": "susmina94",
+      "name": "Susmina_Manandhar",
+      "avatar_url": "https://avatars.githubusercontent.com/u/108750444?v=4",
+      "profile": "https://github.com/susmina94",
+      "contributions": ["doc", "ideas", "bug"]
     }
   ],
   "contributorsPerLine": 7,

.env.example

@@ -21,6 +21,7 @@ OSM_URL=${OSM_URL:-"https://www.openstreetmap.org"}
 OSM_SCOPE=${OSM_SCOPE:-"read_prefs"}
 OSM_LOGIN_REDIRECT_URI="http${FMTM_DOMAIN:+s}://${FMTM_DOMAIN:-127.0.0.1:7051}/osmauth/"
 OSM_SECRET_KEY=${OSM_SECRET_KEY}
+OSM_SVC_ACCOUNT_TOKEN=${OSM_SVC_ACCOUNT_TOKEN}
 
 ### S3 File Storage ###
 S3_ENDPOINT=${S3_ENDPOINT:-"http://s3:9000"}
@@ -41,5 +42,17 @@ FMTM_DB_USER=${FMTM_DB_USER:-"fmtm"}
 FMTM_DB_PASSWORD=${FMTM_DB_PASSWORD:-"fmtm"}
 FMTM_DB_NAME=${FMTM_DB_NAME:-"fmtm"}
 
-### Underpass (optional override) ###
-UNDERPASS_API_URL=${UNDERPASS_API_URL:-"https://api-prod.raw-data.hotosm.org/v1"}
+### raw-data-api (optional override) ###
+RAW_DATA_API_URL=${RAW_DATA_API_URL:-"https://api-prod.raw-data.hotosm.org/v1"}
+RAW_DATA_API_AUTH_TOKEN=${RAW_DATA_API_AUTH_TOKEN}
+
+# Monitoring (OpenTelemetry). Options: 'openobserve', 'sentry'.
+MONITORING=${MONITORING}
+# Monitoring (OpenObserve)
+OPENOBSERVE_USER=${OTEL_USERNAME:-${ODK_CENTRAL_USER:-"test@fmtm.dev"}}
+OPENOBSERVE_PASSWORD=${OPENOBSERVE_PASSWORD:-"testuserpassword"}
+OPENOBSERVE_RETENTION_DAYS=${OPENOBSERVE_RETENTION_DAYS:-90}
+OTEL_ENDPOINT=${OTEL_ENDPOINT}
+OTEL_AUTH_TOKEN=${OTEL_AUTH_TOKEN}
+# Monitoring (Sentry)
+SENTRY_DSN=${SENTRY_DSN}

.github/workflows/build_and_deploy.yml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.5.1
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend
@@ -29,20 +29,20 @@ jobs:
     secrets: inherit
 
   frontend-tests:
-    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.5.1
     with:
       working_dir: src/frontend
 
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     needs: [pytest]
     with:
       context: src/backend
       build_target: prod
       image_name: ghcr.io/${{ github.repository }}/backend
 
   frontend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     needs: [frontend-tests]
     with:
       context: src/frontend
@@ -67,10 +67,10 @@ jobs:
           VARS_CONTEXT: ${{ toJson(vars) }}
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
         run: |
-          # Random delimeter string for security
+          # Random delimiter string for security
           delim=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
 
-          # Parse JSON with multiline strings, using delimeter (Github specific)
+          # Parse JSON with multiline strings, using delimiter (Github specific)
           to_envs() { jq -r "to_entries[] | \"\(.key)<<$delim\n\(.value)\n$delim\n\""; }
 
           # Set all vars
@@ -86,7 +86,7 @@ jobs:
           curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst
           chmod +x envsubst
 
-          echo "Substituing variables from .env.example --> .env"
+          echo "Substituting variables from .env.example --> .env"
           ./envsubst < .env.example > .env
           echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
           echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> .env
@@ -129,7 +129,7 @@ jobs:
           curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst
           chmod +x envsubst
 
-          echo "Substituing variables from .env.example --> .env"
+          echo "Substituting variables from .env.example --> .env"
           ./envsubst < .env.example > .env
 
       - name: Frontend smoke test
@@ -152,7 +152,7 @@ jobs:
     needs:
       - smoke-test-backend
       - smoke-test-frontend
-    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.5.1
     with:
       environment: ${{ github.ref_name }}
       docker_compose_file: "docker-compose.${{ github.ref_name }}.yml"

.github/workflows/build_ci_img.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   backend-ci-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: src/backend
       build_target: ci

.github/workflows/build_odk_imgs.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   build-odkcentral:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: odkcentral/api
       image_tags: |
@@ -26,7 +26,7 @@ jobs:
       # multi_arch: true
 
   build-odkcentral-ui:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: odkcentral/ui
       image_tags: |

.github/workflows/build_proxy_imgs.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build-cert-init-main:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: nginx
       build_target: certs-init-main
@@ -21,7 +21,7 @@ jobs:
       multi_arch: true
 
   build-cert-init-dev:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: nginx
       build_target: certs-init-development
@@ -33,7 +33,7 @@ jobs:
       multi_arch: true
 
   build-proxy-main:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: nginx
       build_target: main
@@ -44,7 +44,7 @@ jobs:
       multi_arch: true
 
   build-proxy-dev:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: nginx
       build_target: development

.github/workflows/docs.yml

@@ -12,19 +12,19 @@ on:
 
 jobs:
   build_doxygen:
-    uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.5.1
     with:
       output_path: docs/apidocs
 
   build_openapi_json:
-    uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.5.1
     with:
       image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
       example_env_file_path: ".env.example"
       output_path: docs/openapi.json
 
   publish_docs:
-    uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.5.1
     needs:
       - build_doxygen
       - build_openapi_json

.github/workflows/pr_test_backend.yml

@@ -14,7 +14,7 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.5.1
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend

.github/workflows/pr_test_frontend.yml

@@ -13,7 +13,14 @@ on:
   workflow_dispatch:
 
 jobs:
-  frontend-tests:
-    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.9
+  unit-tests:
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.5.1
     with:
       working_dir: src/frontend
+
+  e2e-tests:
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main
+    with:
+      container_config: '{"image": "mcr.microsoft.com/playwright:v1.43.0"}'
+      working_dir: src/frontend
+      run_command: "test:e2e"

.github/workflows/tag_build.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.5.1
     with:
       context: src/backend
       build_target: prod

.github/workflows/tests/pytest.yml

@@ -28,10 +28,10 @@ jobs:
           VARS_CONTEXT: ${{ toJson(vars) }}
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
         run: |
-          # Random delimeter string for security
+          # Random delimiter string for security
           delim=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
 
-          # Parse JSON with multiline strings, using delimeter (Github specific)
+          # Parse JSON with multiline strings, using delimiter (Github specific)
           to_envs() { jq -r "to_entries[] | \"\(.key)<<$delim\n\(.value)\n$delim\n\""; }
 
           # Set VARS_CONTEXT if not null

.github/workflows/wiki.yml

@@ -10,6 +10,6 @@ on:
 
 jobs:
   publish-docs-to-wiki:
-    uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.4.9
+    uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.5.1
     with:
       homepage_path: "wiki_redirect.md"

.gitignore

@@ -21,6 +21,7 @@ env/
 build/
 develop-eggs/
 dist/
+dev-dist/
 downloads/
 eggs/
 .eggs/
@@ -89,3 +90,6 @@ envsubst
 
 # Scripts
 **/openapi.json
+
+# Chart dependencies
+chart/charts

.pre-commit-config.yaml

@@ -19,6 +19,21 @@ repos:
       - id: ruff-format
         files: ^src/backend/(?:.*/)*.*$
 
+  # # Dead code: Python usused code detection
+  # - repo: https://github.com/asottile/dead
+  #   rev: v1.5.2
+  #   hooks:
+  #     - id: dead
+  #       args: ["--exclude", "contrib"]
+
+  # Spelling: Checks for common misspellings in text files.
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        types: [text]
+        args: [--toml, src/backend/pyproject.toml]
+
   # Autoformat: YAML, JSON, Markdown, etc.
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.1.0
@@ -48,6 +63,17 @@ repos:
   #           "--ignore=DL3059",
   #         ]
 
+  # Lint & Autoformat: SQL
+  - repo: https://github.com/sqlfluff/sqlfluff
+    rev: 3.0.6
+    hooks:
+      - id: sqlfluff-lint
+        files: ^src/backend/migrations/(?:.*/)*.*$
+        types: [sql]
+      - id: sqlfluff-fix
+        files: ^src/backend/migrations/(?:.*/)*.*$
+        types: [sql]
+
   # Lint: Bash scripts
   - repo: https://github.com/openstack-dev/bashate.git
     rev: 2.1.1