2024-01-25 merge development --> staging

.github/labeler.yml

@@ -1,16 +1,16 @@
 "frontend":
-  - any: ["src/frontend/**/*", "!src/frontend/public/install-fmtm.sh"]
+  - "src/frontend/**/*"
 "backend":
   - "src/backend/**/*"
 "devops":
   - ".github/**/*"
   - "nginx/**/*"
   - "scripts/**/*"
-  - "src/frontend/public/install-fmtm.sh"
   - "docker-*.yml"
   - "**/Dockerfile"
   - "**/*.dockerfile"
   - "**/*entrypoint.sh"
+  - "Justfile"
 "migration":
   - "src/backend/migrations/**/*"
 "documentation":

.github/workflows/build_and_deploy.yml

@@ -17,31 +17,32 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.5
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend
       pre_command: docker compose up -d proxy && docker compose exec api /app-entrypoint.sh
       compose_service: api
       compose_command: pytest
       tag_override: ci-${{ github.ref_name }}
+      coverage: true
     secrets: inherit
 
   frontend-tests:
-    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.5
     with:
       working_dir: src/frontend
 
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     needs: [pytest]
     with:
       context: src/backend
       build_target: prod
       image_name: ghcr.io/${{ github.repository }}/backend
 
   frontend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     needs: [frontend-tests]
     with:
       context: src/frontend
@@ -118,14 +119,37 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Create .env file
+        run: |
+          # Get a8m/envsubst (required for default vals syntax ${VAR:-default})
+          echo "Downloading envsubst"
+          curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst
+          chmod +x envsubst
+
+          echo "Substituing variables from .env.example --> .env"
+          ./envsubst < .env.example > .env
+
       - name: Frontend smoke test
-        run: echo "Not implemented"
+        run: |
+          if docker compose up --detach \
+              --no-deps --wait --wait-timeout 30 \
+              ui
+          then
+              docker compose logs ui
+              echo "Sleeping 5 seconds to wait for dev server"
+              sleep 5
+              curl --fail http://localhost:7051 || exit 1
+          else
+              echo "Application not healthy after 30s. Exiting."
+              docker compose logs ui
+              exit 1
+          fi
 
   deploy-containers:
     needs:
       - smoke-test-backend
       - smoke-test-frontend
-    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.4.5
     with:
       environment: ${{ github.ref_name }}
       docker_compose_file: "docker-compose.${{ github.ref_name }}.yml"

.github/workflows/build_ci_img.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   backend-ci-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: src/backend
       build_target: ci

.github/workflows/build_odk_imgs.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   build-odkcentral:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: odkcentral/api
       image_tags: |
@@ -26,7 +26,7 @@ jobs:
       # multi_arch: true
 
   build-odkcentral-ui:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: odkcentral/ui
       image_tags: |

.github/workflows/build_proxy_imgs.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build-cert-init-main:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: nginx
       build_target: certs-init-main
@@ -21,7 +21,7 @@ jobs:
       multi_arch: true
 
   build-cert-init-dev:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: nginx
       build_target: certs-init-development
@@ -33,7 +33,7 @@ jobs:
       multi_arch: true
 
   build-proxy-main:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: nginx
       build_target: main
@@ -43,19 +43,8 @@ jobs:
         NGINX_TAG=${{ vars.NGINX_TAG }}
       multi_arch: true
 
-  build-proxy-main-plus-script:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
-    with:
-      context: nginx
-      build_target: main-plus-script
-      image_tags: |
-        "ghcr.io/${{ github.repository }}/proxy:main-plus-script"
-      extra_build_args: |
-        NGINX_TAG=${{ vars.NGINX_TAG }}
-      multi_arch: true
-
   build-proxy-dev:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: nginx
       build_target: development

.github/workflows/docs.yml

@@ -12,23 +12,22 @@ on:
 
 jobs:
   build_doxygen:
-    uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.4.5
     with:
       output_path: docs/apidocs
 
   build_openapi_json:
-    uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.4.5
     with:
       image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
       example_env_file_path: ".env.example"
       output_path: docs/openapi.json
 
   publish_docs:
-    uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.4.5
     needs:
       - build_doxygen
       - build_openapi_json
     with:
-      image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
       doxygen: true
       openapi: true

.github/workflows/pr_test_backend.yml

@@ -14,7 +14,7 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.5
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend

.github/workflows/pr_test_frontend.yml

@@ -14,6 +14,6 @@ on:
 
 jobs:
   frontend-tests:
-    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.5
     with:
       working_dir: src/frontend

.github/workflows/tag_build.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.5
     with:
       context: src/backend
       build_target: prod

.github/workflows/tests/pytest.yml

@@ -9,6 +9,9 @@ on:
       - staging
       - development
 
+permissions:
+  contents: write
+
 jobs:
   run-tests:
     runs-on: ubuntu-latest
@@ -20,6 +23,7 @@ jobs:
       - name: Vars and Secrets to Env
         env:
           TAG_OVERRIDE: ${{ env.TAG_OVERRIDE || 'ci-development' }}
+          TARGET_OVERRIDE: ${{ env.TARGET_OVERRIDE || 'ci' }}
           GIT_BRANCH: ${{ github.ref_name }}
           VARS_CONTEXT: ${{ toJson(vars) }}
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
@@ -30,10 +34,6 @@ jobs:
           # Parse JSON with multiline strings, using delimeter (Github specific)
           to_envs() { jq -r "to_entries[] | \"\(.key)<<$delim\n\(.value)\n$delim\n\""; }
 
-          # Set all vars
-          echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> $GITHUB_ENV
-          echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV
-
           # Set VARS_CONTEXT if not null
           if [ "${VARS_CONTEXT}" != "null" ]; then
             echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV
@@ -51,18 +51,37 @@ jobs:
           curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst
           chmod +x envsubst
 
-          # Check if .env.example exists
-          if [ -f .env.example ]; then
-            echo "Substituting variables from .env.example --> .env"
-            ./envsubst < .env.example > .env
-          else
-            echo ".env.example not found, creating .env with GIT_BRANCH only"
-            echo "GIT_BRANCH=${GIT_BRANCH}" > .env
-          fi
+          echo "Substituting variables from .env.example --> .env"
+          ./envsubst < .env.example > .env
 
+          # Set all vars
+          echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> .env
+          echo "TARGET_OVERRIDE=${TARGET_OVERRIDE}" >> .env
           echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
 
-      - name: Run Tests
+      - name: Run Tests With Coverage
         run: |
           docker compose up -d proxy
-          docker compose run api pytest
+          docker compose run --entrypoint "sh -c" \
+            --volume ${{ github.workspace }}/coverage:/tmp/coverage api \
+            "coverage run -m pytest \
+            && coverage report && coverage html \
+            && coverage-badge -o coverage.svg \
+            && mv htmlcov/index.html /tmp/coverage/coverage.html \
+            && mv coverage.svg /tmp/coverage/coverage.svg"
+
+      - name: Upload Coverage
+        run: |
+          # Checkout to gh-pages
+          git config user.name svchot
+          git config user.email sysadmin@hotosm.org
+          git checkout gh-pages
+
+          # Overwrite coverage index and badge
+          echo "Coverage dir contents:"
+          ls ${{ github.workspace }}/coverage
+          echo ""
+          mv ${{ github.workspace }}/coverage/* ${{ github.workspace }}
+
+          # Assess diff
+          git --no-pager diff

.github/workflows/tests/test_ci.sh

@@ -7,22 +7,25 @@ set -e
 ########################################
 
 # TODO read personal access token
-# read -p
+# read -erp
 # GITHUB_TOKEN=input
 # Feed to act using -s flag: -s GITHUB_TOKEN=input_personal_access_token
 
-# Run backend PyTest manually
-docker compose build api
-act pull_request -W .github/workflows/tests/pytest.yml \
-    -e .github/workflows/tests/pr_payload.json \
-    --var-file=.env --secret-file=.env
+export TAG_OVERRIDE=ci
+export TARGET_OVERRIDE=ci
 
 # # PR Test Backend
-# Includes image build, which fails due to registry auth
+# NOTE: Includes image build, which fails due to registry auth
 # act pull_request -W .github/workflows/pr_test_backend.yml \
 #     -e .github/workflows/tests/pr_payload.json \
 #     --var-file=.env --secret-file=.env
 
+# Instead, run backend PyTest manually
+TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose build api
+act pull_request -W .github/workflows/tests/pytest.yml \
+    -e .github/workflows/tests/pr_payload.json \
+    --var-file=.env --secret-file=.env
+
 # PR Test Frontend
 act pull_request -W .github/workflows/pr_test_frontend.yml \
     -e .github/workflows/tests/pr_payload.json \

.github/workflows/wiki.yml

@@ -10,6 +10,6 @@ on:
 
 jobs:
   publish-docs-to-wiki:
-    uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.4.5
     with:
       homepage_path: "wiki_redirect.md"

.gitignore

@@ -80,6 +80,9 @@ temp_webmaps/Naivasha
 
 # mkdocs
 site
+docs/apidocs
+coverage
+**/**/coverage*
 
 # Bash install script
 envsubst