feat: add test coverage metric (#1129)

* build: add coverage.py to test dependencies * docs: add dev info about coverage and profiling * refactor: add doxygen output to gitignore * build: add coverage-badge to test deps * ci: update all workflows v1.4.2 --> v1.4.3 * ci: add coverage output during deploy pytest * docs: add coverage stats to readme * build: lock dependencies after merge coverage+fmtm-splitter * build: add coverage files to gitignore * ci: update pytest workflow test to use coverage * ci: update all workflows v1.4.3 --> v1.4.4
hotosm · Jan 22, 2024 · 148de05 · 148de05
1 parent 66b3607
commit 148de05
Show file tree

Hide file tree

Showing 16 changed files with 160 additions and 55 deletions.
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
@@ -17,31 +17,32 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.4
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend
       pre_command: docker compose up -d proxy && docker compose exec api /app-entrypoint.sh
       compose_service: api
       compose_command: pytest
       tag_override: ci-${{ github.ref_name }}
+      coverage: true
     secrets: inherit
 
   frontend-tests:
-    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.4
     with:
       working_dir: src/frontend
 
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     needs: [pytest]
     with:
       context: src/backend
       build_target: prod
       image_name: ghcr.io/${{ github.repository }}/backend
 
   frontend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     needs: [frontend-tests]
     with:
       context: src/frontend
@@ -125,7 +126,7 @@ jobs:
     needs:
       - smoke-test-backend
       - smoke-test-frontend
-    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.4.4
     with:
       environment: ${{ github.ref_name }}
       docker_compose_file: "docker-compose.${{ github.ref_name }}.yml"

diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml
@@ -16,7 +16,7 @@ on:
 
 jobs:
   backend-ci-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: src/backend
       build_target: ci

diff --git a/.github/workflows/build_odk_imgs.yml b/.github/workflows/build_odk_imgs.yml
@@ -13,7 +13,7 @@ on:
 
 jobs:
   build-odkcentral:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: odkcentral/api
       image_tags: |
@@ -26,7 +26,7 @@ jobs:
       # multi_arch: true
 
   build-odkcentral-ui:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: odkcentral/ui
       image_tags: |

diff --git a/.github/workflows/build_proxy_imgs.yml b/.github/workflows/build_proxy_imgs.yml
@@ -10,7 +10,7 @@ on:
 
 jobs:
   build-cert-init-main:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: nginx
       build_target: certs-init-main
@@ -21,7 +21,7 @@ jobs:
       multi_arch: true
 
   build-cert-init-dev:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: nginx
       build_target: certs-init-development
@@ -33,7 +33,7 @@ jobs:
       multi_arch: true
 
   build-proxy-main:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: nginx
       build_target: main
@@ -44,7 +44,7 @@ jobs:
       multi_arch: true
 
   build-proxy-dev:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: nginx
       build_target: development

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -12,19 +12,19 @@ on:
 
 jobs:
   build_doxygen:
-    uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@1.4.4
     with:
       output_path: docs/apidocs
 
   build_openapi_json:
-    uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@1.4.4
     with:
       image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
       example_env_file_path: ".env.example"
       output_path: docs/openapi.json
 
   publish_docs:
-    uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@1.4.4
     needs:
       - build_doxygen
       - build_openapi_json

diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml
@@ -14,7 +14,7 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.4.4
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend

diff --git a/.github/workflows/pr_test_frontend.yml b/.github/workflows/pr_test_frontend.yml
@@ -14,6 +14,6 @@ on:
 
 jobs:
   frontend-tests:
-    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@1.4.4
     with:
       working_dir: src/frontend
diff --git a/.github/workflows/tag_build.yml b/.github/workflows/tag_build.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.4.4
     with:
       context: src/backend
       build_target: prod

diff --git a/.github/workflows/tests/pytest.yml b/.github/workflows/tests/pytest.yml
@@ -20,6 +20,7 @@ jobs:
       - name: Vars and Secrets to Env
         env:
           TAG_OVERRIDE: ${{ env.TAG_OVERRIDE || 'ci-development' }}
+          TARGET_OVERRIDE: ${{ env.TARGET_OVERRIDE || 'ci' }}
           GIT_BRANCH: ${{ github.ref_name }}
           VARS_CONTEXT: ${{ toJson(vars) }}
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
@@ -32,6 +33,7 @@ jobs:
 
           # Set all vars
           echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> $GITHUB_ENV
+          echo "TARGET_OVERRIDE=${TARGET_OVERRIDE}" >> $GITHUB_ENV
           echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV
 
           # Set VARS_CONTEXT if not null
@@ -61,8 +63,38 @@ jobs:
           fi
 
           echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
+          echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
 
-      - name: Run Tests
+      - name: Run Tests With Coverage
         run: |
           docker compose up -d proxy
-          docker compose run api pytest
+          docker compose run --entrypoint "sh -c" \
+            --volume ${{ github.workspace }}/coverage:/tmp/coverage api \
+            "coverage run -m pytest \
+            && coverage report && coverage html \
+            && coverage-badge -o coverage.svg \
+            && mv htmlcov/index.html /tmp/coverage/coverage.html \
+            && mv coverage.svg /tmp/coverage/coverage.svg"
+
+      - name: Upload Coverage
+        run: |
+          # Pull content from gh-pages
+          mkdir tmp_pages
+          cd tmp_pages
+          git init
+          git pull https://github.com/${{ github.repository }}.wiki
+          cd ..
+
+          # Sync coverage index and badge using rsync
+          sudo apt-get update && sudo apt-get install -y rsync --no-install-recommends
+
+          echo "Coverage dir contents:"
+          ls ${{ github.workspace }}/coverage
+          echo ""
+
+          cp ${{ github.workspace }}/coverage/coverage.* docs/
+          rsync -av --delete docs/ tmp_pages/ --exclude .git
+
+          # Assess diff
+          cd tmp_pages
+          git --no-pager diff
diff --git a/.github/workflows/tests/test_ci.sh b/.github/workflows/tests/test_ci.sh
@@ -11,18 +11,21 @@ set -e
 # GITHUB_TOKEN=input
 # Feed to act using -s flag: -s GITHUB_TOKEN=input_personal_access_token
 
-# Run backend PyTest manually
-docker compose build api
-act pull_request -W .github/workflows/tests/pytest.yml \
-    -e .github/workflows/tests/pr_payload.json \
-    --var-file=.env --secret-file=.env
+export TAG_OVERRIDE=ci
+export TARGET_OVERRIDE=ci
 
 # # PR Test Backend
-# Includes image build, which fails due to registry auth
+# NOTE: Includes image build, which fails due to registry auth
 # act pull_request -W .github/workflows/pr_test_backend.yml \
 #     -e .github/workflows/tests/pr_payload.json \
 #     --var-file=.env --secret-file=.env
 
+# Instead, run backend PyTest manually
+docker compose build api
+act pull_request -W .github/workflows/tests/pytest.yml \
+    -e .github/workflows/tests/pr_payload.json \
+    --var-file=.env --secret-file=.env
+
 # PR Test Frontend
 act pull_request -W .github/workflows/pr_test_frontend.yml \
     -e .github/workflows/tests/pr_payload.json \

diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml
@@ -10,6 +10,6 @@ on:
 
 jobs:
   publish-docs-to-wiki:
-    uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.4.2
+    uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.4.4
     with:
       homepage_path: "wiki_redirect.md"
diff --git a/.gitignore b/.gitignore
@@ -80,6 +80,8 @@ temp_webmaps/Naivasha
 
 # mkdocs
 site
+docs/apidocs
+**/**/coverage*
 
 # Bash install script
 envsubst

diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@
 | **Docs**        | [![Publish Docs](https://github.com/hotosm/fmtm/actions/workflows/docs.yml/badge.svg?branch=development)](https://github.com/hotosm/fmtm/actions/workflows/docs.yml) [![Publish Docs to Wiki](https://github.com/hotosm/fmtm/actions/workflows/wiki.yml/badge.svg?branch=development)](https://github.com/hotosm/fmtm/actions/workflows/wiki.yml)                                                                                                                                                                                                                                                                                                                                                    |
 | **Tech Stack**  | ![FastAPI](https://img.shields.io/badge/FastAPI-005571?style=for-the-badge&logo=fastapi) ![React](https://img.shields.io/badge/react-%2320232a.svg?style=for-the-badge&logo=react&logoColor=%2361DAFB) ![Postgres](https://img.shields.io/badge/postgres-%23316192.svg?style=for-the-badge&logo=postgresql&logoColor=white) ![Kubernetes](https://img.shields.io/badge/kubernetes-%23326ce5.svg?style=for-the-badge&logo=kubernetes&logoColor=white) ![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white)                                                                                                                                    |
 | **Code Style**  | ![Backend Style](https://img.shields.io/badge/code%20style-black-black) ![Frontend Style](https://img.shields.io/badge/code%20style-prettier-F7B93E?logo=Prettier) ![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| **Other Info**  | [![GitHub Sponsors](https://img.shields.io/badge/sponsor-30363D?logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/hotosm) [![License](https://img.shields.io/github/license/hotosm/fmtm.svg)](https://github.com/hotosm/fmtm/blob/main/LICENSE.md) [![All Contributors](https://img.shields.io/github/all-contributors/hotosm/fmtm?color=ee8449&style=flat-square)](#contributors-)                                                                                                                                                                                                                                                                                               |
+| **Other Info**  | [![GitHub Sponsors](https://img.shields.io/badge/sponsor-30363D?logo=GitHub-Sponsors&logoColor=#EA4AAA)](https://github.com/sponsors/hotosm) [![License](https://img.shields.io/github/license/hotosm/fmtm.svg)](https://github.com/hotosm/fmtm/blob/main/LICENSE.md) [![All Contributors](https://img.shields.io/github/all-contributors/hotosm/fmtm?color=ee8449&style=flat-square)](#contributors-) [![Coverage](https://hotosm.github.io/fmtm/coverage.svg)](https://hotosm.github.io/fmtm/coverage.html)                                                                                                                                                                                        |
 
 <!-- markdownlint-restore -->
 

diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md
@@ -208,6 +208,19 @@ To run the backend tests locally, run:
 docker compose run --rm api pytest
 ```
 
+To assess coverage of tests, run:
+
+```bash
+docker compose run --rm --entrypoint='sh -c' api \
+  'coverage run -m pytest && coverage report -m'
+```
+
+To assess performance of endpoints:
+
+- We can use the pyinstrument profiler.
+- While in debug mode (DEBUG=True), access any endpoint.
+- Add the `?profile=true` arg to the URL to view the execution time.
+
 ## Using the local version of ODK Central
 
 - During project creation a Central ODK URL must be provided.