Update build workflows to use image caching, multi-arch, auto-tag

.env.example

@@ -1,7 +1,6 @@
 ### copy to .env and set variables
 
 ### ODK Central ###
-ODK_CENTRAL_VERSION=v2023.2.1
 ODK_CENTRAL_URL=https://central-proxy
 ODK_CENTRAL_USER=dev@fmtm.hotosm.org
 ODK_CENTRAL_PASSWD=testuserpassword

.github/labeler.yml

@@ -9,7 +9,6 @@
   - "**/Dockerfile"
   - "**/*.dockerfile"
   - "**/*entrypoint.sh"
-  - "josm/**/*"
 "migration":
   - "src/backend/migrations/**/*"
 "documentation":
@@ -20,3 +19,5 @@
   - "INSTALL.md"
 "ODK":
   - "odkcentral/**/*"
+"contrib":
+  - "contrib/**/*"

.github/workflows/build_and_deploy.yml

@@ -1,3 +1,5 @@
+# Workflow for build and auto-deploy of branches
+
 name: Build and Deploy
 
 on:
@@ -15,52 +17,47 @@ on:
 
 jobs:
   pytest:
-    uses: ./.github/workflows/r-pytest.yml
+    uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main
     with:
-      image_tag: ci-${{ github.ref_name }}
+      image_name: ghcr.io/${{ github.repository }}/backend
+      build_context: src/backend
+      build_args: |
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
+      docker_compose_service: api
+      tag_override: ci-${{ github.ref_name }}
     secrets: inherit
 
   frontend-tests:
-    uses: ./.github/workflows/r-frontend_tests.yml
-
-  extract-vars:
-    needs:
-      - pytest
-      - frontend-tests
-    uses: ./.github/workflows/r-extract_vars.yml
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main
     with:
-      environment: ${{ github.ref_name }}
+      working_dir: src/frontend
 
   backend-build:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
-    needs: [extract-vars]
     with:
       context: src/backend
       build_target: prod
-      image_tags: |
-        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-${{ github.ref_name }}"
-        "ghcr.io/hotosm/fmtm/backend:latest"
+      image_name: ghcr.io/${{ github.repository }}/backend
       build_args: |
-        APP_VERSION=${{ needs.extract-vars.outputs.api_version }}
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
 
   frontend-main-build:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
-    needs: [extract-vars]
     with:
       context: src/frontend
       dockerfile: prod.dockerfile
       build_target: prod
-      image_tags: |
-        "ghcr.io/hotosm/fmtm/frontend:${{ needs.extract-vars.outputs.frontend_main_version }}-${{ github.ref_name }}"
-        "ghcr.io/hotosm/fmtm/frontend:latest"
+      image_name: ghcr.io/${{ github.repository }}/frontend
       build_args: |
-        APP_VERSION=${{ needs.extract-vars.outputs.frontend_main_version }}
-        VITE_API_URL=${{ needs.extract-vars.outputs.api_url }}
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
+        VITE_API_URL=${{ vars.URL_SCHEME }}://${{ vars.API_URL }}"
 
   smoke-test-backend:
     runs-on: ubuntu-latest
     needs:
-      - extract-vars
       - backend-build
     environment:
       name: ${{ github.ref_name }}
@@ -69,55 +66,28 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Environment to .env
+      - name: Vars and Secrets to Env
         env:
           GIT_BRANCH: ${{ github.ref_name }}
-          API_VERSION: ${{ needs.extract-vars.outputs.api_version }}
-          FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }}
+          VARS_CONTEXT: ${{ toJson(vars) }}
+          SECRETS_CONTEXT: ${{ toJson(secrets) }}
         run: |
-          echo "${{ secrets.DOTENV }}" > .env
-          echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
-          echo "API_VERSION=${API_VERSION}" >> .env
-          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
+          to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; }
+
+          echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV
+          echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV
+          echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV
+
+      - name: Create .env file
+        run: env > .env
 
       - name: Backend smoke test
         run: |
-          echo "Not implemented" 
-          # source .env
-          # docker network create fmtm
-
-          # docker pull "postgis/postgis:14-3.3-alpine"
-          # docker run --rm -d \
-          #   --name=fmtm-db \
-          #   --network=fmtm \
-          #   -e POSTGRES_USER=fmtm \
-          #   -e POSTGRES_PASSWORD=fmtm \
-          #   -e POSTGRES_DB=fmtm \
-          #   "postgis/postgis:14-3.3-alpine"
-
-          # docker pull "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
-          # docker run --rm -d \
-          #   --network=fmtm \
-          #   -p 8080:8080 \
-          #   -e OSM_CLIENT_ID="test" \
-          #   -e OSM_CLIENT_SECRET="test" \
-          #   -e OSM_SECRET_KEY="test" \
-          #   -e S3_ACCESS_KEY="fmtm" \
-          #   -e S3_SECRET_KEY="somelongpassword" \
-          #   "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}"
-
-          # # First wait 10 seconds for API
-          # sleep 10
-          # # Check the exit status of curl and exit the job if it fails
-          # if ! curl -f http://localhost:8080/docs; then
-          #   echo "curl failed to access http://localhost:8080/docs"
-          #   exit 1
-          # fi
+          docker compose up -d api --wait --wait-timeout 60
 
   smoke-test-frontend:
     runs-on: ubuntu-latest
     needs:
-      - extract-vars
       - frontend-main-build
     environment:
       name: ${{ github.ref_name }}
@@ -130,40 +100,11 @@ jobs:
         run: echo "Not implemented"
 
   deploy-containers:
-    runs-on: ubuntu-latest
     needs:
-      - extract-vars
       - smoke-test-backend
       - smoke-test-frontend
-    environment:
-      name: ${{ github.ref_name }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Environment to .env
-        env:
-          GIT_BRANCH: ${{ github.ref_name }}
-          API_VERSION: ${{ needs.extract-vars.outputs.api_version }}
-          FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }}
-        run: |
-          echo "${{ secrets.DOTENV }}" > .env
-          echo "GIT_BRANCH=${GIT_BRANCH}" >> .env
-          echo "API_VERSION=${API_VERSION}" >> .env
-          echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env
-
-      - uses: webfactory/ssh-agent@v0.7.0
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Disable Host key verification
-        # Hack to prevent "Host key verification failed". Should be replaced with a ssh-keyscan based solution
-        run: echo "StrictHostKeyChecking no" >> ~/.ssh/config
-
-      - name: Deploy
-        run: |
-          docker compose --file "docker-compose.${{ github.ref_name }}.yml" pull
-          docker compose --file "docker-compose.${{ github.ref_name }}.yml" up --detach --remove-orphans
-        env:
-          DOCKER_HOST: ${{ vars.DOCKER_HOST }}
+    uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@main
+    with:
+      environment: ${{ github.ref_name }}
+      docker_compose_file: "docker-compose.${{ github.ref_name }}.yml"
+    secrets: inherit

.github/workflows/build_ci_img.yml

@@ -15,17 +15,13 @@ on:
   workflow_dispatch:
 
 jobs:
-  extract-vars:
-    uses: ./.github/workflows/r-extract_vars.yml
-
   backend-ci-build:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
-    needs: [extract-vars]
     with:
       context: src/backend
       build_target: ci
       image_tags: |
-        "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-ci-${{ github.ref_name }}"
-        "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}"
+        "ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}"
       build_args: |
-        APP_VERSION=${{ needs.extract-vars.outputs.api_version }}
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}

.github/workflows/build_odk_imgs.yml

@@ -17,15 +17,15 @@ jobs:
     with:
       context: odkcentral/api
       image_tags: |
-        "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_VERSION }}"
-        "ghcr.io/hotosm/fmtm/odkcentral:latest"
+        "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}"
+        "ghcr.io/${{ github.repository }}/odkcentral:latest"
       build_args: |
-        ODK_CENTRAL_VERSION=${{ vars.ODK_CENTRAL_VERSION }}
+        ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }}
 
   build-proxy:
     uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main
     with:
       context: odkcentral/proxy
       image_tags: |
-        "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_VERSION }}"
-        "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest"
+        "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}"
+        "ghcr.io/${{ github.repository }}/odkcentral-proxy:latest"

.github/workflows/docs.yml

@@ -6,37 +6,29 @@ on:
       - docs/**
       - src/**
       - mkdocs.yml
-    branches: [main]
+    branches: [development]
   # Allow manual trigger (workflow_dispatch)
   workflow_dispatch:
 
 jobs:
   build_doxygen:
     uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@main
     with:
-      cache_paths: |
-        docs/apidocs
-        docs/openapi.json
-      cache_key: docs-build
+      output_path: docs/apidocs
 
   build_openapi_json:
     uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@main
     with:
-      image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}
+      image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
       example_env_file_path: ".env.example"
-      cache_paths: |
-        docs/apidocs
-        docs/openapi.json
-      cache_key: docs-build
+      output_path: docs/openapi.json
 
   publish_docs:
     uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@main
     needs:
       - build_doxygen
       - build_openapi_json
     with:
-      image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}
-      cache_paths: |
-        docs/apidocs
-        docs/openapi.json
-      cache_key: docs-build
+      image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
+      doxygen: true
+      openapi: true

.github/workflows/pr_test_backend.yml

@@ -0,0 +1,31 @@
+name: PR Test Backend
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - development
+    # Workflow is triggered only if src/backend changes
+    paths:
+      - src/backend/**
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  pytest:
+    uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main
+    with:
+      image_name: ghcr.io/${{ github.repository }}/backend
+      build_context: src/backend
+      build_args: |
+        APP_VERSION=${{ github.ref_name }}
+        COMMIT_REF=${{ github.sha }}
+      docker_compose_service: api
+      cache_extra_imgs: |
+        "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}"
+        "docker.io/minio/minio:${{ vars.MINIO_TAG }}"
+      # For caching odk central images, add:
+      # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}"
+      # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}"
+    secrets: inherit

.github/workflows/pr_test_frontend.yml

@@ -0,0 +1,19 @@
+name: PR Tests Frontend
+
+on:
+  pull_request:
+    branches:
+      - main
+      - staging
+      - development
+    # Workflow is triggered only if src/frontend changes
+    paths:
+      - src/frontend/**
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  frontend-tests:
+    uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main
+    with:
+      working_dir: src/frontend