From cdf2dc564a22c41540a497b4d468961bd91177a5 Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:54:00 +0700 Subject: [PATCH] build: major deveops refactor, use nginx/certbot + add install script (#931) * build: replace docker-compose yamls with new config * build: replace odkcentral proxy with repo level proxy * build: replace URL_SCHEME, API_URL, FRONTEND_MAIN_URL --> FMTM_DOMAIN * docs: update refs to https://central-proxy --> https://proxy * docs: remove Docker-Tips file * build: rename docker compose files to match branches * docs: replace wiki homepage with redirect to docs * build: set VITE_API_URL automatically, unless override * build: update lables for all dockerfiles * build: add certbot and netcat to nginx image * build: script to build nginx certbot images * build: add renew-certs.sh script, certbot init template * docs: extra info for production / vars * build: certbot compose config, set all depends conditions * build: compose add required: false for odk central svc * build: add recommended letsencrypt ssl params nginx * build: add PUSH_IMGS option to certs-init build * build: add renew-certs-manual.sh script * ci: update compose command for pytest * build: add mc client to backend, migrate backup to s3 * fix: add default for UNDERPASS_API_URL in .env.example * build: set default dockerfile user to non-system >999 * build: add db backups to production deploys * build: use staging certbot config * build: fix app dist dir for proxy copy * build: fix fmtm-central container name for dev compose * build: fix default_server for cert init nginx configs * build: update cert init build stages for clarity * build: don't cache local cert-init img builds * build: revert certbot staging, run non-interactive * build: set certbot non-interactive via -n flag * build: fix certbot non-interactive before certonly * build: remove dup ssl_dhparam directive for prod nginx * build: cache 2nd cert nginx build, logs for backup entry * build: fix default_server for port 80 minio/odk * build: combine domains into single cert for nginx * build: add http --> https redirects on root * build: add healthcheck on db compose backup service * docs: update license code for frontend + author * build: add root dir for nginx frontend conf * build: move ssl-dhparams to certs dir * build: remove redundant FMTM_PORT in proxy env * build: update certbot certs-init to single stage * build: fix certbot development command * build: rename build_img --> build_imgs, build all * build: remove dist from proxy, load via compose volume * build: allow caching for image_builds.sh * build: fix FROM in proxy dockerfile * build: update spa-to-http img, use entrypoint for ui-build * build: specify certbot img command via array * build: set certbot domains via entrypoint (vs command) * build: move frontend volume from /app/dist to /app * build: rename ui-build service to ui * build: set frontend build args correctly in prod * build: update VITE_API_URL without protocol * build: change spa-to-http to rclone, add sync entrypoint * build: add entrypoint to prod.dockerfile frontend * build: use sh for container-entrypoint (no bash) * build: update frontend prod dockerfile to use root * build: optimise odk central build (use slim img, save 800mb) * ci: update pytest to run proxy service instead of api * ci: bump remote_deploy workflow --> 1.1.3 to force redeploy * build: allow TAG_OVERRIDE for debug frontend * build: add VOLUME to prod dockerfile /frontend * build: add security headers to all prod nginx * build: add central frontend to dev stack * ci: remove wait-for-it from api, update test_backend workflows * build: correctly unset ENTRYPOINT in api ci image * build: default ci img command sleep infinity * build: fix renew_certs_manual script * ci: update pr labeller criteria * build: add curlable bash script for easy install * build: fix curlable bash install script * build: always clone repo during install (frontend build) * refactor: rename install-fmtm.sh --> install.sh * build: all changing port during development install * feat: major updates to install scripts, capture sigterms * build: update install bash script * docs: add info about using install script * build: pass DOCKER_HOST to machinectl cmd * build: neater install outputs, progress, default redirect * build: replace frontend as default_server (over api) * build: install allow priv port access, progress install bar * build: rename certs-init image to proxy:certs-init tag * build: check odk password length during install * build: add install script to nginx proxy * build: optimise install script, allow existing .env file * build: add default for FMTM_SCRIPT_DOMAIN * build: fix using existing .env in install script * build: use cert init images specific to branches * build: remove set -u from cert renew (unbound vars) * build: rename proxy entrypoint file * build: remove FMTM_SCRIPT_DOMAIN from dev setup * build: fix ref to container entrypoint for proxy * build: set default FMTM_SCRIPT_DOMAIN to ignore * ci: add workflow to build proxy images * build: fix cert renew with FMTM_SCRIPT_DOMAIN * build: fix typo in nginx container entrypoint * build: rename FMTM_PORT --> FMTM_DEV_PORT for clarity * build: add defaults for FMTM_SCRIPT_DOMAIN (nginx fails) * build: fix remove user from sudoers in script * build: install script fix unary operator expected * build: add names to compose volumes (multiple deploy) * build: fix add fmtm_data volume to dev compose * build: fix install script find .env for non-root usr * build: fix script.conf nginx, get the script only * build: add DEBIAN_FRONTEND=noninteractive to install script * build: always revalidate cache for script proxy * refactor: add s3 bucket name to prod install script * ci: gh workflow schedule cron in single quotes * build: install script replace == with =, fix remote execution * docs: update latest docs with install / prod details * docs: tweak install docs * docs: update install docs * build: replace this branch in install.sh with env var * fix: set OSM_LOGIN_REDIRECT_URI automatically * build: fix gen-env scripts for osm redirect uri * docs: update install docs for .env generation * docs: add info for restoring from db backup --- .env.example | 16 +- .github/labeler.yml | 4 +- .github/workflows/build_and_deploy.yml | 7 +- .github/workflows/build_odk_imgs.yml | 10 +- .github/workflows/build_proxy_imgs.yml | 45 + .github/workflows/pr_label.yml | 2 +- .github/workflows/pr_test_backend.yml | 10 +- .github/workflows/wiki.yml | 2 +- .gitignore | 5 +- INSTALL.md | 287 +++--- Makefile | 1 - README.md | 15 +- contrib/josm/Dockerfile | 3 +- docker-compose.development.yml | 286 ++++-- docker-compose.main.yml | 174 ++-- docker-compose.noodk.yml | 135 --- docker-compose.staging.yml | 75 ++ docker-compose.yml | 226 +++-- docs/_Sidebar.md | 29 - docs/dev/Backend.md | 14 +- docs/dev/Database-Tips.md | 6 +- docs/dev/Docker-Tips.md | 138 --- docs/dev/Frontend.md | 2 +- docs/dev/Production.md | 131 ++- docs/dev/Setup.md | 71 +- docs/dev/Troubleshooting.md | 2 +- docs/wiki_redirect.md | 4 + gen-env.sh | 226 ----- mkdocs.yml | 1 - nginx/Dockerfile | 130 +++ nginx/build_imgs.sh | 63 ++ nginx/certs/ca.crt | 19 + nginx/certs/central.crt | 21 + nginx/certs/central.key | 28 + nginx/certs/ssl-dhparams.pem | 8 + nginx/container-entrypoint.sh | 81 ++ nginx/nginx.conf | 105 ++ nginx/options-security.conf | 6 + nginx/options-ssl-nginx.conf | 10 + nginx/templates/api.conf.template | 75 ++ .../templates/cert-init/api.conf.template | 19 +- nginx/templates/cert-init/fmtm.conf.template | 26 + nginx/templates/cert-init/minio.conf.template | 26 + nginx/templates/cert-init/odk.conf.template | 26 + .../templates/cert-init/script.conf.template | 30 + .../templates/dev/api.conf.template | 38 +- nginx/templates/dev/fmtm.conf.template | 64 ++ nginx/templates/dev/minio.conf.template | 57 ++ nginx/templates/dev/odk.conf.template | 118 +++ nginx/templates/fmtm.conf.template | 52 + nginx/templates/minio.conf.template | 73 ++ nginx/templates/odk.conf.template | 91 ++ nginx/templates/script.conf.template | 61 ++ odkcentral/api/.dockerignore | 2 - odkcentral/api/Dockerfile | 20 +- odkcentral/enketo/Dockerfile | 86 ++ odkcentral/proxy/ca.crt | 21 - odkcentral/proxy/central.crt | 21 - odkcentral/proxy/central.key | 27 - .../{proxy/nginx.conf => ui/Dockerfile} | 42 +- odkcentral/ui/container-entrypoint.sh | 11 + scripts/README.md | 8 + scripts/gen-env.sh | 441 +++++++++ scripts/install-fmtm.sh | 9 - scripts/renew-certs-manual.sh | 44 + scripts/setup/README.md | 5 + scripts/setup/docker.sh | 212 ++++ scripts/setup/podman.sh | 74 ++ src/backend/.dockerignore | 1 + src/backend/Dockerfile | 23 +- src/backend/app/auth/auth_routes.py | 2 +- src/backend/app/config.py | 16 +- src/backend/backup-entrypoint.sh | 142 +++ src/backend/migrate-entrypoint.sh | 14 +- src/frontend/container-entrypoint.sh | 11 + src/frontend/debug.dockerfile | 4 +- src/frontend/package.json | 6 +- src/frontend/prod.dockerfile | 29 +- src/frontend/public/install.sh | 916 ++++++++++++++++++ src/frontend/src/views/Organization.tsx | 2 +- src/frontend/vite.config.ts | 2 +- 81 files changed, 4141 insertions(+), 1204 deletions(-) create mode 100644 .github/workflows/build_proxy_imgs.yml delete mode 100644 docker-compose.noodk.yml create mode 100644 docker-compose.staging.yml delete mode 100644 docs/_Sidebar.md delete mode 100644 docs/dev/Docker-Tips.md create mode 100644 docs/wiki_redirect.md delete mode 100644 gen-env.sh create mode 100644 nginx/Dockerfile create mode 100644 nginx/build_imgs.sh create mode 100644 nginx/certs/ca.crt create mode 100644 nginx/certs/central.crt create mode 100644 nginx/certs/central.key create mode 100644 nginx/certs/ssl-dhparams.pem create mode 100644 nginx/container-entrypoint.sh create mode 100644 nginx/nginx.conf create mode 100644 nginx/options-security.conf create mode 100644 nginx/options-ssl-nginx.conf create mode 100644 nginx/templates/api.conf.template rename odkcentral/proxy/Dockerfile => nginx/templates/cert-init/api.conf.template (65%) create mode 100644 nginx/templates/cert-init/fmtm.conf.template create mode 100644 nginx/templates/cert-init/minio.conf.template create mode 100644 nginx/templates/cert-init/odk.conf.template create mode 100644 nginx/templates/cert-init/script.conf.template rename odkcentral/proxy/conf.d/central.conf => nginx/templates/dev/api.conf.template (62%) create mode 100644 nginx/templates/dev/fmtm.conf.template create mode 100644 nginx/templates/dev/minio.conf.template create mode 100644 nginx/templates/dev/odk.conf.template create mode 100644 nginx/templates/fmtm.conf.template create mode 100644 nginx/templates/minio.conf.template create mode 100644 nginx/templates/odk.conf.template create mode 100644 nginx/templates/script.conf.template delete mode 100644 odkcentral/api/.dockerignore create mode 100644 odkcentral/enketo/Dockerfile delete mode 100644 odkcentral/proxy/ca.crt delete mode 100644 odkcentral/proxy/central.crt delete mode 100644 odkcentral/proxy/central.key rename odkcentral/{proxy/nginx.conf => ui/Dockerfile} (51%) create mode 100644 odkcentral/ui/container-entrypoint.sh create mode 100644 scripts/README.md create mode 100644 scripts/gen-env.sh delete mode 100644 scripts/install-fmtm.sh create mode 100644 scripts/renew-certs-manual.sh create mode 100644 scripts/setup/README.md create mode 100644 scripts/setup/docker.sh create mode 100644 scripts/setup/podman.sh create mode 100644 src/backend/backup-entrypoint.sh create mode 100644 src/frontend/container-entrypoint.sh create mode 100644 src/frontend/public/install.sh diff --git a/.env.example b/.env.example index 25a07d47d3..cfe6f94809 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,15 @@ -### copy to .env and set variables - ### ODK Central ### -ODK_CENTRAL_URL=${ODK_CENTRAL_URL:-"https://central-proxy"} -ODK_CENTRAL_USER=${ODK_CENTRAL_USER:-"dev@fmtm.hotosm.org"} +ODK_CENTRAL_URL=${ODK_CENTRAL_URL:-"https://proxy"} +ODK_CENTRAL_USER=${ODK_CENTRAL_USER:-"test@fmtm.dev"} ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD:-"testuserpassword"} ### FMTM ### DEBUG=${DEBUG:-False} LOG_LEVEL=${LOG_LEVEL:-INFO} EXTRA_CORS_ORIGINS=${EXTRA_CORS_ORIGINS} -URL_SCHEME=${URL_SCHEME:-http} -API_URL=${API_URL:-"127.0.0.1:8000"} -FRONTEND_MAIN_URL=${FRONTEND_MAIN_URL:-"127.0.0.1:8080"} +FMTM_DOMAIN=${FMTM_DOMAIN:-"fmtm.localhost"} +FMTM_DEV_PORT=${FMTM_DEV_PORT:-7050} +CERT_EMAIL=${CERT_EMAIL} # Use API_PREFIX if running behind a proxy subpath (e.g. /api) API_PREFIX=${API_PREFIX:-/} @@ -20,7 +18,7 @@ OSM_CLIENT_ID=${OSM_CLIENT_ID} OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET} OSM_URL=${OSM_URL:-"https://www.openstreetmap.org"} OSM_SCOPE=${OSM_SCOPE:-"read_prefs"} -OSM_LOGIN_REDIRECT_URI=${OSM_LOGIN_REDIRECT_URI:-"http://127.0.0.1:8080/osmauth/"} +OSM_LOGIN_REDIRECT_URI="http${FMTM_DOMAIN:+s}://${FMTM_DOMAIN:-127.0.0.1:7051}/osmauth/" OSM_SECRET_KEY=${OSM_SECRET_KEY} ### S3 File Storage ### @@ -41,4 +39,4 @@ FMTM_DB_PASSWORD=${FMTM_DB_PASSWORD:-"fmtm"} FMTM_DB_NAME=${FMTM_DB_NAME:-"fmtm"} ### Underpass (optional override) ### -UNDERPASS_API_URL=${UNDERPASS_API_URL} +UNDERPASS_API_URL=${UNDERPASS_API_URL:-"https://raw-data-api0.hotosm.org/v1"} diff --git a/.github/labeler.yml b/.github/labeler.yml index 3407c15cf5..e18983173e 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,10 +1,12 @@ "frontend": - - "src/frontend/**/*" + - any: ["src/frontend/**/*", "!src/frontend/public/install-fmtm.sh"] "backend": - "src/backend/**/*" "devops": - ".github/**/*" + - "nginx/**/*" - "scripts/**/*" + - "src/frontend/public/install-fmtm.sh" - "docker-*.yml" - "**/Dockerfile" - "**/*.dockerfile" diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index a405e18736..a24863f7bc 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -17,12 +17,13 @@ on: jobs: pytest: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.1.2 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1 with: image_name: ghcr.io/${{ github.repository }}/backend build_context: src/backend + pre_command: docker compose up -d proxy compose_service: api - compose_command: wait-for-it fmtm-db:5432 --strict -- wait-for-it central:8383 --strict --timeout=30 -- pytest + compose_command: pytest tag_override: ci-${{ github.ref_name }} secrets: inherit @@ -124,7 +125,7 @@ jobs: needs: - smoke-test-backend - smoke-test-frontend - uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.1.2 + uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@1.1.3 with: environment: ${{ github.ref_name }} docker_compose_file: "docker-compose.${{ github.ref_name }}.yml" diff --git a/.github/workflows/build_odk_imgs.yml b/.github/workflows/build_odk_imgs.yml index 94a58826a1..8b672f557e 100644 --- a/.github/workflows/build_odk_imgs.yml +++ b/.github/workflows/build_odk_imgs.yml @@ -22,10 +22,12 @@ jobs: extra_build_args: | ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }} - build-proxy: + build-odkcentral-ui: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 with: - context: odkcentral/proxy + context: odkcentral/ui image_tags: | - "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" - "ghcr.io/${{ github.repository }}/odkcentral-proxy:latest" + "ghcr.io/${{ github.repository }}/odkcentral-ui:${{ vars.ODK_CENTRAL_TAG }}" + "ghcr.io/${{ github.repository }}/odkcentral-ui:latest" + build_args: | + ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }} diff --git a/.github/workflows/build_proxy_imgs.yml b/.github/workflows/build_proxy_imgs.yml new file mode 100644 index 0000000000..29857251eb --- /dev/null +++ b/.github/workflows/build_proxy_imgs.yml @@ -0,0 +1,45 @@ +name: 🔧 Build Proxy Images + +on: + # Trigger on schedule + schedule: + # Run midnight 1st and 15th of every month (must be single quote) + # prettier-ignore + - cron: '0 0 * * 1,15' + # Allow manual trigger + workflow_dispatch: + +jobs: + build-cert-init-main: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + with: + context: nginx + target: certs-init-main + image_tags: | + "ghcr.io/${{ github.repository }}/proxy:certs-init-main" + + build-cert-init-dev: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + with: + context: nginx + target: certs-init-development + image_tags: | + "ghcr.io/${{ github.repository }}/proxy:certs-init-development" + "ghcr.io/${{ github.repository }}/proxy:certs-init-staging" + + build-proxy-main: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + with: + context: nginx + target: main + image_tags: | + "ghcr.io/${{ github.repository }}/proxy:main" + + build-proxy-dev: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.1.2 + with: + context: nginx + target: development + image_tags: | + "ghcr.io/${{ github.repository }}/proxy:development" + "ghcr.io/${{ github.repository }}/proxy:staging" diff --git a/.github/workflows/pr_label.yml b/.github/workflows/pr_label.yml index c73a1f84d4..1e5a389561 100644 --- a/.github/workflows/pr_label.yml +++ b/.github/workflows/pr_label.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/labeler@v3 + - uses: actions/labeler@v4 # Uses .github/labeler.yml definitions with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml index ebed4109ad..ec357840f3 100644 --- a/.github/workflows/pr_test_backend.yml +++ b/.github/workflows/pr_test_backend.yml @@ -14,16 +14,18 @@ on: jobs: pytest: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.1.2 + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@1.2.1 with: image_name: ghcr.io/${{ github.repository }}/backend build_context: src/backend + pre_command: docker compose up -d proxy compose_service: api - compose_command: wait-for-it fmtm-db:5432 --strict -- wait-for-it central:8383 --strict --timeout=30 -- pytest + compose_command: pytest cache_extra_imgs: | "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" "docker.io/minio/minio:${{ vars.MINIO_TAG }}" - # For caching odk central images, add: + # For caching odk central image & proxy image, add: # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" - # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" + # "ghcr.io/${{ github.repository }}/proxy:${{ github.head_ref }}" + # ${{ github.head_ref }} --> target branch in PR secrets: inherit diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml index c147fa88fe..ff5866dca5 100644 --- a/.github/workflows/wiki.yml +++ b/.github/workflows/wiki.yml @@ -12,4 +12,4 @@ jobs: publish-docs-to-wiki: uses: hotosm/gh-workflows/.github/workflows/wiki.yml@1.1.2 with: - homepage_path: "index.md" + homepage_path: "wiki_redirect.md" diff --git a/.gitignore b/.gitignore index ed487806dc..5e9993d08c 100644 --- a/.gitignore +++ b/.gitignore @@ -57,7 +57,7 @@ db.sqlite3 .DS_Store # ignore settings -.env +**/*.env # ignore python environments venv @@ -81,3 +81,6 @@ temp_webmaps/Naivasha # mkdocs site + +# Bash install script +envsubst diff --git a/INSTALL.md b/INSTALL.md index ddf3d760ad..583b450907 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,208 +1,245 @@ -> NOTE: This is an installation guide to quickly get the fmtm app up and running. For a detailed guide on how to install the fmtm app using different methods and contributing, checkout the [dev docs](https://hotosm.github.io/fmtm/dev/Setup/) +# Installation -# Table of Contents +## Easy Way -1. [Software Requirements](#software-requirements) +Use the provided bash script: -2. [Setting up the Backend](#setting-up-the-backend) +```bash +# Option A) If you already cloned the repo +bash src/frontend/public/install.sh - - [Fork and Clone the FMTM repository](#fork-and-clone-the-fmtm-repository) - - [Development: Setup Your Local Environment](#development-setup-your-local-environment) - - [Start the API with Docker](#start-the-api-with-docker) - - [Setup ODK Central User](#setup-odk-central-user) - - [Import Test Data](#import-test-data) - - [Check Authentication](#check-authentication) +# Option B) Download the script & run +curl -L https://get.fmtm.dev -o install.sh +bash install.sh +# Alternative URL: https://fmtm.hotosm.org/install.sh -3. [Setting up the Frontend](#setting-up-the-frontend) +# Then follow the prompts +``` - - [Fork and Clone the FMTM repository](#fork-and-clone-the-fmtm-repository-1) - - [Start the Frontends with Docker](#start-the-frontends-with-docker) +> Note: it is best to run this script as a user other than root. +> +> However, if you run as root, a user svcfmtm will be created for you. -# 1. Software Requirements +## Manual Way -Before you can install and use this application, you will need to have the following software installed and configured on your system +If more details are required, check out the +[dev docs](https://hotosm.github.io/fmtm/dev/Setup/) -- [Git(or any other Terminal)](https://git-scm.com/) -- [Docker](https://docs.docker.com/) +### Table of Contents -To install Git, please follow the instructions on the official Git website: +1. [Software Requirements](#software-requirements) - https://git-scm.com/downloads +2. [Setting up FMTM](#setting-up-fmtm) -To install Docker, please follow the instructions on the official Docker website: + - [Fork and Clone the FMTM repository](#fork-and-clone-the-fmtm-repository) + - [Development: Setup Your Local Environment](#setup-your-local-environment) + - [Start the API with Docker](#start-the-api-with-docker) + - [Setup ODK Central User (Optional)](#setup-odk-central-user-optional) + - [Import Test Data (Optional)](#import-test-data-optional) + - [Check Authentication (Optional)](#check-authentication-optional) + +## Software Requirements - https://docs.docker.com/engine/install/ +Before you can install and use this application, +you will need to have the following software +installed and configured on your system: + +- [Git](https://git-scm.com/) +- [Docker](https://docs.docker.com/) -# 2. Setting up the Backend +To install Git, please follow the instructions on the +[official Git website](https://git-scm.com/downloads) -## Fork and Clone the FMTM repository +To install Docker, please follow the instructions on the +[official Docker website](https://docs.docker.com/engine/install/) -### 1. Fork the repository +## Setting up FMTM + +### Fork and Clone the FMTM repository + +#### 1. Fork the repository Forking creates a copy of the repository in your own GitHub account. -Go to the [Field Mapping Tasking Manager repository](https://github.com/hotosm/fmtm) and click the "Fork" button in the top right corner of the page. +Go to the [Field Mapping Tasking Manager repository](https://github.com/hotosm/fmtm) +and click the "Fork" button in the top right corner of the page. -### 2. Clone the forked repository +#### 2. Clone the forked repository Clone the forked repository to your local machine using the following command: -`git clone https://github.com//fmtm.git` +```bash +git clone https://github.com//fmtm.git + +# If you wish to deploy for production, change to the main branch +git checkout main +``` Make sure to replace `` with your GitHub username. -## Development: Setup Your Local Environment +### Setup Your Local Environment These steps are essential to run and test your code! -### 1. Setup OSM OAUTH 2.0 +#### 1. Setup OSM OAUTH 2.0 -The FMTM uses OAUTH2 with OSM to authenticate users. To properly configure your FMTM project, you will need to create keys for OSM. +The FMTM uses OAUTH2 with OSM to authenticate users. -1. [Login to OSM](https://www.openstreetmap.org/login) (_If you do not have an account yet, click the signup button at the top navigation bar to create one_). Click the drop down arrow on the extreme right of the navigation bar and select My Settings. +To properly configure your FMTM project, you will need to create keys for OSM. -2. Register your FMTM instance to OAuth 2 applications. Put your login redirect url as `http://127.0.0.1:8080/osmauth/`, For Production replace the URL as production API Url +1. [Login to OSM](https://www.openstreetmap.org/login) + (_If you do not have an account yet, click the signup + button at the top navigation bar to create one_). + Click the drop down arrow on the top right of the navigation bar + and select My Settings. -> Note: `127.0.0.1` is required instead of `localhost` due to OSM restrictions. +2. Register your FMTM instance to OAuth 2 applications. + Put your login redirect url as `http://127.0.0.1:7051/osmauth/` if running locally, + or for production replace with https://{YOUR_DOMAIN}/osmauth/ -image + > Note: `127.0.0.1` is required for debugging instead of `localhost` + > due to OSM restrictions. -3. Right now _read user preferences permission_ is enough later on fmtm may need permission to modify the map option which should be updated on OSM_SCOPE variable on .env , Keep read_prefs for now. + ![image](https://user-images.githubusercontent.com/36752999/216319298-1444a62f-ba6b-4439-bb4f-2075fdf03291.png) -4. Now Copy your Client ID and Client Secret. Put them in the `OSM_CLIENT_ID` and `OSM_CLIENT_SECRET` of your `.env` file +3. Only the _read user preferences permission_ is required as of now. -### 2. Create an `.env` File +4. Now save your Client ID and Client Secret for the next step. -Environmental variables are used throughout this project. To get started, create `.env` file in the top level dir, a sample is located at `.env.example` +#### 2. Create an `.env` File - cp .env.example .env +Environmental variables are used throughout this project. +To get started, create `.env` file in the top level dir, +a sample is located at `.env.example`. -Your env should look like this: +This can be created interactively by running: -```dotenv -### ODK Central ### -ODK_CENTRAL_URL=https://central-proxy -ODK_CENTRAL_USER=`` -ODK_CENTRAL_PASSWD=`` +```bash +bash scripts/gen-env.sh +``` -### FMTM ### -# DEBUG=True -# LOG_LEVEL=DEBUG -URL_SCHEME=http -API_URL=127.0.0.1:8000 -FRONTEND_MAIN_URL=localhost:8080 -# API_PREFIX=/api +> Note: If extra cors origins are required for testing, the variable +> `EXTRA_CORS_ORIGINS` is a set of comma separated strings, e.g.: +> -### OSM ### -OSM_CLIENT_ID=`` -OSM_CLIENT_SECRET=`` -OSM_URL=https://www.openstreetmap.org -OSM_SCOPE=read_prefs -OSM_LOGIN_REDIRECT_URI=http://127.0.0.1:8080/osmauth/ -OSM_SECRET_KEY= +### Start the API with Docker -### S3 File Storage ### -S3_ENDPOINT="http://s3:9000" -S3_ACCESS_KEY=`` -S3_SECRET_KEY=`` +This is the easiest way to get started with FMTM. -### Database (optional) ### -CENTRAL_DB_HOST=central-db -CENTRAL_DB_USER=odk -CENTRAL_DB_PASSWORD=odk -CENTRAL_DB_NAME=odk +Docker runs each service inside **containers**, fully isolated from your +host operating system. -FMTM_DB_HOST=fmtm-db -FMTM_DB_USER=fmtm -FMTM_DB_PASSWORD=fmtm -FMTM_DB_NAME=fmtm -``` +#### Prerequisite -## Start the API with Docker +You will need to [Install Docker](https://docs.docker.com/engine/install/) +and ensure that it is running on your local machine. -The easiest way to get up and running is by using the FMTM Docker deployment. Docker creates a virtual environment, isolated from your computer's environment, installs all necessary dependencies, and creates a container for the database, the api, and the frontend. These containers talk to each other via the URLs defined in the docker-compose file and your env file. +Then from the command line, navigate to the top level directory of the FMTM project. -### Starting the Containers +#### Select the install type -1. You will need to [Install Docker](https://docs.docker.com/engine/install/) and ensure that it is running on your local machine. -2. From the command line, navigate to the top level directory of the FMTM project. -3. From the command line run: `docker compose pull`. - This will pull the latest container builds from **main** branch. -4. Once everything is pulled, from the command line run: `docker compose up -d api` -5. If everything goes well you should now be able to **navigate to the project in your browser:** `http://127.0.0.1:8000/docs` +Determine the what type of FMTM install you would like: -> Note: If those link doesn't work, check the logs with `docker logs fmtm_api`. +```text +main - the latest production +staging - the latest staging +development - the latest development (warning: may be unstable) +local test - used during development, or to start a test version +``` -## Setup ODK Central User +The corresponding docker-compose files are: -The FMTM uses ODK Central to store ODK data. +```text +main - docker-compose.main.yml +staging - docker-compose.staging.yml +development - docker-compose.development.yml +local test - docker-compose.yml +``` -- By default, the docker setup includes a Central server. -- The credentials should have been provided in your `.env` file to automatically create a user. -- To create a user manually: +Set your selection to a terminal variable to make the next step easier: ```bash -docker compose exec central odk-cmd --email YOUREMAIL@ADDRESSHERE.com user-create -docker-compose exec central odk-cmd --email YOUREMAIL@ADDRESSHERE.com user-promote +export GIT_BRANCH={your_selection} + +# E.g. +export GIT_BRANCH=development ``` -> Note: Alternatively, you may use an external Central server and user. +#### Pull the Images -## Import Test Data +```bash +docker compose -f "docker-compose.${GIT_BRANCH}.yml" pull +``` -Some test data is available to get started quickly. +> This will pull the latest containers for the branch you selected. -- Navigate to the `import-test-data` endpoint in the API docs page: - -- Click `Try it out`, then `execute`. +#### Build the Frontend -## Check Authentication +Before we can run, you need to build your version of the frontend. -Once you have deployed, you will need to check that you can properly authenticate. +This is because the frontend contains variable specific to your deployment. + +```bash +docker compose -f "docker-compose.${GIT_BRANCH}.yml" build ui +``` + +#### Start the Containers -1. Navigate to `{URL_SCHEME}://{API_URL}/docs` +```bash +docker compose -f "docker-compose.${GIT_BRANCH}.yml" up -d +``` - Three endpoints are responsible for oauth - image +You should see the containers start up in order. -2. Hit `/auth/osm_login/` : This will give you the Login URL where you can supply your osm username/password +Once complete, you should now be able to **navigate to the project in your browser:** + +```text +https://{YOUR_DOMAIN} + +# For the local test setup, this will be +http://fmtm.localhost:7050 +``` - Response should be like this : +> Note: If those link doesn't work, check the logs with `docker logs fmtm-api`. +> +> Note: Use `docker ps` to view all container names. - {"login_url": "https://www.openstreetmap.org/oauth2/authorize/?response_type=code&client_id=xxxx"} +### Setup ODK Central User (Optional) - Now Copy your login_url and hit it in new tab, and you will be redirected to OSM for your LOGIN. Give FMTM the necessary permission +The FMTM uses ODK Central to store ODK data. - After successful login, you will get your `access_token` for FMTM Copy it and now you can use it for rest of the endpoints that need authorizations +- By default, the docker setup includes a Central server. +- The credentials should have been provided in your `.env` + file to automatically create a user. +- To create a user manually: -3. Check your access token: Hit `/auth/me/` and pass your `access_token` You should get your osm id, username and profile picture id +```bash +docker compose exec central odk-cmd --email YOUREMAIL@ADDRESSHERE.com user-create +docker-compose exec central odk-cmd --email YOUREMAIL@ADDRESSHERE.com user-promote +``` -That's it, you have successfully set up the backend!! +> Note: Alternatively, you may use an external Central server and user. -# 3. Setting up the Frontend +### Import Test Data (Optional) -## Fork and Clone the FMTM repository +If running a local test version, test data is available to get started quickly. -### 1. Fork the repository +- Navigate to the `import-test-data` endpoint in the API docs page: -Forking creates a copy of the repository in your own GitHub account. -Go to the [Field Mapping Tasking Manager repository](https://github.com/hotosm/fmtm) and click the "Fork" button in the top right corner of the page. + -### 2. Clone the forked repository +- Click `Try it out`, then `execute`. -Clone the forked repository to your local machine using the following command: +### Check Authentication (Optional) -`git clone https://github.com//fmtm.git` +Once you have deployed, you will need to check that you can properly authenticate. -Make sure to replace `` with your GitHub username. +1. Navigate to your frontend (e.g. `http://fmtm.localhost:7050`) -## Start the Frontend with Docker +2. Click the 'Sign In' button and follow the popup prompts. -1. You will need to [Install Docker](https://docs.docker.com/engine/install/) and ensure that it is running on your local machine. -2. From the command line: navigate to the top level directory of the FMTM project. -3. From the command line run: `docker compose build ui` - This is essential, as the development container for the frontend is different to production. -4. Once everything is built, from the command line run: `docker compose up -d ui` +3. If successful, you should see your username in the header. -5. If everything goes well you should now be able to **navigate to the project in your browser:** +4. If you see an error instead, double check your credentials and + redirect URL in the openstreetmap.org settings. -That's it, you have successfully set up the frontend!! +That's it, you have successfully set up FMTM!! diff --git a/Makefile b/Makefile index ae7aa6cfce..50cb41945a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ MDS := \ docs/dev/Backend.md \ docs/dev/Database-Tips.md \ docs/dev/Deployment-Flow.md \ - docs/dev/Docker-Tips.md \ docs/dev/Frontend.md \ docs/dev/Production.md \ docs/dev/Version-Control.md \ diff --git a/README.md b/README.md index 0cde3b3572..0ecc5bb999 100644 --- a/README.md +++ b/README.md @@ -139,9 +139,18 @@ or inform the field mappers that they need to fix it. They need to: - Merge good-quality data into OSM (probably from JOSM). - Mark areas as completed and merged. -## Info for developers +## Install -The basic setup here is: +To install for a quick test, or on a production instance, +use the convenience script: + +```bash +curl -fsSL https://get.fmtm.dev | bash +``` + +## Info For Developers + +A breakdown of the components: ### ODK Collect @@ -154,7 +163,7 @@ to become OSM tags associated with those features). The ODK Collect app connects to a back-end server (in this case ODK Central), which provides the features to be mapped and the survey form definitions. -### ODK Central server +### ODK Central Server An ODK Central server functions as the back end for the field data collectors. ODK Collect is an application that can be downloaded on Android phones. diff --git a/contrib/josm/Dockerfile b/contrib/josm/Dockerfile index 0b7bb5885b..aa0c1263b7 100644 --- a/contrib/josm/Dockerfile +++ b/contrib/josm/Dockerfile @@ -16,8 +16,7 @@ # FROM docker.io/debian:bookworm as base -ARG MAINTAINER=admin@hotosm.org -LABEL org.hotosm.fmtm.maintainer="${MAINTAINER}" \ +LABEL org.hotosm.fmtm.maintainer="sysadmin@hotosm.org" \ org.hotosm.fmtm.josm-port="8111" \ org.hotosm.fmtm.nginx-port="80" RUN set -ex \ diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 9de9548a45..b472a8ae7a 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -18,136 +18,146 @@ version: "3" volumes: - fmtm_data: + fmtm_frontend: + name: fmtm-frontend-${GIT_BRANCH} fmtm_db_data: + name: fmtm-db-data-${GIT_BRANCH} + fmtm_data: + name: fmtm-s3-data-${GIT_BRANCH} fmtm_logs: + name: fmtm-logs-${GIT_BRANCH} fmtm_images: + name: fmtm-images-${GIT_BRANCH} fmtm_tiles: - traefik-public-certificates: + name: fmtm-tiles-${GIT_BRANCH} + certs: + name: fmtm-certs-${GIT_BRANCH} + certbot_data: + name: fmtm-certbot-data-${GIT_BRANCH} + central_db_data: + name: fmtm-central-db-data-${GIT_BRANCH} + central_frontend: + name: fmtm-central-frontend-${GIT_BRANCH} networks: fmtm-net: name: fmtm-${GIT_BRANCH} services: - traefik: - image: "docker.io/traefik:v2.8" - container_name: fmtm_proxy + proxy: + image: "ghcr.io/hotosm/fmtm/proxy:${GIT_BRANCH}" + container_name: fmtm-${GIT_BRANCH} + depends_on: + api: + condition: service_started + central: + condition: service_started + central-ui: + condition: service_completed_successfully + s3: + condition: service_started + certbot: + condition: service_completed_successfully + ui: + condition: service_completed_successfully volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - "traefik-public-certificates:/certificates" + - fmtm_frontend:/usr/share/nginx/html/fmtm/ + - central_frontend:/usr/share/nginx/html/central/ + - certs:/etc/letsencrypt + - certbot_data:/var/www/certbot + environment: + FMTM_DOMAIN: ${FMTM_DOMAIN} + FMTM_API_DOMAIN: ${FMTM_API_DOMAIN:-${VITE_API_URL:-api.${FMTM_DOMAIN}}} + FMTM_ODK_DOMAIN: ${FMTM_ODK_DOMAIN:-odk.${FMTM_DOMAIN}} + FMTM_S3_DOMAIN: ${FMTM_S3_DOMAIN:-s3.${FMTM_DOMAIN}} + FMTM_SCRIPT_DOMAIN: ${FMTM_SCRIPT_DOMAIN:-_} ports: - 80:80 - 443:443 networks: - fmtm-net restart: "unless-stopped" - command: - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" - - "--certificatesresolvers.letsencrypt.acme.email=admin@hotosm.org" - - "--certificatesresolvers.letsencrypt.acme.storage=/certificates/acme.json" - ## Enable below for staging tests - # - "--certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory" - labels: - - "traefik.enable=true" - - "traefik.docker.network=fmtm-${GIT_BRANCH}" - - "traefik.api.dashboard=false" - - "traefik.http.middlewares.https_redirect.redirectscheme.scheme=https" - - "traefik.http.middlewares.https_redirect.redirectscheme.permanent=true" - - "traefik.http.routers.http_catchall.rule=HostRegexp(`{any:.+}`) && !PathPrefix(`/.well-known/acme-challenge/`)" - - "traefik.http.routers.http_catchall.entrypoints=web" - - "traefik.http.routers.http_catchall.middlewares=https_redirect" - - fmtm-db: - image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" - container_name: fmtm_db - volumes: - - fmtm_db_data:/var/lib/postgresql/data/ - environment: - - POSTGRES_USER=${FMTM_DB_USER:-fmtm} - - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD:-fmtm} - - POSTGRES_DB=${FMTM_DB_NAME:-fmtm} - ports: - - "5433:5432" - networks: - - fmtm-net - restart: "unless-stopped" - healthcheck: - test: pg_isready -U ${FMTM_DB_USER:-fmtm} -d ${FMTM_DB_NAME:-fmtm} - start_period: 5s - interval: 10s - timeout: 5s - retries: 3 api: image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" - container_name: fmtm_api + container_name: fmtm-api-${GIT_BRANCH} volumes: - fmtm_logs:/opt/logs - fmtm_images:/opt/app/images - fmtm_tiles:/opt/tiles depends_on: - - fmtm-db - - migrations - - traefik + fmtm-db: + condition: service_healthy + migrations: + condition: service_completed_successfully + s3: + condition: service_started env_file: - .env networks: - fmtm-net restart: "unless-stopped" - labels: - - "traefik.enable=true" - - "traefik.http.routers.api.tls=true" - - "traefik.http.routers.api.tls.certresolver=letsencrypt" - - "traefik.http.routers.api.rule=Host(`${API_URL}`)" - - "traefik.http.services.api-svc.loadbalancer.server.port=8000" - - "traefik.http.routers.api.service=api-svc" - - migrations: - image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" - container_name: fmtm_migrations - depends_on: - - fmtm-db - env_file: - - .env - networks: - - fmtm-net - entrypoint: ["/migrate-entrypoint.sh"] - restart: "on-failure:3" ui: - image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH}" + # This service simply builds the frontend to a volume + # accessible to the proxy, then shuts down + image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH:-development}" build: context: src/frontend dockerfile: prod.dockerfile args: - VITE_API_URL: ${URL_SCHEME}://${API_URL} - container_name: fmtm + APP_VERSION: ${GIT_BRANCH} + VITE_API_URL: ${VITE_API_URL:-https://api.${FMTM_DOMAIN}} + container_name: fmtm-ui-${GIT_BRANCH} + volumes: + - fmtm_frontend:/frontend + network_mode: none + restart: "on-failure:2" + + central: + image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_TAG:-v2023.4.0}" + container_name: fmtm-central-${GIT_BRANCH} depends_on: - - api - - traefik + central-db: + condition: service_healthy + environment: + - DOMAIN=local + - SYSADMIN_EMAIL=${ODK_CENTRAL_USER} + - SYSADMIN_PASSWD=${ODK_CENTRAL_PASSWD} + - HTTPS_PORT=443 + - DB_HOST=central-db + - DB_USER=${CENTRAL_DB_USER} + - DB_PASSWORD=${CENTRAL_DB_PASSWORD} + - DB_NAME=${CENTRAL_DB_NAME} + - DB_SSL=null + - EMAIL_FROM=${ODK_CENTRAL_USER} + - EMAIL_HOST=${EMAIL_HOST:-mail} + - EMAIL_PORT=${EMAIL_PORT:-25} + - EMAIL_SECURE=${EMAIL_SECURE:-false} + - EMAIL_IGNORE_TLS=${EMAIL_IGNORE_TLS:-true} + - EMAIL_USER=${EMAIL_USER:-''} + - EMAIL_PASSWORD=${EMAIL_PASSWORD:-''} + - OIDC_ENABLED=${OIDC_ENABLED:-false} + - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} + - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} + - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} networks: - fmtm-net - environment: - - BROTLI=true - - VITE_API_URL=${URL_SCHEME}://${API_URL} restart: "unless-stopped" - labels: - - "traefik.enable=true" - - "traefik.http.routers.ui-main.tls=true" - - "traefik.http.routers.ui-main.tls.certresolver=letsencrypt" - - "traefik.http.routers.ui-main.rule=Host(`${FRONTEND_MAIN_URL}`)" - - "traefik.http.services.ui-main-svc.loadbalancer.server.port=8080" - - "traefik.http.routers.ui-main.service=ui-main-svc" + + central-ui: + # This service simply builds the frontend to a volume + # accessible to the proxy, then shuts down + image: "ghcr.io/hotosm/fmtm/odkcentral-ui:${ODK_CENTRAL_TAG:-v2023.4.0}" + container_name: fmtm-central-ui-${GIT_BRANCH} + volumes: + - central_frontend:/frontend + network_mode: none + restart: "on-failure:2" s3: - image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" - container_name: fmtm_s3 + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-25T06-33-25Z}" + container_name: fmtm-s3-${GIT_BRANCH} environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY} MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY} @@ -165,3 +175,103 @@ services: interval: 5s timeout: 5s retries: 3 + + fmtm-db: + image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}" + container_name: fmtm-db-${GIT_BRANCH} + volumes: + - fmtm_db_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=${FMTM_DB_USER} + - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD} + - POSTGRES_DB=${FMTM_DB_NAME} + ports: + - "5433:5432" + networks: + - fmtm-net + restart: "unless-stopped" + healthcheck: + test: pg_isready -U ${FMTM_DB_USER} -d ${FMTM_DB_NAME} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + central-db: + image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}" + container_name: fmtm-central-db-${GIT_BRANCH} + volumes: + - central_db_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=${CENTRAL_DB_USER} + - POSTGRES_PASSWORD=${CENTRAL_DB_PASSWORD} + - POSTGRES_DB=${CENTRAL_DB_NAME} + ports: + - "5434:5432" + networks: + - fmtm-net + restart: "on-failure:3" + healthcheck: + test: pg_isready -U ${CENTRAL_DB_USER} -d ${CENTRAL_DB_NAME} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + migrations: + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" + container_name: fmtm-migrations-${GIT_BRANCH} + depends_on: + fmtm-db: + condition: service_healthy + s3: + condition: service_healthy + env_file: + - .env + networks: + - fmtm-net + entrypoint: ["/migrate-entrypoint.sh"] + restart: "on-failure:3" + + backups: + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" + container_name: fmtm-backups-${GIT_BRANCH} + depends_on: + fmtm-db: + condition: service_healthy + central-db: + condition: service_healthy + s3: + condition: service_healthy + env_file: + - .env + networks: + - fmtm-net + entrypoint: ["/backup-entrypoint.sh"] + restart: "on-failure:3" + healthcheck: + test: pg_isready -U ${FMTM_DB_USER} -d ${FMTM_DB_NAME} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + certbot: + image: "ghcr.io/hotosm/fmtm/proxy:certs-init-development" + container_name: fmtm-cert-renew-${GIT_BRANCH} + volumes: + - certs:/etc/letsencrypt + - certbot_data:/var/www/certbot + environment: + FMTM_DOMAIN: ${FMTM_DOMAIN} + FMTM_API_DOMAIN: ${FMTM_API_DOMAIN:-${VITE_API_URL:-api.${FMTM_DOMAIN}}} + FMTM_ODK_DOMAIN: ${FMTM_ODK_DOMAIN:-odk.${FMTM_DOMAIN}} + FMTM_S3_DOMAIN: ${FMTM_S3_DOMAIN:-s3.${FMTM_DOMAIN}} + FMTM_SCRIPT_DOMAIN: ${FMTM_SCRIPT_DOMAIN:-_} + CERT_EMAIL: ${CERT_EMAIL} + ports: + - 80:80 + - 443:443 + networks: + - fmtm-net + restart: "on-failure:2" diff --git a/docker-compose.main.yml b/docker-compose.main.yml index 4429fa57a8..54ab3ac621 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -18,128 +18,152 @@ version: "3" volumes: + fmtm_frontend: + name: fmtm-frontend-${GIT_BRANCH} fmtm_db_data: + name: fmtm-db-data-${GIT_BRANCH} fmtm_logs: + name: fmtm-logs-${GIT_BRANCH} fmtm_images: + name: fmtm-images-${GIT_BRANCH} fmtm_tiles: - traefik-public-certificates: + name: fmtm-tiles-${GIT_BRANCH} + certs: + name: fmtm-certs-${GIT_BRANCH} + certbot_data: + name: fmtm-certbot-data-${GIT_BRANCH} networks: fmtm-net: name: fmtm-${GIT_BRANCH} services: - traefik: - image: "docker.io/traefik:v2.8" - container_name: fmtm_proxy + proxy: + image: "ghcr.io/hotosm/fmtm/proxy:${GIT_BRANCH}" + container_name: fmtm-${GIT_BRANCH} + depends_on: + api: + condition: service_started + certbot: + condition: service_completed_successfully + ui: + condition: service_completed_successfully volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - "traefik-public-certificates:/certificates" + - fmtm_frontend:/usr/share/nginx/html/fmtm/ + - certs:/etc/letsencrypt + - certbot_data:/var/www/certbot + environment: + FMTM_DOMAIN: ${FMTM_DOMAIN} + FMTM_API_DOMAIN: ${VITE_API_URL:-api.${FMTM_DOMAIN}} + FMTM_SCRIPT_DOMAIN: ${FMTM_SCRIPT_DOMAIN:-_} ports: - 80:80 - 443:443 networks: - fmtm-net restart: "unless-stopped" - command: - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" - - "--certificatesresolvers.letsencrypt.acme.email=admin@hotosm.org" - - "--certificatesresolvers.letsencrypt.acme.storage=/certificates/acme.json" - ## Enable below for staging tests - # - "--certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory" - labels: - - "traefik.enable=true" - - "traefik.docker.network=fmtm-${GIT_BRANCH}" - - "traefik.api.dashboard=false" - - "traefik.http.middlewares.https_redirect.redirectscheme.scheme=https" - - "traefik.http.middlewares.https_redirect.redirectscheme.permanent=true" - - "traefik.http.routers.http_catchall.rule=HostRegexp(`{any:.+}`) && !PathPrefix(`/.well-known/acme-challenge/`)" - - "traefik.http.routers.http_catchall.entrypoints=web" - - "traefik.http.routers.http_catchall.middlewares=https_redirect" + + api: + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" + container_name: fmtm-api-${GIT_BRANCH} + volumes: + - fmtm_logs:/opt/logs + - fmtm_images:/opt/app/images + - fmtm_tiles:/opt/tiles + depends_on: + fmtm-db: + condition: service_healthy + migrations: + condition: service_completed_successfully + env_file: + - .env + networks: + - fmtm-net + restart: "unless-stopped" + + ui: + # This service simply builds the frontend to a volume + # accessible to the proxy, then shuts down + image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH:-main}" + build: + context: src/frontend + dockerfile: prod.dockerfile + args: + APP_VERSION: ${GIT_BRANCH} + VITE_API_URL: ${VITE_API_URL:-https://api.${FMTM_DOMAIN}} + container_name: fmtm-ui-${GIT_BRANCH} + volumes: + - fmtm_frontend:/frontend + network_mode: none + restart: "on-failure:2" fmtm-db: - image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" - container_name: fmtm_db + image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}" + container_name: fmtm-db-${GIT_BRANCH} volumes: - fmtm_db_data:/var/lib/postgresql/data/ environment: - - POSTGRES_USER=${FMTM_DB_USER:-fmtm} - - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD:-fmtm} - - POSTGRES_DB=${FMTM_DB_NAME:-fmtm} + - POSTGRES_USER=${FMTM_DB_USER} + - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD} + - POSTGRES_DB=${FMTM_DB_NAME} ports: - "5433:5432" networks: - fmtm-net restart: "unless-stopped" healthcheck: - test: pg_isready -U ${FMTM_DB_USER:-fmtm} -d ${FMTM_DB_NAME:-fmtm} + test: pg_isready -U ${FMTM_DB_USER} -d ${FMTM_DB_NAME} start_period: 5s interval: 10s timeout: 5s retries: 3 - api: + migrations: image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" - container_name: fmtm_api - volumes: - - fmtm_logs:/opt/logs - - fmtm_images:/opt/app/images - - fmtm_tiles:/opt/tiles + container_name: fmtm-migrations-${GIT_BRANCH} depends_on: - - fmtm-db - - migrations - - traefik + fmtm-db: + condition: service_healthy env_file: - .env networks: - fmtm-net - restart: "unless-stopped" - labels: - - "traefik.enable=true" - - "traefik.http.routers.api.tls=true" - - "traefik.http.routers.api.tls.certresolver=letsencrypt" - - "traefik.http.routers.api.rule=Host(`${API_URL}`)" - - "traefik.http.services.api-svc.loadbalancer.server.port=8000" - - "traefik.http.routers.api.service=api-svc" + entrypoint: ["/migrate-entrypoint.sh"] + restart: "on-failure:3" - migrations: + backups: image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" - container_name: fmtm_migrations + container_name: fmtm-backups-${GIT_BRANCH} depends_on: - - fmtm-db + fmtm-db: + condition: service_healthy env_file: - .env networks: - fmtm-net - entrypoint: ["/migrate-entrypoint.sh"] + entrypoint: ["/backup-entrypoint.sh"] restart: "on-failure:3" + healthcheck: + test: pg_isready -U ${FMTM_DB_USER} -d ${FMTM_DB_NAME} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 - ui: - image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH}" - build: - context: src/frontend - dockerfile: prod.dockerfile - args: - VITE_API_URL: ${URL_SCHEME}://${API_URL} - container_name: fmtm - depends_on: - - api - - traefik + certbot: + image: "ghcr.io/hotosm/fmtm/proxy:certs-init-main" + container_name: fmtm-cert-renew-${GIT_BRANCH} + volumes: + - certs:/etc/letsencrypt + - certbot_data:/var/www/certbot + environment: + FMTM_DOMAIN: ${FMTM_DOMAIN} + FMTM_API_DOMAIN: ${FMTM_API_DOMAIN:-${VITE_API_URL:-api.${FMTM_DOMAIN}}} + FMTM_SCRIPT_DOMAIN: ${FMTM_SCRIPT_DOMAIN:-_} + CERT_EMAIL: ${CERT_EMAIL} + ports: + - 80:80 + - 443:443 networks: - fmtm-net - environment: - - BROTLI=true - - VITE_API_URL=${URL_SCHEME}://${API_URL} - restart: "unless-stopped" - labels: - - "traefik.enable=true" - - "traefik.http.routers.ui-main.tls=true" - - "traefik.http.routers.ui-main.tls.certresolver=letsencrypt" - - "traefik.http.routers.ui-main.rule=Host(`${FRONTEND_MAIN_URL}`)" - - "traefik.http.services.ui-main-svc.loadbalancer.server.port=8080" - - "traefik.http.routers.ui-main.service=ui-main-svc" + restart: "on-failure:2" diff --git a/docker-compose.noodk.yml b/docker-compose.noodk.yml deleted file mode 100644 index 1e8d68f0de..0000000000 --- a/docker-compose.noodk.yml +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team -# This file is part of FMTM. -# -# FMTM is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# FMTM is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with FMTM. If not, see . -# - -version: "3" - -volumes: - fmtm_db_data: - fmtm_data: - fmtm_logs: - fmtm_images: - fmtm_tiles: - -networks: - fmtm-dev: - name: fmtm-dev - -services: - fmtm-db: - image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" - container_name: fmtm_db - volumes: - - fmtm_db_data:/var/lib/postgresql/data/ - environment: - - POSTGRES_USER=${FMTM_DB_USER:-fmtm} - - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD:-fmtm} - - POSTGRES_DB=${FMTM_DB_NAME:-fmtm} - ports: - - "5433:5432" - networks: - - fmtm-dev - restart: "unless-stopped" - healthcheck: - test: pg_isready -U ${FMTM_DB_USER:-fmtm} -d ${FMTM_DB_NAME:-fmtm} - start_period: 5s - interval: 10s - timeout: 5s - retries: 3 - - api: - image: "ghcr.io/hotosm/fmtm/backend:debug" - build: - context: src/backend - target: debug-no-odk - args: - APP_VERSION: debug - container_name: fmtm_api - volumes: - - fmtm_logs:/opt/logs - - fmtm_images:/opt/app/images - - fmtm_tiles:/opt/tiles - - ./src/backend/app:/opt/app - depends_on: - - fmtm-db - - migrations - env_file: - - .env - ports: - - "7050:8000" - - "5678:5678" - networks: - - fmtm-dev - restart: "unless-stopped" - - migrations: - image: "ghcr.io/hotosm/fmtm/backend:debug" - container_name: fmtm_migrations - depends_on: - - fmtm-db - env_file: - - .env - networks: - - fmtm-dev - entrypoint: ["/migrate-entrypoint.sh"] - restart: "on-failure:3" - - ui: - image: "ghcr.io/hotosm/fmtm/frontend:debug" - build: - context: src/frontend - dockerfile: debug.dockerfile - args: - VITE_API_URL: ${URL_SCHEME}://${API_URL} - container_name: fmtm - depends_on: - - api - volumes: - - ./src/frontend:/app - - /app/node_modules/ - environment: - - VITE_API_URL=${URL_SCHEME}://${API_URL} - - VITE_ODK_CENTRAL_URL=${ODK_CENTRAL_URL} - - VITE_ODK_CENTRAL_USER=${ODK_CENTRAL_USER} - - VITE_ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD} - ports: - - "8081:8081" - networks: - - fmtm-dev - restart: "unless-stopped" - - s3: - image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" - container_name: fmtm_s3 - environment: - MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm} - MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-somelongpassword} - MINIO_VOLUMES: "/mnt/data" - MINIO_BROWSER: "off" - volumes: - - fmtm_data:/mnt/data - ports: - - 9000:9000 - networks: - - fmtm-dev - command: minio server # --console-address ":9090" - restart: "unless-stopped" - healthcheck: - test: curl --fail http://localhost:9000/minio/health/live || exit 1 - start_period: 5s - interval: 5s - timeout: 5s - retries: 3 diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml new file mode 100644 index 0000000000..68d53796b9 --- /dev/null +++ b/docker-compose.staging.yml @@ -0,0 +1,75 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +version: "3" + +volumes: + fmtm_data: + fmtm_db_data: + fmtm_logs: + fmtm_images: + fmtm_tiles: + central_db_data: + certs: + +networks: + fmtm-net: + name: fmtm-${GIT_BRANCH} + +services: + proxy: + extends: + file: docker-compose.development.yml + service: proxy + api: + extends: + file: docker-compose.development.yml + service: api + ui: + extends: + file: docker-compose.development.yml + service: ui + image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH:-staging}" + central: + extends: + file: docker-compose.development.yml + service: central + s3: + extends: + file: docker-compose.development.yml + service: s3 + fmtm-db: + extends: + file: docker-compose.development.yml + service: fmtm-db + central-db: + extends: + file: docker-compose.development.yml + service: central-db + migrations: + extends: + file: docker-compose.development.yml + service: migrations + backups: + extends: + file: docker-compose.development.yml + service: backups + certbot: + extends: + file: docker-compose.development.yml + service: certbot + image: "ghcr.io/hotosm/fmtm/proxy:certs-init-staging" diff --git a/docker-compose.yml b/docker-compose.yml index f52994d8bd..a638d80e17 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,36 +20,45 @@ version: "3" volumes: fmtm_data: fmtm_db_data: - central_db_data: fmtm_logs: fmtm_images: fmtm_tiles: + central_db_data: + central_frontend: networks: - fmtm-dev: - name: fmtm-dev + fmtm-net: + name: fmtm-local services: - fmtm-db: - image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" - container_name: fmtm_db + proxy: + image: "ghcr.io/hotosm/fmtm/proxy:debug" + build: + context: nginx + target: debug + args: + NGINX_TAG: "${NGINX_TAG:-1.25.2}" + container_name: fmtm + depends_on: + api: + condition: service_started + ui: + condition: service_started + central: + condition: service_started + required: false + central-ui: + condition: service_completed_successfully + required: false + s3: + condition: service_started volumes: - - fmtm_db_data:/var/lib/postgresql/data/ - environment: - - POSTGRES_USER=${FMTM_DB_USER:-fmtm} - - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD:-fmtm} - - POSTGRES_DB=${FMTM_DB_NAME:-fmtm} + - central_frontend:/usr/share/nginx/html/central ports: - - "5438:5432" + - ${FMTM_DEV_PORT:-7050}:80 networks: - - fmtm-dev + - fmtm-net restart: "unless-stopped" - healthcheck: - test: pg_isready -U ${FMTM_DB_USER:-fmtm} -d ${FMTM_DB_NAME:-fmtm} - start_period: 5s - interval: 10s - timeout: 5s - retries: 3 api: image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" @@ -57,8 +66,8 @@ services: context: src/backend target: debug-with-odk args: - APP_VERSION: "debug" - container_name: fmtm_api + APP_VERSION: "${TAG_OVERRIDE:-debug}" + container_name: fmtm-api # Uncomment these to debug with a terminal debugger like pdb # Then `docker attach fmtm_api` to debug # stdin_open: true @@ -72,87 +81,58 @@ services: - ./src/backend/tests:/opt/tests # - ../osm-fieldwork/osm_fieldwork:/home/appuser/.local/lib/python3.10/site-packages/osm_fieldwork depends_on: - - fmtm-db - - migrations - - s3 - - central-proxy + fmtm-db: + condition: service_healthy + central: + condition: service_healthy + required: false + migrations: + condition: service_completed_successfully + s3: + condition: service_started env_file: - .env - ports: - - "8000:8000" - - "5678:5678" + # ports: + # - "7052:8000" + # - "5678:5678" # Debugger port networks: - - fmtm-dev + - fmtm-net restart: "unless-stopped" - migrations: - image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" - container_name: fmtm_migrations - depends_on: - - fmtm-db - env_file: - - .env - networks: - - fmtm-dev - entrypoint: ["/migrate-entrypoint.sh"] - restart: "on-failure:3" - healthcheck: - test: [] # Set the health check test to an empty value to disable it - ui: image: "ghcr.io/hotosm/fmtm/frontend:debug" build: context: src/frontend dockerfile: debug.dockerfile - args: - VITE_API_URL: ${URL_SCHEME}://${API_URL} - container_name: fmtm + container_name: fmtm-ui depends_on: - - api + api: + condition: service_started volumes: - ./src/frontend:/app - /app/node_modules/ environment: - - VITE_API_URL=${URL_SCHEME}://${API_URL} + - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} - VITE_ODK_CENTRAL_URL=${ODK_CENTRAL_URL} - VITE_ODK_CENTRAL_USER=${ODK_CENTRAL_USER} - VITE_ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD} ports: - - "8080:8080" - networks: - - fmtm-dev - restart: "unless-stopped" - - central-db: - image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" - container_name: central_db - volumes: - - central_db_data:/var/lib/postgresql/data/ - environment: - - POSTGRES_USER=${CENTRAL_DB_USER:-odk} - - POSTGRES_PASSWORD=${CENTRAL_DB_PASSWORD:-odk} - - POSTGRES_DB=${CENTRAL_DB_NAME:-odk} - ports: - - "5434:5432" + - "7051:7051" networks: - - fmtm-dev + - fmtm-net restart: "unless-stopped" - healthcheck: - test: pg_isready -U ${CENTRAL_DB_USER:-odk} -d ${CENTRAL_DB_NAME:-odk} - start_period: 5s - interval: 10s - timeout: 5s - retries: 3 central: + profiles: ["", "central"] image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_TAG:-v2023.4.0}" build: context: odkcentral/api args: - ODK_CENTRAL_TAG: v2023.4.0 - container_name: central_api + ODK_CENTRAL_TAG: ${ODK_CENTRAL_TAG:-v2023.4.0} + container_name: fmtm-central depends_on: - - central-db + central-db: + condition: service_healthy environment: - DOMAIN=local - SYSADMIN_EMAIL=${ODK_CENTRAL_USER} @@ -174,26 +154,30 @@ services: - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} - ports: - - "8383:8383" + # ports: + # - "8383:8383" networks: - - fmtm-dev + - fmtm-net restart: "unless-stopped" - central-proxy: - image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:${ODK_CENTRAL_TAG:-v2023.4.0}" + central-ui: + # This service simply builds the frontend to a volume + # accessible to the proxy, then shuts down + profiles: ["central"] + image: "ghcr.io/hotosm/fmtm/odkcentral-ui:${ODK_CENTRAL_TAG:-v2023.4.0}" build: - context: odkcentral/proxy - container_name: central_proxy - depends_on: - - central - networks: - - fmtm-dev - restart: "unless-stopped" + context: odkcentral/ui + args: + ODK_CENTRAL_TAG: ${ODK_CENTRAL_TAG:-v2023.4.0} + container_name: fmtm-central-ui + volumes: + - central_frontend:/frontend + network_mode: none + restart: "on-failure:2" s3: - image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" - container_name: fmtm_s3 + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-25T06-33-25Z}" + container_name: fmtm-s3 environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm} MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-somelongpassword} @@ -201,10 +185,10 @@ services: MINIO_BROWSER: "off" volumes: - fmtm_data:/mnt/data - ports: - - 9000:9000 + # ports: + # - 9000:9000 networks: - - fmtm-dev + - fmtm-net command: minio server # --console-address ":9090" restart: "unless-stopped" healthcheck: @@ -213,3 +197,63 @@ services: interval: 5s timeout: 5s retries: 3 + + fmtm-db: + image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}" + container_name: fmtm-db + volumes: + - fmtm_db_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=${FMTM_DB_USER:-fmtm} + - POSTGRES_PASSWORD=${FMTM_DB_PASSWORD:-fmtm} + - POSTGRES_DB=${FMTM_DB_NAME:-fmtm} + ports: + - "5438:5432" + networks: + - fmtm-net + restart: "unless-stopped" + healthcheck: + test: pg_isready -U ${FMTM_DB_USER:-fmtm} -d ${FMTM_DB_NAME:-fmtm} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + central-db: + profiles: ["", "central"] + image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}" + container_name: fmtm-central-db + volumes: + - central_db_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=${CENTRAL_DB_USER:-odk} + - POSTGRES_PASSWORD=${CENTRAL_DB_PASSWORD:-odk} + - POSTGRES_DB=${CENTRAL_DB_NAME:-odk} + ports: + - "5434:5432" + networks: + - fmtm-net + restart: "on-failure:3" + healthcheck: + test: pg_isready -U ${CENTRAL_DB_USER:-odk} -d ${CENTRAL_DB_NAME:-odk} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + migrations: + image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" + container_name: fmtm-migrations + depends_on: + fmtm-db: + condition: service_healthy + s3: + condition: service_healthy + env_file: + - .env + networks: + - fmtm-net + entrypoint: ["/migrate-entrypoint.sh"] + restart: "on-failure:3" + healthcheck: + test: [] # Set the health check test to an empty value to disable it diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md deleted file mode 100644 index ab57cfe195..0000000000 --- a/docs/_Sidebar.md +++ /dev/null @@ -1,29 +0,0 @@ -# [Home](https://github.com/hotosm/fmtm/wiki) - -[About](https://github.com/hotosm/fmtm/wiki/About) - -[Code of Conduct](https://github.com/hotosm/fmtm/wiki/CODE_OF_CONDUCT) - -[User Manual for FMTM](https://github.com/hotosm/fmtm/wiki/User-Manual-For-Project-Managers) - -[Contribution](https://github.com/hotosm/fmtm/wiki/CONTRIBUTING) - -[FAQ](https://github.com/hotosm/fmtm/wiki/FAQ) - -## For Developers - -[1. Getting Started](https://github.com/hotosm/fmtm/wiki/dev/Setup) - -[2. Backend](https://github.com/hotosm/fmtm/wiki/dev/Backend) - -[3. Frontend](https://github.com/hotosm/fmtm/wiki/Dev/Frontend) - -[4. Database Tips](https://github.com/hotosm/fmtm/wiki/dev/Database-Tips) - -[5. Docker Tips](https://github.com/hotosm/fmtm/wiki/dev/Docker-Tips) - -[6. Production Deployment](https://github.com/hotosm/fmtm/wiki/dev/Production) - -[7. Deployment Flow](https://github.com/hotosm/fmtm/wiki/dev/Deployment-Flow) - -[8. Troubleshooting](https://github.com/hotosm/fmtm/wiki/dev/Troubleshooting) diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md index 9ef6ae7ec9..ea4af0a69f 100644 --- a/docs/dev/Backend.md +++ b/docs/dev/Backend.md @@ -18,9 +18,9 @@ The easiest way to get up and running is by using the FMTM Docker deployment. Do This will pull the latest container builds from **main** branch. 4. Make sure you have a `.env` file with all required variables, see [Getting Started](https://github.com/hotosm/fmtm/blob/main/docs/DEV-1.-Getting-Started.md). 5. Once everything is pulled, from the command line run: `docker compose up -d api` -6. If everything goes well you should now be able to **navigate to the project in your browser:** `http://127.0.0.1:8000/docs` +6. If everything goes well you should now be able to **navigate to the project in your browser:** `http://api.fmtm.localhost:7050/docs` -> Note: If that link doesn't work, check the logs with `docker log fmtm_api`. +> Note: If that link doesn't work, check the logs with `docker log fmtm-api`. > Note: the database host `fmtm-db` is automatically resolved by docker compose to the database container IP. @@ -35,7 +35,7 @@ The easiest way to get up and running is by using the FMTM Docker deployment. Do Some test data is available to get started quickly. 1. Navigate to the `import-test-data` endpoint in the API docs page: - + 2. Click `Try it out`, then `execute`. ## 2. Start the API without Docker @@ -52,7 +52,9 @@ Running the database in Docker means postgres does not need to be installed on y 2. Start an instance of Postgres (with Postgis): ```bash -docker run -d --name fmtm_db -e POSTGRES_PASSWORD=xxxx -p 5432:5432 postgis/postgis:15-3.3 +GIT_BRANCH=development + +docker run -d --name fmtm-db-${GIT_BRANCH} -e POSTGRES_PASSWORD=xxxx -p 5432:5432 postgis/postgis:15-3.3 ``` The database should be accessible at localhost:5432. @@ -72,7 +74,7 @@ After starting the database, from the command line: 3. Install backend dependencies with PDM: `pdm install` 4. Run the Fast API backend with: `pdm run uvicorn app.main:api --host 0.0.0.0 --port 8000` -The API should now be accessible at: +The API should now be accessible at: ## 3. Hybrid Docker/Local @@ -148,7 +150,7 @@ Example launch.json config for vscode: - During project creation a Central ODK URL must be provided. - If you set up FMTM with docker and have ODK Central running in a container, you can use the URL: - `https://central-proxy` + `https://proxy` - The credentials should be present in your `.env` file. ## Debugging osm-fieldwork diff --git a/docs/dev/Database-Tips.md b/docs/dev/Database-Tips.md index 37eeab4539..17f9faa227 100644 --- a/docs/dev/Database-Tips.md +++ b/docs/dev/Database-Tips.md @@ -12,10 +12,12 @@ psql -d fmtm -U fmtm -h localhost ### Option 2 -Access a PostgreSQL shell inside the fmtm_db container: +Access a PostgreSQL shell inside the fmtm-db container: ```bash -docker exec -it fmtm_db psql -U fmtm fmtm +GIT_BRANCH=development + +docker exec -it fmtm-db-${GIT_BRANCH} psql -U fmtm fmtm ``` And then connect to the database using this command: diff --git a/docs/dev/Docker-Tips.md b/docs/dev/Docker-Tips.md deleted file mode 100644 index 68d465a833..0000000000 --- a/docs/dev/Docker-Tips.md +++ /dev/null @@ -1,138 +0,0 @@ -# Docker organization - -This section explains how the Docker deployment works when running -`docker-compose`. It gives an overview of what happens when the -scripts in `Dockerfile` are run and what happens for each container. - -When running `docker-compose`, it sets up the environment for the -project and deploys the containers specified in the -`docker-compose.yml` file. The scripts in the `Dockerfile` are used to -set up the environment and dependencies for each container. - -### For example, let's say that we have the following `docker-compose.yml` file - - version: "3" - services: - db: - image: postgres:latest - environment: - POSTGRES_USER: user - POSTGRES_PASSWORD: password - api: - build: - context: . - dockerfile: Dockerfile - environment: - DATABASE_URL: postgresql://user:password@db:5432/mydb - ports: - - "8000:8000" - web: - build: - context: . - dockerfile: Dockerfile - environment: - API_URL: http://api:8000 - ports: - - "3000:3000" - -In this file, we have three containers: `db`, `api`, and `web`. For -each container, we specify the `image` or `build` to use, any -environment variables to set, and any ports to expose. - -When we run `docker-compose up`, the scripts in the `Dockerfile` will -be run for each container, setting up the environment and dependencies -needed for the container to run. Then, for each container, the -environment variables will be set, the execute command will be run, -and the container will be deployed at the given port. - -> Note: this Docker deployment automatically sets up a traefik server with ssh. - -# When updating docker - -This section explains what to do when updating Docker. It's important -to make sure that all the necessary files are updated, including -`.prod` and `.local` `docker-compose` or `Dockerfiles`. - -For example, if we update the `Dockerfile` to include a new package -that our application needs, we need to make sure that we also update -the `.prod` and `.local` files as needed. Otherwise, our production -deployment may be missing the necessary dependencies, causing our -application to fail. - -# Debugging when using docker - -This section explains how to debug an application when using -Docker. It gives an example of how to open a TTY to a container and -add a debug line in code using IPython debugger (ipdb). - -When we are developing an application, we may encounter bugs that we -need to debug. Docker can make this process a bit more complicated, as -we need to access the running container to debug our code. - -To open a TTY to a container, we can use the following command: - - docker attach - -For example, if we want to open a TTY to the `fmtm-web-1` container, we would run: - - docker attach fmtm-web-1 - -Once we have a TTY open, we can add a debug line in our code using -IPython debugger (ipdb). This allows us to pause our code at a -specific point and interactively debug it using IPython commands. - -For example, let's say that we have the following Python code: - - def add_numbers(a, b): - import ipdb; ipdb.set_trace() - return a + b - -When we call the add_numbers function, our code will pause at the -import ipdb; ipdb.set_trace() line, and we can use IPython commands to -inspect variables - -When this line is reached in the code then the attached tty window will -become interactive with ipdb. - -A few of those commands: -[Command CheatSheet](https://wangchuan.github.io/coding/2017/07/12/ipdb-cheat-sheet.html) - -- `help`: displays the list of available IPython commands -- `h(elp) `: displays help for the specified command -- `w(here)`: shows the current position in the code -- `d(own)`: moves down one level in the call stack -- `u(p)`: moves up one level in the call stack -- `b(reak) [condition]`: sets a breakpoint at the - specified - location, with an optional condition -- `tbreak [condition]`: sets a temporary breakpoint at - the - specified location, with an optional condition -- `cl(ear) [bpnumber]`: clears the specified breakpoint, or all - breakpoints if no number is given -- `disable [ ...]`: disables the specified breakpoint(s) -- `enable [ ...]`: enables the specified breakpoint(s) -- `ignore `: sets the number of times to ignore the - specified breakpoint -- `condition `: sets a new condition for the - specified breakpoint -- `s(tep)`: steps to the next line of code -- `n(ext)`: executes the next line of code, without stepping into functions -- `unt(il)`: runs the code until it reaches a line with a greater - number than the current line -- `r(eturn)`: runs the code until it returns from the current function -- `run [args ...]`: starts the program with the specified arguments -- `c(ont(inue))`: continues running the program until the next - breakpoint or until it finishes -- `l(ist) [ []]`: displays the source code around the - current position, with an optional range of lines to show -- `a(rgs)`: shows the arguments of the current function -- `p `: evaluates the expression and prints its value - -To exit IPython debugging mode, we can press CTRL + D. - -### Conclusion - -Overall, the documentation provides a clear overview of how the Docker -deployment works when running `docker-compose`, what to do when -updating Docker, and how to debug an application when using Docker -using IPython debugger (ipdb). The provided examples make it easier to -understand how to implement these concepts in practice. diff --git a/docs/dev/Frontend.md b/docs/dev/Frontend.md index 31a6befbad..94e519a0b2 100644 --- a/docs/dev/Frontend.md +++ b/docs/dev/Frontend.md @@ -16,7 +16,7 @@ For details on how to run the API first, please see: [DEV 2. Backend](https://gi This is essential, as the development container for the frontend is different to production. 4. Once everything is built, from the command line run: `docker compose up -d ui` -5. If everything goes well you should now be able to **navigate to the project in your browser:** +5. If everything goes well you should now be able to **navigate to the project in your browser:** > Note: during development, if you rebuild the frontend, then > run 'docker compose up -d', the node_modules directory may diff --git a/docs/dev/Production.md b/docs/dev/Production.md index 2a8d8812e3..5dbefe9a44 100644 --- a/docs/dev/Production.md +++ b/docs/dev/Production.md @@ -49,56 +49,125 @@ with `cd fmtm`. ### Set up the environment and utilities to launch -Create the env file from the example with `cp .env.example .env`. Edit -that file to contain the needful (it should look like this): +Create the env file interactively with: - # ODK Central - ODK_CENTRAL_URL=https://central-proxy - ODK_CENTRAL_USER=`` - ODK_CENTRAL_PASSWD=`` +```bash +bash scripts/gen-env.sh +``` + +OR + +```bash +cp .env.example .env + +# Then edit values manually +``` + +Main variables of note to update: + +```dotenv +ODK_CENTRAL_USER=`` +ODK_CENTRAL_PASSWD=`` + +CERT_EMAIL=`` +OSM_CLIENT_ID=`` +OSM_CLIENT_SECRET=`` + +S3_ACCESS_KEY=`` +S3_SECRET_KEY=`` +``` + +> Note: If extra cors origins are required for testing, the variable +> `EXTRA_CORS_ORIGINS` is a set of comma separated strings, e.g.: +> + +#### API_PREFIX + +It is also possible to use the API_PREFIX variable if the api +is served under, e.g. /api on the domain. +However, this isn't the recommended approach, and testing is minimal. + +#### VITE_API_URL + +By default, the API URL for the frontend to use is: + +`api.${FMTM_DOMAIN}` + +If you wish to change this and use another domain, +then add the environment variable during the +frontend build: + +`VITE_API_URL=some.other.domain.org` - # FMTM - API_URL=https://fmtm-api.hotosm.org - FRONTEND_MAIN_URL=https://fmtm.hotosm.org - # API_PREFIX=/api +> Note: this is only used for the **frontend** build. - # OSM - OSM_CLIENT_ID=`` - OSM_CLIENT_SECRET=`` - OSM_URL=https://www.openstreetmap.org - OSM_SCOPE=read_prefs - OSM_LOGIN_REDIRECT_URI=``/osmauth/ - OSM_SECRET_KEY=`` +#### ODK\_ Variables - ### S3 File Storage ### - S3_ENDPOINT="http://s3:9000" - S3_ACCESS_KEY=`` - S3_SECRET_KEY=`` +These can point to an externally hosted instance of ODK Central. - FMTM_DB_HOST=fmtm-db - FMTM_DB_USER=fmtm - FMTM_DB_PASSWORD=`` - FMTM_DB_NAME=fmtm +Or ODK Central can be started as part of the FMTM docker compose +stack, and variables should be set accordingly. -> Note: It is also possible to use the API_PREFIX variable if the api is served under, e.g. /api on the domain. +#### Other Domains -> Note: You must have an existing version of ODKCentral running, to provide the URL and credentials here. +If you run FMTM with ODK and Minio (S3) included, then the +domains will default to: + +``` +${FMTM_DOMAIN} --> Frontend +api.${FMTM_DOMAIN} --> Backend +odk.${FMTM_DOMAIN} --> ODK Central +s3.${FMTM_DOMAIN} --> S3 / Minio +``` + +These defaults can be overriden with respective environment variables: + +``` +FMTM_API_DOMAIN (defaults to the value of VITE_API_URL if provided) +FMTM_ODK_DOMAIN +FMTM_S3_DOMAIN +``` + +### Start the Compose Stack Run the production docker-compose config: -`docker compose -f docker-compose.prod.yml up -d` +`docker compose -f docker-compose.main.yml up -d` -> Note: The images should be built already on Quay. If they don't exist, use the `--build` flag during run. +> Note: The images should be built already on Github. With any luck, this will launch the docker container where the project runs, and you can access the working website from the domain name! -## Connecting to a remote database +### Connecting to a remote database - A database may be located on a headless Linux server in the cloud. - To access the database via GUI tool such as PGAdmin, it is possible using port tunneling. ```bash -ssh username@server.domain -N -f -L 5430:localhost:5432 +ssh username@server.domain -N -f -L {local_port}:localhost:{remote_port} + +# Example +ssh root@fmtm.hotosm.org -N -f -L 5430:localhost:5433 ``` This will map port 5432 on the remote machine to port 5430 on your local machine. + +## Manual Database Backups + +```bash +GIT_BRANCH=development +backup_filename="fmtm-db-backup-$(date +'%Y-%m-%d').sql.gz" +echo $backup_filename + +docker exec -i -e PGPASSWORD=PASSWORD_HERE fmtm-db-${GIT_BRANCH} pg_dump --verbose --format c -U fmtm fmtm | gzip -9 > "$backup_filename" +``` + +## Manual Database Restores + +```bash +# On a different machine (else change the container name) +GIT_BRANCH=development +backup_filename=fmtm-db-backup-XXXX-XX-XX-sql.gz + +cat "$backup_filename" | gunzip | docker exec -i -e PGPASSWORD=NEW_PASSWORD_HERE fmtm-db-${GIT_BRANCH} pg_restore --verbose -U fmtm -d fmtm +``` diff --git a/docs/dev/Setup.md b/docs/dev/Setup.md index eec20af1de..43f88634b3 100644 --- a/docs/dev/Setup.md +++ b/docs/dev/Setup.md @@ -240,7 +240,7 @@ The FMTM uses OAUTH2 with OSM to authenticate users. To properly configure your 1. [Login to OSM](https://www.openstreetmap.org/login) (_If you do not have an account yet, click the signup button at the top navigation bar to create one_). Click the drop down arrow on the extreme right of the navigation bar and select My Settings. -2. Register your FMTM instance to OAuth 2 applications. Put your login redirect url as `http://127.0.0.1:8080/osmauth/`, For Production replace the URL as production API Url +2. Register your FMTM instance to OAuth 2 applications. Put your login redirect url as `http://127.0.0.1:7051/osmauth/`, For Production replace the URL as production API Url > Note: `127.0.0.1` is required instead of `localhost` due to OSM restrictions. @@ -250,54 +250,21 @@ The FMTM uses OAUTH2 with OSM to authenticate users. To properly configure your 4. Now Copy your Client ID and Client Secret. Put them in the `OSM_CLIENT_ID` and `OSM_CLIENT_SECRET` field of your `.env` file -### 2. Create an `.env` File - -Environmental variables are used throughout this project. To get started, create `.env` file in the top level dir, Sample is `.env.example` - - cp .env.example .env - -Your env should look like this: - -```dotenv -### ODK Central ### -ODK_CENTRAL_URL=https://central-proxy -ODK_CENTRAL_USER=`` -ODK_CENTRAL_PASSWD=`` - -### FMTM ### -# DEBUG=True -# LOG_LEVEL=DEBUG -URL_SCHEME=http -API_URL=127.0.0.1:8000 -FRONTEND_MAIN_URL=localhost:8080 -# API_PREFIX=/api - -### OSM ### -OSM_CLIENT_ID=`` -OSM_CLIENT_SECRET=`` -OSM_URL=https://www.openstreetmap.org -OSM_SCOPE=read_prefs -OSM_LOGIN_REDIRECT_URI=http://127.0.0.1:8080/osmauth/ -OSM_SECRET_KEY= - -### S3 File Storage ### -S3_ENDPOINT="http://s3:9000" -S3_ACCESS_KEY=`` -S3_SECRET_KEY=`` - -### Database (optional) ### -CENTRAL_DB_HOST=central-db -CENTRAL_DB_USER=odk -CENTRAL_DB_PASSWORD=odk -CENTRAL_DB_NAME=odk - -FMTM_DB_HOST=fmtm-db -FMTM_DB_USER=fmtm -FMTM_DB_PASSWORD=fmtm -FMTM_DB_NAME=fmtm' +#### 2. Create an `.env` File + +Environmental variables are used throughout this project. +To get started, create `.env` file in the top level dir, +a sample is located at `.env.example`. + +This can be created interactively by running: + +```bash +bash scripts/gen-env.sh ``` -> Note: If extra cors origins are required for testing, the variable `EXTRA_CORS_ORIGINS` is a set of comma separated strings, e.g. +> Note: If extra cors origins are required for testing, the variable +> `EXTRA_CORS_ORIGINS` is a set of comma separated strings, e.g.: +> ## Verify Setup @@ -309,7 +276,7 @@ For details on how to run this project locally for development, please look at: Once you have deployed, you will need to check that you can properly authenticate. -1. Navigate to `http://127.0.0.1:8000/docs` +1. Navigate to `http://api.fmtm.localhost:7050/docs` Three endpoints are responsible for oauth image @@ -336,3 +303,11 @@ Don't forget to review [Contribution](https://github.com/hotosm/fmtm/wiki/Contri Happy coding! The FMTM Developer Team + +## Note + +To run the local development setup without ODK Central (use external server): + +```bash +dc --profile no-odk up -d +``` diff --git a/docs/dev/Troubleshooting.md b/docs/dev/Troubleshooting.md index 290edaf7e0..cf9488bd92 100644 --- a/docs/dev/Troubleshooting.md +++ b/docs/dev/Troubleshooting.md @@ -42,7 +42,7 @@ If you would rather not do this, an alternative can be to feed them into the pdm command: ```bash -FRONTEND_MAIN_URL="" \ +FMTM_DOMAIN="" \ OSM_CLIENT_ID="" OSM_CLIENT_SECRET="" OSM_SECRET_KEY="" \ S3_ACCESS_KEY="" S3_SECRET_KEY="" \ pdm run uvicorn app.main:api --host 0.0.0.0 --port 8000 diff --git a/docs/wiki_redirect.md b/docs/wiki_redirect.md new file mode 100644 index 0000000000..68af28ab40 --- /dev/null +++ b/docs/wiki_redirect.md @@ -0,0 +1,4 @@ +# FMTM + +Please see the docs page at: +[https://hotosm.github.io/fmtm/](https://hotosm.github.io/fmtm/) diff --git a/gen-env.sh b/gen-env.sh deleted file mode 100644 index f876c85131..0000000000 --- a/gen-env.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/bin/bash -echo "Generate dotenv config for FMTM" - -echo "Reading .env.example" -source .env.example - -DOTENV_NAME=.env - -if [ -f "${DOTENV_NAME}" ] -then - echo "WARNING: ${DOTENV_NAME} file already exists." - echo "This script will overwrite the content of this file." - until [ "$conf" = "y" -o "$conf" = "n" ] - do - read -e -p "Do you want to overwrite it? y/n " conf - if [ "$conf" = "y" ] - then - conf="" - break - elif [ "$conf" = "n" ] - then - echo "Aborting." - exit 0 - else - echo "Invalid input!" - fi - done -fi - -# Debug -echo -echo "Is this a development deployment?" -while true -do - read -e -p "Enter y for development, anything else to continue: " debug - - if [ "$debug" = "y" ] - then - DEBUG=True - LOG_LEVEL="DEBUG" - echo "Using debug configuration." - else - DEBUG=False - LOG_LEVEL="INFO" - break - fi - break -done - -# External ODKCentral Creds -echo -echo "Do you want to use an external ODKCentral?" -while true -do - read -e -p "Enter y for external, anything else to continue: " externalodk - - if [ "$externalodk" = "y" ] - then - EXTERNAL_ODK="True" - echo "Using external ODKCentral." - fi - break -done - -if [ "$EXTERNAL_ODK" = "True" ] -then - echo - echo "Please enter the ODKCentral URL." - read -e -p "ODKCentral URL: " ODK_CENTRAL_URL - - echo "Please enter the ODKCentral User." - read -e -p "ODKCentral User: " ODK_CENTRAL_USER - - echo "Please enter the ODKCentral Password." - read -e -p "ODKCentral Password: " ODK_CENTRAL_PASSWD -fi - -# External DB Creds -echo -echo "Do you want to use an external database?" -while true -do - read -e -p "Enter y for external, anything else to continue: " externaldb - - if [ "$externaldb" = "y" ] - then - EXTERNAL_DB="True" - echo "Using external database." - fi - break -done - -if [ "$EXTERNAL_DB" = "True" ] -then - echo - echo "Please enter the database host." - read -e -p "FMTM DB Host: " FMTM_DB_HOST - - echo "Please enter the database name." - read -e -p "FMTM DB Name: " FMTM_DB_NAME - - echo "Please enter the database user." - read -e -p "FMTM DB User: " FMTM_DB_USER - - echo "Please enter the database password." - read -e -p "FMTM DB Password: " FMTM_DB_PASSWORD -fi - -echo -echo "Do you want access FMTM securely over https?" -echo "**If yes, you need to provide valid domain names with certificates later.**" -while true -do - read -e -p "Enter y for https, anything else for http: " https - - if [ "$https" = "y" ] - then - echo "Using https." - URL_SCHEME="https" - fi - break -done - -# API -echo -echo "Enter the FMTM API URL." -echo "If you have a valid domain name, enter it here." -while true -do - read -e -p "Enter d for default 127.0.0.1:8000, else your IP/domain: " api_url - - if [ "$api_url" != "d" ] - then - echo "Using $API_URL" - break - elif [ "$api_url" = "" ] - then - echo "Invalid input!" - else - echo "Using $api_url" - API_URL="api_url" - break - fi -done - -# FRONTEND -echo -echo "Enter the FMTM Frontend URL." -echo "If you have a valid domain name, enter it here." -while true -do - read -e -p "Enter d for default 127.0.0.1:8080, else your IP/domain: " frontend_url - - if [ "$frontend_url" != "d" ] - then - echo "Using $FRONTEND_MAIN_URL" - break - elif [ "$api_url" = "" ] - then - echo "Invalid input!" - else - echo "Using $FRONTEND_MAIN_URL" - FRONTEND_MAIN_URL="frontend_url" - break - fi -done - -echo -echo "Please enter your OSM authentication details" -read -e -p "Client ID: " OSM_CLIENT_ID -read -e -p "Client Secret: " OSM_CLIENT_SECRET -read -e -p "Secret Key: " OSM_SECRET_KEY -echo "Login redirect URI (default http://127.0.0.1:8080/osmauth/): " -while true -do - read -e -p "Enter a URI, or nothing for default: " auth_redirect_uri - - if [ "$auth_redirect_uri" == "" ] - then - echo "Using $OSM_LOGIN_REDIRECT_URI" - break - else - echo "Using $auth_redirect_uri" - OSM_LOGIN_REDIRECT_URI="auth_redirect_uri" - break - fi -done - -echo -echo "Generating dotenv file ${DOTENV_NAME}" - -echo "### ODK Central ###" -echo "ODK_CENTRAL_URL=${ODK_CENTRAL_URL}" >> "${DOTENV_NAME}" -echo "ODK_CENTRAL_USER=${ODK_CENTRAL_USER}" >> "${DOTENV_NAME}" -echo "ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}" >> "${DOTENV_NAME}" - -echo "### Debug ###" -echo "DEBUG=${DEBUG}" >> "${DOTENV_NAME}" -echo "LOG_LEVEL=${LOG_LEVEL}" >> "${DOTENV_NAME}" - -echo "### FMTM ###" -echo "URL_SCHEME=${URL_SCHEME}" >> "${DOTENV_NAME}" -echo "API_URL=${API_URL}" >> "${DOTENV_NAME}" -echo "FRONTEND_MAIN_URL=${FRONTEND_MAIN_URL}" >> "${DOTENV_NAME}" - -echo "### OSM ###" -echo "OSM_CLIENT_ID=${OSM_CLIENT_ID}" >> "${DOTENV_NAME}" -echo "OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET}" >> "${DOTENV_NAME}" -echo "OSM_URL=${OSM_URL}" >> "${DOTENV_NAME}" -echo "OSM_SCOPE=${OSM_SCOPE}" >> "${DOTENV_NAME}" -echo "OSM_LOGIN_REDIRECT_URI=${OSM_LOGIN_REDIRECT_URI}" >> "${DOTENV_NAME}" -echo "OSM_SECRET_KEY=${OSM_SECRET_KEY}" >> "${DOTENV_NAME}" - -echo "### Database (optional) ###" -echo "CENTRAL_DB_HOST=${CENTRAL_DB_HOST}" >> "${DOTENV_NAME}" -echo "CENTRAL_DB_USER=${CENTRAL_DB_USER}" >> "${DOTENV_NAME}" -echo "CENTRAL_DB_PASSWORD=${CENTRAL_DB_PASSWORD}" >> "${DOTENV_NAME}" -echo "CENTRAL_DB_NAME=${CENTRAL_DB_NAME}" >> "${DOTENV_NAME}" - -echo "FMTM_DB_HOST=${FMTM_DB_HOST}" >> "${DOTENV_NAME}" -echo "FMTM_DB_USER=${FMTM_DB_USER}" >> "${DOTENV_NAME}" -echo "FMTM_DB_PASSWORD=${FMTM_DB_PASSWORD}" >> "${DOTENV_NAME}" -echo "FMTM_DB_NAME=${FMTM_DB_NAME}" >> "${DOTENV_NAME}" - -echo "Completed dotenv file generation" -exit 0 diff --git a/mkdocs.yml b/mkdocs.yml index 99e156be26..1397639241 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -73,7 +73,6 @@ nav: - Frontend: dev/Frontend.md - Production Deployment: dev/Production.md - Deployment Flow: dev/Deployment-Flow.md - - Docker Tips: dev/Docker-Tips.md - Database Tips: dev/Database-Tips.md - Troubleshooting: dev/Troubleshooting.md - Version Control: dev/Version-Control.md diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 0000000000..6f8b094cb2 --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,130 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# +ARG NGINX_TAG + + +FROM docker.io/nginx:${NGINX_TAG}-bookworm as brotli-module +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ + "wget" \ + "git" \ + "build-essential" \ + "libpcre3-dev" \ + "zlib1g-dev" \ + "libssl-dev" \ + "libgd-dev" \ + "libxml2-dev" \ + "uuid-dev" \ + "libxslt-dev" \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && rm -rf /var/lib/apt/lists/* +ARG NGINX_TAG +RUN wget "http://nginx.org/download/nginx-${NGINX_TAG}.tar.gz" \ + && tar -xzvf "nginx-${NGINX_TAG}.tar.gz" +RUN git clone https://github.com/google/ngx_brotli.git --recursive +RUN CONFARGS=$(nginx -V 2>&1 | sed -n -e 's/^.*arguments: //p') \ + && cd "nginx-${NGINX_TAG}" \ + && ./configure --with-compat "${CONFARGS}" \ + --add-dynamic-module=../ngx_brotli \ + && make modules \ + && mv objs/ngx_http_brotli_filter_module.so \ + objs/ngx_http_brotli_static_module.so / + + + +FROM docker.io/nginx:${NGINX_TAG}-bookworm as base +ARG NGINX_TAG +ARG GIT_COMMIT +LABEL org.hotosm.fmtm.app-name="proxy" \ + org.hotosm.fmtm.app-version="${NGINX_TAG}" \ + org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ + org.hotosm.fmtm.maintainer="sysadmin@hotosm.org" +# Install certbot, netcat +RUN set -ex \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ + "netcat-traditional" \ + "certbot" \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && rm -rf /var/lib/apt/lists/* +# Add modules +COPY --from=brotli-module \ + /ngx_http_brotli_filter_module.so \ + /ngx_http_brotli_static_module.so \ + /usr/lib/nginx/modules/ +WORKDIR /usr/share/nginx/html +# Remove default Nginx static assets +RUN rm -rf ./* /etc/nginx/conf.d/default.conf /etc/nginx/nginx.conf +COPY nginx.conf \ + options-ssl-nginx.conf \ + options-security.conf \ + certs/ssl-dhparams.pem \ + /etc/nginx/ +# Add Healthcheck +HEALTHCHECK --start-period=5s --interval=5s --retries=8 \ + CMD nc -z localhost 80 || exit 1 + + + +FROM base as debug +COPY certs /etc/nginx/ +RUN cat /etc/nginx/central.crt /etc/nginx/ca.crt \ + >> /etc/nginx/central-fullchain.crt +COPY templates/dev/fmtm.conf.template \ + templates/dev/api.conf.template \ + templates/dev/odk.conf.template \ + templates/dev/minio.conf.template \ + /etc/nginx/templates/ + + + +FROM base as certs-init-main +# Replace existing /docker-entrypoint.sh +COPY container-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh +COPY templates/cert-init/fmtm.conf.template \ + templates/cert-init/api.conf.template \ + templates/cert-init/script.conf.template \ + /etc/nginx/templates/ + + + +FROM certs-init-main as certs-init-development +COPY templates/cert-init/odk.conf.template \ + templates/cert-init/minio.conf.template \ + /etc/nginx/templates/ + + + +FROM base as main +# API & Frontend only +COPY templates/fmtm.conf.template \ + templates/api.conf.template \ + templates/script.conf.template \ + /etc/nginx/templates/ + + + +FROM main as development +# API, Frontend, ODK, S3 +COPY templates/odk.conf.template \ + templates/minio.conf.template \ + /etc/nginx/templates/ diff --git a/nginx/build_imgs.sh b/nginx/build_imgs.sh new file mode 100644 index 0000000000..bd9a514d63 --- /dev/null +++ b/nginx/build_imgs.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Dev certs init +echo "Building proxy:certs-init-development" +docker build nginx \ + --tag "ghcr.io/hotosm/fmtm/proxy:certs-init-development" \ + --target certs-init-development \ + --build-arg NGINX_TAG="${NGINX_TAG:-1.25.2}" + +if [[ -n "$PUSH_IMGS" ]]; then + docker push "ghcr.io/hotosm/fmtm/proxy:certs-init-development" +fi + +# Staging certs init +echo "Tagging proxy:certs-init-staging" +docker tag "ghcr.io/hotosm/fmtm/proxy:certs-init-development" \ + "ghcr.io/hotosm/fmtm/proxy:certs-init-staging" + +if [[ -n "$PUSH_IMGS" ]]; then + docker push "ghcr.io/hotosm/fmtm/proxy:certs-init-staging" +fi + +# Main certs init +echo "Building proxy:certs-init-main" +docker build nginx \ + --tag "ghcr.io/hotosm/fmtm/proxy:certs-init-main" \ + --target certs-init-main \ + --build-arg NGINX_TAG="${NGINX_TAG:-1.25.2}" + +if [[ -n "$PUSH_IMGS" ]]; then + docker push "ghcr.io/hotosm/fmtm/proxy:certs-init-main" +fi + +# Dev proxy +echo "Building proxy:development" +docker build nginx \ + --tag "ghcr.io/hotosm/fmtm/proxy:development" \ + --target development \ + --build-arg NGINX_TAG="${NGINX_TAG:-1.25.2}" + +if [[ -n "$PUSH_IMGS" ]]; then + docker push "ghcr.io/hotosm/fmtm/proxy:development" +fi + +# Staging proxy +echo "Tagging proxy:staging" +docker tag "ghcr.io/hotosm/fmtm/proxy:development" \ + "ghcr.io/hotosm/fmtm/proxy:staging" + +if [[ -n "$PUSH_IMGS" ]]; then + docker push "ghcr.io/hotosm/fmtm/proxy:staging" +fi + +# Main proxy +echo "Building proxy:main" +docker build nginx \ + --tag "ghcr.io/hotosm/fmtm/proxy:main" \ + --target main \ + --build-arg NGINX_TAG="${NGINX_TAG:-1.25.2}" + +if [[ -n "$PUSH_IMGS" ]]; then + docker push "ghcr.io/hotosm/fmtm/proxy:main" +fi diff --git a/nginx/certs/ca.crt b/nginx/certs/ca.crt new file mode 100644 index 0000000000..effe9c16ca --- /dev/null +++ b/nginx/certs/ca.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDATCCAemgAwIBAgIUdmj//UNKfI9wXlMg37EJDHQ3azQwDQYJKoZIhvcNAQEL +BQAwEDEOMAwGA1UEAwwFcHJveHkwHhcNMjMxMDI0MTI1MDUxWhcNMjMxMTIzMTI1 +MDUxWjAQMQ4wDAYDVQQDDAVwcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAJpOnKnhCXOZjnYsVNlOP3dZmRsVI+I70iWqPXlZT9jDlINgP/5Vu+p7 +QPttlGl1hiJeRoAPdtvb1CM36IDg+djcZ51iUtR6BDEipYO5YOakUNNUj0tm7IVY +ipiAppHJeogZfFm9Cikk6KCByNAaOCL6hRhJG/zK2PxxUdCuAhp5h/7CU49wuWdq +1vOwiZqmoK2KSfQLH3/y62wABxDAM29Bfr/RNqTGqYZpcJlf5y8DDYDjGnmM4f6C +xuxFHRQPUj+HZGraJXEjrjMK3zuhYrL0plKHWGqs7KfCg6RjblJZrDWI1liJfX3s +spNJqdxoY/BB+raed3T0YOnYPMpM3ZsCAwEAAaNTMFEwHQYDVR0OBBYEFIeKocq4 +Z30JVNRM3MNTiq7CHAX9MB8GA1UdIwQYMBaAFIeKocq4Z30JVNRM3MNTiq7CHAX9 +MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAI7TGY6DavFvH+3k +NcG1V1SvnpZ+g84/yaz3nKxM7Uj/DsyIFhpIdrKQUeIGmC3SOSQT4D2L2cMLSU4q +W1b8jQ2Fg0Z62B+zZx5BjtQn5QtcaRD+TvPCHaixSjfwab0W7c5bKrJeeVuGVO9y +kCxoCM6Zcbd+IYnTRpqlpbhST7X+/h+AHnYgjmGM4bz+ra7h+WujTsweW/4bcuxh +3FXTGTQr5hIbjn3FC0p8OlfD6xQifXh2GSs2s9liuZMk1ncUB3Vi5bQOp6UDgAoo +hfhz5RZfr/9yFrnpS0DmSaXUSIqj9E/V3gV0tdC3la36+o19zl0gAJq5kmdv7pJ0 +EIotYLs= +-----END CERTIFICATE----- diff --git a/nginx/certs/central.crt b/nginx/certs/central.crt new file mode 100644 index 0000000000..2fda7271c8 --- /dev/null +++ b/nginx/certs/central.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgIUHthK7+y82CxbCxN5Fcu9cS1Pr1cwDQYJKoZIhvcNAQEL +BQAwEDEOMAwGA1UEAwwFcHJveHkwHhcNMjMxMDI0MTI1MTAxWhcNMjUwMzA3MTI1 +MTAxWjCBgjELMAkGA1UEBhMCVVMxFjAUBgNVBAgMDVdhc2hpbmd0b24gREMxFjAU +BgNVBAcMDVdhc2hpbmd0b24gREMxDzANBgNVBAoMBkhPVE9TTTEiMCAGCSqGSIb3 +DQEJARYTc3lzYWRtaW5AaG90b3NtLmNvbTEOMAwGA1UEAwwFcHJveHkwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZC5dgeWFZINWhNa3B1IcwDgep/sfL +XoYzLnoDOp77hNmqSiL8Mbdqn8DfOOWqvNSTo7juYgkURLco+KBIgs9rumV25/p3 +5Fw1pkB8ijswCsD51Lu0Vh2dmbz83XnaZ+Ugb/hT/XHWL0vTJV/EkO4KLlstpDFW +7FoEKQlN/wRZDL+Rr12Z0irmhaNFwnI0YyAgiyr5EPArGHEky0NjIkBROGn9C2TG +y4yfwURbflP8S8tFWxGi8mx9PTDNTByFs+ixQQjc1PBCViCaj35I4KygVulW1rfc +73g5wfh65wkJx1YKu4lgIgPs5iE5NyisA3Jc0BYgGleXSLiHnTao9QElAgMBAAGj +VDBSMBAGA1UdEQQJMAeCBXByb3h5MB0GA1UdDgQWBBTcL9cWKJgOs2P2BnOhssND +DF8dVDAfBgNVHSMEGDAWgBSHiqHKuGd9CVTUTNzDU4quwhwF/TANBgkqhkiG9w0B +AQsFAAOCAQEAFfdDY5UA9ZHyrhEc5xaMSw7gj3vAZ3OMUjMtPLewvY0FMSEjnl0a +zYJcHg69VCHy/8FFp8bSQJMIstrdWTU71rq3ybvHx3KzcNBcPwWTMH/npjiR7d/2 +lnVQcPCK2kh6lnJ1vzNRP9NcA7hxrF4KRjcSeroUq5GGgVtNEm8bZBuxHMhgr5w5 +0dPK2Cz1PxsZJD6U8PFGHhFwPBuzLHmX99aLR+8lDfGSZFBp6UP93PN5MsoAdHm2 +aZZtzH4WoMJsr0HKkJHcdx/RiZiwB3c9brpSg3hrZLjhZy0DFyU9hxub4n+WM+d1 +PGUUp3jHA3dDo8jrHndgjrAsfBsIkgYo5A== +-----END CERTIFICATE----- diff --git a/nginx/certs/central.key b/nginx/certs/central.key new file mode 100644 index 0000000000..5378e2920e --- /dev/null +++ b/nginx/certs/central.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDZC5dgeWFZINWh +Na3B1IcwDgep/sfLXoYzLnoDOp77hNmqSiL8Mbdqn8DfOOWqvNSTo7juYgkURLco ++KBIgs9rumV25/p35Fw1pkB8ijswCsD51Lu0Vh2dmbz83XnaZ+Ugb/hT/XHWL0vT +JV/EkO4KLlstpDFW7FoEKQlN/wRZDL+Rr12Z0irmhaNFwnI0YyAgiyr5EPArGHEk +y0NjIkBROGn9C2TGy4yfwURbflP8S8tFWxGi8mx9PTDNTByFs+ixQQjc1PBCViCa +j35I4KygVulW1rfc73g5wfh65wkJx1YKu4lgIgPs5iE5NyisA3Jc0BYgGleXSLiH +nTao9QElAgMBAAECggEAC1rAYhMnhRajJmpc+zFnmxt4yitif2CkC6RoTKfx7j2O +d4qmVuQw8LqHyAW9ehej1U9w2sO1LuLDZDjRhcmWGVAog7ZyE3iWXFYktH80xngh +jVBUO4RvyzFiQiZB/CuIQJCrAJXXMTnLMTIY1vswAMdZDMO0sxPtR59v9a0cKJ5Z +8AxraJtEG2w3FQgqk4B0YFSXm6aUaqo7RfmtsCuCRijhTt1H1+H5PA52IVcCY8Gi +4XTJLizA7FKVY4MoKNC4B5cyXLuHKGcv0ZMWqugDUciJYeMspM5g06dmScUNm/Xg +aIybtkxF4h55DS13T7O1tIsxa/AF6CPqz3qaWV64VwKBgQDznPZs2Y6R/5uvrIxP +O8INxgxul280iRB2k1Lk5EQt8YZ6Z0S/4mgIXbeuZ1SEERuD6vYR63tckI2fTBTj +sdkU3eFmtY+LIeKX067k+gGRxhiNVJGuIwCWY9GFzVS3jx/Au406bPPqzLQWKbaA +2FEfr19MyQCmxdrV5kTuQ40hGwKBgQDkFM2W4sj54cZovAaQm+OSpkkg8PsRhTxJ +2bcdlTdEY5D45nxVbtZY/EheJ3Mxo7qqgW7+hmArSOaLtfck0YyWM4eCf9X1/P7Z +P+vknINSH0j8vOYOxJJbuqsfcphzOmdXTYl1csoHeYv35ZzE3D6YGZrm5is2Rnwc +/cDUxj7KvwKBgQDsFflak5vvMV+XaAix4J+26W+y3V8P8qTZxPxYhj7sp1xD00B1 +FNzs/qW7hrgJJOsddN7zjtoGYrBc6EZ2Qyy1ZDHEb7+kfjoyGRstHHvY4bl5fbOQ +pqszE5lBqjOnkDSdeyTqshvQOIFt2Q7c3sX/ht08dK8E8EmbcYTrwttKcQKBgQDP +wcdX5vJs8+TYaFWOigFtBv1b/UalufilyKSNVOuH5y4aG1EsPQ8Q9ZUx9rtufxa8 +07P4Mw0MwJYIs1MRfbOS6hNIowuRy8eUuy40CwFv38GLPsJMt3AeEFZqCeemJpAV +rTJGAq7WyjfeMpBvnbOCtugr5YN9KA1MFzT8XCy8XwKBgBlTPQ/Wu5e3DI46SDFV +6hN8KPZGtY36T7+OVS8u9Fed0GxRRY6xaz5wtMcaM2REX2uuEJwdc9E4YHc215nu +iQxdDVF1Cp0BnlzqywtDulA2XgaxdJPiMH1HKSGsqbtNGfcu4BrP71CKXD1smXVD +0kesIuOuT0r8NvwvVObUVoBF +-----END PRIVATE KEY----- diff --git a/nginx/certs/ssl-dhparams.pem b/nginx/certs/ssl-dhparams.pem new file mode 100644 index 0000000000..088f9673dc --- /dev/null +++ b/nginx/certs/ssl-dhparams.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== +-----END DH PARAMETERS----- \ No newline at end of file diff --git a/nginx/container-entrypoint.sh b/nginx/container-entrypoint.sh new file mode 100644 index 0000000000..6bda9b35f4 --- /dev/null +++ b/nginx/container-entrypoint.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -eo pipefail + +echo +echo "Substituing env vars" +echo +bash /docker-entrypoint.d/20-envsubst-on-templates.sh + +echo +echo "Starting NGINX in the background" +echo +nginx -g "daemon off;" > /dev/null 2>&1 & + +# Wait for NGINX to start with a maximum timeout of 20 seconds +timeout=20 +while [ $timeout -gt 0 ]; do + if nc -z localhost 80; then + break + fi + + echo "" + echo "Waiting for NGINX to be running..." + sleep 2 + timeout=$((timeout - 2)) +done + +# Check if the timeout was reached +if [ $timeout -eq 0 ]; then + echo "NGINX did not start within the timeout." + exit 1 +fi + +# Check if FMTM_DOMAIN is set +if [ -z "${FMTM_DOMAIN}" ]; then + echo "${FMTM_DOMAIN} variable is not set. Exiting." + exit 1 +fi + +# Check if FMTM_API_DOMAIN is set +if [ -z "${FMTM_API_DOMAIN}" ]; then + echo "${FMTM_API_DOMAIN} variable is not set. Exiting." + exit 1 +fi + +# Renew certs arg (default api & frontend only) +certbot_args=( + "--webroot" "--webroot-path=/var/www/certbot" \ + "--email" "${CERT_EMAIL}" "--agree-tos" "--no-eff-email" \ + "-d" "${FMTM_DOMAIN}" "-d" "${FMTM_API_DOMAIN}" \ +) + +# Add FMTM_ODK_DOMAIN if present +if [ -n "${FMTM_ODK_DOMAIN}" ]; then + echo "Adding ${FMTM_ODK_DOMAIN} to certificate for domain ${FMTM_DOMAIN}." + certbot_args+=("-d" "${FMTM_ODK_DOMAIN}") +fi + +# Add FMTM_S3_DOMAIN if present +if [ -n "${FMTM_S3_DOMAIN}" ]; then + echo "Adding ${FMTM_S3_DOMAIN} to certificate for domain ${FMTM_DOMAIN}." + certbot_args+=("-d" "${FMTM_S3_DOMAIN}") +fi + +# Run certbot with the constructed arguments +echo "Running command: certbot --non-interactive certonly ${certbot_args[@]}" +certbot --non-interactive certonly "${certbot_args[@]}" +echo "Certificate generated under: /etc/letsencrypt/live/${FMTM_DOMAIN}/" + +# Add FMTM_SCRIPT_DOMAIN if present +if [ -n "${FMTM_SCRIPT_DOMAIN}" ] && [ "${FMTM_SCRIPT_DOMAIN}" != "_" ]; then + echo + echo "FMTM_SCRIPT_DOMAIN variable set. Generating separate certificate." + certbot --non-interactive certonly \ + --webroot --webroot-path=/var/www/certbot \ + --email "${CERT_EMAIL}" --agree-tos --no-eff-email \ + -d "${FMTM_SCRIPT_DOMAIN}" +fi + +# Successful exit (stop container) +exit 0 diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000000..a9259eabc3 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,105 @@ +# Load brotli module +load_module /usr/lib/nginx/modules/ngx_http_brotli_filter_module.so; +load_module /usr/lib/nginx/modules/ngx_http_brotli_static_module.so; + +user nginx; +worker_processes auto; +worker_rlimit_nofile 4096; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; + use epoll; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + server_tokens off; + types_hash_max_size 2048; + + keepalive_timeout 65; + + brotli on; + brotli_comp_level 6; # Adjust compression level as needed + brotli_static on; + brotli_buffers 16 8k; + brotli_min_length 20; + brotli_types + application/atom+xml + application/geo+json + application/javascript + application/x-javascript + application/json + application/ld+json + application/manifest+json + application/rdf+xml + application/rss+xml + application/vnd.ms-fontobject + application/wasm + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/eot + font/otf + font/ttf + image/bmp + image/svg+xml + text/cache-manifest + text/calendar + text/css + text/javascript + text/markdown + text/plain + text/xml + text/vcard + text/vnd.rim.location.xloc + text/vtt + text/x-component + text/x-cross-domain-policy; + + # gzip compatibility backup (older clients) + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_min_length 256; + gzip_types + application/atom+xml + application/geo+json + application/javascript + application/x-javascript + application/json + application/ld+json + application/manifest+json + application/rdf+xml + application/rss+xml + application/xhtml+xml + application/xml + font/eot + font/otf + font/ttf + image/svg+xml + text/css + text/javascript + text/plain + text/xml + application/xml+rss; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/nginx/options-security.conf b/nginx/options-security.conf new file mode 100644 index 0000000000..7b62ca3ca1 --- /dev/null +++ b/nginx/options-security.conf @@ -0,0 +1,6 @@ +add_header Referrer-Policy "no-referrer" always; +add_header X-Content-Type-Options "nosniff" always; +add_header X-Frame-Options "SAMEORIGIN" always; +add_header X-Permitted-Cross-Domain-Policies "none" always; +add_header X-Robots-Tag "noindex, nofollow" always; +add_header X-XSS-Protection "1; mode=block" always; diff --git a/nginx/options-ssl-nginx.conf b/nginx/options-ssl-nginx.conf new file mode 100644 index 0000000000..35b9895d46 --- /dev/null +++ b/nginx/options-ssl-nginx.conf @@ -0,0 +1,10 @@ +ssl_session_cache shared:le_nginx_SSL:10m; +ssl_session_timeout 1440m; +ssl_session_tickets off; + +ssl_protocols TLSv1.2 TLSv1.3; +ssl_prefer_server_ciphers off; + +ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + +ssl_dhparam /etc/nginx/ssl-dhparams.pem; diff --git a/nginx/templates/api.conf.template b/nginx/templates/api.conf.template new file mode 100644 index 0000000000..7730db4c6c --- /dev/null +++ b/nginx/templates/api.conf.template @@ -0,0 +1,75 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream backend { + server api:8000 max_fails=1 fail_timeout=2s; + keepalive 32; +} + +server { + # Default handler for port 80 + listen 80; + server_name ${FMTM_API_DOMAIN}; + return 301 https://$host$request_uri; +} + +server { + # Default handler for port 443 + listen 443 ssl reuseport; + server_name ${FMTM_API_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${FMTM_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${FMTM_DOMAIN}/privkey.pem; + include /etc/nginx/options-ssl-nginx.conf; + include /etc/nginx/options-security.conf; + + # Max upload size 1GB + client_max_body_size 1G; + + # Response headers (Access-Control-Allow-Origin set by FastAPI, not required) + add_header 'Content-Security-Policy' 'upgrade-insecure-requests'; + # For opentelemetry + add_header 'Access-Control-Allow-Headers' 'traceparent,tracestate'; + + location / { + proxy_read_timeout 40s; + proxy_connect_timeout 20s; + + # Requests headers + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $http_host; + proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + proxy_pass http://backend; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/odkcentral/proxy/Dockerfile b/nginx/templates/cert-init/api.conf.template similarity index 65% rename from odkcentral/proxy/Dockerfile rename to nginx/templates/cert-init/api.conf.template index 664784c58a..1ddea7d23e 100644 --- a/odkcentral/proxy/Dockerfile +++ b/nginx/templates/cert-init/api.conf.template @@ -16,13 +16,12 @@ # along with FMTM. If not, see . # -FROM docker.io/nginx:1.23-alpine as prod -WORKDIR /usr/share/nginx/html -# Remove default Nginx static assets -RUN rm -rf ./* /etc/nginx/conf.d/default.conf /etc/nginx/nginx.conf -COPY . /etc/nginx -RUN cat /etc/nginx/central.crt /etc/nginx/ca.crt \ - >> /etc/nginx/central-fullchain.crt -# Add Healthcheck -HEALTHCHECK --start-period=5s --interval=5s --retries=20 \ - CMD nc -z localhost 443 || exit 1 +server { + # Default handler for port 80 + listen 80; + server_name ${FMTM_API_DOMAIN}; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } +} diff --git a/nginx/templates/cert-init/fmtm.conf.template b/nginx/templates/cert-init/fmtm.conf.template new file mode 100644 index 0000000000..deff4f7df2 --- /dev/null +++ b/nginx/templates/cert-init/fmtm.conf.template @@ -0,0 +1,26 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +server { + listen 80 default_server; + server_name ${FMTM_DOMAIN}; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } +} diff --git a/nginx/templates/cert-init/minio.conf.template b/nginx/templates/cert-init/minio.conf.template new file mode 100644 index 0000000000..5d75f89c6b --- /dev/null +++ b/nginx/templates/cert-init/minio.conf.template @@ -0,0 +1,26 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +server { + listen 80; + server_name ${FMTM_S3_DOMAIN}; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } +} diff --git a/nginx/templates/cert-init/odk.conf.template b/nginx/templates/cert-init/odk.conf.template new file mode 100644 index 0000000000..c29aaec4f9 --- /dev/null +++ b/nginx/templates/cert-init/odk.conf.template @@ -0,0 +1,26 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +server { + listen 80; + server_name ${FMTM_ODK_DOMAIN}; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } +} diff --git a/nginx/templates/cert-init/script.conf.template b/nginx/templates/cert-init/script.conf.template new file mode 100644 index 0000000000..e1f4b46346 --- /dev/null +++ b/nginx/templates/cert-init/script.conf.template @@ -0,0 +1,30 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +server { + listen 80; + server_name ${FMTM_SCRIPT_DOMAIN}; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + if ($server_name = "") { + return 444; + } +} diff --git a/odkcentral/proxy/conf.d/central.conf b/nginx/templates/dev/api.conf.template similarity index 62% rename from odkcentral/proxy/conf.d/central.conf rename to nginx/templates/dev/api.conf.template index 70c309f6b8..0bfcffbd64 100644 --- a/odkcentral/proxy/conf.d/central.conf +++ b/nginx/templates/dev/api.conf.template @@ -15,27 +15,26 @@ # You should have received a copy of the GNU General Public License # along with FMTM. If not, see . # -upstream centralapi { - server central:8383; -} -server { - listen 80; - server_name localhost; - return 301 https://$host:$request_uri; +upstream backend { + server api:8000 max_fails=1 fail_timeout=2s; + keepalive 32; } server { - listen 443 ssl; - server_name localhost; + # Default handler for port 80 + listen 80 reuseport; + server_name api.fmtm.localhost; - ssl_certificate /etc/nginx/central-fullchain.crt; - ssl_certificate_key /etc/nginx/central.key; + # Max upload size 1GB + client_max_body_size 1G; - error_page 497 https://$host:$server_port$request_uri; + # Response headers (note: Access-Control-Allow-Origin already set by FastAPI, not required) location / { - proxy_pass http://centralapi; + proxy_read_timeout 40s; + proxy_connect_timeout 20s; + # Requests headers proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; @@ -44,5 +43,18 @@ server { proxy_set_header X-Forwarded-Host $http_host; proxy_set_header X-Forwarded-Server $http_host; proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + proxy_pass http://backend; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; } } diff --git a/nginx/templates/dev/fmtm.conf.template b/nginx/templates/dev/fmtm.conf.template new file mode 100644 index 0000000000..fe5d41e43c --- /dev/null +++ b/nginx/templates/dev/fmtm.conf.template @@ -0,0 +1,64 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream frontend { + # Enable sticky sessions based on an incoming client IP address + ip_hash; + + server ui:7051; +} + +server { + listen 80 default_server; + server_name fmtm.localhost; + + # Max upload size 1GB + client_max_body_size 1G; + + location / { + proxy_read_timeout 40s; + proxy_connect_timeout 20s; + + # Requests headers + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $http_host; + proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + # Config to enable websockets + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_pass http://frontend; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/nginx/templates/dev/minio.conf.template b/nginx/templates/dev/minio.conf.template new file mode 100644 index 0000000000..d60d6d1d49 --- /dev/null +++ b/nginx/templates/dev/minio.conf.template @@ -0,0 +1,57 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream minio { + server s3:9000; + # No keepalive +} + +server { + listen 80; + server_name s3.fmtm.localhost; + + # Allow special characters in headers (Minio) + ignore_invalid_headers off; + + # Max upload size 10GB + client_max_body_size 10G; + + location / { + # Disable buffering + proxy_buffering off; + proxy_request_buffering off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/nginx/templates/dev/odk.conf.template b/nginx/templates/dev/odk.conf.template new file mode 100644 index 0000000000..67fc94dc21 --- /dev/null +++ b/nginx/templates/dev/odk.conf.template @@ -0,0 +1,118 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream centralapi { + server central:8383 max_fails=1 fail_timeout=2s; + keepalive 32; +} + +server { + listen 80; + server_name odk.fmtm.localhost; + + # Max upload size 100MB + client_max_body_size 100m; + + # The frontend + location / { + root /usr/share/nginx/html/central; + + #location /index.html { + # include /usr/share/odk/nginx/common-headers.conf; + # add_header Cache-Control no-cache; + #} + } + + # The API + location ~ ^/v\d { + proxy_redirect off; + # buffer requests, but not responses, so streaming out works. + proxy_request_buffering on; + proxy_buffering off; + proxy_read_timeout 2m; + proxy_connect_timeout 1m; + + # Requests headers + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $http_host; + proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + proxy_pass http://centralapi; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} + +server { + listen 443 ssl; + server_name odk.fmtm.localhost; + + ssl_certificate /etc/nginx/central-fullchain.crt; + ssl_certificate_key /etc/nginx/central.key; + + # Max upload size 100MB + client_max_body_size 100m; + + # The API + location ~ ^/v\d { + proxy_redirect off; + # buffer requests, but not responses, so streaming out works. + proxy_request_buffering on; + proxy_buffering off; + proxy_read_timeout 2m; + proxy_connect_timeout 1m; + + # Requests headers + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $http_host; + proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + proxy_pass http://centralapi; + } + + # Required redirect + error_page 497 https://$host:$server_port$request_uri; + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/nginx/templates/fmtm.conf.template b/nginx/templates/fmtm.conf.template new file mode 100644 index 0000000000..f1a35f35a5 --- /dev/null +++ b/nginx/templates/fmtm.conf.template @@ -0,0 +1,52 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +server { + listen 80 default_server; + server_name ${FMTM_DOMAIN}; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl default_server; + server_name ${FMTM_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${FMTM_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${FMTM_DOMAIN}/privkey.pem; + include /etc/nginx/options-ssl-nginx.conf; + include /etc/nginx/options-security.conf; + + # Max upload size 1GB + client_max_body_size 1G; + + # Response headers + add_header 'Content-Security-Policy' 'upgrade-insecure-requests'; + # For opentelemetry + add_header 'Access-Control-Allow-Headers' 'traceparent,tracestate'; + + location / { + # Serve FMTM frontend under /usr/share/nginx/html + root /usr/share/nginx/html/fmtm; + try_files $uri $uri/ /index.html; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/nginx/templates/minio.conf.template b/nginx/templates/minio.conf.template new file mode 100644 index 0000000000..c6e6493ae5 --- /dev/null +++ b/nginx/templates/minio.conf.template @@ -0,0 +1,73 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream minio { + server s3:9000; + # No keepalive +} + +server { + listen 80; + server_name ${FMTM_S3_DOMAIN}; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name ${FMTM_S3_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${FMTM_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${FMTM_DOMAIN}/privkey.pem; + include /etc/nginx/options-ssl-nginx.conf; + include /etc/nginx/options-security.conf; + + # Allow special characters in headers (Minio) + ignore_invalid_headers off; + + # Max upload size 10GB + client_max_body_size 10G; + + # Response headers + add_header 'Content-Security-Policy' 'upgrade-insecure-requests'; + # For opentelemetry + add_header 'Access-Control-Allow-Headers' 'traceparent,tracestate'; + + location / { + # Disable buffering + proxy_buffering off; + proxy_request_buffering off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/nginx/templates/odk.conf.template b/nginx/templates/odk.conf.template new file mode 100644 index 0000000000..a0a49eded3 --- /dev/null +++ b/nginx/templates/odk.conf.template @@ -0,0 +1,91 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream centralapi { + server central:8383 max_fails=1 fail_timeout=2s; + keepalive 32; +} + +server { + listen 80; + server_name ${FMTM_ODK_DOMAIN}; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name ${FMTM_ODK_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${FMTM_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${FMTM_DOMAIN}/privkey.pem; + include /etc/nginx/options-ssl-nginx.conf; + include /etc/nginx/options-security.conf; + + # Max upload size 1GB + client_max_body_size 100m; + + # Response headers (Access-Control-Allow-Origin set by FastAPI, not required) + add_header 'Content-Security-Policy' 'upgrade-insecure-requests'; + # For opentelemetry + add_header 'Access-Control-Allow-Headers' 'traceparent,tracestate'; + + # The frontend + location / { + root /usr/share/nginx/html/central; + + location /index.html { + include /etc/nginx/options-security.conf; + add_header Cache-Control no-cache; + } + } + + # The API + location ~ ^/v\d { + proxy_redirect off; + # buffer requests, but not responses, so streaming out works. + proxy_request_buffering on; + proxy_buffering off; + proxy_read_timeout 2m; + proxy_connect_timeout 1m; + + # Requests headers + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $http_host; + proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + proxy_pass http://centralapi; + } + + # Required redirect + error_page 497 https://$host:$server_port$request_uri; + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/nginx/templates/script.conf.template b/nginx/templates/script.conf.template new file mode 100644 index 0000000000..41a53744b0 --- /dev/null +++ b/nginx/templates/script.conf.template @@ -0,0 +1,61 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +server { + listen 80; + server_name ${FMTM_SCRIPT_DOMAIN}; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name ${FMTM_SCRIPT_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${FMTM_SCRIPT_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${FMTM_SCRIPT_DOMAIN}/privkey.pem; + include /etc/nginx/options-ssl-nginx.conf; + include /etc/nginx/options-security.conf; + + # Disable file uploads + client_max_body_size 0; + + # Response headers + add_header 'Content-Security-Policy' 'upgrade-insecure-requests'; + # For opentelemetry + add_header 'Access-Control-Allow-Headers' 'traceparent,tracestate'; + + location / { + # Serve FMTM install script /usr/share/nginx/html/fmtm/install.sh + root /usr/share/nginx/html/fmtm; + try_files /install.sh /install.sh; + + # Allow executing shell scripts directly in the browser + default_type text/plain; + + # Prevent caching + add_header Cache-Control "no-cache, no-store, must-revalidate"; + + # Prevent directory listing + autoindex off; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/odkcentral/api/.dockerignore b/odkcentral/api/.dockerignore deleted file mode 100644 index c478ee1372..0000000000 --- a/odkcentral/api/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -**/* -!init-user-and-start.sh diff --git a/odkcentral/api/Dockerfile b/odkcentral/api/Dockerfile index c8898da9de..c98355bc93 100644 --- a/odkcentral/api/Dockerfile +++ b/odkcentral/api/Dockerfile @@ -20,10 +20,6 @@ ARG node_version=18 FROM docker.io/bitnami/git:2 as repo -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - "jq" \ - && rm -rf /var/lib/apt/lists/* ARG ODK_CENTRAL_TAG RUN git clone --depth 1 --branch ${ODK_CENTRAL_TAG} \ "https://github.com/getodk/central.git" \ @@ -31,7 +27,7 @@ RUN git clone --depth 1 --branch ${ODK_CENTRAL_TAG} \ -FROM docker.io/node:${node_version} +FROM docker.io/node:${node_version}-slim WORKDIR /usr/odk @@ -45,10 +41,16 @@ COPY init-user-and-start.sh / COPY --from=repo central/server/package*.json ./ # Install system deps -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODENAME=\K\w+' /etc/os-release)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg && \ - apt-get update && \ - apt-get install -y cron wait-for-it gettext postgresql-client-14 netcat-traditional \ +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + gpg \ + cron \ + wait-for-it \ + gettext \ + postgresql-client \ + netcat-traditional \ + && rm -rf /var/lib/apt/lists/* \ # Install node_modules && npm clean-install --omit=dev --legacy-peer-deps --no-audit \ --fund=false --update-notifier=false \ diff --git a/odkcentral/enketo/Dockerfile b/odkcentral/enketo/Dockerfile new file mode 100644 index 0000000000..dfd359077f --- /dev/null +++ b/odkcentral/enketo/Dockerfile @@ -0,0 +1,86 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +# TODO this dockerfile is not complete, but is largely optimise +# over the official image (>800MB reduction) +ARG node_version=16 + + +FROM docker.io/bitnami/git:2 as enketo +ARG ENKETO_TAG=6.2.2 +RUN git clone --depth 1 --branch ${ENKETO_TAG} \ + "https://github.com/enketo/enketo-express.git" + + + +FROM docker.io/bitnami/git:2 as central +ARG ODK_CENTRAL_TAG +RUN git clone --depth 1 --branch ${ODK_CENTRAL_TAG} \ + "https://github.com/getodk/central.git" + + + +FROM docker.io/node:${node_version}-slim as build +WORKDIR /repo +COPY --from=enketo enketo-express/ . +RUN npm install -g npm@^6 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + build-essential \ + python3 \ + openssh-client \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && update-ca-certificates +# Skip chrome install for puppeteer +ENV PUPPETEER_SKIP_DOWNLOAD='true' +RUN npm ci +RUN npx grunt +RUN npm prune --production + + + +FROM docker.io/node:${node_version}-slim as runtime +ENV ENKETO_SRC_DIR=/srv/src/enketo_express +WORKDIR ${ENKETO_SRC_DIR} +RUN npm install -g pm2@$(npm info pm2 version) +# Persist the `secrets` directory so the encryption key remains consistent. +RUN mkdir -p ${ENKETO_SRC_DIR}/setup/docker/secrets +VOLUME ${ENKETO_SRC_DIR}/setup/docker/secrets + +# From ODK Central Config +COPY --from=central central/files/enketo/config.json.template \ + ${ENKETO_SRC_DIR}/config/config.json.template +COPY --from=central central/files/enketo/config.json.template \ + ${ENKETO_SRC_DIR}/config/config.json +COPY --from=central central/files/enketo/start-enketo.sh \ + ${ENKETO_SRC_DIR}/start-enketo.sh + +RUN apt-get update && \ + apt-get install gettext-base \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir /etc/secrets \ + && echo 'jhs9udhy987gyds98gfyds98f' > /etc/secrets/enketo-api-key \ + && echo 'jhs9udhy987gyds98gfyds98f' > /etc/secrets/enketo-secret \ + && echo 'jhs9udhy987gyds98gfyds98f' > /etc/secrets/enketo-less-secret + +COPY --from=build /repo/ ./ + +EXPOSE 8005 + +CMD ["./start-enketo.sh"] diff --git a/odkcentral/proxy/ca.crt b/odkcentral/proxy/ca.crt deleted file mode 100644 index 2678ff8a06..0000000000 --- a/odkcentral/proxy/ca.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDbTCCAlWgAwIBAgIUCBSpLXlqds6qzp34urVda3761ecwDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMzA1MTgyMDI1MDFaGA8yNTIz -MDExNzIwMjUwMVowRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx -ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN -AQEBBQADggEPADCCAQoCggEBAPFuKskPINSCjUEUf+3uVmVESuUo9aNPFJyTvPzB -uGvH+R1/a5PchIz15oyotVWN75//WAWsukz0vfHvEpyYqdz69LxBH7UPZYJA3EHW -32oZXfnpkC0pPFyyBPkVhITFjtXB96zCFVg+VImRsa/BdEJ4/DssHXHRedB/oM95 -7+wc8X2sYB+4xtnApBjqFj3Fca23n8FXko6WP7N1W4GZWVcJBwfwHJX/CTD0mYj5 -tlxPTWQbo20YtD+tqq7aGLo/OT4UgGgwA63HHsY0HfPvhuvUa8ZCgVU2eKIwrtBR -B5Nd3DIAK/GKLKcAqqyihqu3Wz4HlNj9J5kNgDjSRyyutfcCAwEAAaNTMFEwHQYD -VR0OBBYEFJ4TKxxddkR9AhDfMXceE1XbUtfOMB8GA1UdIwQYMBaAFJ4TKxxddkR9 -AhDfMXceE1XbUtfOMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB -AGcHNSpDFpL+zOyIV0aeSjkroA8avHnoEkcRbpdkRptMOwdVQcVD0DRRSzAImBBs -E8E1Oa2OuSpeUk7AkQVDbQHK1wXmJGnQBsRZVlz6/pU9Tp3Pysqw75GZWzbg9mFt -oTT/+0Ks64wxFByA7EyXHiegEwTCmO9aqGL3w+EKI/Xg+yaYq86Qv5QF3AAhZU0i -jaC0vk+WnaHQpEGBMaO77lMMeHLf+R5POOq8OqVQ8P09hWztxjwOcCzEZbnD13e+ -2CBEUn/ouuyB2KmXj1JyFKJ2xlJ8fFUEx5DHeUJK0RwZUlpJUcws6Ym753Dz5lKj -VawBxgYMF4GIAcG42to3uPw= ------END CERTIFICATE----- diff --git a/odkcentral/proxy/central.crt b/odkcentral/proxy/central.crt deleted file mode 100644 index 54db5ca85a..0000000000 --- a/odkcentral/proxy/central.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDYDCCAkigAwIBAgIUeyhILi+kuTlNYcgCmwLBEesmBfQwDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMzA1MTgyMDI1MjlaFw0yNDA1 -MTcyMDI1MjlaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQC903qRRkKuiS+RoL6y/zOwSlgiGGJlyX1IhojJaYgR -hlx3yAI56RHc2g0GrSyn7Pt3wHZx1kgWMgxUl4im9fqC4X2I/D6C8XHUoiMvRyB0 -nMvwo6V3sNPAQRALjIcA+a7rjTLCjkcayl96KbfzmshwfCYAhXrUjwE+l2BoxzMm -NnkfqpIcaW6qBKTXqb7ZJW+pW5a+7/JP/0G8YAYyURu32ksMe2O9JND9vM7TMXgp -GvFtEI9Y/GC7Hp72hOdAgKIz/0MYFOgScMgO3CBmeAETqNR3zXG1XDO73pj+XBbu -qZqAwePK/YQyBcTm4H1ViyLlZJq/FQiFbfZgDtnvWhi3AgMBAAGjSDBGMB8GA1Ud -IwQYMBaAFJ4TKxxddkR9AhDfMXceE1XbUtfOMAkGA1UdEwQCMAAwGAYDVR0RBBEw -D4INY2VudHJhbC1wcm94eTANBgkqhkiG9w0BAQsFAAOCAQEA7wIfcgu2czFMYSXW -pO6hZhRAdbXmIeX0UkH5C+K9LWSHDgnACUqpnjTXsqgg2wbE/Lad6p3mZZ1ytMgL -1W8RDcW6e2HsVvEnjWRwFDuaCIz94f+QNBbtWFP/9o6tIX39pO6ZLz1DLJ/6igf3 -UrG1DtnIzcauQyrdJBlgzg2Zmc/jcl+vb9bz3Hy+H7liKVV2GFnvzv9NUyk4B+Q1 -oJY2p24+wMB1meKfoDj5m0ADeJacY95KLa/4qWYAVhLfWdYBLk8/BszXsYx2xwJR -tKwpMITqXGIIl9RaaaHasTzaDp8a+VqBVYf/gDA5ij/XgYEBz38YsEl1WQWLVA3Q -QVDwig== ------END CERTIFICATE----- diff --git a/odkcentral/proxy/central.key b/odkcentral/proxy/central.key deleted file mode 100644 index 3a93a5cf2b..0000000000 --- a/odkcentral/proxy/central.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAvdN6kUZCrokvkaC+sv8zsEpYIhhiZcl9SIaIyWmIEYZcd8gC -OekR3NoNBq0sp+z7d8B2cdZIFjIMVJeIpvX6guF9iPw+gvFx1KIjL0cgdJzL8KOl -d7DTwEEQC4yHAPmu640ywo5HGspfeim385rIcHwmAIV61I8BPpdgaMczJjZ5H6qS -HGluqgSk16m+2SVvqVuWvu/yT/9BvGAGMlEbt9pLDHtjvSTQ/bzO0zF4KRrxbRCP -WPxgux6e9oTnQICiM/9DGBToEnDIDtwgZngBE6jUd81xtVwzu96Y/lwW7qmagMHj -yv2EMgXE5uB9VYsi5WSavxUIhW32YA7Z71oYtwIDAQABAoIBAAcmwXZZpEd5CZ69 -5JFPxf29P0bBjmtxRNQHpX2ZCTRH0JH+qf7OhgrA+D6pJYaKCgs/5/Zv8TVM22QQ -dWh82dw2bIlNn3lFaVluZk7wqgtaoOBG+gPhWIjkqpsUxz1FKs7a+e+Udp9OwnwS -uEhoK2b5dmEMpkz9ujlGcK5h2s50rCTuk5VtEN0dJUgOicJkJ6lSA7LrUazOqsnC -UppwQ8mdO2V58eMdv+KU0ISMxYDN9CCrpalKg+W9YOZlpI6kQCuX4gz6YMnVYaQo -ginIUYm9VsSRc1YtEW+vzgOD51GMGf2V7cCHyoYkyAOCL8wl7UDh9gUrTGZzQwPB -gp2utPECgYEA/QH6ZT0MscF2vTUxdF69g9bEEU4T7vxeezgkpVFgSlDhFKzxuMAF -IXposkQ6mcyyNqljsxDxOC8wOjOxfgcNvmMAGLGuVwxtlW8SFVcYHVOCgX8846Qk -v7l2KmanNsomRzYMhfHceeAk9eLElJ5ei3s5kPxURdQtC/TlJAPZy2kCgYEAwBI1 -QRdJAfHICBE5lIReYGt1TY+m983xEfSdoolXB1TTksmSpqCsfZsajkOOCW87h5Uk -ylmm3/CvY9nkNn4b0Dxo27eKxusJXqJ4M6y4MlacRAzCv6rwx5sCIBHA0Q+7d51A -Kq758fXglcBhqGVKg9Z/WDzCDKV37LKm0ho63x8CgYAFkR+TTq6S0rDsJNl7uAHv -frA9uTwTDBaAG5Ii2RGHURRCwjNnX0I9pFXdZkbrz2c0cLGy5PKJy/ABt9V18m3I -/KCp6m7tl2Zp3VHGim4LZyT9+HC8iYYNeeUxtaSEPhptgrDv2YyCWf42MtEke/+U -jheMKyEnkC2Vj8Pi8Pq/0QKBgQClj0xeNh8VUP8WjRpv/YWIYTg0yg2nbtpMmulC -EFZ3GFRjI+OIQ9lLT+YdGPgXA81xekrDLXIdeE8Gp+wCWMPKxDeypMw32KMz8qHg -ERKZOXTrZ4B7THLN67xyPjtROdljKqYD5N3IBiQ3kSPczqrjZ4JPYdf2pK7x1bDc -TAXRCwKBgQCY113RKVxVa7RhzXF4TpX3Sd60xuflD/5hG4koMdVo0PwbdlQYIVOh -blNUOYMs1Y5Zc/+acxf8ZxtbdU/916Uku138yCVA2yAdVNmttAxnZk7D5W4l34Ei -B9PmdCQgorrJNzOAfy4t68kE/rMMFNUzu/dFTt2r3B3wz5fd96NqZA== ------END RSA PRIVATE KEY----- diff --git a/odkcentral/proxy/nginx.conf b/odkcentral/ui/Dockerfile similarity index 51% rename from odkcentral/proxy/nginx.conf rename to odkcentral/ui/Dockerfile index bb975cc803..aaa3d791fc 100644 --- a/odkcentral/proxy/nginx.conf +++ b/odkcentral/ui/Dockerfile @@ -1,5 +1,4 @@ # Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team -# # This file is part of FMTM. # # FMTM is free software: you can redistribute it and/or modify @@ -16,35 +15,30 @@ # along with FMTM. If not, see . # -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; +ARG node_version=18 -events { - worker_connections 1024; -} +FROM docker.io/bitnami/git:2 as repo +ARG ODK_CENTRAL_TAG +RUN git clone --depth 1 --branch ${ODK_CENTRAL_TAG} \ + "https://github.com/getodk/central.git" \ + && cd central && git submodule update --init -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; +FROM docker.io/node:${node_version}-slim as build +WORKDIR /frontend +COPY --from=repo central/client/ /frontend/ +RUN npm ci --no-audit --fund=false --update-notifier=false +RUN VUE_APP_OIDC_ENABLED="false" npm run build - sendfile on; - tcp_nopush on; - tcp_nodelay on; - server_tokens off; - types_hash_max_size 2048; - keepalive_timeout 65; - include /etc/nginx/conf.d/*.conf; -} +FROM docker.io/rclone/rclone:1.64 as prod +VOLUME /frontend +COPY container-entrypoint.sh / +RUN chmod +x /container-entrypoint.sh +ENTRYPOINT ["/container-entrypoint.sh"] +WORKDIR /app +COPY --from=build /frontend/dist . diff --git a/odkcentral/ui/container-entrypoint.sh b/odkcentral/ui/container-entrypoint.sh new file mode 100644 index 0000000000..cca44b6b70 --- /dev/null +++ b/odkcentral/ui/container-entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +# Copy frontend to attached volume +echo "Syncing files from /app --> /frontend." +rclone sync /app /frontend +echo "Sync done." + +# Successful exit (stop container) +exit 0 diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..d3bb2ccbfa --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,8 @@ +# Scripts + +- Directory of helper scripts related to FMTM. +- The install-fmtm.sh script is located under: + +`src/frontend/public/install-fmtm.sh` + +So it is available to install via URL. diff --git a/scripts/gen-env.sh b/scripts/gen-env.sh new file mode 100644 index 0000000000..e16c59c871 --- /dev/null +++ b/scripts/gen-env.sh @@ -0,0 +1,441 @@ +#!/bin/bash + +DOTENV_PATH=.env +IS_TEST=false +BRANCH_NAME= + +pretty_echo() { + local message="$1" + local length=${#message} + local separator="" + + for ((i=0; i /dev/null; then + sudo apt-get update + sudo apt-get install -y curl + fi + + echo + # Get a8m/envsubst (required for default vals syntax ${VAR:-default}) + # Use local version, as envsubst may be installed on system already + if [ -f ./envsubst ]; then + echo "envsubst already exists. Continuing." + else + echo "Downloading a8m/envsubst" + echo + curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst + chmod +x envsubst + fi +} + +check_if_test() { + pretty_echo "Test Deployment?" + + echo "Is this a test deployment?" + echo + while true + do + read -e -p "Enter 'y' if yes, anything else to continue: " test + + if [[ "$test" = "y" || "$test" = "yes" ]] + then + IS_TEST=true + export DEBUG=True + export LOG_LEVEL="DEBUG" + echo "Using debug configuration." + else + IS_TEST=false + export DEBUG=False + export LOG_LEVEL="INFO" + break + fi + break + done +} + +check_existing_dotenv() { + if [ -f "${DOTENV_PATH}" ] + then + echo "WARNING: ${DOTENV_PATH} file already exists." + echo "This script will overwrite the content of this file." + echo + echo "Do you want to overwrite "${DOTENV_PATH}"? y/n" + until [ "$overwrite" = "y" -o "$overwrite" = "n" ] + do + read -e -p "Enter 'y' to overwrite, anything else to continue: " overwrite + + if [ "$overwrite" = "y" ] + then + return 1 + elif [ "$overwrite" = "n" ] + then + echo "Continuing with existing .env file." + return 0 + else + echo "Invalid input!" + fi + done + fi + + return 1 +} + +check_existing_dotenv() { + if [ -f "${DOTENV_PATH}" ] + then + echo "WARNING: ${DOTENV_PATH} file already exists." + echo "This script will overwrite the content of this file." + echo + echo "Do you want to overwrite file '"${DOTENV_PATH}"'? y/n" + echo + while true + do + read -e -p "Enter 'y' to overwrite, anything else to continue: " overwrite + + if [[ "$overwrite" = "y" || "$overwrite" = "yes" ]] + then + return 1 + else + echo "Continuing with existing .env file." + return 0 + fi + done + fi + + return 1 +} + +set_deploy_env() { + pretty_echo "Deployment Environment" + + while true + do + echo "Which environment do you wish to run? (dev/staging/prod)" + echo + echo "Both dev & staging include ODK Central and S3 buckets." + echo "For prod, it is expected you provide and external instances of:" + echo + echo "- ODK Central" + echo "- S3 Buckets" + echo + read -e -p "Enter the environment (dev/staging/prod): " environment + + case "$environment" in + dev) + export BRANCH_NAME="development" + break + ;; + staging) + export BRANCH_NAME="staging" + break + ;; + prod) + export BRANCH_NAME="main" + break + ;; + *) + echo "Invalid environment name. Please enter dev, staging, or prod." + ;; + esac + done +} + +set_external_odk() { + pretty_echo "External ODK Central Host" + + echo "Please enter the ODKCentral URL." + read -e -p "ODKCentral URL: " ODK_CENTRAL_URL + echo + export ODK_CENTRAL_URL=${ODK_CENTRAL_URL} + + set_odk_user_creds +} + +set_fmtm_db_pass() { + db_pass=$(tr -dc 'a-zA-Z0-9' 10 characters long." + echo + read -e -p "ODKCentral Password: " ODK_CENTRAL_PASSWD + echo + + # Check the length of the entered password + if [ ${#ODK_CENTRAL_PASSWD} -ge 10 ]; then + echo "Password is at least 10 characters long. Proceeding..." + export ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD} + break # Exit the loop if a valid password is entered + else + echo "Password is too short. It must be at least 10 characters long." + echo + fi + done +} + +check_external_database() { + pretty_echo "External Database" + + echo "Do you want to use an external database instead of local?" + while true + do + read -e -p "Enter y for external, anything else to continue: " externaldb + + if [ "$externaldb" = "y" ] + then + EXTERNAL_DB="True" + echo "Using external database." + fi + break + done + + if [ "$EXTERNAL_DB" = "True" ] + then + echo + echo "Please enter the database host." + read -e -p "FMTM DB Host: " FMTM_DB_HOST + echo + export FMTM_DB_HOST=${FMTM_DB_HOST} + + echo "Please enter the database name." + read -e -p "FMTM DB Name: " FMTM_DB_NAME + echo + export FMTM_DB_NAME=${FMTM_DB_NAME} + + echo "Please enter the database user." + read -e -p "FMTM DB User: " FMTM_DB_USER + echo + export FMTM_DB_USER=${FMTM_DB_USER} + + echo "Please enter the database password." + read -e -p "FMTM DB Password: " FMTM_DB_PASSWORD + echo + export FMTM_DB_PASSWORD=${FMTM_DB_PASSWORD} + + else + set_fmtm_db_pass + fi +} + +set_external_s3() { + pretty_echo "S3 Credentials" + + echo "Please enter the S3 host endpoint." + read -e -p "S3 Endpoint: " S3_ENDPOINT + echo + export S3_ENDPOINT=${S3_ENDPOINT} + + echo "Please enter the access key." + read -e -p "S3 Access Key: " S3_ACCESS_KEY + echo + export S3_ACCESS_KEY=${S3_ACCESS_KEY} + + echo "Please enter the secret key." + read -e -p "S3 Secret Key: " S3_SECRET_KEY + echo + export S3_SECRET_KEY=${S3_SECRET_KEY} + + if [ "$BRANCH_NAME" == "main" ]; then + echo "Production deployments require a preconfigured S3 bucket." + echo + echo "The bucket should be public." + echo + echo "Please enter the bucket name." + read -e -p "S3 Bucket Name: " S3_BUCKET_NAME_BASEMAPS + echo + export S3_BUCKET_NAME_BASEMAPS=${S3_BUCKET_NAME_BASEMAPS} + fi +} + +set_minio_s3_creds() { + access_key=$(tr -dc 'a-zA-Z0-9' $current_ip" + echo "api.$fmtm_domain --> $current_ip" + + if [ "$BRANCH_NAME" != "main" ] + then + echo "s3.$fmtm_domain --> $current_ip" + echo "odk.$fmtm_domain --> $current_ip" + fi + + echo + read -e -p "Once these DNS entries are set and valid, press ENTER to continue." valid + + pretty_echo "Certificates" + echo "FMTM will automatically generate SSL (HTTPS) certificates for your domain name." + while true + do + echo "Enter an email address you wish to use for certificate generation." + read -e -p "This will be used by LetsEncrypt, but for no other purpose: " cert_email + + if [ "$cert_email" = "" ] + then + echo "Invalid input!" + else + export CERT_EMAIL="${cert_email}" + break + fi + done +} + +set_osm_credentials() { + pretty_echo "OSM OAuth2 Credentials" + + redirect_uri="http${FMTM_DOMAIN:+s}://${FMTM_DOMAIN:-127.0.0.1:7051}/osmauth/" + + echo "App credentials are generated from your OSM user profile." + echo + echo "If you need to generate new OAuth2 App credentials, visit:" + echo + echo "> https://www.openstreetmap.org/oauth2/applications" + echo + echo "Set the redirect URI to: ${redirect_uri}" + echo + + echo "Please enter your OSM authentication details" + echo + read -e -p "Client ID: " OSM_CLIENT_ID + echo + read -e -p "Client Secret: " OSM_CLIENT_SECRET + + export OSM_CLIENT_ID=${OSM_CLIENT_ID} + export OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET} + secret_key=$(tr -dc 'a-zA-Z0-9' ${DOTENV_PATH}" + ./envsubst < .env.example > ${DOTENV_PATH} + else + echo "Downloading .env.example from repo." + echo + curl -LO "https://raw.githubusercontent.com/hotosm/fmtm/${BRANCH_NAME:-development}/.env.example" + + echo "substituting variables from .env.example --> ${DOTENV_PATH}" + ./envsubst < .env.example > ${DOTENV_PATH} + + echo "Deleting .env.example" + rm .env.example + fi +} + +prompt_user_gen_dotenv() { + pretty_echo "Generate dotenv config for FMTM" + check_existing_dotenv + install_envsubst_if_missing + check_if_test + + if [ $IS_TEST != true ]; then + set_deploy_env + + if [ "$BRANCH_NAME" == "main" ] + then + set_external_odk + check_external_database + set_external_s3 + else + set_fmtm_db_pass + set_odk_db_pass + set_odk_user_creds + set_minio_s3_creds + fi + + set_domains + + else + check_change_port + fi + + set_osm_credentials + generate_dotenv + + pretty_echo "Completed dotenv file generation" +} + +trap cleanup_and_exit INT +prompt_user_gen_dotenv diff --git a/scripts/install-fmtm.sh b/scripts/install-fmtm.sh deleted file mode 100644 index 8ecda1c3f4..0000000000 --- a/scripts/install-fmtm.sh +++ /dev/null @@ -1,9 +0,0 @@ - - -# 1. Need to configure two DNS entries to point to FMTM server -# Prompt user to go and do this first, then come back and insert domain name -# 2. -# 3. Use script to gen_env.sh -# 4. Pull backend and db containers only -# 5. Need to rebuild frontend using .env vars -# 6. docker compose deploy everything \ No newline at end of file diff --git a/scripts/renew-certs-manual.sh b/scripts/renew-certs-manual.sh new file mode 100644 index 0000000000..0b380c670c --- /dev/null +++ b/scripts/renew-certs-manual.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -euo pipefail + +cleanup_and_exit() { + echo + echo "CTRL+C received, exiting..." + exit 1 +} + +# Capture CTRL+C +trap cleanup_and_exit INT + +# Prompt the user for input and set the BRANCH_NAME variable +read -p "Enter the environment (dev/staging/prod): " ENVIRONMENT + +case "$ENVIRONMENT" in + dev) + BRANCH_NAME="development" + ;; + staging) + BRANCH_NAME="staging" + ;; + prod) + BRANCH_NAME="main" + ;; + *) + echo "Invalid environment. Please enter dev, staging, or prod." + exit 1 + ;; +esac + +# Check if any containers using the 'ghcr.io/hotosm/fmtm/proxy:${BRANCH_NAME}' image are running +if [[ -z $(docker ps -q -f "ancestor=ghcr.io/hotosm/fmtm/proxy:${BRANCH_NAME}") ]]; then + echo "No containers using the 'ghcr.io/hotosm/fmtm/proxy:${BRANCH_NAME}' image are running." + echo "You must first start the containers using:" + echo + echo "docker-compose -f docker-compose.${BRANCH_NAME}.yml up -d" + echo + exit 1 +fi + +# Execute the Docker Compose command with the determined BRANCH_NAME +docker compose exec "fmtm-$BRANCH_NAME" certbot --non-interactive renew diff --git a/scripts/setup/README.md b/scripts/setup/README.md new file mode 100644 index 0000000000..ebdb92fb23 --- /dev/null +++ b/scripts/setup/README.md @@ -0,0 +1,5 @@ +# Setup Scripts + +- Scripts used to setup a machine to use Docker & Podman. +- Docker is installed in rootless mode. +- Podman is an alternative to Docker, and by default rootless. diff --git a/scripts/setup/docker.sh b/scripts/setup/docker.sh new file mode 100644 index 0000000000..c4da462f07 --- /dev/null +++ b/scripts/setup/docker.sh @@ -0,0 +1,212 @@ +#!/bin/bash + +# Tested for Debian 11 Bookworm & Ubuntu 22.04 LTS +# Note: this script must be run as a non-root user +# Note: The user must be logged in directly (not via su) + +OS_NAME="debian" + +pretty_echo() { + local message="$1" + local length=${#message} + local separator="" + + for ((i=0; i /dev/null + echo "Done" +} + +apt_install_docker() { + pretty_echo "Installing Docker" + sudo apt-get update + sudo apt-get install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin \ + docker-ce-rootless-extras +} + +check_user_not_root() { + pretty_echo "Use non-root user" + + if [ "$(id -u)" -eq 0 ]; then + if id "fmtm" &>/dev/null; then + echo "Current user is root. Switching to existing non-privileged user 'fmtm'." + else + echo "Current user is root. Creating a non-privileged user 'fmtm'." + useradd -m -s /bin/bash fmtm + fi + + echo "Rerunning this script as user 'fmtm'." + sudo -u fmtm bash -c "$0 $*" + exit 0 + fi +} + +update_to_rootless() { + pretty_echo "Disabling docker service if running" + sudo systemctl disable --now docker.service docker.socket + + pretty_echo "Install rootless docker" + dockerd-rootless-setuptool.sh install +} + +restart_docker_rootless() { + heading_echo "Restarting Docker Service" + echo "This is required as sometimes docker doesn't init correctly." + systemctl --user daemon-reload + systemctl --user restart docker + echo + echo "Done." +} + +allow_priv_port_access() { + pretty_echo "Allowing Privileged Port Usage" + sudo tee -a /etc/sysctl.conf <> ~/.bashrc + fi + + echo "Done" + echo + + heading_echo "Adding dc='docker compose' alias" + + # Check if the alias already exists + if ! grep -q "$dc_alias_cmd" ~/.bashrc; then + echo "Adding 'dc' alias to ~/.bashrc." + echo "$dc_alias_cmd" >> ~/.bashrc + fi + + echo "Done" +} + +install_docker() { + check_os + remove_old_docker_installs + install_dependencies + add_gpg_key + add_to_apt + apt_install_docker + update_to_rootless + allow_priv_port_access + restart_docker_rootless + update_docker_ps_format + add_vars_to_bashrc +} + +check_user_not_root +trap cleanup_and_exit INT +install_docker diff --git a/scripts/setup/podman.sh b/scripts/setup/podman.sh new file mode 100644 index 0000000000..3bf6b79d44 --- /dev/null +++ b/scripts/setup/podman.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Tested for Ubuntu 22.04 LTS & Debian 11 Bookworm + +# TODO docker-compose is required, install equivalent for podman and alias + +pretty_echo() { + local message="$1" + local length=${#message} + local separator="" + + for ((i=0; i> ~/.bashrc + echo "Done" +} + + +# Podman install process +check_os +install_podman +allow_priv_port_usage +create_docker_alias diff --git a/src/backend/.dockerignore b/src/backend/.dockerignore index 2a9161252e..07cc234666 100644 --- a/src/backend/.dockerignore +++ b/src/backend/.dockerignore @@ -7,6 +7,7 @@ !tests !app-entrypoint.sh !migrate-entrypoint.sh +!backup-entrypoint.sh !pyproject.toml !pdm.lock !migrations diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 4a18a6e3d3..67e759c150 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -15,16 +15,20 @@ # along with FMTM. If not, see . # ARG PYTHON_IMG_TAG=3.10 +ARG MINIO_TAG=${MINIO_TAG:-RELEASE.2023-10-25T06-33-25Z} +FROM docker.io/minio/minio:${MINIO_TAG} as minio + + FROM docker.io/python:${PYTHON_IMG_TAG}-slim-bookworm as base ARG APP_VERSION ARG COMMIT_REF ARG PYTHON_IMG_TAG -ARG MAINTAINER=admin@hotosm.org -LABEL org.hotosm.fmtm.app-version="${APP_VERSION}" \ +LABEL org.hotosm.fmtm.app-name="backend" \ + org.hotosm.fmtm.app-version="${APP_VERSION}" \ org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ org.hotosm.fmtm.python-img-tag="${PYTHON_IMG_TAG}" \ - org.hotosm.fmtm.maintainer="${MAINTAINER}" \ + org.hotosm.fmtm.maintainer="sysadmin@hotosm.org" \ org.hotosm.fmtm.api-port="8000" RUN set -ex \ && apt-get update \ @@ -92,7 +96,6 @@ RUN set -ex \ -y --no-install-recommends \ "nano" \ "curl" \ - "wait-for-it" \ "libpcre3" \ "mime-support" \ "postgresql-client" \ @@ -101,6 +104,8 @@ RUN set -ex \ "libproj25" \ "libgeos-c1v5" \ && rm -rf /var/lib/apt/lists/* +# Copy minio mc client +COPY --from=minio /opt/bin/mc /usr/local/bin/ COPY *-entrypoint.sh / ENTRYPOINT ["/app-entrypoint.sh"] # Copy Python deps from build to runtime @@ -112,10 +117,10 @@ WORKDIR /opt COPY app/ /opt/app/ COPY migrations/ /opt/migrations/ # Add non-root user, permissions -RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \ +RUN useradd -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \ && mkdir -p /opt/logs /opt/tiles \ && chown -R appuser:appuser /opt /home/appuser \ - && chmod +x /app-entrypoint.sh /migrate-entrypoint.sh + && chmod +x /app-entrypoint.sh /migrate-entrypoint.sh /backup-entrypoint.sh # Add volumes for persistence VOLUME /opt/logs VOLUME /opt/tiles @@ -145,7 +150,7 @@ CMD ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", \ FROM debug-no-odk as debug-with-odk USER root # Add the SSL cert for debug odkcentral -COPY --from=ghcr.io/hotosm/fmtm/odkcentral-proxy:latest \ +COPY --from=ghcr.io/hotosm/fmtm/proxy:debug \ /etc/nginx/central-fullchain.crt /usr/local/share/ca-certificates/ RUN update-ca-certificates USER appuser @@ -175,8 +180,8 @@ RUN mv /home/appuser/.local/bin/* /usr/local/bin/ \ # Pre-compile packages to .pyc (init speed gains) && python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)" # Override entrypoint, as not possible in Github action -ENTRYPOINT [""] -CMD [""] +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/src/backend/app/auth/auth_routes.py b/src/backend/app/auth/auth_routes.py index 1c6c60cbea..fa61715276 100644 --- a/src/backend/app/auth/auth_routes.py +++ b/src/backend/app/auth/auth_routes.py @@ -69,7 +69,7 @@ def callback(request: Request, osm_auth=Depends(init_osm_auth)): Returns: access_token(string): The access token provided by the login URL request. """ - print("Call back api requested", request.url) + log.debug(f"Callback url requested: {request.url}") # Enforce https callback url callback_url = str(request.url).replace("http://", "https://") diff --git a/src/backend/app/config.py b/src/backend/app/config.py index edf0a9571f..1c780c3a21 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -31,8 +31,8 @@ class Settings(BaseSettings): DEBUG: bool = False LOG_LEVEL: str = "INFO" - URL_SCHEME: Optional[str] = "http" - FRONTEND_MAIN_URL: Optional[str] + FMTM_DOMAIN: Optional[str] + FMTM_DEV_PORT: Optional[str] = "7050" EXTRA_CORS_ORIGINS: Optional[Union[str, list[str]]] = [] @@ -51,11 +51,13 @@ def assemble_cors_origins( default_origins = [] # Build default origins from env vars - url_scheme = info.data.get("URL_SCHEME") - main_url = info.data.get("FRONTEND_MAIN_URL") - if url_scheme and main_url: + url_scheme = "http" if info.data.get("DEBUG") else "https" + local_server_port = ( + f":{info.data.get('FMTM_DEV_PORT')}" if info.data.get("DEBUG") else "" + ) + if frontend_domain := info.data.get("FMTM_DOMAIN"): default_origins = [ - f"{url_scheme}://{main_url}", + f"{url_scheme}://{frontend_domain}{local_server_port}", ] if val is None: @@ -105,7 +107,7 @@ def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any: OSM_SECRET_KEY: str OSM_URL: str = "https://www.openstreetmap.org" OSM_SCOPE: str = "read_prefs" - OSM_LOGIN_REDIRECT_URI: str = "http://127.0.0.1:8080/osmauth/" + OSM_LOGIN_REDIRECT_URI: str = "http://127.0.0.1:7051/osmauth/" S3_ENDPOINT: str = "http://s3:9000" S3_ACCESS_KEY: str diff --git a/src/backend/backup-entrypoint.sh b/src/backend/backup-entrypoint.sh new file mode 100644 index 0000000000..6275b2bdda --- /dev/null +++ b/src/backend/backup-entrypoint.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +set -eo pipefail + +### Functions START ### + +pretty_echo() { + local message="$1" + local length=${#message} + local separator="" + + for ((i=0; i> /dev/null 2>&1 + + echo "gzipping file --> ${db_backup_file}.gz" + gzip "$db_backup_file" + db_backup_file="${db_backup_file}.gz" + + BUCKET_NAME="fmtm-db-backups" + echo "Uploading to S3 bucket ${BUCKET_NAME}" + mc alias set s3 $S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY + mc mb "s3/${BUCKET_NAME}" --ignore-existing + mc anonymous set download "s3/${BUCKET_NAME}" + mc cp "${db_backup_file}" "s3/${BUCKET_NAME}/${db_name}/" +} +### Functions END ### + + + +#################### +### Script START ### +#################### + +echo "Waiting 5 minutes (for migrations) before first backup." +sleep 600 + +while true; do + pretty_echo "### Backup FMTM $(date +%Y-%m-%d_%H:%M:%S) ###" + check_fmtm_db_vars_present + wait_for_db ${FMTM_DB_HOST:-"fmtm-db"} + backup_db ${FMTM_DB_HOST:-"fmtm-db"} ${FMTM_DB_USER:-"fmtm"} \ + ${FMTM_DB_NAME:-"fmtm"} ${FMTM_DB_PASSWORD} + pretty_echo "### Backup FMTM Complete ###" + + # Only run ODK Central DB Backups if variables set + if [ -n "${CENTRAL_DB_HOST}" ]; then + pretty_echo "### Backup ODK Central $(date +%Y-%m-%d_%H:%M:%S) ###" + check_central_db_vars_present + wait_for_db ${CENTRAL_DB_HOST:-"central-db"} + backup_db ${CENTRAL_DB_HOST:-"central-db"} ${CENTRAL_DB_USER:-"odk"} \ + ${CENTRAL_DB_NAME:-"odk"} ${CENTRAL_DB_PASSWORD} + pretty_echo "### Backup ODK Central Complete ###" + fi + + echo "Waiting 24hrs until next backup." + sleep 86400 +done + +#################### +### Script END ### +#################### diff --git a/src/backend/migrate-entrypoint.sh b/src/backend/migrate-entrypoint.sh index 19e978f7dc..366f8c703d 100644 --- a/src/backend/migrate-entrypoint.sh +++ b/src/backend/migrate-entrypoint.sh @@ -122,13 +122,23 @@ backup_db() { PGPASSWORD="$FMTM_DB_PASSWORD" psql --host "$FMTM_DB_HOST" \ --username "$FMTM_DB_USER" "$FMTM_DB_NAME" -c "VACUUM ANALYZE;" - pretty_echo "Dumping current database to backup file." + pretty_echo "Dumping current database to backup file: $db_backup_file" PGPASSWORD="$FMTM_DB_PASSWORD" pg_dump --verbose --format c \ --file "${db_backup_file}" \ --host "$FMTM_DB_HOST" --username "$FMTM_DB_USER" "$FMTM_DB_NAME" + echo "gzipping file --> ${db_backup_file}.gz" gzip "$db_backup_file" - pretty_echo "Backup complete: $db_backup_file.gz" + db_backup_file="${db_backup_file}.gz" + + BUCKET_NAME="fmtm-db-backups" + echo "Uploading to S3 bucket ${BUCKET_NAME}" + mc alias set s3 $S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY + mc mb "s3/${BUCKET_NAME}" --ignore-existing + mc anonymous set download "s3/${BUCKET_NAME}" + mc cp "${db_backup_file}" s3/${BUCKET_NAME}/pre-migrate/ + + pretty_echo "Backup complete: $db_backup_file to bucket ${BUCKET_NAME}/pre-migrate/" } execute_migrations() { diff --git a/src/frontend/container-entrypoint.sh b/src/frontend/container-entrypoint.sh new file mode 100644 index 0000000000..cca44b6b70 --- /dev/null +++ b/src/frontend/container-entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +# Copy frontend to attached volume +echo "Syncing files from /app --> /frontend." +rclone sync /app /frontend +echo "Sync done." + +# Successful exit (stop container) +exit 0 diff --git a/src/frontend/debug.dockerfile b/src/frontend/debug.dockerfile index b6c669fb50..d196516f39 100755 --- a/src/frontend/debug.dockerfile +++ b/src/frontend/debug.dockerfile @@ -1,6 +1,4 @@ -FROM docker.io/node:18 -ARG VITE_API_URL -ENV VITE_API_URL="${VITE_API_URL}" +FROM docker.io/node:18-slim WORKDIR /app COPY ./package.json ./pnpm-lock.yaml ./ ENV PNPM_HOME="/pnpm" diff --git a/src/frontend/package.json b/src/frontend/package.json index 737dd5d3b4..cc25dde4e7 100755 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -9,10 +9,10 @@ "start:live": "vite dev", "test": "vitest tests/" }, - "license": "MIT", + "license": "GPL-3.0-only", "author": { - "name": "Jack Herrington", - "email": "jherr@pobox.com" + "name": "HOTOSM", + "email": "sysadmin@hotosm.org" }, "devDependencies": { "@testing-library/jest-dom": "^6.1.3", diff --git a/src/frontend/prod.dockerfile b/src/frontend/prod.dockerfile index a53ddcdd1e..9514c36d54 100644 --- a/src/frontend/prod.dockerfile +++ b/src/frontend/prod.dockerfile @@ -1,17 +1,8 @@ FROM docker.io/node:18 as builder -ARG MAINTAINER=admin@hotosm.org -ARG APP_VERSION -ARG COMMIT_REF ARG VITE_API_URL ENV VITE_API_URL="${VITE_API_URL}" -LABEL org.hotosm.fmtm.app-name="fmtm-frontend" \ - org.hotosm.fmtm.app-version="${APP_VERSION}" \ - org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ - org.hotosm.fmtm.maintainer="${MAINTAINER}" \ - org.hotosm.fmtm.api-url="${VITE_API_URL}" - WORKDIR /app COPY ./package.json ./pnpm-lock.yaml ./ ENV PNPM_HOME="/pnpm" @@ -24,9 +15,19 @@ COPY . . RUN pnpm run build -FROM docker.io/devforth/spa-to-http:1.0.3 as prod + +FROM docker.io/rclone/rclone:1.64 as prod +ARG APP_VERSION +ARG COMMIT_REF +ARG VITE_API_URL +LABEL org.hotosm.fmtm.app-name="frontend" \ + org.hotosm.fmtm.app-version="${APP_VERSION}" \ + org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ + org.hotosm.fmtm.maintainer="sysadmin@hotosm.org" \ + org.hotosm.fmtm.api-url="${VITE_API_URL}" +VOLUME /frontend +COPY container-entrypoint.sh / +RUN chmod +x /container-entrypoint.sh +ENTRYPOINT ["/container-entrypoint.sh"] WORKDIR /app -# Add non-root user, permissions -RUN adduser -D -u 900 -h /home/appuser appuser -USER appuser -COPY --from=builder --chown=appuser:appuser /app/dist . +COPY --from=builder /app/dist . diff --git a/src/frontend/public/install.sh b/src/frontend/public/install.sh new file mode 100644 index 0000000000..b049d7ba70 --- /dev/null +++ b/src/frontend/public/install.sh @@ -0,0 +1,916 @@ +#!/bin/bash + +set -o pipefail + +# Tested for Debian 11 Bookworm & Ubuntu 22.04 LTS + +# Auto accept all apt prompts +export DEBIAN_FRONTEND=noninteractive + +# Global Vars +RANDOM_DIR="${RANDOM}${RANDOM}" +DOTENV_NAME=.env +OS_NAME="debian" +IS_TEST=false +BRANCH_NAME=development +COMPOSE_FILE=docker-compose.yml + +heading_echo() { + local message="$1" + local color="${2:-blue}" + local separator="--------------------------------------------------------" + local sep_length=${#separator} + local pad_length=$(( (sep_length - ${#message}) / 2 )) + local pad="" + + case "$color" in + "black") color_code="\e[0;30m" ;; + "red") color_code="\e[0;31m" ;; + "green") color_code="\e[0;32m" ;; + "yellow") color_code="\e[0;33m" ;; + "blue") color_code="\e[0;34m" ;; + "purple") color_code="\e[0;35m" ;; + "cyan") color_code="\e[0;36m" ;; + "white") color_code="\e[0;37m" ;; + *) color_code="\e[0m" ;; # Default: reset color + esac + + for ((i=0; i/dev/null; then + yellow_echo "User 'svcfmtm' found." + else + yellow_echo "Creating user 'svcfmtm'." + useradd -m -d /home/svcfmtm -s /bin/bash svcfmtm 2>/dev/null + fi + + echo + yellow_echo "Enable login linger for user svcfmtm (docker if logged out)." + loginctl enable-linger svcfmtm + + echo + yellow_echo "Temporarily adding to sudoers list." + echo "svcfmtm ALL=(ALL) NOPASSWD:ALL" | tee /etc/sudoers.d/fmtm-sudoers >/dev/null + + echo + yellow_echo "Rerunning this script as user 'svcfmtm'." + echo + + if ! command -v machinectl &>/dev/null; then + # Start the installation process in the background with spinner + ( apt-get update > /dev/null + wait # Wait for 'apt-get update' to complete + apt-get install -y systemd-container --no-install-recommends > /dev/null ) & + install_progress $! + echo + fi + + # Check if input is direct bash script call (i.e. ends in .sh) + ext="$(basename "$0")" + if [ "${ext: -3}" = ".sh" ]; then + # User called script directly, copy to /home/svcfmtm/install.sh + root_script_path="$(readlink -f "$0")" + user_script_path="/home/svcfmtm/$(basename "$0")" + cp "$root_script_path" "$user_script_path" + chmod +x "$user_script_path" + + machinectl --quiet shell \ + --setenv=RUN_AS_ROOT=true \ + --setenv=DOCKER_HOST=${DOCKER_HOST} \ + svcfmtm@ /bin/bash -c "$user_script_path" + else + # User called script remotely, so do the same + machinectl --quiet shell \ + --setenv=RUN_AS_ROOT=true \ + --setenv=DOCKER_HOST=${DOCKER_HOST} \ + svcfmtm@ /bin/bash -c "curl -fsSL https://get.fmtm.dev | bash" + fi + + exit 0 + fi +} + +check_os() { + heading_echo "Checking Current OS" + + if [ -e /etc/os-release ]; then + source /etc/os-release + case "$ID" in + debian) + export OS_NAME=${ID} + echo "Current OS is ${PRETTY_NAME}." + ;; + ubuntu) + export OS_NAME=${ID} + echo "Current OS is ${PRETTY_NAME}." + ;; + *) + echo "Current OS is not Debian or Ubuntu. Exiting." + exit 1 + ;; + esac + else + echo "Could not determine the operating system. Exiting." + exit 1 + fi +} + +remove_old_docker_installs() { + heading_echo "Removing Old Versions of Docker" + packages=( + docker.io + docker-doc + docker-compose + podman-docker + containerd + runc + ) + for pkg in "${packages[@]}"; do + sudo apt-get remove "$pkg" + done +} + +install_dependencies() { + heading_echo "Installing Dependencies" + sudo apt-get update + sudo apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + uidmap \ + dbus-user-session \ + slirp4netns + + if [ "$OS_NAME" = "debian" ]; then + sudo apt-get install -y fuse-overlayfs + fi +} + +add_gpg_key() { + heading_echo "Adding Docker GPG Key" + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/${ID}/gpg | sudo gpg --yes --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + echo "Done" +} + +add_to_apt() { + heading_echo "Adding Docker to Apt Source" + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/${ID} \ + $(. /etc/os-release && echo $VERSION_CODENAME) stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + echo "Done" +} + +apt_install_docker() { + heading_echo "Installing Docker" + sudo apt-get update + sudo apt-get install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin \ + docker-ce-rootless-extras +} + +update_to_rootless() { + heading_echo "Disabling Docker Service (If Running)" + sudo systemctl disable --now docker.service docker.socket + + heading_echo "Install Rootless Docker" + dockerd-rootless-setuptool.sh install +} + +restart_docker_rootless() { + heading_echo "Restarting Docker Service" + echo "This is required as sometimes docker doesn't init correctly." + systemctl --user daemon-reload + systemctl --user restart docker + echo + echo "Done." +} + +allow_priv_port_access() { + heading_echo "Allowing Privileged Port Usage" + sudo tee -a /etc/sysctl.conf < /dev/null 2>&1 +net.ipv4.ip_unprivileged_port_start=0 +EOF + sudo sysctl -p + echo "Done" +} + +update_docker_ps_format() { + heading_echo "Updating docker ps Formatting" + + # Root user + if [ "$RUN_AS_ROOT" = true ]; then + sudo mkdir -p /root/.docker + sudo touch /root/.docker/config.json + sudo tee /root/.docker/config.json < /dev/null 2>&1 +{ + "psFormat": "table {{.ID}}\\t{{.Image}}\\t{{.Status}}\\t{{.Names}}" +} +EOF + fi + + # svcfmtm user + mkdir -p ~/.docker + touch ~/.docker/config.json + tee ~/.docker/config.json < /dev/null 2>&1 +{ + "psFormat": "table {{.ID}}\\t{{.Image}}\\t{{.Status}}\\t{{.Names}}" +} +EOF + +echo "Done" +} + + +add_vars_to_bashrc() { + heading_echo "Adding DOCKER_HOST and 'dc' alias to bashrc" + + user_id=$(id -u) + docker_host_var="export DOCKER_HOST=unix:///run/user/$user_id/docker.sock" + dc_alias_cmd="alias dc='docker compose'" + + if [ "$RUN_AS_ROOT" = true ]; then + # Check if DOCKER_HOST is already defined in /root/.bashrc + if ! sudo grep -q "$docker_host_var" /root/.bashrc; then + echo "Adding DOCKER_HOST var to /root/.bashrc." + echo "$docker_host_var" | sudo tee -a /root/.bashrc > /dev/null + echo + fi + + # Check if the 'dc' alias already exists in /root/.bashrc + if ! sudo grep -q "$dc_alias_cmd" /root/.bashrc; then + echo "Adding 'dc' alias to /root/.bashrc." + echo "$dc_alias_cmd" | sudo tee -a /root/.bashrc > /dev/null + echo + fi + fi + + # Check if DOCKER_HOST is already defined in ~/.bashrc + if ! grep -q "$docker_host_var" ~/.bashrc; then + echo "Adding DOCKER_HOST var to ~/.bashrc." + echo "$docker_host_var" >> ~/.bashrc + echo + fi + + # Check if the 'dc' alias already exists in ~/.bashrc + if ! grep -q "$dc_alias_cmd" ~/.bashrc; then + echo "Adding 'dc' alias to ~/.bashrc." + echo "$dc_alias_cmd" >> ~/.bashrc + echo + fi + + echo "Setting DOCKER_HOST for the current session." + export DOCKER_HOST=unix:///run/user/$(id -u)/docker.sock + + echo + echo "Done" +} + +install_docker() { + heading_echo "Docker Install" + + if command -v docker &> /dev/null; then + echo "Docker already installed: $(which docker)" + echo "Skipping." + return 0 + fi + + echo "Docker is required for FMTM to run." + echo + echo "Do you want to install Docker? (y/n)" + echo + read -rp "Enter 'y' to install, anything else to continue: " install_docker + + if [[ "$install_docker" = "y" || "$install_docker" = "yes" ]]; then + check_os + remove_old_docker_installs + install_dependencies + add_gpg_key + add_to_apt + apt_install_docker + update_to_rootless + allow_priv_port_access + restart_docker_rootless + update_docker_ps_format + add_vars_to_bashrc + else + heading_echo "Docker is Required. Aborting." "red" + exit 1 + fi +} + +install_envsubst_if_missing() { + if ! command -v curl &> /dev/null; then + sudo apt-get update + sudo apt-get install -y curl --no-install-recommends + fi + + echo + # Get a8m/envsubst (required for default vals syntax ${VAR:-default}) + # Use local version, as envsubst may be installed on system already + if [ -f ./envsubst ]; then + echo "envsubst already exists. Continuing." + else + echo "Downloading a8m/envsubst" + echo + curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst + chmod +x envsubst + fi +} + +check_existing_dotenv() { + if [ -f "${DOTENV_NAME}" ] + then + echo "WARNING: ${DOTENV_NAME} file already exists." + echo "This script will overwrite the content of this file." + echo + echo "Do you want to overwrite file '"${DOTENV_NAME}"'? y/n" + echo + while true + do + read -e -p "Enter 'y' to overwrite, anything else to continue: " overwrite + + if [[ "$overwrite" = "y" || "$overwrite" = "yes" ]] + then + return 1 + else + echo "Continuing with existing .env file." + return 0 + fi + done + fi + + return 1 +} + +check_if_test() { + heading_echo "Test Deployment?" + + echo "Is this a test deployment?" + echo + while true + do + read -e -p "Enter 'y' if yes, anything else to continue: " test + + if [[ "$test" = "y" || "$test" = "yes" ]] + then + IS_TEST=true + export DEBUG="True" + export LOG_LEVEL="DEBUG" + echo "Using debug configuration." + else + IS_TEST=false + export DEBUG="False" + export LOG_LEVEL="INFO" + break + fi + break + done +} + +get_repo() { + heading_echo "Getting Necessary Files" + + current_dir="${PWD}" + + if ! command -v git &>/dev/null; then + yellow_echo "Downloading GIT." + echo + sudo apt-get update + sudo apt-get install -y git --no-install-recommends + echo + fi + + # Files in a random temp dir + mkdir -p "/tmp/${RANDOM_DIR}" + cd "/tmp/${RANDOM_DIR}" + + repo_url="https://github.com/hotosm/fmtm.git" + + echo "Cloning repo $repo_url to dir: /tmp/${RANDOM_DIR}" + echo + git clone --branch "${BRANCH_NAME}" --depth 1 "$repo_url" + + # Check for existing .env files + existing_dotenv="" + if [ "${RUN_AS_ROOT}" = true ] && sudo test -f "/root/fmtm/${DOTENV_NAME}"; then + existing_dotenv="/root/fmtm/${DOTENV_NAME}" + elif [ -f "${current_dir}/${DOTENV_NAME}" ]; then + existing_dotenv="${current_dir}/${DOTENV_NAME}" + fi + + if [ -n "$existing_dotenv" ]; then + echo + echo "Found existing dotenv file." + echo + echo "Copying $existing_dotenv --> /tmp/${RANDOM_DIR}/fmtm/${DOTENV_NAME}" + if [ "${RUN_AS_ROOT}" = true ]; then + sudo cp "$existing_dotenv" "/tmp/${RANDOM_DIR}/fmtm/" + else + cp "$existing_dotenv" "/tmp/${RANDOM_DIR}/fmtm/" + fi + fi +} + +set_deploy_env() { + heading_echo "Deployment Environment" + + while true + do + echo "Which environment do you wish to run? (dev/staging/prod)" + echo + echo "Both dev & staging include ODK Central and S3 buckets." + echo "For prod, it is expected you provide and external instances of:" + echo + echo "- ODK Central" + echo "- S3 Buckets" + echo + read -e -p "Enter the environment (dev/staging/prod): " environment + + case "$environment" in + dev) + BRANCH_NAME="development" + ;; + staging) + BRANCH_NAME="staging" + ;; + prod) + BRANCH_NAME="main" + ;; + *) + echo "Invalid environment name. Please enter dev, staging, or prod." + ;; + + esac + + export GIT_BRANCH="${BRANCH_NAME}" + COMPOSE_FILE="docker-compose.${BRANCH_NAME}.yml" + break + done +} + +set_external_odk() { + heading_echo "External ODK Central Host" + + echo "Please enter the ODKCentral URL." + read -e -p "ODKCentral URL: " ODK_CENTRAL_URL + echo + export ODK_CENTRAL_URL=${ODK_CENTRAL_URL} + + set_odk_user_creds +} + +set_fmtm_db_pass() { + db_pass=$(tr -dc 'a-zA-Z0-9' 10 characters long." + while true; do + echo + read -e -p "ODKCentral Password: " ODK_CENTRAL_PASSWD + echo + + # Check the length of the entered password + if [ ${#ODK_CENTRAL_PASSWD} -ge 10 ]; then + export ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD} + break + else + yellow_echo "Password is too short. It must be at least 10 characters long." + fi + done +} + +check_external_database() { + heading_echo "External Database" + + echo "Do you want to use an external database instead of local?" + echo + while true + do + read -e -p "Enter y for external, anything else to continue: " externaldb + + if [ "$externaldb" = "y" ] + then + EXTERNAL_DB="True" + echo "Using external database." + fi + break + done + + if [ "$EXTERNAL_DB" = "True" ] + then + echo + echo "Please enter the database host." + read -e -p "FMTM DB Host: " FMTM_DB_HOST + echo + export FMTM_DB_HOST=${FMTM_DB_HOST} + + echo "Please enter the database name." + read -e -p "FMTM DB Name: " FMTM_DB_NAME + echo + export FMTM_DB_NAME=${FMTM_DB_NAME} + + echo "Please enter the database user." + read -e -p "FMTM DB User: " FMTM_DB_USER + echo + export FMTM_DB_USER=${FMTM_DB_USER} + + echo "Please enter the database password." + read -e -p "FMTM DB Password: " FMTM_DB_PASSWORD + echo + export FMTM_DB_PASSWORD=${FMTM_DB_PASSWORD} + + else + set_fmtm_db_pass + fi +} + +set_external_s3() { + heading_echo "S3 Credentials" + + echo "Please enter the S3 host endpoint." + read -e -p "S3 Endpoint: " S3_ENDPOINT + echo + export S3_ENDPOINT=${S3_ENDPOINT} + + echo "Please enter the access key." + read -e -p "S3 Access Key: " S3_ACCESS_KEY + echo + export S3_ACCESS_KEY=${S3_ACCESS_KEY} + + echo "Please enter the secret key." + read -e -p "S3 Secret Key: " S3_SECRET_KEY + echo + export S3_SECRET_KEY=${S3_SECRET_KEY} + + if [ "$BRANCH_NAME" = "main" ]; then + yellow_echo "Production deployments require a preconfigured S3 bucket." + echo + yellow_echo "The bucket should be public." + echo + echo "Please enter the bucket name." + read -e -p "S3 Bucket Name: " S3_BUCKET_NAME_BASEMAPS + echo + export S3_BUCKET_NAME_BASEMAPS=${S3_BUCKET_NAME_BASEMAPS} + fi +} + +set_minio_s3_creds() { + access_key=$(tr -dc 'a-zA-Z0-9' $current_ip" + yellow_echo "api.$fmtm_domain --> $current_ip" + + if [ "$BRANCH_NAME" != "main" ] + then + yellow_echo "s3.$fmtm_domain --> $current_ip" + yellow_echo "odk.$fmtm_domain --> $current_ip" + fi + + echo + read -e -p "Once these DNS entries are set and valid, press ENTER to continue." valid + + heading_echo "Certificates" + echo "FMTM will automatically generate SSL (HTTPS) certificates for your domain name." + echo + while true + do + echo "Enter an email address you wish to use for certificate generation." + echo "This will be used by LetsEncrypt, but for no other purpose." + echo + read -e -p "Email: " cert_email + + if [ "$cert_email" = "" ] + then + echo "Invalid input!" + else + export CERT_EMAIL="${cert_email}" + break + fi + done +} + +set_osm_credentials() { + heading_echo "OSM OAuth2 Credentials" + + redirect_uri="http${FMTM_DOMAIN:+s}://${FMTM_DOMAIN:-127.0.0.1:7051}/osmauth/" + + yellow_echo "App credentials are generated from your OSM user profile." + echo + yellow_echo "If you need to generate new OAuth2 App credentials, visit:" + echo + yellow_echo "> https://www.openstreetmap.org/oauth2/applications" + echo + yellow_echo "Set the redirect URI to: ${redirect_uri}" + echo + + echo "Please enter your OSM authentication details" + echo + read -e -p "Client ID: " OSM_CLIENT_ID + echo + read -e -p "Client Secret: " OSM_CLIENT_SECRET + + export OSM_CLIENT_ID=${OSM_CLIENT_ID} + export OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET} + secret_key=$(tr -dc 'a-zA-Z0-9' ${DOTENV_NAME}" + ./envsubst < .env.example > ${DOTENV_NAME} + else + echo "Downloading .env.example from repo." + echo + curl -LO "https://raw.githubusercontent.com/hotosm/fmtm/${BRANCH_NAME:-development}/.env.example" + + echo + echo "substituting variables from .env.example --> ${DOTENV_NAME}" + ./envsubst < .env.example > ${DOTENV_NAME} + + echo + echo "Deleting .env.example" + rm .env.example + fi + + heading_echo "Completed Dotenv File Generation." "green" + echo "File ${DOTENV_NAME} content:" + echo + cat ${DOTENV_NAME} + echo + if [ "${RUN_AS_ROOT}" = true ] && sudo test ! -f "/root/fmtm/${DOTENV_NAME}"; then + echo "Copying generated dotenv to /root/fmtm/${DOTENV_NAME}" + cp "${DOTENV_NAME}" "/root/fmtm/${DOTENV_NAME}" || true + elif [ ! -f "/home/svcfmtm/${DOTENV_NAME}" ]; then + echo "Copying generated dotenv to /home/svcfmtm/fmtm/${DOTENV_NAME}" + cp "${DOTENV_NAME}" "/home/svcfmtm/fmtm/${DOTENV_NAME}" || true + fi +} + +prompt_user_gen_dotenv() { + heading_echo "Generate dotenv config for FMTM" + + # Exit if user does not overwrite existing dotenv + if check_existing_dotenv; then + return + fi + + install_envsubst_if_missing + + if [ $IS_TEST != true ]; then + if [ "$BRANCH_NAME" = "main" ]; then + set_external_odk + check_external_database + set_external_s3 + else + set_fmtm_db_pass + set_odk_db_pass + set_odk_user_creds + set_minio_s3_creds + fi + + set_domains + + else + check_change_port + fi + + set_osm_credentials + generate_dotenv +} + +run_compose_stack() { + # Workaround if DOCKER_HOST is missed (i.e. docker just installed) + if [ -z "$DOCKER_HOST" ]; then + export DOCKER_HOST=unix:///run/user/$(id -u)/docker.sock + fi + + heading_echo "Pulling Required Images" + docker compose -f ${COMPOSE_FILE} pull + heading_echo "Building Frontend Image" + docker compose -f ${COMPOSE_FILE} build ui + + heading_echo "Starting FMTM" + docker compose -f ${COMPOSE_FILE} up \ + --detach --remove-orphans --force-recreate +} + +final_output() { + # Source env vars + . .env + + proto="http" + suffix="" + + if [ "$IS_TEST" != true ]; then + proto="https" + else + suffix=":${FMTM_DEV_PORT:-7050}" + fi + + heading_echo "FMTM Setup Complete" + heading_echo "Services" "green" + echo "Frontend: ${proto}://${FMTM_DOMAIN}${suffix}" + echo "API: ${proto}://api.${FMTM_DOMAIN}${suffix}" + echo "S3 Buckets: ${proto}://s3.${FMTM_DOMAIN}${suffix}" + echo "ODK Central: ${proto}://odk.${FMTM_DOMAIN}${suffix}" + heading_echo "Inspect Containers" "green" + echo "To login as svcfmtm and inspect the containers, run:" + echo + echo "$ machinectl shell svcfmtm@" + echo "$ docker ps" + echo + echo "Alternatively, to run as the current user:" + echo + echo "$ export DOCKER_HOST=unix:///run/user/$(id -u svcfmtm)/docker.sock" + echo "$ docker ps" + echo + heading_echo "ODK Central Credentials" "green" + echo "URL: ${ODK_CENTRAL_URL}" + echo "Email: ${ODK_CENTRAL_USER}" + echo "Password: ${ODK_CENTRAL_PASSWD}" + echo +} + +install_fmtm() { + check_user_not_root + display_logo + + trap cleanup_and_exit INT + install_docker + + check_if_test + if [ $IS_TEST != true ]; then + set_deploy_env + fi + + get_repo + # Work in generated temp dir + local repo_dir="/tmp/${RANDOM_DIR}/fmtm" + cd "${repo_dir}" + + if [ -f "${repo_dir}/${DOTENV_NAME}" ]; then + heading_echo "Skip Dotenv Generation" + echo "Using existing dotenv file." + else + prompt_user_gen_dotenv + fi + + run_compose_stack + final_output + + if [[ "$RUN_AS_ROOT" = true ]]; then + # Remove from sudoers + sudo rm /etc/sudoers.d/fmtm-sudoers + fi + + # Cleanup files + if [[ "$IS_TEST" != true ]]; then + rm -rf "/tmp/${RANDOM_DIR:-tmp}" + fi + +} + +install_fmtm diff --git a/src/frontend/src/views/Organization.tsx b/src/frontend/src/views/Organization.tsx index 37bbe63b81..a4a1a8db4e 100644 --- a/src/frontend/src/views/Organization.tsx +++ b/src/frontend/src/views/Organization.tsx @@ -117,7 +117,7 @@ const Organization = () => { src={ data.logo ? `${import.meta.env.VITE_API_URL}/images/${data.logo}` - : 'http://localhost:8080/d907cf67fe587072a592.png' + : 'http://localhost:7051/d907cf67fe587072a592.png' } sx={{ width: '150px' }} /> diff --git a/src/frontend/vite.config.ts b/src/frontend/vite.config.ts index c101d9eed3..a1322b69dd 100644 --- a/src/frontend/vite.config.ts +++ b/src/frontend/vite.config.ts @@ -7,7 +7,7 @@ import { VitePWA } from 'vite-plugin-pwa'; export default defineConfig({ plugins: [react(), VitePWA({ registerType: 'autoUpdate' })], server: { - port: 8080, + port: 7051, host: '0.0.0.0', watch: { usePolling: true,