From bdee3bd826a68e7ac2c195826155aac46a59a770 Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Mon, 4 Dec 2023 06:09:44 -0500 Subject: [PATCH 01/23] Web container fixes and backups templates (#301) * Add osmcha container * Add chartpress config for osmcha * Update start point in osmcha * Add osmcha database helm template * Update osmcha db config * Add helm template for osmcha-web * Rename files * Update osmcha db helm template * Update osmcha-web deployment template * Add osm server env var for osmcha * Update OSMCha Dockerfile and start script * Remove pip uninstall command * Update env vars for osmcha web * Update env var * Open port 81 for cgimap * Fix service - web config * Add cgimap endpoint * Update to web-cgimap endpoint * Update cgimap endpoint * Build osmcha to ohm config + add env vars * Fix git clone url * Use npm install instead of yarn * Install yarnpkg and use it instead of npm * Try to fix error on yarn command * Fix yarn instalation for osmcha * Add extra osmcha env vars in deployent * Fix the path for static files - osmcha * Build osmcha app * Remove cgimap from web config * Add multiple backup database jobs * Add supervisor + nginx * Update docker container for backup db * Update Script to backup and restore db * Update path for bucket and env vars for backups db * Change port for osmcha web * Add env vars for backups * Update bakup db template * Add function to clean up backup older than x days * Update target port for osmcha * Update env vars for backup db * Update log and config for backups * Add node selector for backups config * Raising shared memory limit for web api * Install wget for db-bakups container * Store results of restoring db * Fix name for backup * Set file name * osm-seed/values.yaml * Add sharedMemorySize for db * Remove comment * Add liveness using bash --- chartpress.yaml | 88 ++++----- compose/db-backup-restore.yml | 9 +- compose/osmcha.yml | 24 +++ envs/.env.osmcha.example | 24 +++ images/backup-restore/Dockerfile | 30 +-- images/backup-restore/start.sh | 152 +++++++++------ images/osmcha-db/Dockerfile | 9 + images/osmcha-db/README.md | 12 ++ images/osmcha-db/init-postgis.sql | 1 + images/osmcha-web/Dockerfile | 66 +++++++ images/osmcha-web/README.md | 14 ++ images/osmcha-web/django.conf | 31 ++++ images/osmcha-web/gunicorn.conf | 6 + images/osmcha-web/start.sh | 15 ++ images/osmcha-web/supervisord.conf | 5 + images/web/Dockerfile | 1 + images/web/liveness.sh | 9 + osm-seed/templates/db/db-backup-job.yaml | 142 +++++++++----- osm-seed/templates/db/db-statefulset.yaml | 8 +- .../templates/osmcha-db/osmcha-db-pd.yaml | 98 ++++++++++ .../osmcha-db/osmcha-db-service.yaml | 19 ++ .../osmcha-db/osmcha-db-statefulset.yaml | 98 ++++++++++ .../osmcha-web/osmcha-web-deployment.yaml | 127 +++++++++++++ .../osmcha-web/osmcha-web-ingress.yaml | 26 +++ .../osmcha-web/osmcha-web-service.yaml | 43 +++++ osm-seed/templates/web/web-deployment.yaml | 20 ++ osm-seed/values.yaml | 173 +++++++++++++++--- 27 files changed, 1068 insertions(+), 182 deletions(-) create mode 100644 compose/osmcha.yml create mode 100644 envs/.env.osmcha.example create mode 100644 images/osmcha-db/Dockerfile create mode 100644 images/osmcha-db/README.md create mode 100644 images/osmcha-db/init-postgis.sql create mode 100644 images/osmcha-web/Dockerfile create mode 100644 images/osmcha-web/README.md create mode 100644 images/osmcha-web/django.conf create mode 100644 images/osmcha-web/gunicorn.conf create mode 100755 images/osmcha-web/start.sh create mode 100644 images/osmcha-web/supervisord.conf create mode 100755 images/web/liveness.sh create mode 100644 osm-seed/templates/osmcha-db/osmcha-db-pd.yaml create mode 100644 osm-seed/templates/osmcha-db/osmcha-db-service.yaml create mode 100644 osm-seed/templates/osmcha-db/osmcha-db-statefulset.yaml create mode 100644 osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml create mode 100644 osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml create mode 100644 osm-seed/templates/osmcha-web/osmcha-web-service.yaml diff --git a/chartpress.yaml b/chartpress.yaml index 98f0f471..df170850 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -1,44 +1,48 @@ charts: - - name: osm-seed - imagePrefix: developmentseed/osmseed- - repo: - git: developmentseed/osm-seed-chart - published: https://devseed.com/osm-seed-chart +- name: osm-seed + imagePrefix: developmentseed/osmseed- + repo: + git: developmentseed/osm-seed-chart + published: https://devseed.com/osm-seed-chart - images: - web: - valuesPath: web.image - db: - valuesPath: db.image - backup-restore: - valuesPath: dbBackupRestore.image - planet-dump: - valuesPath: planetDump.image - full-history: - valuesPath: fullHistory.image - replication-job: - valuesPath: replicationJob.image - populate-apidb: - valuesPath: populateApidb.image - osm-processor: - valuesPath: osmProcessor.image - tiler-db: - valuesPath: tilerDb.image - tiler-imposm: - valuesPath: tilerImposm.image - tiler-server: - valuesPath: tilerServer.image - tasking-manager-api: - valuesPath: tmApi.image - # tiler-visor: - # valuesPath: tilerVisor.image - nominatim: - valuesPath: nominatimApi.image - overpass-api: - valuesPath: overpassApi.image - taginfo: - valuesPath: taginfo.image - osm-simple-metrics: - valuesPath: osmSimpleMetrics.image - changeset-replication-job: - valuesPath: changesetReplicationJob.image \ No newline at end of file + images: + web: + valuesPath: web.image + db: + valuesPath: db.image + backup-restore: + valuesPath: dbBackupRestore.image + planet-dump: + valuesPath: planetDump.image + full-history: + valuesPath: fullHistory.image + replication-job: + valuesPath: replicationJob.image + populate-apidb: + valuesPath: populateApidb.image + osm-processor: + valuesPath: osmProcessor.image + tiler-db: + valuesPath: tilerDb.image + tiler-imposm: + valuesPath: tilerImposm.image + tiler-server: + valuesPath: tilerServer.image + tasking-manager-api: + valuesPath: tmApi.image + # tiler-visor: + # valuesPath: tilerVisor.image + nominatim: + valuesPath: nominatimApi.image + overpass-api: + valuesPath: overpassApi.image + taginfo: + valuesPath: taginfo.image + osm-simple-metrics: + valuesPath: osmSimpleMetrics.image + changeset-replication-job: + valuesPath: changesetReplicationJob.image + osmcha-web: + valuesPath: osmchaWeb.image + osmcha-db: + valuesPath: osmchaDb.image diff --git a/compose/db-backup-restore.yml b/compose/db-backup-restore.yml index 7f98764e..7641cde8 100644 --- a/compose/db-backup-restore.yml +++ b/compose/db-backup-restore.yml @@ -9,13 +9,14 @@ services: context: ../images/backup-restore dockerfile: Dockerfile volumes: - - ../data/db-backup-restore-data:/mnt/data + - ../data/db-backup-restore-data:/mnt/data command: > /bin/bash -c " echo 'Set cron job for backing up DB, every 4 minutes'; while :; do echo 'Creating DB backup...'; /start.sh; sleep 4m; done; " env_file: - - ../envs/.env.db - - ../envs/.env.db-utils - - ../envs/.env.cloudprovider + - ../envs/.env.db + - ../envs/.env.osmcha + - ../envs/.env.db-utils + - ../envs/.env.cloudprovider diff --git a/compose/osmcha.yml b/compose/osmcha.yml new file mode 100644 index 00000000..5a1ae4e6 --- /dev/null +++ b/compose/osmcha.yml @@ -0,0 +1,24 @@ +version: '3' +services: + osmcha-db: + platform: linux/amd64 + image: osmseed-osmcha-db:v1 + build: + context: ../images/osmcha-db + dockerfile: Dockerfile + ports: + - '5432:5432' + volumes: + - ../data/osmcha-db-data:/var/lib/postgresql/data + env_file: + - ../envs/.env.osmcha + osmcha-web: + platform: linux/amd64 + image: osmseed-osmcha-web:v1 + build: + context: ../images/osmcha-web + dockerfile: Dockerfile + ports: + - '8000:8000' + env_file: + - ../envs/.env.osmcha diff --git a/envs/.env.osmcha.example b/envs/.env.osmcha.example new file mode 100644 index 00000000..7c3d265c --- /dev/null +++ b/envs/.env.osmcha.example @@ -0,0 +1,24 @@ +POSTGRES_DB=osmcha +POSTGRES_USER=postgres +POSTGRES_PASSWORD=1234 +PGHOST=osmcha-db +DJANGO_DEBUG=true +DJANGO_SECRET_KEY=abc +OAUTH_OSM_KEY=xya +OAUTH_OSM_SECRET=xya +DJANGO_ANON_USER_THROTTLE_RATE=30/min +DJANGO_COMMON_USER_THROTTLE_RATE=180/min +DJANGO_NON_STAFF_USER_THROTTLE_RATE=3/min +OSMCHA_URL=public-url +OAUTH_REDIRECT_URI=$OSMCHA_URL/oauth-landing.html +DJANGO_ENABLE_CHANGESET_COMMENTS=False +DJANGO_OSM_COMMENTS_API_KEY="" +DJANGO_ROOT=/app +DJANGO_SETTINGS_MODULE=config.settings.local +C_FORCE_ROOT=True +REACT_APP_OSM_URL=https://www.openhistoricalmap.org +REACT_APP_OSM_API=https://www.openhistoricalmap.org/api/0.6 +REACT_APP_OVERPASS_BASE=https://overpass-api.openhistoricalmap.org/api/interpreter +REACT_APP_NOMINATIM_URL=https://www.openhistoricalmap.org/geocoder/search_osm_nominatim +REACT_APP_PRODUCTION_API_URL=$OSMCHA_URL/api/v1 +REACT_APP_MAPBOX_ACCESS_TOKEN='' diff --git a/images/backup-restore/Dockerfile b/images/backup-restore/Dockerfile index b4356155..95996a98 100644 --- a/images/backup-restore/Dockerfile +++ b/images/backup-restore/Dockerfile @@ -1,19 +1,23 @@ -FROM python:3.9.9 -RUN apt-get update -RUN apt-get install -y \ - curl \ - postgresql-client +FROM python:3.9.9-slim -# Install AWS CLI -RUN pip install awscli +# Install Postgres client, GCP CLI, and Azure CLI +RUN apt-get update \ + && apt-get install -y curl apt-transport-https lsb-release gnupg \ + && echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ + && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ + && apt-get update \ + && apt-get install -y postgresql-client-14 wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # Install GCP CLI -RUN curl -sSL https://sdk.cloud.google.com | bash -RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil +RUN curl -sSL https://sdk.cloud.google.com | bash \ + && ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil +RUN rm -rf /root/google-cloud-sdk/.install/.backup -# Install Azure CLI -RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash +# Install AWS CLI +RUN pip install awscli -VOLUME /mnt/data COPY ./start.sh / -CMD /start.sh +CMD ["/start.sh"] diff --git a/images/backup-restore/start.sh b/images/backup-restore/start.sh index 2f21a0ad..2648f321 100755 --- a/images/backup-restore/start.sh +++ b/images/backup-restore/start.sh @@ -1,65 +1,111 @@ #!/usr/bin/env bash +set -e export PGPASSWORD=$POSTGRES_PASSWORD -export VOLUME_DIR=/mnt/data +# Upload files +cloudStorageOps() { + local LOCAL_STATE_FILE=state.txt + local filepath=$1 + local cloudpath=$2 -date=$(date '+%y%m%d_%H%M') -local_backupFile=$VOLUME_DIR/osmseed-db-${date}.sql.gz -cloud_backupFile=database/osmseed-db-${date}.sql.gz -stateFile=$VOLUME_DIR/state.txt -restoreFile=$VOLUME_DIR/backup.sql.gz - -echo "Start...$DB_ACTION action" -# Backing up DataBase -if [ "$DB_ACTION" == "backup" ]; then - # Backup database and make maximum compression at the slowest speed - pg_dump -h $POSTGRES_HOST -U $POSTGRES_USER $POSTGRES_DB | gzip -9 >$local_backupFile - - # AWS - if [ "$CLOUDPROVIDER" == "aws" ]; then - echo "$AWS_S3_BUCKET/$cloud_backupFile" > $stateFile - # Upload db backup file - aws s3 cp $local_backupFile $AWS_S3_BUCKET/$cloud_backupFile - # Upload state.txt file - aws s3 cp $stateFile $AWS_S3_BUCKET/database/state.txt - fi - - # GCP - if [ "$CLOUDPROVIDER" == "gcp" ]; then - echo "$GCP_STORAGE_BUCKET/$cloud_backupFile" > $stateFile - # Upload db backup file - gsutil cp $local_backupFile $GCP_STORAGE_BUCKET/$cloud_backupFile - # Upload state.txt file - gsutil cp $stateFile $GCP_STORAGE_BUCKET/database/state.txt - fi - - # Azure - if [ "$CLOUDPROVIDER" == "azure" ]; then - # Save the path file - echo "blob://$AZURE_STORAGE_ACCOUNT/$AZURE_CONTAINER_NAME/$cloud_backupFile" > $stateFile - # Upload db backup file + case "${CLOUDPROVIDER}" in + aws) + aws s3 cp ${filepath} s3://${AWS_S3_BUCKET}/${cloudpath} + echo s3://${AWS_S3_BUCKET}/${cloudpath} >${LOCAL_STATE_FILE} + aws s3 cp ${LOCAL_STATE_FILE} s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/state.txt + ;; + gcp) + gsutil cp ${filepath} gs://${GCP_STORAGE_BUCKET}/${cloudpath} + echo "gs://${GCP_STORAGE_BUCKET}/${CLOUD_BACKUP_FILE}" >${LOCAL_STATE_FILE} + gsutil cp ${LOCAL_STATE_FILE} gs://${GCP_STORAGE_BUCKET}/${BACKUP_CLOUD_FOLDER}/state.txt + ;; + azure) az storage blob upload \ - --container-name $AZURE_CONTAINER_NAME \ - --file $local_backupFile \ - --name $cloud_backupFile \ + --container-name ${AZURE_CONTAINER_NAME} \ + --file ${filepath} \ + --name ${cloudpath} \ --output table - # Upload state.txt file + echo "blob://${AZURE_STORAGE_ACCOUNT}/${AZURE_CONTAINER_NAME}/${CLOUD_BACKUP_FILE}" >${LOCAL_STATE_FILE} az storage blob upload \ - --container-name $AZURE_CONTAINER_NAME \ - --file $stateFile \ - --name database/state.txt \ + --container-name ${AZURE_CONTAINER_NAME} \ + --file ${LOCAL_STATE_FILE} \ + --name ${BACKUP_CLOUD_FOLDER}/state.txt \ --output table + ;; + esac +} + +backupDB() { + local LOCAL_BACKUP_FILE=${BACKUP_CLOUD_FILE}.sql.gz + local CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}.sql.gz" + if [ "$SET_DATE_AT_NAME" == "true" ]; then + local CURRENT_DATE=$(date '+%Y%m%d-%H%M') + LOCAL_BACKUP_FILE="${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.sql.gz" + CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.sql.gz" fi -fi -# Restoring DataBase -if [ "$DB_ACTION" == "restore" ]; then - # AWS - flag=true + # Backup database with max compression + echo "Backing up DB ${POSTGRES_DB} into ${LOCAL_BACKUP_FILE}" + pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} ${POSTGRES_DB} | gzip -9 >${LOCAL_BACKUP_FILE} + + # Handle cloud storage based on the provider + cloudStorageOps "${LOCAL_BACKUP_FILE}" "${CLOUD_BACKUP_FILE}" +} + +restoreDB() { + local CURRENT_DATE=$(date '+%Y%m%d-%H%M') + local RESTORE_FILE="backup.sql.gz" + local LOG_RESULT_FILE="restore_results-${CURRENT_DATE}.log" + local flag=true + while "$flag" = true; do - pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue - flag=false - wget -O $restoreFile $RESTORE_URL_FILE - gunzip <$restoreFile | psql -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB - echo " Import data to $POSTGRES_DB has finished ..." + pg_isready -h ${POSTGRES_HOST} -p 5432 >/dev/null 2>&2 || continue + flag=false + wget -O ${RESTORE_FILE} ${RESTORE_URL_FILE} + echo "Restoring ${RESTORE_URL_FILE} in ${POSTGRES_DB}" + gunzip -c <${RESTORE_FILE} | psql -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DB} | tee ${LOG_RESULT_FILE} + aws s3 cp ${LOG_RESULT_FILE} s3://${AWS_S3_BUCKET}/${LOG_RESULT_FILE} + echo "Import data to ${POSTGRES_DB} has finished ..." + done +} + +delete_old_s3_files() { + # Use RETENTION_DAYS from environment variable or default to 30 days + if [ -z "${RETENTION_DAYS}" ]; then + DAYS_AGO=30 + else + DAYS_AGO="${RETENTION_DAYS}" + fi + + echo "Files older than $DAYS_AGO days will be deleted." + echo "Processing s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/" + TARGET_DATE=$(date -d "${DAYS_AGO} days ago" +%Y-%m-%d) + aws s3 ls "s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/" --recursive | while read -r line; do + FILE_DATE=$(echo "$line" | awk '{print $1}') + FILE_PATH=$(echo "$line" | awk '{print $4}') + if [[ "$FILE_DATE" < "$TARGET_DATE" && ! -z "$FILE_PATH" ]]; then + echo "Deleting ${FILE_PATH} which was modified on ${FILE_DATE}" + aws s3 rm "s3://${AWS_S3_BUCKET}/${FILE_PATH}" + fi done +} + +# Main logic +case "${DB_ACTION}" in +backup) + backupDB + ;; +restore) + restoreDB + ;; +*) + echo "Unknown action: ${DB_ACTION}" + exit 1 + ;; +esac + +# Check for the CLEAN_BACKUPS var +if [ "$CLEANUP_BACKUPS" == "true" ]; then + delete_old_s3_files +else + echo "CLEANUP_BACKUPS is not set to true. Skipping deletion." fi diff --git a/images/osmcha-db/Dockerfile b/images/osmcha-db/Dockerfile new file mode 100644 index 00000000..7fcb57d9 --- /dev/null +++ b/images/osmcha-db/Dockerfile @@ -0,0 +1,9 @@ +FROM postgres:14 +# Install PostGIS +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + postgis \ + postgresql-14-postgis-3 && \ + rm -rf /var/lib/apt/lists/* +# Initialize PostGIS +COPY init-postgis.sql /docker-entrypoint-initdb.d/ diff --git a/images/osmcha-db/README.md b/images/osmcha-db/README.md new file mode 100644 index 00000000..d751f703 --- /dev/null +++ b/images/osmcha-db/README.md @@ -0,0 +1,12 @@ +# Postgres 14 for OpenStreetMap Changeset Analyzer (OSMCha) + +## Build +```sh +docker compose -f /compose/osmcha.yml build +``` + +## Run + +```sh +docker compose -f /compose/osmcha.yml up +``` \ No newline at end of file diff --git a/images/osmcha-db/init-postgis.sql b/images/osmcha-db/init-postgis.sql new file mode 100644 index 00000000..576e5426 --- /dev/null +++ b/images/osmcha-db/init-postgis.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS postgis; diff --git a/images/osmcha-web/Dockerfile b/images/osmcha-web/Dockerfile new file mode 100644 index 00000000..ae3c6898 --- /dev/null +++ b/images/osmcha-web/Dockerfile @@ -0,0 +1,66 @@ +FROM ubuntu:20.04 +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y -qq +RUN apt-get install -y curl python3 python3-dev \ + python3-pip git libgeos-dev libyaml-dev \ + libcurl4-gnutls-dev librtmp-dev python3-gdal \ + locales nginx supervisor yarnpkg nodejs npm +RUN locale-gen en_US.UTF-8 +RUN apt-get clean +RUN rm -rf /var/lib/apt/lists/* + +# Install yarn. +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN apt-get update -y +RUN apt-get remove -y yarnpkg cmdtest && apt-get install -y yarn && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Setup nginx +RUN rm /etc/nginx/sites-enabled/default +COPY django.conf /etc/nginx/sites-available/ +RUN ln -s /etc/nginx/sites-available/django.conf /etc/nginx/sites-enabled/django.conf +RUN echo "daemon off;" >> /etc/nginx/nginx.conf + + +# Setting locale is required by unicode-slugify (osmcha-django python requirement) +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +WORKDIR /app +ENV BACKEND_VERSION v4.19.0 +RUN git clone https://github.com/willemarcel/osmcha-django . +RUN git checkout $BACKEND_VERSION +RUN git pull origin $BACKEND_VERSION +RUN pip3 install -r requirements/production.txt +RUN pip3 install -r requirements/local.txt + +ADD . /app +ENV DJANGO_ROOT /app +ENV DJANGO_SETTINGS_MODULE config.settings.aws_production + +# Replace social-auth-core by a modified version that provides OHM authentication +RUN pip3 install git+https://github.com/OpenHistoricalMap/social-core.git@master + +# Upgrading requests to the latest version. +RUN pip3 install -U requests +RUN pip3 install certifi +RUN pip3 install django-extensions + +# Clone and build the frontend +RUN git clone https://github.com/mapbox/osmcha-frontend.git /osmcha-frontend +RUN cd /osmcha-frontend && yarn install + +RUN echo "alias python=python3" >> ~/.bashrc +RUN echo "alias pip=pip3" >> ~/.bashrc + +EXPOSE 80 + +# Supervisor config +RUN mkdir -p /var/log/supervisor +COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY gunicorn.conf /etc/supervisor/conf.d/gunicorn.conf + +COPY ./start.sh . +CMD ./start.sh diff --git a/images/osmcha-web/README.md b/images/osmcha-web/README.md new file mode 100644 index 00000000..c653ed65 --- /dev/null +++ b/images/osmcha-web/README.md @@ -0,0 +1,14 @@ +# OpenStreetMap Changeset Analyzer (OSMCha) + +OSMCha in a Docker containe that integrated with the OSM-Seed Ecosystem + +## Build +```sh +docker compose -f /compose/osmcha.yml build +``` + +## Run + +```sh +docker compose -f /compose/osmcha.yml up +``` \ No newline at end of file diff --git a/images/osmcha-web/django.conf b/images/osmcha-web/django.conf new file mode 100644 index 00000000..ff788d99 --- /dev/null +++ b/images/osmcha-web/django.conf @@ -0,0 +1,31 @@ +server { + listen 80; + charset utf-8; + client_max_body_size 20M; + + location /static/ { + gzip_static on; + alias /app/staticfiles/; + } + + location /index.html { + gzip_static on; + alias /app/osmchadjango/frontend/templates/frontend/index.html; + } + + location / { + # checks for static file, if not found proxy to app + try_files $uri @proxy_to_app; + } + + # cookiecutter-django app + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://127.0.0.1:5000; + fastcgi_connect_timeout 130; + fastcgi_send_timeout 130; + fastcgi_read_timeout 130; + } +} diff --git a/images/osmcha-web/gunicorn.conf b/images/osmcha-web/gunicorn.conf new file mode 100644 index 00000000..21d99bb7 --- /dev/null +++ b/images/osmcha-web/gunicorn.conf @@ -0,0 +1,6 @@ +[program:gunicorn] +command=/usr/local/bin/gunicorn config.wsgi -b localhost:5000 -w 4 --log-file /var/log/gunicorn.error --access-logfile /var/log/gunicorn.access --capture-output --timeout 120 +directory=/app +autostart=true +autorestart=true +redirect_stderr=true diff --git a/images/osmcha-web/start.sh b/images/osmcha-web/start.sh new file mode 100755 index 00000000..b37767e0 --- /dev/null +++ b/images/osmcha-web/start.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -ex + +# Build frontend +cd /osmcha-frontend +REACT_APP_VERSION=ohm REACT_APP_STACK=PRODUCTION PUBLIC_URL=$OSMCHA_URL npx react-scripts build +cp -R build/*.html /app/osmchadjango/frontend/templates/frontend/ +cp -R build/* /app/osmchadjango/static/ +cp -R build/static/* /app/osmchadjango/static/ + +# Start service +cd /app +python3 manage.py collectstatic --noinput +python3 manage.py migrate +supervisord -c /etc/supervisor/supervisord.conf diff --git a/images/osmcha-web/supervisord.conf b/images/osmcha-web/supervisord.conf new file mode 100644 index 00000000..8d520a8c --- /dev/null +++ b/images/osmcha-web/supervisord.conf @@ -0,0 +1,5 @@ +[supervisord] +nodaemon=true + +[program:nginx] +command=/usr/sbin/nginx diff --git a/images/web/Dockerfile b/images/web/Dockerfile index ce843aca..bf8b4fed 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -101,5 +101,6 @@ RUN chown -R www-data: $workdir ADD config/settings.yml $workdir/config/settings.yml COPY start.sh $workdir/ +COPY liveness.sh $workdir/ CMD $workdir/start.sh diff --git a/images/web/liveness.sh b/images/web/liveness.sh new file mode 100755 index 00000000..5d30dfd1 --- /dev/null +++ b/images/web/liveness.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# This is a script for the complex evaluation of whether Apache or other processes are running in the container. +if [ $(ps -ef | grep -E 'httpd|apache2' | grep -v grep | wc -l) -ge 1 ]; then + echo "Apache is running." + exit 0 +else + echo "Apache is not running!" 1>&2 + exit 1 +fi diff --git a/osm-seed/templates/db/db-backup-job.yaml b/osm-seed/templates/db/db-backup-job.yaml index 8cabd74a..40e0b5d6 100644 --- a/osm-seed/templates/db/db-backup-job.yaml +++ b/osm-seed/templates/db/db-backup-job.yaml @@ -1,15 +1,15 @@ -{{- if .Values.dbBackupRestore.enabled -}} +{{- range $i, $job := .Values.dbBackupRestore.cronjobs }} +{{- if $job.enabled -}} apiVersion: batch/v1 kind: CronJob metadata: - name: {{ .Release.Name }}-db-backup-job + name: {{ $.Release.Name }}-{{ $job.name }}-backup-job labels: - app: {{ template "osm-seed.name" . }} - component: db-backup-job - environment: {{ .Values.environment }} - release: {{ .Release.Name }} + component: {{ $job.name }}-job + environment: {{ $.Values.environment }} + release: {{ $.Release.Name }} spec: - schedule: {{ quote .Values.dbBackupRestore.schedule }} + schedule: {{ quote $job.schedule }} successfulJobsHistoryLimit: 2 failedJobsHistoryLimit: 2 startingDeadlineSeconds: 100 @@ -19,57 +19,109 @@ spec: template: spec: containers: - - name: {{ .Release.Name }}-db-backup-job - image: {{ .Values.dbBackupRestore.image.name }}:{{ .Values.dbBackupRestore.image.tag }} + - name: {{ $job.name }}-backup-job + image: {{ $.Values.dbBackupRestore.image.name }}:{{ $.Values.dbBackupRestore.image.tag }} command: ['/start.sh'] - {{- if .Values.dbBackupRestore.resources.enabled }} - resources: - requests: - memory: {{ .Values.dbBackupRestore.resources.requests.memory }} - cpu: {{ .Values.dbBackupRestore.resources.requests.cpu }} - limits: - memory: {{ .Values.dbBackupRestore.resources.limits.memory }} - cpu: {{ .Values.dbBackupRestore.resources.limits.cpu }} - {{- end }} env: + # Env vars for api-web database + {{ if eq $job.name "web-db" }} + # DB credentials - name: POSTGRES_HOST - value: {{ .Release.Name }}-db + value: {{ $.Release.Name }}-db - name: POSTGRES_DB - value: {{ .Values.db.env.POSTGRES_DB }} + value: {{ $.Values.db.env.POSTGRES_DB }} - name: POSTGRES_PASSWORD - value: {{ quote .Values.db.env.POSTGRES_PASSWORD }} + value: {{ quote $.Values.db.env.POSTGRES_PASSWORD }} - name: POSTGRES_USER - value: {{ .Values.db.env.POSTGRES_USER }} + value: {{ $.Values.db.env.POSTGRES_USER }} + # DB action: backup/restore - name: DB_ACTION - value: {{ .Values.dbBackupRestore.env.DB_ACTION }} + value: {{quote $job.env.DB_ACTION }} + # Backup naming + - name: SET_DATE_AT_NAME + value: {{quote $job.env.SET_DATE_AT_NAME }} + - name: BACKUP_CLOUD_FOLDER + value: {{ $job.env.BACKUP_CLOUD_FOLDER }} + - name: BACKUP_CLOUD_FILE + value: {{ $job.env.BACKUP_CLOUD_FILE }} + - name: AWS_S3_BUCKET + value: {{ $job.env.AWS_S3_BUCKET }} + # Clean up options + - name: CLEANUP_BACKUPS + value: {{ quote $job.env.CLEANUP_BACKUPS }} + - name: RETENTION_DAYS + value: {{ quote $job.env.RETENTION_DAYS }} + # In case DB_ACTION=restore, adn the backup url - name: RESTORE_URL_FILE - value: {{.Values.dbBackupRestore.env.RESTORE_URL_FILE}} - - name: CLEAN_BACKUPS - value: {{ quote .Values.dbBackupRestore.env.CLEAN_BACKUPS }} - - name: CLOUDPROVIDER - value: {{ .Values.cloudProvider }} - # In case cloudProvider=aws - {{- if eq .Values.cloudProvider "aws" }} + value: {{ $job.env.RESTORE_URL_FILE}} + {{- end }} + # Env vars for osmcha database + {{ if eq $job.name "osmcha-db" }} + # DB credentials + - name: POSTGRES_HOST + value: {{ $.Release.Name }}-osmcha-db + - name: POSTGRES_DB + value: {{ $.Values.osmchaDb.env.POSTGRES_DB }} + - name: POSTGRES_PASSWORD + value: {{ quote $.Values.osmchaDb.env.POSTGRES_PASSWORD }} + - name: POSTGRES_USER + value: {{ $.Values.osmchaDb.env.POSTGRES_USER }} + # DB action: backup/restore + - name: DB_ACTION + value: {{ quote $job.env.DB_ACTION }} + # Backup naming + - name: SET_DATE_AT_NAME + value: {{quote $job.env.SET_DATE_AT_NAME }} + - name: BACKUP_CLOUD_FOLDER + value: {{ $job.env.BACKUP_CLOUD_FOLDER }} + - name: BACKUP_CLOUD_FILE + value: {{ $job.env.BACKUP_CLOUD_FILE }} - name: AWS_S3_BUCKET - value: {{ .Values.dbBackupRestore.env.AWS_S3_BUCKET }} - {{- end }} - # In case cloudProvider=gcp - {{- if eq .Values.cloudProvider "gcp" }} + value: {{ $job.env.AWS_S3_BUCKET }} + # Clean up options + - name: CLEANUP_BACKUPS + value: {{ quote $job.env.CLEANUP_BACKUPS }} + - name: RETENTION_DAYS + value: {{ quote $job.env.RETENTION_DAYS }} + # In case DB_ACTION=restore, adn the backup url + - name: RESTORE_URL_FILE + value: {{ $job.env.RESTORE_URL_FILE}} + {{- end }} + # Cloud provider + - name: CLOUDPROVIDER + value: {{ $.Values.cloudProvider }} + # Credentials or configurations for each cloud provider + # {{- if eq $.Values.cloudProvider "aws" }} + # - name: AWS_S3_BUCKET + # value: {{ $job.env.AWS_S3_BUCKET }} + # {{- end }} + {{- if eq $.Values.cloudProvider "gcp" }} - name: GCP_STORAGE_BUCKET - value: {{ .Values.GCP_STORAGE_BUCKET }} - {{- end }} - # In case cloudProvider=azure - {{- if eq .Values.cloudProvider "azure" }} + value: {{ $job.env.GCP_STORAGE_BUCKET }} + {{- end }} + {{- if eq $.Values.cloudProvider "azure" }} - name: AZURE_STORAGE_ACCOUNT - value: {{ .Values.AZURE_STORAGE_ACCOUNT }} + value: {{ $.Values.AZURE_STORAGE_ACCOUNT }} - name: AZURE_CONTAINER_NAME - value: {{ .Values.AZURE_CONTAINER_NAME }} + value: {{ $job.AZURE_CONTAINER_NAME }} - name: AZURE_STORAGE_CONNECTION_STRING - value: {{ .Values.AZURE_STORAGE_CONNECTION_STRING }} - {{- end }} + value: {{ $.Values.AZURE_STORAGE_CONNECTION_STRING }} + {{- end }} + # Node selector + {{- if $job.resources.enabled }} + resources: + requests: + memory: {{ $job.resources.requests.memory }} + cpu: {{ $job.resources.requests.cpu }} + limits: + memory: {{ $job.resources.limits.memory }} + cpu: {{ $job.resources.limits.cpu }} + {{- end }} restartPolicy: OnFailure - {{- if .Values.dbBackupRestore.nodeSelector.enabled }} + {{- if $job.nodeSelector.enabled }} nodeSelector: - {{ .Values.dbBackupRestore.nodeSelector.label_key }} : {{ .Values.dbBackupRestore.nodeSelector.label_value }} + {{ $job.nodeSelector.label_key }} : {{ $job.nodeSelector.label_value }} {{- end }} -{{- end }} \ No newline at end of file +--- +{{- end }} +{{- end }} diff --git a/osm-seed/templates/db/db-statefulset.yaml b/osm-seed/templates/db/db-statefulset.yaml index 4422f232..a8fc04a0 100644 --- a/osm-seed/templates/db/db-statefulset.yaml +++ b/osm-seed/templates/db/db-statefulset.yaml @@ -9,7 +9,7 @@ metadata: environment: {{ .Values.environment }} release: {{ .Release.Name }} spec: - replicas: 1 # we only ever want one replica of the db + replicas: 1 selector: matchLabels: app: {{ template "osm-seed.name" . }} @@ -69,6 +69,8 @@ spec: - name: postgres-storage mountPath: {{ .Values.db.persistenceDisk.mountPath }} subPath: {{ .Values.db.persistenceDisk.subPath }} + - name: shared-memory + mountPath: /dev/shm {{- if .Values.db.resources.enabled }} resources: requests: @@ -86,6 +88,10 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: shared-memory + emptyDir: + medium: Memory + sizeLimit: {{ .Values.db.sharedMemorySize }} {{- if .Values.db.nodeSelector.enabled }} nodeSelector: {{ .Values.db.nodeSelector.label_key }} : {{ .Values.db.nodeSelector.label_value }} diff --git a/osm-seed/templates/osmcha-db/osmcha-db-pd.yaml b/osm-seed/templates/osmcha-db/osmcha-db-pd.yaml new file mode 100644 index 00000000..e3ad67da --- /dev/null +++ b/osm-seed/templates/osmcha-db/osmcha-db-pd.yaml @@ -0,0 +1,98 @@ +{{- if .Values.osmchaDb.enabled -}} +{{- if .Values.osmchaDb.persistenceDisk.enabled -}} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ .Release.Name }}-osmcha-db-pv + labels: + app: {{ template "osm-seed.name" . }} + component: db-osmcha-pv + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + accessModes: + - ReadWriteOnce + ############## Minikube ############## + {{- if eq .Values.cloudProvider "minikube" }} + storageClassName: local-storage + capacity: + storage: {{ .Values.osmchaDb.persistenceDisk.localVolumeSize }} + hostPath: + path: {{ .Values.osmchaDb.persistenceDisk.localVolumeHostPath | quote }} + {{- end }} + + ############## AWS Provider ############## + {{- if eq .Values.cloudProvider "aws" }} + storageClassName: "" + capacity: + storage: {{ .Values.osmchaDb.persistenceDisk.AWS_ElasticBlockStore_size }} + awsElasticBlockStore: + volumeID: {{ .Values.osmchaDb.persistenceDisk.AWS_ElasticBlockStore_volumeID }} + fsType: ext4 + {{- end }} + + ############## GCP Provider ############## + {{- if eq .Values.cloudProvider "gcp" }} + storageClassName: "" + capacity: + storage: {{ .Values.osmchaDb.persistenceDisk.GCP_gcePersistentDisk_size }} + gcePersistentDisk: + pdName: {{ .Values.osmchaDb.persistenceDisk.GCP_gcePersistentDisk_pdName }} + fsType: ext4 + {{- end }} + + ############## Azure Provider ############## + {{- if eq .Values.cloudProvider "azure" }} + storageClassName: "" + capacity: + storage: {{ .Values.osmchaDb.persistenceDisk.AZURE_diskSize }} + persistentVolumeReclaimPolicy: Retain + azureDisk: + kind: Managed + diskName: {{ .Values.osmchaDb.persistenceDisk.AZURE_diskName }} + diskURI: {{ .Values.osmchaDb.persistenceDisk.AZURE_diskURI }} + {{- end }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-osmcha-db-pv-claim + labels: + app: {{ template "osm-seed.name" . }} + component: db-osmcha-pvc + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + ############## Minikube ############## + {{- if eq .Values.cloudProvider "minikube" }} + storageClassName: local-storage + {{- else }} + ############## Other providers ############## + storageClassName: "" + {{- end }} + volumeName: {{ .Release.Name }}-osmcha-db-pv + accessModes: + - ReadWriteOnce + resources: + requests: + ############## Minikube ############## + {{- if eq .Values.cloudProvider "minikube" }} + storage: {{ .Values.osmchaDb.persistenceDisk.localVolumeSize }} + {{- end }} + + ############## AWS Provider ############## + {{- if eq .Values.cloudProvider "aws" }} + storage: {{ .Values.osmchaDb.persistenceDisk.AWS_ElasticBlockStore_size }} + {{- end }} + + ############## GCP Provider ############## + {{- if eq .Values.cloudProvider "gcp" }} + storage: {{ .Values.osmchaDb.persistenceDisk.GCP_gcePersistentDisk_size }} + {{- end }} + + ############## AZURE Provider ############## + {{- if eq .Values.cloudProvider "azure" }} + storage: {{ .Values.osmchaDb.persistenceDisk.AZURE_diskSize }} + {{- end }} +{{- end }} +{{- end }} diff --git a/osm-seed/templates/osmcha-db/osmcha-db-service.yaml b/osm-seed/templates/osmcha-db/osmcha-db-service.yaml new file mode 100644 index 00000000..4ce6b3a7 --- /dev/null +++ b/osm-seed/templates/osmcha-db/osmcha-db-service.yaml @@ -0,0 +1,19 @@ +{{- if .Values.osmchaDb.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-osmcha-db + labels: + app: {{ template "osm-seed.name" . }} + component: osmcha-db-service + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + ports: + - port: 5432 + targetPort: 5432 + selector: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-db +{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/osmcha-db/osmcha-db-statefulset.yaml b/osm-seed/templates/osmcha-db/osmcha-db-statefulset.yaml new file mode 100644 index 00000000..78d32dbd --- /dev/null +++ b/osm-seed/templates/osmcha-db/osmcha-db-statefulset.yaml @@ -0,0 +1,98 @@ +{{- if .Values.osmchaDb.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }}-osmcha-db + labels: + app: {{ template "osm-seed.name" . }} + component: db-tiler-statefulset + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-db + serviceName: {{ .Release.Name }}-osmcha-db + template: + metadata: + labels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-db + spec: + # Remove lost+found in EBS disk + initContainers: + - name: remove-lost-found + image: busybox + command: ['sh', '-c', 'rm -rf /var/lib/postgresql/data/lost+found'] + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + containers: + - name: {{ .Chart.Name }}-osmcha-db + image: "{{ .Values.osmchaDb.image.name }}:{{ .Values.osmchaDb.image.tag }}" + ports: + - name: postgres + containerPort: 5432 + protocol: TCP + env: + - name: PGHOST + value: {{ .Release.Name }}-osmcha-db + - name: POSTGRES_DB + value: {{ .Values.osmchaDb.env.POSTGRES_DB }} + - name: POSTGRES_PASSWORD + value: {{ quote .Values.osmchaDb.env.POSTGRES_PASSWORD }} + - name: POSTGRES_USER + value: {{ default "postgres" .Values.osmchaDb.env.POSTGRES_USER | quote }} + - name: PGUSER + value: {{ default "postgres" .Values.osmchaDb.env.POSTGRES_USER | quote }} + - name: PGDATA + value: {{ .Values.osmchaDb.persistenceDisk.mountPath }} + - name: POD_IP + valueFrom: { fieldRef: { fieldPath: status.podIP } } + livenessProbe: + exec: + command: + - sh + - -c + - exec pg_isready --host $POD_IP + initialDelaySeconds: 120 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + exec: + command: + - sh + - -c + - exec pg_isready --host $POD_IP + initialDelaySeconds: 5 + timeoutSeconds: 3 + periodSeconds: 5 + volumeMounts: + - name: postgres-storage + mountPath: {{ .Values.osmchaDb.persistenceDisk.mountPath }} + {{- if .Values.osmchaDb.resources.enabled }} + resources: + requests: + memory: {{ .Values.osmchaDb.resources.requests.memory }} + cpu: {{ .Values.osmchaDb.resources.requests.cpu }} + limits: + memory: {{ .Values.osmchaDb.resources.limits.memory }} + cpu: {{ .Values.osmchaDb.resources.limits.cpu }} + {{- end }} + volumes: + - name: postgres-storage + {{- if .Values.osmchaDb.persistenceDisk.enabled }} + persistentVolumeClaim: + claimName: {{ .Release.Name }}-osmcha-db-pv-claim + {{- else }} + emptyDir: {} + {{- end }} + {{- if .Values.osmchaDb.nodeSelector.enabled }} + nodeSelector: + {{ .Values.osmchaDb.nodeSelector.label_key }} : {{ .Values.osmchaDb.nodeSelector.label_value }} + {{- end }} +{{- end }} diff --git a/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml b/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml new file mode 100644 index 00000000..81a86f9c --- /dev/null +++ b/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml @@ -0,0 +1,127 @@ +{{- if .Values.osmchaWeb.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-osmcha-web + labels: + app: {{ template "osm-seed.name" . }} + component: osmcha-web-deployment + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.osmchaWeb.replicaCount }} + selector: + matchLabels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-web + template: + metadata: + labels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-web + spec: + containers: + - name: {{ .Chart.Name }}-osmcha-web + image: "{{ .Values.osmchaWeb.image.name }}:{{ .Values.osmchaWeb.image.tag }}" + ports: + - name: http + containerPort: 5000 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 5000 + initialDelaySeconds: 600 # 10 min, because the compile process takes time. + timeoutSeconds: 30 + {{- if .Values.osmchaWeb.resources.enabled }} + resources: + requests: + memory: {{ .Values.osmchaWeb.resources.requests.memory }} + cpu: {{ .Values.osmchaWeb.resources.requests.cpu }} + limits: + memory: {{ .Values.osmchaWeb.resources.limits.memory }} + cpu: {{ .Values.osmchaWeb.resources.limits.cpu }} + {{- end }} + env: + - name: OSM_SERVER_URL + value: {{ .Values.osmchaDb.env.OSM_SERVER_URL | quote }} + - name: POSTGRES_USER + value: {{ .Values.osmchaDb.env.POSTGRES_USER | quote }} + - name: POSTGRES_PASSWORD + value: {{ .Values.osmchaDb.env.POSTGRES_PASSWORD | quote }} + - name: POSTGRES_DB + value: {{ .Values.osmchaDb.env.POSTGRES_DB | quote }} + - name: PGHOST + value: {{ .Release.Name }}-osmcha-db + - name: POSTGRES_HOST + value: {{ .Release.Name }}-osmcha-db + - name: DJANGO_CACHES + value: {{ .Values.osmchaWeb.env.DJANGO_CACHES | quote }} + - name: DJANGO_DEBUG + value: {{ .Values.osmchaWeb.env.DJANGO_DEBUG | quote }} + - name: DJANGO_SECRET_KEY + value: {{ .Values.osmchaWeb.env.DJANGO_SECRET_KEY | quote }} + - name: DJANGO_SECURE_BROWSER_XSS_FILTER + value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_BROWSER_XSS_FILTER | quote }} + - name: DJANGO_SECURE_SSL_REDIRECT + value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_SSL_REDIRECT | quote }} + - name: DJANGO_SECURE_CONTENT_TYPE_NOSNIFF + value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_CONTENT_TYPE_NOSNIFF | quote }} + - name: DJANGO_SECURE_FRAME_DENY + value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_FRAME_DENY | quote }} + - name: DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS + value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS | quote }} + - name: DJANGO_SESSION_COOKIE_HTTPONLY + value: {{ .Values.osmchaWeb.env.DJANGO_SESSION_COOKIE_HTTPONLY | quote }} + - name: DJANGO_SESSION_COOKIE_SECURE + value: {{ .Values.osmchaWeb.env.DJANGO_SESSION_COOKIE_SECURE | quote }} + - name: DJANGO_DEFAULT_FROM_EMAIL + value: {{ .Values.osmchaWeb.env.DJANGO_DEFAULT_FROM_EMAIL | quote }} + - name: DJANGO_SERVER_EMAIL + value: {{ .Values.osmchaWeb.env.DJANGO_SERVER_EMAIL | quote }} + - name: DJANGO_EMAIL_SUBJECT_PREFIX + value: {{ .Values.osmchaWeb.env.DJANGO_EMAIL_SUBJECT_PREFIX | quote }} + - name: DJANGO_CHANGESETS_FILTER + value: {{ .Values.osmchaWeb.env.DJANGO_CHANGESETS_FILTER | quote }} + - name: OAUTH_OSM_KEY + value: {{ .Values.osmchaWeb.env.OAUTH_OSM_KEY | quote }} + - name: OAUTH_OSM_SECRET + value: {{ .Values.osmchaWeb.env.OAUTH_OSM_SECRET | quote }} + - name: DJANGO_ANON_USER_THROTTLE_RATE + value: {{ .Values.osmchaWeb.env.DJANGO_ANON_USER_THROTTLE_RATE | quote }} + - name: DJANGO_COMMON_USER_THROTTLE_RATE + value: {{ .Values.osmchaWeb.env.DJANGO_COMMON_USER_THROTTLE_RATE | quote }} + - name: DJANGO_NON_STAFF_USER_THROTTLE_RATE + value: {{ .Values.osmchaWeb.env.DJANGO_NON_STAFF_USER_THROTTLE_RATE | quote }} + - name: OAUTH_REDIRECT_URI + value: {{ .Values.osmchaWeb.env.OAUTH_REDIRECT_URI | quote }} + - name: OSMCHA_FRONTEND_VERSION + value: {{ .Values.osmchaWeb.env.OSMCHA_FRONTEND_VERSION | quote }} + - name: DJANGO_ENABLE_CHANGESET_COMMENTS + value: {{ .Values.osmchaWeb.env.DJANGO_ENABLE_CHANGESET_COMMENTS | quote }} + - name: DJANGO_OSM_COMMENTS_API_KEY + value: {{ .Values.osmchaWeb.env.DJANGO_OSM_COMMENTS_API_KEY | quote }} + - name: DJANGO_ROOT + value: {{ .Values.osmchaWeb.env.DJANGO_ROOT | quote }} + - name: DJANGO_SETTINGS_MODULE + value: {{ .Values.osmchaWeb.env.DJANGO_SETTINGS_MODULE | quote }} + - name: C_FORCE_ROOT + value: {{ .Values.osmchaWeb.env.C_FORCE_ROOT | quote }} + - name: REACT_APP_OSM_URL + value: {{ .Values.osmchaWeb.env.REACT_APP_OSM_URL | quote }} + - name: REACT_APP_OSM_API + value: {{ .Values.osmchaWeb.env.REACT_APP_OSM_API | quote }} + - name: REACT_APP_OVERPASS_BASE + value: {{ .Values.osmchaWeb.env.REACT_APP_OVERPASS_BASE | quote }} + - name: REACT_APP_NOMINATIM_URL + value: {{ .Values.osmchaWeb.env.REACT_APP_NOMINATIM_URL | quote }} + - name: OSMCHA_URL + value: {{ .Values.osmchaWeb.env.OSMCHA_URL | quote }} + {{- if .Values.osmchaWeb.nodeSelector.enabled }} + nodeSelector: + {{ .Values.osmchaWeb.nodeSelector.label_key }} : {{ .Values.osmchaWeb.nodeSelector.label_value }} + {{- end }} +{{- end }} + diff --git a/osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml b/osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml new file mode 100644 index 00000000..cf2df4bc --- /dev/null +++ b/osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.osmchaWeb.enabled (eq .Values.serviceType "ClusterIP") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ template "osm-seed.fullname" . }}-ingress-osmcha-web + annotations: + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod-issuer +spec: + tls: + - hosts: + - osmcha.{{ .Values.domain }} + secretName: {{ template "osm-seed.fullname" . }}-secret-osmcha + + rules: + - host: osmcha.{{ .Values.domain }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ template "osm-seed.fullname" . }}-osmcha-web + port: + number: 80 +{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/osmcha-web/osmcha-web-service.yaml b/osm-seed/templates/osmcha-web/osmcha-web-service.yaml new file mode 100644 index 00000000..612a35ab --- /dev/null +++ b/osm-seed/templates/osmcha-web/osmcha-web-service.yaml @@ -0,0 +1,43 @@ +{{- if .Values.osmchaWeb.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "osm-seed.fullname" . }}-osmcha-web + labels: + app: {{ template "osm-seed.name" . }} + component: osmcha-web-service + environment: {{ .Values.environment }} + release: {{ .Release.Name }} + annotations: + {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https + {{- end }} + {{- if eq .Values.serviceType "ClusterIP" }} + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod-issuer + {{- else }} + fake.annotation: fake + {{- end }} + {{- with .Values.osmchaWeb.serviceAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.serviceType }} + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} + - port: 443 + targetPort: http + protocol: TCP + name: https + {{- end }} + selector: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-web +{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/web/web-deployment.yaml b/osm-seed/templates/web/web-deployment.yaml index 7f07f511..f154c5cb 100644 --- a/osm-seed/templates/web/web-deployment.yaml +++ b/osm-seed/templates/web/web-deployment.yaml @@ -30,11 +30,23 @@ spec: containerPort: 80 protocol: TCP livenessProbe: + {{- if .Values.web.livenessProbeExec }} + exec: + command: + - /bin/bash + - -c + - ./liveness.sh + initialDelaySeconds: 600 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + {{- else }} httpGet: path: /api/0.6/capabilities port: 80 initialDelaySeconds: 600 # 10 min, because the compile process takes time. timeoutSeconds: 30 + {{- end }} {{- if .Values.web.resources.enabled }} resources: requests: @@ -107,6 +119,14 @@ spec: value: {{ .Values.web.env.NEW_RELIC_LICENSE_KEY | default ""}} - name: NEW_RELIC_APP_NAME value: {{ .Values.web.env.NEW_RELIC_APP_NAME | default ""}} + volumeMounts: + - mountPath: /dev/shm + name: shared-memory + volumes: + - emptyDir: + medium: Memory + sizeLimit: {{ .Values.web.sharedMemorySize }} + name: shared-memory {{- if .Values.web.nodeSelector.enabled }} nodeSelector: {{ .Values.web.nodeSelector.label_key }} : {{ .Values.web.nodeSelector.label_value }} diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index b9f45f6c..a1a534f0 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -75,7 +75,7 @@ adminEmail: admin@example.com # Variables for osm-seed database # ==================================================================================================== db: - enabled: true + enabled: false image: name: '' tag: '' @@ -110,12 +110,12 @@ db: cpu: '1' nodeSelector: enabled: false - + sharedMemorySize: 64Mi # ==================================================================================================== # Variables for osm-seed website # ==================================================================================================== web: - enabled: true + enabled: false image: name: '' tag: '' @@ -147,7 +147,8 @@ web: minReplicas: 1 maxReplicas: 3 cpuUtilization: 80 - + sharedMemorySize: 256Mi + livenessProbeExec: true # ==================================================================================================== # Variables for memcached. Memcached is used to store session cookies # ==================================================================================================== @@ -196,27 +197,62 @@ fullHistory: memory: "16Gi" cpu: "4" -# ==================================================================================================== -# Variables for osm-seed database backup and restore -# ==================================================================================================== + # ==================================================================================================== + # Variables for osm-seed database backup and restore + # ==================================================================================================== dbBackupRestore: - enabled: false image: - name: '' - tag: '' - schedule: '* * * * *' - env: - DB_ACTION: backup - resources: - enabled: false - requests: - memory: '300Mi' - cpu: '0.5' - limits: - memory: '400Mi' - cpu: '0.6' - nodeSelector: - enabled: false + name: 'developmentseed/osmseed-backup-restore' + tag: '0.1.0-0.dev.git.942.h994af82' + cronjobs: + - name: web-db + enabled: true + schedule: '* * * * *' + env: + # DB action: backup/restore + DB_ACTION: backup + # Naming backup files + SET_DATE_AT_NAME: true + BACKUP_CLOUD_FOLDER: database/web-db + BACKUP_CLOUD_FILE: osmseed-api-web-db + AWS_S3_BUCKET: secure-bucket + # Clean up backups options + CLEANUP_BACKUPS: true + RETENTION_DAYS: "30" + resources: + enabled: false + requests: + memory: '300Mi' + cpu: '0.5' + limits: + memory: '400Mi' + cpu: '0.6' + nodeSelector: + enabled: false + - name: osmcha-db + enabled: true + schedule: '* * * * *' + env: + # DB action: backup/restore + DB_ACTION: backup + # Naming backup files + SET_DATE_AT_NAME: true + BACKUP_CLOUD_FOLDER: database/osmcha-db + BACKUP_CLOUD_FILE: osmseed-osmcha-db + AWS_S3_BUCKET: secure-bucket + # Clean up backups options + CLEANUP_BACKUPS: true + RETENTION_DAYS: "30" + resources: + enabled: false + requests: + memory: '300Mi' + cpu: '0.5' + limits: + memory: '400Mi' + cpu: '0.6' + nodeSelector: + enabled: false # ==================================================================================================== # Variables for osm-seed for osmosis, this configuration os to get the planet dump files from apidb @@ -535,7 +571,7 @@ tmApi: # Variables for nominatim api # ==================================================================================================== nominatimApi: - enabled: true + enabled: false image: name: '' tag: '' @@ -716,3 +752,92 @@ changesetReplicationJob: cpu: '10' nodeSelector: enabled: false + + +# ==================================================================================================== +# Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day +# ==================================================================================================== +osmchaWeb: + enabled: false + image: + name: ghcr.io/willemarcel/osmcha-django + tag: b1f4e6afc90e08707cadc4d74580632ca3b93dd2 + env: + OSM_SERVER_URL: 'https://staging.openstreetmap.org' + DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" + DJANGO_DEBUG: "False" + DJANGO_SECRET_KEY: abc + DJANGO_SECURE_BROWSER_XSS_FILTER: "True" + DJANGO_SECURE_SSL_REDIRECT: "True" + DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" + DJANGO_SECURE_FRAME_DENY: "True" + DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True" + DJANGO_SESSION_COOKIE_HTTPONLY: "True" + DJANGO_SESSION_COOKIE_SECURE: "True" + DJANGO_DEFAULT_FROM_EMAIL: "webmaster@openstreetmap.org" + DJANGO_SERVER_EMAIL: "server@openstreetmap.org" + DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" + DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" + OAUTH_OSM_KEY: abc + OAUTH_OSM_SECRET: abc + DJANGO_ANON_USER_THROTTLE_RATE: "10/h" + DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" + DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" + OAUTH_REDIRECT_URI: https://osmcha.openstreetmap.org/oauth-landing.html + OSMCHA_FRONTEND_VERSION: "v0.86.0-production" + DJANGO_ENABLE_CHANGESET_COMMENTS: "True" + DJANGO_OSM_COMMENTS_API_KEY: "test" + DJANGO_ROOT: /app + DJANGO_SETTINGS_MODULE: config.settings.local + C_FORCE_ROOT: true + REACT_APP_OSM_URL: https://staging.openstreetmap.org + REACT_APP_OSM_API: https://staging.openstreetmap.org/api/0.6 + REACT_APP_OVERPASS_BASE: https://overpass-api-staging.openstreetmap.org/api/interpreter + REACT_APP_NOMINATIM_URL: https://nominatim-api-staging.openstreetmap.org + OSMCHA_URL: https://osmcha-staging.openstreetmap.org + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + nodeSelector: + enabled: false + +# ==================================================================================================== +# Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day +# ==================================================================================================== +osmchaDb: + enabled: false + image: + name: '' + tag: '' + env: + POSTGRES_DB: osmcha-db + POSTGRES_USER: postgres + POSTGRES_PASSWORD: '1234' + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + persistenceDisk: + enabled: false + accessMode: ReadWriteOnce + mountPath: /var/lib/postgresql/data + # Minikube + localVolumeHostPath: /mnt/db-data/osmcha-data + localVolumeSize: 10Gi + # AWS + AWS_ElasticBlockStore_volumeID: vol-1234bcd + AWS_ElasticBlockStore_size: 50Gi + # GCP + GCP_gcePersistentDisk_pdName: osmseed-osmcha-disk--v1 + GCP_gcePersistentDisk_size: 50Gi + nodeSelector: + enabled: false From 0ec63170b5b4d5f753d6cc4f4ee821ed795af2ad Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Fri, 5 Jan 2024 12:21:20 -0500 Subject: [PATCH 02/23] OSMCha containers and helm templates (#305) * docker images for osmcha * Add root user to install missing modules Update values.yaml * Refactoring osmcha Update osmcha-deployments Remove osmcha-api in chartpress * Update Ruby version Set ubuntu 20.04 Add missing libs Add libv8-dev Update ruby version Fix web continaer * Update ruby for taginfo Add port names for osmcha * Update osmcha deployment Update osmcha deployment Update services * Update osmcha values * Update osmcha wbe contianer and helm templates * Update osmcha template * Update config * Update osmcha config * Update osmcha service * Update values * Update overpass start script * Update entrypoint overpass * Update cmd for overpass * Update overpas container * Fix cli for overpass * Update overpass api * Update helm template for overpass * Enable livenessProbe for overpass --- .gitignore | 3 +- chartpress.yaml | 1 + compose/osmcha.yml | 50 +- images/osmcha-web/Dockerfile | 75 +-- images/osmcha-web/README.md | 14 - images/osmcha-web/django.conf | 31 -- images/osmcha-web/gunicorn.conf | 6 - images/osmcha-web/start.sh | 21 +- images/osmcha-web/supervisord.conf | 5 - images/overpass-api/Dockerfile | 2 +- images/overpass-api/docker-entrypoint.sh | 201 ++++---- images/taginfo/Dockerfile | 2 +- images/web/Dockerfile | 60 ++- osm-seed/templates/osmcha-app/configMap.yaml | 126 +++++ osm-seed/templates/osmcha-app/cronJob.yaml | 36 ++ osm-seed/templates/osmcha-app/deployment.yaml | 124 +++++ .../ingress.yaml} | 8 +- .../service.yaml} | 22 +- .../osmcha-web/osmcha-web-deployment.yaml | 127 ----- .../overpass-api-statefulset.yaml | 13 +- osm-seed/values.yaml | 443 +++++++++--------- 21 files changed, 743 insertions(+), 627 deletions(-) delete mode 100644 images/osmcha-web/README.md delete mode 100644 images/osmcha-web/django.conf delete mode 100644 images/osmcha-web/gunicorn.conf delete mode 100644 images/osmcha-web/supervisord.conf create mode 100644 osm-seed/templates/osmcha-app/configMap.yaml create mode 100644 osm-seed/templates/osmcha-app/cronJob.yaml create mode 100644 osm-seed/templates/osmcha-app/deployment.yaml rename osm-seed/templates/{osmcha-web/osmcha-web-ingress.yaml => osmcha-app/ingress.yaml} (71%) rename osm-seed/templates/{osmcha-web/osmcha-web-service.yaml => osmcha-app/service.yaml} (76%) delete mode 100644 osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml diff --git a/.gitignore b/.gitignore index c2eaf90e..4c32d0b3 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,5 @@ tiler-server/imposm/ overpass-api-db/ data/*/ !data/README.md -images/changeset-replication-job/config.yaml \ No newline at end of file +images/changeset-replication-job/config.yaml +values_copy.yaml \ No newline at end of file diff --git a/chartpress.yaml b/chartpress.yaml index df170850..f5af3ddc 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -46,3 +46,4 @@ charts: valuesPath: osmchaWeb.image osmcha-db: valuesPath: osmchaDb.image + \ No newline at end of file diff --git a/compose/osmcha.yml b/compose/osmcha.yml index 5a1ae4e6..0761809d 100644 --- a/compose/osmcha.yml +++ b/compose/osmcha.yml @@ -1,24 +1,60 @@ -version: '3' +version: "3.8" services: - osmcha-db: + web: + platform: linux/amd64 + image: developmentseed/osmseed-osmcha-web:v16 + build: + context: ../images/osmcha-web + dockerfile: Dockerfile + env_file: + - ../envs/.env.osmcha + volumes: + - ../data/osmcha/staticfiles:/staticfiles + db: platform: linux/amd64 image: osmseed-osmcha-db:v1 build: context: ../images/osmcha-db dockerfile: Dockerfile ports: - - '5432:5432' + - "5432:5432" volumes: - ../data/osmcha-db-data:/var/lib/postgresql/data env_file: - ../envs/.env.osmcha - osmcha-web: + init: platform: linux/amd64 - image: osmseed-osmcha-web:v1 + image: ghcr.io/willemarcel/osmcha-django:b1f4e6afc90e08707cadc4d74580632ca3b93dd2 + command: > + /bin/bash -c " + set -x + python manage.py collectstatic + python manage.py migrate + mkdir -p /staticfiles/static + cp -r /app/staticfiles/* /staticfiles/static/ + " + env_file: + - ../envs/.env.osmcha + volumes: + - ../data/osmcha/staticfiles:/staticfiles + api: + platform: linux/amd64 + image: ghcr.io/willemarcel/osmcha-django:b1f4e6afc90e08707cadc4d74580632ca3b93dd2 build: - context: ../images/osmcha-web + context: ../images/osmcha-api dockerfile: Dockerfile + command: > + /bin/bash -c " + set -x + python manage.py collectstatic + python manage.py migrate + mkdir -p /staticfiles/static + cp -r /app/staticfiles/* /staticfiles/static/ + gunicorn --workers 4 --bind 0.0.0.0:5000 --log-file - --access-logfile - config.wsgi + " ports: - - '8000:8000' + - "5000:5000" env_file: - ../envs/.env.osmcha + volumes: + - ../data/osmcha/staticfiles:/staticfiles diff --git a/images/osmcha-web/Dockerfile b/images/osmcha-web/Dockerfile index ae3c6898..9b5f4181 100644 --- a/images/osmcha-web/Dockerfile +++ b/images/osmcha-web/Dockerfile @@ -1,66 +1,11 @@ -FROM ubuntu:20.04 -ARG DEBIAN_FRONTEND=noninteractive - -RUN apt-get update -y -qq -RUN apt-get install -y curl python3 python3-dev \ - python3-pip git libgeos-dev libyaml-dev \ - libcurl4-gnutls-dev librtmp-dev python3-gdal \ - locales nginx supervisor yarnpkg nodejs npm -RUN locale-gen en_US.UTF-8 -RUN apt-get clean -RUN rm -rf /var/lib/apt/lists/* - -# Install yarn. -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -RUN apt-get update -y -RUN apt-get remove -y yarnpkg cmdtest && apt-get install -y yarn && apt-get clean && rm -rf /var/lib/apt/lists/* - -# Setup nginx -RUN rm /etc/nginx/sites-enabled/default -COPY django.conf /etc/nginx/sites-available/ -RUN ln -s /etc/nginx/sites-available/django.conf /etc/nginx/sites-enabled/django.conf -RUN echo "daemon off;" >> /etc/nginx/nginx.conf - - -# Setting locale is required by unicode-slugify (osmcha-django python requirement) -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en - +FROM node:16-slim as builder +ENV DEBIAN_FRONTEND noninteractive +ENV BUILD_ENV=prod +ENV REACT_APP_PRODUCTION_API_URL=/api/v1 +RUN apt-get update && apt-get install -y git +RUN mkdir /app WORKDIR /app -ENV BACKEND_VERSION v4.19.0 -RUN git clone https://github.com/willemarcel/osmcha-django . -RUN git checkout $BACKEND_VERSION -RUN git pull origin $BACKEND_VERSION -RUN pip3 install -r requirements/production.txt -RUN pip3 install -r requirements/local.txt - -ADD . /app -ENV DJANGO_ROOT /app -ENV DJANGO_SETTINGS_MODULE config.settings.aws_production - -# Replace social-auth-core by a modified version that provides OHM authentication -RUN pip3 install git+https://github.com/OpenHistoricalMap/social-core.git@master - -# Upgrading requests to the latest version. -RUN pip3 install -U requests -RUN pip3 install certifi -RUN pip3 install django-extensions - -# Clone and build the frontend -RUN git clone https://github.com/mapbox/osmcha-frontend.git /osmcha-frontend -RUN cd /osmcha-frontend && yarn install - -RUN echo "alias python=python3" >> ~/.bashrc -RUN echo "alias pip=pip3" >> ~/.bashrc - -EXPOSE 80 - -# Supervisor config -RUN mkdir -p /var/log/supervisor -COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf -COPY gunicorn.conf /etc/supervisor/conf.d/gunicorn.conf - -COPY ./start.sh . -CMD ./start.sh +RUN git clone https://github.com/osmus/osmcha-frontend.git /app +RUN yarn install +COPY start.sh . +CMD [ "/app/start.sh" ] diff --git a/images/osmcha-web/README.md b/images/osmcha-web/README.md deleted file mode 100644 index c653ed65..00000000 --- a/images/osmcha-web/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# OpenStreetMap Changeset Analyzer (OSMCha) - -OSMCha in a Docker containe that integrated with the OSM-Seed Ecosystem - -## Build -```sh -docker compose -f /compose/osmcha.yml build -``` - -## Run - -```sh -docker compose -f /compose/osmcha.yml up -``` \ No newline at end of file diff --git a/images/osmcha-web/django.conf b/images/osmcha-web/django.conf deleted file mode 100644 index ff788d99..00000000 --- a/images/osmcha-web/django.conf +++ /dev/null @@ -1,31 +0,0 @@ -server { - listen 80; - charset utf-8; - client_max_body_size 20M; - - location /static/ { - gzip_static on; - alias /app/staticfiles/; - } - - location /index.html { - gzip_static on; - alias /app/osmchadjango/frontend/templates/frontend/index.html; - } - - location / { - # checks for static file, if not found proxy to app - try_files $uri @proxy_to_app; - } - - # cookiecutter-django app - location @proxy_to_app { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_pass http://127.0.0.1:5000; - fastcgi_connect_timeout 130; - fastcgi_send_timeout 130; - fastcgi_read_timeout 130; - } -} diff --git a/images/osmcha-web/gunicorn.conf b/images/osmcha-web/gunicorn.conf deleted file mode 100644 index 21d99bb7..00000000 --- a/images/osmcha-web/gunicorn.conf +++ /dev/null @@ -1,6 +0,0 @@ -[program:gunicorn] -command=/usr/local/bin/gunicorn config.wsgi -b localhost:5000 -w 4 --log-file /var/log/gunicorn.error --access-logfile /var/log/gunicorn.access --capture-output --timeout 120 -directory=/app -autostart=true -autorestart=true -redirect_stderr=true diff --git a/images/osmcha-web/start.sh b/images/osmcha-web/start.sh index b37767e0..661b6d28 100755 --- a/images/osmcha-web/start.sh +++ b/images/osmcha-web/start.sh @@ -1,15 +1,8 @@ #!/usr/bin/env bash -set -ex - -# Build frontend -cd /osmcha-frontend -REACT_APP_VERSION=ohm REACT_APP_STACK=PRODUCTION PUBLIC_URL=$OSMCHA_URL npx react-scripts build -cp -R build/*.html /app/osmchadjango/frontend/templates/frontend/ -cp -R build/* /app/osmchadjango/static/ -cp -R build/static/* /app/osmchadjango/static/ - -# Start service -cd /app -python3 manage.py collectstatic --noinput -python3 manage.py migrate -supervisord -c /etc/supervisor/supervisord.conf +set -x +export BUILD_ENV=prod +export REACT_APP_PRODUCTION_API_URL=/api/v1 +sed -i "s|https://osmcha.org|$OSMCHA_URL|g" package.json +yarn build:${BUILD_ENV} +find /app/build -type f -exec sed -i "s/www.openstreetmap.org/$OSMCHA_API_URL/g" {} + +cp -r /app/build/* /assets/ diff --git a/images/osmcha-web/supervisord.conf b/images/osmcha-web/supervisord.conf deleted file mode 100644 index 8d520a8c..00000000 --- a/images/osmcha-web/supervisord.conf +++ /dev/null @@ -1,5 +0,0 @@ -[supervisord] -nodaemon=true - -[program:nginx] -command=/usr/sbin/nginx diff --git a/images/overpass-api/Dockerfile b/images/overpass-api/Dockerfile index 9528668b..de488a70 100644 --- a/images/overpass-api/Dockerfile +++ b/images/overpass-api/Dockerfile @@ -1,4 +1,4 @@ -FROM wiktorn/overpass-api:0.7.56.8 +FROM wiktorn/overpass-api:latest COPY docker-entrypoint.sh /app/ RUN chmod a+rx /app/docker-entrypoint.sh CMD ["/app/docker-entrypoint.sh"] diff --git a/images/overpass-api/docker-entrypoint.sh b/images/overpass-api/docker-entrypoint.sh index 243ddac6..60b94687 100644 --- a/images/overpass-api/docker-entrypoint.sh +++ b/images/overpass-api/docker-entrypoint.sh @@ -1,119 +1,124 @@ #!/bin/bash -set -eo pipefail +set -eox pipefail shopt -s nullglob OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} OVERPASS_FLUSH_SIZE=${OVERPASS_FLUSH_SIZE:-16} -OVERPASS_CLONE_SOURCE=${OVERPASS_CLONE_SOURCE:-http://dev.overpass-api.de/api_drolbr/} +OVERPASS_CLONE_SOURCE=${OVERPASS_CLONE_SOURCE:-https://dev.overpass-api.de/api_drolbr/} # this is used by other processes, so needs to be exported export OVERPASS_MAX_TIMEOUT=${OVERPASS_MAX_TIMEOUT:-1000s} -if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" -elif [[ "${OVERPASS_META}" == "yes" ]] ; then - META="--meta" +if [[ "$OVERPASS_META" == "attic" ]]; then + META="--keep-attic" +elif [[ "${OVERPASS_META}" == "yes" ]]; then + META="--meta" else - META="" + META="" fi for f in /docker-entrypoint-initdb.d/*; do - case "$f" in - *.sh) - if [[ -x "$f" ]]; then - echo "$0: running $f" - "$f" - else - echo "$0: sourcing $f" - . "$f" - fi - ;; - *) echo "$0: ignoring $f" ;; - esac - echo + case "$f" in + *.sh) + if [[ -x "$f" ]]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + # shellcheck disable=SC1090 # ignore SC1090 (unable to follow file) because they are dynamically provided + . "$f" + fi + ;; + *) echo "$0: ignoring $f" ;; + esac + echo done -function startAPIServer(){ - envsubst '${OVERPASS_MAX_TIMEOUT}' < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf - echo "Starting supervisord process" - exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf -} +if [[ ! -f /db/init_done ]]; then + echo "No database directory. Initializing" + if [[ "${USE_OAUTH_COOKIE_CLIENT}" = "yes" ]]; then + /app/venv/bin/python /app/bin/oauth_cookie_client.py -o /db/cookie.jar -s /secrets/oauth-settings.json --format netscape + # necessary to add newline at the end as oauth_cookie_client doesn't do that + echo >>/db/cookie.jar + else + echo "# Netscape HTTP Cookie File" >/db/cookie.jar + echo "${OVERPASS_COOKIE_JAR_CONTENTS}" >>/db/cookie.jar + fi + chown overpass /db/cookie.jar -if [[ ! -f /db/init_done ]] ; then - echo "No database directory. Initializing" - if [[ "${USE_OAUTH_COOKIE_CLIENT}" = "yes" ]]; then - /app/venv/bin/python /app/bin/oauth_cookie_client.py -o /db/cookie.jar -s /secrets/oauth-settings.json --format netscape - # necessary to add newline at the end as oauth_cookie_client doesn't do that - echo >> /db/cookie.jar - else - echo "# Netscape HTTP Cookie File" > /db/cookie.jar - echo "${OVERPASS_COOKIE_JAR_CONTENTS}" >> /db/cookie.jar - fi - chown overpass /db/cookie.jar + if [[ "$OVERPASS_MODE" = "clone" ]]; then + ( + mkdir -p /db/db && + /app/bin/download_clone.sh --db-dir=/db/db --source="${OVERPASS_CLONE_SOURCE}" --meta="${OVERPASS_META}" && + cp /db/db/replicate_id /db/replicate_id && + cp -r /app/etc/rules /db/db && + chown -R overpass:overpass /db/* && + touch /db/init_done + ) || ( + echo "Failed to clone overpass repository" + exit 1 + ) + if [[ "${OVERPASS_STOP_AFTER_INIT}" == "false" ]]; then + echo "Overpass container ready to receive requests" + else + echo "Overpass container initialization complete. Exiting." + exit 0 + fi + fi - if [[ "$OVERPASS_MODE" = "clone" ]]; then - mkdir -p /db/db \ - && /app/bin/download_clone.sh --db-dir=/db/db --source="${OVERPASS_CLONE_SOURCE}" --meta="${OVERPASS_META}" \ - && cp /db/db/replicate_id /db/replicate_id \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && touch /db/init_done \ - && echo "Overpass ready, you can start your container with docker start" - exit - fi - - if [[ "$OVERPASS_MODE" = "init" ]]; then - while true ; do - CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") - case "${CURL_STATUS_CODE}" in - 429) - echo "Server responded with 429 Too many requests. Trying again in 5 minutes..." - sleep 300 - continue - ;; - # for `file:///` scheme curl returns `000` HTTP status code - 200 | 000) - ( - if [[ ! -z "${OVERPASS_PLANET_PREPROCESS+x}" ]]; then - echo "Running preprocessing command: ${OVERPASS_PLANET_PREPROCESS}" - eval "${OVERPASS_PLANET_PREPROCESS}" - fi \ - && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "${META}" "--version=$(osmium fileinfo -e -g data.timestamp.last /db/planet.osm.bz2) --compression-method=${OVERPASS_COMPRESSION} --map-compression-method=${OVERPASS_COMPRESSION} --flush-size=${OVERPASS_FLUSH_SIZE}" \ - && echo "Database created. Now updating it." \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && echo "Updating" \ - && /app/bin/update_overpass.sh "-O /db/planet.osm.bz2" \ - && /app/bin/osm3s_query --progress --rules --db-dir=/db/db < /db/db/rules/areas.osm3s \ - && touch /db/init_done \ - && rm /db/planet.osm.bz2 \ - && chown -R overpass:overpass /db \ - && echo $OVERPASS_REPLICATION_SEQUENCE_NUMBER > /db/replicate_id \ - && chmod 777 /db/replicate_id \ - && echo "Overpass ready, you can start your container with docker start" \ - && startAPIServer - ) || ( - echo "Failed to process planet file" - exit - ) - ;; - 403) - echo "Access denied when downloading planet file. Check your OVERPASS_PLANET_URL and OVERPASS_COOKIE_JAR_CONTENTS or USE_OAUTH_COOKIE_CLIENT" - cat /db/cookie.jar - exit - ;; - *) - echo "Failed to download planet file. HTTP status code: ${CURL_STATUS_CODE}" - cat /db/planet.osm.bz2 - exit - ;; - esac - exit - done - fi + if [[ "$OVERPASS_MODE" = "init" ]]; then + CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") + # try again until it's allowed + while [ "$CURL_STATUS_CODE" = "429" ]; do + echo "Server responded with 429 Too many requests. Trying again in 5 minutes..." + sleep 300 + CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") + done + # for `file:///` scheme curl returns `000` HTTP status code + if [[ $CURL_STATUS_CODE = "200" || $CURL_STATUS_CODE = "000" ]]; then + ( + if [[ -n "${OVERPASS_PLANET_PREPROCESS+x}" ]]; then + echo "Running preprocessing command: ${OVERPASS_PLANET_PREPROCESS}" + eval "${OVERPASS_PLANET_PREPROCESS}" + fi && + /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "${META}" "--version=$(osmium fileinfo -e -g data.timestamp.last /db/planet.osm.bz2) --compression-method=${OVERPASS_COMPRESSION} --map-compression-method=${OVERPASS_COMPRESSION} --flush-size=${OVERPASS_FLUSH_SIZE}" && + echo "Database created. Now updating it." && + cp -r /app/etc/rules /db/db && + chown -R overpass:overpass /db/* && + echo "Updating" && + /app/bin/update_overpass.sh -O /db/planet.osm.bz2 && + if [[ "${OVERPASS_USE_AREAS}" = "true" ]]; then + echo "Generating areas..." && /app/bin/osm3s_query --progress --rules --db-dir=/db/db /etc/nginx/nginx.conf + +echo "Starting supervisord process" +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf \ No newline at end of file diff --git a/images/taginfo/Dockerfile b/images/taginfo/Dockerfile index 1721e262..16745806 100644 --- a/images/taginfo/Dockerfile +++ b/images/taginfo/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:2.7 +FROM ruby:3.0 ENV workdir /apps # Install Taginfo site diff --git a/images/web/Dockerfile b/images/web/Dockerfile index bf8b4fed..0a1e5fb8 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -1,16 +1,52 @@ FROM ubuntu:20.04 + ENV DEBIAN_FRONTEND=noninteractive -ENV workdir /var/www +ENV RUBY_MAJOR 3.0 +ARG RUBY_VERSION=3.0.6 +ENV RUBY_VERSION $RUBY_VERSION +ENV PATH /opt/ruby/bin:$PATH:/opt/rbenv/plugins/ruby-build/bin + +# ruby-build +RUN set -ex \ + && mkdir -p /etc/network/interfaces.d \ + && BaseDeps=' \ + git \ + gcc \ + autoconf \ + bison \ + build-essential \ + libssl-dev \ + libyaml-dev \ + libreadline6-dev \ + zlib1g-dev \ + libncurses5-dev \ + libffi-dev \ + libgdbm6 \ + libgdbm-dev \ + make \ + wget \ + curl \ + iproute2 \ + net-tools \ + tzdata \ + locales \ + ca-certificates' \ + && apt-get update \ + && DEBCONF_NOWARNINGS=yes apt-get -y upgrade \ + && DEBCONF_NOWARNINGS=yes DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $BaseDeps \ + && rm -rf /var/lib/apt/lists/* \ + && git clone https://github.com/sstephenson/ruby-build.git /opt/rbenv/plugins/ruby-build \ + && ruby-build ${RUBY_VERSION} /opt/ruby -# Production OSM setup +ENV workdir /var/www ENV RAILS_ENV=production # Install the openstreetmap-website dependencies RUN apt-get update \ && apt-get install -y \ - ruby2.7 libruby2.7 ruby2.7-dev libmagickwand-dev libxml2-dev libxslt1-dev \ - nodejs npm apache2 apache2-dev build-essential git-core firefox-geckodriver postgresql-client \ - libpq-dev libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev yarnpkg curl \ + libmagickwand-dev libxml2-dev libxslt1-dev \ + nodejs npm libv8-dev apache2 apache2-dev build-essential git-core postgresql-client \ + libpq-dev libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev yarnpkg curl unzip \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -23,7 +59,8 @@ RUN apt-get update \ libcrypto++-dev libargon2-dev libyajl-dev automake autoconf libtool \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - + + # Install cgimap ENV cgimap /tmp/openstreetmap-cgimap RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap @@ -40,7 +77,7 @@ RUN npm install -g svgo # Install openstreetmap-website RUN rm -rf $workdir/html -# GITSHA value at 15-02-2022 + ENV OPENSTREETMAP_WEBSITE_GITSHA=c24b5481812aba9e83da1fd855ccb37f92c5d75e RUN curl -L https://github.com/openstreetmap/openstreetmap-website/archive/$OPENSTREETMAP_WEBSITE_GITSHA.zip --output website.zip && unzip website.zip RUN mv openstreetmap-website-$OPENSTREETMAP_WEBSITE_GITSHA/* $workdir/ @@ -50,6 +87,7 @@ RUN echo "gem 'image_optim_pack', :git => 'https://github.com/toy/image_optim_pa # Install Ruby packages RUN gem install bundler && bundle install + # Configure database.yml and secrets.yml RUN cp $workdir/config/example.database.yml $workdir/config/database.yml RUN touch $workdir/config/settings.local.yml @@ -78,9 +116,9 @@ RUN apt-get update && apt-get install -y libapache2-mod-passenger RUN echo "ServerName $(cat /etc/hostname)" >> /etc/apache2/apache2.conf RUN a2enmod passenger -# Check installation -RUN /usr/bin/passenger-config validate-install -RUN /usr/sbin/passenger-memory-stats +# # Check installation +# RUN /usr/bin/passenger-config validate-install +# RUN /usr/sbin/passenger-memory-stats # Enable required apache modules for the cgimap Apache service RUN a2enmod proxy proxy_http rewrite @@ -104,3 +142,5 @@ COPY start.sh $workdir/ COPY liveness.sh $workdir/ CMD $workdir/start.sh + + diff --git a/osm-seed/templates/osmcha-app/configMap.yaml b/osm-seed/templates/osmcha-app/configMap.yaml new file mode 100644 index 00000000..a3aa6ba3 --- /dev/null +++ b/osm-seed/templates/osmcha-app/configMap.yaml @@ -0,0 +1,126 @@ +{{- if .Values.osmchaApi.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-osmcha-common-env +data: + # DB + POSTGRES_HOST: {{ .Release.Name }}-osmcha-db + PGHOST: {{ .Release.Name }}-osmcha-db + POSTGRES_PORT: "5432" + POSTGRES_USER: {{ .Values.osmchaDb.env.POSTGRES_USER | quote }} + POSTGRES_PASSWORD: {{ .Values.osmchaDb.env.POSTGRES_PASSWORD | quote }} + POSTGRES_DATABASE: {{ .Values.osmchaDb.env.POSTGRES_DB | quote }} + POSTGRES_DB: {{ .Values.osmchaDb.env.POSTGRES_DB | quote }} + # API + DJANGO_SECRET_KEY: {{ .Values.osmchaApi.env.DJANGO_SECRET_KEY | quote }} + DJANGO_SETTINGS_MODULE: {{ .Values.osmchaApi.env.DJANGO_SETTINGS_MODULE | quote }} + OSMCHA_FRONTEND_VERSION: {{ .Values.osmchaApi.env.OSMCHA_FRONTEND_VERSION | quote }} + OAUTH_OSM_KEY: {{ .Values.osmchaApi.env.OAUTH_OSM_KEY | quote }} + OAUTH_OSM_SECRET: {{ .Values.osmchaApi.env.OAUTH_OSM_SECRET | quote }} + OAUTH_REDIRECT_URI: {{ .Values.osmchaApi.env.OAUTH_REDIRECT_URI | quote }} + DJANGO_SECURE_SSL_REDIRECT: "False" + DJANGO_DEBUG: "False" + DJANGO_ENV: "production" + OSM_SERVER_URL: {{ .Values.osmchaApi.env.OSM_SERVER_URL | quote }} + DJANGO_DEBUG: "False" + DJANGO_ENV: "production" + REDIS_URL: "redis://localhost:6379" + OSM_PLANET_BASE_URL: {{ .Values.osmchaApi.env.OSM_PLANET_BASE_URL | quote }} + # frontend + OSMCHA_URL: {{ .Values.osmchaApi.env.OSMCHA_URL | quote }} + OSMCHA_API_URL: {{ .Values.osmchaApi.env.OSMCHA_API_URL | quote }} + REACT_APP_OSM_URL: {{ .Values.osmchaApi.env.REACT_APP_OSM_URL | quote }} + REACT_APP_OSM_API: {{ .Values.osmchaApi.env.REACT_APP_OSM_API | quote }} + REACT_APP_OVERPASS_BASE: {{ .Values.osmchaApi.env.REACT_APP_OVERPASS_BASE | quote }} + REACT_APP_DISABLE_REAL_CHANGESETS: {{ .Values.osmchaApi.env.REACT_APP_DISABLE_REAL_CHANGESETS | quote }} + REACT_APP_MAPBOX_ACCESS_TOKEN: {{ .Values.osmchaApi.env.REACT_APP_MAPBOX_ACCESS_TOKEN | quote }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-osmcha-nginx-config +data: + nginx.conf: | + worker_processes 4; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + upstream app { + server localhost:5000; + } + + server { + listen 80; + charset utf-8; + + location ~ ^/static.* { + root /staticfiles; + try_files $uri @proxy_to_frontend; + } + + location @proxy_to_frontend { + root /assets; + try_files $uri $uri/ =404; + } + + location /api { + try_files $uri $uri/ @django; + } + + location /admin { + try_files $uri $uri/ @django; + } + + location @django { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://app; + } + + location / { + root /assets; + # checks for static file, if not found proxy to app + try_files $uri $uri/ /index.html; + } + } + } + mime.types: | + types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/x-icon ico; + image/x-jng jng; + application/font-woff woff; + application/json json; + application/zip zip; + } + +{{- end }} diff --git a/osm-seed/templates/osmcha-app/cronJob.yaml b/osm-seed/templates/osmcha-app/cronJob.yaml new file mode 100644 index 00000000..036d8df5 --- /dev/null +++ b/osm-seed/templates/osmcha-app/cronJob.yaml @@ -0,0 +1,36 @@ +{{- if .Values.osmchaApi.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-osmcha-fetch-changesets-cronjob + labels: + app: {{ template "osm-seed.name" . }} + component: osmcha-app-deployment + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + schedule: "*/2 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 2 + jobTemplate: + spec: + template: + spec: + containers: + - name: {{ .Release.Name }}-osmcha-fetch-changesets + image: "{{ .Values.osmchaApi.image.name }}:{{ .Values.osmchaApi.image.tag }}" + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - python manage.py fetchchangesets + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + restartPolicy: Never + {{- if .Values.osmchaApi.nodeSelector.enabled }} + nodeSelector: + {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }} + {{- end }} +{{- end }} diff --git a/osm-seed/templates/osmcha-app/deployment.yaml b/osm-seed/templates/osmcha-app/deployment.yaml new file mode 100644 index 00000000..43e2b981 --- /dev/null +++ b/osm-seed/templates/osmcha-app/deployment.yaml @@ -0,0 +1,124 @@ +{{- if .Values.osmchaApi.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-osmcha-app + labels: + app: {{ template "osm-seed.name" . }} + component: osmcha-app-deployment + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.osmchaApi.replicaCount }} + selector: + matchLabels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-app + template: + metadata: + labels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-app + spec: + initContainers: + - name: web-builder + image: "{{ .Values.osmchaWeb.image.name }}:{{ .Values.osmchaWeb.image.tag }}" + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + volumeMounts: + - name: nginx-assets + mountPath: /assets + - name: init + image: "{{ .Values.osmchaApi.image.name }}:{{ .Values.osmchaApi.image.tag }}" + command: + - /bin/sh + - -c + - | + set -x + flag=true + while "$flag" = true; do + pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue + flag=false + python manage.py migrate + python manage.py collectstatic + mkdir -p /staticfiles/static + cp -r /app/staticfiles/* /staticfiles/static/ + done + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + volumeMounts: + - name: staticfiles + mountPath: /staticfiles + # securityContext: + # runAsUser: 0 + containers: + - name: api + # securityContext: + # runAsUser: 0 + image: "{{ .Values.osmchaApi.image.name }}:{{ .Values.osmchaApi.image.tag }}" + command: + - /bin/sh + - -c + - | + set -x + echo "start app...." + gunicorn --workers 4 --bind 0.0.0.0:5000 --log-file - --access-logfile - config.wsgi + ports: + - containerPort: 5000 + livenessProbe: + httpGet: + path: / + port: 5000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + volumeMounts: + - name: staticfiles + mountPath: /staticfiles + {{- if .Values.osmchaApi.resources.enabled }} + resources: + requests: + memory: {{ .Values.osmchaApi.resources.requests.memory }} + cpu: {{ .Values.osmchaApi.resources.requests.cpu }} + limits: + memory: {{ .Values.osmchaApi.resources.limits.memory }} + cpu: {{ .Values.osmchaApi.resources.limits.cpu }} + {{- end }} + + - name: nginx + image: "nginx:latest" + ports: + - containerPort: 80 + volumeMounts: + - name: osmcha-frontend-nginx-config + mountPath: /etc/nginx + readOnly: true + - name: staticfiles + mountPath: /staticfiles + - name: nginx-assets + mountPath: /assets + - name: redis + image: "redis:latest" + ports: + - containerPort: 6379 + volumes: + - name: staticfiles + emptyDir: {} + - name: nginx-assets + emptyDir: {} + - name: osmcha-frontend-nginx-config + configMap: + name: {{ .Release.Name }}-osmcha-nginx-config + defaultMode: 0777 + {{- if .Values.osmchaApi.nodeSelector.enabled }} + nodeSelector: + {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }} + {{- end }} +{{- end }} + diff --git a/osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml b/osm-seed/templates/osmcha-app/ingress.yaml similarity index 71% rename from osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml rename to osm-seed/templates/osmcha-app/ingress.yaml index cf2df4bc..7360e2d1 100644 --- a/osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml +++ b/osm-seed/templates/osmcha-app/ingress.yaml @@ -1,8 +1,8 @@ -{{- if and .Values.osmchaWeb.enabled (eq .Values.serviceType "ClusterIP") }} +{{- if and .Values.osmchaApi.enabled (eq .Values.serviceType "ClusterIP") }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ template "osm-seed.fullname" . }}-ingress-osmcha-web + name: {{ template "osm-seed.fullname" . }}-ingress-osmcha-app annotations: kubernetes.io/ingress.class: nginx cert-manager.io/cluster-issuer: letsencrypt-prod-issuer @@ -20,7 +20,7 @@ spec: pathType: Prefix backend: service: - name: {{ template "osm-seed.fullname" . }}-osmcha-web + name: {{ template "osm-seed.fullname" . }}-osmcha-app port: number: 80 -{{- end }} \ No newline at end of file +{{- end }} diff --git a/osm-seed/templates/osmcha-web/osmcha-web-service.yaml b/osm-seed/templates/osmcha-app/service.yaml similarity index 76% rename from osm-seed/templates/osmcha-web/osmcha-web-service.yaml rename to osm-seed/templates/osmcha-app/service.yaml index 612a35ab..985cf171 100644 --- a/osm-seed/templates/osmcha-web/osmcha-web-service.yaml +++ b/osm-seed/templates/osmcha-app/service.yaml @@ -1,18 +1,18 @@ -{{- if .Values.osmchaWeb.enabled -}} +{{- if .Values.osmchaApi.enabled -}} apiVersion: v1 kind: Service metadata: - name: {{ template "osm-seed.fullname" . }}-osmcha-web + name: {{ template "osm-seed.fullname" . }}-osmcha-app labels: app: {{ template "osm-seed.name" . }} - component: osmcha-web-service + component: osmcha-app-service environment: {{ .Values.environment }} release: {{ .Release.Name }} annotations: {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }} service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https {{- end }} {{- if eq .Values.serviceType "ClusterIP" }} kubernetes.io/ingress.class: nginx @@ -20,24 +20,24 @@ metadata: {{- else }} fake.annotation: fake {{- end }} - {{- with .Values.osmchaWeb.serviceAnnotations }} + {{- with .Values.osmchaApi.serviceAnnotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: type: {{ .Values.serviceType }} ports: - - port: 80 - targetPort: http - protocol: TCP + - protocol: TCP + port: 80 + targetPort: 80 name: http {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} - port: 443 - targetPort: http + targetPort: 80 protocol: TCP name: https {{- end }} selector: app: {{ template "osm-seed.name" . }} release: {{ .Release.Name }} - run: {{ .Release.Name }}-osmcha-web -{{- end }} \ No newline at end of file + run: {{ .Release.Name }}-osmcha-app +{{- end }} diff --git a/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml b/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml deleted file mode 100644 index 81a86f9c..00000000 --- a/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml +++ /dev/null @@ -1,127 +0,0 @@ -{{- if .Values.osmchaWeb.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Release.Name }}-osmcha-web - labels: - app: {{ template "osm-seed.name" . }} - component: osmcha-web-deployment - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: - replicas: {{ .Values.osmchaWeb.replicaCount }} - selector: - matchLabels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-osmcha-web - template: - metadata: - labels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-osmcha-web - spec: - containers: - - name: {{ .Chart.Name }}-osmcha-web - image: "{{ .Values.osmchaWeb.image.name }}:{{ .Values.osmchaWeb.image.tag }}" - ports: - - name: http - containerPort: 5000 - protocol: TCP - livenessProbe: - httpGet: - path: / - port: 5000 - initialDelaySeconds: 600 # 10 min, because the compile process takes time. - timeoutSeconds: 30 - {{- if .Values.osmchaWeb.resources.enabled }} - resources: - requests: - memory: {{ .Values.osmchaWeb.resources.requests.memory }} - cpu: {{ .Values.osmchaWeb.resources.requests.cpu }} - limits: - memory: {{ .Values.osmchaWeb.resources.limits.memory }} - cpu: {{ .Values.osmchaWeb.resources.limits.cpu }} - {{- end }} - env: - - name: OSM_SERVER_URL - value: {{ .Values.osmchaDb.env.OSM_SERVER_URL | quote }} - - name: POSTGRES_USER - value: {{ .Values.osmchaDb.env.POSTGRES_USER | quote }} - - name: POSTGRES_PASSWORD - value: {{ .Values.osmchaDb.env.POSTGRES_PASSWORD | quote }} - - name: POSTGRES_DB - value: {{ .Values.osmchaDb.env.POSTGRES_DB | quote }} - - name: PGHOST - value: {{ .Release.Name }}-osmcha-db - - name: POSTGRES_HOST - value: {{ .Release.Name }}-osmcha-db - - name: DJANGO_CACHES - value: {{ .Values.osmchaWeb.env.DJANGO_CACHES | quote }} - - name: DJANGO_DEBUG - value: {{ .Values.osmchaWeb.env.DJANGO_DEBUG | quote }} - - name: DJANGO_SECRET_KEY - value: {{ .Values.osmchaWeb.env.DJANGO_SECRET_KEY | quote }} - - name: DJANGO_SECURE_BROWSER_XSS_FILTER - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_BROWSER_XSS_FILTER | quote }} - - name: DJANGO_SECURE_SSL_REDIRECT - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_SSL_REDIRECT | quote }} - - name: DJANGO_SECURE_CONTENT_TYPE_NOSNIFF - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_CONTENT_TYPE_NOSNIFF | quote }} - - name: DJANGO_SECURE_FRAME_DENY - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_FRAME_DENY | quote }} - - name: DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS | quote }} - - name: DJANGO_SESSION_COOKIE_HTTPONLY - value: {{ .Values.osmchaWeb.env.DJANGO_SESSION_COOKIE_HTTPONLY | quote }} - - name: DJANGO_SESSION_COOKIE_SECURE - value: {{ .Values.osmchaWeb.env.DJANGO_SESSION_COOKIE_SECURE | quote }} - - name: DJANGO_DEFAULT_FROM_EMAIL - value: {{ .Values.osmchaWeb.env.DJANGO_DEFAULT_FROM_EMAIL | quote }} - - name: DJANGO_SERVER_EMAIL - value: {{ .Values.osmchaWeb.env.DJANGO_SERVER_EMAIL | quote }} - - name: DJANGO_EMAIL_SUBJECT_PREFIX - value: {{ .Values.osmchaWeb.env.DJANGO_EMAIL_SUBJECT_PREFIX | quote }} - - name: DJANGO_CHANGESETS_FILTER - value: {{ .Values.osmchaWeb.env.DJANGO_CHANGESETS_FILTER | quote }} - - name: OAUTH_OSM_KEY - value: {{ .Values.osmchaWeb.env.OAUTH_OSM_KEY | quote }} - - name: OAUTH_OSM_SECRET - value: {{ .Values.osmchaWeb.env.OAUTH_OSM_SECRET | quote }} - - name: DJANGO_ANON_USER_THROTTLE_RATE - value: {{ .Values.osmchaWeb.env.DJANGO_ANON_USER_THROTTLE_RATE | quote }} - - name: DJANGO_COMMON_USER_THROTTLE_RATE - value: {{ .Values.osmchaWeb.env.DJANGO_COMMON_USER_THROTTLE_RATE | quote }} - - name: DJANGO_NON_STAFF_USER_THROTTLE_RATE - value: {{ .Values.osmchaWeb.env.DJANGO_NON_STAFF_USER_THROTTLE_RATE | quote }} - - name: OAUTH_REDIRECT_URI - value: {{ .Values.osmchaWeb.env.OAUTH_REDIRECT_URI | quote }} - - name: OSMCHA_FRONTEND_VERSION - value: {{ .Values.osmchaWeb.env.OSMCHA_FRONTEND_VERSION | quote }} - - name: DJANGO_ENABLE_CHANGESET_COMMENTS - value: {{ .Values.osmchaWeb.env.DJANGO_ENABLE_CHANGESET_COMMENTS | quote }} - - name: DJANGO_OSM_COMMENTS_API_KEY - value: {{ .Values.osmchaWeb.env.DJANGO_OSM_COMMENTS_API_KEY | quote }} - - name: DJANGO_ROOT - value: {{ .Values.osmchaWeb.env.DJANGO_ROOT | quote }} - - name: DJANGO_SETTINGS_MODULE - value: {{ .Values.osmchaWeb.env.DJANGO_SETTINGS_MODULE | quote }} - - name: C_FORCE_ROOT - value: {{ .Values.osmchaWeb.env.C_FORCE_ROOT | quote }} - - name: REACT_APP_OSM_URL - value: {{ .Values.osmchaWeb.env.REACT_APP_OSM_URL | quote }} - - name: REACT_APP_OSM_API - value: {{ .Values.osmchaWeb.env.REACT_APP_OSM_API | quote }} - - name: REACT_APP_OVERPASS_BASE - value: {{ .Values.osmchaWeb.env.REACT_APP_OVERPASS_BASE | quote }} - - name: REACT_APP_NOMINATIM_URL - value: {{ .Values.osmchaWeb.env.REACT_APP_NOMINATIM_URL | quote }} - - name: OSMCHA_URL - value: {{ .Values.osmchaWeb.env.OSMCHA_URL | quote }} - {{- if .Values.osmchaWeb.nodeSelector.enabled }} - nodeSelector: - {{ .Values.osmchaWeb.nodeSelector.label_key }} : {{ .Values.osmchaWeb.nodeSelector.label_value }} - {{- end }} -{{- end }} - diff --git a/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml b/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml index 0f5902e1..3b0e6af0 100644 --- a/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml +++ b/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml @@ -30,12 +30,12 @@ spec: - name: http containerPort: 80 protocol: TCP - # livenessProbe: - # httpGet: - # path: / - # port: 80 - # initialDelaySeconds: 600 # 10 min, because the compile process takes time. - # timeoutSeconds: 30 + livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 14400 # Normal update takes 2 hours aprrox, afther that start checking liveness + timeoutSeconds: 30 {{- if .Values.overpassApi.resources.enabled }} resources: requests: @@ -46,7 +46,6 @@ spec: cpu: {{ .Values.overpassApi.resources.limits.cpu }} {{- end }} env: - # overpass-api env variables - name: OVERPASS_META value: {{ .Values.overpassApi.env.OVERPASS_META | quote}} - name: OVERPASS_MODE diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index a1a534f0..4f4c5e79 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -15,7 +15,7 @@ cloudProvider: minikube AWS_S3_BUCKET: s3://osm-seed # ==================================================================================================== -# GCP: In case you are using the cloudProvider=gcp set the below variables, We are assuming the nodes has a policies to upload files to cloud storage +# GCP: In case you are using the cloudProvider=gcp set the below variables, We are assuming the nodes has a policies to upload files to cloud storage # ==================================================================================================== GCP_STORAGE_BUCKET: gs://osm-seed-test @@ -44,9 +44,9 @@ AWS_SSL_ARN: false # If you specify `LoadBalancer` as the service type, if you also specify # an `AWS_SSL_ARN` that is a wildcart certificate, that will be configured # as the SSL certificate for your services. Else, you will need to configure -# SSL separately. -serviceType: NodePort - +# SSL separately. +# serviceType: NodePort +serviceType: LoadBalancer # Domain that is pointed to the clusterIP # You will need to create an A record like *.osmseed.example.com pointed to the ClusterIP @@ -77,12 +77,12 @@ adminEmail: admin@example.com db: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: POSTGRES_DB: openstreetmap POSTGRES_USER: postgres - POSTGRES_PASSWORD: '1234' + POSTGRES_PASSWORD: "1234" persistenceDisk: enabled: false mountPath: /var/lib/postgresql/data @@ -103,11 +103,11 @@ db: resources: enabled: false requests: - memory: '1Gi' - cpu: '1' + memory: "1Gi" + cpu: "1" limits: - memory: '2Gi' - cpu: '1' + memory: "2Gi" + cpu: "1" nodeSelector: enabled: false sharedMemorySize: 64Mi @@ -117,29 +117,29 @@ db: web: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" replicaCount: 1 serviceAnnotations: {} env: MAILER_ADDRESS: smtp.gmail.com MAILER_DOMAIN: gmail.com MAILER_USERNAME: example@gmail.com - MAILER_PASSWORD: '1234' + MAILER_PASSWORD: "1234" MAILER_FROM: no-reply@example.com MAILER_PORT: 25 - NOMINATIM_URL: 'nominatim' - OVERPASS_URL: 'overpassApi' - OAUTH_CLIENT_ID: 'abcd' # Create client ID and key as OAuth2 Application - OAUTH_KEY: 'xyz' + NOMINATIM_URL: "nominatim" + OVERPASS_URL: "overpassApi" + OAUTH_CLIENT_ID: "abcd" # Create client ID and key as OAuth2 Application + OAUTH_KEY: "xyz" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false autoscaling: @@ -161,11 +161,11 @@ memcached: resources: enabled: false requests: - memory: '8Gi' - cpu: '2' + memory: "8Gi" + cpu: "2" limits: - memory: '8Gi' - cpu: '2' + memory: "8Gi" + cpu: "2" # ==================================================================================================== # Variables for id-editor @@ -179,13 +179,13 @@ idEditor: fullHistory: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" nodeSelector: enabled: false label_key: nodegroup-type label_value: ohm-non-tiler - schedule: '* * * * *' + schedule: "* * * * *" env: OVERWRITE_FHISTORY_FILE: false resources: @@ -202,12 +202,12 @@ fullHistory: # ==================================================================================================== dbBackupRestore: image: - name: 'developmentseed/osmseed-backup-restore' - tag: '0.1.0-0.dev.git.942.h994af82' + name: "developmentseed/osmseed-backup-restore" + tag: "0.1.0-0.dev.git.942.h994af82" cronjobs: - name: web-db - enabled: true - schedule: '* * * * *' + enabled: false + schedule: "* * * * *" env: # DB action: backup/restore DB_ACTION: backup @@ -222,16 +222,16 @@ dbBackupRestore: resources: enabled: false requests: - memory: '300Mi' - cpu: '0.5' + memory: "300Mi" + cpu: "0.5" limits: - memory: '400Mi' - cpu: '0.6' + memory: "400Mi" + cpu: "0.6" nodeSelector: enabled: false - name: osmcha-db - enabled: true - schedule: '* * * * *' + enabled: false + schedule: "* * * * *" env: # DB action: backup/restore DB_ACTION: backup @@ -246,11 +246,11 @@ dbBackupRestore: resources: enabled: false requests: - memory: '300Mi' - cpu: '0.5' + memory: "300Mi" + cpu: "0.5" limits: - memory: '400Mi' - cpu: '0.6' + memory: "400Mi" + cpu: "0.6" nodeSelector: enabled: false @@ -260,19 +260,19 @@ dbBackupRestore: planetDump: enabled: false image: - name: '' - tag: '' - schedule: '* * * * *' + name: "" + tag: "" + schedule: "* * * * *" env: OVERWRITE_PLANET_FILE: false resources: enabled: false requests: - memory: '14Gi' - cpu: '4' + memory: "14Gi" + cpu: "4" limits: - memory: '16Gi' - cpu: '4' + memory: "16Gi" + cpu: "4" nodeSelector: enabled: false @@ -282,16 +282,16 @@ planetDump: replicationJob: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" nodeSelector: enabled: false @@ -301,18 +301,18 @@ replicationJob: populateApidb: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: - URL_FILE_TO_IMPORT: 'http://download.geofabrik.de/europe/monaco-latest.osm.pbf' + URL_FILE_TO_IMPORT: "http://download.geofabrik.de/europe/monaco-latest.osm.pbf" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2.5' + memory: "2Gi" + cpu: "2.5" # ==================================================================================================== # Variables to start a pod to process osm files @@ -320,19 +320,19 @@ populateApidb: osmProcessor: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: - URL_FILE_TO_PROCESS: 'http://download.geofabrik.de/europe/monaco-latest.osm.pbf' + URL_FILE_TO_PROCESS: "http://download.geofabrik.de/europe/monaco-latest.osm.pbf" OSM_FILE_ACTION: simple_pbf resources: enabled: false requests: - memory: '14Gi' - cpu: '4' + memory: "14Gi" + cpu: "4" limits: - memory: '16Gi' - cpu: '4' + memory: "16Gi" + cpu: "4" # ==================================================================================================== # Variables for tiler-db @@ -340,13 +340,13 @@ osmProcessor: tilerDb: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: POSTGRES_HOST: tiler-db POSTGRES_DB: tiler-osm POSTGRES_USER: postgres - POSTGRES_PASSWORD: '1234' + POSTGRES_PASSWORD: "1234" POSTGRES_PORT: 5432 POSTGRES_DB_MAX_CONNECTIONS: 100 POSTGRES_DB_SHARED_BUFFERS: 128MB @@ -370,11 +370,11 @@ tilerDb: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2.5' + memory: "2Gi" + cpu: "2.5" nodeSelector: enabled: false label_key: nodegroup_type @@ -386,8 +386,8 @@ tilerDb: tilerImposm: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: TILER_IMPORT_FROM: osm TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf @@ -408,11 +408,11 @@ tilerImposm: resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" nodeSelector: enabled: false label_key: nodegroup_type @@ -423,8 +423,8 @@ tilerImposm: tilerServer: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" replicaCount: 1 serviceAnnotations: {} env: @@ -437,7 +437,7 @@ tilerServer: TILER_CACHE_REGION: us-east-1 TILER_CACHE_AWS_ACCESS_KEY_ID: xyz TILER_CACHE_AWS_SECRET_ACCESS_KEY: xyz/q - command: './start.sh' # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh + command: "./start.sh" # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh # In case you use TILER_CACHE_TYPE: file with persistenceDisk persistenceDisk: enabled: false @@ -455,30 +455,30 @@ tilerServer: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false label_key: nodegroup_type label_value: tiler # ==================================================================================================== -# Variables for tiler-server cache cleaner, only avaliable in case the TILER_CACHE_TYPE = s3 +# Variables for tiler-server cache cleaner, only avaliable in case the TILER_CACHE_TYPE = s3 # ==================================================================================================== tilerServerCacheCleaner: enabled: false replicaCount: 1 - command: './tile_cache_downloader.sh & ./expire-watcher.sh' + command: "./tile_cache_downloader.sh & ./expire-watcher.sh" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" env: KILL_PROCESS: manually MAX_NUM_PS: 3 @@ -498,8 +498,8 @@ tilerServerCacheCleaner: tilerVisor: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} replicaCount: 1 env: @@ -508,11 +508,11 @@ tilerVisor: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false label_key: nodegroup_type @@ -524,48 +524,47 @@ tilerVisor: tmApi: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" nodeSelector: enabled: false replicaCount: 1 serviceAnnotations: {} env: - POSTGRES_HOST: 'tm' - POSTGRES_DB: 'tm' - POSTGRES_PASSWORD: 'tm' - POSTGRES_USER: 'postgres' + POSTGRES_HOST: "tm" + POSTGRES_DB: "tm" + POSTGRES_PASSWORD: "tm" + POSTGRES_USER: "postgres" POSTGRES_PORT: 5432 - TM_ORG_NAME: 'OpenStreetMap' - TM_ORG_CODE: 'OSM' - TM_ORG_URL: 'openstreetmap.org' - TM_ORG_PRIVACY_POLICY_URL: 'openstreetmap.org/copyright' - TM_ORG_GITHUB: 'github.com/openstreetmap' - OSM_SERVER_URL: 'https://openstreetmap.org' - OSM_NOMINATIM_SERVER_URL: 'https://nominatim.openstreetmap.org' - OSM_REGISTER_URL: 'https://openstreetmap.org/user/new' - ID_EDITOR_URL: 'https://openstreetmap.org/edit?editor=id' - POTLATCH2_EDITOR_URL: 'https://openstreetmap.org/edit?editor=potlatch2' - TM_SECRET: 's0m3l0ngr4nd0mstr1ng-b3cr34tiv3' - TM_CONSUMER_KEY: 'example' - TM_CONSUMER_SECRET: 'example' - TM_EMAIL_FROM_ADDRESS: 'no-reply@openstreetmap.org' - TM_SMTP_HOST: 'email-smtp.us-east-1.amazonaws.com' + TM_ORG_NAME: "OpenStreetMap" + TM_ORG_CODE: "OSM" + TM_ORG_URL: "openstreetmap.org" + TM_ORG_PRIVACY_POLICY_URL: "openstreetmap.org/copyright" + TM_ORG_GITHUB: "github.com/openstreetmap" + OSM_SERVER_URL: "https://openstreetmap.org" + OSM_NOMINATIM_SERVER_URL: "https://nominatim.openstreetmap.org" + OSM_REGISTER_URL: "https://openstreetmap.org/user/new" + ID_EDITOR_URL: "https://openstreetmap.org/edit?editor=id" + POTLATCH2_EDITOR_URL: "https://openstreetmap.org/edit?editor=potlatch2" + TM_SECRET: "s0m3l0ngr4nd0mstr1ng-b3cr34tiv3" + TM_CONSUMER_KEY: "example" + TM_CONSUMER_SECRET: "example" + TM_EMAIL_FROM_ADDRESS: "no-reply@openstreetmap.org" + TM_SMTP_HOST: "email-smtp.us-east-1.amazonaws.com" TM_SMTP_PORT: 25 - TM_SMTP_USER: '' - TM_SMTP_PASSWORD: '' - TM_DEFAULT_LOCALE: 'en' - TM_APP_API_URL: '' - TM_APP_BASE_URL: '' + TM_SMTP_USER: "" + TM_SMTP_PASSWORD: "" + TM_DEFAULT_LOCALE: "en" + TM_APP_API_URL: "" + TM_APP_BASE_URL: "" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' - + memory: "2Gi" + cpu: "2" # ==================================================================================================== # Variables for nominatim api @@ -573,8 +572,8 @@ tmApi: nominatimApi: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} replicaCount: 1 env: @@ -596,11 +595,11 @@ nominatimApi: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" persistenceDisk: enabled: true accessMode: ReadWriteOnce @@ -624,17 +623,16 @@ nominatimApi: overpassApi: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} env: - OVERPASS_META: 'yes' + OVERPASS_META: "yes" OVERPASS_MODE: init OVERPASS_PLANET_URL: http://download.geofabrik.de/europe/monaco-latest.osm.bz2 OVERPASS_DIFF_URL: http://download.openstreetmap.fr/replication/europe/monaco/minute/ OVERPASS_RULES_LOAD: 10 - #OVERPASS_PLANET_PREPROCESS: 'mv /db/planet.osm.bz2 /db/planet.osm.pbf && osmium cat -o /db/planet.osm.bz2 /db/planet.osm.pbf && rm /db/planet.osm.pbf' # it is in case we pass planet files as PBF file uncommment this line - OVERPASS_PLANET_PREPROCESS: 'ls' + OVERPASS_PLANET_PREPROCESS: 'mv /db/planet.osm.bz2 /db/planet.osm.pbf && osmium cat -o /db/planet.osm.bz2 /db/planet.osm.pbf && rm /db/planet.osm.pbf' OVERPASS_REPLICATION_SEQUENCE_NUMBER: 5201000 persistenceDisk: enabled: false @@ -652,11 +650,11 @@ overpassApi: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false # ==================================================================================================== @@ -665,8 +663,8 @@ overpassApi: taginfo: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} env: URL_PLANET_FILE_STATE: https://planet.osm.org/pbf/state.txt @@ -679,16 +677,16 @@ taginfo: INSTANCE_ICON: /img/logo/world.png INSTANCE_CONTACT: Anonymous TAGINFO_PROJECT_REPO: https://github.com/taginfo/taginfo-projects.git - DOWNLOAD_DB: 'languages wiki' - CREATE_DB: 'db projects chronology' + DOWNLOAD_DB: "languages wiki" + CREATE_DB: "db projects chronology" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false # ==================================================================================================== @@ -697,17 +695,17 @@ taginfo: osmSimpleMetrics: enabled: false image: - name: '' - tag: '' - schedule: '* */1 * * *' + name: "" + tag: "" + schedule: "* */1 * * *" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false @@ -717,20 +715,20 @@ osmSimpleMetrics: monitoringReplication: enabled: false image: - name: 'developmentseed/osmseed-replication-job' - tag: '0.1.0-n770.hd15c9f2' - schedule: '* */1 * * *' + name: "developmentseed/osmseed-replication-job" + tag: "0.1.0-n770.hd15c9f2" + schedule: "* */1 * * *" env: CREATE_MISSING_FILES: empty REPLICATION_SEQUENCE_NUMBER: "109789" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false @@ -740,92 +738,87 @@ monitoringReplication: changesetReplicationJob: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" nodeSelector: enabled: false - - # ==================================================================================================== -# Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day +# Variables for osmcha web builder # ==================================================================================================== osmchaWeb: - enabled: false + enabled: true + image: + name: "" + tag: "" +# ==================================================================================================== +# Variables for osmcha Api +# ==================================================================================================== +osmchaApi: + enabled: true image: - name: ghcr.io/willemarcel/osmcha-django - tag: b1f4e6afc90e08707cadc4d74580632ca3b93dd2 + name: "ghcr.io/willemarcel/osmcha-django" + tag: "10d7f6748d9e55d439339ed2c35509db15f782a6" env: - OSM_SERVER_URL: 'https://staging.openstreetmap.org' - DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" - DJANGO_DEBUG: "False" + DJANGO_SETTINGS_MODULE: "config.settings.production" + OSMCHA_FRONTEND_VERSION: "v0.86.0-production" DJANGO_SECRET_KEY: abc - DJANGO_SECURE_BROWSER_XSS_FILTER: "True" - DJANGO_SECURE_SSL_REDIRECT: "True" - DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" - DJANGO_SECURE_FRAME_DENY: "True" - DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True" - DJANGO_SESSION_COOKIE_HTTPONLY: "True" - DJANGO_SESSION_COOKIE_SECURE: "True" - DJANGO_DEFAULT_FROM_EMAIL: "webmaster@openstreetmap.org" - DJANGO_SERVER_EMAIL: "server@openstreetmap.org" - DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" - DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" + ## ohm OAUTH_OSM_KEY: abc - OAUTH_OSM_SECRET: abc - DJANGO_ANON_USER_THROTTLE_RATE: "10/h" - DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" - DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" - OAUTH_REDIRECT_URI: https://osmcha.openstreetmap.org/oauth-landing.html - OSMCHA_FRONTEND_VERSION: "v0.86.0-production" - DJANGO_ENABLE_CHANGESET_COMMENTS: "True" - DJANGO_OSM_COMMENTS_API_KEY: "test" - DJANGO_ROOT: /app - DJANGO_SETTINGS_MODULE: config.settings.local - C_FORCE_ROOT: true - REACT_APP_OSM_URL: https://staging.openstreetmap.org - REACT_APP_OSM_API: https://staging.openstreetmap.org/api/0.6 - REACT_APP_OVERPASS_BASE: https://overpass-api-staging.openstreetmap.org/api/interpreter - REACT_APP_NOMINATIM_URL: https://nominatim-api-staging.openstreetmap.org - OSMCHA_URL: https://osmcha-staging.openstreetmap.org + OAUTH_OSM_SECRET: bdc + DJANGO_SECURE_SSL_REDIRECT: "False" + OSM_SERVER_URL: https://www.osmseed.org + OAUTH_REDIRECT_URI: https://osmcha-staging.osmseed.org/oauth-landing.html + OSM_PLANET_BASE_URL: https://s3.amazonaws.com/planet.osmseed.org/replication/changesets/ + + ## frontend + OSMCHA_URL: https://osmcha-staging.osmseed.org + OSMCHA_API_URL: www.osmseed.org + REACT_APP_OSM_URL: https://www.osmseed.org + REACT_APP_OSM_API: https://www.osmseed.org/api/0.6 + REACT_APP_OVERPASS_BASE: //overpass-api.osmseed.org/api/interpreter + REACT_APP_DISABLE_REAL_CHANGESETS: 1 + REACT_APP_MAPBOX_ACCESS_TOKEN: "pk.abc.df" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "512Mi" + cpu: "1" limits: - memory: '24Gi' - cpu: '10' + memory: "512Mi" + cpu: "1" nodeSelector: enabled: false + label_key: nodegroup_type + label_value: db # ==================================================================================================== -# Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day +# Variables for osmcha DB # ==================================================================================================== osmchaDb: - enabled: false + enabled: true image: - name: '' - tag: '' + name: "" + tag: "" env: POSTGRES_DB: osmcha-db POSTGRES_USER: postgres - POSTGRES_PASSWORD: '1234' + POSTGRES_PASSWORD: "1234" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" persistenceDisk: enabled: false accessMode: ReadWriteOnce From 4591171a5b7befa78341f09a841588dbd7aa2aee Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Tue, 6 Feb 2024 08:31:36 -0500 Subject: [PATCH 03/23] Update taginfo container (#307) Co-authored-by: Rub21 --- images/taginfo/Dockerfile | 54 +++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/images/taginfo/Dockerfile b/images/taginfo/Dockerfile index 16745806..c4dde56a 100644 --- a/images/taginfo/Dockerfile +++ b/images/taginfo/Dockerfile @@ -1,25 +1,13 @@ -FROM ruby:3.0 +FROM ruby:2.4 ENV workdir /apps - -# Install Taginfo site -RUN apt-get update && apt-get -y install \ +WORKDIR $workdir +RUN apt-get update && apt-get install -y \ curl \ sqlite3 \ sqlite3-pcre \ ruby-passenger \ libapache2-mod-passenger \ - git - -# Commit ae5a950f7aa4c0de4e706839619a1dc05fc4450a, at 2021-10-18 -RUN git clone https://github.com/taginfo/taginfo.git $workdir/taginfo -WORKDIR $workdir/taginfo -RUN git checkout ae5a950f7aa4c0de4e706839619a1dc05fc4450a -RUN echo "gem 'thin' " >>Gemfile -RUN gem install bundler -RUN bundle install - -# Install Taginfo tools -RUN apt-get -y install \ + git \ cmake \ libbz2-dev \ libexpat1-dev \ @@ -31,25 +19,31 @@ RUN apt-get -y install \ make \ zlib1g-dev \ jq \ - ca-certificates - -# Other useful packages -RUN apt-get install -y \ - git \ + ca-certificates \ osmium-tool \ pyosmium \ rsync \ tmux \ - zsh + zsh \ + nano \ + vim \ + && rm -rf /var/lib/apt/lists/* # Clean up to reduce image size -RUN git clone https://github.com/taginfo/taginfo-tools.git $workdir/taginfo-tools -WORKDIR $workdir/taginfo-tools -RUN git submodule update --init -RUN mkdir build && cd build && cmake .. && make +# Clone and setup taginfo-tools +RUN git clone https://github.com/taginfo/taginfo-tools.git $workdir/taginfo-tools && \ + cd $workdir/taginfo-tools && \ + git submodule update --init && \ + mkdir build && cd build && \ + cmake .. && make + +# Clone and setup taginfo +RUN git clone https://github.com/taginfo/taginfo.git $workdir/taginfo && \ + cd $workdir/taginfo && \ + git checkout ae5a950f7aa4c0de4e706839619a1dc05fc4450a && \ + echo "gem 'thin' " >> Gemfile && \ + gem install bundler -v 2.3.27 && \ + bundle install -RUN apt-get install -y nano vim COPY overwrite_config.py $workdir/ COPY start.sh $workdir/ - -WORKDIR $workdir/ -CMD $workdir/start.sh \ No newline at end of file +CMD $workdir/start.sh From d91475f77a01f69b6bd1170ae8bfaf9727beb4a8 Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Wed, 7 Feb 2024 13:12:54 -0500 Subject: [PATCH 04/23] Update osmcha frontend container (#308) Co-authored-by: Rub21 --- images/osmcha-web/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/images/osmcha-web/Dockerfile b/images/osmcha-web/Dockerfile index 9b5f4181..04867f44 100644 --- a/images/osmcha-web/Dockerfile +++ b/images/osmcha-web/Dockerfile @@ -2,10 +2,12 @@ FROM node:16-slim as builder ENV DEBIAN_FRONTEND noninteractive ENV BUILD_ENV=prod ENV REACT_APP_PRODUCTION_API_URL=/api/v1 -RUN apt-get update && apt-get install -y git +RUN apt-get update && apt-get install -y git \ + && rm -rf /var/lib/apt/lists/* RUN mkdir /app WORKDIR /app -RUN git clone https://github.com/osmus/osmcha-frontend.git /app +ARG GIT_TAG=master +RUN git clone --branch $GIT_TAG https://github.com/osmus/osmcha-frontend.git /app RUN yarn install COPY start.sh . CMD [ "/app/start.sh" ] From 4a88a934fd7789106c3c2eeae069989eec2d42ae Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Thu, 8 Feb 2024 14:59:23 -0500 Subject: [PATCH 05/23] Fix ruby version (#310) --- images/web/config/production.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 8b9f3c2b..34dcb9e3 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -2,7 +2,7 @@ # ServerName localhost # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public - PassengerRuby /usr/bin/ruby + PassengerRuby /opt/ruby/bin/ruby RewriteEngine On RewriteCond %{HTTP:X-Forwarded-Proto} =http From f87b744d0b30b0586359c3b23ba7147b3dd93f6f Mon Sep 17 00:00:00 2001 From: Sanjay Bhangar Date: Fri, 16 Feb 2024 14:31:56 +0000 Subject: [PATCH 06/23] fix env var name for id key - OSM_id_key -> OPENSTREETMAP_id_key (#303) * fix env var name for id key - OSM_id_key -> OPENSTREETMAP_id_key * Update env var OPENSTREETMAP_id_key and add comments --------- Co-authored-by: Rub21 --- envs/.env.web.example | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/envs/.env.web.example b/envs/.env.web.example index e20a31d2..9c720436 100644 --- a/envs/.env.web.example +++ b/envs/.env.web.example @@ -24,8 +24,9 @@ MAILER_PORT=25 # Nominatim settings NOMINATIM_URL=nominatim-api -# TODO: Describe those vars -OSM_id_key="abcd..." +# Make sure that the first time this (OPENSTREETMAP_id_key) value is empty. +# Once you start you server and create your OAuth 1, fill the value with the Consumer Key. +OPENSTREETMAP_id_key="" OSM_memcache_servers="" # NEW_RELIC settings From 2a7e77ff24247b780367f09d55446bc3e32752a7 Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Tue, 20 Feb 2024 10:22:15 -0500 Subject: [PATCH 07/23] ImpOSM update sequence (#313) * Update script to keep state of sequence files - imposm * Update docker file and start.sh - imposm * Add liveness probe - imposm * Add UPLOAD_EXPIRED_FILES env var for imposm * Add default value for UPLOAD_EXPIRED_FILES in imposm * set initialDelaySeconds: 3600 in liveness probe - imposm * Increase initialDelaySeconds to 2 hours --------- --- compose/tiler.yml | 57 +++-- images/tiler-imposm/Dockerfile | 10 +- images/tiler-imposm/liveness.sh | 8 + images/tiler-imposm/start.sh | 237 +++++++++--------- .../tiler-imposm-statefulset.yaml | 28 ++- osm-seed/values.yaml | 1 + 6 files changed, 186 insertions(+), 155 deletions(-) create mode 100755 images/tiler-imposm/liveness.sh diff --git a/compose/tiler.yml b/compose/tiler.yml index a880d3d8..b94acd66 100644 --- a/compose/tiler.yml +++ b/compose/tiler.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.8' services: # ###################################################### # ### Tiler DB @@ -9,11 +9,12 @@ services: context: ../images/tiler-db dockerfile: Dockerfile ports: - - "5433:5432" + - "5433:5432" volumes: - - ../data/tiler-db-data:/var/lib/postgresql/data + - ../data/tiler-db-data:/var/lib/postgresql/data env_file: - - ../envs/.env.tiler-db + - ../envs/.env.tiler-db + restart: always # ###################################################### # ### Tiler imposm for importing data # ###################################################### @@ -23,18 +24,23 @@ services: context: ../images/tiler-imposm dockerfile: Dockerfile volumes: - - ../data/tiler-imposm-data:/mnt/data + - ../data/tiler-imposm-data:/mnt/data + # - ../images/tiler-imposm:/osm depends_on: - - tiler-db - command: > - /bin/bash -c " - echo starting imposm!; - sleep 1m; - ./start.sh; - " + - tiler-db + command: + - sh + - -c + - "sleep 60 && ./start.sh" env_file: - - ../envs/.env.tiler-db - - ../envs/.env.tiler-imposm + - ../envs/.env.tiler-db + - ../envs/.env.tiler-imposm + healthcheck: + test: ["CMD", "./liveness.sh"] + interval: 30s + timeout: 10s + retries: 3 + restart: always # ###################################################### # ### Tiler imposm for importing data # ###################################################### @@ -44,18 +50,17 @@ services: context: ../images/tiler-server dockerfile: Dockerfile volumes: - - ../data/tiler-server-data:/mnt/data + - ../data/tiler-server-data:/mnt/data depends_on: - - tiler-db - - tiler-imposm + - tiler-db + - tiler-imposm ports: - - "9090:9090" - command: > - /bin/bash -c " - echo Starting tiles server!; - sleep 2m; - ./start.sh; - " + - "9090:9090" + command: + - sh + - -c + - "sleep 60 && ./start.sh" env_file: - - ../envs/.env.tiler-db - - ../envs/.env.tiler-server \ No newline at end of file + - ../envs/.env.tiler-db + - ../envs/.env.tiler-server + restart: always diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile index 0c38c2d1..10b7fd2b 100644 --- a/images/tiler-imposm/Dockerfile +++ b/images/tiler-imposm/Dockerfile @@ -19,12 +19,14 @@ RUN apt-get install -y \ curl \ wget \ unzip \ - software-properties-common + software-properties-common && \ + rm -rf /var/lib/apt/lists/* # # Install python RUN add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \ + rm -rf /var/lib/apt/lists/* && \ python3 -m pip install pip --upgrade && \ python3 -m pip install wheel @@ -53,7 +55,5 @@ WORKDIR $IMPOSMDATA VOLUME $IMPOSMDATA WORKDIR /osm -COPY config config -COPY scripts scripts -COPY start.sh start.sh -CMD ./start.sh +COPY . . +CMD ./start.sh \ No newline at end of file diff --git a/images/tiler-imposm/liveness.sh b/images/tiler-imposm/liveness.sh new file mode 100755 index 00000000..a7b249ea --- /dev/null +++ b/images/tiler-imposm/liveness.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +if ps aux | grep -v grep | grep "imposm" >/dev/null; then + echo "imposm process is running." + exit 0 +else + echo "imposm process is not running." 1>&2 + exit 1 +fi diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh index cf2d7922..c9fa48a0 100755 --- a/images/tiler-imposm/start.sh +++ b/images/tiler-imposm/start.sh @@ -1,175 +1,180 @@ #!/bin/bash set -e -stateFile="state.txt" -PBFFile="osm.pbf" -limitFile="limitFile.geojson" + +STATEFILE="state.txt" +PBFFILE="osm.pbf" +LIMITFILE="limitFile.geojson" # directories to keep the imposm's cache for updating the db -workDir=/mnt/data -cachedir=$workDir/cachedir -mkdir -p $cachedir -diffdir=$workDir/diff -mkdir -p $diffdir -imposm3_expire_dir=$workDir/imposm3_expire_dir -mkdir -p $imposm3_expire_dir -# imposm3_expire_state_dir=$workDir/imposm3_expire_state -# mkdir -p $imposm3_expire_state_dir -# Setting directory -settingDir=/osm +WORKDIR=/mnt/data +CACHE_DIR=$WORKDIR/cachedir +DIFF_DIR=$WORKDIR/diff +IMPOSM3_EXPIRE_DIR=$WORKDIR/imposm3_expire_dir + +# # Setting directory +# settingDir=/osm # Folder to store the imposm expider files in s3 or gs BUCKET_IMPOSM_FOLDER=imposm INIT_FILE=/mnt/data/init_done -# Create config file to set variable for imposm -echo "{" > $workDir/config.json -echo "\"cachedir\": \"$cachedir\"," >> $workDir/config.json -echo "\"diffdir\": \"$diffdir\"," >> $workDir/config.json -echo "\"connection\": \"postgis://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB\"," >> $workDir/config.json -echo "\"mapping\": \"config/imposm3.json\"," >> $workDir/config.json -echo "\"replication_url\": \"$REPLICATION_URL\"" >> $workDir/config.json -echo "}" >> $workDir/config.json - -function getData () { - # Import from pubic url, usualy it come from osm - if [ $TILER_IMPORT_FROM == "osm" ]; then - wget $TILER_IMPORT_PBF_URL -O $PBFFile - fi - if [ $TILER_IMPORT_FROM == "osmseed" ]; then - if [ $CLOUDPROVIDER == "aws" ]; then +mkdir -p "$CACHE_DIR" "$DIFF_DIR" "$IMPOSM3_EXPIRE_DIR" + +# Create config file to set variables for imposm +{ + echo "{" + echo "\"cachedir\": \"$CACHE_DIR\"," + echo "\"diffdir\": \"$DIFF_DIR\"," + echo "\"connection\": \"postgis://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB\"," + echo "\"mapping\": \"config/imposm3.json\"," + echo "\"replication_url\": \"$REPLICATION_URL\"" + echo "}" +} >"$WORKDIR/config.json" + +function getData() { + ### Get the PBF file from the cloud provider or public URL + if [ "$TILER_IMPORT_FROM" == "osm" ]; then + wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE" + elif [ "$TILER_IMPORT_FROM" == "osmseed" ]; then + if [ "$CLOUDPROVIDER" == "aws" ]; then # Get the state.txt file from S3 - aws s3 cp $AWS_S3_BUCKET/planet/full-history/$stateFile . - PBFCloudPath=$(tail -n +1 $stateFile) - aws s3 cp $PBFCloudPath $PBFFile - fi - # Google storage - if [ $CLOUDPROVIDER == "gcp" ]; then + aws s3 cp "$AWS_S3_BUCKET/planet/full-history/$STATEFILE" . + PBFCloudPath=$(tail -n +1 "$STATEFILE") + aws s3 cp "$PBFCloudPath" "$PBFFILE" + elif [ "$CLOUDPROVIDER" == "gcp" ]; then # Get the state.txt file from GS - gsutil cp $GCP_STORAGE_BUCKET/planet/full-history/$stateFile . - PBFCloudPath=$(tail -n +1 $stateFile) - gsutil cp $PBFCloudPath $PBFFile + gsutil cp "$GCP_STORAGE_BUCKET/planet/full-history/$STATEFILE" . + PBFCloudPath=$(tail -n +1 "$STATEFILE") + gsutil cp "$PBFCloudPath" "$PBFFILE" + fi + fi +} + +getFormattedDate() { + local file_path="$1" + if command -v stat >/dev/null 2>&1; then + local modification_date=$(stat -c %Y "$file_path") + if [ $? -eq 0 ]; then + local formatted_date=$(date -d "@$modification_date" "+%Y-%m-%d:%H:%M:%S") + echo "Created/Updated date of $file_path: $formatted_date" + else + echo "Error: Unable to get file modification date for file ${file_path}" fi + else + echo "Error: 'stat' command not found. Unable to get file modification date, for file ${file_path}" fi } -function uploadExpiredFiles(){ - # create statte file - # dateStr=$(date '+%y%m%d%H%M%S') - # stateFile=$imposm3_expire_state_dir/expired_${dateStr}.txt - # bucketStateFile=${stateFile#*"$workDir"} - - for file in $(find $imposm3_expire_dir -type f -cmin -1); do - bucketFile=${file#*"$workDir"} - echo $(date +%F_%H:%M:%S)": New file..." $file - # echo $file >> $stateFile +function uploadExpiredFiles() { + # Upload the expired files to the cloud provider + for file in $(find "$IMPOSM3_EXPIRE_DIR" -type f -cmin -1); do + bucketFile=${file#*"$WORKDIR"} + getFormattedDate "$file" + # UPLOAD_EXPIRED_FILES=true to upload the expired to cloud provider + if [ "$UPLOAD_EXPIRED_FILES" == "true" ]; then + echo "Uploading expired file ${file} to ${AWS_S3_BUCKET}" + # AWS if [ "$CLOUDPROVIDER" == "aws" ]; then - aws s3 cp $file ${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}${bucketFile} --acl public-read + aws s3 cp "$file" "${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}${bucketFile}" --acl public-read fi + # Google Storage if [ "$CLOUDPROVIDER" == "gcp" ]; then - gsutil cp -a public-read $file ${GCP_STORAGE_BUCKET}${BUCKET_IMPOSM_FOLDER}${bucketFile} + gsutil cp -a public-read "$file" "${GCP_STORAGE_BUCKET}${BUCKET_IMPOSM_FOLDER}${bucketFile}" fi - done - # Upload state File - # if [[ -f "$stateFile" ]]; then - # # AWS - # if [ "$CLOUDPROVIDER" == "aws" ]; then - # aws s3 cp $stateFile ${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}${bucketStateFile} --acl public-read - # fi - # # Google Storage - # if [ "$CLOUDPROVIDER" == "gcp" ]; then - # gsutil cp -a public-read $stateFile ${GCP_STORAGE_BUCKET}${BUCKET_IMPOSM_FOLDER}${bucketStateFile} - # fi - # fi + else + echo "Expired files were not uploaded because UPLOAD_EXPIRED_FILES=${UPLOAD_EXPIRED_FILES}" + fi + done } -function updateData(){ +function updateData() { + ### Update the DB with the new data form minute replication if [ "$OVERWRITE_STATE" = "true" ]; then - rm $diffdir/last.state.txt + rm $DIFF_DIR/last.state.txt fi - # Verify if last.state.txt exist - if [ -f "$diffdir/last.state.txt" ]; then - echo "Exist... $diffdir/last.state.txt" - else - # OverWrite the last.state.txt file with REPLICATION_URL and sequenceNumber=0 + + # Check if last.state.txt exists + if [ -f "$DIFF_DIR/last.state.txt" ]; then + echo "Exist... $DIFF_DIR/last.state.txt" + else + # Create last.state.txt file with REPLICATION_URL and SEQUENCE_NUMBER from env vars echo "timestamp=0001-01-01T00\:00\:00Z sequenceNumber=$SEQUENCE_NUMBER - replicationUrl=$REPLICATION_URL" > $diffdir/last.state.txt + replicationUrl=$REPLICATION_URL" >$DIFF_DIR/last.state.txt fi + # Check if the limit file exists if [ -z "$TILER_IMPORT_LIMIT" ]; then - imposm run -config $workDir/config.json -expiretiles-dir $imposm3_expire_dir & - while true - do - echo "Updating...$(date +%F_%H-%M-%S)" - uploadExpiredFiles - sleep 1m - done + imposm run -config "$WORKDIR/config.json" -expiretiles-dir "$IMPOSM3_EXPIRE_DIR" & else - imposm run -config $workDir/config.json -limitto $workDir/$limitFile -expiretiles-dir $imposm3_expire_dir & - while true - do - echo "Updating...$(date +%F_%H-%M-%S)" - uploadExpiredFiles - sleep 1m - done + imposm run -config "$WORKDIR/config.json" -limitto "$WORKDIR/$LIMITFILE" -expiretiles-dir "$IMPOSM3_EXPIRE_DIR" & fi + + while true; do + echo "Upload expired files... $(date +%F_%H-%M-%S)" + uploadExpiredFiles + sleep 1m + done } -function importData () { +function importData() { + ### Import the PBF and Natural Earth files to the DB echo "Execute the missing functions" psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_helpers.sql - echo "Import Natural Earth" + + echo "Import Natural Earth..." ./scripts/natural_earth.sh - echo "Import OSM Land" + + echo "Import OSM Land..." ./scripts/osm_land.sh - echo "Import PBF file" + + echo "Import PBF file..." if [ -z "$TILER_IMPORT_LIMIT" ]; then imposm import \ - -config $workDir/config.json \ - -read $PBFFile \ - -write \ - -diff -cachedir $cachedir -diffdir $diffdir + -config $WORKDIR/config.json \ + -read $PBFFILE \ + -write \ + -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR else - wget $TILER_IMPORT_LIMIT -O $workDir/$limitFile + wget $TILER_IMPORT_LIMIT -O $WORKDIR/$LIMITFILE imposm import \ - -config $workDir/config.json \ - -read $PBFFile \ - -write \ - -diff -cachedir $cachedir -diffdir $diffdir \ - -limitto $workDir/$limitFile + -config $WORKDIR/config.json \ + -read $PBFFILE \ + -write \ + -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR \ + -limitto $WORKDIR/$LIMITFILE fi imposm import \ - -config $workDir/config.json \ - -deployproduction - # -diff -cachedir $cachedir -diffdir $diffdir + -config $WORKDIR/config.json \ + -deployproduction # These index will help speed up tegola tile generation psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_index.sql touch $INIT_FILE + # Update the DB updateData } - echo "Connecting to $POSTGRES_HOST DB" flag=true while "$flag" = true; do pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue - # Change flag to false to stop ping the DB - flag=false - echo "Check if $INIT_FILE exists" - if ([[ -f $INIT_FILE ]]); then - echo "Update the DB with osm data" - updateData - else - echo "Import PBF data to DB" - getData - if [ -f $PBFFile ]; then - echo "Start importing the data" - importData - fi + # Change flag to false to stop ping the DB + flag=false + echo "Check if $INIT_FILE exists" + if ([[ -f $INIT_FILE ]]); then + echo "Update the DB with osm data" + updateData + else + echo "Import PBF data to DB" + getData + if [ -f $PBFFILE ]; then + echo "Start importing the data" + importData fi + fi done diff --git a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml index f25ff266..ab344bab 100644 --- a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml +++ b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml @@ -24,6 +24,16 @@ spec: - name: {{ .Release.Name }}-tiler-imposm-statefulset image: {{ .Values.tilerImposm.image.name }}:{{ .Values.tilerImposm.image.tag }} command: ['./start.sh'] + livenessProbe: + exec: + command: + - /bin/bash + - -c + - ./liveness.sh + initialDelaySeconds: 7200 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 3 {{- if .Values.tilerImposm.resources.enabled }} resources: requests: @@ -37,23 +47,25 @@ spec: - name: POSTGRES_HOST value: {{ .Release.Name }}-tiler-db - name: POSTGRES_DB - value: {{ .Values.tilerDb.env.POSTGRES_DB }} + value: {{ .Values.tilerDb.env.POSTGRES_DB | quote }} - name: POSTGRES_PASSWORD - value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }} + value: {{ .Values.tilerDb.env.POSTGRES_PASSWORD | quote }} - name: POSTGRES_USER - value: {{ .Values.tilerDb.env.POSTGRES_USER }} + value: {{ .Values.tilerDb.env.POSTGRES_USER | quote }} - name: POSTGRES_PORT value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }} - name: TILER_IMPORT_FROM - value: {{ .Values.tilerImposm.env.TILER_IMPORT_FROM }} + value: {{ .Values.tilerImposm.env.TILER_IMPORT_FROM | quote }} - name: TILER_IMPORT_PBF_URL - value: {{ .Values.tilerImposm.env.TILER_IMPORT_PBF_URL }} + value: {{ .Values.tilerImposm.env.TILER_IMPORT_PBF_URL | quote }} - name: REPLICATION_URL - value: {{ .Values.tilerImposm.env.REPLICATION_URL }} + value: {{ .Values.tilerImposm.env.REPLICATION_URL | quote }} - name: SEQUENCE_NUMBER - value: {{ .Values.tilerImposm.env.SEQUENCE_NUMBER | quote}} + value: {{ .Values.tilerImposm.env.SEQUENCE_NUMBER | quote }} - name: OVERWRITE_STATE - value: {{ quote .Values.tilerImposm.env.OVERWRITE_STATE }} + value: {{ .Values.tilerImposm.env.OVERWRITE_STATE | quote }} + - name: UPLOAD_EXPIRED_FILES + value: {{ .Values.tilerImposm.env.UPLOAD_EXPIRED_FILES | default "true" | quote }} - name: CLOUDPROVIDER value: {{ .Values.cloudProvider }} # In case cloudProvider=aws diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 4f4c5e79..b9bfc19d 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -392,6 +392,7 @@ tilerImposm: TILER_IMPORT_FROM: osm TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf TILER_IMPORT_LIMIT: https://gist.githubusercontent.com/Rub21/96bdcac5eb11f0b36ba8d0352ac537f4/raw/2606f2e207d4a0d895897a83efa1efacefd36eb4/monaco.geojson + UPLOAD_EXPIRED_FILES: true persistenceDisk: enabled: false accessMode: ReadWriteOnce From cfaec17da2f257bcc2c36ee2c911dca595f4a9d4 Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Wed, 21 Feb 2024 07:54:16 -0500 Subject: [PATCH 08/23] Planet server files (#314) * Update replication job and add a log file to optimize upload to the cloud provider. * Add planet files server * Update container and create image using chartpress --- chartpress.yaml | 4 +- compose/db-backup-restore.yml | 6 +- compose/full-history.yml | 22 --- compose/planet-dump.yml | 30 ---- compose/planet.yml | 89 ++++++++++++ compose/replication.yml | 32 ----- compose/web.yml | 16 +-- images/full-history/README.md | 16 +-- images/planet-dump/README.md | 16 +-- images/planet-files/Dockerfile | 10 ++ images/planet-files/README.MD | 11 ++ images/planet-files/index.html | 105 ++++++++++++++ images/planet-files/server/package.json | 12 ++ images/planet-files/server/server.js | 49 +++++++ images/replication-job/Dockerfile | 7 +- images/replication-job/start.sh | 180 +++++++++++++++--------- osm-seed/values.yaml | 15 +- 17 files changed, 424 insertions(+), 196 deletions(-) delete mode 100644 compose/full-history.yml delete mode 100644 compose/planet-dump.yml create mode 100644 compose/planet.yml delete mode 100644 compose/replication.yml create mode 100644 images/planet-files/Dockerfile create mode 100644 images/planet-files/README.MD create mode 100644 images/planet-files/index.html create mode 100644 images/planet-files/server/package.json create mode 100644 images/planet-files/server/server.js diff --git a/chartpress.yaml b/chartpress.yaml index f5af3ddc..388d0d44 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -46,4 +46,6 @@ charts: valuesPath: osmchaWeb.image osmcha-db: valuesPath: osmchaDb.image - \ No newline at end of file + planet-files: + valuesPath: planetFiles.image + diff --git a/compose/db-backup-restore.yml b/compose/db-backup-restore.yml index 7641cde8..982e94ad 100644 --- a/compose/db-backup-restore.yml +++ b/compose/db-backup-restore.yml @@ -10,11 +10,7 @@ services: dockerfile: Dockerfile volumes: - ../data/db-backup-restore-data:/mnt/data - command: > - /bin/bash -c " - echo 'Set cron job for backing up DB, every 4 minutes'; - while :; do echo 'Creating DB backup...'; /start.sh; sleep 4m; done; - " + command: /start.sh env_file: - ../envs/.env.db - ../envs/.env.osmcha diff --git a/compose/full-history.yml b/compose/full-history.yml deleted file mode 100644 index f5bf586a..00000000 --- a/compose/full-history.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: '3' -services: - ##################################################### - ## OSM full planet replication - ##################################################### - full-history: - image: osmseed-full-history:v1 - build: - context: ../images/full-history - dockerfile: Dockerfile - volumes: - - ../data/full-history-data:/mnt/data - command: > - /bin/bash -c " - echo 'Set cron job for full history PBF file, every 4 minutes'; - while :; do echo 'Creating full history PBF file...'; /start.sh; sleep 4m; done; - " - env_file: - - ../envs/.env.db - - ../envs/.env.db-utils - - ../envs/.env.cloudprovider - \ No newline at end of file diff --git a/compose/planet-dump.yml b/compose/planet-dump.yml deleted file mode 100644 index b265a768..00000000 --- a/compose/planet-dump.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: '3' -services: - ###################################################### - ### OSM processor and also use this image as base for other containers - ###################################################### - osm-processor: - image: osmseed-osm-processor:v1 - build: - context: ../images/osm-processor - dockerfile: Dockerfile - ###################################################### - ### Planet replication section - ###################################################### - planet-dump: - image: osmseed-planet-dump:v1 - build: - context: ./images/planet-dump - dockerfile: Dockerfile - volumes: - - ../data/planet-dump-data:/mnt/data - command: > - /bin/bash -c " - echo 'Set cronjob for planet-dump, every 4 minutes'; - while :; do echo 'Creating the planet dump file...'; /start.sh; sleep 4m; done; - " - env_file: - - ../envs/.env.db - - ../envs/.env.db-utils - - ../envs/.env.cloudprovider - diff --git a/compose/planet.yml b/compose/planet.yml new file mode 100644 index 00000000..bb0e31c9 --- /dev/null +++ b/compose/planet.yml @@ -0,0 +1,89 @@ +version: '3' +services: + ###################################################### + ### OSM minute replication files section + ###################################################### + replication-job: + image: osmseed-replication-job:v1 + build: + context: ../images/replication-job + dockerfile: Dockerfile + volumes: + - ../data/replication-job-data:/mnt/data + # - ./../images/replication-job:/app + command: /start.sh + env_file: + - ../envs/.env.db + - ../envs/.env.db-utils + - ../envs/.env.cloudprovider + + ###################################################### + ### Planet replication section + ###################################################### + planet-dump: + image: osmseed-planet-dump:v1 + build: + context: ../images/planet-dump + dockerfile: Dockerfile + volumes: + - ../data/planet-dump-data:/mnt/data + command: /start.sh; + env_file: + - ../envs/.env.db + - ../envs/.env.db-utils + - ../envs/.env.cloudprovider + + ##################################################### + ## OSM full planet replication + ##################################################### + full-history: + image: osmseed-full-history:v1 + build: + context: ../images/full-history + dockerfile: Dockerfile + volumes: + - ../data/full-history-data:/mnt/data + command: /start.sh + env_file: + - ../envs/.env.db + - ../envs/.env.db-utils + - ../envs/.env.cloudprovider + + ##################################################### + ## Changeset replications + ##################################################### + changeset-replication-job: + image: osmseed-changeset-replication-job:v1 + build: + context: ../images/changeset-replication-job + dockerfile: Dockerfile + volumes: + - ../data/changeset-replication-job-data:/mnt/changesets + command: > + /bin/bash -c "./start.sh" + env_file: + - ../envs/.env.db + - ../envs/.env.db-utils + - ../envs/.env.cloudprovider + + ###################################################### + ### NGINX container for serving files + ###################################################### + planet-files: + image: osmseed-planet-files:v1 + build: + context: ../images/planet-files + dockerfile: Dockerfile + ports: + - "8081:80" + - "3000:3000" + volumes: + # Serve minute replication files + - ../data/replication-job-data:/usr/share/nginx/html/server/static-files/replication/minute + # Serve planet dump files + - ../data/planet-dump-data:/usr/share/nginx/html/server/static-files/planet + # Serve full history planet file + - '.../data/full-history-data:/usr/share/nginx/html/server/static-files/full-planet' + - '../data/changeset-replication-job-data:/usr/share/nginx/html/server/static-files/changesets' + # Development mode + # - ./../images/planet-files/:/usr/share/nginx/html/ diff --git a/compose/replication.yml b/compose/replication.yml deleted file mode 100644 index 6cdba218..00000000 --- a/compose/replication.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '3' -services: - ###################################################### - ### OSM minute replication files section - ###################################################### - replication-job: - image: osmseed-replication-job:v1 - build: - context: ../images/replication-job - dockerfile: Dockerfile - volumes: - - ../data/replication-job-data:/mnt/data - command: > - /bin/bash -c " echo Sleep the replication-job for 1 minute; sleep 1m; echo Creating the replication files!; /start.sh " - env_file: - - ../envs/.env.db - - ../envs/.env.db-utils - - ../envs/.env.cloudprovider - changeset-replication-job: - image: osmseed-changeset-replication-job:v1 - build: - context: ../images/changeset-replication-job - dockerfile: Dockerfile - volumes: - - ../data/changeset-replication-job-data:/mnt/changesets - # - ./../images/changeset-replication-job:/openstreetmap-changeset-replication - command: > - /bin/bash -c "./start.sh" - env_file: - - ../envs/.env.db - - ../envs/.env.db-utils - - ../envs/.env.cloudprovider diff --git a/compose/web.yml b/compose/web.yml index 1b90b4b1..ef571d6c 100644 --- a/compose/web.yml +++ b/compose/web.yml @@ -10,20 +10,20 @@ services: context: ../images/db dockerfile: Dockerfile ports: - - '5432:5432' - volumes: - - ../data/db-data:/var/lib/postgresql/data + - '5432:5432' + # volumes: + # - ../data/db-data:/var/lib/postgresql/data env_file: - - ../envs/.env.db + - ../envs/.env.db web: image: osmseed-web:v1 build: context: ../images/web dockerfile: Dockerfile ports: - - '80:80' + - '80:80' env_file: - - ../envs/.env.web - - ../envs/.env.db + - ../envs/.env.web + - ../envs/.env.db depends_on: - - db + - db diff --git a/images/full-history/README.md b/images/full-history/README.md index 58900aa2..0d841e22 100644 --- a/images/full-history/README.md +++ b/images/full-history/README.md @@ -12,18 +12,8 @@ In order to run this container we need environment variables, these can be found **Note**: Rename the above files as `.env.db`, `.env.db-utils` and `.env.cloudprovider` -#### Running full-history container - +### Build and bring up the container ```sh - # Docker compose - docker-compose run full-history - - # Docker - docker run \ - --env-file ./envs/.env.db \ - --env-file ./envs/.env.full-history \ - --env-file ./envs/.env.cloudprovider \ - -v ${PWD}/data/full-history-data:/mnt/data \ - --network osm-seed_default \ - -it osmseed-full-history:v1 +docker compose -f ./compose/planet.yml build +docker compose -f ./compose/planet.yml up full-history ``` diff --git a/images/planet-dump/README.md b/images/planet-dump/README.md index bc6f7031..bbb7ea17 100644 --- a/images/planet-dump/README.md +++ b/images/planet-dump/README.md @@ -12,18 +12,8 @@ In order to run this container we need environment variables, these can be found **Note**: Rename the above files as `.env.db`, `.env.db-utils` and `.env.cloudprovider` -#### Running planet-dump container - +### Build and bring up the container ```sh - # Docker compose - docker-compose run planet-dump - - # Docker - docker run \ - --env-file ./envs/.env.db \ - --env-file ./envs/.env.planet-dump \ - --env-file ./envs/.env.cloudprovider \ - -v ${PWD}/data/planet-dump-data:/mnt/data \ - --network osm-seed_default \ - -it osmseed-planet-dump:v1 +docker compose -f ./compose/planet.yml build +docker compose -f ./compose/planet.yml up planet-dump ``` diff --git a/images/planet-files/Dockerfile b/images/planet-files/Dockerfile new file mode 100644 index 00000000..4f9aa907 --- /dev/null +++ b/images/planet-files/Dockerfile @@ -0,0 +1,10 @@ +FROM nginx:latest +RUN apt-get update && \ + apt-get install -y nodejs npm && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /usr/share/nginx/html/server/ +COPY server/package.json ./ +RUN npm install +COPY server/server.js ./ +COPY index.html /usr/share/nginx/html/ +CMD ["sh", "-c", "nginx -g 'daemon off;' & node server.js"] diff --git a/images/planet-files/README.MD b/images/planet-files/README.MD new file mode 100644 index 00000000..79407e01 --- /dev/null +++ b/images/planet-files/README.MD @@ -0,0 +1,11 @@ + +# Planet server files + +This container is based on Nginx and serves data related to planet, replication, and changesets for easy access and download, similar to how it is done in OpenStreetMap. + + +# Build and bring up the container +```sh +docker compose -f ./compose/planet.yml build +docker compose -f ./compose/planet.yml up planet-files +``` \ No newline at end of file diff --git a/images/planet-files/index.html b/images/planet-files/index.html new file mode 100644 index 00000000..b1b9a205 --- /dev/null +++ b/images/planet-files/index.html @@ -0,0 +1,105 @@ + + + + + + Planet Replication + + + +
+ + + diff --git a/images/planet-files/server/package.json b/images/planet-files/server/package.json new file mode 100644 index 00000000..f91d0fca --- /dev/null +++ b/images/planet-files/server/package.json @@ -0,0 +1,12 @@ +{ + "name": "websocket-server", + "version": "1.0.0", + "main": "server.js", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "cors": "^2.8.5", + "express": "^4.17.1" + } +} diff --git a/images/planet-files/server/server.js b/images/planet-files/server/server.js new file mode 100644 index 00000000..227a23b8 --- /dev/null +++ b/images/planet-files/server/server.js @@ -0,0 +1,49 @@ +const express = require("express"); +const cors = require("cors"); +const fs = require("fs"); +const path = require("path"); +const app = express(); +const port = 3000; + +app.use(cors()); +app.get("/:directory(*)", (req, res) => { + const directory = req.params.directory; + const files = getFilesRecursive(path.join("./static-files", directory)); + res.json(files); +}); + +function formatSize(sizeInBytes) { + const KB = 1024; + const MB = KB * 1024; + if (sizeInBytes < KB) { + return (sizeInBytes / KB).toFixed(1) + " kB"; + } else if (sizeInBytes < MB) { + const sizeInKB = (sizeInBytes / KB).toFixed(2); + return sizeInKB + " kB"; + } else { + const sizeInMB = (sizeInBytes / MB).toFixed(2); + return sizeInMB + " mB"; + } + } + +function getFilesRecursive(dir) { + const files = fs.readdirSync(dir); + return files.map((file) => { + const filePath = path.join(dir, file); + const stats = fs.statSync(filePath); + const isDirectory = stats.isDirectory(); + const size = isDirectory ? null : formatSize(stats.size); + const creationDate = isDirectory ? null : stats.mtime.toISOString(); + return { + name: file, + isDirectory, + path: filePath, + size, + creationDate, + }; + }); +} + +app.listen(port, () => { + console.log(`Server is listening at http://localhost:${port}`); +}); diff --git a/images/replication-job/Dockerfile b/images/replication-job/Dockerfile index 2b90ac07..8683fb42 100644 --- a/images/replication-job/Dockerfile +++ b/images/replication-job/Dockerfile @@ -1,7 +1,12 @@ FROM developmentseed/osmseed-osm-processor:v2 +# Install Nginx +RUN apt-get update && \ + apt-get install -y nginx && \ + rm -rf /var/lib/apt/lists/* + RUN pip3 install boto3 -COPY ./start.sh / +COPY ./*.sh / COPY monitoring.py / WORKDIR /mnt/data CMD /start.sh diff --git a/images/replication-job/start.sh b/images/replication-job/start.sh index 0435ca85..5653eb7a 100755 --- a/images/replication-job/start.sh +++ b/images/replication-job/start.sh @@ -11,44 +11,101 @@ fi workingDirectory="/mnt/data" mkdir -p $workingDirectory -# Check if state.txt exist in the workingDirectory, -# in case the file does not exist locally and does not exist in the cloud the replication will start from 0 -if [ ! -f $workingDirectory/state.txt ]; then - echo "File $workingDirectory/state.txt does not exist in local storage" - ### AWS - if [ $CLOUDPROVIDER == "aws" ]; then - aws s3 ls $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt - if [[ $? -eq 0 ]]; then - echo "File exist, let's get it from $CLOUDPROVIDER - $AWS_S3_BUCKET" - aws s3 cp $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt +# Remove files that are not required +[ -e /mnt/data/replicate.lock ] && rm -f /mnt/data/replicate.lock +# [ -e /mnt/data/processed_files.log ] && rm -f /mnt/data/processediles.log + +function get_current_state_file() { + # Check if state.txt exist in the workingDirectory, + # in case the file does not exist locally and does not exist in the cloud the replication will start from 0 + if [ ! -f $workingDirectory/state.txt ]; then + echo "File $workingDirectory/state.txt does not exist in local storage" + ### AWS + if [ $CLOUDPROVIDER == "aws" ]; then + aws s3 ls $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt + if [[ $? -eq 0 ]]; then + echo "File exist, let's get it from $CLOUDPROVIDER - $AWS_S3_BUCKET" + aws s3 cp $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt + fi fi - fi - - ### GCP - if [ $CLOUDPROVIDER == "gcp" ]; then - gsutil ls $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt - if [[ $? -eq 0 ]]; then - echo "File exist, let's get it from $CLOUDPROVIDER - $GCP_STORAGE_BUCKET" - gsutil cp $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt + + ### GCP + if [ $CLOUDPROVIDER == "gcp" ]; then + gsutil ls $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt + if [[ $? -eq 0 ]]; then + echo "File exist, let's get it from $CLOUDPROVIDER - $GCP_STORAGE_BUCKET" + gsutil cp $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt + fi fi - fi - - ### Azure - if [ $CLOUDPROVIDER == "azure" ]; then - state_file_exists=$(az storage blob exists --container-name $AZURE_CONTAINER_NAME --name $REPLICATION_FOLDER/state.txt --query="exists") - if [[ $state_file_exists=="true" ]]; then - echo "File exist, let's get it from $CLOUDPROVIDER - $AZURE_CONTAINER_NAME" - az storage blob download \ - --container-name $AZURE_CONTAINER_NAME \ - --name $REPLICATION_FOLDER/state.txt \ - --file $workingDirectory/state.txt --query="name" + + ### Azure + if [ $CLOUDPROVIDER == "azure" ]; then + state_file_exists=$(az storage blob exists --container-name $AZURE_CONTAINER_NAME --name $REPLICATION_FOLDER/state.txt --query="exists") + if [[ $state_file_exists=="true" ]]; then + echo "File exist, let's get it from $CLOUDPROVIDER - $AZURE_CONTAINER_NAME" + az storage blob download \ + --container-name $AZURE_CONTAINER_NAME \ + --name $REPLICATION_FOLDER/state.txt \ + --file $workingDirectory/state.txt --query="name" + fi fi fi - mkdir -p $workingDirectory -fi +} + +function upload_file_cloud() { + # Upload files to cloud provider + local local_file="$1" + local cloud_file="$REPLICATION_FOLDER/${local_file#*"$workingDirectory/"}" + echo "$(date +%F_%H:%M:%S): Upload file $local_file to ...$CLOUDPROVIDER...$cloud_file" + if [ "$CLOUDPROVIDER" == "aws" ]; then + aws s3 cp "$local_file" "$AWS_S3_BUCKET/$cloud_file" --acl public-read + elif [ "$CLOUDPROVIDER" == "gcp" ]; then + gsutil cp -a public-read "$local_file" "$GCP_STORAGE_BUCKET/$cloud_file" + elif [ "$CLOUDPROVIDER" == "azure" ]; then + az storage blob upload \ + --container-name "$AZURE_CONTAINER_NAME" \ + --file "$local_file" \ + --name "$cloud_file" \ + --output none + fi +} + +function monitor_minute_replication() { + # Function to handle continuous monitoring, minutminutes replication and upload to cloud provider + # Directory to store a log of processed files + processed_files_log="$workingDirectory/processed_files.log" + max_log_size_mb=1 + + while true; do + upload_file_cloud /mnt/data/state.txt + sleep 60s + done & -# Creating the replication files -function generateReplication() { + while true; do + if [ -e "$processed_files_log" ]; then + log_size=$(du -m "$processed_files_log" | cut -f1) + if [ "$log_size" -gt "$max_log_size_mb" ]; then + echo $(date +%F_%H:%M:%S)": Cleaning processed_files_log..." >"$processed_files_log" + fi + for local_minute_file in $(find $workingDirectory/ -cmin -1); do + if [ -f "$local_minute_file" ]; then + if grep -q "$local_minute_file" "$processed_files_log"; then + continue + fi + upload_file_cloud $local_minute_file + echo "$local_minute_file" >>"$processed_files_log" + fi + done + else + echo "File $processed_files_log not found." + echo $processed_files_log >$processed_files_log + fi + sleep 10s + done +} + +function generate_replication() { + # Replicate the API database using Osmosis osmosis -q \ --replicate-apidb \ iterations=0 \ @@ -60,45 +117,32 @@ function generateReplication() { password=$POSTGRES_PASSWORD \ validateSchemaVersion=no \ --write-replication \ - workingDirectory=$workingDirectory & - while true; do - for local_file in $(find $workingDirectory/ -cmin -1); do - if [ -f "$local_file" ]; then - - cloud_file=$REPLICATION_FOLDER/${local_file#*"$workingDirectory/"} - echo $(date +%F_%H:%M:%S)": Copy file...$local_file to $cloud_file" - - ### AWS - if [ $CLOUDPROVIDER == "aws" ]; then - aws s3 cp $local_file $AWS_S3_BUCKET/$cloud_file --acl public-read - fi - - ### GCP - if [ $CLOUDPROVIDER == "gcp" ]; then - #TODO, emable public acces - gsutil cp -a public-read $local_file $GCP_STORAGE_BUCKET/$cloud_file - fi - - ### Azure - if [ $CLOUDPROVIDER == "azure" ]; then - #TODO, emable public acces - az storage blob upload \ - --container-name $AZURE_CONTAINER_NAME \ - --file $local_file \ - --name $cloud_file \ - --output none - fi - fi - done - sleep 15s - done + workingDirectory=$workingDirectory } -# Check if Postgres is ready +# function start_nginx() { +# if [ "$STAR_NGINX_SERVER" = "true" ]; then +# echo 'server { +# listen 8080; +# server_name localhost; + +# location / { +# root /mnt/data; +# index index.html; +# } +# }' >/etc/nginx/nginx.conf +# service nginx restart +# else +# echo "STAR_NGINX_SERVER is either not set or not set to true." +# fi +# } + +######################## Start minutes replication process ######################## +get_current_state_file flag=true while "$flag" = true; do pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue - # Change flag to false to stop ping the DB flag=false - generateReplication + generate_replication & + monitor_minute_replication done diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index b9bfc19d..122c95e3 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -755,7 +755,7 @@ changesetReplicationJob: # Variables for osmcha web builder # ==================================================================================================== osmchaWeb: - enabled: true + enabled: false image: name: "" tag: "" @@ -763,7 +763,7 @@ osmchaWeb: # Variables for osmcha Api # ==================================================================================================== osmchaApi: - enabled: true + enabled: false image: name: "ghcr.io/willemarcel/osmcha-django" tag: "10d7f6748d9e55d439339ed2c35509db15f782a6" @@ -804,7 +804,7 @@ osmchaApi: # Variables for osmcha DB # ==================================================================================================== osmchaDb: - enabled: true + enabled: false image: name: "" tag: "" @@ -835,3 +835,12 @@ osmchaDb: GCP_gcePersistentDisk_size: 50Gi nodeSelector: enabled: false + +# ==================================================================================================== +# Planet server files +# ==================================================================================================== +planetFiles: + enabled: false + image: + name: "" + tag: "" From 5f1dfdd63f2d15c0a920b30a47e04f450e009705 Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Thu, 14 Mar 2024 15:53:39 -0500 Subject: [PATCH 09/23] Update web container and TM config (#315) * Update config for TM to use a db * Update web container * Update config to set up web and tm * Update env vars for web and tm * Restart containers * Update port for planet api * Update tiler containers * Update config * Update compose config * Delete tasking manager web * Update website version * Update config * Generate doorkeeper_signing_key and set in config * Remove upload to s3 the logs from restore db * Add env var to change org name * Disable cgimap * Update the private key creation * Comment cgimap cli * Update private key creation * Update website version * Update api file service * Update api db --- compose/nominatim.yml | 3 +- compose/planet.yml | 10 +- compose/tasking-manager.yml | 33 ++++-- compose/tiler.yml | 10 +- compose/web.yml | 15 ++- envs/.env.tasking-manager-db.example | 4 + envs/.env.tasking-manager.example | 11 +- envs/.env.web.example | 17 ++- images/backup-restore/start.sh | 2 +- images/db/Dockerfile | 17 +-- images/planet-files/index.html | 2 +- images/tiler-server/Dockerfile | 4 +- images/tiler-server/expire-watcher.sh | 2 +- images/tiler-server/seed-by-diffs.sh | 45 +++----- images/web/Dockerfile | 117 ++++++++------------- images/web/config/cgimap.conf | 43 -------- images/web/config/production.conf | 42 +++++++- images/web/config/settings.yml | 78 ++++++++++---- images/web/start.sh | 52 +++++++-- osm-seed/templates/web/web-deployment.yaml | 2 + osm-seed/values.yaml | 1 + 21 files changed, 277 insertions(+), 233 deletions(-) create mode 100644 envs/.env.tasking-manager-db.example delete mode 100644 images/web/config/cgimap.conf diff --git a/compose/nominatim.yml b/compose/nominatim.yml index cb0a1c07..ff3349d2 100644 --- a/compose/nominatim.yml +++ b/compose/nominatim.yml @@ -10,9 +10,10 @@ services: dockerfile: Dockerfile ports: - '8080:8080' - - '5432:5432' + # - '5432:5432' volumes: - ../data/nominatim-data:/var/lib/postgresql/14/main # - ../../Nominatim/nominatim/tokenizer/sanitizers:/usr/local/lib/nominatim/lib-python/nominatim/tokenizer/sanitizers env_file: - ../envs/.env.nominatim + restart: always diff --git a/compose/planet.yml b/compose/planet.yml index bb0e31c9..c16f9990 100644 --- a/compose/planet.yml +++ b/compose/planet.yml @@ -16,6 +16,7 @@ services: - ../envs/.env.db - ../envs/.env.db-utils - ../envs/.env.cloudprovider + restart: always ###################################################### ### Planet replication section @@ -32,6 +33,7 @@ services: - ../envs/.env.db - ../envs/.env.db-utils - ../envs/.env.cloudprovider + restart: always ##################################################### ## OSM full planet replication @@ -48,6 +50,7 @@ services: - ../envs/.env.db - ../envs/.env.db-utils - ../envs/.env.cloudprovider + restart: always ##################################################### ## Changeset replications @@ -65,6 +68,7 @@ services: - ../envs/.env.db - ../envs/.env.db-utils - ../envs/.env.cloudprovider + restart: always ###################################################### ### NGINX container for serving files @@ -76,7 +80,7 @@ services: dockerfile: Dockerfile ports: - "8081:80" - - "3000:3000" + - "8082:3000" volumes: # Serve minute replication files - ../data/replication-job-data:/usr/share/nginx/html/server/static-files/replication/minute @@ -87,3 +91,7 @@ services: - '../data/changeset-replication-job-data:/usr/share/nginx/html/server/static-files/changesets' # Development mode # - ./../images/planet-files/:/usr/share/nginx/html/ + restart: always +networks: + osm-seed: + driver: bridge \ No newline at end of file diff --git a/compose/tasking-manager.yml b/compose/tasking-manager.yml index 178e87cb..d61b52e0 100644 --- a/compose/tasking-manager.yml +++ b/compose/tasking-manager.yml @@ -1,16 +1,33 @@ version: '3' services: - ###################################################### - ### Tasking Manager section - ###################################################### - tasking-manager-api: + tmdb: + image: mdillon/postgis:11 + # ports: + # - '5432:5432' + volumes: + - ../data/tasking-manager-db:/db + env_file: + - ../envs/.env.tasking-manager-db + restart: always + migration: image: osmseed-tasking-manager-api:v1 build: context: ../images/tasking-manager-api dockerfile: Dockerfile + restart: on-failure + command: > + sh -c "sleep 10s && python3 manage.py db upgrade" + env_file: + - ../envs/.env.tasking-manager + - ../envs/.env.tasking-manager-db + tmapi: + build: + context: ../images/tasking-manager-api + dockerfile: Dockerfile + image: osmseed-tasking-manager-api:v1 ports: - - '5000:5000' - volumes: - - ../data/tasking-manager-api-db:/db + - '5000:5000' env_file: - - ../envs/.env.tasking-manager \ No newline at end of file + - ../envs/.env.tasking-manager + - ../envs/.env.tasking-manager-db + restart: always diff --git a/compose/tiler.yml b/compose/tiler.yml index b94acd66..11d15502 100644 --- a/compose/tiler.yml +++ b/compose/tiler.yml @@ -8,8 +8,8 @@ services: build: context: ../images/tiler-db dockerfile: Dockerfile - ports: - - "5433:5432" + # ports: + # - "5433:5432" volumes: - ../data/tiler-db-data:/var/lib/postgresql/data env_file: @@ -42,7 +42,7 @@ services: retries: 3 restart: always # ###################################################### - # ### Tiler imposm for importing data + # ### Tiler server # ###################################################### tiler-server: image: osmseed-tiler-server:v1 @@ -51,6 +51,7 @@ services: dockerfile: Dockerfile volumes: - ../data/tiler-server-data:/mnt/data + - ../data/tiler-imposm-data:/mnt/data/imposm depends_on: - tiler-db - tiler-imposm @@ -59,8 +60,9 @@ services: command: - sh - -c - - "sleep 60 && ./start.sh" + - "./start.sh & ./expire-watcher.sh" env_file: - ../envs/.env.tiler-db - ../envs/.env.tiler-server restart: always + \ No newline at end of file diff --git a/compose/web.yml b/compose/web.yml index ef571d6c..cc6bbf10 100644 --- a/compose/web.yml +++ b/compose/web.yml @@ -9,12 +9,13 @@ services: build: context: ../images/db dockerfile: Dockerfile - ports: - - '5432:5432' - # volumes: - # - ../data/db-data:/var/lib/postgresql/data + # ports: + # - '5432:5432' + volumes: + - ../data/db-data-pg14:/var/lib/postgresql/data env_file: - ../envs/.env.db + restart: always web: image: osmseed-web:v1 build: @@ -27,3 +28,9 @@ services: - ../envs/.env.db depends_on: - db + restart: always + memcache: + image: memcached:latest + ports: + - '11211:11211' + restart: always diff --git a/envs/.env.tasking-manager-db.example b/envs/.env.tasking-manager-db.example new file mode 100644 index 00000000..9176a7d1 --- /dev/null +++ b/envs/.env.tasking-manager-db.example @@ -0,0 +1,4 @@ +POSTGRES_DB=tm +POSTGRES_USER=postgres +POSTGRES_PASSWORD=1234 +POSTGRES_PORT=5432 \ No newline at end of file diff --git a/envs/.env.tasking-manager.example b/envs/.env.tasking-manager.example index 902a262c..5887b66e 100644 --- a/envs/.env.tasking-manager.example +++ b/envs/.env.tasking-manager.example @@ -37,20 +37,15 @@ POTLATCH2_EDITOR_URL=https://www.openstreetmap.org/edit?editor=potlatch2 # TM_SECRET=xyz -# OpenStreetMap OAuth consumer key and secret (required) -# +# OpenStreetMap OAuth 1 consumer key and secret (required) + TM_CONSUMER_KEY=foo TM_CONSUMER_SECRET=xyz TM_DEFAULT_CHANGESET_COMMENT="#{osmseed}-project" # The connection to the postgres database (required) -# -POSTGRES_DB=tm -POSTGRES_USER=postgres -POSTGRES_PASSWORD=tm -POSTGRES_ENDPOINT=localhost -POSTGRES_PORT=5432 +POSTGRES_HOST=tmdb # The address to use as the sender on auto generated emails (optional) # diff --git a/envs/.env.web.example b/envs/.env.web.example index 9c720436..81e62ab7 100644 --- a/envs/.env.web.example +++ b/envs/.env.web.example @@ -20,16 +20,27 @@ MAILER_USERNAME=test@gmail.com MAILER_PASSWORD=abc... MAILER_FROM=no-reply@osmseed.org MAILER_PORT=25 +OSM_memcache_servers="" # Nominatim settings NOMINATIM_URL=nominatim-api # Make sure that the first time this (OPENSTREETMAP_id_key) value is empty. -# Once you start you server and create your OAuth 1, fill the value with the Consumer Key. -OPENSTREETMAP_id_key="" -OSM_memcache_servers="" +# Once you start you server and create your OAuth 2. +### Create OAuth 2 applications +### Client ID=OPENSTREETMAP_id_key +OPENSTREETMAP_id_key="xyz..." + +### OpenStreetMap Web Site +# Client ID=OAUTH_CLIENT_ID +# Client Secret=OAUTH_KEY +OAUTH_CLIENT_ID="abc..." +OAUTH_KEY="efg..." # NEW_RELIC settings # TODO: Needs to be implemented the New Relic functions in order to make running in the web container NEW_RELIC_LICENSE_KEY="..." NEW_RELIC_APP_NAME="..." + +### Set organization name, by default : OpenStreetMap +ORGANIZATION_NAME=OSMSeed diff --git a/images/backup-restore/start.sh b/images/backup-restore/start.sh index 2648f321..e9fa1240 100755 --- a/images/backup-restore/start.sh +++ b/images/backup-restore/start.sh @@ -63,7 +63,7 @@ restoreDB() { wget -O ${RESTORE_FILE} ${RESTORE_URL_FILE} echo "Restoring ${RESTORE_URL_FILE} in ${POSTGRES_DB}" gunzip -c <${RESTORE_FILE} | psql -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DB} | tee ${LOG_RESULT_FILE} - aws s3 cp ${LOG_RESULT_FILE} s3://${AWS_S3_BUCKET}/${LOG_RESULT_FILE} + # aws s3 cp ${LOG_RESULT_FILE} s3://${AWS_S3_BUCKET}/${LOG_RESULT_FILE} echo "Import data to ${POSTGRES_DB} has finished ..." done } diff --git a/images/db/Dockerfile b/images/db/Dockerfile index c9a5c1e7..a7e6a566 100644 --- a/images/db/Dockerfile +++ b/images/db/Dockerfile @@ -1,25 +1,18 @@ -FROM postgres:11 -RUN rm /etc/apt/sources.list.d/pgdg.list -RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list -RUN sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list -RUN sed -i '/stretch-updates/d' /etc/apt/sources.list -RUN apt-get update && apt-get -y install apt-transport-https -RUN echo "deb [ signed-by=/usr/local/share/keyrings/postgres.gpg.asc ] https://apt-archive.postgresql.org/pub/repos/apt/ stretch-pgdg main 11" >/etc/apt/sources.list.d/pgdg.list +FROM postgres:14 RUN apt-get update \ && apt-get install -y \ - postgresql-server-dev-11 \ + postgresql-server-dev-14 \ make \ build-essential \ - postgresql-11-postgis-2.5 \ + postgresql-14-postgis-3 \ && apt-get clean && rm -rf /var/lib/apt/lists/* - + ADD functions/functions.sql /usr/local/share/osm-db-functions.sql ADD docker_postgres.sh /docker-entrypoint-initdb.d/ RUN mkdir -p db RUN mkdir -p lib ADD functions/ db/functions/ ADD lib/quad_tile/ lib/quad_tile/ - RUN make -C db/functions/ RUN chown -R postgres lib/ -RUN chown -R postgres db/ +RUN chown -R postgres db/ \ No newline at end of file diff --git a/images/planet-files/index.html b/images/planet-files/index.html index b1b9a205..4c1eac05 100644 --- a/images/planet-files/index.html +++ b/images/planet-files/index.html @@ -22,7 +22,7 @@