diff --git a/Dockerfile b/Dockerfile index 9787ab90a..e6afa3f86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,108 +1,106 @@ FROM nikolaik/python-nodejs:python3.8-nodejs10 +# Declare environment variables ENV DEBIAN_FRONTEND=noninteractive ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 -ENV VIRTUAL_ENV=/opt/venv - -ENV KOBOCAT_LOGS_DIR=/srv/logs \ - DJANGO_SETTINGS_MODULE=kobo.settings.prod \ +ENV VIRTUAL_ENV=/opt/venv \ + KOBOCAT_LOGS_DIR=/srv/logs \ + DJANGO_SETTINGS_MODULE=onadata.settings.prod \ # The mountpoint of a volume shared with the `nginx` container. Static files will - # be copied there. + # be copied there. NGINX_STATIC_DIR=/srv/static \ KOBOCAT_SRC_DIR=/srv/src/kobocat \ BACKUPS_DIR=/srv/backups \ - TMP_PATH=/srv/tmp \ + TMP_DIR=/srv/tmp \ + UWSGI_USER=kobo \ + UWSGI_GROUP=kobo \ + SERVICES_DIR=/etc/service \ + CELERY_PID_DIR=/var/run/celery \ INIT_PATH=/srv/init -# Install Dockerize. +# Install Dockerize ENV DOCKERIZE_VERSION v0.6.1 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -P /tmp \ && tar -C /usr/local/bin -xzvf /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -########################################## -# Create build directories # -########################################## - -RUN mkdir -p "${NGINX_STATIC_DIR}" && \ - mkdir -p "${KOBOCAT_SRC_DIR}" && \ - mkdir -p "${TMP_PATH}" && \ - mkdir -p "${BACKUPS_DIR}" && \ - mkdir -p "${INIT_PATH}" - -########################################## -# Install `apt` packages. # -########################################## +# Create needed directories +RUN mkdir -p ${NGINX_STATIC_DIR} && \ + mkdir -p ${KOBOCAT_SRC_DIR} && \ + mkdir -p ${TMP_DIR} && \ + mkdir -p ${BACKUPS_DIR} && \ + mkdir -p ${CELERY_PID_DIR} && \ + mkdir -p ${SERVICES_DIR}/uwsgi && \ + mkdir -p ${SERVICES_DIR}/uwsgi_wrong_port_warning && \ + mkdir -p ${SERVICES_DIR}/celery && \ + mkdir -p ${SERVICES_DIR}/celery_beat && \ + mkdir -p ${KOBOCAT_LOGS_DIR}/ && \ + mkdir -p ${KOBOCAT_SRC_DIR}/emails && \ + mkdir -p ${INIT_PATH} + +# Install `apt` packages. RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN apt -qq update && \ - apt -qq -y install \ +RUN apt-get -qq update && \ + apt-get -qq -y install \ gdal-bin \ libproj-dev \ gettext \ postgresql-client \ + openjdk-11-jre \ locales \ runit-init \ rsync \ - vim && \ - apt clean && \ + vim \ + gosu \ + cron && \ + apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -########################### -# Install locales # -########################### - -RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen -RUN locale-gen && dpkg-reconfigure locales -f noninteractive +# Install locales +RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && \ + locale-gen && dpkg-reconfigure locales -f noninteractive -########################### -# Copy KoBoCAT directory # -########################### +# Create local user UWSGI_USER` +RUN adduser --disabled-password --gecos '' "$UWSGI_USER" +# Copy KoBoCAT directory COPY . "${KOBOCAT_SRC_DIR}" -########################### -# Install `pip` packages. # -########################### - +# Install `pip` packages RUN virtualenv "$VIRTUAL_ENV" ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip install --quiet --upgrade pip && \ pip install --quiet pip-tools -COPY ./dependencies/pip/prod.txt /srv/tmp/pip_dependencies.txt -RUN pip-sync /srv/tmp/pip_dependencies.txt 1>/dev/null && \ +COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt" +RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null && \ rm -rf ~/.cache/pip -########################################## -# Persist the log and email directories. # -########################################## - -RUN mkdir -p "${KOBOCAT_LOGS_DIR}/" "${KOBOCAT_SRC_DIR}/emails" && \ - chown -R "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/emails/" && \ - chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}" - -################################################# -# Handle runtime tasks and create main process. # -################################################# - # Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides. -RUN echo "export PATH=${PATH}" >> /etc/profile -RUN echo 'source /etc/profile' >> /root/.bashrc - -# Prepare for execution. -RUN mkdir -p /etc/service/uwsgi_wrong_port_warning && \ - cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi_wrong_port_warning.bash" /etc/service/uwsgi_wrong_port_warning/run && \ - mkdir -p /etc/service/uwsgi && \ - # Remove getty* services - rm -rf /etc/runit/runsvdir/default/getty-tty* && \ - cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash" /etc/service/uwsgi/run && \ - mkdir -p /etc/service/celery && \ - ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery.bash" /etc/service/celery/run && \ - mkdir -p /etc/service/celery_beat && \ - ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash" /etc/service/celery_beat/run && \ +RUN echo "export PATH=${PATH}" >> /etc/profile && \ + echo 'source /etc/profile' >> /root/.bashrc && \ + echo 'source /etc/profile' >> /home/${UWSGI_USER}/.bashrc + +# Remove getty* services to avoid errors of absent tty at sv start-up +RUN rm -rf /etc/runit/runsvdir/default/getty-tty* + +# Create symlinks for runsv services +RUN ln -s "${KOBOCAT_SRC_DIR}/docker/run_uwsgi_wrong_port_warning.bash" "${SERVICES_DIR}/uwsgi_wrong_port_warning/run" && \ + ln -s "${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash" "${SERVICES_DIR}/uwsgi/run" && \ + ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery.bash" "${SERVICES_DIR}/celery/run" && \ + ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash" "${SERVICES_DIR}/celery_beat/run" + +# Add/Restore `UWSGI_USER`'s permissions +RUN chown -R ":${UWSGI_GROUP}" ${CELERY_PID_DIR} && \ + chmod g+w ${CELERY_PID_DIR} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_SRC_DIR}/emails/ && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_LOGS_DIR} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${TMP_DIR} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${VIRTUAL_ENV} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${BACKUPS_DIR} WORKDIR "${KOBOCAT_SRC_DIR}" diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt index bd146cbe4..59256fab4 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev.txt @@ -36,10 +36,11 @@ django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in django-registration-redux==2.8 # via -r dependencies/pip/requirements.in django-render-block==0.7 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-storages==1.9.1 # via -r dependencies/pip/requirements.in django-taggit==1.3.0 # via -r dependencies/pip/requirements.in django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in @@ -59,7 +60,6 @@ gitdb==4.0.5 # via gitpython gitpython==3.1.7 # via transifex-client httmock==1.3.0 # via -r dependencies/pip/dev.in idna==2.10 # via requests -importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path, pluggy, pytest ipdb==0.13.3 # via -r dependencies/pip/dev.in ipython-genutils==0.2.0 # via traitlets ipython==7.16.1 # via -r dependencies/pip/dev.in, ipdb @@ -82,7 +82,7 @@ openpyxl==3.0.4 # via -r dependencies/pip/requirements.in packaging==20.4 # via pytest pandas==1.0.5 # via -r dependencies/pip/requirements.in parso==0.7.0 # via jedi -path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack +path.py==12.4.0 # via formpack path==13.1.0 # via path.py pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython @@ -130,7 +130,6 @@ werkzeug==1.0.1 # via -r dependencies/pip/dev.in xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform xlsxwriter==1.2.9 # via formpack xlwt==1.3.0 # via -r dependencies/pip/requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt index f58b461bd..73e534ce2 100644 --- a/dependencies/pip/prod.txt +++ b/dependencies/pip/prod.txt @@ -33,10 +33,11 @@ django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in django-registration-redux==2.8 # via -r dependencies/pip/requirements.in django-render-block==0.7 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-storages==1.9.1 # via -r dependencies/pip/requirements.in django-taggit==1.3.0 # via -r dependencies/pip/requirements.in django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in @@ -55,7 +56,6 @@ geojson-rewind==0.2.0 # via formpack gitdb==4.0.5 # via gitpython gitpython==3.1.7 # via transifex-client idna==2.10 # via requests -importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path jdcal==1.4.1 # via openpyxl jsonfield==3.1.0 # via -r dependencies/pip/requirements.in jsonschema==3.2.0 # via formpack @@ -68,8 +68,8 @@ numpy==1.19.0 # via pandas oauthlib==3.1.0 # via django-oauth-toolkit openpyxl==3.0.4 # via -r dependencies/pip/requirements.in pandas==1.0.5 # via -r dependencies/pip/requirements.in -path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack -path==13.1.0 # via path.py +path.py==12.5.0 # via formpack +path==15.0.0 # via path.py pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in pymongo==3.10.1 # via -r dependencies/pip/requirements.in @@ -96,11 +96,11 @@ transifex-client==0.13.11 # via -r dependencies/pip/requirements.in unicodecsv==0.14.1 # via djangorestframework-csv, pyxform unittest2==1.1.0 # via pyxform urllib3==1.25.9 # via requests, transifex-client +uwsgi==2.0.19.1 # via -r dependencies/pip/prod.in vine==1.3.0 # via amqp, celery xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform xlsxwriter==1.2.9 # via formpack xlwt==1.3.0 # via -r dependencies/pip/requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 09d8e92dc..6b2ccbb54 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -33,10 +33,11 @@ django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in django-registration-redux==2.8 # via -r dependencies/pip/requirements.in django-render-block==0.7 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-storages==1.9.1 # via -r dependencies/pip/requirements.in django-taggit==1.3.0 # via -r dependencies/pip/requirements.in django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in @@ -55,7 +56,6 @@ geojson-rewind==0.2.0 # via formpack gitdb==4.0.5 # via gitpython gitpython==3.1.7 # via transifex-client idna==2.10 # via requests -importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path jdcal==1.4.1 # via openpyxl jsonfield==3.1.0 # via -r dependencies/pip/requirements.in jsonschema==3.2.0 # via formpack @@ -68,7 +68,7 @@ numpy==1.19.0 # via pandas oauthlib==3.1.0 # via django-oauth-toolkit openpyxl==3.0.4 # via -r dependencies/pip/requirements.in pandas==1.0.5 # via -r dependencies/pip/requirements.in -path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack +path.py==12.4.0 # via formpack path==13.1.0 # via path.py pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in @@ -100,7 +100,6 @@ vine==1.3.0 # via amqp, celery xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform xlsxwriter==1.2.9 # via formpack xlwt==1.3.0 # via -r dependencies/pip/requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/docker/init.bash b/docker/init.bash index 48e903295..0b635530c 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -13,34 +13,21 @@ if [[ -z $DATABASE_URL ]]; then exit 1 fi -echo 'Running migrations...' -python manage.py migrate --noinput - -rm -f /etc/cron.d/clean_up_tmp -cp docker/cron/clean_up_tmp /etc/cron.d/ -echo 'KoBoCat tmp clean-up cron installed' - -rm -f /etc/cron.d/backup_media_crontab -if [[ -z "${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" ]]; then - echo 'KoBoCAT media automatic backups disabled.' -else - # Should we first validate the schedule e.g. with `chkcrontab`? - cat "${KOBOCAT_SRC_DIR}/docker/backup_media_crontab.envsubst" | envsubst > /etc/cron.d/backup_media_crontab - echo "KoBoCAT media automatic backup schedule: ${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" -fi +# Wait for databases to be up & running before going further +/bin/bash "${INIT_PATH}/wait_for_mongo.bash" +/bin/bash "${INIT_PATH}/wait_for_postgres.bash" -/bin/bash ${KOBOCAT_SRC_DIR}/docker/sync_static.sh +echo 'Running migrations...' +gosu "${UWSGI_USER}" python manage.py migrate --noinput -# Keep it as is, not tested with Python3 -rm -rf /etc/profile.d/pydev_debugger.bash.sh -if [[ -d /srv/pydev_orig && -n "${KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then - echo 'Enabling PyDev remote debugging.' - "${KOBOCAT_SRC_DIR}/docker/setup_pydev.bash" -fi +echo 'Setting up cron tasks...' +/bin/bash "${KOBOCAT_SRC_DIR}/docker/setup_cron.bash" +/bin/bash "${KOBOCAT_SRC_DIR}/docker/setup_pydev_debugger.bash" +/bin/bash "${KOBOCAT_SRC_DIR}/docker/sync_static.bash" echo 'Cleaning up Celery PIDs...' -rm -rf /tmp/celery*.pid +rm -rf ${CELERY_PID_DIR}/*.pid echo 'KoBoCAT initialization complete.' -exec /usr/bin/runsvdir /etc/service +exec /usr/bin/runsvdir "${SERVICES_DIR}" diff --git a/docker/run_celery.bash b/docker/run_celery.bash index d043a82a9..f5af0deef 100755 --- a/docker/run_celery.bash +++ b/docker/run_celery.bash @@ -4,7 +4,9 @@ source /etc/profile # Run the main Celery worker cd "${KOBOCAT_SRC_DIR}" -exec /sbin/setuser "${UWSGI_USER}" celery worker -A onadata -Ofair --loglevel=info \ +exec celery worker -A onadata -Ofair --loglevel=info \ --hostname=kobocat_main_worker@%h \ --logfile=${KOBOCAT_LOGS_DIR}/celery.log \ - --pidfile=/tmp/celery.pid + --pidfile=${CELERY_PID_DIR}/celery.pid \ + --uid=${UWSGI_USER} \ + --gid=${UWSGI_GROUP} diff --git a/docker/run_celery_beat.bash b/docker/run_celery_beat.bash index e777dff6c..faf270444 100755 --- a/docker/run_celery_beat.bash +++ b/docker/run_celery_beat.bash @@ -4,7 +4,9 @@ source /etc/profile # Run the main Celery beat worker cd "${KOBOCAT_SRC_DIR}" -exec /sbin/setuser "${UWSGI_USER}" celery beat -A onadata --loglevel=info \ +exec celery beat -A onadata --loglevel=info \ --logfile=${KOBOCAT_LOGS_DIR}/celery_beat.log \ - --pidfile=/tmp/celery_beat.pid \ - --scheduler django_celery_beat.schedulers:DatabaseScheduler + --pidfile=${CELERY_PID_DIR}/celery_beat.pid \ + --scheduler django_celery_beat.schedulers:DatabaseScheduler \ + --uid=${UWSGI_USER} \ + --gid=${UWSGI_GROUP} diff --git a/docker/run_uwsgi.bash b/docker/run_uwsgi.bash index b3ac46c51..463f4e810 100755 --- a/docker/run_uwsgi.bash +++ b/docker/run_uwsgi.bash @@ -1,41 +1,55 @@ #!/usr/bin/env bash -set -e source /etc/profile - if [[ "$(stat -c '%U' ${KOBOCAT_LOGS_DIR})" != "${UWSGI_USER}" ]]; then echo 'Restoring ownership of Logs directory.' - chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}" + chown -R "${UWSGI_USER}":"${UWSGI_GROUP}" "${KOBOCAT_LOGS_DIR}" fi KOBOCAT_WEB_SERVER="${KOBOCAT_WEB_SERVER:-uWSGI}" -uwsgi_command="/sbin/setuser ${UWSGI_USER} $(command -v uwsgi) --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]]; then cd "${KOBOCAT_SRC_DIR}" - DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "/srv/tmp/pip_dependencies.txt") + DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt") if [[ -n "$DIFF" ]]; then + # Celery services need to be stopped to avoid raising errors + # during pip-sync + echo "Stopping Celery services..." + sv stop celery + sv stop celery_beat echo "Syncing pip dependencies..." pip-sync dependencies/pip/prod.txt 1>/dev/null - cp "dependencies/pip/prod.txt" "/srv/tmp/pip_dependencies.txt" + cp "dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" + echo "Restarting Celery..." + sv start celery + sv start celery_beat fi - echo 'Running `kobocat` container with uWSGI application server.' - exec ${uwsgi_command} + echo "Running \`KoBoCAT\` container with uWSGI application server." + UWSGI_COMMAND="$(command -v uwsgi) --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" else cd "${KOBOCAT_SRC_DIR}" - DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "/srv/tmp/pip_dependencies.txt") + DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt") if [[ -n "$DIFF" ]]; then + # Celery services need to be stopped to avoid raising errors during + # pip-sync + echo "Stopping Celery services..." + sv stop celery + sv stop celery_beat echo "Syncing pip dependencies..." - pip-sync dependencies/pip/dev.txt 1>/dev/null - cp "dependencies/pip/dev.txt" "/srv/tmp/pip_dependencies.txt" + gosu "${UWSGI_USER}" pip-sync dependencies/pip/dev.txt 1>/dev/null + cp "dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" + echo "Restarting Celery..." + sv start celery + sv start celery_beat fi if [[ -n "$RAVEN_DSN" ]]; then echo "Sentry detected. Installing \`raven\` pip dependency..." - pip install raven + gosu "${UWSGI_USER}" pip install raven fi - echo 'Running `kobocat` container with `runserver_plus` debugging application server.' - exec python manage.py runserver_plus 0:8001 + echo "Running KoBoCAT container with \`runserver_plus\` debugging application server." + UWSGI_COMMAND="gosu $UWSGI_USER python manage.py runserver_plus 0:8001" fi +exec ${UWSGI_COMMAND} diff --git a/docker/run_uwsgi_wrong_port_warning.bash b/docker/run_uwsgi_wrong_port_warning.bash index ec8daca7d..61bae4375 100755 --- a/docker/run_uwsgi_wrong_port_warning.bash +++ b/docker/run_uwsgi_wrong_port_warning.bash @@ -5,10 +5,9 @@ source /etc/profile # Per kobotoolbox/kobo-docker#301, we have changed the uWSGI port to 8001. This # provides a helpful message to anyone still trying to use port 8000 - if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]] then - exec /usr/local/bin/uwsgi --uid "${UWSGI_USER}" --socket :8000 --wsgi-file "${KOBOCAT_SRC_DIR}/docker/wrong_port_warning.wsgi" + exec $(command -v uwsgi) --uid "${UWSGI_USER}" --gid "${UWSGI_GROUP}" --socket :8000 --wsgi-file "${KOBOCAT_SRC_DIR}/docker/wrong_port_warning.wsgi" else - exec "${KOBOCAT_SRC_DIR}/docker/dev_wrong_port_warning.py" 8000 + exec gosu "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/docker/dev_wrong_port_warning.py" 8000 fi diff --git a/docker/setup_cron.bash b/docker/setup_cron.bash new file mode 100755 index 000000000..9ad6556fb --- /dev/null +++ b/docker/setup_cron.bash @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +rm -f /etc/cron.d/clean_up_tmp +cp docker/cron/clean_up_tmp /etc/cron.d/ +echo 'KoBoCAT tmp clean-up cron installed' + +rm -f /etc/cron.d/backup_media_crontab +if [[ -z "${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" ]]; then + echo 'KoBoCAT media automatic backups disabled.' +else + # Should we first validate the schedule e.g. with `chkcrontab`? + cat "${KOBOCAT_SRC_DIR}/docker/backup_media_crontab.envsubst" | envsubst > /etc/cron.d/backup_media_crontab + echo "KoBoCAT media automatic backup schedule: ${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" +fi diff --git a/docker/setup_pydev_debugger.bash b/docker/setup_pydev_debugger.bash new file mode 100755 index 000000000..c8f7dfc50 --- /dev/null +++ b/docker/setup_pydev_debugger.bash @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +# Keep it as is, not tested with Python3 +rm -rf /etc/profile.d/pydev_debugger.bash.sh +if [[ -d /srv/pydev_orig && -n "${KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then + echo 'Enabling PyDev remote debugging.' + "${KOBOCAT_SRC_DIR}/docker/setup_pydev.bash" +fi diff --git a/docker/sync_static.sh b/docker/sync_static.bash similarity index 71% rename from docker/sync_static.sh rename to docker/sync_static.bash index c752e0e8d..317887ab3 100755 --- a/docker/sync_static.sh +++ b/docker/sync_static.bash @@ -1,12 +1,10 @@ #!/bin/bash set -e -source /etc/profile - -mkdir -p "${KOBOCAT_SRC_DIR}/onadata/static" +gosu "${UWSGI_USER}" mkdir -p "${KOBOCAT_SRC_DIR}/onadata/static" echo "Collecting static files..." -python manage.py collectstatic -v 0 --noinput +gosu "${UWSGI_USER}" "${VIRTUAL_ENV}/bin/python" manage.py collectstatic -v 0 --noinput echo "Done" # `chown -R` becomes very slow once a fair amount of media has been collected, @@ -18,10 +16,10 @@ echo '%%%%%%% NOTICE %%%%%%%' echo '% To avoid long delays, we no longer reset ownership *recursively*' echo '% every time this container starts. If you have trouble with' echo '% permissions, please run the following command inside the' -echo '% `kobocat` container:' +echo '% KoBoCAT container:' echo "% chown -R \"${UWSGI_USER}\" \"${KOBOCAT_SRC_DIR}\"" echo '%%%%%%%%%%%%%%%%%%%%%%' echo "Syncing to nginx folder..." -rsync -aq --delete --chown=www-data "${KPI_SRC_DIR}/onadata/static/" "${NGINX_STATIC_DIR}/" +rsync -aq --delete --chown=www-data "${KOBOCAT_SRC_DIR}/onadata/static/" "${NGINX_STATIC_DIR}/" echo "Done" diff --git a/requirements/README.md b/requirements/README.md deleted file mode 100644 index c346eb8d8..000000000 --- a/requirements/README.md +++ /dev/null @@ -1,7 +0,0 @@ -#REQUIREMENTS - -- ***base.pip*** - Contains requirements common to all environs and deploys. -- ***dev.pip*** - Requirements used for development and testing. - -- ***s3.pip*** - AWS S3 requirements used mainly in production. -- ***ses.pip*** - SES Mail backend requirements. diff --git a/requirements/base.pip b/requirements/base.pip deleted file mode 100644 index c59f47a28..000000000 --- a/requirements/base.pip +++ /dev/null @@ -1,82 +0,0 @@ -pytz==2018.9 - -Django>=1.8,<1.9 -dj-database-url==0.4.0 -django-guardian==1.4.1 -django-registration-redux==1.3 -django-templated-email==0.4.9 -gdata==2.0.18 -httplib2==0.9 -modilabs-python-utils==0.1.5 -Pillow==2.5.3 -poster==0.8.1 -psycopg2-binary==2.7.7 -pymongo==3.7.2 -lxml==3.4.0 -pyxform==0.13.1 -django-reversion==2.0.8 -xlrd==1.1.0 -xlwt==1.3.0 -openpyxl==2.0.5 -celery==4.2.1 -django-celery-beat==1.1.1 -amqp==2.4.0 -vine==1.2.0 -django-nose==1.4.2 -python-digest==1.7 -raven==6.1.0 --e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest - -# new export code relies on -pandas>=0.12.0 -elaphe==0.5.6 -requests==2.21.0 - -# formpack exports -git+https://github.com/kobotoolbox/formpack@kobocat-requirement -django-pure-pagination==0.3.0 -path.py==8.1.2 - -# sms support -dict2xml==1.3 - -# api support -djangorestframework==3.3.2 -djangorestframework-csv==1.4.1 -djangorestframework-jsonp==1.0.2 -djangorestframework-xml==1.3.0 - -# cors -django-cors-headers==0.13 -Markdown==2.5 -django-filter==0.7 - -# captcha -recaptcha-client==1.0.6 -unicodecsv==0.9.4 -dpath==1.2-70 - -# tagging -django-taggit==0.18.0 - -# oath2 support -django-oauth-toolkit==0.9.0 - -# spss -https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter - -# JSON data type support -jsonfield<1.0 -django-db-readonly==0.3.2 - -transifex-client - -# Shell Plus -django-extensions==1.6.7 - -# redis-sessions -django-redis-sessions==0.6.1 -redis==3.2.0 - -# ssrf --e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect diff --git a/requirements/dev.pip b/requirements/dev.pip deleted file mode 100644 index 99329a739..000000000 --- a/requirements/dev.pip +++ /dev/null @@ -1,22 +0,0 @@ -#clint -#django-kombu -#django-snippetscream -#django-statsd-mozilla -# fabric -# gunicorn -ipdb -ipython -shell_command -#statsd -#twill -Werkzeug -sqlparse -pytest -pytest-django -pytest-env -mongomock -mock - - -#python 3 -httmock diff --git a/requirements/s3.pip b/requirements/s3.pip deleted file mode 100644 index 277c5b31c..000000000 --- a/requirements/s3.pip +++ /dev/null @@ -1,5 +0,0 @@ -boto==2.48.0 - -# Necessary to use this fork until the resolution of -# https://github.com/jschneier/django-storages/issues/566 --e git+https://github.com/jnm/django-storages@s3boto_accurate_tell#egg=django_storages diff --git a/requirements/ses.pip b/requirements/ses.pip deleted file mode 100644 index 23c028d3f..000000000 --- a/requirements/ses.pip +++ /dev/null @@ -1,2 +0,0 @@ -boto==2.1.1 -django-ses==0.4.0