Merge pull request #681 from kobotoolbox/kobocat2.0-step5-new-docker-…

…image [Python3 and Django2.2 upgrade] part 5 - Updated docker image with non-root user
kobotoolbox · Mar 12, 2021 · 1b91ec2 · 1b91ec2
2 parents 0314ba3 + 314f492
commit 1b91ec2
Show file tree

Hide file tree

Showing 17 changed files with 152 additions and 248 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,108 +1,106 @@
 FROM nikolaik/python-nodejs:python3.8-nodejs10
 
+# Declare environment variables
 ENV DEBIAN_FRONTEND=noninteractive
 ENV LANG=en_US.UTF-8
 ENV LANGUAGE=en_US:en
 ENV LC_ALL=en_US.UTF-8
-ENV VIRTUAL_ENV=/opt/venv
 
-
-ENV KOBOCAT_LOGS_DIR=/srv/logs \
-    DJANGO_SETTINGS_MODULE=kobo.settings.prod \
+ENV VIRTUAL_ENV=/opt/venv \
+    KOBOCAT_LOGS_DIR=/srv/logs \
+    DJANGO_SETTINGS_MODULE=onadata.settings.prod \
     # The mountpoint of a volume shared with the `nginx` container. Static files will
-    #   be copied there.
+    # be copied there.
     NGINX_STATIC_DIR=/srv/static \
     KOBOCAT_SRC_DIR=/srv/src/kobocat \
     BACKUPS_DIR=/srv/backups \
-    TMP_PATH=/srv/tmp \
+    TMP_DIR=/srv/tmp \
+    UWSGI_USER=kobo \
+    UWSGI_GROUP=kobo \
+    SERVICES_DIR=/etc/service \
+    CELERY_PID_DIR=/var/run/celery \
     INIT_PATH=/srv/init
 
-# Install Dockerize.
+# Install Dockerize
 ENV DOCKERIZE_VERSION v0.6.1
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -P /tmp \
     && tar -C /usr/local/bin -xzvf /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && rm /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
-##########################################
-# Create build directories               #
-##########################################
-
-RUN mkdir -p "${NGINX_STATIC_DIR}" && \
-    mkdir -p "${KOBOCAT_SRC_DIR}" && \
-    mkdir -p "${TMP_PATH}" && \
-    mkdir -p "${BACKUPS_DIR}" && \
-    mkdir -p "${INIT_PATH}"
-
-##########################################
-# Install `apt` packages.                #
-##########################################
+# Create needed directories
+RUN mkdir -p ${NGINX_STATIC_DIR} && \
+    mkdir -p ${KOBOCAT_SRC_DIR} && \
+    mkdir -p ${TMP_DIR} && \
+    mkdir -p ${BACKUPS_DIR} && \
+    mkdir -p ${CELERY_PID_DIR} && \
+    mkdir -p ${SERVICES_DIR}/uwsgi && \
+    mkdir -p ${SERVICES_DIR}/uwsgi_wrong_port_warning && \
+    mkdir -p ${SERVICES_DIR}/celery && \
+    mkdir -p ${SERVICES_DIR}/celery_beat && \
+    mkdir -p ${KOBOCAT_LOGS_DIR}/ && \
+    mkdir -p ${KOBOCAT_SRC_DIR}/emails && \
+    mkdir -p ${INIT_PATH}
+
+# Install `apt` packages.
 RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 
-RUN apt -qq update && \
-    apt -qq -y install \
+RUN apt-get -qq update && \
+    apt-get -qq -y install \
         gdal-bin \
         libproj-dev \
         gettext \
         postgresql-client \
+        openjdk-11-jre \
         locales \
         runit-init \
         rsync \
-        vim && \
-    apt clean && \
+        vim \
+        gosu \
+        cron && \
+    apt-get clean && \
         rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-###########################
-# Install locales         #
-###########################
-
-RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen
-RUN locale-gen && dpkg-reconfigure locales -f noninteractive
+# Install locales
+RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && \
+    locale-gen && dpkg-reconfigure locales -f noninteractive
 
-###########################
-# Copy KoBoCAT directory  #
-###########################
+# Create local user UWSGI_USER`
+RUN adduser --disabled-password --gecos '' "$UWSGI_USER"
 
+# Copy KoBoCAT directory
 COPY . "${KOBOCAT_SRC_DIR}"
 
-###########################
-# Install `pip` packages. #
-###########################
-
+# Install `pip` packages
 RUN virtualenv "$VIRTUAL_ENV"
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 RUN pip install  --quiet --upgrade pip && \
     pip install  --quiet pip-tools
-COPY ./dependencies/pip/prod.txt /srv/tmp/pip_dependencies.txt
-RUN pip-sync /srv/tmp/pip_dependencies.txt 1>/dev/null && \
+COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt"
+RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null && \
     rm -rf ~/.cache/pip
 
-##########################################
-# Persist the log and email directories. #
-##########################################
-
-RUN mkdir -p "${KOBOCAT_LOGS_DIR}/" "${KOBOCAT_SRC_DIR}/emails" && \
-    chown -R "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/emails/" && \
-    chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}"
-
-#################################################
-# Handle runtime tasks and create main process. #
-#################################################
-
 # Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides.
-RUN echo "export PATH=${PATH}" >> /etc/profile
-RUN echo 'source /etc/profile' >> /root/.bashrc
-
-# Prepare for execution.
-RUN mkdir -p /etc/service/uwsgi_wrong_port_warning && \
-    cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi_wrong_port_warning.bash" /etc/service/uwsgi_wrong_port_warning/run && \
-    mkdir -p /etc/service/uwsgi && \
-    # Remove getty* services
-    rm -rf /etc/runit/runsvdir/default/getty-tty* && \
-    cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash" /etc/service/uwsgi/run && \
-    mkdir -p /etc/service/celery && \
-    ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery.bash" /etc/service/celery/run && \
-    mkdir -p /etc/service/celery_beat && \
-    ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash" /etc/service/celery_beat/run && \
+RUN echo "export PATH=${PATH}" >> /etc/profile && \
+    echo 'source /etc/profile' >> /root/.bashrc && \
+    echo 'source /etc/profile' >> /home/${UWSGI_USER}/.bashrc
+
+# Remove getty* services to avoid errors of absent tty at sv start-up
+RUN rm -rf /etc/runit/runsvdir/default/getty-tty*
+
+# Create symlinks for runsv services
+RUN ln -s "${KOBOCAT_SRC_DIR}/docker/run_uwsgi_wrong_port_warning.bash" "${SERVICES_DIR}/uwsgi_wrong_port_warning/run" && \
+    ln -s "${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash" "${SERVICES_DIR}/uwsgi/run" && \
+    ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery.bash" "${SERVICES_DIR}/celery/run" && \
+    ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash" "${SERVICES_DIR}/celery_beat/run"
+
+# Add/Restore `UWSGI_USER`'s permissions
+RUN chown -R ":${UWSGI_GROUP}" ${CELERY_PID_DIR} && \
+    chmod g+w ${CELERY_PID_DIR} && \
+    chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_SRC_DIR}/emails/ && \
+    chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_LOGS_DIR} && \
+    chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${TMP_DIR} && \
+    chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${VIRTUAL_ENV} && \
+    chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${BACKUPS_DIR}
 
 WORKDIR "${KOBOCAT_SRC_DIR}"
 

diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt
@@ -36,10 +36,11 @@ django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.8  # via -r dependencies/pip/requirements.in
 django-render-block==0.7  # via django-templated-email
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
+django-storages==1.9.1    # via -r dependencies/pip/requirements.in
 django-taggit==1.3.0      # via -r dependencies/pip/requirements.in
 django-templated-email==2.3.0  # via -r dependencies/pip/requirements.in
 django-timezone-field==4.0  # via -r dependencies/pip/requirements.in, django-celery-beat
-django==2.2.14            # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield
+django==2.2.14            # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield
 djangorestframework-csv==2.1.0  # via -r dependencies/pip/requirements.in
 djangorestframework-guardian==0.3.0  # via -r dependencies/pip/requirements.in
 djangorestframework-jsonp==1.0.2  # via -r dependencies/pip/requirements.in
@@ -59,7 +60,6 @@ gitdb==4.0.5              # via gitpython
 gitpython==3.1.7          # via transifex-client
 httmock==1.3.0            # via -r dependencies/pip/dev.in
 idna==2.10                # via requests
-importlib-metadata==1.7.0  # via jsonschema, kombu, markdown, path, pluggy, pytest
 ipdb==0.13.3              # via -r dependencies/pip/dev.in
 ipython-genutils==0.2.0   # via traitlets
 ipython==7.16.1           # via -r dependencies/pip/dev.in, ipdb
@@ -82,7 +82,7 @@ openpyxl==3.0.4           # via -r dependencies/pip/requirements.in
 packaging==20.4           # via pytest
 pandas==1.0.5             # via -r dependencies/pip/requirements.in
 parso==0.7.0              # via jedi
-path.py==12.4.0           # via -r dependencies/pip/requirements.in, formpack
+path.py==12.4.0           # via formpack
 path==13.1.0              # via path.py
 pexpect==4.8.0            # via ipython
 pickleshare==0.7.5        # via ipython
@@ -130,7 +130,6 @@ werkzeug==1.0.1           # via -r dependencies/pip/dev.in
 xlrd==1.2.0               # via -r dependencies/pip/requirements.in, pyxform
 xlsxwriter==1.2.9         # via formpack
 xlwt==1.3.0               # via -r dependencies/pip/requirements.in
-zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt
@@ -33,10 +33,11 @@ django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.8  # via -r dependencies/pip/requirements.in
 django-render-block==0.7  # via django-templated-email
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
+django-storages==1.9.1    # via -r dependencies/pip/requirements.in
 django-taggit==1.3.0      # via -r dependencies/pip/requirements.in
 django-templated-email==2.3.0  # via -r dependencies/pip/requirements.in
 django-timezone-field==4.0  # via -r dependencies/pip/requirements.in, django-celery-beat
-django==2.2.14            # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield
+django==2.2.14            # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield
 djangorestframework-csv==2.1.0  # via -r dependencies/pip/requirements.in
 djangorestframework-guardian==0.3.0  # via -r dependencies/pip/requirements.in
 djangorestframework-jsonp==1.0.2  # via -r dependencies/pip/requirements.in
@@ -55,7 +56,6 @@ geojson-rewind==0.2.0     # via formpack
 gitdb==4.0.5              # via gitpython
 gitpython==3.1.7          # via transifex-client
 idna==2.10                # via requests
-importlib-metadata==1.7.0  # via jsonschema, kombu, markdown, path
 jdcal==1.4.1              # via openpyxl
 jsonfield==3.1.0          # via -r dependencies/pip/requirements.in
 jsonschema==3.2.0         # via formpack
@@ -68,8 +68,8 @@ numpy==1.19.0             # via pandas
 oauthlib==3.1.0           # via django-oauth-toolkit
 openpyxl==3.0.4           # via -r dependencies/pip/requirements.in
 pandas==1.0.5             # via -r dependencies/pip/requirements.in
-path.py==12.4.0           # via -r dependencies/pip/requirements.in, formpack
-path==13.1.0              # via path.py
+path.py==12.5.0           # via formpack
+path==15.0.0              # via path.py
 pillow==7.2.0             # via -r dependencies/pip/requirements.in, elaphe3
 psycopg2-binary==2.8.5    # via -r dependencies/pip/requirements.in
 pymongo==3.10.1           # via -r dependencies/pip/requirements.in
@@ -96,11 +96,11 @@ transifex-client==0.13.11  # via -r dependencies/pip/requirements.in
 unicodecsv==0.14.1        # via djangorestframework-csv, pyxform
 unittest2==1.1.0          # via pyxform
 urllib3==1.25.9           # via requests, transifex-client
+uwsgi==2.0.19.1           # via -r dependencies/pip/prod.in
 vine==1.3.0               # via amqp, celery
 xlrd==1.2.0               # via -r dependencies/pip/requirements.in, pyxform
 xlsxwriter==1.2.9         # via formpack
 xlwt==1.3.0               # via -r dependencies/pip/requirements.in
-zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt
@@ -33,10 +33,11 @@ django-redis-sessions==0.6.1  # via -r dependencies/pip/requirements.in
 django-registration-redux==2.8  # via -r dependencies/pip/requirements.in
 django-render-block==0.7  # via django-templated-email
 django-reversion==3.0.1   # via -r dependencies/pip/requirements.in
+django-storages==1.9.1    # via -r dependencies/pip/requirements.in
 django-taggit==1.3.0      # via -r dependencies/pip/requirements.in
 django-templated-email==2.3.0  # via -r dependencies/pip/requirements.in
 django-timezone-field==4.0  # via -r dependencies/pip/requirements.in, django-celery-beat
-django==2.2.14            # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield
+django==2.2.14            # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield
 djangorestframework-csv==2.1.0  # via -r dependencies/pip/requirements.in
 djangorestframework-guardian==0.3.0  # via -r dependencies/pip/requirements.in
 djangorestframework-jsonp==1.0.2  # via -r dependencies/pip/requirements.in
@@ -55,7 +56,6 @@ geojson-rewind==0.2.0     # via formpack
 gitdb==4.0.5              # via gitpython
 gitpython==3.1.7          # via transifex-client
 idna==2.10                # via requests
-importlib-metadata==1.7.0  # via jsonschema, kombu, markdown, path
 jdcal==1.4.1              # via openpyxl
 jsonfield==3.1.0          # via -r dependencies/pip/requirements.in
 jsonschema==3.2.0         # via formpack
@@ -68,7 +68,7 @@ numpy==1.19.0             # via pandas
 oauthlib==3.1.0           # via django-oauth-toolkit
 openpyxl==3.0.4           # via -r dependencies/pip/requirements.in
 pandas==1.0.5             # via -r dependencies/pip/requirements.in
-path.py==12.4.0           # via -r dependencies/pip/requirements.in, formpack
+path.py==12.4.0           # via formpack
 path==13.1.0              # via path.py
 pillow==7.2.0             # via -r dependencies/pip/requirements.in, elaphe3
 psycopg2-binary==2.8.5    # via -r dependencies/pip/requirements.in
@@ -100,7 +100,6 @@ vine==1.3.0               # via amqp, celery
 xlrd==1.2.0               # via -r dependencies/pip/requirements.in, pyxform
 xlsxwriter==1.2.9         # via formpack
 xlwt==1.3.0               # via -r dependencies/pip/requirements.in
-zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/docker/init.bash b/docker/init.bash
@@ -13,34 +13,21 @@ if [[ -z $DATABASE_URL ]]; then
     exit 1
 fi
 
-echo 'Running migrations...'
-python manage.py migrate --noinput
-
-rm -f /etc/cron.d/clean_up_tmp
-cp docker/cron/clean_up_tmp /etc/cron.d/
-echo 'KoBoCat tmp clean-up cron installed'
-
-rm -f /etc/cron.d/backup_media_crontab
-if [[ -z "${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" ]]; then
-    echo 'KoBoCAT media automatic backups disabled.'
-else
-    # Should we first validate the schedule e.g. with `chkcrontab`?
-    cat "${KOBOCAT_SRC_DIR}/docker/backup_media_crontab.envsubst" | envsubst > /etc/cron.d/backup_media_crontab
-    echo "KoBoCAT media automatic backup schedule: ${KOBOCAT_MEDIA_BACKUP_SCHEDULE}"
-fi
+# Wait for databases to be up & running before going further
+/bin/bash "${INIT_PATH}/wait_for_mongo.bash"
+/bin/bash "${INIT_PATH}/wait_for_postgres.bash"
 
-/bin/bash ${KOBOCAT_SRC_DIR}/docker/sync_static.sh
+echo 'Running migrations...'
+gosu "${UWSGI_USER}" python manage.py migrate --noinput
 
-# Keep it as is, not tested with Python3
-rm -rf /etc/profile.d/pydev_debugger.bash.sh
-if [[ -d /srv/pydev_orig && -n "${KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then
-    echo 'Enabling PyDev remote debugging.'
-    "${KOBOCAT_SRC_DIR}/docker/setup_pydev.bash"
-fi
+echo 'Setting up cron tasks...'
+/bin/bash "${KOBOCAT_SRC_DIR}/docker/setup_cron.bash"
+/bin/bash "${KOBOCAT_SRC_DIR}/docker/setup_pydev_debugger.bash"
+/bin/bash "${KOBOCAT_SRC_DIR}/docker/sync_static.bash"
 
 echo 'Cleaning up Celery PIDs...'
-rm -rf /tmp/celery*.pid
+rm -rf ${CELERY_PID_DIR}/*.pid
 
 echo 'KoBoCAT initialization complete.'
 
-exec /usr/bin/runsvdir /etc/service
+exec /usr/bin/runsvdir "${SERVICES_DIR}"
diff --git a/docker/run_celery.bash b/docker/run_celery.bash
@@ -4,7 +4,9 @@ source /etc/profile
 
 # Run the main Celery worker
 cd "${KOBOCAT_SRC_DIR}"
-exec /sbin/setuser "${UWSGI_USER}" celery worker -A onadata -Ofair --loglevel=info \
+exec celery worker -A onadata -Ofair --loglevel=info \
     --hostname=kobocat_main_worker@%h \
     --logfile=${KOBOCAT_LOGS_DIR}/celery.log \
-    --pidfile=/tmp/celery.pid
+    --pidfile=${CELERY_PID_DIR}/celery.pid \
+    --uid=${UWSGI_USER} \
+    --gid=${UWSGI_GROUP}
diff --git a/docker/run_celery_beat.bash b/docker/run_celery_beat.bash
@@ -4,7 +4,9 @@ source /etc/profile
 
 # Run the main Celery beat worker
 cd "${KOBOCAT_SRC_DIR}"
-exec /sbin/setuser "${UWSGI_USER}" celery beat -A onadata --loglevel=info \
+exec celery beat -A onadata --loglevel=info \
     --logfile=${KOBOCAT_LOGS_DIR}/celery_beat.log \
-    --pidfile=/tmp/celery_beat.pid \
-    --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    --pidfile=${CELERY_PID_DIR}/celery_beat.pid \
+    --scheduler django_celery_beat.schedulers:DatabaseScheduler \
+    --uid=${UWSGI_USER} \
+    --gid=${UWSGI_GROUP}