Docker support [WIP]

.dockerignore

@@ -0,0 +1,12 @@
+.git
+htdocs/media/logo
+mygpo/settings_prod.py
+doc/
+res/
+tools/
+locale/
+venv*
+*.pyc
+*/*.pyc
+*/*/*.pyc
+*/*/*/*.pyc

Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.6
+MAINTAINER Stefan Kögl <stefan@skoegl.net>
+
+RUN apt-get update && apt-get install -y git make
+
+# copy source
+WORKDIR /srv/mygpo
+
+COPY makefile .
+
+# install all packaged runtime dependencies
+RUN yes | make install-deps APT=apt-get
+
+# create log directories
+RUN mkdir -p /var/log/gunicorn
+
+COPY requirements.txt .
+COPY requirements-setup.txt .
+
+# install all runtime dependencies
+RUN pip install \
+    -r requirements.txt \
+    -r requirements-setup.txt
+
+# copy source
+COPY . .
+
+# Post-deployment actions
+ENV SECRET_KEY temp
+RUN python manage.py collectstatic --no-input
+RUN python manage.py compilemessages
+
+# set up missing environment variables
+ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"]
+
+# default launch command
+CMD ["/srv/mygpo/contrib/wait-for-postgres.py", "gunicorn", "mygpo.wsgi:application", "--access-logfile", "-", "--bind=0.0.0.0:8000"]
+
+EXPOSE 8000

bin/docker-env.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Docker doesn't have a great way to set environment variables at startup.
+# This scripts will set up some defaults.
+
+# if a DATABSE_URL is provided from outside, use it
+if [[ -z "$DATABASE_URL" ]]; then
+    # otherwise construct one using a linked "db" container
+    export DATABASE_URL="postgres://mygpo:mygpo@${DB_PORT_5432_TCP_ADDR}:5432/mygpo"
+fi
+
+# if not SECRET_KEY is provided from outside, create a random one
+if [[ -z "$SECRET_KEY" ]]; then
+    export SECRET_KEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+fi
+
+# Execute the commands passed to this script
+exec "$@"

contrib/Dockerfile.postgres

@@ -0,0 +1,3 @@
+FROM postgres:10
+
+COPY init-db.sql /docker-entrypoint-initdb.d/

contrib/docker.env

@@ -0,0 +1,2 @@
+BROKER_URL=redis://root@redis:6379/0
+DATABASE_URL=postgresql://mygpo:mygpo@postgres:5432/mygpo

contrib/init-db.sql

@@ -0,0 +1,9 @@
+CREATE USER mygpo WITH PASSWORD 'mygpo';
+ALTER USER mygpo CREATEDB;  -- required for creating test database
+CREATE DATABASE mygpo;
+CREATE DATABASE test_mygpo;
+GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo;
+GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo;
+ALTER DATABASE mygpo OWNER TO mygpo;
+ALTER DATABASE test_mygpo OWNER TO mygpo;
+ALTER ROLE mygpo SET statement_timeout = 5000;

contrib/wait-for-postgres.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+import os
+import sys
+import time
+
+import psycopg2cffi as psycopg2
+
+
+if __name__ == "__main__":
+    dburl = os.environ["DATABASE_URL"]
+    print("Trying to connect to {}".format(dburl))
+    conn = None
+    while not conn:
+        try:
+            conn = psycopg2.connect(dburl)
+        except:
+            print("Postgres is unavailable - sleeping")
+            time.sleep(1)
+
+    cur = conn.cursor()
+    cur.execute("""SELECT EXISTS (
+                  SELECT 1
+                  FROM   information_schema.tables
+                  WHERE  table_schema = 'public'
+                  AND    table_name = 'votes_vote'
+     )""")
+    has_table = cur.fetchone()[0]
+    cur.close()
+    conn.close()
+
+    if 'migrate' in sys.argv:
+        if has_table:
+            print("Database already initialized, exiting")
+            sys.exit(0)
+        else:
+            print("Postgres is available => will initialize")
+    else:
+        if not has_table:
+            print("ERROR: Postgres is available but not initialized.\n"
+                   "Please run:\n"
+                   "\tdocker-compose run web /srv/mygpo/contrib/wait-for-postgres.py python manage.py migrate\n"
+                   "and restart.")
+            sys.exit(-1)
+
+    if sys.argv[1]:
+        cmd = sys.argv[1:]
+        print("Postgres is up - executing command {}".format(" ".join(cmd)))
+        sys.stdout.flush()
+        os.execvp(cmd[0], cmd)
+    else:
+        print("Postgres is up - no command given, exiting")

doc/dev/postgres-setup.rst

@@ -5,14 +5,6 @@ PostgreSQL Setup
 
 Use the following to set up a local PostgreSQL.
 
-.. code-block:: sql
-
-    CREATE USER mygpo WITH PASSWORD 'mygpo';
-    ALTER USER mygpo CREATEDB;  -- required for creating test database
-    CREATE DATABASE mygpo;
-    CREATE DATABASE test_mygpo;
-    GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo;
-    GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo;
-    ALTER DATABASE mygpo OWNER TO mygpo;
-    ALTER DATABASE test_mygpo OWNER TO mygpo;
-    ALTER ROLE mygpo SET statement_timeout = 5000;
+.. literalinclude:: ../../contrib/init-db.sql
+   :language: sql
+   :linenos:

doc/index.rst

@@ -24,6 +24,7 @@ Contents
    publisher/index
    api/index
    dev/index
+   ops/index
    Translator Documentation <http://wiki.gpodder.org/wiki/Translations>
 
 

doc/ops/configuration.rst

@@ -0,0 +1,106 @@
+Configuration
+=============
+
+Configuration can be done through the following environment variables.
+
+``DEBUG``
+---------
+Debug mode shows error pages, enables debug output, etc.
+
+
+``DATABASE_URL``
+----------------
+DB connection string in the form of ``postgres://USER:PASSWORD@HOST:PORT/NAME``
+
+
+``ACCOUNT_ACTIVATION_DAYS``
+---------------------------
+Number of days that newly registered users have time to activate their account.
+
+
+``DEFAULT_FROM_EMAIL``
+----------------------
+Default sender address for outgoing emails. See `Django documentation
+<https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-DEFAULT_FROM_EMAIL>`__.
+
+
+``SECRET_KEY``
+--------------
+See `Django documentation
+<https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY>`__.
+
+
+``GOOGLE_ANALYTICS_PROPERTY_ID``
+--------------------------------
+Setting a Google Analytics Property ID activates GA integration.
+
+
+``DIRECTORY_EXCLUDED_TAGS``
+---------------------------
+A comma-separated list of tags that are excluded from the directory.
+
+
+``FLICKR_API_KEY``
+------------------
+Setting a Flickr API key activates Flickr integration.
+
+
+``MAINTENANCE``
+---------------
+Switches the site into a maintenance mode.
+
+
+* ``PODCAST_SLUG_SUBSCRIBER_LIMIT``
+
+* ``MIN_SUBSCRIBERS_CATEGORY``:  minimum number of subscribers that a podcast
+  needs to "push" one of its categories to the top
+
+* ``API_ACTIONS_MAX_NONBG``: maximum number of episode actions that the API
+  processes immediatelly before returning the response. Larger requests will
+  be handled in background.
+
+* ``ADSENSE_CLIENT``
+
+* ``ADSENSE_SLOT_BOTTOM``
+
+* ``STAFF_TOKEN``: enabled access to staff-only areas with ?staff=<STAFF_TOKEN>
+
+* ``FLATTR_KEY``
+
+* ``FLATTR_SECRET``
+
+* ``FLATTR_MYGPO_THING``: Flattr thing of the webservice. Will be flattr'd
+  when a user sets the "Auto-Flattr gpodder.net" option
+
+* ``USER_AGENT``: The User-Agent string used for outgoing HTTP requests
+
+* ``DEFAULT_BASE_URL``: Base URL of the website that is used if the actually
+  used parameters is not available. Request handlers, for example, can access
+  the requested domain. Code that runs in background can not do this, and
+  therefore requires a default value. This should be set to something like
+  ``http://example.com``
+
+* ``BROKER_URL`` Celery Broker URL
+
+* ``CELERY_RESULT_BACKEND``
+
+* ``CELERY_SEND_TASK_ERROR_EMAILS``
+
+* ``SERVER_EMAIL``
+
+* ``GOOGLE_CLIENT_ID``
+
+* ``GOOGLE_CLIENT_SECRET``
+
+* ``SUPPORT_URL``: URL where users of the site can get support
+
+* ``ELASTICSEARCH_SERVER``
+
+* ``ELASTICSEARCH_INDEX``
+
+* ``ELASTICSEARCH_TIMEOUT``
+
+* ``ACTIVATION_VALID_DAYS`` time for how long an activation is valid; after
+  that, an unactivated user will be deleted
+
+* ``INTERNAL_IPS``

doc/ops/docker.rst

@@ -0,0 +1,57 @@
+Using Docker
+============
+
+mygpo can be run using `Docker <https://docker.com/>`_.
+
+
+Database
+--------
+
+The image requires a PostgreSQL server, specified either via
+
+* a `linked container <https://docs.docker.com/userguide/dockerlinks/>`_
+  called ``db`` containing a server with a database called ``mygpo``, a user
+  called ``mygpo`` with a password ``mygpo``.
+* A ``DATABASE_URL`` environment variable (eg
+  ``postgres://USER:PASSWORD@HOST:PORT/NAME``)
+
+Using a PostgreSQL Docker container
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start a container using the `official PostgreSQL image <https://registry.hub.docker.com/_/postgres/>`_. ::
+
+    docker run --name db -d postgres
+
+Create the schema and a corresponding user ::
+
+    docker exec -it db psql -U postgres
+
+And enter the following commands (change passwords as required)
+
+.. literalinclude:: ../../contrib/init-db.sql
+    :language: sql
+    :linenos:
+
+Initialize the tables. This needs needs to be run for every update. ::
+
+    sudo docker run --rm --link db:db -e SECRET_KEY=asdf mygpo/web python manage.py migrate
+
+
+Redis
+-----
+
+
+Web Frontend
+------------
+
+The image exposes the web interface on port 8000.
+
+
+Celery Worker
+-------------
+
+
+Celery Heartbeat
+----------------
+
+

doc/ops/index.rst

@@ -0,0 +1,9 @@
+Operation Documentation
+=======================
+
+
+.. toctree::
+   :maxdepth: 1
+
+   configuration
+   docker

docker-compose-common.yml

@@ -0,0 +1,14 @@
+
+version: '2'
+
+services:
+
+  base:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    env_file: contrib/docker.env
+    volumes:
+      - .:/srv/mygpo
+      - ./contrib/docker.env:/docker.env
+