Docker for local development

.dockerignore

@@ -0,0 +1,15 @@
+# Dot directories
+.git
+.idea
+
+# Python venvs
+venv*
+
+# Ignore submodules?
+#product-details
+#thunderbird_notes
+
+# Ignore local build directories
+site
+thunderbird.net
+

README.md

@@ -24,6 +24,28 @@ git clone https://github.com/thundernest/thunderbird.net-l10n.git locale
 l10n_tools/compile.sh
 ```
 
+## Docker
+As an alternative, you can use [Docker](https://docker.com) for local development. You'll need docker already set-up, and the previously mentioned git repositories.
+
+Once that is done, you'll need to pull down and build a modified apache config (which is done via an automated script), and build the docker containers:
+```
+python docker/apache/build_apache_config.py
+sudo docker build -t thunderbird-website-build . -f website.dockerfile
+sudo docker build -t thunderbird-website-apache . -f apache.dockerfile
+```
+
+You'll also need to set up some local dns entries in `/etc/hosts`:
+```
+127.0.0.1 start.thunderbird.test
+127.0.0.1 www.thunderbird.test
+```
+
+After that, you can simply run `docker-compose up`, and visit [https://www.thunderbird.test](https://www.thunderbird.test). 
+
+Optionally you can do local development against the start page, by running `docker-compose --profile startpage up`, and visiting [https://start.thunderbird.test](https://start.thunderbird.test).
+
+**Note:** The docker container creates and uses a self-signed certificate, which will probably raise a warning on your browser. You can safely ignore this.
+
 ## Run Build
 
 A basic build is `python build-site.py`.

apache.dockerfile

@@ -0,0 +1,51 @@
+FROM python:2.7.18-slim-buster
+
+RUN apt-get update && apt-get install -y apache2 apache2-dev python2 libapache2-mod-wsgi
+
+# Make a few default directories that apache complains about
+RUN mkdir -p /var/www/html/
+RUN mkdir -p /var/www/html/live.momo/htaccess/
+RUN mkdir -p /var/www/html/autoconfig.momo/
+RUN mkdir -p /var/www/services/broker/
+RUN mkdir -p /var/www/services/mx/
+RUN mkdir -p /var/www/html/start/site/
+RUN mkdir -p /var/www/html/start/thunderbird.net/
+RUN mkdir -p /var/www/html/tbstats/docs/
+RUN mkdir -p /var/www/html/style_guide/_site/
+RUN mkdir -p /var/www/html/statictest/www/
+RUN mkdir -p /var/www/html/start/site/en-US/maintenance/
+RUN mkdir -p /var/www/tbservices/
+
+# Log directory
+RUN mkdir -p /etc/apache2/logs/
+RUN mkdir -p /var/log/httpd/autoconfig/
+# SSL directory
+RUN mkdir -p /etc/apache2/ssl/
+
+# Generate SSL certs
+RUN openssl genrsa -out /etc/apache2/ssl/ssl.key 3072
+RUN openssl req -new -out /etc/apache2/ssl/ssl.csr -sha256 -key /etc/apache2/ssl/ssl.key -subj "/C=CA/CN=*.thunderbird.test"
+RUN openssl x509 -req -in /etc/apache2/ssl/ssl.csr -days 3650 -signkey /etc/apache2/ssl/ssl.key -out /etc/apache2/ssl/ssl.crt -outform PEM
+
+# Setup our virtualenv
+RUN pip install virtualenv
+RUN virtualenv -p python2.7 /var/www/tbservices/
+# Install some libs into our virtualenv
+RUN /var/www/tbservices/bin/pip install requests webob lib
+
+# Enable some additional mods
+RUN a2enmod socache_shmcb
+RUN a2enmod rewrite
+RUN a2enmod headers
+RUN a2enmod expires
+RUN a2enmod ssl
+
+# Create a symlink so we can get docker logs working
+RUN ln -sf /proc/self/fd/1 /var/log/apache2/access.log && \
+    ln -sf /proc/self/fd/1 /var/log/apache2/error.log && \
+    ln -sf /proc/self/fd/1 /var/log/apache2/other_vhosts_access.log
+
+# Boot apache
+EXPOSE 80
+EXPOSE 443
+CMD ["apachectl", "-D", "FOREGROUND"]

build-site.py

@@ -13,6 +13,9 @@
                     action='store_true')
 parser.add_argument('--watch', help='Rebuild when template and asset dirs are changed, and run a server on localhost.',
                     action='store_true')
+parser.add_argument('--watch-only', help='Rebuild when template and asset dirs are changed, does not run a server on localhost.',
+                    action='store_true')
+parser.add_argument('--render-path', help='Adjusts the render directory.', type=str)
 parser.add_argument('--port', const=8000, default=8000, type=int,
                     help='Port for the server that runs with --watch.', nargs='?')
 args = parser.parse_args()
@@ -25,9 +28,16 @@
     langmsg = 'in all languages.'
     languages = settings.PROD_LANGUAGES
 
+if not args.render_path:
+    site_render_path = settings.START_RENDERPATH
+    website_render_path = settings.WEBSITE_RENDERPATH
+else:
+    site_render_path = "{}{}".format(args.render_path, settings.START_RENDERPATH)
+    website_render_path = "{}{}".format(args.render_path, settings.WEBSITE_RENDERPATH)
+
 if args.startpage:
     print('Rendering start page ' + langmsg)
-    site = builder.Site(languages, settings.START_PATH, settings.START_RENDERPATH, settings.START_CSS, debug=args.debug)
+    site = builder.Site(languages, settings.START_PATH, site_render_path, settings.START_CSS, debug=args.debug)
     site.build_startpage()
 else:
     print('Rendering www.thunderbird.net ' + langmsg)
@@ -52,9 +62,11 @@
                'blog_data': feedparser.parse(settings.BLOG_FEED_URL)
               }
 
-    site = builder.Site(languages, settings.WEBSITE_PATH, settings.WEBSITE_RENDERPATH,
+    site = builder.Site(languages, settings.WEBSITE_PATH, website_render_path,
         settings.WEBSITE_CSS, js_bundles=settings.WEBSITE_JS, data=context, debug=args.debug)
     site.build_website()
 
 if args.watch:
-    builder.setup_observer(site, args.port)
+    builder.setup_local_watch(site, args.port)
+elif args.watch_only:
+    builder.setup_docker_watch(site)

builder.py

@@ -385,7 +385,7 @@ def setup_httpd(port, path):
     return process
 
 
-def setup_observer(builder_instance, port):
+def setup_observer(builder_instance):
     """Setup and start the watchdog observer for the --watch command."""
     handler = UpdateHandler(builder_instance)
     observer = Observer()
@@ -394,8 +394,25 @@ def setup_observer(builder_instance, port):
     observer.schedule(handler, path=settings.MEDIA_URL.strip('/'), recursive=True)
     observer.daemon = True
     observer.start()
+    return observer
+
+def setup_docker_watch(builder_instance):
+    """Setup and start just the watchdog observer. Temp for now."""
+    observer = setup_observer(builder_instance)
     print("Updating website when templates, CSS, or JS are modified. Press Ctrl-C to end.")
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        print("Shutting down watcher...")
+        observer.stop()
+        observer.join()
+
+def setup_local_watch(builder_instance, port):
+    """Setup and start the watchdog observer, and runs a local webserver."""
+    observer = setup_observer(builder_instance)
     server = setup_httpd(port, builder_instance.renderpath)
+    print("Updating website when templates, CSS, or JS are modified. Press Ctrl-C to end.")
     try:
         while True:
             time.sleep(1)
@@ -404,4 +421,4 @@ def setup_observer(builder_instance, port):
         server.terminate()
         server.join()
         observer.stop()
-        observer.join()
+        observer.join()

docker-compose.yml

@@ -0,0 +1,35 @@
+services:
+  website-builder:
+    image: thunderbird-website-build
+    command: sh -c "/usr/local/bin/python build-site.py --enus --watch-only --render-path='/srv/www/'"
+    working_dir: /website
+    volumes:
+      - ./:/website:ro
+      - srv-directory:/srv/www/
+
+  # Can be started via `docker-compose --profile startpage up`
+  start-builder:
+    image: thunderbird-website-build
+    command: sh -c "/usr/local/bin/python build-site.py --startpage --enus --watch-only --render-path='/srv/www/'"
+    working_dir: /website
+    profiles:
+      - startpage
+    volumes:
+      - ./:/website:ro
+      - srv-directory:/srv/www/
+
+  apache:
+    image: thunderbird-website-apache
+    hostname: www.thunderbird.net
+    ports:
+      - 8000:80
+      - 443:443
+    volumes:
+      - ./docker/apache/built/tb_vhosts.conf:/etc/apache2/conf-enabled/tb_vhosts.conf:ro
+      - srv-directory:/var/www/html/start/
+      # Needed for WSGI rewrites
+      - ./settings.py:/var/www/html/start/settings.py:ro
+      - ./wsgi.py:/var/www/html/start/wsgi.py:ro
+
+volumes:
+  srv-directory:

docker/apache/.gitignore

@@ -0,0 +1,3 @@
+# Remote files
+apache.yml
+vhost_template.j2

docker/apache/build_apache_config.py

@@ -0,0 +1,109 @@
+import os
+import requests
+import yaml
+from jinja2 import FileSystemLoader, Environment
+
+APACHE_CONFIG = './apache.yml'
+VHOST_TEMPLATE = './vhost_template.j2'
+
+APACHE_CONFIG_URL = 'https://raw.githubusercontent.com/thundernest/thundernest-ansible/master/vars/apache.yml'
+VHOST_TEMPLATE_URL = 'https://raw.githubusercontent.com/thundernest/thundernest-ansible/master/vars/vhost_template.j2'
+
+VHOST_CONFIG_OUT = './built/tb_vhosts.conf'
+
+def comment_out_commands(commands, string):
+    """ Goes through a given string and comments out command that are in the commands list. """
+    for cmd in commands:
+        string = string.replace(cmd, '#{}'.format(cmd))
+    return string
+
+def render_config(config_contents):
+    """ Pulls in the apache config, updates some things for local development, and renders it out. """
+    print("Rendering...")
+
+    load = FileSystemLoader('./')
+    env = Environment(loader=load, lstrip_blocks=True, trim_blocks=True)
+
+    config = yaml.load(config_contents, yaml.Loader)
+
+    # Missing defaults
+    config.update({'apache_ignore_missing_ssl_certificate': True})
+    config.update({'apache_listen_ip': '*'})
+    config.update({'apache_listen_port': '80'})
+    config.update({'apache_listen_port_ssl': '443'})
+    config.update({'apache_ssl_cipher_suite': 'DEFAULT'})
+    config.update({'apache_ssl_protocol': 'TLSv1.3'})
+
+    # Use our ssl keys
+    config.update({'apache_certificate_key_file': '/etc/apache2/ssl/ssl.key'})
+    config.update({'apache_certificate_file': '/etc/apache2/ssl/ssl.crt'})
+
+    # Remove the chain
+    config.pop('apache_certificate_chain_file')
+
+    # Some commands that don't clash well with the current apache setup.
+    comment_out = [
+        'Listen 443', # Already done
+        'LoadModule', # Already done
+        # FIXME: Errors due to file/folder not found, yet the folder is there..
+        'ErrorLog',
+        'CustomLog',
+    ]
+
+    vhosts = config.get('apache_vhosts')
+    for (index, vhost_entry) in enumerate(vhosts):
+        server_name = vhost_entry.get('servername')
+        server_alias = vhost_entry.get('serveralias')
+        extra_params = vhost_entry.get('extra_parameters')
+        # Change servername and serveralias (if avail) from .net to .test
+        if server_name is not None:
+            vhosts[index]['servername'] = server_name.replace('.net', '.test')
+        if server_alias is not None:
+            vhosts[index]['serveralias'] = server_alias.replace('.net', '.test')
+        if extra_params is not None:
+            vhosts[index]['extra_parameters'] = comment_out_commands(comment_out, extra_params)
+        else:
+            # Fix a small issue where None (type) is displayed
+            vhosts[index]['extra_parameters'] = ''
+
+    # Set our modified config
+    config.update({'apache_vhosts': vhosts})
+
+    # Remove some problematic config values
+    global_vhost_settings = comment_out_commands(comment_out, config.get('apache_global_vhost_settings'))
+    config.update({'apache_global_vhost_settings': global_vhost_settings})
+
+    # Do the render
+    env.globals.update(**config)
+    template = env.get_template(VHOST_TEMPLATE)
+    template_str = template.render()
+
+    if template_str == '':
+        print("Error: Rendered template is empty")
+        return
+
+    with open(VHOST_CONFIG_OUT, 'w') as fh:
+        fh.write(template_str)
+
+def main():
+    # Ensure we can use relative paths
+    os.chdir(os.path.dirname(__file__))
+
+    print("Retrieving apache.yml from thundernest-ansible.")
+    config = requests.get(APACHE_CONFIG_URL)
+    # Save a local copy for reference
+    with open(APACHE_CONFIG, 'wb') as fh:
+        fh.write(config.content)
+
+    print("Retrieving vhost_template.j2 from thundernest-ansible.")
+    template = requests.get(VHOST_TEMPLATE_URL)
+    # Save a local copy for reference
+    with open(VHOST_TEMPLATE, 'wb') as fh:
+        fh.write(template.content)
+
+    render_config(config.content)
+
+    print("Done!")
+
+if __name__ == '__main__':
+    main()

website.dockerfile

@@ -0,0 +1,14 @@
+FROM python:2.7.18-slim-buster as Build
+WORKDIR /website
+COPY . .
+
+# Dependacies
+RUN apt-get update && apt-get install -y nodejs npm
+RUN npm install -g less
+
+RUN pip install -r requirements-dev.txt
+
+RUN mkdir -p /srv/www/site/
+RUN mkdir -p /srv/www/site/en-US/maintenance/
+RUN mkdir -p /srv/www/thunderbird.net/
+