[EXPERIMENTAL] Dev docker fixes dom2022

.dockerignore

@@ -0,0 +1,4 @@
+/.gitignore
+/.travis.yml
+/package-support
+db-data/

.gitignore

@@ -29,3 +29,4 @@ distribute-*.tar.gz
 distribute_setup.py
 .eggs/
 .coverage
+db-data/

Dockerfile

@@ -1,79 +1,33 @@
-FROM python:3.6.14
+FROM python:3.10-slim-bullseye AS builder
 MAINTAINER RAMS Project "code@magfest.org"
-LABEL version.sideboard ="1.0"
-WORKDIR /app
+LABEL version.sideboard ="1.1"
+EXPOSE 8282
+WORKDIR /app/sideboard
 
-# This is actually the least bad way to compose two Dockerfile tech stacks right now.
-# The following is copied and pasted from the Node Dockerfile at
-# https://github.com/nodejs/docker-node/blob/main/12/buster/Dockerfile
-# Update this comment and change the entire copypasta section to upgrade Node version
+# libcap-dev and gcc is required for python-prctl (for viewing names of python threads in system tools like htop)
+# TODO: need to install npm and nodejs for building static js / gradle stuff
 
-#########################################
-# START NODEJS DOCKERFILE COPYPASTA     #
-# https://github.com/nodejs/docker-node #
-#########################################
-RUN groupadd --gid 1000 node \
-  && useradd --uid 1000 --gid node --shell /bin/bash --create-home node
+RUN apt-get update \
+    && apt-get install -y  \
+    libcap-dev gcc \
+    git \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip3 install virtualenv \
+    && virtualenv --always-copy /app/env/ \
+    && /app/env/bin/pip3 install paver
 
-ENV NODE_VERSION 12.22.3
+# use for production configs. no volume mounting of code/etc
+FROM builder as production
+COPY . /app/sideboard/
 
-RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
-  && case "${dpkgArch##*-}" in \
-    amd64) ARCH='x64';; \
-    ppc64el) ARCH='ppc64le';; \
-    s390x) ARCH='s390x';; \
-    arm64) ARCH='arm64';; \
-    armhf) ARCH='armv7l';; \
-    i386) ARCH='x86';; \
-    *) echo "unsupported architecture"; exit 1 ;; \
-  esac \
-  # gpg keys listed at https://github.com/nodejs/node#release-keys
-  && set -ex \
-  && for key in \
-    4ED778F539E3634C779C87C6D7062848A1AB005C \
-    94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
-    74F12602B6F1C4E913FAA37AD3A89613643B6201 \
-    71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
-    8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
-    C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
-    C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \
-    DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
-    A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
-    108F52B48DB57BB0CC439B2997B01419BD92F80A \
-    B9E2F5981AA6E0CD28160D9FF13993A75599653C \
-  ; do \
-      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \
-      gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \
-  done \
-  && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
-  && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
-  && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
-  && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
-  && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
-  && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
-  # smoke tests
-  && node --version \
-  && npm --version
+RUN cd /app/sideboard/ &&  \
+    /app/env/bin/paver install_deps --env_path=/app/env/
 
-RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" \
-  && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
-  && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
-  && grep " node-v$NODE_VERSION-linux-x64.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
-  && tar -xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 \
-  && rm "node-v$NODE_VERSION-linux-x64.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
-  && ln -s /usr/local/bin/node /usr/local/bin/nodejs
-###################################
-# END NODEJS DOCKERFILE COPYPASTA #
-###################################
+CMD /bin/bash /app/sideboard/run_server.sh
 
-# required for python-prctl
-RUN apt-get update && apt-get install -y libcap-dev && rm -rf /var/lib/apt/lists/*
+FROM builder as dev
 
-ADD . /app/
-RUN pip3 install virtualenv \
-  && virtualenv --always-copy /app/env \
-	&& /app/env/bin/pip3 install paver
-RUN /app/env/bin/paver install_deps
+# use for dev builds. developers: you should mount your local sideboard/ directory (with plugins/etc) in /app/sideboard/
+# then, the startup command will run the paver dependency installations on container startup (instead of image build)
 
-CMD /app/env/bin/python3 /app/sideboard/run_server.py
-EXPOSE 8282
+CMD /bin/bash /app/sideboard/run_dev_server.sh

docker-compose-nginx.yaml

@@ -0,0 +1,38 @@
+version: '3.9'
+
+# NOTE: to use this file, you need to set HTTPS_SERVER_HOSTNAME in an "environment" section.
+# in uber this is done one level up in docker-compose-config.yaml
+# also need to add files to ./nginx/templates
+
+services:
+  frontend: 
+    image: nginx:stable-alpine
+    restart: "always"
+    depends_on:
+      - backend
+    ports: 
+      - "80:80"   # warning: if backend service is also listening on 80 on the host, this will conflict and fail to start
+      - "443:443"
+    volumes:
+      - ./nginx/templates:/etc/nginx/templates/:ro
+
+      # NOTE: it is expected that deploys on real servers will override these values in an external downstream config file.
+      # the two root paths here are the ones you need to use though. they match what's in nginx.conf
+      #
+      # DEV NOTE: This gets weird under windows sometimes, be careful if you're hitting errors, could be a windows-specific thing.
+      # If you have issues, try this from inside WSL2 instead of native windows.
+      # example1:
+      # - /etc/ssl/certs/_.uber.org/final-bundle.crt:/__server_bundle.crt:ro
+      # - /etc/ssl/certs/_.uber.org/_.uber.org.key:/__server.key:ro
+      #
+      # example2: (works on the prod server where the config dir is two down from uber_server/ root dir)
+      # - ../../config/certs/_.uber.org/final-bundle.crt:/__server_bundle.crt:ro
+      # - ../../config/certs/_.uber.org/_.uber.org.key:/__server.key:ro
+      #
+      # NOTE2: anything in templates directory will automatically have envsubst ran against it.
+      # ENV vars defined in docker-compose can be used in the nginx conf files like this: ${SOME_ENV_VAR}
+
+      # defaults: use self-signed certs so at least the server actually starts
+      # these files need to exist and should be checked in
+      - ./self-signed.crt:/__server_bundle.crt:ro
+      - ./self-signed.key:/__server.key:ro

docker-compose.dev.yaml

@@ -0,0 +1,21 @@
+version: '3.9'
+
+services:
+  backend:
+    # for live-editing code
+    volumes:
+      - ./:/app/sideboard/
+
+    restart: "no"
+
+    ports:
+     - "8282:8282"
+
+    # option 1: build container every time (decent for development)
+    build:
+      context: ./
+      dockerfile: Dockerfile
+      target: dev   # if using multi-stage dockerfiles, useful for selection of a particular stage
+
+    # option 2: use a prebuilt image (better for production)
+    # image: ghcr.io/magfest/sideboard-whatever:dev

docker-compose.yaml

@@ -0,0 +1,41 @@
+version: '3.9'
+
+# default settings. it's expected that other YAML files will override these in production
+x-postgres-default-vars: &postgres-default-vars
+  POSTGRES_USER: uber_db_user
+  POSTGRES_PASSWORD: uber_db_password
+  POSTGRES_DB: uber_db
+
+x-http-default-vars: &http-default-vars
+  HTTPS_SERVER_HOSTNAME: changeme.ubersystem.com
+
+services:
+  db:
+    restart: "always"
+    image: 'postgres:latest'
+
+    environment:
+      <<: *postgres-default-vars
+    ports:
+      - "5432:5432"
+
+    # to prevent heartache from accidentally deleting the DB, our default is to have the 
+    # volume be served from a local directory. feel free to remove or change this.
+    volumes:
+      - ./db-data/:/var/lib/postgresql/data/
+
+  # our backend uber app. expose via HTTP internally only (no SSL, and not publicly exposed)
+  backend:
+    # image: ghcr.io/magfest/ubersystem_prime:v1.0.0.0   # for production, populate this with container builds
+    restart: "always"
+    depends_on:
+      - db
+    environment:
+      <<: *postgres-default-vars
+      <<: *http-default-vars
+      # example of how to set other env vars
+      SOME_OTHER_ENV_DB: "some_value"
+
+    # if you want to expose the backend ports directly (not recommended. go through nginx)
+    # ports:
+    #  - "80:80"   # warning: if nginx is also trying to listen on 80, one of the two services will fail to start.

pavement.py

@@ -75,16 +75,17 @@ def make_venv():
     develop_sideboard()
 
 
-def install_pip_requirements_in_dir(dir_of_requirements_txt):
-    path_to_pip = __here__ / path('env/bin/pip')
+def install_pip_requirements_in_dir(dir_of_requirements_txt, env_path=None):
+    if env_path is None:
+        pip = __here__ / path('env/bin/pip')
+    else:
+        pip = path(env_path) / path('bin/pip')
 
-    print("---- installing dependencies in {} ----"
-        .format(dir_of_requirements_txt))
+    print("---- **PAVER** installing dependencies in {} ----".format(dir_of_requirements_txt))\
 
-    sh('{pip} install -e {dir_of_requirements_txt}'
-        .format(
-            pip=path_to_pip,
-            dir_of_requirements_txt=dir_of_requirements_txt))
+    # "-e" means "editable mode" which installs .egg-info in the local dir
+    sh(f'{pip} install -e {dir_of_requirements_txt}')
+    sh(f'{pip} install -r {dir_of_requirements_txt}/requirements.txt')
 
 
 def run_setup_py(path):
@@ -207,10 +208,13 @@ def create_plugin(options):
 
 
 @task
-def install_deps():
-    install_pip_requirements_in_dir(__here__)
-    for pdir in collect_plugin_dirs():
-        install_pip_requirements_in_dir(pdir)
+@cmdopts([
+    ('env_path=', 'e', "override path to virtualenv"),  # -e /some/path or --env_path=/some/path
+    ])
+def install_deps(options):
+    install_pip_requirements_in_dir(__here__, options.env_path)
+    for plugin_dir in collect_plugin_dirs():
+        install_pip_requirements_in_dir(plugin_dir, options.env_path)
 
 
 @task

requirements.txt

@@ -5,7 +5,9 @@ SQLAlchemy>=1.1.0
 six>=1.5.2
 Jinja2>=2.7
 rpctools>=0.3.1
-logging_unterpolation>=0.2.0
+# logging_unterpolation>=0.2.1   # this is right and fixes a bug we need, BUT it hasn't been published yet
+# instead, we can do this:
+git+https://github.com/robdennis/logging_unterpolation@62bbb658493f5f0cf04ad3b82a521b3b330a11f7#egg=logging_unterpolation
 requests>=2.2.1
 paver>=1.2.2
 wheel>=0.24.0

run_dev_server.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# run a "dev" server (for local development)
+
+set -e
+cd /app/sideboard/
+
+# TODO: need to add 'develop' in here
+/app/env/bin/paver install_deps --env_path=/app/env/
+
+bash ./run_server.sh

run_server.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+echo "-----------STARTING UBER SERVER--------------"
+
+cd /app/sideboard/
+/app/env/bin/python3 sideboard/run_server.py

setup.py

@@ -1,4 +1,3 @@
-import os
 import platform
 import sys
 import os.path
@@ -11,45 +10,40 @@
 with open(os.path.join(__here__, pkg_name, '_version.py')) as version:
     exec(version.read())
 # __version__ is now defined
-req_data = open(os.path.join(__here__, 'requirements.txt')).readlines()
-raw_requires = [r.strip() for r in req_data if r.strip() != '']
 
-# Ugly hack to reconcile pip requirements.txt and setup.py install_requires
 os_name = os.name
 sys_platform = sys.platform
 platform_release = platform.release()
 implementation_name = sys.implementation.name
 platform_machine = platform.machine()
 platform_python_implementation = platform.python_implementation()
-requires = []
-for s in reversed(raw_requires):
-    if ';' in s:
-        req, env_marker = s.split(';')
-        if eval(env_marker):
-            requires.append(s)
-    else:
-        requires.append(s)
 
+# --------------
+# Ugly, old hack to reconcile pip requirements.txt and setup.py install_requires
+# right now try to get away without this.
+# -------------
+# req_data = open(os.path.join(__here__, 'requirements.txt')).readlines()
+# raw_requires = [r.strip() for r in req_data if r.strip() != '']
+# requires = []
+# for s in reversed(raw_requires):
+#     if ';' in s:
+#         req, env_marker = s.split(';')
+#         if eval(env_marker):
+#             requires.append(s)
+#     else:
+#         requires.append(s)
 # testing dependencies
-req_data = open(os.path.join(__here__, 'test_requirements.txt')).read()
-tests_require = [r.strip() for r in req_data.split() if r.strip() != '']
-tests_require = list(reversed(tests_require))
-
-# temporary workaround for a Python 2 CherryPy bug, for which we opened a pull request:
-# https://bitbucket.org/cherrypy/cherrypy/pull-request/85/1285-python-2-now-accepts-both-bytestrings/
-if sys.version_info[0] == 2:
-    requires = ['CherryPy==3.2.2' if 'cherrypy' in r.lower() else r for r in requires]
+# req_data = open(os.path.join(__here__, 'test_requirements.txt')).read()
+# tests_require = [r.strip() for r in req_data.split() if r.strip() != '']
+# tests_require = list(reversed(tests_require))
 
 if __name__ == '__main__':
-    setup_requires = {'setup_requires': ['distribute']} if sys.version_info[0] == 2 else {}
     setup(
         name=pkg_name,
         version=__version__,
         description='Sideboard plugin container.',
         license='BSD',
         scripts=[],
-        install_requires=requires,
-        tests_require=tests_require,
         packages=find_packages(),
         include_package_data=True,
         package_data={pkg_name: []},
@@ -59,8 +53,10 @@
                 'sep = sideboard.sep:run_plugin_entry_point'
             ]
         },
-        extras_require={
-            'perftrace': ['python-prctl>=1.6.1', 'psutil>=4.3.0']
-        },
-        **setup_requires
+
+        # extras_require={
+        #     'perftrace': ['python-prctl>=1.6.1', 'psutil>=4.3.0']
+        # },
+        # install_requires=requires,
+        # tests_require=tests_require,
     )

sideboard/config.py

@@ -3,7 +3,7 @@
 import re
 
 from os import unlink
-from collections import Sized, Iterable, Mapping
+from collections.abc import Sized, Iterable, Mapping
 from copy import deepcopy
 from tempfile import NamedTemporaryFile
 

sideboard/lib/_utils.py

@@ -5,7 +5,8 @@
 from datetime import datetime, date
 from contextlib import contextmanager
 from threading import RLock, Condition, current_thread
-from collections import Sized, Iterable, Mapping, defaultdict
+from collections.abc import Sized, Iterable, Mapping
+from collections import defaultdict
 
 
 def is_listy(x):