[wip]: pr for testing auto deployment to multi env

.gitignore

@@ -27,6 +27,7 @@ __pycache__
 *.log
 *.pid
 /tests/.pytest_cache/
+/node_modules
 
 # Pycharm defaults
 .idea
@@ -79,3 +80,7 @@ docker/local.env
 
 # config dumps from make docker-compose-config-to-file
 config-compose-*
+
+# Frontend
+frontend/node_modules
+frontend/dist

checks/models.py

@@ -16,17 +16,6 @@ class ListField(models.TextField):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def from_db_value(self, value, expression, connection, context="Null"):
-        if value is None:
-            return value
-        try:
-            return ast.literal_eval(value)
-        except SyntaxError:
-            raise SyntaxError(
-                f"Syntax error while attempting to parse value as python, in ListField,"
-                f" possibly raw value has been stored instead of valid python code: {value}"
-            )
-
     def to_python(self, value):
         if not value:
             value = []
@@ -42,8 +31,11 @@ def get_prep_value(self, value):
         return str(value)
 
     def value_to_string(self, obj):
-        value = self._get_val_from_obj(obj)
-        return self.get_db_prep_value(value)
+        value = self.value_from_object(obj)
+        return self.get_prep_value(value)
+
+    def from_db_value(self, value, expression, connection):
+        return self.to_python(value)
 
 
 class AutoConfOption(Enum):

docker/Dockerfile

@@ -134,6 +134,18 @@ USER unbound
 
 ENTRYPOINT ["/entrypoint.sh"]
 
+# build frontend image target
+FROM --platform=linux/amd64 debian:bullseye-20241111-slim AS frontend
+
+# install npm from official image
+COPY --from=node:22-slim /usr/local/bin/ /usr/local/bin/
+COPY --from=node:22-slim /usr/local/lib/node_modules /usr/local/lib/node_modules
+
+ADD frontend/ /app/frontend
+WORKDIR /app/frontend
+RUN npm install
+RUN npm run docker:build
+
 # build main application image target
 FROM --platform=linux/amd64 debian:bullseye-20241111-slim AS build-app
 ARG PYTHON_VERSION
@@ -205,8 +217,10 @@ RUN install -d -m 0755 -o nobody -g nogroup /app/batch_results
 # some js/css files use a separate script to generate minimized versions
 # storage these in a place for django to find
 RUN mkdir -p /app/static_frontend
-RUN python3 bin/frontend.py js
-RUN python3 bin/frontend.py css
+# copy build/minimized CSS files from frontend stage
+COPY --from=frontend --chown=nobody:nogroup /app/static_frontend/css/*.css /app/static_frontend/css/
+# copy build/minimized JS files from frontend stage
+COPY --from=frontend --chown=nobody:nogroup /app/static_frontend/js/ /app/static_frontend/js/
 
 # generate language files
 RUN python3 bin/pofiles.py to_django
@@ -228,6 +242,19 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION=$RELEASE
 ENTRYPOINT [ "python3", "./manage.py"]
 CMD ["runserver", "0.0.0.0:8080"]
 
+# app image with development tools for better live reloading
+FROM app AS app-dev
+
+# required for compose `develop` mode file sync/copy
+USER root
+
+# copy dependencies to rebuild CSS (postcss)
+COPY --from=frontend /usr/local/bin/ /usr/local/bin/
+COPY --from=frontend /usr/local/lib/node_modules /usr/local/lib/node_modules
+
+# copy browser reload module
+COPY --from=linttest-deps /usr/local/lib/python${PYTHON_VERSION}/dist-packages/django_browser_reload /usr/local/lib/python${PYTHON_VERSION}/dist-packages/django_browser_reload
+
 # supplement application with unittest, lint dependencies
 FROM build-app AS linttest
 ARG PYTHON_VERSION

docker/compose.development.yaml

@@ -17,32 +17,23 @@ services:
     environment:
       - INTERNETNL_DOMAINNAME
 
-  webserver:
-    build:
-      context: ..
-      dockerfile: docker/webserver.Dockerfile
-    develop:
-      watch:
-        # auto rebuild/reload when config files change
-        - path: ./webserver/
-          action: rebuild
-
   app:
-    develop:
-      watch:
-        # auto rebuild/reload when CSS/JS changes
-        - path: ../frontend/
-          action: sync+exec
-          target: /app/frontend
-          exec:
-            command: /bin/sh -c "python3 bin/frontend.py js;python3 bin/frontend.py css;"
+    build:
+      target: app-dev
     # use Django runserver for better debug abilities during development
     entrypoint: ["/bin/bash"]
     # run CSS auto rebuild in the background and start devserver
     command:
       - "-c"
       - >
         if [ "$INTERNETNL_AUTORELOAD" = "True" ]; then
+          # install npm dependencies and watch CSS and JS files for rebuild
+          cd /app/frontend;
+          # It's needed to install the rollup linux dependency
+          npm install;
+          npm run docker:watch &
+          cd /app;
+          # start development server with auto reloading
           ./manage.py runserver 0.0.0.0:8080
         else
           ./manage.py runserver 0.0.0.0:8080 --noreload
@@ -55,6 +46,8 @@ services:
       - ../checks:/app/checks
       - ../interface:/app/interface
       - ../internetnl:/app/internetnl
+      # mount frontend files so build tools can detect changes and rebuild
+      - ../frontend/src:/app/frontend/src
 
   worker:
     volumes:

docker/webserver.Dockerfile

@@ -34,6 +34,8 @@ RUN mkdir -p /var/www/internet.nl/.well-known/
 # copy all security*.txt files
 COPY .well-known/security*.txt /var/www/internet.nl/.well-known/
 COPY interface/static/favicon.ico /var/www/internet.nl/
+COPY interface/static/favicon.svg /var/www/internet.nl/
+COPY interface/static/favicon.png /var/www/internet.nl/
 
 COPY docker/webserver/nginx_templates/* /etc/nginx/templates/
 COPY docker/webserver/mime.types /etc/nginx/

docker/webserver/nginx_templates/app.conf.template

@@ -249,6 +249,12 @@ server {
     location = /favicon.ico {
         alias /var/www/internet.nl/favicon.ico;
     }
+    location = /favicon.svg {
+        alias /var/www/internet.nl/favicon.svg;
+    }
+    location = /favicon.png {
+        alias /var/www/internet.nl/favicon.png;
+    }
 
     # static files served from app
     location /static {

documentation/Docker-development-environment.md

@@ -408,3 +408,31 @@ There are various cron jobs configured during normal deployments that perform ta
 ## Routinator
 
 For the development environment the Routinator service/container is not started but instead an external routinator service is used. This reduces resource usage and wait time.
+
+## Testdata/fixtures
+
+For frontend development purposes a set of testresults can be loaded into the database to so these don't have to be generated manually.
+
+To load this testdata run:
+
+    docker exec -ti internetnl-develop-app-1 ./manage.py loaddata example_reports
+
+The following urls can be used to view the testresults:
+
+http://localhost:8080/site/internet.nl/9000/
+http://localhost:8080/site/example.nl/9001/
+http://localhost:8080/site/ipv6.internet.nl/9002/
+http://localhost:8080/site/forfun.net/9003/
+http://localhost:8080/site/ipv6.google.com/9004/
+http://localhost:8080/site/brokendnssec.net/9005/
+http://localhost:8080/site/ok.bogussig.ok.bad-dnssec.wb.sidnlabs.nl/9006/
+http://localhost:8080/site/badhash.dane.huque.com/9007/
+http://localhost:8080/site/expired.badssl.com/9008/
+http://localhost:8080/site/wrong.host.badssl.com/9009/
+http://localhost:8080/site/self-signed.badssl.com/9010/
+http://localhost:8080/site/untrusted-root.badssl.com/9011/
+http://localhost:8080/site/revoked.badssl.com/9012/
+http://localhost:8080/site/pinning-test.badssl.com/9013/
+http://localhost:8080/site/invalid.rpki.isbgpsafeyet.com/9014/
+
+There are several scenarios with different failures in the testresults.