OBS-385: Introduce ESBuild for CSS and Images (#6864)

* OBS-385: use ESBuild to bundle crashstats_base as first test case

* OBS-385: remove unwanted npm scripts

* OBS-385: create anonymous volumes for node_modules

fix lint

* OBS-385: add `webapp/static` anonymous volume

OBS-385: fix tests

* OBS-385: ignore collectstatic css and other things

* OBS-385: update ESBuild entrypoints

* OBS-385: use staticfiles backend storage instead of pipeline

* OBS-385: handle media assets with ESBuild

* OBS-385: build CSS and images using ESBuild

* OBS-385: update HTML files to use ESBuild CSS and images

* OBS-385: revert work-around for status.css

* OBS-385: revert staticfiles storage to fix js files not served in production

* OBS-385: update collectstatic "ignore" specificity to allow admin css

* OBS-385: use npm script in test.sh for consolidation

* OBS-397: add ESBuild "watch" mode

* OBS-385: remove STATIC_ROOT declaration from local_dev.env

* OBS-385: restore bundle quality check, removing PIPELINE_CSS

* OBS-385: update webapp documentation

Author: toufali

.devcontainer/Dockerfile

@@ -36,11 +36,11 @@ RUN pip install --no-cache-dir --no-deps -r requirements.txt && \
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     PYTHONPATH=/app \
-    UGLIFYJS_BINARY=/webapp-frontend-deps/node_modules/.bin/uglifyjs \
-    CSSMIN_BINARY=/webapp-frontend-deps/node_modules/.bin/cssmin \
-    NPM_ROOT_PATH=/webapp-frontend-deps/ \
-    NODE_PATH=/webapp-frontend-deps/node_modules/
+    UGLIFYJS_BINARY=/app/webapp/node_modules/.bin/uglifyjs \
+    NPM_ROOT_PATH=/app/webapp/ \
+    NODE_PATH=/app/webapp/node_modules/
 
 # Install frontend JS deps
-COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/
-RUN cd /webapp-frontend-deps/ && npm install
+COPY --chown=app:app ./webapp/package*.json /app/webapp/
+RUN cd /app/webapp/ && npm ci
+

.dockerignore

@@ -4,6 +4,5 @@ build/
 **/*.pyc
 __pycache__
 .cache
-webapp-django/node_modules
-webapp-django/static
-symbols/
+node_modules
+symbols/

.eslintrc.json

@@ -1,7 +1,8 @@
 {
   "env": {
       "browser": true,
-      "jquery": true
+      "jquery": true,
+      "node": true
   },
   "extends": ["plugin:prettier/recommended", "eslint:recommended"],
   "parserOptions": {

bin/lint.sh

@@ -46,5 +46,5 @@ else
 
     echo ">>> eslint (js)"
     cd /app/webapp
-    /webapp-frontend-deps/node_modules/.bin/eslint .
-fi
+    /app/webapp/node_modules/.bin/eslint .
+fi

bin/run_service_webapp.sh

@@ -29,7 +29,7 @@ if [ "${1:-}" == "--dev" ]; then
     echo "Running webapp in local dev environment."
     echo "Connect with your browser using: http://localhost:8000/ "
     echo "******************************************************************"
-    cd /app/webapp/ && exec ${CMDPREFIX} python manage.py runserver 0.0.0.0:8000
+    cd /app/webapp/ && (node esbuild --watch & exec ${CMDPREFIX} python manage.py runserver 0.0.0.0:8000)
 
 else
     exec ${CMDPREFIX} gunicorn \

bin/test.sh

@@ -50,6 +50,6 @@ echo ">>> run tests"
 
 # Collect static and then run pytest in the webapp
 pushd webapp
-${PYTHON} manage.py collectstatic --noinput
+npm run build
 "${PYTEST}"
 popd

docker-compose.override.yml

@@ -4,9 +4,14 @@ services:
   app:
     volumes:
       - .:/app
+      - /app/webapp/node_modules
+      - /app/webapp/static
+
   test:
     volumes:
       - .:/app
+      - /app/webapp/node_modules
+      - /app/webapp/static
 
   processor:
     volumes:
@@ -19,6 +24,8 @@ services:
   webapp:
     volumes:
       - .:/app
+      - /app/webapp/node_modules
+      - /app/webapp/static
 
   stage_submitter:
     volumes:

docker/Dockerfile

@@ -33,25 +33,23 @@ RUN pip install --no-cache-dir --no-deps -r requirements.txt && \
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     PYTHONPATH=/app \
-    UGLIFYJS_BINARY=/webapp-frontend-deps/node_modules/.bin/uglifyjs \
-    CSSMIN_BINARY=/webapp-frontend-deps/node_modules/.bin/cssmin \
-    NPM_ROOT_PATH=/webapp-frontend-deps/ \
-    NODE_PATH=/webapp-frontend-deps/node_modules/
+    UGLIFYJS_BINARY=/app/webapp/node_modules/.bin/uglifyjs \
+    NPM_ROOT_PATH=/app/webapp/ \
+    NODE_PATH=/app/webapp/node_modules/
 
 # Install frontend JS deps
-COPY --chown=app:app ./webapp/package*.json /webapp-frontend-deps/
-RUN cd /webapp-frontend-deps/ && npm install
+COPY --chown=app:app ./webapp/package*.json /app/webapp/
+RUN cd /app/webapp/ && npm ci
 
 # app should own everything under /app in the container
 USER app
 
 # Copy everything over
 COPY --chown=app:app . /app/
 
-# Run collectstatic in container which puts files in the default place for
-# static files
-RUN cd /app/webapp/ && TOOL_ENV=True python manage.py collectstatic --noinput
+# Build front-end static files. Runs ESBuild and collectstatic
+RUN cd /app/webapp/ && npm run build
 
 # Set entrypoint for this image. The entrypoint script takes a service
 # to run as the first argument. See the script for available arguments.
-ENTRYPOINT ["/app/bin/entrypoint.sh"]
+ENTRYPOINT ["/app/bin/entrypoint.sh"]

docker/config/local_dev.env

@@ -62,16 +62,6 @@ VERSIONS_COUNT_THRESHOLD=0
 # Django DEBUG mode which shows settings and tracebacks on errors
 DEBUG=True
 
-# Static files are generated as part of the image and reside in
-# /app/webapp/static/ which is the default location. Thus for server
-# environments, you can leave STATIC_ROOT unset.
-#
-# For local development, the local directory is mounted as /app so the static
-# files generated in the image are not available. For local development,
-# static files for the webapp get put in /tmp/crashstats-static/ so we
-# need to set STATIC_ROOT to that.
-STATIC_ROOT=/tmp/crashstats-static/
-
 # For webapp sessions in the local dev environment, we need to allow cookies to
 # be sent insecurely since it's using HTTP and not HTTPS.
 SESSION_COOKIE_SECURE=False

docs/service/webapp.rst

@@ -8,25 +8,79 @@ Code is in ``webapp/``.
 
 Run script is ``/app/bin/run_service_webapp.sh``.
 
+Running in a local dev environment
+==================================
+
+This documentation assumes you've gone through the setup steps described in the Development chapter :ref:`setup-quickstart`, in particular:
+
+  .. code-block:: shell
+
+      $ just build
+      $ just setup
+
+To run the webapp...
+
+  .. code-block:: shell
+
+      $ docker compose up webapp
+
+...or if you don't like typing:
+
+  .. code-block:: shell
+
+      $ just run
 
-Configuration
+To ease debugging, you can run a shell in the container:
+
+  .. code-block:: shell
+      $ docker compose run --service-ports webapp shell
+
+Then you can start and stop the webapp, adjust files, and debug.  
+The webapp runs ESBuild's watch mode and Django's StatReloader to reload static file changes automatically.
+This avoids needing to stop, rebuild, and restart the container/server on every change.
+
+Static Assets
 =============
 
-FIXME
+At the time of this writing, JS files are collected and processed by collectstatic and django-pipeline.  All other static assets (CSS, images, fonts, etc) are collected and processed by ESBuild.
+Migration of JS to ESBuild is currently in progress, with the intent to retire django-pipeline when complete.  The collectstatic package will continue to be used in support of the internal Django admin pages.
 
+Static asset builds are triggered by NPM scripts in ``webapp/package.json``.  The assets are built into ``/app/webapp/static`` also known as ``STATIC_ROOT``.
 
-Running in a local dev environment
-==================================
+Production-style Assets
+=======================
 
-To run the webapp, do::
+When you run ``docker compose up webapp`` in the local development environment,
+it starts the web app using Django's ``runserver`` command. ``DEBUG=True`` is
+set in the ``docker/config/local_dev.env`` file, so static assets are
+automatically served from within the individual Django apps rather than serving
+the minified and concatenated static assets you'd get in a production-like
+environment.
 
-  $ docker compose up webapp
+If you want to run the web app in a more "prod-like manner", you want to run the
+webapp using ``gunicorn`` and with ``DEBUG=False``. Here's how you do that.
 
-To ease debugging, you can run a shell in the container::
+First start a ``bash`` shell with service ports::
 
   $ docker compose run --service-ports webapp shell
 
-Then you can start and stop the webapp, adjust files, and debug.
+Compile the static assets (if needed)::
+
+  app@socorro:/app$ npm run build --prefix webapp
+
+Then run the webapp with ``gunicorn`` and ``DEBUG=False``::
+
+  app@socorro:/app$ DEBUG=False bash bin/run_service_webapp.sh
+
+You will now be able to open ``http://localhost:8000`` on the host and if you
+view the source you see that the minified and concatenated static assets are
+served instead.
+
+Because static assets are compiled, if you change JS or CSS files, you'll need
+to re-run ``npm run build --prefix webapp`` - the "watch mode" feature is not enabled in production.
+
+Admin Account
+=============
 
 If you want to do anything in the webapp admin, you'll need to create a
 superuser in the Crash Stats webapp and a OIDC account to authenticate against
@@ -80,87 +134,4 @@ A logged-in user can view their detailed permissions on the
 
 The groups and their permissions are defined in
 ``webapp/crashstats/crashstats/signals.py``. These are applied to
-the database in a "post-migrate" signal handler.
-
-
-Static Assets
-=============
-
-In the development environment, the ``STATIC_ROOT`` is set to
-``/tmp/crashstats-static/`` rather than ``/app/webapp/static``.
-The process in the container creates files with the uid 10001, and Linux users
-will have permissions-related problems if these are mounted on the host
-computer.
-
-The problem this creates is that ``/tmp/crashstats-static/`` is ephemeral
-and any changes there disappear when you stop the container.
-
-If you are on macOS or Windows, then Docker uses a shared file system that
-creates files with your user ID. This makes it safe to persist static assets,
-at the cost of slower file system performance. Linux users can manually set
-the uid and gid to match their account, for the same effect. See "Set UID and
-GID for Docker container user" in :ref:`setup-quickstart`.
-
-If you want static assets to persist between container restarts, then you
-can override ``STATIC_ROOT`` in ``my.env`` to return it to the ``app`` folder::
-
-    STATIC_ROOT=/app/static
-
-Alternatively, you can mount ``/tmp/crashstats-static/`` using ``volumes``
-in a ``docker compose.override.yml`` file:
-
-.. code-block:: yaml
-
-    version: "2"
-    services:
-      webapp:
-        volumes:
-          # Persist the static files folder
-          - ./static:/tmp/crashstats-static
-
-
-Production-style Assets
-=======================
-
-When you run ``docker compose up webapp`` in the local development environment,
-it starts the web app using Django's ``runserver`` command. ``DEBUG=True`` is
-set in the ``docker/config/local_dev.env`` file, so static assets are
-automatically served from within the individual Django apps rather than serving
-the minified and concatenated static assets you'd get in a production-like
-environment.
-
-If you want to run the web app in a more "prod-like manner", you want to run the
-webapp using ``gunicorn`` and with ``DEBUG=False``. Here's how you do that.
-
-First start a ``bash`` shell with service ports::
-
-  $ docker compose run --service-ports webapp shell
-
-Then compile the static assets::
-
-  app@socorro:/app$ cd webapp/
-  app@socorro:/app/webapp$ ./manage.py collectstatic --noinput
-  app@socorro:/app/webapp$ cd ..
-
-Now run the webapp with ``gunicorn`` and ``DEBUG=False``::
-
-  app@socorro:/app$ DEBUG=False bash bin/run_service_webapp.sh
-
-You will now be able to open ``http://localhost:8000`` on the host and if you
-view the source you see that the minified and concatenated static assets are
-served instead.
-
-Because static assets are compiled, if you change JS or CSS files, you'll need
-to re-run ``./manage.py collectstatic``.
-
-
-Running in a server environment
-===============================
-
-Add configuration to ``webapp.env`` file.
-
-Run the docker image using the ``webapp`` command. Something like this::
-
-    docker run \
-        --env-file=webapp.env \
-        mozilla/socorro_app webapp
+the database in a "post-migrate" signal handler.

webapp/crashstats/api/jinja2/api/documentation.html

@@ -2,7 +2,7 @@
 
 {% block site_css %}
 {{ super() }}
-{% stylesheet 'api_documentation' %}
+<link rel="stylesheet" href="/static/api/css/documentation.min.css">
 {% endblock %}
 
 {% block site_js %}
@@ -111,7 +111,7 @@ <h2><a href="#{{ endpoint.name }}">{{ endpoint.name }}</a></h2>
       <div class="run-test">
         {% if endpoint.test_drive %}
           <button type="submit">Run Test Drive!</button>
-          <img src="{{ static('img/ajax-loader16x16.gif') }}" alt="Loading..." class="loading-ajax">
+          <img src="/static/img/ajax-loader16x16.gif" alt="Loading..." class="loading-ajax">
           <button type="button" class="close">&times; Close</button>
         {% else %}
           <em>Test drive not supported.</em>

webapp/crashstats/crashstats/finders.py

@@ -20,16 +20,15 @@ def find(self, path, all=False):
         # staticfiles finders. Before we raise an error, try to find out where,
         # in the bundles, this was defined. This will make it easier to correct
         # the mistake.
-        for config_name in "STYLESHEETS", "JAVASCRIPT":
-            config = settings.PIPELINE[config_name]
-            for key, directive in config.items():
-                if path in directive["source_filenames"]:
-                    raise ImproperlyConfigured(
-                        "Static file {} can not be found anywhere. Defined in "
-                        "PIPELINE[{!r}][{!r}]['source_filenames']".format(
-                            path, config_name, key
-                        )
+        config = settings.PIPELINE["JAVASCRIPT"]
+        for key, directive in config.items():
+            if path in directive["source_filenames"]:
+                raise ImproperlyConfigured(
+                    "Static file {} can not be found anywhere. Defined in "
+                    "PIPELINE[{!r}][{!r}]['source_filenames']".format(
+                        path, "JAVASCRIPT", key
                     )
+                )
         # If the file can't be found AND it's not in bundles, there's
         # got to be something else really wrong.
         raise NotImplementedError(path)

webapp/crashstats/crashstats/jinja2/crashstats/product_home.html

@@ -2,7 +2,7 @@
 
 {% block site_css %}
 {{ super() }}
-{% stylesheet 'product_home' %}
+<link rel="stylesheet" href="/static/crashstats/css/pages/product_home.min.css">
 {% endblock %}
 
 {% block page_title %}