merge main into the branch

.github/workflows/dockerhub-publish.yml

@@ -5,7 +5,11 @@ on:
   release:
     types: [published]
 
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Docker image tag'
+        required: true
 
 jobs:
   publish_on_dockerhub:
@@ -21,9 +25,22 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Load .env file
+        uses: aarcangeli/load-dotenv@v1
+        with:
+          filenames: |
+            config.env
+
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
           push: true
-          tags: bhavyatech/celeryviz:${{ github.ref_name }}
+          tags: bhavyatech/celeryviz:${{ github.event.inputs.tag || github.ref_name }}
           platforms: linux/amd64,linux/arm64
+          build-args: FRONTEND_VERSION=${{ env.CELERYVIZ_FRONTEND_BUNDLED_VERSION }}

.github/workflows/python-publish.yml

@@ -11,26 +11,20 @@ name: Upload Python Package
 on:
   release:
     types: [published]
-
-  workflow_dispatch:
-    inputs:
-      environment:
-        type: environment
-        description: 'Manually trigger a deployment to the test PyPI environment'
-
+  workflow_dispatch: {}
 
 permissions:
   contents: read
 
 jobs:
-
   publish:
-    runs-on: ubuntu-latest
-
+    runs-on: ubuntu-latest 
     steps:
     - uses: actions/checkout@v4
     - name: Build UI
-      run: docker build --target export --output ./celeryviz/static ./ 
+      run: |
+        source config.env
+        docker build --target webapp-prebuilt --output . . --build-arg FRONTEND_VERSION=$CELERYVIZ_FRONTEND_BUNDLED_VERSION
     - name: Set up Python
       uses: actions/setup-python@v3
       with:

.gitignore

@@ -4,4 +4,5 @@
 */__pycache__/*
 celeryviz.egg-info/
 celeryviz/static/
-dist/*
+dist/*
+*/.DS_Store

Dockerfile

@@ -1,10 +1,25 @@
+#########################################
+# Docs                                  #
+#########################################
+# There are two ways to build the UI:
+# 1. Download the prebuilt UI
+#    a. If you want to build complete image with prebuilt UI, use the `celeryviz` target.
+#       Run: `docker build --target celeryviz`
+#    b. If you want to download only the prebuilt UI, use the `prebuilt` target.
+#       Run: `docker build --target webapp-prebuilt --output . .`
+# 2. Build from source
+#    a. If you want to build complete image from source, use the `celeryviz-with-frontend-build` target.
+#       Run: `docker build --target celeryviz-with-frontend-build`
+#    b. If you want to build only the UI, use the `webapp-build` target.
+#       Run: `docker build --target webapp-build --output . .`
+#    - The `GIT_REPO` and `SOURCE` build args can be passed for specific builds.
+#########################################
+
+
 #####################################
 # Docker stages for building the UI #
 #####################################
-
-# Run `docker build --target export --output ./celeryviz/static ./` to build the UI locally.
-
-FROM instrumentisto/flutter:3 AS base
+FROM instrumentisto/flutter:3 AS flutter_build_base
 
 # Set the working directory inside the container
 WORKDIR /app
@@ -13,7 +28,7 @@ WORKDIR /app
 RUN flutter precache
 
 # Build stage
-FROM base AS build
+FROM flutter_build_base AS webapp-compile
 
 ARG SOURCE="main"
 ARG GIT_REPO="https://github.com/bhavya-tech/celeryviz_with_lib.git"
@@ -24,45 +39,71 @@ RUN git checkout $SOURCE
 
 # Enable web support for Flutter
 RUN flutter config --enable-web
-
-# Now that the repo is cloned, we can run 'flutter pub get'
 RUN flutter pub get
-
-# Build the Flutter web app with CanvasKit renderer
 RUN flutter build web --release
 
-# Final stage: export the build files
-# This is needed other wise all the linux files will be copied to the final image.
-FROM scratch AS export
-COPY --from=build /app/celeryviz_with_lib/build/web /
+# Use this stage if static files are needed in the actual folder structure.
+FROM scratch AS webapp-build
+COPY --from=webapp-compile /app/celeryviz_with_lib/build/web /celeryviz/static/
 ###########################################
 ###########################################
 
 
+################################################
+# Docker stage to download the prebuilt webapp #
+################################################
+
+# Download and extract the prebuilt webapp
+# This is needed for users who do not want to build the webapp from source.
+FROM alpine:3.14 AS download-and-extract-prebuilt
+ARG FRONTEND_VERSION="main"
+ADD https://github.com/bhavya-tech/celeryviz_with_lib/releases/download/$FRONTEND_VERSION/webapp-build.zip /app/webapp-build.zip
+RUN apk add --no-cache unzip curl && \
+    unzip /app/webapp-build.zip -d /app/static && \
+    rm /app/webapp-build.zip
+
+# Use this stage if static files are needed in the actual folder structure.
+FROM scratch AS webapp-prebuilt
+COPY --from=download-and-extract-prebuilt /app/static/ /celeryviz/static/
+###########################################
+###########################################
+
 
 ###########################################
 # Docker stage for the Python application #
 ###########################################
 
-FROM python:3.9
+FROM python:3.9-alpine AS setup-celeryviz-dependency
 
 WORKDIR /app
-
 COPY . .
 
-# Build UI
-COPY --from=export / /app/celeryviz/static/
-
 # Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install --no-cache-dir redis
-RUN pip install .
+RUN pip install --no-cache-dir redis 
 
 # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`)
 # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`)
 # PYTHONDONTWRITEBYTECODE: Do not write byte files to disk, since we maintain it as readonly. (equivalent to `python -B`)
 ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1
-
 EXPOSE 9095
+###########################################
+###########################################
+
+
+#######################
+# Final docker stages #
+#######################
+
+# This stage builds celeryviz with the frontend built from the source.
+FROM setup-celeryviz-dependency AS celeryviz-with-frontend-build
+COPY --from=webapp-compile /app/celeryviz_with_lib/build/web /app/celeryviz/static/
+RUN pip install .
+
+# This stage builds celeryviz with the prebuilt frontend.
+FROM setup-celeryviz-dependency AS celeryviz
+WORKDIR /app
+COPY --from=download-and-extract-prebuilt /app/static/ /app/celeryviz/static/
+RUN pip install .
+RUN cd .. && rm -rf /app/
 
-CMD ["celery", "celeryviz"]

config.env

@@ -0,0 +1,5 @@
+# Version name for the CeleryViz Python package
+CELERYVIZ_PYTHON_VERSION=0.0.5
+
+# Version name for the CeleryViz bundled frontend package.
+CELERYVIZ_FRONTEND_BUNDLED_VERSION=0.0.3

example/Dockerfile

@@ -5,5 +5,4 @@ WORKDIR /app
 
 RUN pip install redis celery celeryviz
 
-COPY . .
-CMD ["celery", "-A", "example_app", "worker", "--loglevel=info", "-E"]
+COPY . .

example/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   redis:
     image: redis:alpine
@@ -29,6 +27,8 @@ services:
     build:
       context: ../.
       dockerfile: Dockerfile
+      target: celeryviz
+
     command: celery --broker='redis://redis:6379/0' celeryviz
     depends_on:
       - redis

pyproject.toml

@@ -1,4 +1,4 @@
 [build-system]
-requires = ["setuptools", "setuptools-scm"]
+requires = ["setuptools", "setuptools-scm", "python-dotenv"]
 build-backend = "setuptools.build_meta"
 

setup.py

@@ -1,8 +1,12 @@
+from dotenv import load_dotenv
+import os
 from setuptools import setup, find_packages
 
 def get_requirements():
     return open('./requirements.txt').read().splitlines()
 
+load_dotenv("config.env")
+
 classes = """
     Development Status :: 1 - Planning
     Intended Audience :: Developers
@@ -25,7 +29,7 @@ def get_requirements():
     description= "A UI centric tool for visualising Celery task execution.",
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
-    version='0.0.3',
+    version=os.getenv("CELERYVIZ_PYTHON_VERSION", "latest"),
     classifiers=classifiers,
     author = "Bhavya Peshavaria",
     packages=find_packages(),

tests/test_eventlistener.py

@@ -25,7 +25,7 @@ def run(self):
 class TestEventListener(unittest.TestCase):
 
     def setUp(self) -> None:
-        self.app = Celery('example_app')
+        self.app = Celery('example_app', broker='redis://127.0.0.1:6379/0')
         self.event_handler_process = EventHandlerProcess(
             self.app, AsyncMock(), daemon=True)
         return super().setUp()

tests/test_integration.py

@@ -1,6 +1,7 @@
 import multiprocessing
 import threading
 import time
+from types import SimpleNamespace
 import unittest
 from unittest.mock import Mock
 from celery import Celery
@@ -25,21 +26,27 @@ def run(self):
         self.client.wait()
 
 
-# This test is not working. It is not able to connect to the server. I am not sure why.
-# Keeping it disconnected until its resolved, maybe someone can lok into this.
-class TestIntegration():
+class MockCtx:
     """
-    This test the backend end to end. It starts the server and client and checks if the client receives the event.
+    Custom mock context class as unittest.mock.Mock does not pickle.
+    """
+
+    def __init__(self, app):
+        self.obj = SimpleNamespace()
+        self.obj.app = app
 
+
+class TestIntegration(unittest.TestCase):
+    """
+    This test the backend end to end. It starts the server and client and checks if the client receives the event.
     """
 
     payload = {"utcoffset": -6, "uuid": "4897c640-a023-4cb8-ae8e-1df4641a3ba1", "name": "basic_task.add", "args": "()", "kwargs": "{}",
                "root_id": "4897c640-a023-4cb8-ae8e-1df4641a3ba1", "parent_id": None, "retries": 0, "eta": None, "expires": None, "timestamp": 1685367125.9317474, "type": "task-received"}
 
     def setUp(self) -> None:
-        self.app = Celery('example_app')
-        self.mock_ctx = Mock()
-        self.mock_ctx.obj.app = self.app
+        self.app = Celery('example_app', broker='redis://127.0.0.1:6379/0')
+        self.mock_ctx = MockCtx(self.app)
 
         self.on_event = Mock()
         self.server_process = multiprocessing.Process(