From ebe1c4cb4b08cc49b6dd8ba88d585c2285464799 Mon Sep 17 00:00:00 2001 From: Bhavya Peshavaria <64970880+bhavya-tech@users.noreply.github.com> Date: Sat, 23 Aug 2025 16:52:03 +0530 Subject: [PATCH 1/8] User prebuilt webapp (#20) * modified the dockerfile to use prebuilt webapp * modified to use the celeryviz_with_lib repo --- .github/workflows/greetings.yml | 16 ------ .github/workflows/python-publish.yml | 13 +---- .gitignore | 3 +- Dockerfile | 82 +++++++++++++++++++++------- example/Dockerfile | 3 +- example/docker-compose.yml | 4 +- 6 files changed, 68 insertions(+), 53 deletions(-) delete mode 100644 .github/workflows/greetings.yml diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml deleted file mode 100644 index c776f54..0000000 --- a/.github/workflows/greetings.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Greetings - -on: [pull_request_target, issues] - -jobs: - greeting: - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - steps: - - uses: actions/first-interaction@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: "๐Ÿš€ **Welcome, First-Time Contributor!** ๐Ÿš€ \nThanks for jumping in! Whether itโ€™s an issue or a PR, weโ€™re excited to have you on board. Your contribution is on its way to being reviewed. Feel free to explore or reach out with any questions. \ Letโ€™s make something awesome together! ๐ŸŒŸ" - pr-message: "๐Ÿš€ **Welcome, First-Time Contributor!** ๐Ÿš€ \nThanks for jumping in! Whether itโ€™s an issue or a PR, weโ€™re excited to have you on board. Your contribution is on its way to being reviewed. Feel free to explore or reach out with any questions. \ Letโ€™s make something awesome together! ๐ŸŒŸ" diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 9cc3f70..6380526 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,26 +11,17 @@ name: Upload Python Package on: release: types: [published] - - workflow_dispatch: - inputs: - environment: - type: environment - description: 'Manually trigger a deployment to the test PyPI environment' - permissions: contents: read jobs: - publish: - runs-on: ubuntu-latest - + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build UI - run: docker build --target export --output ./celeryviz/static ./ + run: docker build --target webapp-build --output . . - name: Set up Python uses: actions/setup-python@v3 with: diff --git a/.gitignore b/.gitignore index 8d6ae47..d7f1ba4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ */__pycache__/* celeryviz.egg-info/ celeryviz/static/ -dist/* \ No newline at end of file +dist/* +celeryviz/.DS_Store diff --git a/Dockerfile b/Dockerfile index c76298e..4576e95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,25 @@ +######################################### +# Docs # +######################################### +# There are two ways to build the UI: +# 1. Download the prebuilt UI +# a. If you want to build complete image with prebuilt UI, use the `celeryviz` target. +# Run: `docker build --target celeryviz` +# b. If you want to download only the prebuilt UI, use the `prebuilt` target. +# Run: `docker build --target webapp-prebuilt --output . .` +# 2. Build from source +# a. If you want to build complete image from source, use the `celeryviz-with-frontend-build` target. +# Run: `docker build --target celeryviz-with-frontend-build` +# b. If you want to build only the UI, use the `webapp-build` target. +# Run: `docker build --target webapp-build --output . .` +# - The `GIT_REPO` and `SOURCE` build args can be passed for specific builds. +######################################### + + ##################################### # Docker stages for building the UI # ##################################### - -# Run `docker build --target export --output ./celeryviz/static ./` to build the UI locally. - -FROM instrumentisto/flutter:3 AS base +FROM instrumentisto/flutter:3 AS flutter_build_base # Set the working directory inside the container WORKDIR /app @@ -13,7 +28,7 @@ WORKDIR /app RUN flutter precache # Build stage -FROM base AS build +FROM flutter_build_base AS webapp-compile ARG SOURCE="main" ARG GIT_REPO="https://github.com/bhavya-tech/celeryviz_with_lib.git" @@ -24,45 +39,70 @@ RUN git checkout $SOURCE # Enable web support for Flutter RUN flutter config --enable-web - -# Now that the repo is cloned, we can run 'flutter pub get' RUN flutter pub get - -# Build the Flutter web app with CanvasKit renderer RUN flutter build web --release -# Final stage: export the build files -# This is needed other wise all the linux files will be copied to the final image. -FROM scratch AS export -COPY --from=build /app/celeryviz_with_lib/build/web / +# Use this stage if static files are needed in the actual folder structure. +FROM scratch AS webapp-build +COPY --from=webapp-compile /app/celeryviz_with_lib/build/web /celeryviz/static/ ########################################### ########################################### +################################################ +# Docker stage to download the prebuilt webapp # +################################################ + +# Download and extract the prebuilt webapp +# This is needed for users who do not want to build the webapp from source. +FROM alpine:3.14 AS download-and-extract-prebuilt +ADD https://github.com/bhavya-tech/celeryviz_with_lib/releases/download/0.0.1/webapp-build.zip /app/webapp-build.zip +RUN apk add --no-cache unzip curl && \ + unzip /app/webapp-build.zip -d /app/static && \ + rm /app/webapp-build.zip + +# Use this stage if static files are needed in the actual folder structure. +FROM scratch AS webapp-prebuilt +COPY --from=download-and-extract-prebuilt /app/static/ /celeryviz/static/ +########################################### +########################################### + ########################################### # Docker stage for the Python application # ########################################### -FROM python:3.9 +FROM python:3.9-alpine AS setup-celeryviz-dependency WORKDIR /app - COPY . . -# Build UI -COPY --from=export / /app/celeryviz/static/ - # Install Python dependencies RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir redis -RUN pip install . # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`) # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`) # PYTHONDONTWRITEBYTECODE: Do not write byte files to disk, since we maintain it as readonly. (equivalent to `python -B`) ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1 - EXPOSE 9095 +########################################### +########################################### + + +####################### +# Final docker stages # +####################### + +# This stage builds celeryviz with the frontend built from the source. +FROM setup-celeryviz-dependency AS celeryviz-with-frontend-build +COPY --from=webapp-compile /app/celeryviz_with_lib/build/web /app/celeryviz/static/ +RUN pip install . + +# This stage builds celeryviz with the prebuilt frontend. +FROM setup-celeryviz-dependency AS celeryviz +WORKDIR /app +COPY --from=download-and-extract-prebuilt /app/static/ /app/celeryviz/static/ +RUN pip install . +RUN cd .. && rm -rf /app/ -CMD ["celery", "celeryviz"] diff --git a/example/Dockerfile b/example/Dockerfile index 5f281aa..bd24053 100644 --- a/example/Dockerfile +++ b/example/Dockerfile @@ -5,5 +5,4 @@ WORKDIR /app RUN pip install redis celery celeryviz -COPY . . -CMD ["celery", "-A", "example_app", "worker", "--loglevel=info", "-E"] \ No newline at end of file +COPY . . \ No newline at end of file diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 957bfb5..a86fa42 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: redis: image: redis:alpine @@ -29,6 +27,8 @@ services: build: context: ../. dockerfile: Dockerfile + target: celeryviz + command: celery --broker='redis://redis:6379/0' celeryviz depends_on: - redis From a56550181e361f64d33cf362c7cdd4510e20c342 Mon Sep 17 00:00:00 2001 From: bhavya-tech <64970880+bhavya-tech@users.noreply.github.com> Date: Sat, 23 Aug 2025 17:04:38 +0530 Subject: [PATCH 2/8] version bump and fixed multiplatofrm docker build --- .github/workflows/dockerhub-publish.yml | 6 ++++++ .gitignore | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-publish.yml b/.github/workflows/dockerhub-publish.yml index 4b28e18..0418174 100644 --- a/.github/workflows/dockerhub-publish.yml +++ b/.github/workflows/dockerhub-publish.yml @@ -21,6 +21,12 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push uses: docker/build-push-action@v5 with: diff --git a/.gitignore b/.gitignore index d7f1ba4..0752264 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ celeryviz.egg-info/ celeryviz/static/ dist/* -celeryviz/.DS_Store +*/.DS_Store diff --git a/setup.py b/setup.py index 4ff2a6e..2f1b72a 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def get_requirements(): description= "A UI centric tool for visualising Celery task execution.", long_description=open('README.md').read(), long_description_content_type='text/markdown', - version='0.0.3', + version='0.0.4', classifiers=classifiers, author = "Bhavya Peshavaria", packages=find_packages(), From d0e370af933934e3d5fd756052fb99eb8a75b333 Mon Sep 17 00:00:00 2001 From: bhavya-tech <64970880+bhavya-tech@users.noreply.github.com> Date: Sat, 23 Aug 2025 17:23:15 +0530 Subject: [PATCH 3/8] add tags for manual workflow trigger --- .github/workflows/dockerhub-publish.yml | 9 +++++++-- .github/workflows/python-publish.yml | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-publish.yml b/.github/workflows/dockerhub-publish.yml index 0418174..66bc509 100644 --- a/.github/workflows/dockerhub-publish.yml +++ b/.github/workflows/dockerhub-publish.yml @@ -5,7 +5,12 @@ on: release: types: [published] - workflow_dispatch: {} + workflow_dispatch: + inputs: + tag: + description: 'Docker image tag' + required: true + default: 'latest' jobs: publish_on_dockerhub: @@ -31,5 +36,5 @@ jobs: uses: docker/build-push-action@v5 with: push: true - tags: bhavyatech/celeryviz:${{ github.ref_name }} + tags: bhavyatech/celeryviz:${{ github.ref_name || github.event.inputs.tag }} platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 6380526..d34f9b6 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,6 +11,7 @@ name: Upload Python Package on: release: types: [published] + workflow_dispatch: {} permissions: contents: read From 466a5941822c6106d6bd2fde5f087e6a60d1418a Mon Sep 17 00:00:00 2001 From: bhavya-tech <64970880+bhavya-tech@users.noreply.github.com> Date: Sat, 23 Aug 2025 17:31:10 +0530 Subject: [PATCH 4/8] minor docker publish fix --- .github/workflows/dockerhub-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-publish.yml b/.github/workflows/dockerhub-publish.yml index 66bc509..483428b 100644 --- a/.github/workflows/dockerhub-publish.yml +++ b/.github/workflows/dockerhub-publish.yml @@ -10,7 +10,6 @@ on: tag: description: 'Docker image tag' required: true - default: 'latest' jobs: publish_on_dockerhub: @@ -36,5 +35,5 @@ jobs: uses: docker/build-push-action@v5 with: push: true - tags: bhavyatech/celeryviz:${{ github.ref_name || github.event.inputs.tag }} + tags: bhavyatech/celeryviz:${{ github.event.inputs.tag || github.ref_name }} platforms: linux/amd64,linux/arm64 From cd4368aa9c5f706f11379f844d9eb89dd28fb0bb Mon Sep 17 00:00:00 2001 From: bhavya-tech <64970880+bhavya-tech@users.noreply.github.com> Date: Tue, 9 Sep 2025 19:23:07 +0530 Subject: [PATCH 5/8] version 0.0.5 --- .github/workflows/dockerhub-publish.yml | 7 +++++++ .github/workflows/python-publish.yml | 4 +++- Dockerfile | 3 ++- config.env | 5 +++++ requirements.txt | 1 + setup.py | 6 +++++- 6 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 config.env diff --git a/.github/workflows/dockerhub-publish.yml b/.github/workflows/dockerhub-publish.yml index 483428b..510342f 100644 --- a/.github/workflows/dockerhub-publish.yml +++ b/.github/workflows/dockerhub-publish.yml @@ -31,9 +31,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Load .env file + uses: aarcangeli/load-dotenv@v1 + with: + filenames: | + config.env + - name: Build and push uses: docker/build-push-action@v5 with: push: true tags: bhavyatech/celeryviz:${{ github.event.inputs.tag || github.ref_name }} platforms: linux/amd64,linux/arm64 + build-args: FRONTEND_VERSION=${{ env.CELERYVIZ_FRONTEND_BUNDLED_VERSION }} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index d34f9b6..7142a77 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -22,7 +22,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build UI - run: docker build --target webapp-build --output . . + run: | + source config.env + docker build --target webapp-prebuilt --output . . --build-arg FRONTEND_VERSION=$CELERYVIZ_FRONTEND_BUNDLED_VERSION - name: Set up Python uses: actions/setup-python@v3 with: diff --git a/Dockerfile b/Dockerfile index 4576e95..79894e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,8 @@ COPY --from=webapp-compile /app/celeryviz_with_lib/build/web /celeryviz/static/ # Download and extract the prebuilt webapp # This is needed for users who do not want to build the webapp from source. FROM alpine:3.14 AS download-and-extract-prebuilt -ADD https://github.com/bhavya-tech/celeryviz_with_lib/releases/download/0.0.1/webapp-build.zip /app/webapp-build.zip +ARG FRONTEND_VERSION="main" +ADD https://github.com/bhavya-tech/celeryviz_with_lib/releases/download/$FRONTEND_VERSION/webapp-build.zip /app/webapp-build.zip RUN apk add --no-cache unzip curl && \ unzip /app/webapp-build.zip -d /app/static && \ rm /app/webapp-build.zip diff --git a/config.env b/config.env new file mode 100644 index 0000000..c44a517 --- /dev/null +++ b/config.env @@ -0,0 +1,5 @@ +# Version name for the CeleryViz Python package +CELERYVIZ_PYTHON_VERSION=0.0.5 + +# Version name for the CeleryViz bundled frontend package. +CELERYVIZ_FRONTEND_BUNDLED_VERSION=0.0.3 diff --git a/requirements.txt b/requirements.txt index 72a541c..78e8269 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ pluggy==1.5.0 prompt-toolkit==3.0.43 pytest==8.2.1 python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 python-engineio==4.9.1 python-socketio==5.11.2 requests==2.32.2 diff --git a/setup.py b/setup.py index 2f1b72a..7d448c1 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,12 @@ +from dotenv import load_dotenv +import os from setuptools import setup, find_packages def get_requirements(): return open('./requirements.txt').read().splitlines() +load_dotenv("config.env") + classes = """ Development Status :: 1 - Planning Intended Audience :: Developers @@ -25,7 +29,7 @@ def get_requirements(): description= "A UI centric tool for visualising Celery task execution.", long_description=open('README.md').read(), long_description_content_type='text/markdown', - version='0.0.4', + version=os.getenv("CELERYVIZ_PYTHON_VERSION", "latest"), classifiers=classifiers, author = "Bhavya Peshavaria", packages=find_packages(), From 5d3ec6f633c7c39360c807fe735c00d5108f1609 Mon Sep 17 00:00:00 2001 From: bhavya-tech <64970880+bhavya-tech@users.noreply.github.com> Date: Tue, 9 Sep 2025 19:27:11 +0530 Subject: [PATCH 6/8] fixed dotenv not found --- .github/workflows/python-publish.yml | 2 +- Dockerfile | 2 +- requirements.txt | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 7142a77..f362b8d 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -32,7 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install build + pip install build python-dotenv==1.1.1 - name: Build package run: python -m build - name: Publish package diff --git a/Dockerfile b/Dockerfile index 79894e7..a997ed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,7 @@ COPY . . # Install Python dependencies RUN pip install --no-cache-dir -r requirements.txt -RUN pip install --no-cache-dir redis +RUN pip install --no-cache-dir redis python-dotenv==1.1.1 # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`) # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`) diff --git a/requirements.txt b/requirements.txt index 78e8269..72a541c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,6 @@ pluggy==1.5.0 prompt-toolkit==3.0.43 pytest==8.2.1 python-dateutil==2.9.0.post0 -python-dotenv==1.1.1 python-engineio==4.9.1 python-socketio==5.11.2 requests==2.32.2 From 868cb2d8dbc624cb1fe0392f706ceb28c47eaa19 Mon Sep 17 00:00:00 2001 From: bhavya-tech <64970880+bhavya-tech@users.noreply.github.com> Date: Tue, 9 Sep 2025 19:45:17 +0530 Subject: [PATCH 7/8] fixed missing load-env build requirement --- .github/workflows/python-publish.yml | 2 +- Dockerfile | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index f362b8d..7142a77 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -32,7 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install build python-dotenv==1.1.1 + pip install build - name: Build package run: python -m build - name: Publish package diff --git a/Dockerfile b/Dockerfile index a997ed6..7de7388 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,7 @@ COPY . . # Install Python dependencies RUN pip install --no-cache-dir -r requirements.txt -RUN pip install --no-cache-dir redis python-dotenv==1.1.1 +RUN pip install --no-cache-dir redis # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`) # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`) diff --git a/pyproject.toml b/pyproject.toml index 7e4a2b8..3ae8231 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ [build-system] -requires = ["setuptools", "setuptools-scm"] +requires = ["setuptools", "setuptools-scm", "python-dotenv"] build-backend = "setuptools.build_meta" From c1d3807a277903e0780a90f593a7c8a6dfc00574 Mon Sep 17 00:00:00 2001 From: bhavya-tech <64970880+bhavya-tech@users.noreply.github.com> Date: Sun, 12 Oct 2025 18:08:10 +0530 Subject: [PATCH 8/8] fixed test cases --- tests/test_eventlistener.py | 2 +- tests/test_integration.py | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_eventlistener.py b/tests/test_eventlistener.py index 6c49fa8..38f3ddd 100644 --- a/tests/test_eventlistener.py +++ b/tests/test_eventlistener.py @@ -25,7 +25,7 @@ def run(self): class TestEventListener(unittest.TestCase): def setUp(self) -> None: - self.app = Celery('example_app') + self.app = Celery('example_app', broker='redis://127.0.0.1:6379/0') self.event_handler_process = EventHandlerProcess( self.app, AsyncMock(), daemon=True) return super().setUp() diff --git a/tests/test_integration.py b/tests/test_integration.py index d06a8b2..21e5774 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,6 +1,7 @@ import multiprocessing import threading import time +from types import SimpleNamespace import unittest from unittest.mock import Mock from celery import Celery @@ -25,21 +26,27 @@ def run(self): self.client.wait() -# This test is not working. It is not able to connect to the server. I am not sure why. -# Keeping it disconnected until its resolved, maybe someone can lok into this. -class TestIntegration(): +class MockCtx: """ - This test the backend end to end. It starts the server and client and checks if the client receives the event. + Custom mock context class as unittest.mock.Mock does not pickle. + """ + + def __init__(self, app): + self.obj = SimpleNamespace() + self.obj.app = app + +class TestIntegration(unittest.TestCase): + """ + This test the backend end to end. It starts the server and client and checks if the client receives the event. """ payload = {"utcoffset": -6, "uuid": "4897c640-a023-4cb8-ae8e-1df4641a3ba1", "name": "basic_task.add", "args": "()", "kwargs": "{}", "root_id": "4897c640-a023-4cb8-ae8e-1df4641a3ba1", "parent_id": None, "retries": 0, "eta": None, "expires": None, "timestamp": 1685367125.9317474, "type": "task-received"} def setUp(self) -> None: - self.app = Celery('example_app') - self.mock_ctx = Mock() - self.mock_ctx.obj.app = self.app + self.app = Celery('example_app', broker='redis://127.0.0.1:6379/0') + self.mock_ctx = MockCtx(self.app) self.on_event = Mock() self.server_process = multiprocessing.Process(