diff --git a/.github/workflows/dockerhub-publish.yml b/.github/workflows/dockerhub-publish.yml index 4b28e18..510342f 100644 --- a/.github/workflows/dockerhub-publish.yml +++ b/.github/workflows/dockerhub-publish.yml @@ -5,7 +5,11 @@ on: release: types: [published] - workflow_dispatch: {} + workflow_dispatch: + inputs: + tag: + description: 'Docker image tag' + required: true jobs: publish_on_dockerhub: @@ -21,9 +25,22 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load .env file + uses: aarcangeli/load-dotenv@v1 + with: + filenames: | + config.env + - name: Build and push uses: docker/build-push-action@v5 with: push: true - tags: bhavyatech/celeryviz:${{ github.ref_name }} + tags: bhavyatech/celeryviz:${{ github.event.inputs.tag || github.ref_name }} platforms: linux/amd64,linux/arm64 + build-args: FRONTEND_VERSION=${{ env.CELERYVIZ_FRONTEND_BUNDLED_VERSION }} diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml deleted file mode 100644 index c776f54..0000000 --- a/.github/workflows/greetings.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Greetings - -on: [pull_request_target, issues] - -jobs: - greeting: - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - steps: - - uses: actions/first-interaction@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: "๐Ÿš€ **Welcome, First-Time Contributor!** ๐Ÿš€ \nThanks for jumping in! Whether itโ€™s an issue or a PR, weโ€™re excited to have you on board. Your contribution is on its way to being reviewed. Feel free to explore or reach out with any questions. \ Letโ€™s make something awesome together! ๐ŸŒŸ" - pr-message: "๐Ÿš€ **Welcome, First-Time Contributor!** ๐Ÿš€ \nThanks for jumping in! Whether itโ€™s an issue or a PR, weโ€™re excited to have you on board. Your contribution is on its way to being reviewed. Feel free to explore or reach out with any questions. \ Letโ€™s make something awesome together! ๐ŸŒŸ" diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 9cc3f70..7142a77 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,26 +11,20 @@ name: Upload Python Package on: release: types: [published] - - workflow_dispatch: - inputs: - environment: - type: environment - description: 'Manually trigger a deployment to the test PyPI environment' - + workflow_dispatch: {} permissions: contents: read jobs: - publish: - runs-on: ubuntu-latest - + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build UI - run: docker build --target export --output ./celeryviz/static ./ + run: | + source config.env + docker build --target webapp-prebuilt --output . . --build-arg FRONTEND_VERSION=$CELERYVIZ_FRONTEND_BUNDLED_VERSION - name: Set up Python uses: actions/setup-python@v3 with: diff --git a/.gitignore b/.gitignore index 8d6ae47..0752264 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ */__pycache__/* celeryviz.egg-info/ celeryviz/static/ -dist/* \ No newline at end of file +dist/* +*/.DS_Store diff --git a/Dockerfile b/Dockerfile index c76298e..7de7388 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,25 @@ +######################################### +# Docs # +######################################### +# There are two ways to build the UI: +# 1. Download the prebuilt UI +# a. If you want to build complete image with prebuilt UI, use the `celeryviz` target. +# Run: `docker build --target celeryviz` +# b. If you want to download only the prebuilt UI, use the `prebuilt` target. +# Run: `docker build --target webapp-prebuilt --output . .` +# 2. Build from source +# a. If you want to build complete image from source, use the `celeryviz-with-frontend-build` target. +# Run: `docker build --target celeryviz-with-frontend-build` +# b. If you want to build only the UI, use the `webapp-build` target. +# Run: `docker build --target webapp-build --output . .` +# - The `GIT_REPO` and `SOURCE` build args can be passed for specific builds. +######################################### + + ##################################### # Docker stages for building the UI # ##################################### - -# Run `docker build --target export --output ./celeryviz/static ./` to build the UI locally. - -FROM instrumentisto/flutter:3 AS base +FROM instrumentisto/flutter:3 AS flutter_build_base # Set the working directory inside the container WORKDIR /app @@ -13,7 +28,7 @@ WORKDIR /app RUN flutter precache # Build stage -FROM base AS build +FROM flutter_build_base AS webapp-compile ARG SOURCE="main" ARG GIT_REPO="https://github.com/bhavya-tech/celeryviz_with_lib.git" @@ -24,45 +39,71 @@ RUN git checkout $SOURCE # Enable web support for Flutter RUN flutter config --enable-web - -# Now that the repo is cloned, we can run 'flutter pub get' RUN flutter pub get - -# Build the Flutter web app with CanvasKit renderer RUN flutter build web --release -# Final stage: export the build files -# This is needed other wise all the linux files will be copied to the final image. -FROM scratch AS export -COPY --from=build /app/celeryviz_with_lib/build/web / +# Use this stage if static files are needed in the actual folder structure. +FROM scratch AS webapp-build +COPY --from=webapp-compile /app/celeryviz_with_lib/build/web /celeryviz/static/ ########################################### ########################################### +################################################ +# Docker stage to download the prebuilt webapp # +################################################ + +# Download and extract the prebuilt webapp +# This is needed for users who do not want to build the webapp from source. +FROM alpine:3.14 AS download-and-extract-prebuilt +ARG FRONTEND_VERSION="main" +ADD https://github.com/bhavya-tech/celeryviz_with_lib/releases/download/$FRONTEND_VERSION/webapp-build.zip /app/webapp-build.zip +RUN apk add --no-cache unzip curl && \ + unzip /app/webapp-build.zip -d /app/static && \ + rm /app/webapp-build.zip + +# Use this stage if static files are needed in the actual folder structure. +FROM scratch AS webapp-prebuilt +COPY --from=download-and-extract-prebuilt /app/static/ /celeryviz/static/ +########################################### +########################################### + ########################################### # Docker stage for the Python application # ########################################### -FROM python:3.9 +FROM python:3.9-alpine AS setup-celeryviz-dependency WORKDIR /app - COPY . . -# Build UI -COPY --from=export / /app/celeryviz/static/ - # Install Python dependencies RUN pip install --no-cache-dir -r requirements.txt -RUN pip install --no-cache-dir redis -RUN pip install . +RUN pip install --no-cache-dir redis # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`) # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`) # PYTHONDONTWRITEBYTECODE: Do not write byte files to disk, since we maintain it as readonly. (equivalent to `python -B`) ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1 - EXPOSE 9095 +########################################### +########################################### + + +####################### +# Final docker stages # +####################### + +# This stage builds celeryviz with the frontend built from the source. +FROM setup-celeryviz-dependency AS celeryviz-with-frontend-build +COPY --from=webapp-compile /app/celeryviz_with_lib/build/web /app/celeryviz/static/ +RUN pip install . + +# This stage builds celeryviz with the prebuilt frontend. +FROM setup-celeryviz-dependency AS celeryviz +WORKDIR /app +COPY --from=download-and-extract-prebuilt /app/static/ /app/celeryviz/static/ +RUN pip install . +RUN cd .. && rm -rf /app/ -CMD ["celery", "celeryviz"] diff --git a/config.env b/config.env new file mode 100644 index 0000000..c44a517 --- /dev/null +++ b/config.env @@ -0,0 +1,5 @@ +# Version name for the CeleryViz Python package +CELERYVIZ_PYTHON_VERSION=0.0.5 + +# Version name for the CeleryViz bundled frontend package. +CELERYVIZ_FRONTEND_BUNDLED_VERSION=0.0.3 diff --git a/example/Dockerfile b/example/Dockerfile index 5f281aa..bd24053 100644 --- a/example/Dockerfile +++ b/example/Dockerfile @@ -5,5 +5,4 @@ WORKDIR /app RUN pip install redis celery celeryviz -COPY . . -CMD ["celery", "-A", "example_app", "worker", "--loglevel=info", "-E"] \ No newline at end of file +COPY . . \ No newline at end of file diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 957bfb5..a86fa42 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: redis: image: redis:alpine @@ -29,6 +27,8 @@ services: build: context: ../. dockerfile: Dockerfile + target: celeryviz + command: celery --broker='redis://redis:6379/0' celeryviz depends_on: - redis diff --git a/pyproject.toml b/pyproject.toml index 7e4a2b8..3ae8231 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ [build-system] -requires = ["setuptools", "setuptools-scm"] +requires = ["setuptools", "setuptools-scm", "python-dotenv"] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 4ff2a6e..7d448c1 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,12 @@ +from dotenv import load_dotenv +import os from setuptools import setup, find_packages def get_requirements(): return open('./requirements.txt').read().splitlines() +load_dotenv("config.env") + classes = """ Development Status :: 1 - Planning Intended Audience :: Developers @@ -25,7 +29,7 @@ def get_requirements(): description= "A UI centric tool for visualising Celery task execution.", long_description=open('README.md').read(), long_description_content_type='text/markdown', - version='0.0.3', + version=os.getenv("CELERYVIZ_PYTHON_VERSION", "latest"), classifiers=classifiers, author = "Bhavya Peshavaria", packages=find_packages(), diff --git a/tests/test_eventlistener.py b/tests/test_eventlistener.py index 6c49fa8..38f3ddd 100644 --- a/tests/test_eventlistener.py +++ b/tests/test_eventlistener.py @@ -25,7 +25,7 @@ def run(self): class TestEventListener(unittest.TestCase): def setUp(self) -> None: - self.app = Celery('example_app') + self.app = Celery('example_app', broker='redis://127.0.0.1:6379/0') self.event_handler_process = EventHandlerProcess( self.app, AsyncMock(), daemon=True) return super().setUp() diff --git a/tests/test_integration.py b/tests/test_integration.py index d06a8b2..21e5774 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,6 +1,7 @@ import multiprocessing import threading import time +from types import SimpleNamespace import unittest from unittest.mock import Mock from celery import Celery @@ -25,21 +26,27 @@ def run(self): self.client.wait() -# This test is not working. It is not able to connect to the server. I am not sure why. -# Keeping it disconnected until its resolved, maybe someone can lok into this. -class TestIntegration(): +class MockCtx: """ - This test the backend end to end. It starts the server and client and checks if the client receives the event. + Custom mock context class as unittest.mock.Mock does not pickle. + """ + + def __init__(self, app): + self.obj = SimpleNamespace() + self.obj.app = app + +class TestIntegration(unittest.TestCase): + """ + This test the backend end to end. It starts the server and client and checks if the client receives the event. """ payload = {"utcoffset": -6, "uuid": "4897c640-a023-4cb8-ae8e-1df4641a3ba1", "name": "basic_task.add", "args": "()", "kwargs": "{}", "root_id": "4897c640-a023-4cb8-ae8e-1df4641a3ba1", "parent_id": None, "retries": 0, "eta": None, "expires": None, "timestamp": 1685367125.9317474, "type": "task-received"} def setUp(self) -> None: - self.app = Celery('example_app') - self.mock_ctx = Mock() - self.mock_ctx.obj.app = self.app + self.app = Celery('example_app', broker='redis://127.0.0.1:6379/0') + self.mock_ctx = MockCtx(self.app) self.on_event = Mock() self.server_process = multiprocessing.Process(