Create a local development stack

Author: benoit74

dev/README.md

@@ -0,0 +1,66 @@
+This is a docker-compose configuration to be used **only** for development purpose. There is
+almost zero security in the stack configuration.
+
+It is composed of the Zimit frontend and API (of course), but also a local Zimfarm DB,
+API and UI, so that you can test the whole integration locally.
+
+Zimit UI and API are not deployed as they would be in production to allow hot reload of
+most modifications done to the source code.
+
+Zimfarm UI, API and DB are deployed with official production Docker images.
+
+## List of containers
+
+### zimit_ui
+
+This container is Zimit frontend web server (UI only)
+
+### zimit_api
+
+This container is Zimit API server (API only)
+
+## zimfarm_db
+
+This container is a local Zimfarm database
+
+## zimfarm_api
+
+This container is a local Zimfarm API
+
+## zimfarm_ui
+
+This container is a local Zimfarm UI
+
+## Instructions
+
+First start the Docker-Compose stack:
+
+```sh
+cd dev
+docker compose -p zimit up -d
+```
+
+If it is your first execution of the dev stack, you need to create a "virtual" worker in Zimfarm DB:
+
+```sh
+dev/create_worker.sh
+```
+
+If you have requested a task via Zimit UI and want to simulate a worker starting this task to observe the consequence in Zimit UI, you might use the `dev/start_first_req_task.sh`.
+
+## Restart the backend
+
+Should the API process fail, you might restart it with:
+```sh
+docker restart zimit-zimit_ui-1
+```
+
+## Browse the web UIs
+
+You might open following URLs in your favorite browser:
+
+- [Zimit UI](http://localhost:8001)
+- [Zimfarm API](http://localhost:8002)
+- [Zimfarm UI](http://localhost:8003)
+
+You can login into Zimfarm UI with username `admin` and password `admin`.

dev/create_worker.sh

@@ -0,0 +1,27 @@
+echo "Retrieving access token"
+
+ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
+    'http://localhost:8002/v1/auth/authorize' \
+    -H 'accept: application/json' \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d 'username=admin&password=admin' \
+    | jq -r '.access_token')"
+
+echo "Worker check-in (will create if missing)"
+
+curl -s -X 'PUT' \
+  'http://localhost:8002/v1/workers/worker/check-in' \
+  -H 'accept: */*' \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+  -d '{
+  "username": "admin",
+  "cpu": 3,
+  "memory": 1024,
+  "disk": 0,
+  "offliners": [
+    "zimit"
+  ]
+}'
+
+echo "DONE"

dev/docker-compose.yml

@@ -0,0 +1,63 @@
+services:
+  zimfarm_db:
+    image: postgres:15.2-bullseye
+    ports:
+      - 127.0.0.1:5432:5432
+    volumes:
+      - zimfarm_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_DB=zimfarm
+      - POSTGRES_USER=zimfarm
+      - POSTGRES_PASSWORD=zimpass
+  zimfarm_api:
+    image: ghcr.io/openzim/zimfarm-dispatcher:latest
+    ports:
+      - 127.0.0.1:8004:80
+    environment:
+      BINDING_HOST: 0.0.0.0
+      JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw
+      POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@zimfarm_db:5432/zimfarm
+      ALEMBIC_UPGRADE_HEAD_ON_START: "1"
+      ZIMIT_USE_RELAXED_SCHEMA: "y"
+    depends_on:
+      - zimfarm_db
+  zimfarm-ui:
+    image: ghcr.io/openzim/zimfarm-ui:latest
+    ports:
+      - 127.0.0.1:8003:80
+    environment:
+      ZIMFARM_WEBAPI: http://localhost:8002/v1
+    depends_on:
+      - zimfarm_api
+  zimit_api:
+    build: ..
+    volumes:
+      - ../api/src:/app
+    command: python main.py
+    ports:
+      - 127.0.0.1:8002:8000
+    environment:
+      BINDING_HOST: 0.0.0.0
+      INTERNAL_ZIMFARM_WEBAPI: http://zimfarm_api:80/v1
+      _ZIMFARM_USERNAME: admin
+      _ZIMFARM_PASSWORD: admin
+      TASK_WORKER: worker
+    depends_on:
+      - zimfarm_api
+  zimit_ui:
+    build:
+      dockerfile: ../dev/zimit_ui_dev/Dockerfile
+      context: ../ui
+    volumes:
+      - ../ui/src:/app/src
+      - ../ui/public:/app/public
+      - ../dev/zimit_ui_dev/environ.json:/app/public/environ.json
+    ports:
+      - 127.0.0.1:8001:80
+    environment:
+      ZIMIT_API_URL: http://localhost:8002
+    depends_on:
+      - zimit_api
+
+volumes:
+  zimfarm_data:

dev/start_first_req_task.sh

@@ -0,0 +1,31 @@
+echo "Retrieving access token"
+
+ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
+    'http://localhost:8002/v1/auth/authorize' \
+    -H 'accept: application/json' \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d 'username=admin&password=admin' \
+    | jq -r '.access_token')"
+
+echo "Get last requested task"
+
+LAST_TASK_ID="$(curl -s -X 'GET' \
+  'http://localhost:8002/v1/requested-tasks/' \
+  -H 'accept: application/json' \
+  -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+  | jq -r '.items[0]._id')"
+
+if [ "$LAST_TASK_ID" = "null" ]; then
+    echo "No pending requested task. Exiting script."
+    exit 1
+fi
+
+echo "Start task"
+
+curl -s -X 'POST' \
+  "http://localhost:8002/v1/tasks/$LAST_TASK_ID?worker_name=worker" \
+  -H 'accept: application/json' \
+  -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+  -d ''
+
+echo "DONE"

dev/zimit_ui_dev/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:14-alpine
+
+RUN apk --no-cache add yarn
+WORKDIR /app
+COPY package.json yarn.lock /app/
+RUN yarn install && yarn cache clean
+COPY *.js /app/
+COPY public /app/public
+COPY src /app/src
+CMD ["yarn", "serve", "--host", "0.0.0.0", "--port", "80"]

dev/zimit_ui_dev/environ.json

@@ -0,0 +1,4 @@
+{
+  "ZIMFARM_WEBAPI": "http://localhost:8004/v1",
+  "ZIMIT_API_URL": "http://localhost:8002/api/v1"
+}