From 20e5926f185fe957d355e5ba820b343f2b13766a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 6 Dec 2024 16:34:12 +0300 Subject: [PATCH] [DOP-22054] Move migrations to separated script --- docker-compose.test.yml | 29 ++++++++++++++++++++++++++++- docker-compose.yml | 18 ++++++++++++++++++ docker/entrypoint_backend.sh | 2 -- docs/scheduler/start_scheduler.rst | 6 ++---- docs/worker/start_worker.rst | 12 +++++------- 5 files changed, 53 insertions(+), 14 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 86cfe463..91a5e219 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -28,6 +28,22 @@ services: timeout: 5s retries: 3 + migrations: + image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test} + restart: unless-stopped + build: + dockerfile: docker/Dockerfile.backend + context: . + target: test + volumes: + - ./syncmaster:/app/syncmaster + entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head] + env_file: .env.docker + depends_on: + db: + condition: service_healthy + profiles: [backend, worker, scheduler, all] + backend: image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test} restart: unless-stopped @@ -41,15 +57,22 @@ services: volumes: - ./syncmaster:/app/syncmaster - ./docs/_static:/app/docs/_static - - ./cached_jars:/root/.ivy2 - ./reports:/app/reports - ./tests:/app/tests - ./pyproject.toml:/app/pyproject.toml depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy + healthcheck: + test: [CMD-SHELL, curl -f http://localhost:8000/monitoring/ping] + interval: 30s + timeout: 5s + retries: 3 + start_period: 5s profiles: [backend, all] scheduler: @@ -68,6 +91,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy profiles: [scheduler, all] @@ -94,6 +119,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, all] diff --git a/docker-compose.yml b/docker-compose.yml index 2cf254b3..bcef34ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,18 @@ services: timeout: 5s retries: 3 + migrations: + image: mtsrus/syncmaster-backend:${TAG:-develop} + restart: unless-stopped + build: + dockerfile: docker/Dockerfile.backend + context: . + entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head] + env_file: .env.docker + depends_on: + db: + condition: service_healthy + backend: image: mtsrus/syncmaster-backend:${TAG:-develop} restart: unless-stopped @@ -49,6 +61,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy healthcheck: @@ -68,6 +82,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy @@ -81,6 +97,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy diff --git a/docker/entrypoint_backend.sh b/docker/entrypoint_backend.sh index 181c41f8..00d75f3b 100755 --- a/docker/entrypoint_backend.sh +++ b/docker/entrypoint_backend.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -e -python -m syncmaster.db.migrations upgrade head - if [[ "x${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" != "x" ]]; then superusers=$(echo "${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" | tr "," " ") python -m syncmaster.backend.scripts.manage_superusers add ${superusers} diff --git a/docs/scheduler/start_scheduler.rst b/docs/scheduler/start_scheduler.rst index b0f82fe8..84e198a4 100644 --- a/docs/scheduler/start_scheduler.rst +++ b/docs/scheduler/start_scheduler.rst @@ -1,7 +1,6 @@ Starting the Scheduler ====================== - With docker ----------- @@ -14,12 +13,12 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../docker-compose.yml - :emphasize-lines: 68-83 + :emphasize-lines: 90-103 .. dropdown:: ``.env.docker`` .. literalinclude:: ../../.env.docker - :emphasize-lines: 11-25 + :emphasize-lines: 11-25 To start the worker container you need to run the command: @@ -28,7 +27,6 @@ To start the worker container you need to run the command: docker compose up scheduler -d --wait --wait-timeout 200 - Without docker -------------- diff --git a/docs/worker/start_worker.rst b/docs/worker/start_worker.rst index 6acb6167..a210333c 100644 --- a/docs/worker/start_worker.rst +++ b/docs/worker/start_worker.rst @@ -6,25 +6,24 @@ Starting the Celery Worker Before starting the worker you need to create a queue. The queue is created by sending a post request to ``/queues`` endpoint (See Swagger doc for details). - With docker ----------- Installation process ~~~~~~~~~~~~~~~~~~~~ -Docker will download worker image of syncmaster worker & broker, and run them. +Docker will download worker image, and then start worker container with dependencies. Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../docker-compose.yml - :emphasize-lines: 55-66 + :emphasize-lines: 75-88 .. dropdown:: ``.env.docker`` .. literalinclude:: ../../.env.docker - :emphasize-lines: 11-22 + :emphasize-lines: 11-22 To start the worker container you need to run the command: @@ -33,7 +32,6 @@ To start the worker container you need to run the command: docker compose up worker -d --wait --wait-timeout 200 - Without docker -------------- @@ -41,13 +39,13 @@ To start the worker you need to run the command .. code-block:: bash - python -m celery -A syncmaster.worker.celery worker + python -m celery -A syncmaster.worker.celery worker --max-tasks-per-child=1 You can specify options like concurrency and queues by adding additional flags: .. code-block:: bash - celery -A -A syncmaster.worker.celery worker --concurrency=4 --max-tasks-per-child=1 --loglevel=info + python -m celery -A syncmaster.worker.celery worker --concurrency=4 --max-tasks-per-child=1 --loglevel=info Refer to the `Celery `_ documentation for more advanced start options.