From edc362154790b60fdec9fbb814dd9d7f20a0dfee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 6 Dec 2024 16:34:12 +0300 Subject: [PATCH] [DOP-22054] Move migrations to separated script --- Makefile | 7 ++- docker-compose.test.yml | 29 +++++++++- docker-compose.yml | 22 ++++++++ docker/Dockerfile.backend | 2 +- docker/Dockerfile.scheduler | 5 +- docker/Dockerfile.worker | 2 +- docker/entrypoint_backend.sh | 2 - docs/changelog/next_release/163.breaking.rst | 1 + docs/scheduler/start_scheduler.rst | 6 +-- docs/worker/start_worker.rst | 12 ++--- poetry.lock | 54 ++----------------- pyproject.toml | 3 -- .../backend/providers/auth/dummy_provider.py | 2 +- 13 files changed, 72 insertions(+), 75 deletions(-) create mode 100644 docs/changelog/next_release/163.breaking.rst diff --git a/Makefile b/Makefile index 27ed459c..924021fe 100644 --- a/Makefile +++ b/Makefile @@ -130,10 +130,13 @@ dev-worker: db-start broker-start ##@Application Run development broker (without prod-build-server: ##@Application Build docker image for server - docker build --progress=plain -t mtsrus/syncmaster-backend:develop -f ./docker/Dockerfile.backend $(ARGS) . + docker build --progress=plain -t mtsrus/syncmaster-backend:develop -f ./docker/Dockerfile.backend --target=prod $(ARGS) . + +prod-build-scheduler: ##@Application Build docker image for scheduler + docker build --progress=plain -t mtsrus/syncmaster-scheduler:develop -f ./docker/Dockerfile.scheduler --target=prod $(ARGS) . prod-build-worker: ##@Application Build docker image for worker - docker build --progress=plain -t mtsrus/syncmaster-worker:develop -f ./docker/Dockerfile.worker $(ARGS) . + docker build --progress=plain -t mtsrus/syncmaster-worker:develop -f ./docker/Dockerfile.worker --target=prod $(ARGS) . prod-build: prod-build-server prod-build-worker ##@Application Build docker images diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 86cfe463..873e22c4 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -28,6 +28,22 @@ services: timeout: 5s retries: 3 + migrations: + image: mtsrus/syncmaster-worker:${WORKER_IMAGE_TAG:-test} + restart: none + build: + dockerfile: docker/Dockerfile.worker + context: . + target: test + volumes: + - ./syncmaster:/app/syncmaster + - ./tests:/app/tests + entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head] + env_file: .env.docker + depends_on: + db: + condition: service_healthy + backend: image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test} restart: unless-stopped @@ -41,15 +57,22 @@ services: volumes: - ./syncmaster:/app/syncmaster - ./docs/_static:/app/docs/_static - - ./cached_jars:/root/.ivy2 - ./reports:/app/reports - ./tests:/app/tests - ./pyproject.toml:/app/pyproject.toml depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy + healthcheck: + test: [CMD-SHELL, curl -f http://localhost:8000/monitoring/ping] + interval: 30s + timeout: 5s + retries: 3 + start_period: 5s profiles: [backend, all] scheduler: @@ -68,6 +91,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy profiles: [scheduler, all] @@ -94,6 +119,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, all] diff --git a/docker-compose.yml b/docker-compose.yml index 2cf254b3..7dc1e0e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,12 +28,26 @@ services: timeout: 5s retries: 3 + migrations: + image: mtsrus/syncmaster-backend:${TAG:-develop} + restart: none + build: + dockerfile: docker/Dockerfile.backend + context: . + target: prod + entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head] + env_file: .env.docker + depends_on: + db: + condition: service_healthy + backend: image: mtsrus/syncmaster-backend:${TAG:-develop} restart: unless-stopped build: dockerfile: docker/Dockerfile.backend context: . + target: prod ports: - 8000:8000 environment: @@ -49,6 +63,8 @@ services: depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy healthcheck: @@ -64,10 +80,13 @@ services: build: dockerfile: docker/Dockerfile.worker context: . + target: prod env_file: .env.docker depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy @@ -77,10 +96,13 @@ services: build: dockerfile: docker/Dockerfile.scheduler context: . + target: prod env_file: .env.docker depends_on: db: condition: service_healthy + migrations: + condition: service_completed_successfully rabbitmq: condition: service_healthy diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index dec88922..86856f11 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -39,7 +39,7 @@ ENV SYNCMASTER__SERVER__OPENAPI__SWAGGER__JS_URL=/static/swagger/swagger-ui-bund SYNCMASTER__SERVER__STATIC_FILES__DIRECTORY=/app/syncmaster/backend/static -FROM base as test +FROM base AS test RUN poetry install --no-root --extras "backend" --with test --without docs,dev RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh diff --git a/docker/Dockerfile.scheduler b/docker/Dockerfile.scheduler index 445573c8..af91bfb7 100644 --- a/docker/Dockerfile.scheduler +++ b/docker/Dockerfile.scheduler @@ -22,7 +22,8 @@ FROM base AS prod COPY ./syncmaster/ /app/syncmaster/ -FROM base as test + +FROM base AS test RUN poetry install --no-root --all-extras --with test --without docs,dev -RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh \ No newline at end of file +RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 0c959ad6..5fb15649 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -42,7 +42,7 @@ FROM base as prod COPY ./syncmaster/ /app/syncmaster/ -FROM base as test +FROM base AS test ENV SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session diff --git a/docker/entrypoint_backend.sh b/docker/entrypoint_backend.sh index 181c41f8..00d75f3b 100755 --- a/docker/entrypoint_backend.sh +++ b/docker/entrypoint_backend.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -e -python -m syncmaster.db.migrations upgrade head - if [[ "x${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" != "x" ]]; then superusers=$(echo "${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" | tr "," " ") python -m syncmaster.backend.scripts.manage_superusers add ${superusers} diff --git a/docs/changelog/next_release/163.breaking.rst b/docs/changelog/next_release/163.breaking.rst new file mode 100644 index 00000000..73dc2500 --- /dev/null +++ b/docs/changelog/next_release/163.breaking.rst @@ -0,0 +1 @@ +Now migrations are executed in a dedicated one-off container, instead of being run as a part of ``backend`` container. diff --git a/docs/scheduler/start_scheduler.rst b/docs/scheduler/start_scheduler.rst index b0f82fe8..84e198a4 100644 --- a/docs/scheduler/start_scheduler.rst +++ b/docs/scheduler/start_scheduler.rst @@ -1,7 +1,6 @@ Starting the Scheduler ====================== - With docker ----------- @@ -14,12 +13,12 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../docker-compose.yml - :emphasize-lines: 68-83 + :emphasize-lines: 90-103 .. dropdown:: ``.env.docker`` .. literalinclude:: ../../.env.docker - :emphasize-lines: 11-25 + :emphasize-lines: 11-25 To start the worker container you need to run the command: @@ -28,7 +27,6 @@ To start the worker container you need to run the command: docker compose up scheduler -d --wait --wait-timeout 200 - Without docker -------------- diff --git a/docs/worker/start_worker.rst b/docs/worker/start_worker.rst index 6acb6167..a210333c 100644 --- a/docs/worker/start_worker.rst +++ b/docs/worker/start_worker.rst @@ -6,25 +6,24 @@ Starting the Celery Worker Before starting the worker you need to create a queue. The queue is created by sending a post request to ``/queues`` endpoint (See Swagger doc for details). - With docker ----------- Installation process ~~~~~~~~~~~~~~~~~~~~ -Docker will download worker image of syncmaster worker & broker, and run them. +Docker will download worker image, and then start worker container with dependencies. Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` .. dropdown:: ``docker-compose.yml`` .. literalinclude:: ../../docker-compose.yml - :emphasize-lines: 55-66 + :emphasize-lines: 75-88 .. dropdown:: ``.env.docker`` .. literalinclude:: ../../.env.docker - :emphasize-lines: 11-22 + :emphasize-lines: 11-22 To start the worker container you need to run the command: @@ -33,7 +32,6 @@ To start the worker container you need to run the command: docker compose up worker -d --wait --wait-timeout 200 - Without docker -------------- @@ -41,13 +39,13 @@ To start the worker you need to run the command .. code-block:: bash - python -m celery -A syncmaster.worker.celery worker + python -m celery -A syncmaster.worker.celery worker --max-tasks-per-child=1 You can specify options like concurrency and queues by adding additional flags: .. code-block:: bash - celery -A -A syncmaster.worker.celery worker --concurrency=4 --max-tasks-per-child=1 --loglevel=info + python -m celery -A syncmaster.worker.celery worker --concurrency=4 --max-tasks-per-child=1 --loglevel=info Refer to the `Celery `_ documentation for more advanced start options. diff --git a/poetry.lock b/poetry.lock index bc21c00f..7d021dd2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -177,24 +177,6 @@ starlette = ">=0.18" [package.extras] celery = ["celery"] -[[package]] -name = "asttokens" -version = "2.4.1" -description = "Annotate AST trees with source code positions" -optional = true -python-versions = "*" -files = [ - {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, - {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, -] - -[package.dependencies] -six = ">=1.12.0" - -[package.extras] -astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] -test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] - [[package]] name = "async-property" version = "0.2.2" @@ -894,22 +876,6 @@ files = [ [package.dependencies] packaging = "*" -[[package]] -name = "devtools" -version = "0.12.2" -description = "Python's missing debug print command, and more." -optional = true -python-versions = ">=3.7" -files = [ - {file = "devtools-0.12.2-py3-none-any.whl", hash = "sha256:c366e3de1df4cdd635f1ad8cbcd3af01a384d7abda71900e68d43b04eb6aaca7"}, - {file = "devtools-0.12.2.tar.gz", hash = "sha256:efceab184cb35e3a11fa8e602cc4fadacaa2e859e920fc6f87bf130b69885507"}, -] - -[package.dependencies] -asttokens = ">=2.0.0,<3.0.0" -executing = ">=1.1.1" -pygments = ">=2.15.0" - [[package]] name = "distlib" version = "0.3.9" @@ -990,20 +956,6 @@ files = [ {file = "evacuator-1.0.4.tar.gz", hash = "sha256:4fac38ee4241e826fced8115ab7cdc8ca2fd18e2e200083405c4a802e836e926"}, ] -[[package]] -name = "executing" -version = "2.1.0" -description = "Get the currently executing AST node of a frame, and other information" -optional = true -python-versions = ">=3.8" -files = [ - {file = "executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf"}, - {file = "executing-2.1.0.tar.gz", hash = "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab"}, -] - -[package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] - [[package]] name = "faker" version = "33.1.0" @@ -3522,11 +3474,11 @@ test = ["coverage[toml]", "zope.event", "zope.testing"] testing = ["coverage[toml]", "zope.event", "zope.testing"] [extras] -backend = ["alembic", "asgi-correlation-id", "asyncpg", "celery", "coloredlogs", "devtools", "fastapi", "itsdangerous", "jinja2", "psycopg", "pydantic-settings", "python-jose", "python-json-logger", "python-keycloak", "python-multipart", "pyyaml", "sqlalchemy", "sqlalchemy-utils", "starlette-exporter", "uuid6", "uvicorn"] +backend = ["alembic", "asgi-correlation-id", "asyncpg", "celery", "coloredlogs", "fastapi", "itsdangerous", "jinja2", "psycopg", "pydantic-settings", "python-jose", "python-json-logger", "python-keycloak", "python-multipart", "pyyaml", "sqlalchemy", "sqlalchemy-utils", "starlette-exporter", "uuid6", "uvicorn"] scheduler = ["apscheduler", "asyncpg", "celery", "coloredlogs", "pydantic-settings", "python-jose", "python-json-logger", "python-multipart", "pyyaml", "sqlalchemy", "sqlalchemy-utils"] -worker = ["asgi-correlation-id", "celery", "coloredlogs", "jinja2", "onetl", "psycopg", "pydantic-settings", "python-json-logger", "python-keycloak", "sqlalchemy", "sqlalchemy-utils", "uuid6"] +worker = ["asgi-correlation-id", "celery", "coloredlogs", "jinja2", "onetl", "psycopg", "pydantic-settings", "python-json-logger", "sqlalchemy", "sqlalchemy-utils", "uuid6"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "8d0b3429bc8759a1e54ab4300022fd083e12f19d8f5cae9a4a8bd8b9c3981349" +content-hash = "22c7277db014088ba6dd9c5ba719ef065b58360c213d437122fb9becc9c7d4cf" diff --git a/pyproject.toml b/pyproject.toml index 7905c32c..611c63f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,6 @@ apscheduler = { version = "^3.10.4", optional = true } starlette-exporter = {version = "^0.23.0", optional = true} itsdangerous = {version = "*", optional = true} python-keycloak = {version = "^4.7.0", optional = true} -devtools = {version = "*", optional = true} [tool.poetry.extras] backend = [ @@ -91,7 +90,6 @@ backend = [ "coloredlogs", "python-json-logger", "asyncpg", - "devtools", "itsdangerous", "python-keycloak", "pyyaml", @@ -110,7 +108,6 @@ worker = [ "uuid6", "coloredlogs", "python-json-logger", - "python-keycloak", ] scheduler = [ diff --git a/syncmaster/backend/providers/auth/dummy_provider.py b/syncmaster/backend/providers/auth/dummy_provider.py index bb5d77b4..972e6a57 100644 --- a/syncmaster/backend/providers/auth/dummy_provider.py +++ b/syncmaster/backend/providers/auth/dummy_provider.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 import logging +from pprint import pformat from time import time from typing import Annotated, Any -from devtools import pformat from fastapi import Depends, FastAPI from syncmaster.backend.dependencies import Stub