diff --git a/.env.docker b/.env.docker index acbb5015..cbdb8764 100644 --- a/.env.docker +++ b/.env.docker @@ -5,6 +5,15 @@ ENV=LOCAL SYNCMASTER__SERVER__DEBUG=true SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} +# Session +SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key + +# Postgres +SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster + +# RabbitMQ +SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ + # Logging SYNCMASTER__LOGGING__SETUP=True SYNCMASTER__LOGGING__PRESET=colored @@ -15,13 +24,6 @@ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= # Scheduler settings SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 -# Session -SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key - -# Postgres -SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster - -# TODO: add to KeycloakAuthProvider documentation about creating new realms, add users, etc. # KEYCLOAK Auth SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created @@ -36,9 +38,6 @@ SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.K SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret -# RabbitMQ -SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ - # S3 MINIO_ACCESS_KEY=syncmaster MINIO_SECRET_KEY=changeme diff --git a/.env.local b/.env.local index b1d73462..7e8abf47 100644 --- a/.env.local +++ b/.env.local @@ -5,21 +5,24 @@ export ENV=LOCAL export SYNCMASTER__SERVER__DEBUG=true export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}" +# Session +export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key + +# Postgres +export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster + +# RabbitMQ +export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/ + # Logging export SYNCMASTER__LOGGING__SETUP=True export SYNCMASTER__LOGGING__PRESET=colored -# Scheduler settings -export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 - -# Session -export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key - # Encrypt / Decrypt credentials data export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= -# Postgres -export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster +# Scheduler settings +export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 # Keycloack Auth export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 @@ -35,9 +38,6 @@ export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_pro export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret -# RabbitMQ -export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/ - export TEST_S3_HOST_FOR_CONFTEST=localhost export TEST_S3_PORT_FOR_CONFTEST=9010 export TEST_S3_HOST_FOR_WORKER=test-s3 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7830619a..127b0ba2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -2,6 +2,8 @@ version: 2 build: os: ubuntu-22.04 + apt_packages: + - make tools: python: '3.12' jobs: @@ -17,7 +19,7 @@ build: - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v - python -m pip list -v - - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json + - make openapi sphinx: configuration: docs/conf.py diff --git a/Makefile b/Makefile index 29ef1288..56527e7c 100644 --- a/Makefile +++ b/Makefile @@ -155,3 +155,6 @@ docs-cleanup: ##@Docs Cleanup docs $(MAKE) -C docs clean docs-fresh: docs-cleanup docs-build ##@Docs Cleanup & build docs + +openapi: ##@Docs Generate OpenAPI schema + python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json \ No newline at end of file diff --git a/docker/Dockerfile.scheduler b/docker/Dockerfile.scheduler index 49c59b7b..445573c8 100644 --- a/docker/Dockerfile.scheduler +++ b/docker/Dockerfile.scheduler @@ -13,7 +13,7 @@ ENV PYTHONPATH=/app COPY ./pyproject.toml ./poetry.lock* /app/ RUN pip install --upgrade pip setuptools wheel packaging -RUN poetry install --no-root --extras "scheduler" --without test,docs,dev +RUN poetry install --no-root --extras "scheduler" --extras "backend" --without test,docs,dev COPY ./docker/entrypoint_scheduler.sh /app/entrypoint.sh ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/docs/_static/architecture.png b/docs/_static/architecture.png index cbbd95d8..4bea0bc5 100644 Binary files a/docs/_static/architecture.png and b/docs/_static/architecture.png differ diff --git a/docs/backend/auth/index.rst b/docs/backend/auth/index.rst index 728b193e..5ad6982f 100644 --- a/docs/backend/auth/index.rst +++ b/docs/backend/auth/index.rst @@ -12,7 +12,7 @@ Syncmaster supports different auth provider implementations. You can change impl :caption: Auth providers dummy - keycloak + keycloak/index .. toctree:: :maxdepth: 2 diff --git a/docs/backend/auth/keycloak/images/keycloak-client-authentication.png b/docs/backend/auth/keycloak/images/keycloak-client-authentication.png new file mode 100644 index 00000000..126067a3 Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-client-authentication.png differ diff --git a/docs/backend/auth/keycloak/images/keycloak-client-redirect_uri.png b/docs/backend/auth/keycloak/images/keycloak-client-redirect_uri.png new file mode 100644 index 00000000..a68b812c Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-client-redirect_uri.png differ diff --git a/docs/backend/auth/keycloak/images/keycloak-client-secret.png b/docs/backend/auth/keycloak/images/keycloak-client-secret.png new file mode 100644 index 00000000..2a5f7edf Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-client-secret.png differ diff --git a/docs/backend/auth/keycloak/images/keycloak-login.png b/docs/backend/auth/keycloak/images/keycloak-login.png new file mode 100644 index 00000000..f6c0b2b8 Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-login.png differ diff --git a/docs/backend/auth/keycloak/images/keycloak-new-client.png b/docs/backend/auth/keycloak/images/keycloak-new-client.png new file mode 100644 index 00000000..7eb31977 Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-new-client.png differ diff --git a/docs/backend/auth/keycloak/images/keycloak-new-client_name.png b/docs/backend/auth/keycloak/images/keycloak-new-client_name.png new file mode 100644 index 00000000..58cd16ce Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-new-client_name.png differ diff --git a/docs/backend/auth/keycloak/images/keycloak-new-realm.png b/docs/backend/auth/keycloak/images/keycloak-new-realm.png new file mode 100644 index 00000000..0819822b Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-new-realm.png differ diff --git a/docs/backend/auth/keycloak/images/keycloak-new-realm_name.png b/docs/backend/auth/keycloak/images/keycloak-new-realm_name.png new file mode 100644 index 00000000..5d152016 Binary files /dev/null and b/docs/backend/auth/keycloak/images/keycloak-new-realm_name.png differ diff --git a/docs/backend/auth/keycloak.rst b/docs/backend/auth/keycloak/index.rst similarity index 72% rename from docs/backend/auth/keycloak.rst rename to docs/backend/auth/keycloak/index.rst index 6fe7381b..436a6af3 100644 --- a/docs/backend/auth/keycloak.rst +++ b/docs/backend/auth/keycloak/index.rst @@ -1,17 +1,18 @@ -.. _backend-auth-ldap: +.. _keycloak-auth-provider: KeyCloak Auth provider -================== +====================== Description ----------- +Keycloak auth provider uses `python-keycloak `_ library to interact with Keycloak server. During the authentication process, +KeycloakAuthProvider redirects user to Keycloak authentication page. -TODO: +After successful authentication, Keycloak redirects user back to Syncmaster with authorization code. +Then KeycloakAuthProvider exchanges authorization code for an access token and uses it to get user information from Keycloak server. +If user is not found in Syncmaster database, KeycloakAuthProvider creates it. Finally, KeycloakAuthProvider returns user with access token. -Strategies ----------- - -TODO: +You can follow interaction schema below. Interaction schema ------------------ @@ -76,6 +77,13 @@ Interaction schema Basic configuration ------------------- -.. autopydantic_model:: syncmaster.settings.auth.keycloak.KeycloakProviderSettings -.. autopydantic_model:: syncmaster.settings.auth.jwt.JWTSettings +.. autopydantic_model:: syncmaster.backend.settings.auth.keycloak.KeycloakAuthProviderSettings +.. autopydantic_model:: syncmaster.backend.settings.auth.keycloak.KeycloakSettings +.. autopydantic_model:: syncmaster.backend.settings.auth.jwt.JWTSettings + +.. toctree:: + :maxdepth: 1 + :caption: Keycloak + :hidden: + local_installation diff --git a/docs/backend/auth/keycloak/local_installation.rst b/docs/backend/auth/keycloak/local_installation.rst new file mode 100644 index 00000000..75533f2c --- /dev/null +++ b/docs/backend/auth/keycloak/local_installation.rst @@ -0,0 +1,121 @@ +Local installation and testing +------------------------------ + +You can test Keycloak auth locally with docker compose: + + +.. code-block:: console + + $ docker compose -f docker-compose.test.yml up keycloak -d + + +Authorize in keycloak +~~~~~~~~~~~~~~~~~~~~~ + +At first, you have to go to `http://localhost:8080/admin `_ and login via login: `admin`, password: `admin` (by default) to create realms. + +.. image:: images/keycloak-login.png + :width: 400px + :align: center + + +Create new realm +~~~~~~~~~~~~~~~~ + +.. image:: images/keycloak-new-realm.png + :width: 400px + :align: center + + +Create new realm name +~~~~~~~~~~~~~~~~~~~~~ + +Pass realm name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME` environment variable: + +.. code-block:: console + + $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm # as on screen below + +.. image:: images/keycloak-new-realm_name.png + :width: 400px + :align: center + + +Create new client +~~~~~~~~~~~~~~~~~ + +.. image:: images/keycloak-new-client.png + :width: 400px + :align: center + + +Create new client name +~~~~~~~~~~~~~~~~~~~~~~ + +In created realm pass client name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID` environment variable: + +.. code-block:: console + + $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client # as on screen below + +.. image:: images/keycloak-new-client_name.png + :width: 400px + :align: center + + +Set ``client_authentication`` **ON** to receive client_secret +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. image:: images/keycloak-client-authentication.png + :width: 400px + :align: center + +Configure Redirect URI +~~~~~~~~~~~~~~~~~~~~~~ + +To configure the redirect URI where the browser will redirect to exchange the code provided from Keycloak for an access token, set the `SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI` environment variable. The default value for local development is `http://localhost:8000/auth/callback`. + +.. code-block:: console + + $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback + +Configure the client redirect URI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Ensure that this URI is also configured as a valid redirect URI in your Keycloak client settings. This allows the browser to redirect to your application after the user successfully authenticates with Keycloak. + +.. image:: images/keycloak-client-redirect_uri.png + :width: 400px + :align: center + +Configure the client secret +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now go to **Credentials** tab and add the client secret to the `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET` environment variable: + +.. code-block:: console + + $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez # as on screen below + +.. image:: images/keycloak-client-secret.png + :width: 400px + :align: center + +Now you can use create users in this realms, check `keycloak documentation `_ on how to manage users creation. + +ENVIRONMENT VARIABLES +~~~~~~~~~~~~~~~~~~~~~ + +After this you can user `KeycloakAuthProvider` in your application with provided environment variables: + + +.. code-block:: console + + $ export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080 + $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback + $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm + $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client + $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez + $ export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email + $ export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False + $ export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider diff --git a/docs/backend/install.rst b/docs/backend/install.rst index 4ba726f8..8d445768 100644 --- a/docs/backend/install.rst +++ b/docs/backend/install.rst @@ -79,15 +79,11 @@ Available *extras* are: Run database ~~~~~~~~~~~~ -Start Postgres instance somewhere, and set up environment variables: +Start Postgres instance somewhere, and set up environment variable: .. code-block:: bash - POSTGRES_HOST=localhost - POSTGRES_PORT=5432 - POSTGRES_DB=postgres - POSTGRES_USER=user - POSTGRES_PASSWORD=password + SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster You can use virtually any database supported by `SQLAlchemy `_, but the only one we really tested is Postgres. @@ -111,33 +107,11 @@ options and commands are just the same. Run RabbitMQ ~~~~~~~~~~~~ -Start RabbitMQ instance somewhere, and set up environment variables: +Start RabbitMQ instance somewhere, and set up environment variable: .. code-block:: bash - RABBITMQ_HOST=somehost - RABBITMQ_PORT=5672 - RABBITMQ_USER=user - RABBITMQ_PASSWORD=password - -Run worker -~~~~~~~~~~ - -.. note:: - - Before starting the worker you need to create a queue. - The queue is created by sending a post request to ``/queues`` endpoint (See Swagger doc for details). - -to start the worker you need to run the command - -.. code-block:: console - - $ celery -A syncmaster.worker.config.celery worker --loglevel=info --max-tasks-per-child=1 -Q queue_name - -.. note:: - - The specified celery options are given as an example, you can specify other options you need. - + SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/ Run backend ~~~~~~~~~~~ diff --git a/docs/conf.py b/docs/conf.py index 5049de40..0ccebebd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -147,6 +147,12 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False +# set the environment variable before imports +# TODO: remove after global init of WorkerAppSettings in worker/__init__.py +os.environ["SYNCMASTER__ENCRYPTION__CRYPTO_KEY"] = "crypto_key" +os.environ["SYNCMASTER__DATABASE__URL"] = "postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster" +os.environ["SYNCMASTER__BROKER__URL"] = "amqp://guest:guest@localhost:5672/" + # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. diff --git a/docs/design/entities.rst b/docs/design/entities.rst index f281160c..a823bb6c 100644 --- a/docs/design/entities.rst +++ b/docs/design/entities.rst @@ -121,7 +121,7 @@ Example: "started_at": "2024-01-19T16:30:07+03:00", "ended_at": null, "status": "STARTED", - "log_url": "https://kinaba.url/...", + "log_url": "https://kibana.url/...", "transfer_dump": { "transfer object JSON" }, diff --git a/docs/index.rst b/docs/index.rst index 9a937b1f..9308729b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,8 +17,8 @@ backend/install backend/architecture backend/auth/index - backend/openapi backend/configuration/index + backend/openapi .. toctree:: @@ -26,6 +26,8 @@ :caption: Worker :hidden: + worker/start_worker + worker/log_url worker/configuration/index @@ -34,6 +36,7 @@ :caption: Scheduler :hidden: + scheduler/start_scheduler scheduler/configuration/index .. toctree:: diff --git a/docs/scheduler/start_scheduler.rst b/docs/scheduler/start_scheduler.rst new file mode 100644 index 00000000..b0f82fe8 --- /dev/null +++ b/docs/scheduler/start_scheduler.rst @@ -0,0 +1,41 @@ +Starting the Scheduler +====================== + + +With docker +----------- + +Installation process +~~~~~~~~~~~~~~~~~~~~ + +Docker will download worker image of syncmaster scheduler (same as backend image) & broker, and run them. +Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` + +.. dropdown:: ``docker-compose.yml`` + + .. literalinclude:: ../../docker-compose.yml + :emphasize-lines: 68-83 + +.. dropdown:: ``.env.docker`` + + .. literalinclude:: ../../.env.docker + :emphasize-lines: 11-25 + +To start the worker container you need to run the command: + +.. code-block:: bash + + docker compose up scheduler -d --wait --wait-timeout 200 + + + +Without docker +-------------- + +To start the scheduler you need to run the command + +.. code-block:: bash + + python -m syncmaster.scheduler + + diff --git a/docs/worker/log_url.rst b/docs/worker/log_url.rst new file mode 100644 index 00000000..856a275b --- /dev/null +++ b/docs/worker/log_url.rst @@ -0,0 +1,13 @@ +Setting the `Run.log_url` value +=============================== + +Each run in the system is linked to a log URL where the Celery worker logs are available. This log URL might point to an Elastic instance or another logging tool such as Grafana. The log URL is generated based on a template configured in the backend. + +The configuration parameter is: + +.. code-block:: bash + + SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} + +You can search for each run by either its correlation id ``CORRELATION_CELERY_HEADER_ID`` in http headers or the ``Run.Id``. + diff --git a/docs/worker/start_worker.rst b/docs/worker/start_worker.rst new file mode 100644 index 00000000..6acb6167 --- /dev/null +++ b/docs/worker/start_worker.rst @@ -0,0 +1,53 @@ +Starting the Celery Worker +========================== + +.. note:: + + Before starting the worker you need to create a queue. + The queue is created by sending a post request to ``/queues`` endpoint (See Swagger doc for details). + + +With docker +----------- + +Installation process +~~~~~~~~~~~~~~~~~~~~ + +Docker will download worker image of syncmaster worker & broker, and run them. +Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml`` + +.. dropdown:: ``docker-compose.yml`` + + .. literalinclude:: ../../docker-compose.yml + :emphasize-lines: 55-66 + +.. dropdown:: ``.env.docker`` + + .. literalinclude:: ../../.env.docker + :emphasize-lines: 11-22 + +To start the worker container you need to run the command: + +.. code-block:: bash + + docker compose up worker -d --wait --wait-timeout 200 + + + +Without docker +-------------- + +To start the worker you need to run the command + +.. code-block:: bash + + python -m celery -A syncmaster.worker.celery worker + +You can specify options like concurrency and queues by adding additional flags: + +.. code-block:: bash + + celery -A -A syncmaster.worker.celery worker --concurrency=4 --max-tasks-per-child=1 --loglevel=info + + +Refer to the `Celery `_ documentation for more advanced start options. diff --git a/syncmaster/scheduler/settings/__init__.py b/syncmaster/scheduler/settings/__init__.py index 9163e993..6ff40ba0 100644 --- a/syncmaster/scheduler/settings/__init__.py +++ b/syncmaster/scheduler/settings/__init__.py @@ -32,7 +32,35 @@ class SchedulerSettings(BaseSettings): class SchedulerAppSettings(BaseSettings): - """Scheduler application settings.""" + """ + Scheduler application settings. + + This class is used to configure various settings for the scheduler application. + The settings can be defined in two ways: + + 1. By explicitly passing a settings object as an argument. + 2. By setting environment variables matching specific keys. + + All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. + Nested items are delimited with ``__``. + + Examples + -------- + + .. code-block:: bash + + # Example of setting a TRANSFER_FETCHING_TIMEOUT_SECONDS via environment variable + SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200 + + # Example of setting a database URL via environment variable + SYNCMASTER__DATABASE__URL=postgresql+asyncpg://user:password@localhost:5432/dbname + + # Example of setting a broker URL via environment variable + SYNCMASTER__BROKER__URL=amqp://user:password@localhost:5672/ + + Refer to `Pydantic documentation `_ + for more details on configuration options and environment variable usage. + """ database: DatabaseSettings = Field(description="Database settings") broker: RabbitMQSettings = Field(description="Broker settings") diff --git a/syncmaster/settings/credentials.py b/syncmaster/settings/credentials.py index f99590de..748d58dc 100644 --- a/syncmaster/settings/credentials.py +++ b/syncmaster/settings/credentials.py @@ -12,7 +12,7 @@ class CredentialsEncryptionSettings(BaseModel): .. code-block:: bash # Set the encryption key - SYNCMASTER_CRYPTO_KEY=secret_key + SYNCMASTER__ENCRYPTION__CRYPTO_KEY=secret_key """ crypto_key: str = Field(description="Key for encrypt/decrypt credentials data") diff --git a/syncmaster/worker/settings/__init__.py b/syncmaster/worker/settings/__init__.py index 1e0061c3..b9cd1719 100644 --- a/syncmaster/worker/settings/__init__.py +++ b/syncmaster/worker/settings/__init__.py @@ -34,6 +34,35 @@ class WorkerSettings(BaseSettings): class WorkerAppSettings(BaseSettings): + """ + Worker application settings. + + This class is used to configure various settings for the worker application. + The settings can be defined in two ways: + + 1. By explicitly passing a settings object as an argument. + 2. By setting environment variables matching specific keys. + + All environment variable names are written in uppercase and should be prefixed with ``SYNCMASTER__``. + Nested items are delimited with ``__``. + + Examples + -------- + + .. code-block:: bash + + # Example of setting a CORRELATION_CELERY_HEADER_ID via environment variable + SYNCMASTER__WORKER__CORRELATION_CELERY_HEADER_ID=CORRELATION_ID_CELERY + + # Example of setting a database URL via environment variable + SYNCMASTER__DATABASE__URL=postgresql+asyncpg://user:password@localhost:5432/dbname + + # Example of setting a broker URL via environment variable + SYNCMASTER__BROKER__URL=amqp://user:password@localhost:5672/ + + Refer to `Pydantic documentation `_ + for more details on configuration options and environment variable usage. + """ database: DatabaseSettings = Field(description="Database settings") broker: RabbitMQSettings = Field(description="Broker settings")