[DOP-19933] - update worker, scheduler, keycloak documentation (#147)

.env.docker

@@ -5,6 +5,15 @@ ENV=LOCAL
 SYNCMASTER__SERVER__DEBUG=true
 SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
 
+# Session
+SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Postgres
+SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
+
+# RabbitMQ
+SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+
 # Logging
 SYNCMASTER__LOGGING__SETUP=True
 SYNCMASTER__LOGGING__PRESET=colored
@@ -15,13 +24,6 @@ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 # Scheduler settings
 SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
-# Session
-SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
-
-# Postgres
-SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
-
-# TODO: add to KeycloakAuthProvider documentation about creating new realms, add users, etc.
 # KEYCLOAK Auth
 SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
 SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
@@ -36,9 +38,6 @@ SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.K
 SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
 SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
 
-# RabbitMQ
-SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
-
 # S3
 MINIO_ACCESS_KEY=syncmaster
 MINIO_SECRET_KEY=changeme

.env.local

@@ -5,21 +5,24 @@ export ENV=LOCAL
 export SYNCMASTER__SERVER__DEBUG=true
 export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
 
+# Session
+export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Postgres
+export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+
+# RabbitMQ
+export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/
+
 # Logging
 export SYNCMASTER__LOGGING__SETUP=True
 export SYNCMASTER__LOGGING__PRESET=colored
 
-# Scheduler settings
-export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
-
-# Session
-export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
-
 # Encrypt / Decrypt credentials data
 export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
-# Postgres
-export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+# Scheduler settings
+export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
 # Keycloack Auth
 export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
@@ -35,9 +38,6 @@ export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_pro
 export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
 export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
 
-# RabbitMQ
-export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/
-
 export TEST_S3_HOST_FOR_CONFTEST=localhost
 export TEST_S3_PORT_FOR_CONFTEST=9010
 export TEST_S3_HOST_FOR_WORKER=test-s3

.readthedocs.yaml

@@ -2,6 +2,8 @@ version: 2
 
 build:
   os: ubuntu-22.04
+  apt_packages:
+    - make
   tools:
     python: '3.12'
   jobs:
@@ -17,7 +19,7 @@ build:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
       - python -m pip list -v
-      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json
+      - make openapi
 
 sphinx:
   configuration: docs/conf.py

Makefile

@@ -155,3 +155,6 @@ docs-cleanup: ##@Docs Cleanup docs
 	$(MAKE) -C docs clean
 
 docs-fresh: docs-cleanup docs-build ##@Docs Cleanup & build docs
+
+openapi: ##@Docs Generate OpenAPI schema
+	python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json

docker/Dockerfile.scheduler

@@ -13,7 +13,7 @@ ENV PYTHONPATH=/app
 COPY ./pyproject.toml ./poetry.lock* /app/
 
 RUN pip install --upgrade pip setuptools wheel packaging
-RUN poetry install --no-root --extras "scheduler" --without test,docs,dev
+RUN poetry install --no-root --extras "scheduler" --extras "backend" --without test,docs,dev
 
 COPY ./docker/entrypoint_scheduler.sh /app/entrypoint.sh
 ENTRYPOINT ["/app/entrypoint.sh"]

docs/backend/auth/index.rst

@@ -12,7 +12,7 @@ Syncmaster supports different auth provider implementations. You can change impl
     :caption: Auth providers
 
     dummy
-    keycloak
+    keycloak/index
 
 .. toctree::
     :maxdepth: 2

docs/backend/auth/keycloak.rst → docs/backend/auth/keycloak/index.rst

@@ -1,17 +1,18 @@
-.. _backend-auth-ldap:
+.. _keycloak-auth-provider:
 
 KeyCloak Auth provider
-==================
+======================
 
 Description
 -----------
+Keycloak auth provider uses `python-keycloak <https://pypi.org/project/python-keycloak/>`_ library to interact with Keycloak server. During the authentication process,
+KeycloakAuthProvider redirects user to Keycloak authentication page.
 
-TODO:
+After successful authentication, Keycloak redirects user back to Syncmaster with authorization code.
+Then KeycloakAuthProvider exchanges authorization code for an access token and uses it to get user information from Keycloak server.
+If user is not found in Syncmaster database, KeycloakAuthProvider creates it. Finally, KeycloakAuthProvider returns user with access token.
 
-Strategies
-----------
-
-TODO:
+You can follow interaction schema below.
 
 Interaction schema
 ------------------
@@ -76,6 +77,13 @@ Interaction schema
 Basic configuration
 -------------------
 
-.. autopydantic_model:: syncmaster.settings.auth.keycloak.KeycloakProviderSettings
-.. autopydantic_model:: syncmaster.settings.auth.jwt.JWTSettings
+.. autopydantic_model:: syncmaster.backend.settings.auth.keycloak.KeycloakAuthProviderSettings
+.. autopydantic_model:: syncmaster.backend.settings.auth.keycloak.KeycloakSettings
+.. autopydantic_model:: syncmaster.backend.settings.auth.jwt.JWTSettings
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Keycloak
+    :hidden:
 
+    local_installation

docs/backend/auth/keycloak/local_installation.rst

@@ -0,0 +1,121 @@
+Local installation and testing
+------------------------------
+
+You can test Keycloak auth locally with docker compose:
+
+
+.. code-block:: console
+
+    $ docker compose -f docker-compose.test.yml up keycloak -d
+
+
+Authorize in keycloak
+~~~~~~~~~~~~~~~~~~~~~
+
+At first, you have to go to `http://localhost:8080/admin <http://localhost:8080/admin>`_ and login via login: `admin`, password: `admin` (by default) to create realms.
+
+.. image:: images/keycloak-login.png
+   :width: 400px
+   :align: center
+
+
+Create new realm
+~~~~~~~~~~~~~~~~
+
+.. image:: images/keycloak-new-realm.png
+   :width: 400px
+   :align: center
+
+
+Create new realm name
+~~~~~~~~~~~~~~~~~~~~~
+
+Pass realm name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME` environment variable:
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm  # as on screen below
+
+.. image:: images/keycloak-new-realm_name.png
+   :width: 400px
+   :align: center
+
+
+Create new client
+~~~~~~~~~~~~~~~~~
+
+.. image:: images/keycloak-new-client.png
+   :width: 400px
+   :align: center
+
+
+Create new client name
+~~~~~~~~~~~~~~~~~~~~~~
+
+In created realm pass client name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID` environment variable:
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client  # as on screen below
+
+.. image:: images/keycloak-new-client_name.png
+   :width: 400px
+   :align: center
+
+
+Set ``client_authentication`` **ON** to receive client_secret
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. image:: images/keycloak-client-authentication.png
+   :width: 400px
+   :align: center
+
+Configure Redirect URI
+~~~~~~~~~~~~~~~~~~~~~~
+
+To configure the redirect URI where the browser will redirect to exchange the code provided from Keycloak for an access token, set the `SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI` environment variable. The default value for local development is `http://localhost:8000/auth/callback`.
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+
+Configure the client redirect URI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ensure that this URI is also configured as a valid redirect URI in your Keycloak client settings. This allows the browser to redirect to your application after the user successfully authenticates with Keycloak.
+
+.. image:: images/keycloak-client-redirect_uri.png
+   :width: 400px
+   :align: center
+
+Configure the client secret
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Now go to **Credentials** tab and add the client secret to the `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET` environment variable:
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez   # as on screen below
+
+.. image:: images/keycloak-client-secret.png
+   :width: 400px
+   :align: center
+
+Now you can use create users in this realms, check `keycloak documentation <https://www.keycloak.org/docs/latest/server_admin/#assembly-managing-users_server_administration_guide>`_ on how to manage users creation.
+
+ENVIRONMENT VARIABLES
+~~~~~~~~~~~~~~~~~~~~~
+
+After this you can user `KeycloakAuthProvider` in your application with provided environment variables:
+
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez
+    $ export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
+    $ export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
+    $ export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider

docs/backend/install.rst

@@ -79,15 +79,11 @@ Available *extras* are:
 Run database
 ~~~~~~~~~~~~
 
-Start Postgres instance somewhere, and set up environment variables:
+Start Postgres instance somewhere, and set up environment variable:
 
 .. code-block:: bash
 
-    POSTGRES_HOST=localhost
-    POSTGRES_PORT=5432
-    POSTGRES_DB=postgres
-    POSTGRES_USER=user
-    POSTGRES_PASSWORD=password
+    SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
 You can use virtually any database supported by `SQLAlchemy <https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls>`_,
 but the only one we really tested is Postgres.
@@ -111,33 +107,11 @@ options and commands are just the same.
 Run RabbitMQ
 ~~~~~~~~~~~~
 
-Start RabbitMQ instance somewhere, and set up environment variables:
+Start RabbitMQ instance somewhere, and set up environment variable:
 
 .. code-block:: bash
 
-    RABBITMQ_HOST=somehost
-    RABBITMQ_PORT=5672
-    RABBITMQ_USER=user
-    RABBITMQ_PASSWORD=password
-
-Run worker
-~~~~~~~~~~
-
-.. note::
-
-    Before starting the worker you need to create a queue.
-    The queue is created by sending a post request to ``/queues`` endpoint (See Swagger doc for details).
-
-to start the worker you need to run the command
-
-.. code-block:: console
-
-    $ celery -A syncmaster.worker.config.celery worker --loglevel=info --max-tasks-per-child=1 -Q queue_name
-
-.. note::
-
-    The specified celery options are given as an example, you can specify other options you need.
-
+    SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
 
 Run backend
 ~~~~~~~~~~~

docs/conf.py

@@ -147,6 +147,12 @@
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
+# set the environment variable before imports
+# TODO: remove after global init of WorkerAppSettings in worker/__init__.py
+os.environ["SYNCMASTER__ENCRYPTION__CRYPTO_KEY"] = "crypto_key"
+os.environ["SYNCMASTER__DATABASE__URL"] = "postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster"
+os.environ["SYNCMASTER__BROKER__URL"] = "amqp://guest:guest@localhost:5672/"
+
 # -- Options for HTMLHelp output ------------------------------------------
 
 # Output file base name for HTML help builder.

docs/design/entities.rst

@@ -121,7 +121,7 @@ Example:
         "started_at": "2024-01-19T16:30:07+03:00",
         "ended_at": null,
         "status": "STARTED",
-        "log_url": "https://kinaba.url/...",
+        "log_url": "https://kibana.url/...",
         "transfer_dump": {
             "transfer object JSON"
         },

docs/index.rst

@@ -17,15 +17,17 @@
     backend/install
     backend/architecture
     backend/auth/index
-    backend/openapi
     backend/configuration/index
+    backend/openapi
 
 
 .. toctree::
     :maxdepth: 2
     :caption: Worker
     :hidden:
 
+    worker/start_worker
+    worker/log_url
     worker/configuration/index
 
 
@@ -34,6 +36,7 @@
     :caption: Scheduler
     :hidden:
 
+    scheduler/start_scheduler
     scheduler/configuration/index
 
 .. toctree::