[DOP-13290] Refact code

.env.docker

@@ -48,6 +48,6 @@ TEST_HIVE_PASSWORD=hive_password
 
 ENV=LOCAL
 
-SPARK_CONF_DIR=/syncmaster/tests/spark/hive/conf/
-HADOOP_CONF_DIR=/syncmaster/tests/spark/hadoop/
-HIVE_CONF_DIR=/syncmaster/tests/spark/hive/conf/
+SPARK_CONF_DIR=/app/tests/spark/hive/conf/
+HADOOP_CONF_DIR=/app/tests/spark/hadoop/
+HIVE_CONF_DIR=/app/tests/spark/hive/conf/

.env.local

@@ -48,6 +48,6 @@ export TEST_HIVE_PASSWORD=hive_password
 
 export ENV=LOCAL
 
-export SPARK_CONF_DIR=./syncmaster/tests/spark/hive/conf/
-export HADOOP_CONF_DIR=./syncmaster/tests/spark/hadoop/
-export HIVE_CONF_DIR=./syncmaster/tests/spark/hive/conf/
+export SPARK_CONF_DIR=./app/tests/spark/hive/conf/
+export HADOOP_CONF_DIR=./app/tests/spark/hadoop/
+export HIVE_CONF_DIR=./app/tests/spark/hive/conf/

.github/workflows/codeql-analysis.yml

@@ -61,7 +61,7 @@ jobs:
         run: poetry run flake8 syncmaster/
 
       - name: Run mypy
-        run: poetry run mypy --config-file ./pyproject.toml ./syncmaster/app
+        run: poetry run mypy --config-file ./pyproject.toml ./syncmaster/backend
 
   codeql:
     name: CodeQL

.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
       #  files: ^(.*__main__\.py)$
       - id: insert-license
         files: .*\.py$
-        exclude: ^(syncmaster/tests/.*\.py)$
+        exclude: ^(tests/.*\.py)$
         args:
           - --license-filepath
           - .spdx-license-header.txt
@@ -98,7 +98,7 @@ repos:
     hooks:
       - id: mypy
         name: mypy
-        entry: mypy ./syncmaster/app --config-file ./pyproject.toml
+        entry: mypy ./syncmaster/backend --config-file ./pyproject.toml
         language: python
         language_version: python3.11
         require_serial: true

Makefile

@@ -37,7 +37,7 @@ run: ##@Application Run backend locally (without docker)
 	RABBITMQ_USER=${RABBITMQ_USER} \
 	RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD} \
 	PYTHONPATH=${APP_PATH} \
-	${POETRY} run python ./syncmaster/app/main.py
+	${POETRY} run python ./syncmaster/backend/main.py
 
 revision: ##@Database Create new revision of migrations
 	@POSTGRES_HOST=${POSTGRES_HOST} \
@@ -107,14 +107,14 @@ check-fixtures: ##@Test Check declared fixtures without using
 
 run_back_docker:
 	docker run --env-file ./.env.docker \
-	-v ./syncmaster:/syncmaster \
+	-v .:/app \
 	-v ./cached_jars:/root/.ivy2 \
 	--net syncmaster_network -p 8000:8000 --rm \
 	 -it --name backend syncmaster_back /bin/bash
 
 run_worker_docker:
 	docker run --env-file ./.env.docker \
-	-v ./syncmaster:/syncmaster \
+	-v .:/app \
 	-v ./cached_jars:/root/.ivy2 \
 	--net syncmaster_network --rm \
 	-it --name worker syncmaster_worker /bin/bash

docker-compose.test.yml

@@ -29,7 +29,7 @@ services:
       - test-s3
     volumes:
       - ./cached_jars:/root/.ivy2
-      - ./reports:/syncmaster/reports
+      - ./reports:/app/reports
     env_file: .env.docker
 
   backend:
@@ -52,9 +52,9 @@ services:
       - test-s3
     volumes:
       - ./cached_jars:/root/.ivy2
-      - ./reports:/syncmaster/reports
+      - ./reports:/app/reports
     env_file: .env.docker
-    command: [coverage, run, app/main.py]
+    command: [coverage, run, /app/syncmaster/backend/main.py]
 
   rabbitmq:
     image: rabbitmq

docker/backend.dockerfile

@@ -20,16 +20,16 @@ RUN apt-get update && apt-get install -y \
 RUN pip install --no-cache-dir --timeout 3 --retries 3 poetry \
     && poetry config virtualenvs.create false
 
-WORKDIR /syncmaster
+WORKDIR /app
 
-COPY ./pyproject.toml ./poetry.lock* /syncmaster/
+COPY ./pyproject.toml ./poetry.lock* /app/
 
 RUN pip install --upgrade pip setuptools wheel packaging
 
 RUN poetry install --no-root --extras "backend"
 
-COPY ./syncmaster/ /syncmaster/
+COPY ./syncmaster /app/
 
-ENV PYTHONPATH=/syncmaster
+ENV PYTHONPATH=/app
 
-CMD [ "python", "app/main.py" ]
+CMD [ "python", "/app/syncmaster/backend/main.py" ]

docker/worker.dockerfile

@@ -20,25 +20,24 @@ RUN apt-get update && apt-get install -y \
 
 RUN pip install --no-cache-dir --timeout 3 --retries 3 poetry && poetry config virtualenvs.create false
 
-WORKDIR /syncmaster
+WORKDIR /app
 
-COPY ./pyproject.toml ./poetry.lock* /syncmaster/
+COPY ./pyproject.toml ./poetry.lock* /app/
 
 RUN pip install --upgrade pip setuptools wheel packaging
 
 RUN poetry install --no-root --extras "worker"
 
-COPY ./syncmaster/ /syncmaster/
-
+COPY . /app/
 
 # https://docs.celeryq.dev/en/stable/userguide/workers.html#max-tasks-per-child-setting
 # Required to start each Celery task in separated process, avoiding issues with global Spark session object
-CMD ["celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"]
+CMD ["celery", "-A" ,"syncmaster.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"]
 
 FROM prod as test
 
 RUN poetry install --no-root --extras "worker backend" --with test
 
 ENV CREATE_SPARK_SESSION_FUNCTION="tests.spark.get_worker_spark_session.get_worker_spark_session"
 # Queue for tests
-CMD ["coverage", "run", "-m", "celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]
+CMD ["coverage", "run", "-m", "celery", "-A" ,"syncmaster.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]

pyproject.toml

@@ -28,15 +28,11 @@ classifiers = [
 keywords = ["Syncmaster", "REST", "API", "Worker", "Replication"]
 
 packages = [
-    { include = "syncmaster/app" },
-]
-
-include = [
-    {path = "syncmaster/*.ini"},
+    { include = "syncmaster" },
 ]
 
 exclude = [
-    {path = "syncmaster/tests"},
+    {path = "tests"},
 ]
 
 [tool.poetry.urls]  # Optional

syncmaster/app/api/router.py → syncmaster/backend/api/router.py

@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 from fastapi import APIRouter
 
-from app.api import monitoring
-from app.api.v1.router import router as v1_router
+from syncmaster.backend.api import monitoring
+from syncmaster.backend.api.v1.router import router as v1_router
 
 api_router = APIRouter()
 api_router.include_router(monitoring.router)

syncmaster/app/api/v1/auth/router.py → syncmaster/backend/api/v1/auth/router.py

@@ -3,12 +3,12 @@
 from fastapi import APIRouter, Depends
 from fastapi.security import OAuth2PasswordRequestForm
 
-from app.api.deps import SettingsMarker, UnitOfWorkMarker
-from app.api.v1.auth.schemas import AuthTokenSchema
-from app.api.v1.auth.utils import sign_jwt
-from app.config import Settings
-from app.exceptions import EntityNotFoundError
-from app.services import UnitOfWork
+from syncmaster.backend.api.deps import SettingsMarker, UnitOfWorkMarker
+from syncmaster.backend.api.v1.auth.utils import sign_jwt
+from syncmaster.backend.services import UnitOfWork
+from syncmaster.config import Settings
+from syncmaster.exceptions import EntityNotFoundError
+from syncmaster.schemas.v1.auth import AuthTokenSchema
 
 router = APIRouter(prefix="/auth", tags=["Auth"])
 

syncmaster/app/api/v1/auth/utils.py → syncmaster/backend/api/v1/auth/utils.py

@@ -5,8 +5,8 @@
 from jose import JWTError, jwt
 from pydantic import ValidationError
 
-from app.api.v1.auth.schemas import TokenPayloadSchema
-from app.config import Settings
+from syncmaster.config import Settings
+from syncmaster.schemas.v1.auth import TokenPayloadSchema
 
 
 def sign_jwt(user_id: int, settings: Settings) -> str:

syncmaster/app/api/v1/connections/router.py → syncmaster/backend/api/v1/connections.py

@@ -6,29 +6,27 @@
 from fastapi import APIRouter, Depends, Query, status
 from pydantic import SecretStr
 
-from app.api.deps import UnitOfWorkMarker
-from app.api.v1.connections.schemas.connection import (
+from syncmaster.backend.api.deps import UnitOfWorkMarker
+from syncmaster.backend.services import UnitOfWork, get_user
+from syncmaster.db.models import User
+from syncmaster.db.utils import Permission
+from syncmaster.exceptions import ActionNotAllowedError
+from syncmaster.exceptions.connection import (
+    ConnectionDeleteError,
+    ConnectionNotFoundError,
+)
+from syncmaster.exceptions.credentials import AuthDataNotFoundError
+from syncmaster.exceptions.group import GroupNotFoundError
+from syncmaster.schemas.v1.connection_types import ORACLE_TYPE, POSTGRES_TYPE
+from syncmaster.schemas.v1.connections.connection import (
     ConnectionCopySchema,
     ConnectionPageSchema,
     CreateConnectionSchema,
     ReadConnectionSchema,
     UpdateConnectionSchema,
 )
-from app.api.v1.schemas import (
-    ORACLE_TYPE,
-    POSTGRES_TYPE,
-    MetaPageSchema,
-    StatusResponseSchema,
-)
-from app.db.models import User
-from app.db.utils import Permission
-from app.exceptions import (
-    ActionNotAllowedError,
-    AuthDataNotFoundError,
-    GroupNotFoundError,
-)
-from app.exceptions.connection import ConnectionDeleteError, ConnectionNotFoundError
-from app.services import UnitOfWork, get_user
+from syncmaster.schemas.v1.page import MetaPageSchema
+from syncmaster.schemas.v1.status import StatusResponseSchema
 
 router = APIRouter(tags=["Connections"])
 

syncmaster/app/api/v1/groups/router.py → syncmaster/backend/api/v1/groups.py

@@ -2,20 +2,21 @@
 # SPDX-License-Identifier: Apache-2.0
 from fastapi import APIRouter, Depends, Query
 
-from app.api.deps import UnitOfWorkMarker
-from app.api.v1.groups.schemas import (
+from syncmaster.backend.api.deps import UnitOfWorkMarker
+from syncmaster.backend.services import UnitOfWork, get_user
+from syncmaster.db.models import User
+from syncmaster.db.utils import Permission
+from syncmaster.exceptions import ActionNotAllowedError
+from syncmaster.exceptions.group import GroupNotFoundError
+from syncmaster.schemas.v1.groups import (
     AddUserSchema,
     CreateGroupSchema,
     GroupPageSchema,
     ReadGroupSchema,
     UpdateGroupSchema,
 )
-from app.api.v1.schemas import StatusResponseSchema
-from app.api.v1.users.schemas import UserPageSchemaAsGroupMember
-from app.db.models import User
-from app.db.utils import Permission
-from app.exceptions import ActionNotAllowedError, GroupNotFoundError
-from app.services import UnitOfWork, get_user
+from syncmaster.schemas.v1.status import StatusResponseSchema
+from syncmaster.schemas.v1.users import UserPageSchemaAsGroupMember
 
 router = APIRouter(tags=["Groups"])