[DOP-13290] Refact code

.github/workflows/codeql-analysis.yml

@@ -61,7 +61,7 @@ jobs:
         run: poetry run flake8 syncmaster/
 
       - name: Run mypy
-        run: poetry run mypy --config-file ./pyproject.toml ./syncmaster/app
+        run: poetry run mypy --config-file ./pyproject.toml ./syncmaster/backend
 
   codeql:
     name: CodeQL

.gitlab-ci.yml

@@ -67,7 +67,7 @@ mypy:
   image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG}
   stage: static analysis
   script:
-    - python -m mypy ./syncmaster/app --config-file ./pyproject.toml
+    - python -m mypy ./syncmaster/backend --config-file ./pyproject.toml
   rules:
     - if: $CI_COMMIT_TAG
       when: never
@@ -78,7 +78,7 @@ flake8:
   image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG}
   stage: static analysis
   script:
-    - python -m flake8 --max-line-length 120 --format=default ./syncmaster/app 2>&1 | tee flake8.txt
+    - python -m flake8 --max-line-length 120 --format=default ./syncmaster/backend 2>&1 | tee flake8.txt
   artifacts:
     when: always
     paths: [flake8.txt]
@@ -92,7 +92,7 @@ black:
   image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG}
   stage: static analysis
   script:
-    - python -m black ./syncmaster/app --check
+    - python -m black ./syncmaster/backend --check
   rules:
     - if: $CI_COMMIT_TAG
       when: never
@@ -103,7 +103,7 @@ bandit:
   image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG}
   stage: static analysis
   script:
-    - python -m bandit -ll -iii -r ./syncmaster/app -f json -o ./bandit.json
+    - python -m bandit -ll -iii -r ./syncmaster/backend -f json -o ./bandit.json
   artifacts:
     when: always
     paths: [./bandit.json]
@@ -228,7 +228,7 @@ tests:
         TEST_S3_BUCKET: syncmaster
 
   script:
-    - python ./syncmaster/app/pre_start.py
+    - python ./syncmaster/backend/pre_start.py
     - python -m pytest -vv ./syncmaster/tests --log-cli-level=INFO -s --junitxml=${JUNIT_FILE}
   artifacts:
     when: always

.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
       #  files: ^(.*__main__\.py)$
       - id: insert-license
         files: .*\.py$
-        exclude: ^(syncmaster/tests/.*\.py)$
+        exclude: ^(tests/.*\.py)$
         args:
           - --license-filepath
           - .spdx-license-header.txt
@@ -98,7 +98,7 @@ repos:
     hooks:
       - id: mypy
         name: mypy
-        entry: mypy ./syncmaster/app --config-file ./pyproject.toml
+        entry: mypy ./syncmaster/backend --config-file ./pyproject.toml
         language: python
         language_version: python3.11
         require_serial: true

Makefile

@@ -108,13 +108,15 @@ check-fixtures: ##@Test Check declared fixtures without using
 run_back_docker:
 	docker run --env-file ./.env.docker \
 	-v ./syncmaster:/syncmaster \
+	-v ./tests:/syncmaster/tests \
 	-v ./cached_jars:/root/.ivy2 \
 	--net syncmaster_network -p 8000:8000 --rm \
 	 -it --name backend syncmaster_back /bin/bash
 
 run_worker_docker:
 	docker run --env-file ./.env.docker \
 	-v ./syncmaster:/syncmaster \
+	-v ./tests:/syncmaster/tests \
 	-v ./cached_jars:/root/.ivy2 \
 	--net syncmaster_network --rm \
 	-it --name worker syncmaster_worker /bin/bash

docker-compose.test.yml

@@ -54,7 +54,7 @@ services:
       - ./cached_jars:/root/.ivy2
       - ./reports:/syncmaster/reports
     env_file: .env.docker
-    command: [coverage, run, app/main.py]
+    command: [coverage, run, backend/main.py]
 
   rabbitmq:
     image: rabbitmq

docker/worker.dockerfile

@@ -33,12 +33,14 @@ COPY ./syncmaster/ /syncmaster/
 
 # https://docs.celeryq.dev/en/stable/userguide/workers.html#max-tasks-per-child-setting
 # Required to start each Celery task in separated process, avoiding issues with global Spark session object
-CMD ["celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"]
+CMD ["celery", "-A" ,"backend.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"]
 
 FROM prod as test
 
+COPY ./tests /syncmaster/tests
+
 RUN poetry install --no-root --extras "worker backend" --with test
 
 ENV CREATE_SPARK_SESSION_FUNCTION="tests.spark.get_worker_spark_session.get_worker_spark_session"
 # Queue for tests
-CMD ["coverage", "run", "-m", "celery", "-A" ,"app.tasks.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]
+CMD ["coverage", "run", "-m", "celery", "-A" ,"backend.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"]

pyproject.toml

@@ -28,15 +28,25 @@ classifiers = [
 keywords = ["Syncmaster", "REST", "API", "Worker", "Replication"]
 
 packages = [
-    { include = "syncmaster/app" },
+    { include = "syncmaster/backend" },
 ]
 
 include = [
-    {path = "syncmaster/*.ini"},
+    {path = "syncmaster/db/*"},
+    {path = "syncmaster/db/migrations/*"},
+    {path = "syncmaster/db/migrations/versions/*"},
+    {path = "syncmaster/db/repositories/*"},
+    {path = "syncmaster/dto/*.py"},
+    {path = "syncmaster/exceptions/*.py"},
+    {path = "syncmaster/schemas/v1/connections/*.py"},
+    {path = "syncmaster/schemas/v1/transfers/*.py"},
+    {path = "syncmaster/schemas/v1/transfers/file/*.py"},
+    {path = "syncmaster/schemas/v1/*.py"},
+    {path = "syncmaster/schemas/*.py"},
 ]
 
 exclude = [
-    {path = "syncmaster/tests"},
+    {path = "tests"},
 ]
 
 [tool.poetry.urls]  # Optional

syncmaster/app/api/router.py → syncmaster/backend/api/router.py

@@ -1,10 +1,9 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
 # SPDX-License-Identifier: Apache-2.0
+from backend.api import monitoring
+from backend.api.v1.router import router as v1_router
 from fastapi import APIRouter
 
-from app.api import monitoring
-from app.api.v1.router import router as v1_router
-
 api_router = APIRouter()
 api_router.include_router(monitoring.router)
 api_router.include_router(v1_router)

syncmaster/app/api/v1/auth/router.py → syncmaster/backend/api/v1/auth/router.py

@@ -1,14 +1,13 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
 # SPDX-License-Identifier: Apache-2.0
+from backend.api.deps import SettingsMarker, UnitOfWorkMarker
+from backend.api.v1.auth.utils import sign_jwt
+from backend.config import Settings
+from backend.services import UnitOfWork
+from exceptions import EntityNotFoundError
 from fastapi import APIRouter, Depends
 from fastapi.security import OAuth2PasswordRequestForm
-
-from app.api.deps import SettingsMarker, UnitOfWorkMarker
-from app.api.v1.auth.schemas import AuthTokenSchema
-from app.api.v1.auth.utils import sign_jwt
-from app.config import Settings
-from app.exceptions import EntityNotFoundError
-from app.services import UnitOfWork
+from schemas.v1.auth import AuthTokenSchema
 
 router = APIRouter(prefix="/auth", tags=["Auth"])
 

syncmaster/app/api/v1/auth/utils.py → syncmaster/backend/api/v1/auth/utils.py

@@ -2,11 +2,10 @@
 # SPDX-License-Identifier: Apache-2.0
 import time
 
+from backend.config import Settings
 from jose import JWTError, jwt
 from pydantic import ValidationError
-
-from app.api.v1.auth.schemas import TokenPayloadSchema
-from app.config import Settings
+from schemas.v1.auth import TokenPayloadSchema
 
 
 def sign_jwt(user_id: int, settings: Settings) -> str:

syncmaster/app/api/v1/connections/router.py → syncmaster/backend/api/v1/connections.py

@@ -3,32 +3,26 @@
 import asyncio
 from typing import get_args
 
+from backend.api.deps import UnitOfWorkMarker
+from backend.services import UnitOfWork, get_user
+from db import Permission, User
+from exceptions import ActionNotAllowedError, AuthDataNotFoundError, GroupNotFoundError
+from exceptions.connection import ConnectionDeleteError, ConnectionNotFoundError
 from fastapi import APIRouter, Depends, Query, status
 from pydantic import SecretStr
-
-from app.api.deps import UnitOfWorkMarker
-from app.api.v1.connections.schemas.connection import (
+from schemas.v1.connections.connection import (
     ConnectionCopySchema,
     ConnectionPageSchema,
     CreateConnectionSchema,
     ReadConnectionSchema,
     UpdateConnectionSchema,
 )
-from app.api.v1.schemas import (
+from schemas.v1.schemas import (
     ORACLE_TYPE,
     POSTGRES_TYPE,
     MetaPageSchema,
     StatusResponseSchema,
 )
-from app.db.models import User
-from app.db.utils import Permission
-from app.exceptions import (
-    ActionNotAllowedError,
-    AuthDataNotFoundError,
-    GroupNotFoundError,
-)
-from app.exceptions.connection import ConnectionDeleteError, ConnectionNotFoundError
-from app.services import UnitOfWork, get_user
 
 router = APIRouter(tags=["Connections"])
 

syncmaster/app/api/v1/groups/router.py → syncmaster/backend/api/v1/groups.py

@@ -1,21 +1,19 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
 # SPDX-License-Identifier: Apache-2.0
+from backend.api.deps import UnitOfWorkMarker
+from backend.services import UnitOfWork, get_user
+from db import Permission, User
+from exceptions import ActionNotAllowedError, GroupNotFoundError
 from fastapi import APIRouter, Depends, Query
-
-from app.api.deps import UnitOfWorkMarker
-from app.api.v1.groups.schemas import (
+from schemas.v1.groups import (
     AddUserSchema,
     CreateGroupSchema,
     GroupPageSchema,
     ReadGroupSchema,
     UpdateGroupSchema,
 )
-from app.api.v1.schemas import StatusResponseSchema
-from app.api.v1.users.schemas import UserPageSchemaAsGroupMember
-from app.db.models import User
-from app.db.utils import Permission
-from app.exceptions import ActionNotAllowedError, GroupNotFoundError
-from app.services import UnitOfWork, get_user
+from schemas.v1.schemas import StatusResponseSchema
+from schemas.v1.users import UserPageSchemaAsGroupMember
 
 router = APIRouter(tags=["Groups"])
 

syncmaster/app/api/v1/queue/router.py → syncmaster/backend/api/v1/queue.py

@@ -1,20 +1,18 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
 # SPDX-License-Identifier: Apache-2.0
+from backend.api.deps import UnitOfWorkMarker
+from backend.services import UnitOfWork, get_user
+from db import Permission, User
+from exceptions import ActionNotAllowedError, GroupNotFoundError
+from exceptions.queue import QueueDeleteError, QueueNotFoundError
 from fastapi import APIRouter, Depends, Query, status
-
-from app.api.deps import UnitOfWorkMarker
-from app.api.v1.queue.schemas import (
+from schemas.v1.queue import (
     CreateQueueSchema,
     QueuePageSchema,
     ReadQueueSchema,
     UpdateQueueSchema,
 )
-from app.api.v1.schemas import StatusResponseSchema
-from app.db.models import User
-from app.db.utils import Permission
-from app.exceptions import ActionNotAllowedError, GroupNotFoundError
-from app.exceptions.queue import QueueDeleteError, QueueNotFoundError
-from app.services import UnitOfWork, get_user
+from schemas.v1.schemas import StatusResponseSchema
 
 router = APIRouter(tags=["Queues"])
 

syncmaster/app/api/v1/router.py → syncmaster/backend/api/v1/router.py

@@ -1,14 +1,13 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
 # SPDX-License-Identifier: Apache-2.0
+from backend.api.v1.auth.router import router as auth_router
+from backend.api.v1.connections import router as connection_router
+from backend.api.v1.groups import router as group_router
+from backend.api.v1.queue import router as queue_router
+from backend.api.v1.transfers.router import router as transfer_router
+from backend.api.v1.users import router as user_router
 from fastapi import APIRouter
 
-from app.api.v1.auth.router import router as auth_router
-from app.api.v1.connections.router import router as connection_router
-from app.api.v1.groups.router import router as group_router
-from app.api.v1.queue.router import router as queue_router
-from app.api.v1.transfers.router import router as transfer_router
-from app.api.v1.users.router import router as user_router
-
 router = APIRouter(prefix="/v1")
 router.include_router(user_router)
 router.include_router(auth_router)

syncmaster/app/api/v1/transfers/router.py → syncmaster/backend/api/v1/transfers/router.py

@@ -2,27 +2,12 @@
 # SPDX-License-Identifier: Apache-2.0
 import asyncio
 
-from fastapi import APIRouter, Depends, Query, status
-from kombu.exceptions import KombuError
-
-from app.api.deps import UnitOfWorkMarker
-from app.api.v1.schemas import StatusCopyTransferResponseSchema, StatusResponseSchema
-from app.api.v1.transfers.schemas.run import (
-    CreateRunSchema,
-    ReadRunSchema,
-    RunPageSchema,
-)
-from app.api.v1.transfers.schemas.transfer import (
-    CopyTransferSchema,
-    CreateTransferSchema,
-    ReadTransferSchema,
-    TransferPageSchema,
-    UpdateTransferSchema,
-)
-from app.api.v1.transfers.utils import process_file_transfer_directory_path
-from app.db.models import Status, User
-from app.db.utils import Permission
-from app.exceptions import (
+from backend.api.deps import UnitOfWorkMarker
+from backend.api.v1.transfers.utils import process_file_transfer_directory_path
+from backend.services import UnitOfWork, get_user
+from backend.worker.config import celery
+from db import Permission, Status, User
+from exceptions import (
     CannotConnectToTaskQueueError,
     ConnectionNotFoundError,
     DifferentTransferAndConnectionsGroupsError,
@@ -31,9 +16,20 @@
     GroupNotFoundError,
     TransferNotFoundError,
 )
-from app.exceptions.base import ActionNotAllowedError
-from app.services import UnitOfWork, get_user
-from app.tasks.config import celery
+from exceptions.base import ActionNotAllowedError
+from fastapi import APIRouter, Depends, Query, status
+from kombu.exceptions import KombuError
+from schemas.v1.schemas import StatusCopyTransferResponseSchema, StatusResponseSchema
+from schemas.v1.transfers import (
+    CopyTransferSchema,
+    CreateRunSchema,
+    CreateTransferSchema,
+    ReadRunSchema,
+    ReadTransferSchema,
+    RunPageSchema,
+    TransferPageSchema,
+    UpdateTransferSchema,
+)
 
 router = APIRouter(tags=["Transfers"])
 

syncmaster/app/api/v1/transfers/utils.py → syncmaster/backend/api/v1/transfers/utils.py

@@ -1,9 +1,6 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems)
 # SPDX-License-Identifier: Apache-2.0
-from app.api.v1.transfers.schemas.transfer import (
-    CreateTransferSchema,
-    UpdateTransferSchema,
-)
+from schemas.v1.transfers import CreateTransferSchema, UpdateTransferSchema
 
 
 def process_file_transfer_directory_path(