From 580873bc961a5b49983e4abb5828e1380b9fd51f Mon Sep 17 00:00:00 2001 From: DMITRIY PEDCHENKO Date: Mon, 18 Mar 2024 17:38:55 +0300 Subject: [PATCH] [DOP-13290] Refact code --- .github/workflows/codeql-analysis.yml | 2 +- .gitlab-ci.yml | 242 ------------------ docker/backend.dockerfile | 6 +- docker/worker.dockerfile | 4 +- pyproject.toml | 16 +- syncmaster/{backend/worker => }/__init__.py | 0 syncmaster/backend/api/router.py | 5 +- syncmaster/backend/api/v1/auth/router.py | 13 +- syncmaster/backend/api/v1/auth/utils.py | 5 +- syncmaster/backend/api/v1/connections.py | 22 +- syncmaster/backend/api/v1/groups.py | 15 +- syncmaster/backend/api/v1/queue.py | 15 +- syncmaster/backend/api/v1/router.py | 13 +- syncmaster/backend/api/v1/transfers/router.py | 28 +- syncmaster/backend/api/v1/transfers/utils.py | 2 +- syncmaster/backend/api/v1/users.py | 13 +- syncmaster/backend/config.py | 2 +- syncmaster/backend/handler.py | 9 +- syncmaster/backend/main.py | 22 +- syncmaster/backend/pre_start.py | 3 +- syncmaster/backend/services/__init__.py | 4 +- syncmaster/backend/services/auth.py | 11 +- syncmaster/backend/services/unit_of_work.py | 11 +- syncmaster/db/__init__.py | 8 +- syncmaster/db/alembic.ini | 2 +- syncmaster/db/factory.py | 5 +- syncmaster/db/migrations/env.py | 5 +- syncmaster/db/models.py | 5 +- syncmaster/db/repositories/__init__.py | 16 +- syncmaster/db/repositories/base.py | 5 +- syncmaster/db/repositories/connection.py | 13 +- .../db/repositories/credentials_repository.py | 11 +- syncmaster/db/repositories/group.py | 13 +- syncmaster/db/repositories/queue.py | 15 +- .../db/repositories/repository_with_owner.py | 7 +- syncmaster/db/repositories/run.py | 17 +- syncmaster/db/repositories/transfer.py | 15 +- syncmaster/db/repositories/user.py | 13 +- syncmaster/db/repositories/utils.py | 3 +- syncmaster/dto/transfers.py | 2 +- syncmaster/exceptions/__init__.py | 20 +- syncmaster/exceptions/connection.py | 2 +- syncmaster/exceptions/credentials.py | 2 +- syncmaster/exceptions/group.py | 2 +- syncmaster/exceptions/queue.py | 2 +- syncmaster/exceptions/run.py | 4 +- syncmaster/exceptions/transfer.py | 2 +- syncmaster/exceptions/user.py | 2 +- syncmaster/schemas/v1/__init__.py | 14 +- syncmaster/schemas/v1/connections/__init__.py | 2 +- .../schemas/v1/connections/connection.py | 13 +- syncmaster/schemas/v1/connections/hdfs.py | 3 +- syncmaster/schemas/v1/connections/hive.py | 3 +- syncmaster/schemas/v1/connections/oracle.py | 3 +- syncmaster/schemas/v1/connections/postgres.py | 3 +- syncmaster/schemas/v1/connections/s3.py | 3 +- syncmaster/schemas/v1/groups.py | 5 +- syncmaster/schemas/v1/queue.py | 3 +- syncmaster/schemas/v1/schemas.py | 3 +- syncmaster/schemas/v1/transfers/__init__.py | 10 +- syncmaster/schemas/v1/transfers/db.py | 3 +- syncmaster/schemas/v1/transfers/file/base.py | 3 +- syncmaster/schemas/v1/transfers/file/hdfs.py | 4 +- syncmaster/schemas/v1/transfers/file/s3.py | 4 +- .../schemas/v1/transfers/file_format.py | 3 +- syncmaster/schemas/v1/transfers/run.py | 5 +- syncmaster/schemas/v1/transfers/strategy.py | 3 +- syncmaster/schemas/v1/transfers/transfer.py | 13 +- syncmaster/schemas/v1/users.py | 5 +- .../worker/handlers => worker}/__init__.py | 0 syncmaster/{backend => }/worker/base.py | 3 +- syncmaster/{backend => }/worker/config.py | 7 +- syncmaster/{backend => }/worker/controller.py | 22 +- .../file => worker/handlers}/__init__.py | 0 .../{backend => }/worker/handlers/base.py | 5 +- syncmaster/worker/handlers/file/__init__.py | 2 + .../worker/handlers/file/base.py | 7 +- .../worker/handlers/file/hdfs.py | 3 +- .../{backend => }/worker/handlers/file/s3.py | 3 +- .../{backend => }/worker/handlers/hive.py | 7 +- .../{backend => }/worker/handlers/oracle.py | 7 +- .../{backend => }/worker/handlers/postgres.py | 7 +- syncmaster/{backend => }/worker/transfer.py | 15 +- syncmaster/{backend => }/worker/utils.py | 5 +- tests/config/ivysettings.xml | 17 -- tests/conftest.py | 16 +- tests/spark/get_worker_spark_session.py | 7 +- .../test_migrationsup_to_date.py | 2 +- .../test_run_transfer/conftest.py | 33 ++- .../test_hdfs_to_postgres.py | 2 +- .../test_hive_to_postgres.py | 2 +- .../test_oracle_to_postgres.py | 2 +- .../test_postgres_to_hdfs.py | 2 +- .../test_postgres_to_hive.py | 2 +- .../test_postgres_to_oracle.py | 2 +- .../test_run_transfer/test_postgres_to_s3.py | 2 +- .../test_read_mixed_column_naming.py | 2 +- .../test_run_transfer/test_s3_to_postgres.py | 2 +- tests/test_unit/conftest.py | 8 +- .../test_connections/test_copy_connection.py | 6 +- .../test_create_hdfs_connection.py | 6 +- .../test_create_hive_connection.py | 6 +- .../test_create_oracle_connection.py | 6 +- .../test_create_postgres_connection.py | 6 +- .../test_create_s3_connection.py | 6 +- .../test_create_connection.py | 6 +- .../test_delete_connection.py | 2 +- .../test_read_connection_types.py | 2 +- .../test_connections/test_read_connections.py | 2 +- .../test_groups/test_create_group.py | 2 +- .../test_groups/test_delete_group_by_id.py | 2 +- .../test_unit/test_queue/test_create_queue.py | 2 +- .../test_unit/test_queue/test_delete_queue.py | 2 +- tests/test_unit/test_queue/test_read_queue.py | 2 +- .../test_unit/test_queue/test_read_queues.py | 2 +- .../test_unit/test_queue/test_update_queue.py | 2 +- .../test_transfers/test_copy_transfer.py | 2 +- .../test_transfers/test_create_transfer.py | 2 +- .../test_transfers/test_delete_transfer.py | 2 +- .../test_create_transfer.py | 2 +- .../test_transfers/test_runs/conftest.py | 4 +- .../test_runs/test_create_run.py | 16 +- .../test_transfers/test_runs/test_stop_run.py | 2 +- .../test_transfers/test_update_transfer.py | 2 +- tests/test_unit/utils.py | 18 +- tests/utils.py | 6 +- 126 files changed, 454 insertions(+), 620 deletions(-) delete mode 100644 .gitlab-ci.yml rename syncmaster/{backend/worker => }/__init__.py (100%) rename syncmaster/{backend/worker/handlers => worker}/__init__.py (100%) rename syncmaster/{backend => }/worker/base.py (88%) rename syncmaster/{backend => }/worker/config.py (72%) rename syncmaster/{backend => }/worker/controller.py (84%) rename syncmaster/{backend/worker/handlers/file => worker/handlers}/__init__.py (100%) rename syncmaster/{backend => }/worker/handlers/base.py (93%) create mode 100644 syncmaster/worker/handlers/file/__init__.py rename syncmaster/{backend => }/worker/handlers/file/base.py (92%) rename syncmaster/{backend => }/worker/handlers/file/hdfs.py (84%) rename syncmaster/{backend => }/worker/handlers/file/s3.py (91%) rename syncmaster/{backend => }/worker/handlers/hive.py (87%) rename syncmaster/{backend => }/worker/handlers/oracle.py (89%) rename syncmaster/{backend => }/worker/handlers/postgres.py (89%) rename syncmaster/{backend => }/worker/transfer.py (88%) rename syncmaster/{backend => }/worker/utils.py (96%) delete mode 100644 tests/config/ivysettings.xml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5f9bdb2f..aafbe36d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -61,7 +61,7 @@ jobs: run: poetry run flake8 syncmaster/ - name: Run mypy - run: poetry run mypy --config-file ./pyproject.toml ./syncmaster/backend + run: poetry run mypy --config-file ./pyproject.toml ./backend codeql: name: CodeQL diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 411b3207..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,242 +0,0 @@ -default: - tags: - - bigdata - - dmz - - docker - -stages: - - build test image - - static analysis - - tests - -include: - - project: DevOps/cicd-store/gitlabci/docker - ref: v9 - file: - - .base_docker_build.yml - - .base_docker_cleaner.yml - -workflow: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: never - - - if: $CI_COMMIT_BRANCH =~ /^(master|main)$/ - when: never - - - when: always - -.build_test_image: - extends: .base_docker_build - stage: build test image - variables: - CICD_DOCKER__IMAGE_TAG: ${CI_COMMIT_REF_SLUG} - CICD_DOCKER__STOP_IF_EXISTS: 'false' - CICD_DOCKER__BUILD_EXTRA_ARGS: --pull --force-rm - CICD_STORE_DEBUG: 'true' - rules: - - if: $CI_COMMIT_TAG - when: never - - when: on_success - interruptible: true - -backend_test_image: - extends: .build_test_image - variables: - CICD_DOCKER__FILE_PATH: ./docker/backend.dockerfile - CICD_DOCKER__IMAGE_NAME: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend - CICD_DOCKER__CACHE_FROM: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:develop - -worker_test_image: - extends: .build_test_image - variables: - CICD_DOCKER__FILE_PATH: ./docker/worker.dockerfile - CICD_DOCKER__IMAGE_NAME: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/worker - CICD_DOCKER__CACHE_FROM: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/worker:develop - CICD_DOCKER__BUILD_EXTRA_ARGS: --target=prod - -worker_prod_image: - extends: .build_test_image - variables: - CICD_DOCKER__FILE_PATH: ./docker/worker.dockerfile - CICD_DOCKER__IMAGE_NAME: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/test/worker - CICD_DOCKER__CACHE_FROM: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/test/worker:develop - CICD_DOCKER__BUILD_EXTRA_ARGS: --target=test - -mypy: - image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG} - stage: static analysis - script: - - python -m mypy ./syncmaster/backend --config-file ./pyproject.toml - rules: - - if: $CI_COMMIT_TAG - when: never - - when: on_success - interruptible: true - -flake8: - image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG} - stage: static analysis - script: - - python -m flake8 --max-line-length 120 --format=default ./syncmaster/backend 2>&1 | tee flake8.txt - artifacts: - when: always - paths: [flake8.txt] - rules: - - if: $CI_COMMIT_TAG - when: never - - when: on_success - interruptible: true - -black: - image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG} - stage: static analysis - script: - - python -m black ./syncmaster/backend --check - rules: - - if: $CI_COMMIT_TAG - when: never - - when: on_success - interruptible: true - -bandit: - image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG} - stage: static analysis - script: - - python -m bandit -ll -iii -r ./syncmaster/backend -f json -o ./bandit.json - artifacts: - when: always - paths: [./bandit.json] - rules: - - if: $CI_COMMIT_TAG - when: never - - when: on_success - interruptible: true - -tests: - image: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/backend:${CI_COMMIT_REF_SLUG} - stage: tests - variables: - CI_DEBUG_SERVICES: 'true' - FF_NETWORK_PER_BUILD: 1 - JUNIT_FILE: junitxml.xml - ENV: GITLAB - - CRYPTO_KEY: UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94= - - POSTGRES_HOST: db - POSTGRES_PORT: 5432 - POSTGRES_DB: syncmaster - POSTGRES_USER: backend - POSTGRES_PASSWORD: backend - - RABBITMQ_HOST: rabbitmq - RABBITMQ_PORT: 5672 - RABBITMQ_USER: guest - RABBITMQ_PASSWORD: guest - - TEST_POSTGRES_HOST: test-postgres - TEST_POSTGRES_PORT: 5432 - TEST_POSTGRES_USER: test_user - TEST_POSTGRES_PASSWORD: test_password - TEST_POSTGRES_DB: test_db - - TEST_ORACLE_HOST: test-oracle - TEST_ORACLE_PORT: 1521 - TEST_ORACLE_USER: test_user - TEST_ORACLE_PASSWORD: test_password - TEST_ORACLE_SERVICE_NAME: test_oracle - - TEST_HIVE_CLUSTER: test-hive - TEST_HIVE_USER: hive_user - TEST_HIVE_PASSWORD: hive_password - - HDFS_HOST: test-hive - HDFS_WEBHDFS_PORT: 9870 - HDFS_IPC_PORT: 9820 - - TEST_S3_HOST: s3 - TEST_S3_PORT: 9000 - TEST_S3_ACCESS_KEY: syncmaster - TEST_S3_SECRET_KEY: a2b6ad842d16f47beaa077b0e50c217a3bded54c - TEST_S3_BUCKET: syncmaster - - SPARK_CONF_DIR: /syncmaster/tests/spark/hive/conf/ - HADOOP_CONF_DIR: /syncmaster/tests/spark/hadoop/ - HIVE_CONF_DIR: /syncmaster/tests/spark/hive/conf/ - - services: - - name: central-mirror.services.mts.ru/postgres:15 - alias: db - variables: - POSTGRES_DB: syncmaster - POSTGRES_USER: backend - POSTGRES_PASSWORD: backend - - - name: central-mirror.services.mts.ru/rabbitmq - alias: rabbitmq - - - name: ${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_PATH}/test/worker:${CI_COMMIT_REF_SLUG} - alias: worker - - - name: central-mirror.services.mts.ru/postgres:15 - alias: test-postgres - variables: - POSTGRES_DB: test_db - POSTGRES_USER: test_user - POSTGRES_PASSWORD: test_password - - - name: central-mirror.services.mts.ru/gvenzl/oracle-xe:slim-faststart - alias: test-oracle - variables: - TZ: UTC - ORACLE_PASSWORD: test_password - ORACLE_DATABASE: test_oracle - APP_USER: test_user - APP_USER_PASSWORD: test_password - - - name: central-mirror.services.mts.ru/postgres:15-alpine - alias: metastore-hive - variables: - POSTGRES_DB: metastore - POSTGRES_USER: test_hive - POSTGRES_PASSWORD: test_hive - - - name: central-mirror.services.mts.ru/mtsrus/hadoop:hadoop2.7.3-hive2.3.9 - alias: test-hive - variables: - HADOOP_CONF_DIR: /etc/hadoop - HIVE_CONF_DIR: /opt/hive/conf - WITH_HIVE_SERVER: 'false' # оставляем только сервер метастора, сам Hive нам не требуется, не тратим на него ресурсы - HIVE_METASTORE_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore - HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver - HIVE_METASTORE_DB_USER: test_hive - HIVE_METASTORE_DB_PASSWORD: test_hive - - - name: central-mirror.services.mts.ru/bitnami/minio:latest - alias: s3 - variables: - MINIO_ACCESS_KEY: syncmaster - MINIO_ROOT_USER: syncmaster - MINIO_SECRET_KEY: a2b6ad842d16f47beaa077b0e50c217a3bded54c - MINIO_ROOT_PASSWORD: a2b6ad842d16f47beaa077b0e50c217a3bded54c - - TEST_S3_HOST: s3 - TEST_S3_PORT: 9000 - TEST_S3_ACCESS_KEY: syncmaster - TEST_S3_SECRET_KEY: a2b6ad842d16f47beaa077b0e50c217a3bded54c - TEST_S3_BUCKET: syncmaster - - script: - - python ./syncmaster/backend/pre_start.py - - python -m pytest -vv ./syncmaster/tests --log-cli-level=INFO -s --junitxml=${JUNIT_FILE} - artifacts: - when: always - reports: - junit: - - ${JUNIT_FILE} - rules: - - if: $CI_COMMIT_TAG - when: never - - when: on_success - interruptible: true diff --git a/docker/backend.dockerfile b/docker/backend.dockerfile index 15befb0a..42dbcc6a 100644 --- a/docker/backend.dockerfile +++ b/docker/backend.dockerfile @@ -30,6 +30,8 @@ RUN poetry install --no-root --extras "backend" COPY ./syncmaster/ /syncmaster/ -ENV PYTHONPATH=/syncmaster +COPY ./tests/ /tests/ -CMD [ "python", "app/main.py" ] \ No newline at end of file +ENV PYTHONPATH=/ + +CMD [ "python", "backend/main.py" ] \ No newline at end of file diff --git a/docker/worker.dockerfile b/docker/worker.dockerfile index 225f835f..52790a1c 100644 --- a/docker/worker.dockerfile +++ b/docker/worker.dockerfile @@ -33,7 +33,7 @@ COPY ./syncmaster/ /syncmaster/ # https://docs.celeryq.dev/en/stable/userguide/workers.html#max-tasks-per-child-setting # Required to start each Celery task in separated process, avoiding issues with global Spark session object -CMD ["celery", "-A" ,"backend.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"] +CMD ["celery", "-A" ,"syncmaster.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1"] FROM prod as test @@ -43,4 +43,4 @@ RUN poetry install --no-root --extras "worker backend" --with test ENV CREATE_SPARK_SESSION_FUNCTION="tests.spark.get_worker_spark_session.get_worker_spark_session" # Queue for tests -CMD ["coverage", "run", "-m", "celery", "-A" ,"backend.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"] +CMD ["coverage", "run", "-m", "celery", "-A" ,"syncmaster.worker.config.celery", "worker", "--loglevel=info", "--max-tasks-per-child=1", "-Q", "test_queue"] diff --git a/pyproject.toml b/pyproject.toml index 0d637fc5..db8d7ba5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,21 +28,7 @@ classifiers = [ keywords = ["Syncmaster", "REST", "API", "Worker", "Replication"] packages = [ - { include = "syncmaster/backend" }, -] - -include = [ - {path = "syncmaster/db/*"}, - {path = "syncmaster/db/migrations/*"}, - {path = "syncmaster/db/migrations/versions/*"}, - {path = "syncmaster/db/repositories/*"}, - {path = "syncmaster/dto/*.py"}, - {path = "syncmaster/exceptions/*.py"}, - {path = "syncmaster/schemas/v1/connections/*.py"}, - {path = "syncmaster/schemas/v1/transfers/*.py"}, - {path = "syncmaster/schemas/v1/transfers/file/*.py"}, - {path = "syncmaster/schemas/v1/*.py"}, - {path = "syncmaster/schemas/*.py"}, + { include = "syncmaster" }, ] exclude = [ diff --git a/syncmaster/backend/worker/__init__.py b/syncmaster/__init__.py similarity index 100% rename from syncmaster/backend/worker/__init__.py rename to syncmaster/__init__.py diff --git a/syncmaster/backend/api/router.py b/syncmaster/backend/api/router.py index 28934384..8a23cdbc 100644 --- a/syncmaster/backend/api/router.py +++ b/syncmaster/backend/api/router.py @@ -1,9 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.api import monitoring -from backend.api.v1.router import router as v1_router from fastapi import APIRouter +from syncmaster.backend.api import monitoring +from syncmaster.backend.api.v1.router import router as v1_router + api_router = APIRouter() api_router.include_router(monitoring.router) api_router.include_router(v1_router) diff --git a/syncmaster/backend/api/v1/auth/router.py b/syncmaster/backend/api/v1/auth/router.py index 9eddfe11..eee2c729 100644 --- a/syncmaster/backend/api/v1/auth/router.py +++ b/syncmaster/backend/api/v1/auth/router.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.api.deps import SettingsMarker, UnitOfWorkMarker -from backend.api.v1.auth.utils import sign_jwt -from backend.config import Settings -from backend.services import UnitOfWork -from exceptions import EntityNotFoundError from fastapi import APIRouter, Depends from fastapi.security import OAuth2PasswordRequestForm -from schemas.v1.auth import AuthTokenSchema + +from syncmaster.backend.api.deps import SettingsMarker, UnitOfWorkMarker +from syncmaster.backend.api.v1.auth.utils import sign_jwt +from syncmaster.backend.config import Settings +from syncmaster.backend.services import UnitOfWork +from syncmaster.exceptions import EntityNotFoundError +from syncmaster.schemas.v1.auth import AuthTokenSchema router = APIRouter(prefix="/auth", tags=["Auth"]) diff --git a/syncmaster/backend/api/v1/auth/utils.py b/syncmaster/backend/api/v1/auth/utils.py index bc9950a8..e65099cf 100644 --- a/syncmaster/backend/api/v1/auth/utils.py +++ b/syncmaster/backend/api/v1/auth/utils.py @@ -2,10 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 import time -from backend.config import Settings from jose import JWTError, jwt from pydantic import ValidationError -from schemas.v1.auth import TokenPayloadSchema + +from syncmaster.backend.config import Settings +from syncmaster.schemas.v1.auth import TokenPayloadSchema def sign_jwt(user_id: int, settings: Settings) -> str: diff --git a/syncmaster/backend/api/v1/connections.py b/syncmaster/backend/api/v1/connections.py index 3e74e9bb..6d6be954 100644 --- a/syncmaster/backend/api/v1/connections.py +++ b/syncmaster/backend/api/v1/connections.py @@ -3,21 +3,29 @@ import asyncio from typing import get_args -from backend.api.deps import UnitOfWorkMarker -from backend.services import UnitOfWork, get_user -from db import Permission, User -from exceptions import ActionNotAllowedError, AuthDataNotFoundError, GroupNotFoundError -from exceptions.connection import ConnectionDeleteError, ConnectionNotFoundError from fastapi import APIRouter, Depends, Query, status from pydantic import SecretStr -from schemas.v1.connections.connection import ( + +from syncmaster.backend.api.deps import UnitOfWorkMarker +from syncmaster.backend.services import UnitOfWork, get_user +from syncmaster.db import Permission, User +from syncmaster.exceptions import ( + ActionNotAllowedError, + AuthDataNotFoundError, + GroupNotFoundError, +) +from syncmaster.exceptions.connection import ( + ConnectionDeleteError, + ConnectionNotFoundError, +) +from syncmaster.schemas.v1.connections.connection import ( ConnectionCopySchema, ConnectionPageSchema, CreateConnectionSchema, ReadConnectionSchema, UpdateConnectionSchema, ) -from schemas.v1.schemas import ( +from syncmaster.schemas.v1.schemas import ( ORACLE_TYPE, POSTGRES_TYPE, MetaPageSchema, diff --git a/syncmaster/backend/api/v1/groups.py b/syncmaster/backend/api/v1/groups.py index 9dd4ea3b..875dbf2f 100644 --- a/syncmaster/backend/api/v1/groups.py +++ b/syncmaster/backend/api/v1/groups.py @@ -1,19 +1,20 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.api.deps import UnitOfWorkMarker -from backend.services import UnitOfWork, get_user -from db import Permission, User -from exceptions import ActionNotAllowedError, GroupNotFoundError from fastapi import APIRouter, Depends, Query -from schemas.v1.groups import ( + +from syncmaster.backend.api.deps import UnitOfWorkMarker +from syncmaster.backend.services import UnitOfWork, get_user +from syncmaster.db import Permission, User +from syncmaster.exceptions import ActionNotAllowedError, GroupNotFoundError +from syncmaster.schemas.v1.groups import ( AddUserSchema, CreateGroupSchema, GroupPageSchema, ReadGroupSchema, UpdateGroupSchema, ) -from schemas.v1.schemas import StatusResponseSchema -from schemas.v1.users import UserPageSchemaAsGroupMember +from syncmaster.schemas.v1.schemas import StatusResponseSchema +from syncmaster.schemas.v1.users import UserPageSchemaAsGroupMember router = APIRouter(tags=["Groups"]) diff --git a/syncmaster/backend/api/v1/queue.py b/syncmaster/backend/api/v1/queue.py index e63820d4..380a46de 100644 --- a/syncmaster/backend/api/v1/queue.py +++ b/syncmaster/backend/api/v1/queue.py @@ -1,18 +1,19 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.api.deps import UnitOfWorkMarker -from backend.services import UnitOfWork, get_user -from db import Permission, User -from exceptions import ActionNotAllowedError, GroupNotFoundError -from exceptions.queue import QueueDeleteError, QueueNotFoundError from fastapi import APIRouter, Depends, Query, status -from schemas.v1.queue import ( + +from syncmaster.backend.api.deps import UnitOfWorkMarker +from syncmaster.backend.services import UnitOfWork, get_user +from syncmaster.db import Permission, User +from syncmaster.exceptions import ActionNotAllowedError, GroupNotFoundError +from syncmaster.exceptions.queue import QueueDeleteError, QueueNotFoundError +from syncmaster.schemas.v1.queue import ( CreateQueueSchema, QueuePageSchema, ReadQueueSchema, UpdateQueueSchema, ) -from schemas.v1.schemas import StatusResponseSchema +from syncmaster.schemas.v1.schemas import StatusResponseSchema router = APIRouter(tags=["Queues"]) diff --git a/syncmaster/backend/api/v1/router.py b/syncmaster/backend/api/v1/router.py index b54c02b3..28da8cc1 100644 --- a/syncmaster/backend/api/v1/router.py +++ b/syncmaster/backend/api/v1/router.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.api.v1.auth.router import router as auth_router -from backend.api.v1.connections import router as connection_router -from backend.api.v1.groups import router as group_router -from backend.api.v1.queue import router as queue_router -from backend.api.v1.transfers.router import router as transfer_router -from backend.api.v1.users import router as user_router from fastapi import APIRouter +from syncmaster.backend.api.v1.auth.router import router as auth_router +from syncmaster.backend.api.v1.connections import router as connection_router +from syncmaster.backend.api.v1.groups import router as group_router +from syncmaster.backend.api.v1.queue import router as queue_router +from syncmaster.backend.api.v1.transfers.router import router as transfer_router +from syncmaster.backend.api.v1.users import router as user_router + router = APIRouter(prefix="/v1") router.include_router(user_router) router.include_router(auth_router) diff --git a/syncmaster/backend/api/v1/transfers/router.py b/syncmaster/backend/api/v1/transfers/router.py index cb591464..4320077d 100644 --- a/syncmaster/backend/api/v1/transfers/router.py +++ b/syncmaster/backend/api/v1/transfers/router.py @@ -2,12 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio -from backend.api.deps import UnitOfWorkMarker -from backend.api.v1.transfers.utils import process_file_transfer_directory_path -from backend.services import UnitOfWork, get_user -from backend.worker.config import celery -from db import Permission, Status, User -from exceptions import ( +from fastapi import APIRouter, Depends, Query, status +from kombu.exceptions import KombuError + +from syncmaster.backend.api.deps import UnitOfWorkMarker +from syncmaster.backend.api.v1.transfers.utils import ( + process_file_transfer_directory_path, +) +from syncmaster.backend.services import UnitOfWork, get_user +from syncmaster.db import Permission, Status, User +from syncmaster.exceptions import ( CannotConnectToTaskQueueError, ConnectionNotFoundError, DifferentTransferAndConnectionsGroupsError, @@ -16,11 +20,12 @@ GroupNotFoundError, TransferNotFoundError, ) -from exceptions.base import ActionNotAllowedError -from fastapi import APIRouter, Depends, Query, status -from kombu.exceptions import KombuError -from schemas.v1.schemas import StatusCopyTransferResponseSchema, StatusResponseSchema -from schemas.v1.transfers import ( +from syncmaster.exceptions.base import ActionNotAllowedError +from syncmaster.schemas.v1.schemas import ( + StatusCopyTransferResponseSchema, + StatusResponseSchema, +) +from syncmaster.schemas.v1.transfers import ( CopyTransferSchema, CreateRunSchema, CreateTransferSchema, @@ -30,6 +35,7 @@ TransferPageSchema, UpdateTransferSchema, ) +from syncmaster.worker.config import celery router = APIRouter(tags=["Transfers"]) diff --git a/syncmaster/backend/api/v1/transfers/utils.py b/syncmaster/backend/api/v1/transfers/utils.py index 980754c6..bc970c28 100644 --- a/syncmaster/backend/api/v1/transfers/utils.py +++ b/syncmaster/backend/api/v1/transfers/utils.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from schemas.v1.transfers import CreateTransferSchema, UpdateTransferSchema +from syncmaster.schemas.v1.transfers import CreateTransferSchema, UpdateTransferSchema def process_file_transfer_directory_path( diff --git a/syncmaster/backend/api/v1/users.py b/syncmaster/backend/api/v1/users.py index 4af7a691..e2b6b8c3 100644 --- a/syncmaster/backend/api/v1/users.py +++ b/syncmaster/backend/api/v1/users.py @@ -2,13 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 import logging -from backend.api.deps import UnitOfWorkMarker -from backend.services import UnitOfWork, get_user -from db import User -from exceptions import ActionNotAllowedError from fastapi import APIRouter, Depends, Query -from schemas.v1.schemas import StatusResponseSchema -from schemas.v1.users import ReadUserSchema, UpdateUserSchema, UserPageSchema + +from syncmaster.backend.api.deps import UnitOfWorkMarker +from syncmaster.backend.services import UnitOfWork, get_user +from syncmaster.db import User +from syncmaster.exceptions import ActionNotAllowedError +from syncmaster.schemas.v1.schemas import StatusResponseSchema +from syncmaster.schemas.v1.users import ReadUserSchema, UpdateUserSchema, UserPageSchema logger = logging.getLogger(__name__) diff --git a/syncmaster/backend/config.py b/syncmaster/backend/config.py index f37169c8..1a5212e2 100644 --- a/syncmaster/backend/config.py +++ b/syncmaster/backend/config.py @@ -71,7 +71,7 @@ def build_rabbit_connection_uri( TOKEN_EXPIRED_TIME: int = 60 * 60 * 10 # 10 hours CREATE_SPARK_SESSION_FUNCTION: PyObject = ( - "backend.worker.utils.get_worker_spark_session" # type: ignore[assignment] + "syncmaster.worker.utils.get_worker_spark_session" # type: ignore[assignment] ) diff --git a/syncmaster/backend/handler.py b/syncmaster/backend/handler.py index a9faaff8..73d7164d 100644 --- a/syncmaster/backend/handler.py +++ b/syncmaster/backend/handler.py @@ -2,7 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 import logging -from exceptions import ( +from fastapi import HTTPException, Request, status +from fastapi.responses import JSONResponse + +from syncmaster.exceptions import ( ActionNotAllowedError, AlreadyIsGroupMemberError, AlreadyIsNotGroupMemberError, @@ -29,9 +32,7 @@ UsernameAlreadyExistsError, UserNotFoundError, ) -from fastapi import HTTPException, Request, status -from fastapi.responses import JSONResponse -from schemas.v1.schemas import StatusResponseSchema +from syncmaster.schemas.v1.schemas import StatusResponseSchema logger = logging.getLogger(__name__) diff --git a/syncmaster/backend/main.py b/syncmaster/backend/main.py index a361530c..5fc40bd7 100644 --- a/syncmaster/backend/main.py +++ b/syncmaster/backend/main.py @@ -1,21 +1,25 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import uvicorn -from backend.api.deps import ( +from fastapi import FastAPI, HTTPException +from starlette.middleware.cors import CORSMiddleware + +from syncmaster.backend.api.deps import ( AuthMarker, DatabaseEngineMarker, DatabaseSessionMarker, SettingsMarker, UnitOfWorkMarker, ) -from backend.api.router import api_router -from backend.config import Settings -from backend.handler import http_exception_handler, syncmsater_exception_handler -from backend.services import get_auth_scheme -from db.factory import create_engine, create_session_factory, get_uow -from exceptions import SyncmasterError -from fastapi import FastAPI, HTTPException -from starlette.middleware.cors import CORSMiddleware +from syncmaster.backend.api.router import api_router +from syncmaster.backend.config import Settings +from syncmaster.backend.handler import ( + http_exception_handler, + syncmsater_exception_handler, +) +from syncmaster.backend.services import get_auth_scheme +from syncmaster.db.factory import create_engine, create_session_factory, get_uow +from syncmaster.exceptions import SyncmasterError def get_application(settings: Settings) -> FastAPI: diff --git a/syncmaster/backend/pre_start.py b/syncmaster/backend/pre_start.py index 28a3a0f6..65849daa 100644 --- a/syncmaster/backend/pre_start.py +++ b/syncmaster/backend/pre_start.py @@ -3,8 +3,7 @@ import logging from time import sleep -from backend.config import Settings, TestSettings - +from syncmaster.backend.config import Settings, TestSettings from tests.test_integration.test_run_transfer.conftest import get_spark_session TIMEOUT = 5 diff --git a/syncmaster/backend/services/__init__.py b/syncmaster/backend/services/__init__.py index ef0369a6..cfea9bad 100644 --- a/syncmaster/backend/services/__init__.py +++ b/syncmaster/backend/services/__init__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.services.auth import get_auth_scheme, get_user -from backend.services.unit_of_work import UnitOfWork +from syncmaster.backend.services.auth import get_auth_scheme, get_user +from syncmaster.backend.services.unit_of_work import UnitOfWork diff --git a/syncmaster/backend/services/auth.py b/syncmaster/backend/services/auth.py index cc02578c..1cd30ea9 100644 --- a/syncmaster/backend/services/auth.py +++ b/syncmaster/backend/services/auth.py @@ -2,15 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 from collections.abc import Awaitable, Callable -from backend.api.deps import AuthMarker, SettingsMarker, UnitOfWorkMarker -from backend.api.v1.auth.utils import decode_jwt -from backend.config import Settings -from backend.services.unit_of_work import UnitOfWork -from db import User from fastapi import Depends, Request, status from fastapi.exceptions import HTTPException from fastapi.security import OAuth2PasswordBearer +from syncmaster.backend.api.deps import AuthMarker, SettingsMarker, UnitOfWorkMarker +from syncmaster.backend.api.v1.auth.utils import decode_jwt +from syncmaster.backend.config import Settings +from syncmaster.backend.services.unit_of_work import UnitOfWork +from syncmaster.db import User + def get_user( is_active: bool = False, diff --git a/syncmaster/backend/services/unit_of_work.py b/syncmaster/backend/services/unit_of_work.py index ff8d7d86..0a7f0d45 100644 --- a/syncmaster/backend/services/unit_of_work.py +++ b/syncmaster/backend/services/unit_of_work.py @@ -1,8 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.config import Settings -from db import AuthData -from db.repositories import ( +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData +from syncmaster.db.repositories import ( ConnectionRepository, GroupRepository, QueueRepository, @@ -10,8 +12,7 @@ TransferRepository, UserRepository, ) -from db.repositories.credentials_repository import CredentialsRepository -from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db.repositories.credentials_repository import CredentialsRepository class UnitOfWork: diff --git a/syncmaster/db/__init__.py b/syncmaster/db/__init__.py index a1627957..021e8390 100644 --- a/syncmaster/db/__init__.py +++ b/syncmaster/db/__init__.py @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from db.base import Base -from db.mixins import DeletableMixin, ResourceMixin, TimestampMixin -from db.models import ( +from syncmaster.db.base import Base +from syncmaster.db.mixins import DeletableMixin, ResourceMixin, TimestampMixin +from syncmaster.db.models import ( AuthData, Connection, Group, @@ -15,7 +15,7 @@ User, UserGroup, ) -from db.utils import Pagination, Permission +from syncmaster.db.utils import Pagination, Permission __all__ = [ "AuthData", diff --git a/syncmaster/db/alembic.ini b/syncmaster/db/alembic.ini index 0bdc4544..0a7110e7 100644 --- a/syncmaster/db/alembic.ini +++ b/syncmaster/db/alembic.ini @@ -1,5 +1,5 @@ [alembic] -script_location = %(here)s/db/migrations +script_location = %(here)s/syncmaster/db/migrations file_template = %%(year)d-%%(month).2d-%%(day).2d_%%(rev)s_%%(slug)s prepend_sys_path = . version_path_separator = os diff --git a/syncmaster/db/factory.py b/syncmaster/db/factory.py index 50ab6ff3..81fbca95 100644 --- a/syncmaster/db/factory.py +++ b/syncmaster/db/factory.py @@ -3,8 +3,6 @@ from collections.abc import AsyncGenerator, Callable from typing import Any -from backend.config import Settings -from backend.services import UnitOfWork from sqlalchemy.ext.asyncio import ( AsyncEngine, AsyncSession, @@ -12,6 +10,9 @@ create_async_engine, ) +from syncmaster.backend.config import Settings +from syncmaster.backend.services import UnitOfWork + def create_engine(connection_uri: str, **engine_kwargs: Any) -> AsyncEngine: return create_async_engine(url=connection_uri, **engine_kwargs) diff --git a/syncmaster/db/migrations/env.py b/syncmaster/db/migrations/env.py index d4767e85..694cf88b 100644 --- a/syncmaster/db/migrations/env.py +++ b/syncmaster/db/migrations/env.py @@ -4,13 +4,14 @@ from logging.config import fileConfig from alembic import context -from backend.config import Settings from celery.backends.database.session import ResultModelBase -from db import Base from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config +from syncmaster.backend.config import Settings +from syncmaster.db import Base + config = context.config diff --git a/syncmaster/db/models.py b/syncmaster/db/models.py index 3109dd5c..350a2938 100644 --- a/syncmaster/db/models.py +++ b/syncmaster/db/models.py @@ -6,8 +6,6 @@ from datetime import datetime from typing import Any -from db.base import Base -from db.mixins import DeletableMixin, ResourceMixin, TimestampMixin from sqlalchemy import ( JSON, BigInteger, @@ -21,6 +19,9 @@ from sqlalchemy.orm import Mapped, declared_attr, mapped_column, relationship from sqlalchemy_utils import ChoiceType +from syncmaster.db.base import Base +from syncmaster.db.mixins import DeletableMixin, ResourceMixin, TimestampMixin + class GroupMemberRole(enum.StrEnum): Maintainer = "Maintainer" diff --git a/syncmaster/db/repositories/__init__.py b/syncmaster/db/repositories/__init__.py index 270e114e..90c6d399 100644 --- a/syncmaster/db/repositories/__init__.py +++ b/syncmaster/db/repositories/__init__.py @@ -1,14 +1,14 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from db.repositories.connection import ConnectionRepository -from db.repositories.group import GroupRepository -from db.repositories.queue import QueueRepository -from db.repositories.repository_with_owner import RepositoryWithOwner -from db.repositories.run import RunRepository -from db.repositories.transfer import TransferRepository -from db.repositories.user import UserRepository -from db.repositories.utils import decrypt_auth_data, encrypt_auth_data +from syncmaster.db.repositories.connection import ConnectionRepository +from syncmaster.db.repositories.group import GroupRepository +from syncmaster.db.repositories.queue import QueueRepository +from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner +from syncmaster.db.repositories.run import RunRepository +from syncmaster.db.repositories.transfer import TransferRepository +from syncmaster.db.repositories.user import UserRepository +from syncmaster.db.repositories.utils import decrypt_auth_data, encrypt_auth_data __all__ = [ "ConnectionRepository", diff --git a/syncmaster/db/repositories/base.py b/syncmaster/db/repositories/base.py index 27f8aa69..925fa490 100644 --- a/syncmaster/db/repositories/base.py +++ b/syncmaster/db/repositories/base.py @@ -3,12 +3,13 @@ from abc import ABC from typing import Any, Generic, TypeVar -from db import Base, Pagination -from exceptions import EntityNotFoundError from sqlalchemy import ScalarResult, Select, and_, delete, func, insert, select, update from sqlalchemy.exc import NoResultFound from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Base, Pagination +from syncmaster.exceptions import EntityNotFoundError + Model = TypeVar("Model", bound=Base) diff --git a/syncmaster/db/repositories/connection.py b/syncmaster/db/repositories/connection.py index 3cc8220e..325172d7 100644 --- a/syncmaster/db/repositories/connection.py +++ b/syncmaster/db/repositories/connection.py @@ -2,9 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Any, NoReturn -from db import Connection, Pagination -from db.repositories.repository_with_owner import RepositoryWithOwner -from exceptions import ( +from sqlalchemy import ScalarResult, insert, select +from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db import Connection, Pagination +from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner +from syncmaster.exceptions import ( ConnectionNotFoundError, ConnectionOwnerError, DuplicatedConnectionNameError, @@ -13,9 +17,6 @@ SyncmasterError, UserNotFoundError, ) -from sqlalchemy import ScalarResult, insert, select -from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound -from sqlalchemy.ext.asyncio import AsyncSession class ConnectionRepository(RepositoryWithOwner[Connection]): diff --git a/syncmaster/db/repositories/credentials_repository.py b/syncmaster/db/repositories/credentials_repository.py index 6ea8635d..c501530a 100644 --- a/syncmaster/db/repositories/credentials_repository.py +++ b/syncmaster/db/repositories/credentials_repository.py @@ -4,15 +4,16 @@ from typing import NoReturn -from backend.config import Settings -from db import AuthData -from db.repositories import decrypt_auth_data, encrypt_auth_data -from db.repositories.base import Repository -from exceptions import AuthDataNotFoundError, SyncmasterError from sqlalchemy import ScalarResult, delete, insert, select from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData +from syncmaster.db.repositories import decrypt_auth_data, encrypt_auth_data +from syncmaster.db.repositories.base import Repository +from syncmaster.exceptions import AuthDataNotFoundError, SyncmasterError + class CredentialsRepository(Repository[AuthData]): def __init__( diff --git a/syncmaster/db/repositories/group.py b/syncmaster/db/repositories/group.py index 3c967f54..f13b0785 100644 --- a/syncmaster/db/repositories/group.py +++ b/syncmaster/db/repositories/group.py @@ -3,9 +3,13 @@ import re from typing import NoReturn -from db import Group, Pagination, Permission, User, UserGroup -from db.repositories.base import Repository -from exceptions import ( +from sqlalchemy import ScalarResult, insert, or_, select, update +from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db import Group, Pagination, Permission, User, UserGroup +from syncmaster.db.repositories.base import Repository +from syncmaster.exceptions import ( AlreadyIsGroupMemberError, AlreadyIsNotGroupMemberError, EntityNotFoundError, @@ -15,9 +19,6 @@ SyncmasterError, UserNotFoundError, ) -from sqlalchemy import ScalarResult, insert, or_, select, update -from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound -from sqlalchemy.ext.asyncio import AsyncSession class GroupRepository(Repository[Group]): diff --git a/syncmaster/db/repositories/queue.py b/syncmaster/db/repositories/queue.py index 8a95996e..7b226581 100644 --- a/syncmaster/db/repositories/queue.py +++ b/syncmaster/db/repositories/queue.py @@ -2,16 +2,21 @@ # SPDX-License-Identifier: Apache-2.0 from typing import NoReturn -from db import Group, GroupMemberRole, Permission, Queue, User, UserGroup -from db.repositories.repository_with_owner import RepositoryWithOwner -from exceptions import EntityNotFoundError, GroupNotFoundError, SyncmasterError -from exceptions.queue import QueueNotFoundError -from schemas.v1.queue import UpdateQueueSchema from sqlalchemy import ScalarResult, insert, select from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload +from syncmaster.db import Group, GroupMemberRole, Permission, Queue, User, UserGroup +from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner +from syncmaster.exceptions import ( + EntityNotFoundError, + GroupNotFoundError, + SyncmasterError, +) +from syncmaster.exceptions.queue import QueueNotFoundError +from syncmaster.schemas.v1.queue import UpdateQueueSchema + class QueueRepository(RepositoryWithOwner[Queue]): def __init__(self, session: AsyncSession): diff --git a/syncmaster/db/repositories/repository_with_owner.py b/syncmaster/db/repositories/repository_with_owner.py index e1c6dc06..323a76fb 100644 --- a/syncmaster/db/repositories/repository_with_owner.py +++ b/syncmaster/db/repositories/repository_with_owner.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Generic, TypeVar -from db import Base, Group, GroupMemberRole, Permission, User, UserGroup -from db.repositories.base import Repository -from exceptions import GroupNotFoundError from sqlalchemy import select +from syncmaster.db import Base, Group, GroupMemberRole, Permission, User, UserGroup +from syncmaster.db.repositories.base import Repository +from syncmaster.exceptions import GroupNotFoundError + Model = TypeVar("Model", bound=Base) diff --git a/syncmaster/db/repositories/run.py b/syncmaster/db/repositories/run.py index 87fa3420..2e70b5a4 100644 --- a/syncmaster/db/repositories/run.py +++ b/syncmaster/db/repositories/run.py @@ -2,19 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Any, NoReturn -from db import Pagination, Run, Status, Transfer -from db.repositories.base import Repository -from exceptions import ( +from sqlalchemy import desc, select +from sqlalchemy.exc import DBAPIError, IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from syncmaster.db import Pagination, Run, Status, Transfer +from syncmaster.db.repositories.base import Repository +from syncmaster.exceptions import ( CannotStopRunError, RunNotFoundError, SyncmasterError, TransferNotFoundError, ) -from schemas.v1.transfers import ReadFullTransferSchema -from sqlalchemy import desc, select -from sqlalchemy.exc import DBAPIError, IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload +from syncmaster.schemas.v1.transfers import ReadFullTransferSchema class RunRepository(Repository[Run]): diff --git a/syncmaster/db/repositories/transfer.py b/syncmaster/db/repositories/transfer.py index 819f2c88..664c808e 100644 --- a/syncmaster/db/repositories/transfer.py +++ b/syncmaster/db/repositories/transfer.py @@ -3,9 +3,14 @@ from collections.abc import Sequence from typing import Any, NoReturn -from db import Pagination, Transfer -from db.repositories.repository_with_owner import RepositoryWithOwner -from exceptions import ( +from sqlalchemy import ScalarResult, insert, or_, select +from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from syncmaster.db import Pagination, Transfer +from syncmaster.db.repositories.repository_with_owner import RepositoryWithOwner +from syncmaster.exceptions import ( ConnectionNotFoundError, DuplicatedTransferNameError, EntityNotFoundError, @@ -16,10 +21,6 @@ TransferOwnerError, UserNotFoundError, ) -from sqlalchemy import ScalarResult, insert, or_, select -from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload class TransferRepository(RepositoryWithOwner[Transfer]): diff --git a/syncmaster/db/repositories/user.py b/syncmaster/db/repositories/user.py index c0e2c1bc..96049b6a 100644 --- a/syncmaster/db/repositories/user.py +++ b/syncmaster/db/repositories/user.py @@ -2,17 +2,18 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Any, NoReturn -from db import Pagination, User -from db.repositories.base import Repository -from exceptions import ( +from sqlalchemy import ScalarResult, insert, select +from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db import Pagination, User +from syncmaster.db.repositories.base import Repository +from syncmaster.exceptions import ( EntityNotFoundError, SyncmasterError, UsernameAlreadyExistsError, UserNotFoundError, ) -from sqlalchemy import ScalarResult, insert, select -from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound -from sqlalchemy.ext.asyncio import AsyncSession class UserRepository(Repository[User]): diff --git a/syncmaster/db/repositories/utils.py b/syncmaster/db/repositories/utils.py index f16b52e8..b8c4c069 100644 --- a/syncmaster/db/repositories/utils.py +++ b/syncmaster/db/repositories/utils.py @@ -2,9 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 import json -from backend.config import Settings from cryptography.fernet import Fernet +from syncmaster.backend.config import Settings + def decrypt_auth_data( value: str, diff --git a/syncmaster/dto/transfers.py b/syncmaster/dto/transfers.py index 541b21a8..93ea0540 100644 --- a/syncmaster/dto/transfers.py +++ b/syncmaster/dto/transfers.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass -from schemas.v1.transfers import CSV, JSON, JSONLine +from syncmaster.schemas.v1.transfers import CSV, JSON, JSONLine @dataclass diff --git a/syncmaster/exceptions/__init__.py b/syncmaster/exceptions/__init__.py index e8da9dae..3ef022c4 100644 --- a/syncmaster/exceptions/__init__.py +++ b/syncmaster/exceptions/__init__.py @@ -1,15 +1,19 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from exceptions.base import ActionNotAllowedError, EntityNotFoundError, SyncmasterError -from exceptions.connection import ( +from syncmaster.exceptions.base import ( + ActionNotAllowedError, + EntityNotFoundError, + SyncmasterError, +) +from syncmaster.exceptions.connection import ( ConnectionDeleteError, ConnectionNotFoundError, ConnectionOwnerError, ConnectionTypeNotRecognizedError, DuplicatedConnectionNameError, ) -from exceptions.credentials import AuthDataNotFoundError -from exceptions.group import ( +from syncmaster.exceptions.credentials import AuthDataNotFoundError +from syncmaster.exceptions.group import ( AlreadyIsGroupMemberError, AlreadyIsNotGroupMemberError, GroupAdminNotFoundError, @@ -17,24 +21,24 @@ GroupNameAlreadyExistsError, GroupNotFoundError, ) -from exceptions.queue import ( +from syncmaster.exceptions.queue import ( DifferentTransferAndQueueGroupError, QueueDeleteError, QueueNotFoundError, ) -from exceptions.run import ( +from syncmaster.exceptions.run import ( CannotConnectToTaskQueueError, CannotStopRunError, RunNotFoundError, ) -from exceptions.transfer import ( +from syncmaster.exceptions.transfer import ( DifferentTransferAndConnectionsGroupsError, DifferentTypeConnectionsAndParamsError, DuplicatedTransferNameError, TransferNotFoundError, TransferOwnerError, ) -from exceptions.user import UsernameAlreadyExistsError, UserNotFoundError +from syncmaster.exceptions.user import UsernameAlreadyExistsError, UserNotFoundError __all__ = [ "ActionNotAllowedError", diff --git a/syncmaster/exceptions/connection.py b/syncmaster/exceptions/connection.py index 9e2dc6e8..51a3b7b7 100644 --- a/syncmaster/exceptions/connection.py +++ b/syncmaster/exceptions/connection.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from exceptions.base import EntityNotFoundError, SyncmasterError +from syncmaster.exceptions.base import EntityNotFoundError, SyncmasterError class ConnectionNotFoundError(EntityNotFoundError): diff --git a/syncmaster/exceptions/credentials.py b/syncmaster/exceptions/credentials.py index b26aa62b..8843c161 100644 --- a/syncmaster/exceptions/credentials.py +++ b/syncmaster/exceptions/credentials.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from exceptions import SyncmasterError +from syncmaster.exceptions import SyncmasterError class AuthDataNotFoundError(SyncmasterError): diff --git a/syncmaster/exceptions/group.py b/syncmaster/exceptions/group.py index 1e084a69..4f7f1163 100644 --- a/syncmaster/exceptions/group.py +++ b/syncmaster/exceptions/group.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from exceptions.base import SyncmasterError +from syncmaster.exceptions.base import SyncmasterError class GroupNameAlreadyExistsError(SyncmasterError): diff --git a/syncmaster/exceptions/queue.py b/syncmaster/exceptions/queue.py index 2cdc8c53..b1a4a124 100644 --- a/syncmaster/exceptions/queue.py +++ b/syncmaster/exceptions/queue.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from exceptions.base import EntityNotFoundError, SyncmasterError +from syncmaster.exceptions.base import EntityNotFoundError, SyncmasterError class QueueNotFoundError(EntityNotFoundError): diff --git a/syncmaster/exceptions/run.py b/syncmaster/exceptions/run.py index 4bf145b0..2151dc5f 100644 --- a/syncmaster/exceptions/run.py +++ b/syncmaster/exceptions/run.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from db import Status -from exceptions.base import EntityNotFoundError, SyncmasterError +from syncmaster.db import Status +from syncmaster.exceptions.base import EntityNotFoundError, SyncmasterError class RunNotFoundError(EntityNotFoundError): diff --git a/syncmaster/exceptions/transfer.py b/syncmaster/exceptions/transfer.py index 684b10bf..61e0752f 100644 --- a/syncmaster/exceptions/transfer.py +++ b/syncmaster/exceptions/transfer.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from exceptions.base import SyncmasterError +from syncmaster.exceptions.base import SyncmasterError class TransferOwnerError(SyncmasterError): diff --git a/syncmaster/exceptions/user.py b/syncmaster/exceptions/user.py index f2f14b83..ae50379c 100644 --- a/syncmaster/exceptions/user.py +++ b/syncmaster/exceptions/user.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from exceptions.base import SyncmasterError +from syncmaster.exceptions.base import SyncmasterError class UsernameAlreadyExistsError(SyncmasterError): diff --git a/syncmaster/schemas/v1/__init__.py b/syncmaster/schemas/v1/__init__.py index e97afd64..8852e376 100644 --- a/syncmaster/schemas/v1/__init__.py +++ b/syncmaster/schemas/v1/__init__.py @@ -1,34 +1,34 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from schemas.v1.auth import AuthTokenSchema, TokenPayloadSchema -from schemas.v1.connections import ( +from syncmaster.schemas.v1.auth import AuthTokenSchema, TokenPayloadSchema +from syncmaster.schemas.v1.connections import ( ConnectionCopySchema, ConnectionPageSchema, CreateConnectionSchema, ReadConnectionSchema, UpdateConnectionSchema, ) -from schemas.v1.groups import ( +from syncmaster.schemas.v1.groups import ( AddUserSchema, CreateGroupSchema, GroupPageSchema, ReadGroupSchema, UpdateGroupSchema, ) -from schemas.v1.queue import ( +from syncmaster.schemas.v1.queue import ( CreateQueueSchema, QueuePageSchema, ReadQueueSchema, UpdateQueueSchema, ) -from schemas.v1.schemas import ( +from syncmaster.schemas.v1.schemas import ( MetaPageSchema, PageSchema, StatusCopyTransferResponseSchema, StatusResponseSchema, ) -from schemas.v1.transfers import ( +from syncmaster.schemas.v1.transfers import ( CSV, JSON, CopyTransferSchema, @@ -49,7 +49,7 @@ TransferPageSchema, UpdateTransferSchema, ) -from schemas.v1.users import ( +from syncmaster.schemas.v1.users import ( FullUserSchema, ReadGroupMember, ReadUserSchema, diff --git a/syncmaster/schemas/v1/connections/__init__.py b/syncmaster/schemas/v1/connections/__init__.py index c8c98c20..ee080a91 100644 --- a/syncmaster/schemas/v1/connections/__init__.py +++ b/syncmaster/schemas/v1/connections/__init__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from schemas.v1.connections.connection import ( +from syncmaster.schemas.v1.connections.connection import ( ConnectionCopySchema, ConnectionPageSchema, CreateConnectionSchema, diff --git a/syncmaster/schemas/v1/connections/connection.py b/syncmaster/schemas/v1/connections/connection.py index f806f318..53a31842 100644 --- a/syncmaster/schemas/v1/connections/connection.py +++ b/syncmaster/schemas/v1/connections/connection.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from pydantic import BaseModel, Field, root_validator -from schemas.v1.connections.hdfs import ( + +from syncmaster.schemas.v1.connections.hdfs import ( HDFSCreateAuthSchema, HDFSCreateConnectionSchema, HDFSReadAuthSchema, @@ -10,7 +11,7 @@ HDFSUpdateAuthSchema, HDFSUpdateConnectionSchema, ) -from schemas.v1.connections.hive import ( +from syncmaster.schemas.v1.connections.hive import ( CreateHiveAuthSchema, CreateHiveConnectionSchema, ReadHiveAuthSchema, @@ -18,7 +19,7 @@ UpdateHiveAuthSchema, UpdateHiveConnectionSchema, ) -from schemas.v1.connections.oracle import ( +from syncmaster.schemas.v1.connections.oracle import ( CreateOracleAuthSchema, CreateOracleConnectionSchema, ReadOracleAuthSchema, @@ -26,7 +27,7 @@ UpdateOracleAuthSchema, UpdateOracleConnectionSchema, ) -from schemas.v1.connections.postgres import ( +from syncmaster.schemas.v1.connections.postgres import ( CreatePostgresAuthSchema, CreatePostgresConnectionSchema, ReadPostgresAuthSchema, @@ -34,7 +35,7 @@ UpdatePostgresAuthSchema, UpdatePostgresConnectionSchema, ) -from schemas.v1.connections.s3 import ( +from syncmaster.schemas.v1.connections.s3 import ( S3CreateAuthSchema, S3CreateConnectionSchema, S3ReadAuthSchema, @@ -42,7 +43,7 @@ S3UpdateAuthSchema, S3UpdateConnectionSchema, ) -from schemas.v1.schemas import NameConstr, PageSchema +from syncmaster.schemas.v1.schemas import NameConstr, PageSchema ReadConnectionDataSchema = ( ReadHiveConnectionSchema diff --git a/syncmaster/schemas/v1/connections/hdfs.py b/syncmaster/schemas/v1/connections/hdfs.py index 244d906d..668fcb40 100644 --- a/syncmaster/schemas/v1/connections/hdfs.py +++ b/syncmaster/schemas/v1/connections/hdfs.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from pydantic import BaseModel, SecretStr -from schemas.v1.schemas import HDFS_TYPE + +from syncmaster.schemas.v1.schemas import HDFS_TYPE class HDFSConnectionBaseSchema(BaseModel): diff --git a/syncmaster/schemas/v1/connections/hive.py b/syncmaster/schemas/v1/connections/hive.py index d011419b..713c0f58 100644 --- a/syncmaster/schemas/v1/connections/hive.py +++ b/syncmaster/schemas/v1/connections/hive.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from pydantic import BaseModel, SecretStr -from schemas.v1.schemas import HIVE_TYPE + +from syncmaster.schemas.v1.schemas import HIVE_TYPE class HiveBaseSchema(BaseModel): diff --git a/syncmaster/schemas/v1/connections/oracle.py b/syncmaster/schemas/v1/connections/oracle.py index bdd44908..3025977b 100644 --- a/syncmaster/schemas/v1/connections/oracle.py +++ b/syncmaster/schemas/v1/connections/oracle.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from pydantic import BaseModel, Field, SecretStr, root_validator -from schemas.v1.schemas import ORACLE_TYPE, NameConstr + +from syncmaster.schemas.v1.schemas import ORACLE_TYPE, NameConstr class OracleBaseSchema(BaseModel): diff --git a/syncmaster/schemas/v1/connections/postgres.py b/syncmaster/schemas/v1/connections/postgres.py index dc37df65..d0a7eaf4 100644 --- a/syncmaster/schemas/v1/connections/postgres.py +++ b/syncmaster/schemas/v1/connections/postgres.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from pydantic import BaseModel, Field, SecretStr -from schemas.v1.schemas import POSTGRES_TYPE + +from syncmaster.schemas.v1.schemas import POSTGRES_TYPE class PostgresBaseSchema(BaseModel): diff --git a/syncmaster/schemas/v1/connections/s3.py b/syncmaster/schemas/v1/connections/s3.py index 266c8ecb..7bffdcde 100644 --- a/syncmaster/schemas/v1/connections/s3.py +++ b/syncmaster/schemas/v1/connections/s3.py @@ -3,7 +3,8 @@ from typing import Literal from pydantic import BaseModel, SecretStr, root_validator -from schemas.v1.schemas import S3_TYPE + +from syncmaster.schemas.v1.schemas import S3_TYPE class S3BaseSchema(BaseModel): diff --git a/syncmaster/schemas/v1/groups.py b/syncmaster/schemas/v1/groups.py index 560b167c..261fdbf7 100644 --- a/syncmaster/schemas/v1/groups.py +++ b/syncmaster/schemas/v1/groups.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from db import GroupMemberRole from pydantic import BaseModel -from schemas.v1.schemas import NameConstr, PageSchema + +from syncmaster.db import GroupMemberRole +from syncmaster.schemas.v1.schemas import NameConstr, PageSchema class UpdateGroupSchema(BaseModel): diff --git a/syncmaster/schemas/v1/queue.py b/syncmaster/schemas/v1/queue.py index 659f01c9..fce83fe8 100644 --- a/syncmaster/schemas/v1/queue.py +++ b/syncmaster/schemas/v1/queue.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 from pydantic import BaseModel, Field, constr -from schemas.v1.schemas import PageSchema + +from syncmaster.schemas.v1.schemas import PageSchema class CreateQueueSchema(BaseModel): diff --git a/syncmaster/schemas/v1/schemas.py b/syncmaster/schemas/v1/schemas.py index 59e846ec..f4abf0ad 100644 --- a/syncmaster/schemas/v1/schemas.py +++ b/syncmaster/schemas/v1/schemas.py @@ -3,9 +3,10 @@ import abc from typing import Any, Literal -from db import Pagination from pydantic import BaseModel, constr +from syncmaster.db import Pagination + # transfer types FULL_TYPE = Literal["full"] INCREMENTAL_TYPE = Literal["incremental"] diff --git a/syncmaster/schemas/v1/transfers/__init__.py b/syncmaster/schemas/v1/transfers/__init__.py index 15a6a4da..de923db5 100644 --- a/syncmaster/schemas/v1/transfers/__init__.py +++ b/syncmaster/schemas/v1/transfers/__init__.py @@ -1,21 +1,21 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from schemas.v1.transfers.db import ( +from syncmaster.schemas.v1.transfers.db import ( HiveReadTransferSourceAndTarget, OracleReadTransferSourceAndTarget, PostgresReadTransferSourceAndTarget, ReadDBTransfer, ) -from schemas.v1.transfers.file_format import CSV, JSON, JSONLine -from schemas.v1.transfers.run import ( +from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine +from syncmaster.schemas.v1.transfers.run import ( CreateRunSchema, ReadRunSchema, RunPageSchema, ShortRunSchema, ) -from schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy -from schemas.v1.transfers.transfer import ( +from syncmaster.schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy +from syncmaster.schemas.v1.transfers.transfer import ( CopyTransferSchema, CreateTransferSchema, ReadFullTransferSchema, diff --git a/syncmaster/schemas/v1/transfers/db.py b/syncmaster/schemas/v1/transfers/db.py index 23d42bbc..b0bd3a21 100644 --- a/syncmaster/schemas/v1/transfers/db.py +++ b/syncmaster/schemas/v1/transfers/db.py @@ -3,7 +3,8 @@ from __future__ import annotations from pydantic import BaseModel -from schemas.v1.schemas import HIVE_TYPE, ORACLE_TYPE, POSTGRES_TYPE + +from syncmaster.schemas.v1.schemas import HIVE_TYPE, ORACLE_TYPE, POSTGRES_TYPE class ReadDBTransfer(BaseModel): diff --git a/syncmaster/schemas/v1/transfers/file/base.py b/syncmaster/schemas/v1/transfers/file/base.py index aaa249ce..d01041c3 100644 --- a/syncmaster/schemas/v1/transfers/file/base.py +++ b/syncmaster/schemas/v1/transfers/file/base.py @@ -5,7 +5,8 @@ from pathlib import PurePosixPath from pydantic import BaseModel, Field, validator -from schemas.v1.transfers.file_format import CSV, JSON, JSONLine + +from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, JSONLine def validate_directory_path(path: str) -> PurePosixPath: diff --git a/syncmaster/schemas/v1/transfers/file/hdfs.py b/syncmaster/schemas/v1/transfers/file/hdfs.py index 1bd8d785..f015da7e 100644 --- a/syncmaster/schemas/v1/transfers/file/hdfs.py +++ b/syncmaster/schemas/v1/transfers/file/hdfs.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from schemas.v1.schemas import HDFS_TYPE -from schemas.v1.transfers.file.base import ( +from syncmaster.schemas.v1.schemas import HDFS_TYPE +from syncmaster.schemas.v1.transfers.file.base import ( CreateFileTransferSource, CreateFileTransferTarget, ReadFileTransferSource, diff --git a/syncmaster/schemas/v1/transfers/file/s3.py b/syncmaster/schemas/v1/transfers/file/s3.py index e132846a..a39660e3 100644 --- a/syncmaster/schemas/v1/transfers/file/s3.py +++ b/syncmaster/schemas/v1/transfers/file/s3.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from schemas.v1.schemas import S3_TYPE -from schemas.v1.transfers.file.base import ( +from syncmaster.schemas.v1.schemas import S3_TYPE +from syncmaster.schemas.v1.transfers.file.base import ( CreateFileTransferSource, CreateFileTransferTarget, ReadFileTransferSource, diff --git a/syncmaster/schemas/v1/transfers/file_format.py b/syncmaster/schemas/v1/transfers/file_format.py index 4fd3cab2..36a2deec 100644 --- a/syncmaster/schemas/v1/transfers/file_format.py +++ b/syncmaster/schemas/v1/transfers/file_format.py @@ -3,7 +3,8 @@ from __future__ import annotations from pydantic import BaseModel -from schemas.v1.schemas import CSV_FORMAT, JSON_FORMAT, JSONLINE_FORMAT + +from syncmaster.schemas.v1.schemas import CSV_FORMAT, JSON_FORMAT, JSONLINE_FORMAT class CSV(BaseModel): diff --git a/syncmaster/schemas/v1/transfers/run.py b/syncmaster/schemas/v1/transfers/run.py index 2de0ef71..da159647 100644 --- a/syncmaster/schemas/v1/transfers/run.py +++ b/syncmaster/schemas/v1/transfers/run.py @@ -4,9 +4,10 @@ from datetime import datetime -from db import Status from pydantic import BaseModel -from schemas.v1.schemas import PageSchema + +from syncmaster.db import Status +from syncmaster.schemas.v1.schemas import PageSchema class ShortRunSchema(BaseModel): diff --git a/syncmaster/schemas/v1/transfers/strategy.py b/syncmaster/schemas/v1/transfers/strategy.py index bda8056c..3e1f0ccb 100644 --- a/syncmaster/schemas/v1/transfers/strategy.py +++ b/syncmaster/schemas/v1/transfers/strategy.py @@ -3,7 +3,8 @@ from __future__ import annotations from pydantic import BaseModel -from schemas.v1.schemas import FULL_TYPE, INCREMENTAL_TYPE + +from syncmaster.schemas.v1.schemas import FULL_TYPE, INCREMENTAL_TYPE class FullStrategy(BaseModel): diff --git a/syncmaster/schemas/v1/transfers/transfer.py b/syncmaster/schemas/v1/transfers/transfer.py index 6f52f45f..b64a71c4 100644 --- a/syncmaster/schemas/v1/transfers/transfer.py +++ b/syncmaster/schemas/v1/transfers/transfer.py @@ -3,26 +3,27 @@ from __future__ import annotations from pydantic import BaseModel, Field, root_validator -from schemas.v1.connections.connection import ReadConnectionSchema -from schemas.v1.schemas import NameConstr, PageSchema -from schemas.v1.transfers.db import ( + +from syncmaster.schemas.v1.connections.connection import ReadConnectionSchema +from syncmaster.schemas.v1.schemas import NameConstr, PageSchema +from syncmaster.schemas.v1.transfers.db import ( HiveReadTransferSourceAndTarget, OracleReadTransferSourceAndTarget, PostgresReadTransferSourceAndTarget, ) -from schemas.v1.transfers.file.hdfs import ( +from syncmaster.schemas.v1.transfers.file.hdfs import ( HDFSCreateTransferSource, HDFSCreateTransferTarget, HDFSReadTransferSource, HDFSReadTransferTarget, ) -from schemas.v1.transfers.file.s3 import ( +from syncmaster.schemas.v1.transfers.file.s3 import ( S3CreateTransferSource, S3CreateTransferTarget, S3ReadTransferSource, S3ReadTransferTarget, ) -from schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy +from syncmaster.schemas.v1.transfers.strategy import FullStrategy, IncrementalStrategy ReadTransferSchemaSource = ( PostgresReadTransferSourceAndTarget diff --git a/syncmaster/schemas/v1/users.py b/syncmaster/schemas/v1/users.py index b410de1b..be76f571 100644 --- a/syncmaster/schemas/v1/users.py +++ b/syncmaster/schemas/v1/users.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from db import GroupMemberRole, Pagination from pydantic import BaseModel, constr -from schemas.v1.schemas import MetaPageSchema, PageSchema + +from syncmaster.db import GroupMemberRole, Pagination +from syncmaster.schemas.v1.schemas import MetaPageSchema, PageSchema class UpdateUserSchema(BaseModel): diff --git a/syncmaster/backend/worker/handlers/__init__.py b/syncmaster/worker/__init__.py similarity index 100% rename from syncmaster/backend/worker/handlers/__init__.py rename to syncmaster/worker/__init__.py diff --git a/syncmaster/backend/worker/base.py b/syncmaster/worker/base.py similarity index 88% rename from syncmaster/backend/worker/base.py rename to syncmaster/worker/base.py index 8caa0628..3e4b6286 100644 --- a/syncmaster/backend/worker/base.py +++ b/syncmaster/worker/base.py @@ -1,9 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.config import Settings from celery import Task from sqlalchemy import create_engine +from syncmaster.backend.config import Settings + class WorkerTask(Task): def __init__(self) -> None: diff --git a/syncmaster/backend/worker/config.py b/syncmaster/worker/config.py similarity index 72% rename from syncmaster/backend/worker/config.py rename to syncmaster/worker/config.py index 3e7fff40..bdd044f0 100644 --- a/syncmaster/backend/worker/config.py +++ b/syncmaster/worker/config.py @@ -1,9 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.config import Settings -from backend.worker.base import WorkerTask from celery import Celery +from syncmaster.backend.config import Settings +from syncmaster.worker.base import WorkerTask + settings = Settings() celery = Celery( @@ -12,6 +13,6 @@ backend="db+" + settings.build_db_connection_uri(driver="psycopg2"), task_cls=WorkerTask, imports=[ - "backend.worker.transfer", + "syncmaster.worker.transfer", ], ) diff --git a/syncmaster/backend/worker/controller.py b/syncmaster/worker/controller.py similarity index 84% rename from syncmaster/backend/worker/controller.py rename to syncmaster/worker/controller.py index 2bb6e5f8..f4f69aba 100644 --- a/syncmaster/backend/worker/controller.py +++ b/syncmaster/worker/controller.py @@ -3,29 +3,29 @@ import logging from typing import Any -from backend.config import Settings -from backend.worker.handlers.base import Handler -from backend.worker.handlers.file.hdfs import HDFSHandler -from backend.worker.handlers.file.s3 import S3Handler -from backend.worker.handlers.hive import HiveHandler -from backend.worker.handlers.oracle import OracleHandler -from backend.worker.handlers.postgres import PostgresHandler -from db import Connection, Transfer -from dto.connections import ( +from syncmaster.backend.config import Settings +from syncmaster.db import Connection, Transfer +from syncmaster.dto.connections import ( HDFSConnectionDTO, HiveConnectionDTO, OracleConnectionDTO, PostgresConnectionDTO, S3ConnectionDTO, ) -from dto.transfers import ( +from syncmaster.dto.transfers import ( HDFSTransferDTO, HiveTransferDTO, OracleTransferDTO, PostgresTransferDTO, S3TransferDTO, ) -from exceptions import ConnectionTypeNotRecognizedError +from syncmaster.exceptions import ConnectionTypeNotRecognizedError +from syncmaster.worker.handlers.base import Handler +from syncmaster.worker.handlers.file.hdfs import HDFSHandler +from syncmaster.worker.handlers.file.s3 import S3Handler +from syncmaster.worker.handlers.hive import HiveHandler +from syncmaster.worker.handlers.oracle import OracleHandler +from syncmaster.worker.handlers.postgres import PostgresHandler logger = logging.getLogger(__name__) diff --git a/syncmaster/backend/worker/handlers/file/__init__.py b/syncmaster/worker/handlers/__init__.py similarity index 100% rename from syncmaster/backend/worker/handlers/file/__init__.py rename to syncmaster/worker/handlers/__init__.py diff --git a/syncmaster/backend/worker/handlers/base.py b/syncmaster/worker/handlers/base.py similarity index 93% rename from syncmaster/backend/worker/handlers/base.py rename to syncmaster/worker/handlers/base.py index 5c330642..b8121df5 100644 --- a/syncmaster/backend/worker/handlers/base.py +++ b/syncmaster/worker/handlers/base.py @@ -2,12 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 from abc import ABC -from dto.connections import ConnectionDTO -from dto.transfers import TransferDTO from onetl.db import DBReader, DBWriter from pyspark.sql import SparkSession from pyspark.sql.dataframe import DataFrame +from syncmaster.dto.connections import ConnectionDTO +from syncmaster.dto.transfers import TransferDTO + class Handler(ABC): def __init__( diff --git a/syncmaster/worker/handlers/file/__init__.py b/syncmaster/worker/handlers/file/__init__.py new file mode 100644 index 00000000..104aecaf --- /dev/null +++ b/syncmaster/worker/handlers/file/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) +# SPDX-License-Identifier: Apache-2.0 diff --git a/syncmaster/backend/worker/handlers/file/base.py b/syncmaster/worker/handlers/file/base.py similarity index 92% rename from syncmaster/backend/worker/handlers/file/base.py rename to syncmaster/worker/handlers/file/base.py index 28c28681..6ae946a0 100644 --- a/syncmaster/backend/worker/handlers/file/base.py +++ b/syncmaster/worker/handlers/file/base.py @@ -3,15 +3,16 @@ import json import pydantic -from backend.worker.handlers.base import Handler -from dto.connections import ConnectionDTO -from dto.transfers import TransferDTO from onetl.base.base_file_df_connection import BaseFileDFConnection from onetl.file import FileDFReader, FileDFWriter from onetl.file.format import CSV, JSON, JSONLine from pyspark.sql.dataframe import DataFrame from pyspark.sql.types import StructType +from syncmaster.dto.connections import ConnectionDTO +from syncmaster.dto.transfers import TransferDTO +from syncmaster.worker.handlers.base import Handler + class FileHandler(Handler): connection: BaseFileDFConnection diff --git a/syncmaster/backend/worker/handlers/file/hdfs.py b/syncmaster/worker/handlers/file/hdfs.py similarity index 84% rename from syncmaster/backend/worker/handlers/file/hdfs.py rename to syncmaster/worker/handlers/file/hdfs.py index bb56d379..ce0a7441 100644 --- a/syncmaster/backend/worker/handlers/file/hdfs.py +++ b/syncmaster/worker/handlers/file/hdfs.py @@ -1,9 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.worker.handlers.file.base import FileHandler from onetl.connection import SparkHDFS +from syncmaster.worker.handlers.file.base import FileHandler + class HDFSHandler(FileHandler): def init_connection(self): diff --git a/syncmaster/backend/worker/handlers/file/s3.py b/syncmaster/worker/handlers/file/s3.py similarity index 91% rename from syncmaster/backend/worker/handlers/file/s3.py rename to syncmaster/worker/handlers/file/s3.py index 4778fab2..69082541 100644 --- a/syncmaster/backend/worker/handlers/file/s3.py +++ b/syncmaster/worker/handlers/file/s3.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.worker.handlers.file.base import FileHandler from onetl.connection import SparkS3 +from syncmaster.worker.handlers.file.base import FileHandler + class S3Handler(FileHandler): def init_connection(self): diff --git a/syncmaster/backend/worker/handlers/hive.py b/syncmaster/worker/handlers/hive.py similarity index 87% rename from syncmaster/backend/worker/handlers/hive.py rename to syncmaster/worker/handlers/hive.py index 99cbd4eb..3646e1c2 100644 --- a/syncmaster/backend/worker/handlers/hive.py +++ b/syncmaster/worker/handlers/hive.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.worker.handlers.base import Handler -from dto.connections import HiveConnectionDTO -from dto.transfers import HiveTransferDTO from onetl.connection import Hive from onetl.db import DBReader, DBWriter from pyspark.sql.dataframe import DataFrame +from syncmaster.dto.connections import HiveConnectionDTO +from syncmaster.dto.transfers import HiveTransferDTO +from syncmaster.worker.handlers.base import Handler + class HiveHandler(Handler): connection: Hive diff --git a/syncmaster/backend/worker/handlers/oracle.py b/syncmaster/worker/handlers/oracle.py similarity index 89% rename from syncmaster/backend/worker/handlers/oracle.py rename to syncmaster/worker/handlers/oracle.py index 61469d47..f8a79ff9 100644 --- a/syncmaster/backend/worker/handlers/oracle.py +++ b/syncmaster/worker/handlers/oracle.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.worker.handlers.base import Handler -from dto.connections import OracleConnectionDTO -from dto.transfers import OracleTransferDTO from onetl.connection import Oracle from onetl.db import DBReader, DBWriter from pyspark.sql.dataframe import DataFrame +from syncmaster.dto.connections import OracleConnectionDTO +from syncmaster.dto.transfers import OracleTransferDTO +from syncmaster.worker.handlers.base import Handler + class OracleHandler(Handler): connection: Oracle diff --git a/syncmaster/backend/worker/handlers/postgres.py b/syncmaster/worker/handlers/postgres.py similarity index 89% rename from syncmaster/backend/worker/handlers/postgres.py rename to syncmaster/worker/handlers/postgres.py index 25947313..25ddf337 100644 --- a/syncmaster/backend/worker/handlers/postgres.py +++ b/syncmaster/worker/handlers/postgres.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 -from backend.worker.handlers.base import Handler -from dto.connections import PostgresConnectionDTO -from dto.transfers import PostgresTransferDTO from onetl.connection import Postgres from onetl.db import DBReader, DBWriter from pyspark.sql.dataframe import DataFrame +from syncmaster.dto.connections import PostgresConnectionDTO +from syncmaster.dto.transfers import PostgresTransferDTO +from syncmaster.worker.handlers.base import Handler + class PostgresHandler(Handler): connection: Postgres diff --git a/syncmaster/backend/worker/transfer.py b/syncmaster/worker/transfer.py similarity index 88% rename from syncmaster/backend/worker/transfer.py rename to syncmaster/worker/transfer.py index 129786b7..517e34ac 100644 --- a/syncmaster/backend/worker/transfer.py +++ b/syncmaster/worker/transfer.py @@ -4,16 +4,17 @@ from datetime import datetime import onetl -from backend.config import Settings -from backend.worker.base import WorkerTask -from backend.worker.config import celery -from backend.worker.controller import TransferController -from db import AuthData, Run, Status, Transfer -from db.repositories import decrypt_auth_data -from exceptions.run import RunNotFoundError from sqlalchemy import select from sqlalchemy.orm import Session, selectinload +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Run, Status, Transfer +from syncmaster.db.repositories import decrypt_auth_data +from syncmaster.exceptions.run import RunNotFoundError +from syncmaster.worker.base import WorkerTask +from syncmaster.worker.config import celery +from syncmaster.worker.controller import TransferController + logger = logging.getLogger(__name__) diff --git a/syncmaster/backend/worker/utils.py b/syncmaster/worker/utils.py similarity index 96% rename from syncmaster/backend/worker/utils.py rename to syncmaster/worker/utils.py index 6895bba1..057158ad 100644 --- a/syncmaster/backend/worker/utils.py +++ b/syncmaster/worker/utils.py @@ -3,11 +3,12 @@ import logging import pyspark -from backend.config import Settings -from dto.connections import ConnectionDTO from onetl.connection import Oracle, Postgres, SparkS3 from pyspark.sql import SparkSession +from syncmaster.backend.config import Settings +from syncmaster.dto.connections import ConnectionDTO + log = logging.getLogger(__name__) diff --git a/tests/config/ivysettings.xml b/tests/config/ivysettings.xml deleted file mode 100644 index ff694d67..00000000 --- a/tests/config/ivysettings.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/tests/conftest.py b/tests/conftest.py index 71df57ce..3657b1fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,11 +9,6 @@ import pytest import pytest_asyncio from alembic.config import Config as AlembicConfig -from backend.api.v1.auth.utils import sign_jwt -from backend.config import Settings, TestSettings -from backend.main import get_application -from db import Base, Connection, Queue -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy.ext.asyncio import ( AsyncEngine, @@ -22,6 +17,11 @@ create_async_engine, ) +from syncmaster.backend.api.v1.auth.utils import sign_jwt +from syncmaster.backend.config import Settings, TestSettings +from syncmaster.backend.main import get_application +from syncmaster.db import Base, Connection, Queue +from syncmaster.db.repositories import decrypt_auth_data from tests.test_unit.conftest import add_user_to_group, create_group_member from tests.test_unit.utils import ( create_connection, @@ -65,8 +65,8 @@ def test_settings(): @pytest.fixture(scope="session") def alembic_config(settings: Settings) -> AlembicConfig: - alembic_cfg = AlembicConfig(PROJECT_PATH / "db" / "alembic.ini") - alembic_cfg.set_main_option("script_location", os.fspath(PROJECT_PATH / "db/migrations")) + alembic_cfg = AlembicConfig(PROJECT_PATH / "syncmaster" / "db" / "alembic.ini") + alembic_cfg.set_main_option("script_location", os.fspath(PROJECT_PATH / "syncmaster/db/migrations")) alembic_cfg.set_main_option("sqlalchemy.url", settings.build_db_connection_uri()) return alembic_cfg @@ -107,9 +107,11 @@ async def client( settings: Settings, async_engine: AsyncEngine, ) -> AsyncGenerator: + print(10 * "-", "START CLIENT FIXTURE", 10 * "-") app = get_application(settings=settings) async with AsyncClient(app=app, base_url="http://testserver") as client: yield client + print(10 * "-", "END CLIENT FIXTURE", 10 * "-") @pytest_asyncio.fixture diff --git a/tests/spark/get_worker_spark_session.py b/tests/spark/get_worker_spark_session.py index 7e4034ca..6fbd553b 100644 --- a/tests/spark/get_worker_spark_session.py +++ b/tests/spark/get_worker_spark_session.py @@ -4,15 +4,16 @@ import logging -from backend.config import Settings -from backend.worker.utils import get_spark_session_conf from celery.signals import worker_process_init, worker_process_shutdown from coverage import Coverage -from dto.connections import ConnectionDTO from onetl.connection import SparkHDFS from onetl.hooks import hook from pyspark.sql import SparkSession +from syncmaster.backend.config import Settings +from syncmaster.dto.connections import ConnectionDTO +from syncmaster.worker.utils import get_spark_session_conf + log = logging.getLogger(__name__) diff --git a/tests/test_database/test_migrationsup_to_date.py b/tests/test_database/test_migrationsup_to_date.py index f7026c38..9de0026f 100644 --- a/tests/test_database/test_migrationsup_to_date.py +++ b/tests/test_database/test_migrationsup_to_date.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import pytest from celery.backends.database.session import ResultModelBase -from db import Base from sqlalchemy.ext.asyncio import AsyncEngine +from syncmaster.db import Base from tests.utils import get_diff_db_metadata pytestmark = [pytest.mark.backend] diff --git a/tests/test_integration/test_run_transfer/conftest.py b/tests/test_integration/test_run_transfer/conftest.py index b05608fe..780ff262 100644 --- a/tests/test_integration/test_run_transfer/conftest.py +++ b/tests/test_integration/test_run_transfer/conftest.py @@ -10,15 +10,6 @@ import pytest import pytest_asyncio -from backend.api.v1.auth.utils import sign_jwt -from backend.config import Settings, TestSettings -from dto.connections import ( - HDFSConnectionDTO, - HiveConnectionDTO, - OracleConnectionDTO, - PostgresConnectionDTO, - S3ConnectionDTO, -) from onetl.connection import Hive, Oracle, Postgres, SparkS3 from onetl.db import DBWriter from onetl.file.format import CSV, JSON, JSONLine @@ -34,6 +25,15 @@ ) from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.api.v1.auth.utils import sign_jwt +from syncmaster.backend.config import Settings, TestSettings +from syncmaster.dto.connections import ( + HDFSConnectionDTO, + HiveConnectionDTO, + OracleConnectionDTO, + PostgresConnectionDTO, + S3ConnectionDTO, +) from tests.resources.file_df_connection.test_data import data from tests.test_unit.utils import ( create_connection, @@ -65,6 +65,7 @@ def spark(settings: Settings) -> SparkSession: def get_spark_session(connection_settings: Settings) -> SparkSession: + print(10 * "-", "START GET SPARK SESSION", 10 * "-") maven_packages = [p for connection in (Postgres, Oracle) for p in connection.get_packages()] maven_s3_packages = [p for p in SparkS3.get_packages(spark_version="3.4.1")] maven_packages.extend(maven_s3_packages) @@ -166,7 +167,9 @@ def s3(test_settings: TestSettings) -> S3ConnectionDTO: @pytest.fixture def init_df(spark: SparkSession) -> DataFrame: + print(10 * "-", "START INIT DF", 10 * "-") df = spark.createDataFrame(data, df_schema) # type: ignore + print(10 * "-", "END INIT DF", 10 * "-") return df @@ -177,6 +180,7 @@ def prepare_postgres( postgres: PostgresConnectionDTO, init_df: DataFrame, ) -> Postgres: + print(10 * "-", "START PREPARE POSTGRES", 10 * "-") postgres_connection = Postgres( host=postgres.host, port=postgres.port, @@ -193,6 +197,7 @@ def prepare_postgres( options=Postgres.WriteOptions(if_exists="append"), ) db_writer.run(init_df) + print(10 * "-", "END PREPARE POSTGRES", 10 * "-") return postgres_connection @@ -357,8 +362,10 @@ def prepare_hdfs( hdfs_file_connection, resource_path, ): + print(10 * "-", "START PREPARE HDFS", 10 * "-") connection, upload_to = hdfs_file_df_connection_with_path files = upload_files(resource_path, upload_to, hdfs_file_connection) + print(10 * "-", "END PREPARE HDFS", 10 * "-") return connection, upload_to, files @@ -368,6 +375,7 @@ def prepare_hive( hive: HiveConnectionDTO, init_df: DataFrame, ) -> Hive: + print(10 * "-", "START PREPARE HIVE", 10 * "-") hive_connection = Hive( cluster=hive.cluster, spark=spark, @@ -381,6 +389,7 @@ def prepare_hive( ) db_writer.run(init_df) spark.catalog.refreshTable("public.source_table") + print(10 * "-", "END PREPARE HIVE", 10 * "-") return hive_connection @@ -390,6 +399,7 @@ def prepare_oracle( oracle: OracleConnectionDTO, spark: SparkSession, ) -> Oracle: + print(10 * "-", "START PREPARE ORACLE", 10 * "-") oracle_connection = Oracle( host=oracle.host, port=oracle.port, @@ -413,6 +423,7 @@ def prepare_oracle( options=Oracle.WriteOptions(if_exists="append"), ) db_writer.run(init_df) + print(10 * "-", "END PREPARE ORACLE", 10 * "-") return oracle_connection @@ -460,6 +471,7 @@ async def transfers( hdfs: HDFSConnectionDTO, s3: S3ConnectionDTO, ): + print(10 * "-", "START TRANSFERS FIXTURE", 10 * "-") s3_file_format, file_format_object = choice_file_format _, source_path, _ = prepare_s3 @@ -589,7 +601,7 @@ async def transfers( queue = await create_queue( session=session, - name=f"test_queue_{secrets.token_hex(5)}", + name="test_queue", group_id=group.id, ) @@ -661,6 +673,7 @@ async def transfers( ), } data.update(transfers) # type: ignore + print(10 * "-", "END TRANSFERS FIXTURE", 10 * "-") yield data for transfer in transfers.values(): await session.delete(transfer) diff --git a/tests/test_integration/test_run_transfer/test_hdfs_to_postgres.py b/tests/test_integration/test_run_transfer/test_hdfs_to_postgres.py index c285924c..14cd8db7 100644 --- a/tests/test_integration/test_run_transfer/test_hdfs_to_postgres.py +++ b/tests/test_integration/test_run_transfer/test_hdfs_to_postgres.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.db import DBReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.test_integration.test_run_transfer.conftest import df_schema from tests.utils import MockUser, get_run_on_end diff --git a/tests/test_integration/test_run_transfer/test_hive_to_postgres.py b/tests/test_integration/test_run_transfer/test_hive_to_postgres.py index 58d632a5..4cd5ae9d 100644 --- a/tests/test_integration/test_run_transfer/test_hive_to_postgres.py +++ b/tests/test_integration/test_run_transfer/test_hive_to_postgres.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.db import DBReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.utils import MockUser, get_run_on_end pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.hive, pytest.mark.postgres] diff --git a/tests/test_integration/test_run_transfer/test_oracle_to_postgres.py b/tests/test_integration/test_run_transfer/test_oracle_to_postgres.py index d11ef414..9886d867 100644 --- a/tests/test_integration/test_run_transfer/test_oracle_to_postgres.py +++ b/tests/test_integration/test_run_transfer/test_oracle_to_postgres.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.db import DBReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.utils import MockUser, get_run_on_end pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.oracle, pytest.mark.postgres] diff --git a/tests/test_integration/test_run_transfer/test_postgres_to_hdfs.py b/tests/test_integration/test_run_transfer/test_postgres_to_hdfs.py index 126fa8c1..d052d26a 100644 --- a/tests/test_integration/test_run_transfer/test_postgres_to_hdfs.py +++ b/tests/test_integration/test_run_transfer/test_postgres_to_hdfs.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.file import FileDFReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.utils import MockUser, get_run_on_end pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.hdfs, pytest.mark.postgres] diff --git a/tests/test_integration/test_run_transfer/test_postgres_to_hive.py b/tests/test_integration/test_run_transfer/test_postgres_to_hive.py index acd68a21..f3354a66 100644 --- a/tests/test_integration/test_run_transfer/test_postgres_to_hive.py +++ b/tests/test_integration/test_run_transfer/test_postgres_to_hive.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.db import DBReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.utils import MockUser, get_run_on_end pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.hive, pytest.mark.postgres] diff --git a/tests/test_integration/test_run_transfer/test_postgres_to_oracle.py b/tests/test_integration/test_run_transfer/test_postgres_to_oracle.py index aad01bdd..cad2e5ca 100644 --- a/tests/test_integration/test_run_transfer/test_postgres_to_oracle.py +++ b/tests/test_integration/test_run_transfer/test_postgres_to_oracle.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.db import DBReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.utils import MockUser, get_run_on_end pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.oracle, pytest.mark.postgres] diff --git a/tests/test_integration/test_run_transfer/test_postgres_to_s3.py b/tests/test_integration/test_run_transfer/test_postgres_to_s3.py index c1bb530e..93089246 100644 --- a/tests/test_integration/test_run_transfer/test_postgres_to_s3.py +++ b/tests/test_integration/test_run_transfer/test_postgres_to_s3.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.file import FileDFReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.utils import MockUser, get_run_on_end pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.s3, pytest.mark.postgres] diff --git a/tests/test_integration/test_run_transfer/test_read_mixed_column_naming.py b/tests/test_integration/test_run_transfer/test_read_mixed_column_naming.py index baffff0a..e471038f 100644 --- a/tests/test_integration/test_run_transfer/test_read_mixed_column_naming.py +++ b/tests/test_integration/test_run_transfer/test_read_mixed_column_naming.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.db import DBReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.utils import MockUser, get_run_on_end pytestmark = [pytest.mark.asyncio, pytest.mark.worker, pytest.mark.oracle, pytest.mark.postgres] diff --git a/tests/test_integration/test_run_transfer/test_s3_to_postgres.py b/tests/test_integration/test_run_transfer/test_s3_to_postgres.py index 23641b7f..95b4d109 100644 --- a/tests/test_integration/test_run_transfer/test_s3_to_postgres.py +++ b/tests/test_integration/test_run_transfer/test_s3_to_postgres.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status, Transfer from httpx import AsyncClient from onetl.db import DBReader from pyspark.sql import DataFrame +from syncmaster.db import Status, Transfer from tests.test_integration.test_run_transfer.conftest import df_schema from tests.utils import MockUser, get_run_on_end diff --git a/tests/test_unit/conftest.py b/tests/test_unit/conftest.py index 44266ec4..19ead156 100644 --- a/tests/test_unit/conftest.py +++ b/tests/test_unit/conftest.py @@ -3,12 +3,12 @@ import secrets import pytest_asyncio -from backend.api.v1.auth.utils import sign_jwt -from backend.config import Settings -from db import Queue, User, UserGroup -from db.repositories import decrypt_auth_data from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.api.v1.auth.utils import sign_jwt +from syncmaster.backend.config import Settings +from syncmaster.db import Queue, User, UserGroup +from syncmaster.db.repositories import decrypt_auth_data from tests.test_unit.utils import ( create_connection, create_credentials, diff --git a/tests/test_unit/test_connections/test_copy_connection.py b/tests/test_unit/test_connections/test_copy_connection.py index 474eda4e..56e5bea3 100644 --- a/tests/test_unit/test_connections/test_copy_connection.py +++ b/tests/test_unit/test_connections/test_copy_connection.py @@ -3,13 +3,13 @@ import secrets import pytest -from backend.config import Settings -from db import AuthData, Connection -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Connection +from syncmaster.db.repositories import decrypt_auth_data from tests.utils import MockConnection, MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_connections/test_create_all_connection/test_create_hdfs_connection.py b/tests/test_unit/test_connections/test_create_all_connection/test_create_hdfs_connection.py index 35255dd6..c055f165 100644 --- a/tests/test_unit/test_connections/test_create_all_connection/test_create_hdfs_connection.py +++ b/tests/test_unit/test_connections/test_create_all_connection/test_create_hdfs_connection.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from backend.config import Settings -from db import AuthData, Connection -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Connection +from syncmaster.db.repositories import decrypt_auth_data from tests.utils import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.hdfs] diff --git a/tests/test_unit/test_connections/test_create_all_connection/test_create_hive_connection.py b/tests/test_unit/test_connections/test_create_all_connection/test_create_hive_connection.py index 2d27a269..f6d3b8fa 100644 --- a/tests/test_unit/test_connections/test_create_all_connection/test_create_hive_connection.py +++ b/tests/test_unit/test_connections/test_create_all_connection/test_create_hive_connection.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from backend.config import Settings -from db import AuthData, Connection -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Connection +from syncmaster.db.repositories import decrypt_auth_data from tests.utils import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.hive] diff --git a/tests/test_unit/test_connections/test_create_all_connection/test_create_oracle_connection.py b/tests/test_unit/test_connections/test_create_all_connection/test_create_oracle_connection.py index 896a69ac..f8302ec9 100644 --- a/tests/test_unit/test_connections/test_create_all_connection/test_create_oracle_connection.py +++ b/tests/test_unit/test_connections/test_create_all_connection/test_create_oracle_connection.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from backend.config import Settings -from db import AuthData, Connection -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Connection +from syncmaster.db.repositories import decrypt_auth_data from tests.utils import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.oracle] diff --git a/tests/test_unit/test_connections/test_create_all_connection/test_create_postgres_connection.py b/tests/test_unit/test_connections/test_create_all_connection/test_create_postgres_connection.py index 6736d8cf..d389fd6b 100644 --- a/tests/test_unit/test_connections/test_create_all_connection/test_create_postgres_connection.py +++ b/tests/test_unit/test_connections/test_create_all_connection/test_create_postgres_connection.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from backend.config import Settings -from db import AuthData, Connection -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Connection +from syncmaster.db.repositories import decrypt_auth_data from tests.utils import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.postgres] diff --git a/tests/test_unit/test_connections/test_create_all_connection/test_create_s3_connection.py b/tests/test_unit/test_connections/test_create_all_connection/test_create_s3_connection.py index 1a6d6a27..0da96d16 100644 --- a/tests/test_unit/test_connections/test_create_all_connection/test_create_s3_connection.py +++ b/tests/test_unit/test_connections/test_create_all_connection/test_create_s3_connection.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from backend.config import Settings -from db import AuthData, Connection -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Connection +from syncmaster.db.repositories import decrypt_auth_data from tests.utils import MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.s3] diff --git a/tests/test_unit/test_connections/test_create_connection.py b/tests/test_unit/test_connections/test_create_connection.py index 68100bd4..c003b3ca 100644 --- a/tests/test_unit/test_connections/test_create_connection.py +++ b/tests/test_unit/test_connections/test_create_connection.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from backend.config import Settings -from db import AuthData, Connection -from db.repositories import decrypt_auth_data from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings +from syncmaster.db import AuthData, Connection +from syncmaster.db.repositories import decrypt_auth_data from tests.test_unit.conftest import ALLOWED_SOURCES from tests.utils import MockConnection, MockGroup, MockUser, UserTestRoles diff --git a/tests/test_unit/test_connections/test_delete_connection.py b/tests/test_unit/test_connections/test_delete_connection.py index 40c07d02..3aaf8d8a 100644 --- a/tests/test_unit/test_connections/test_delete_connection.py +++ b/tests/test_unit/test_connections/test_delete_connection.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Connection from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Connection from tests.utils import MockConnection, MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_connections/test_read_connection_types.py b/tests/test_unit/test_connections/test_read_connection_types.py index f371bbcd..30c70538 100644 --- a/tests/test_unit/test_connections/test_read_connection_types.py +++ b/tests/test_unit/test_connections/test_read_connection_types.py @@ -3,9 +3,9 @@ from typing import get_args import pytest -from backend.api.v1.connections import CONNECTION_TYPES from httpx import AsyncClient +from syncmaster.backend.api.v1.connections import CONNECTION_TYPES from tests.utils import MockUser pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_connections/test_read_connections.py b/tests/test_unit/test_connections/test_read_connections.py index 4d4bf3a1..04711eb5 100644 --- a/tests/test_unit/test_connections/test_read_connections.py +++ b/tests/test_unit/test_connections/test_read_connections.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from backend.config import Settings from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.config import Settings from tests.utils import MockConnection, MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_groups/test_create_group.py b/tests/test_unit/test_groups/test_create_group.py index 11a148aa..a19878f2 100644 --- a/tests/test_unit/test_groups/test_create_group.py +++ b/tests/test_unit/test_groups/test_create_group.py @@ -3,11 +3,11 @@ import secrets import pytest -from db import Group from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Group from tests.utils import MockUser pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_groups/test_delete_group_by_id.py b/tests/test_unit/test_groups/test_delete_group_by_id.py index f84a4ebf..2e7a608b 100644 --- a/tests/test_unit/test_groups/test_delete_group_by_id.py +++ b/tests/test_unit/test_groups/test_delete_group_by_id.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Group from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Group from tests.utils import MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_queue/test_create_queue.py b/tests/test_unit/test_queue/test_create_queue.py index 4216920e..bcbaa659 100644 --- a/tests/test_unit/test_queue/test_create_queue.py +++ b/tests/test_unit/test_queue/test_create_queue.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Queue from tests.utils import MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_queue/test_delete_queue.py b/tests/test_unit/test_queue/test_delete_queue.py index 8f1bf570..aa19a0b0 100644 --- a/tests/test_unit/test_queue/test_delete_queue.py +++ b/tests/test_unit/test_queue/test_delete_queue.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Queue from tests.utils import MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_queue/test_read_queue.py b/tests/test_unit/test_queue/test_read_queue.py index b18e368c..da3a1afe 100644 --- a/tests/test_unit/test_queue/test_read_queue.py +++ b/tests/test_unit/test_queue/test_read_queue.py @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue from httpx import AsyncClient +from syncmaster.db import Queue from tests.utils import MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_queue/test_read_queues.py b/tests/test_unit/test_queue/test_read_queues.py index 97c30a44..faeea8db 100644 --- a/tests/test_unit/test_queue/test_read_queues.py +++ b/tests/test_unit/test_queue/test_read_queues.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Queue from tests.utils import MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_queue/test_update_queue.py b/tests/test_unit/test_queue/test_update_queue.py index 6e66a35a..36d6f5a2 100644 --- a/tests/test_unit/test_queue/test_update_queue.py +++ b/tests/test_unit/test_queue/test_update_queue.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Queue from tests.utils import MockGroup, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_transfers/test_copy_transfer.py b/tests/test_unit/test_transfers/test_copy_transfer.py index 2b1f5f56..7dd8171c 100644 --- a/tests/test_unit/test_transfers/test_copy_transfer.py +++ b/tests/test_unit/test_transfers/test_copy_transfer.py @@ -4,11 +4,11 @@ from operator import or_ import pytest -from db import Connection, Queue from httpx import AsyncClient from sqlalchemy import delete from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Connection, Queue from tests.utils import MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_create_transfer.py index 1216617c..de24f7a2 100644 --- a/tests/test_unit/test_transfers/test_create_transfer.py +++ b/tests/test_unit/test_transfers/test_create_transfer.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue, Transfer from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Queue, Transfer from tests.utils import MockConnection, MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_transfers/test_delete_transfer.py b/tests/test_unit/test_transfers/test_delete_transfer.py index 8c23bd2a..6f0c4abc 100644 --- a/tests/test_unit/test_transfers/test_delete_transfer.py +++ b/tests/test_unit/test_transfers/test_delete_transfer.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Transfer from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Transfer from tests.utils import MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py index b15aa9bf..2afa9847 100644 --- a/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py +++ b/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue, Transfer from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Queue, Transfer from tests.utils import MockConnection, MockGroup, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_transfers/test_runs/conftest.py b/tests/test_unit/test_transfers/test_runs/conftest.py index b70df4ea..4383984f 100644 --- a/tests/test_unit/test_transfers/test_runs/conftest.py +++ b/tests/test_unit/test_transfers/test_runs/conftest.py @@ -3,10 +3,10 @@ import secrets import pytest_asyncio -from backend.api.v1.auth.utils import sign_jwt -from backend.config import Settings from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.backend.api.v1.auth.utils import sign_jwt +from syncmaster.backend.config import Settings from tests.test_unit.conftest import create_group_member from tests.test_unit.utils import ( create_connection, diff --git a/tests/test_unit/test_transfers/test_runs/test_create_run.py b/tests/test_unit/test_transfers/test_runs/test_create_run.py index faddfc13..cd75ac56 100644 --- a/tests/test_unit/test_transfers/test_runs/test_create_run.py +++ b/tests/test_unit/test_transfers/test_runs/test_create_run.py @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Run, Status from httpx import AsyncClient from sqlalchemy import desc, select from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Run, Status from tests.utils import MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] @@ -20,7 +20,7 @@ async def test_developer_plus_can_create_run_of_transfer_his_group( ) -> None: # Arrange user = group_transfer.owner_group.get_member_of_role(role_developer_plus) - mocker.patch("backend.worker.config.celery.send_task") + mocker.patch("syncmaster.worker.config.celery.send_task") run = ( await session.scalars( @@ -64,7 +64,7 @@ async def test_groupless_user_cannot_create_run( mocker, ) -> None: # Arrange - mocker.patch("backend.worker.config.celery.send_task") + mocker.patch("syncmaster.worker.config.celery.send_task") # Act result = await client.post( @@ -91,7 +91,7 @@ async def test_group_member_cannot_create_run_of_other_group_transfer( role_guest_plus: UserTestRoles, ): # Arrange - mocker.patch("backend.worker.config.celery.send_task") + mocker.patch("syncmaster.worker.config.celery.send_task") user = group.get_member_of_role(role_guest_plus) # Act @@ -124,7 +124,7 @@ async def test_superuser_can_create_run( mocker, ) -> None: # Arrange - mocker.patch("backend.worker.config.celery.send_task") + mocker.patch("syncmaster.worker.config.celery.send_task") # Act result = await client.post( @@ -157,7 +157,7 @@ async def test_unauthorized_user_cannot_create_run( mocker, ) -> None: # Arrange - mocker.patch("backend.worker.config.celery.send_task") + mocker.patch("syncmaster.worker.config.celery.send_task") # Act result = await client.post( @@ -183,7 +183,7 @@ async def test_group_member_cannot_create_run_of_unknown_transfer_error( ) -> None: # Arrange user = group_transfer.owner_group.get_member_of_role(role_guest_plus) - mocker.patch("backend.worker.config.celery.send_task") + mocker.patch("syncmaster.worker.config.celery.send_task") # Act result = await client.post( @@ -208,7 +208,7 @@ async def test_superuser_cannot_create_run_of_unknown_transfer_error( mocker, ) -> None: # Arrange - mocker.patch("backend.worker.config.celery.send_task") + mocker.patch("syncmaster.worker.config.celery.send_task") # Act result = await client.post( diff --git a/tests/test_unit/test_transfers/test_runs/test_stop_run.py b/tests/test_unit/test_transfers/test_runs/test_stop_run.py index 0c133642..be765f3f 100644 --- a/tests/test_unit/test_transfers/test_runs/test_stop_run.py +++ b/tests/test_unit/test_transfers/test_runs/test_stop_run.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Status from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from syncmaster.db import Status from tests.utils import MockGroup, MockRun, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/test_transfers/test_update_transfer.py b/tests/test_unit/test_transfers/test_update_transfer.py index fe9220bd..d32559a7 100644 --- a/tests/test_unit/test_transfers/test_update_transfer.py +++ b/tests/test_unit/test_transfers/test_update_transfer.py @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: 2023-2024 MTS (Mobile Telesystems) # SPDX-License-Identifier: Apache-2.0 import pytest -from db import Queue from httpx import AsyncClient +from syncmaster.db import Queue from tests.utils import MockConnection, MockGroup, MockTransfer, MockUser, UserTestRoles pytestmark = [pytest.mark.asyncio, pytest.mark.backend] diff --git a/tests/test_unit/utils.py b/tests/test_unit/utils.py index 74332350..8aac2d08 100644 --- a/tests/test_unit/utils.py +++ b/tests/test_unit/utils.py @@ -8,15 +8,25 @@ from datetime import datetime from typing import Any -from backend.config import Settings -from db import AuthData, Connection, Group, Queue, Run, Status, Transfer, User -from db.repositories import encrypt_auth_data from onetl.connection import FileConnection from onetl.impl import LocalPath, RemotePath -from schemas.v1.transfers import ReadFullTransferSchema from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload +from syncmaster.backend.config import Settings +from syncmaster.db import ( + AuthData, + Connection, + Group, + Queue, + Run, + Status, + Transfer, + User, +) +from syncmaster.db.repositories import encrypt_auth_data +from syncmaster.schemas.v1.transfers import ReadFullTransferSchema + @asynccontextmanager async def create_user_cm( diff --git a/tests/utils.py b/tests/utils.py index e572007d..3c94a0c1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -11,8 +11,6 @@ from alembic.runtime.environment import EnvironmentContext from alembic.runtime.migration import MigrationContext from alembic.script import ScriptDirectory -from backend.config import Settings -from db import Connection, Group, Run, Status, Transfer, User from httpx import AsyncClient from sqlalchemy import Connection as AlchConnection from sqlalchemy import MetaData, pool, text @@ -22,6 +20,9 @@ create_async_engine, ) +from syncmaster.backend.config import Settings +from syncmaster.db import Connection, Group, Run, Status, Transfer, User + logger = logging.getLogger(__name__) @@ -201,6 +202,7 @@ async def drop_database(connection: AsyncConnection, db_name: str) -> None: async def get_run_on_end(client: AsyncClient, run_id: int, token: str) -> dict[str, Any]: while True: + print(10 * "-", "WAITING FOR THE RUN STATUS", 10 * "-") result = await client.get( f"v1/runs/{run_id}", headers={"Authorization": f"Bearer {token}"},