Merge branch 'develop'

MobileTeleSystems · Apr 15, 2024 · cd81768 · cd81768
2 parents d570433 + 23f4f44
commit cd81768
Show file tree

Hide file tree

Showing 53 changed files with 493 additions and 677 deletions.
diff --git a/.env.docker b/.env.docker
@@ -51,3 +51,4 @@ ENV=LOCAL
 SPARK_CONF_DIR=/app/tests/spark/hive/conf/
 HADOOP_CONF_DIR=/app/tests/spark/hadoop/
 HIVE_CONF_DIR=/app/tests/spark/hive/conf/
+CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session.get_worker_spark_session
diff --git a/.env.local b/.env.local
@@ -50,4 +50,5 @@ export ENV=LOCAL
 
 export SPARK_CONF_DIR=./app/tests/spark/hive/conf/
 export HADOOP_CONF_DIR=./app/tests/spark/hadoop/
-export HIVE_CONF_DIR=./app/tests/spark/hive/conf/
+export HIVE_CONF_DIR=./app/tests/spark/hive/conf/
+export CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session.get_worker_spark_session
diff --git a/.github/workflows/backend_docker_image.yml b/.github/workflows/backend_docker_image.yml
@@ -48,7 +48,7 @@ jobs:
         with:
           tags: ${{ env.TAG }}
           context: .
-          file: docker/backend.dockerfile
+          file: docker/Dockerfile.backend
           pull: true
           push: true
           cache-to: type=gha,mode=max

diff --git a/.github/workflows/hdfs-tests.yml b/.github/workflows/hdfs-tests.yml
@@ -47,9 +47,9 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          tags: syncmaster_worker:${{ github.sha }}
+          tags: mtsrus/syncmaster-worker:${{ github.sha }}
           target: test
-          file: docker/worker.dockerfile
+          file: docker/Dockerfile.worker
           load: true
           cache-to: type=gha,mode=max
           cache-from: type=gha
@@ -60,22 +60,16 @@ jobs:
           docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-hive test-postgres --wait --wait-timeout 200
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
-      - name: Run HDFS Tests.
-        # Tests for the backend are run on the worker.
-        # Backend and worker on the same container.
+      # This is important, as coverage is exported after receiving SIGTERM
+      - name: Run HDFS Tests
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m hdfs
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and hdfs"
 
       - name: Shutdown
         if: always()
         run: |
           docker compose -f docker-compose.test.yml down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml
@@ -47,9 +47,9 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          tags: syncmaster_worker:${{ github.sha }}
+          tags: mtsrus/syncmaster-worker:${{ github.sha }}
           target: test
-          file: docker/worker.dockerfile
+          file: docker/Dockerfile.worker
           load: true
           cache-to: type=gha,mode=max
           cache-from: type=gha
@@ -60,22 +60,16 @@ jobs:
           docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-hive test-postgres --wait --wait-timeout 200
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
-      - name: Run Hive Tests.
-        # Tests for the backend are run on the worker.
-        # Backend and worker on the same container.
+      # This is important, as coverage is exported after receiving SIGTERM
+      - name: Run Hive Tests
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m hive
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and hive"
 
       - name: Shutdown
         if: always()
         run: |
           docker compose -f docker-compose.test.yml down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/oracle-tests.yml b/.github/workflows/oracle-tests.yml
@@ -47,9 +47,9 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          tags: syncmaster_worker:${{ github.sha }}
+          tags: mtsrus/syncmaster-worker:${{ github.sha }}
           target: test
-          file: docker/worker.dockerfile
+          file: docker/Dockerfile.worker
           load: true
           cache-to: type=gha,mode=max
           cache-from: type=gha
@@ -60,22 +60,16 @@ jobs:
           docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-oracle test-postgres --wait --wait-timeout 200
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
-      - name: Run Oracle Tests.
-        # Tests for the backend are run on the worker.
-        # Backend and worker on the same container.
+      # This is important, as coverage is exported after receiving SIGTERM
+      - name: Run Oracle Tests
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m oracle
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and oracle"
 
       - name: Shutdown
         if: always()
         run: |
           docker compose -f docker-compose.test.yml down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/s3-tests.yml b/.github/workflows/s3-tests.yml
@@ -47,8 +47,8 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          tags: syncmaster_worker:${{ github.sha }}
-          file: docker/worker.dockerfile
+          tags: mtsrus/syncmaster-worker:${{ github.sha }}
+          file: docker/Dockerfile.worker
           target: test
           load: true
           cache-to: type=gha,mode=max
@@ -60,22 +60,16 @@ jobs:
           docker compose -f docker-compose.test.yml up -d db worker rabbitmq test-s3 test-postgres --wait --wait-timeout 200
         env:
           WORKER_IMAGE_TAG: ${{ github.sha }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
-      - name: Run S3 Tests.
-        # Tests for the backend are run on the worker.
-        # Backend and worker on the same container.
+      - name: Run S3 Tests
         run: |
-          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m s3
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+          docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and s3"
 
+      # This is important, as coverage is exported after receiving SIGTERM
       - name: Shutdown
         if: always()
         run: |
           docker compose -f docker-compose.test.yml down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -52,22 +52,17 @@ jobs:
         run: |
           docker compose -f docker-compose.test.yml down -v --remove-orphans
           docker compose -f docker-compose.test.yml up -d db --wait --wait-timeout 200
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
-      - name: Run Unit Tests.
+      - name: Run Unit Tests
         run: |
           source .env.local
-          ./pytest_runner.sh -m backend
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+          poetry run python -m syncmaster.db.migrations upgrade head
+          poetry run coverage run -m pytest -vvv -s -m backend
 
       - name: Shutdown
         if: always()
         run: |
           docker compose -f docker-compose.test.yml down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/worker_docker_image.yml b/.github/workflows/worker_docker_image.yml
@@ -49,7 +49,7 @@ jobs:
           tags: ${{ env.TAG }}
           context: .
           target: prod
-          file: docker/worker.dockerfile
+          file: docker/Dockerfile.worker
           pull: true
           push: true
           cache-to: type=gha,mode=max

diff --git a/.gitignore b/.gitignore
@@ -158,3 +158,5 @@ cython_debug/
 **/metastore_db/
 **/spark-warehouse/
 cached_jars/
+
+.DS_Store
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
@@ -40,13 +40,13 @@ repos:
       - id: chmod
         args: ['644']
         exclude_types: [shell]
-      #  exclude: ^(.*__main__\.py)$
-      #- id: chmod
-      #  args: ['755']
-      #  types: [shell]
-      #- id: chmod
-      #  args: ['755']
-      #  files: ^(.*__main__\.py)$
+        exclude: ^(.*__main__\.py)$
+      - id: chmod
+        args: ['755']
+        types: [shell]
+      - id: chmod
+        args: ['755']
+        files: ^(.*__main__\.py)$
       - id: insert-license
         files: .*\.py$
         exclude: ^(tests/.*\.py)$
@@ -111,7 +111,7 @@ repos:
       - id: docker-compose-check
 
   - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-    rev: v2.12.0
+    rev: v2.13.0
     hooks:
       - id: pretty-format-yaml
         args: [--autofix, --indent, '2', --offset, '2']

diff --git a/Makefile b/Makefile
@@ -50,7 +50,7 @@ revision: ##@Database Create new revision of migrations
 	RABBITMQ_USER=${RABBITMQ_USER} \
 	RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD} \
 	PYTHONPATH=${APP_PATH} \
-	${POETRY} run alembic -c ${APP_PATH}/alembic.ini revision --autogenerate
+	${POETRY} run python -m syncmaster.db.migrations --autogenerate
 
 migrate: ##@Database Upgdade database to last migration
 	@POSTGRES_HOST=${POSTGRES_HOST} \
@@ -63,7 +63,7 @@ migrate: ##@Database Upgdade database to last migration
 	RABBITMQ_USER=${RABBITMQ_USER} \
 	RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD} \
 	PYTHONPATH=${APP_PATH} \
-	${POETRY} run alembic -c ${APP_PATH}/alembic.ini upgrade head
+	${POETRY} run python -m syncmaster.db.migrations upgrade head
 
 test: ##@Test Run tests
 	@POSTGRES_HOST=${POSTGRES_HOST} \

diff --git a/README.rst b/README.rst
@@ -17,9 +17,9 @@ SyncMaster
     :target: https://badge.fury.io/py/data-syncmaster
 .. |Docker image| image:: https://img.shields.io/docker/v/mtsrus/syncmaster-backend?sort=semver&label=docker
     :target: https://hub.docker.com/r/mtsrus/syncmaster-backend
-.. |Documentation| image:: https://readthedocs.org/projects/data-syncmaster/badge/?version=stable
+.. |Documentation| image:: https://readthedocs.org/projects/syncmaster/badge/?version=stable
     :target: https://syncmaster.readthedocs.io
-.. |Build Status| image:: https://github.com/MobileTeleSystems/syncmaster/workflows/Tests/badge.svg
+.. |Build Status| image:: https://github.com/MobileTeleSystems/syncmaster/workflows/Run%20All%20Tests/badge.svg
     :target: https://github.com/MobileTeleSystems/syncmaster/actions
 .. |Coverage| image:: https://codecov.io/gh/MobileTeleSystems/syncmaster/graph/badge.svg?token=ky7UyUxolB
     :target: https://codecov.io/gh/MobileTeleSystems/syncmaster