[DOP-11711] Run tests in github actions

.env.docker

@@ -24,7 +24,7 @@ MINIO_ROOT_USER=syncmaster
 MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
 MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
 
-TEST_S3_HOST=s3
+TEST_S3_HOST=test-s3
 TEST_S3_PORT=9000
 TEST_S3_ACCESS_KEY=syncmaster
 TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c

.env.github

@@ -0,0 +1,53 @@
+export TZ=UTC
+
+# Postgres
+export POSTGRES_HOST=localhost
+export POSTGRES_PORT=5432
+export POSTGRES_DB=syncmaster
+export POSTGRES_USER=user
+export POSTGRES_PASSWORD=secret
+
+# RabbitMQ
+export RABBITMQ_HOST=localhost
+export RABBITMQ_PORT=5672
+export RABBITMQ_USER=guest
+export RABBITMQ_PASSWORD=guest
+
+# HDFS
+export HDFS_HOST=localhost
+export HDFS_WEBHDFS_PORT=9870
+export HDFS_IPC_PORT=9820
+
+# S3
+export MINIO_ACCESS_KEY=syncmaster
+export MINIO_ROOT_USER=syncmaster
+export MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+export MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+
+export TEST_S3_HOST=localhost
+export TEST_S3_PORT=9010
+export TEST_S3_ACCESS_KEY=syncmaster
+export TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+export TEST_S3_BUCKET=syncmaster
+
+export TEST_POSTGRES_HOST=localhost
+export TEST_POSTGRES_PORT=5433
+export TEST_POSTGRES_USER=user
+export TEST_POSTGRES_PASSWORD=secret
+export TEST_POSTGRES_DB=syncmaster
+
+export TEST_ORACLE_HOST=localhost
+export TEST_ORACLE_PORT=1522
+export TEST_ORACLE_USER=test_user
+export TEST_ORACLE_PASSWORD=test_password
+export TEST_ORACLE_SERVICE_NAME=XEPDB1
+
+export TEST_HIVE_CLUSTER=test-hive
+export TEST_HIVE_USER=hive_user
+export TEST_HIVE_PASSWORD=hive_password
+
+export ENV=LOCAL
+
+export SPARK_CONF_DIR=./syncmaster/tests/spark/hive/conf/
+export HADOOP_CONF_DIR=./syncmaster/tests/spark/hadoop/
+export HIVE_CONF_DIR=./syncmaster/tests/spark/hive/conf/

.github/workflows/unit_test.yml

@@ -0,0 +1,79 @@
+name: Tests
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    branches-ignore:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEFAULT_PYTHON: '3.11'
+
+jobs:
+  tests:
+    name: Run unit tests (Python 3.11, Pydantic v1 on ubuntu-latest)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    env:
+      POETRY_VERSION: 1.6.1
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.6.1
+
+      - name: Install dependencies
+        run: |
+          poetry install --no-root --no-cache --with backend,dev,test
+
+      - name: Start docker compose
+        run: |
+          docker compose -f docker-compose.test.yml down -v --remove-orphans
+          docker compose -f docker-compose.test.yml up -d db --wait --wait-timeout 200
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Run Unit Tests.
+        run: |
+          source .env.github
+          mkdir -p reports/
+          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_unit
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Run DB Tests.
+        run: |
+          source .env.github
+          mkdir -p reports/
+          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_database
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Shutdown
+        if: always()
+        run: |
+          docker compose -f docker-compose.test.yml down -v --remove-orphans
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster

docker-compose.yml → docker-compose.test.yml

@@ -3,33 +3,14 @@ services:
   db:
     image: postgres:15
     restart: unless-stopped
-    env_file: .env.dev
+    env_file: .env.docker
     ports:
       - 5432:5432
     volumes:
       - postgres_data:/var/lib/postgresql/data
     networks:
       - network
 
-  rabbitmq:
-    image: rabbitmq
-    restart: unless-stopped
-    ports:
-      - 5672:5672
-    networks:
-      - network
-
-  s3:
-    image: bitnami/minio:latest
-    container_name: s3
-    restart: unless-stopped
-    env_file: .env.docker
-    ports:
-      - 9010:9000
-      - 9011:9001
-    networks:
-      - network
-
   worker:
     image: syncmaster_worker
     restart: unless-stopped
@@ -39,12 +20,13 @@ services:
       network: host
     networks:
       - network
+    volumes:
+      - /mnt/.ivy2:/root/.ivy2
     env_file: .env.docker
-    command: [celery, -A, app.tasks.config.celery, worker, --loglevel=info, --max-tasks-per-child=1, -Q, test_queue]
 
   backend:
     image: syncmaster_back
-    restart: unless-stopped
+    container_name: backend
     build:
       dockerfile: docker/backend.dockerfile
       context: .
@@ -56,31 +38,40 @@ services:
     depends_on:
       - db
       - rabbitmq
-      - worker
       - test-oracle
       - test-postgres
       - test-hive
-    environment:
-      POSTGRES_HOST: db
-      POSGRES_PORT: 5432
-      POSTGRES_DB: ${POSTGRES_DB}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      - test-s3
+    volumes:
+      - /mnt/.ivy2:/root/.ivy2
+    env_file: .env.docker
+
 
-      RABBITMQ_HOST: rabbitmq
-      RABBITMQ_PORT: 5672
-      RABBITMQ_USER: guest
-      RABBITMQ_PASSWORD: guest
+  rabbitmq:
+    image: rabbitmq
+    restart: unless-stopped
+    ports:
+      - 5672:5672
+    networks:
+      - network
+
+  test-s3:
+    image: bitnami/minio:latest
+    container_name: test-s3
+    restart: unless-stopped
+    env_file: .env.docker
+    ports:
+      - 9010:9000
+      - 9011:9001
+    networks:
+      - network
 
   test-postgres:
     image: postgres:15
     restart: unless-stopped
     ports:
       - 5433:5432
-    environment:
-      POSTGRES_DB: syncmaster
-      POSTGRES_USER: user
-      POSTGRES_PASSWORD: secret
+    env_file: .env.docker
     networks:
       - network
 
@@ -99,7 +90,7 @@ services:
       - network
 
   metastore-hive:
-    image: central-mirror.services.mts.ru/postgres:15-alpine
+    image: postgres:15
     restart: unless-stopped
     environment:
       POSTGRES_DB: metastore
@@ -111,8 +102,9 @@ services:
       - network
 
   test-hive:
-    image: mtsrus/hadoop:hadoop2.7.3-hive2.3.9
+    image: mtsrus/hadoop:hadoop2-hdfs
     restart: unless-stopped
+    hostname: test-hive
     depends_on:
       - metastore-hive
     ports:
@@ -124,8 +116,9 @@ services:
       - 10002:10002  # Hive server Admin UI
       - 19888:19888  # MapReduce JobServer History UI
       - 9083:9083  # Hive Metastore server
+      - 9864:9864 # Datanode UI
     environment:
-      WITH_HIVE_SERVER: 'false'  # оставляем только сервер метастора, сам Hive нам не требуется, не тратим на него ресурсы
+      WITH_HIVE_SERVER: 'false'  # We leave only the metastore server, we don’t need Hive itself, we don’t waste resources on it
       HIVE_METASTORE_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore
       HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver
       HIVE_METASTORE_DB_USER: test_hive
@@ -136,6 +129,5 @@ services:
 networks:
   network:
 
-
 volumes:
   postgres_data:

docker/backend.dockerfile

@@ -1,16 +1,32 @@
-FROM sregistry.mts.ru/bigdata/platform/docker-images/python:3.11-slim
+FROM python:3.11-slim
 
-RUN microdnf install java-17-openjdk libnghttp2-1.33.0-3.el8_2.1  # fix version https://github.com/nghttp2/nghttp2/issues/2003
+RUN apt-get update && apt-get install -y \
+        libkrb5-dev \
+        libsasl2-dev \
+        libsasl2-modules-gssapi-mit \
+        libsasl2-modules-ldap \
+        libsasl2-modules \
+        openjdk-17-jdk \
+        libssl-dev \
+        libldap2-dev \
+        autoconf \
+        gcc \
+        g++ \
+        make \
+        libnghttp2-dev \
+        libffi-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir --timeout 3 --retries 3 poetry \
     && poetry config virtualenvs.create false
 
 WORKDIR /syncmaster
 
 COPY ./pyproject.toml ./poetry.lock* /syncmaster/
-RUN poetry export -f requirements.txt --with backend,dev,test --output /syncmaster/requirements.txt \
-    --without-hashes --with-credentials --without-urls \
-    && pip install --timeout 5 --retries 5 --no-cache-dir -r /syncmaster/requirements.txt
+
+RUN pip install --upgrade pip setuptools wheel packaging
+
+RUN poetry install --no-root --no-cache --with backend
 
 COPY ./syncmaster/ /syncmaster/