[DOP-11711] Run tests in github actions

.dockerignore

@@ -12,4 +12,5 @@ __pycache__/
 .idea
 .vscode
 .venv/
-venv/
+venv/
+cached_jars/

.env.docker

@@ -24,7 +24,7 @@ MINIO_ROOT_USER=syncmaster
 MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
 MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
 
-TEST_S3_HOST=s3
+TEST_S3_HOST=test-s3
 TEST_S3_PORT=9000
 TEST_S3_ACCESS_KEY=syncmaster
 TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c

.env.local

@@ -0,0 +1,53 @@
+export TZ=UTC
+
+# Postgres
+export POSTGRES_HOST=localhost
+export POSTGRES_PORT=5432
+export POSTGRES_DB=syncmaster
+export POSTGRES_USER=user
+export POSTGRES_PASSWORD=secret
+
+# RabbitMQ
+export RABBITMQ_HOST=localhost
+export RABBITMQ_PORT=5672
+export RABBITMQ_USER=guest
+export RABBITMQ_PASSWORD=guest
+
+# HDFS
+export HDFS_HOST=localhost
+export HDFS_WEBHDFS_PORT=9870
+export HDFS_IPC_PORT=9820
+
+# S3
+export MINIO_ACCESS_KEY=syncmaster
+export MINIO_ROOT_USER=syncmaster
+export MINIO_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+export MINIO_ROOT_PASSWORD=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+
+export TEST_S3_HOST=localhost
+export TEST_S3_PORT=9010
+export TEST_S3_ACCESS_KEY=syncmaster
+export TEST_S3_SECRET_KEY=a2b6ad842d16f47beaa077b0e50c217a3bded54c
+export TEST_S3_BUCKET=syncmaster
+
+export TEST_POSTGRES_HOST=localhost
+export TEST_POSTGRES_PORT=5433
+export TEST_POSTGRES_USER=user
+export TEST_POSTGRES_PASSWORD=secret
+export TEST_POSTGRES_DB=syncmaster
+
+export TEST_ORACLE_HOST=localhost
+export TEST_ORACLE_PORT=1522
+export TEST_ORACLE_USER=test_user
+export TEST_ORACLE_PASSWORD=test_password
+export TEST_ORACLE_SERVICE_NAME=XEPDB1
+
+export TEST_HIVE_CLUSTER=test-hive
+export TEST_HIVE_USER=hive_user
+export TEST_HIVE_PASSWORD=hive_password
+
+export ENV=LOCAL
+
+export SPARK_CONF_DIR=./syncmaster/tests/spark/hive/conf/
+export HADOOP_CONF_DIR=./syncmaster/tests/spark/hadoop/
+export HIVE_CONF_DIR=./syncmaster/tests/spark/hive/conf/

.github/workflows/unit_test.yml

@@ -0,0 +1,89 @@
+name: Unit Tests
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    branches-ignore:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEFAULT_PYTHON: '3.11'
+
+jobs:
+  tests:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    env:
+      POETRY_VERSION: 1.6.1
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+
+      - name: Install poetry ${{ env.POETRY_VERSION }}
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+
+      - name: Cache poetry
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-tests-unit-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-tests-unit-${{ hashFiles('**/poetry.lock') }}
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-tests-unit-
+            ${{ runner.os }}-python
+            ${{ runner.os }}
+
+      - name: Install dependencies
+        run: |
+          poetry install --no-root --with backend,test
+
+      - name: Start docker compose
+        run: |
+          docker compose -f docker-compose.test.yml down -v --remove-orphans
+          docker compose -f docker-compose.test.yml up -d db --wait --wait-timeout 200
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Run DB Tests.
+        run: |
+          source .env.local
+          mkdir -p reports/
+          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_database
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Run Unit Tests.
+        run: |
+          source .env.local
+          poetry run coverage run -m pytest -vvv -s ./syncmaster/tests/test_unit
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster
+
+      - name: Shutdown
+        if: always()
+        run: |
+          docker compose -f docker-compose.test.yml down -v --remove-orphans
+        env:
+          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-syncmaster

.gitignore

@@ -157,4 +157,4 @@ cython_debug/
 
 **/metastore_db/
 **/spark-warehouse/
-/cached_jars/
+cached_jars/

docker-compose.test.yml

@@ -0,0 +1,133 @@
+version: '3.8'
+services:
+  db:
+    image: postgres:15
+    restart: unless-stopped
+    env_file: .env.docker
+    ports:
+      - 5432:5432
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    networks:
+      - network
+
+  worker:
+    image: syncmaster_worker
+    restart: unless-stopped
+    build:
+      dockerfile: docker/worker.dockerfile
+      context: .
+      network: host
+    networks:
+      - network
+    volumes:
+      - ./cached_jars:/root/.ivy2
+    env_file: .env.docker
+
+  backend:
+    image: syncmaster_back
+    restart: unless-stopped
+    build:
+      dockerfile: docker/backend.dockerfile
+      context: .
+      network: host
+    ports:
+      - 8000:8000
+    networks:
+      - network
+    depends_on:
+      - db
+      - rabbitmq
+      - test-oracle
+      - test-postgres
+      - test-hive
+      - test-s3
+    volumes:
+      - ./cached_jars:/root/.ivy2
+    env_file: .env.docker
+
+
+  rabbitmq:
+    image: rabbitmq
+    restart: unless-stopped
+    ports:
+      - 5672:5672
+    networks:
+      - network
+
+  test-s3:
+    image: bitnami/minio:latest
+    container_name: test-s3
+    restart: unless-stopped
+    env_file: .env.docker
+    ports:
+      - 9010:9000
+      - 9011:9001
+    networks:
+      - network
+
+  test-postgres:
+    image: postgres:15
+    restart: unless-stopped
+    ports:
+      - 5433:5432
+    env_file: .env.docker
+    networks:
+      - network
+
+  test-oracle:
+    image: gvenzl/oracle-xe:slim-faststart
+    restart: unless-stopped
+    ports:
+      - 1522:1521
+    environment:
+      TZ: UTC
+      ORACLE_PASSWORD: test_password
+      ORACLE_DATABASE: test_oracle
+      APP_USER: test_user
+      APP_USER_PASSWORD: test_password
+    networks:
+      - network
+
+  metastore-hive:
+    image: postgres:15
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: metastore
+      POSTGRES_USER: test_hive
+      POSTGRES_PASSWORD: test_hive
+    ports:
+      - 5440:5432
+    networks:
+      - network
+
+  test-hive:
+    image: mtsrus/hadoop:hadoop2-hdfs
+    restart: unless-stopped
+    hostname: test-hive
+    depends_on:
+      - metastore-hive
+    ports:
+      - 9820:9820  # HDFS IPC
+      - 9870:9870  # HDFS WebHDFS
+      - 8088:8088  # Yarn UI
+      - 8042:8042  # NodeManager UI
+      - 10000:10000  # Hive server
+      - 10002:10002  # Hive server Admin UI
+      - 19888:19888  # MapReduce JobServer History UI
+      - 9083:9083  # Hive Metastore server
+      - 9864:9864 # Datanode UI
+    environment:
+      WITH_HIVE_SERVER: 'false'  # We leave only the metastore server, we don’t need Hive itself, we don’t waste resources on it
+      HIVE_METASTORE_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore
+      HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver
+      HIVE_METASTORE_DB_USER: test_hive
+      HIVE_METASTORE_DB_PASSWORD: test_hive
+    networks:
+      - network
+
+networks:
+  network:
+
+volumes:
+  postgres_data: