test_cornflow_client.yml

name: cornflow-client tests

on:
  pull_request:
    types: [ opened, edited, synchronize, reopened ]
    paths:
      - 'libs/client/**'
      - '.github/workflows/test_cornflow_client.yml'
      - '!libs/client/README.rst'
      - '!libs/client/setup.py'
  push:
    branches:
      - master
      - develop
    paths:
      - 'libs/client/**'
      - '.github/workflows/test_cornflow_client.yml'
      - '!libs/client/README.rst'
      - '!libs/client/setup.py'


jobs:
  unit:
    runs-on: ${{ matrix.os }}
    name: Run unit test on cornflow-client
    defaults:
      run:
        working-directory: ./libs/client
    strategy:
      max-parallel: 21
      matrix:
        python-version: [ 3.8, 3.9, '3.10', '3.11' ]
        os: [ubuntu-latest, macOS-latest, windows-latest]

    steps:
    - uses: actions/checkout@v1
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements-dev.txt
        pip install .
    - name: Test
      run: |
        coverage run --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/unit
        coverage report -m
  
  integration:

    name: Run all test suites on cornflow-client
    needs: unit
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        working-directory: ./libs/client
    strategy:
      max-parallel: 21
      matrix:
        python-version: [3.8, 3.9, '3.10', '3.11']
        os: [ ubuntu-latest ]

    steps:
      - uses: actions/checkout@v1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version:  ${{ matrix.python-version }}
      - name: Copy DAG files
        run: |
          cd ..
          cd ..
          cp -r cornflow-dags/DAG/* cornflow-server/airflow_config/dags/
          cp cornflow-dags/requirements.txt cornflow-server/airflow_config/
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-dev.txt
          pip install .
      - name: Install airflow
        run: |
          cd ..
          cd ..
          cd cornflow-server
          python -m venv afvenv
          source afvenv/bin/activate
          AIRFLOW_VERSION=2.7.1
          PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
          CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}"
          CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
          python -m pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
          python -m pip install -U orloge pulp
          python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
          python -m pip install -U -r airflow_config/requirements.txt
          airflow db init
          airflow users create -u admin -f admin -l admin -r Admin -p admin -e admin@example.org
          airflow webserver -p 8080 &
          airflow scheduler &
          deactivate
          sleep 5s
        env:
          AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT: 0
          AIRFLOW_HOME: "${GITHUB_WORKSPACE}/cornflow-server/airflow_config"
          AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 0
          AIRFLOW__API__AUTH_BACKEND: airflow.api.auth.backend.basic_auth
          AIRFLOW__WEBSERVER__SECRET_KEY: e9adafa751fd35adfc1fdd3285019be15eea0758f76e38e1e37a1154fb36
          AIRFLOW__CORE__LOAD_EXAMPLES: 0
          AIRFLOW_CONN_CF_URI: http://airflow:Airflow_test_password1@localhost:5050
      - name: Run cornflow
        run: |
          cd ..
          cd ..
          cd cornflow-server
          python -m venv cfvenv
          source cfvenv/bin/activate
          python -m pip install -U -r requirements.txt
          CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}"
          python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
          flask db upgrade -d cornflow/migrations/
          flask access_init
          flask register_deployed_dags -r http://127.0.0.1:8080 -u admin -p admin
          flask create_admin_user -u admin -e admin@cornflow.org -p Adminpassword1!
          flask create_service_user -u airflow -e airflow@cornflow.org -p Airflow_test_password1
          flask create_base_user -u user -e user@cornflow.org -p UserPassword1!
          flask register_dag_permissions -o 1
          flask run -p 5050 &
          deactivate
          sleep 5s
        env:
          FLASK_ENV: testing
          FLASK_APP: cornflow.app
          AIRFLOW_URL: http://127.0.0.1:8080
          AIRFLOW_USER: admin
          AIRFLOW_PWD: admin
          SECRET_KEY: THISNEEDSTOBECHANGED
          OPEN_DEPLOYMENT: 1
          AUTH_TYPE: 1
          LOG_LEVEL: 10
          CORNFLOW_SERVICE_USER: airflow
      - name: Run unit tests
        run: |
          coverage run --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/unit
          coverage report -m
      - name: Run integration tests
        run: |
          coverage run -a --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/integration
          coverage report -m
          coverage xml
      - name: Upload coverage to codecov
        uses: codecov/codecov-action@v3
        with:
          flags: client-tests
          token: ${{secrets.CODECOV_TOKEN}}