test_cornflow_server.yml

name: cornflow-server tests

on:
  pull_request:
    types: [ opened, edited, synchronize, reopened ]
    paths: 
      - 'cornflow-server/**'
      - '.github/workflows/test_cornflow_server.yml'
      - '!cornflow-server/docs/**'
      - '!cornflow-server/README.rst'
  push:
    branches:
      - master
      - development
    paths: 
      - 'cornflow-server/**'
      - '!cornflow-server/docs/**'
      - '!cornflow-server/README.rst'

jobs:
  testing:

    name: Run all test suites on cornflow-server
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        working-directory: ./cornflow-server
    strategy:
      max-parallel: 21
      matrix:
        python-version: [3.8, 3.9, '3.10', '3.11']
        os: [ubuntu-latest]

    services:
      postgres:
        image: postgres
        env:
          POSTGRES_DB: cornflow-test
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgresadmin
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432

    steps:
    - uses: actions/checkout@v1
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Copy DAG files
      run: |
        cd ..
        cp -r cornflow-dags/DAG/* cornflow-server/airflow_config/dags/
        cp cornflow-dags/requirements.txt cornflow-server/airflow_config/
    - name: Install dependencies
      run: |
        docker run --volume $GITHUB_WORKSPACE/cornflow-server/airflow_config:/etc/ldap.dist/prepopulate \
        -e SLAPD_DOMAIN=example.org \
        -e SLAPD_PASSWORD=admin \
        -d -p 389:389 dinkel/openldap
        CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}"
        python -m pip install --upgrade pip
        python -m pip install -U -r requirements-dev.txt
        python -m pip uninstall cornflow-client -y
        python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
    - name: Install airflow
      run: |
        python -m venv afvenv
        source afvenv/bin/activate
        AIRFLOW_VERSION=2.7.1
        PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
        CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}"
        CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
        python -m pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
        python -m pip install -U orloge pulp
        python -m pip uninstall cornflow-client -y
        python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
        python -m pip install -U -r airflow_config/requirements.txt
        airflow db init
        airflow users create \
              --username admin \
              --firstname admin \
              --lastname admin \
              --role Admin \
              --password admin \
              --email admin@example.org
        airflow webserver -p 8080 &
        airflow scheduler &
        deactivate
        sleep 5s
      env:
        AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT: 0
        AIRFLOW_HOME: "${GITHUB_WORKSPACE}/cornflow-server/airflow_config"
        AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 0
        AIRFLOW__API__AUTH_BACKEND: airflow.api.auth.backend.basic_auth
        AIRFLOW__WEBSERVER__SECRET_KEY: e9adafa751fd35adfc1fdd3285019be15eea0758f76e38e1e37a1154fb36
        AIRFLOW__CORE__LOAD_EXAMPLES: 0
        AIRFLOW_CONN_CF_URI: http://airflow:Airflow_test_password1@localhost:5050
    - name: Run unit tests
      run: | 
        coverage run --source=./cornflow/ --omit="*/tests/data/*" -m unittest discover -s cornflow/tests/unit
        coverage report -m
      env:
        FLASK_ENV: testing
        OPEN_DEPLOYMENT: 1
        DATABASE_URL: sqlite:///cornflow_test.db
        LOG_LEVEL: 30
        AIRFLOW_USER: notadmin
        AIRFLOW_PWD: notadmin
        CF_ALARMS_ENDPOINT: 1
    - name: Run ldap unit tests
      run: | 
        coverage run -a --source=./cornflow/ -m unittest discover -s cornflow/tests/ldap
        coverage report -m
      env:
        FLASK_ENV: testing
        DATABASE_URL: sqlite:///cornflow_test.db
        AUTH_TYPE: 2
        OPEN_DEPLOYMENT: 1
        LDAP_HOST: ldap://localhost:389
        LDAP_BIND_PASSWORD: admin
        LDAP_GROUP_BASE: dc=example,dc=org
        LDAP_GROUP_OBJECT_CLASS: groupOfNames
        LDAP_PROTOCOL_VERSION: 3
        LDAP_BIND_DN: cn=admin,dc=example,dc=org
        LDAP_USE_TLS: False
        LDAP_USERNAME_ATTRIBUTE: cn
        LDAP_USER_BASE: ou=users,dc=example,dc=org
        LDAP_SERVICE_BASE: ou=service,dc=example,dc=org
        LDAP_EMAIL_ATTRIBUTE: mail
        LDAP_USER_OBJECT_CLASS: inetOrgPerson
        LDAP_GROUP_ATTRIBUTE: cn
        LDAP_GROUP_TO_ROLE_SERVICE: services
        LDAP_GROUP_TO_ROLE_ADMIN: administrators
        LDAP_GROUP_TO_ROLE_VIEWER: viewers
        LDAP_GROUP_TO_ROLE_PLANNER: planners
        LOG_LEVEL: 30
        CORNFLOW_SERVICE_USER: cornflow
    - name: Run postgres unit tests
      run: | 
        coverage run -a --source=./cornflow/ -m unittest cornflow/tests/unit/test_commands.py
        coverage report -m
      env:
        FLASK_ENV: testing
        OPEN_DEPLOYMENT: 1
        DATABASE_URL: postgresql://postgres:postgresadmin@127.0.0.1:5432/cornflow-test
        LOG_LEVEL: 30
        CF_ALARMS_ENDPOINT: 1
    - name: Run integration tests
      run: |
        coverage run -a --source=./cornflow/ -m unittest discover -s cornflow/tests/integration
        coverage report -m
        coverage xml
      env:
        FLASK_ENV: testing
        OPEN_DEPLOYMENT: 1
        DATABASE_URL: sqlite:///cornflow_test.db
        LOG_LEVEL: 30
    - name: Upload coverage to codecov
      uses: codecov/codecov-action@v3
      with:
        flags: server-tests