Skip to content
Python Presto/Hive

fix: requirements/testing.txt to reduce vulnerabilities #41

Sign in to view logs

fix: requirements/testing.txt to reduce vulnerabilities

fix: requirements/testing.txt to reduce vulnerabilities #41

Workflow file for this run

.github/workflows/superset-python-presto-hive.yml at f332f50
# Python Presto/Hive unit tests
name: Python Presto/Hive
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
test-postgres-presto:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
presto:
image: starburstdata/presto:350-e.6
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15433:8080
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F presto
test-postgres-hive:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
UPLOAD_FOLDER: /tmp/.superset/uploads/
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Create csv upload directory
if: steps.check.outcome == 'failure'
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
if: steps.check.outcome == 'failure'
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
if: steps.check.outcome == 'failure'
run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F hive