Skip to content

SNOWVATION: Replace parameter boilerplate with utility classes. #13571

SNOWVATION: Replace parameter boilerplate with utility classes.

SNOWVATION: Replace parameter boilerplate with utility classes. #13571

Workflow file for this run

name: precommit test
on:
push:
branches: [ main ]
pull_request:
branches: '**'
workflow_dispatch:
inputs:
logLevel:
default: warning
description: "Log level"
required: true
tags:
description: "Test scenario tags"
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
name: Check linting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Display Python version
run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Run fix_lint
run: python -m tox -e fix_lint
type_checking:
name: Type Checking
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Display Python version
run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Run pyright on Selected Files
run: python -m tox -e pyright
build:
needs: lint
name: Build Wheel File
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip wheel
- name: Generate wheel
run: python -m pip wheel -v -w dist --no-deps .
- name: Show wheels generated
run: ls -lh dist
- uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: wheel
path: dist/
test:
name: Test py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
python-version: ["3.9", "3.10", "3.11"]
cloud-provider: [aws, gcp, azure]
exclude:
# only run macos with aws py3.9 for doctest
- os: macos-latest
python-version: "3.10"
- os: macos-latest
python-version: "3.11"
- os: macos-latest
python-version: "3.9"
cloud-provider: azure
- os: macos-latest
python-version: "3.9"
cloud-provider: gcp
# only run ubuntu with py3.9 on aws and py3.10 on azure
- os: ubuntu-latest-64-cores
python-version: "3.11"
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: azure
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: gcp
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: aws
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: gcp
# only run windows with py3.10 on gcp
- os: windows-latest-64-cores
python-version: "3.9"
- os: windows-latest-64-cores
python-version: "3.10"
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: aws
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: azure
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Add protoc to Windows path
if: runner.os == 'Windows'
run: |
echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
# we only run doctest on macos
- if: ${{ matrix.os == 'macos-latest' }}
name: Run doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
# do not run other tests for macos
- if: ${{ matrix.os != 'macos-latest' }}
name: Run tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
path: |
.tox/.coverage
.tox/coverage.xml
test-local-testing:
name: Test Local Testing Module py-${{ matrix.os }}-${{ matrix.python-version }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest, ubuntu-latest]
python-version: [ "3.8", "3.9", "3.10", "3.11" ]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Add protoc to Windows path
if: runner.os == 'Windows'
run: |
echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run tests
run: python -m tox -e "py${PYTHON_VERSION/\./}-local"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-local-testing
path: |
.tox/.coverage
.tox/coverage.xml
test-ast-encoding:
name: Test AST Encoding py-${{ matrix.os }}-${{ matrix.python-version }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest, ubuntu-latest]
# TODO SNOW-1763185 use py39 by default
# Protobuf encoding differs per python version, use 3.8 by default.
python-version: [ "3.8"]
cloud-provider: [aws]
protobuf-version: ["3.20.1"]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install protobuf
run: python -m pip install protobuf==${{ matrix.protobuf-version }}
- name: Install tox
run: python -m pip install tox
- name: Set up Windows environment
if: runner.os == 'Windows'
# windows timezone management is broken, that's why we need a special fix here
run: |
"TZ=EST" >> $env:GITHUB_ENV
echo "TZ=EST" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Run tests
# TODO: SNOW-1763186 use pytest marker to test multiple TZs
run: TZ=EST python -m tox -e "py${PYTHON_VERSION/\./}-ast"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
TZ: EST
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
path: |
.tox/.coverage
.tox/coverage.xml
doctest-ast-phase0:
name: Doctest AST (phase0) py-${{ matrix.os }}-${{ matrix.python-version }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ macos-latest ]
python-version: [ "3.9"]
cloud-provider: [ aws ]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run doctests for AST (I/II)
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --enable_ast
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
test-ast-phase0:
name: Test AST enabled (phase0) py-${{ matrix.os }}-${{ matrix.python-version }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-latest ]
python-version: [ "3.9", "3.11" ]
cloud-provider: [ aws ]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
- name: Run tests for AST (II/II)
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short --enable_ast
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
path: |
.tox/.coverage
.tox/coverage.xml
test-snowpark-pandas:
name: Test modin-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
needs: build
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
python-version: [ "3.9", "3.10", "3.11" ]
cloud-provider: [aws, gcp, azure]
exclude:
# only run macos with aws py3.9 for doctest
- os: macos-latest
python-version: "3.10"
- os: macos-latest
python-version: "3.11"
- os: macos-latest
python-version: "3.9"
cloud-provider: azure
- os: macos-latest
python-version: "3.9"
cloud-provider: gcp
# only run ubuntu with py3.9 on aws and py3.10 on azure
- os: ubuntu-latest-64-cores
python-version: "3.11"
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: azure
- os: ubuntu-latest-64-cores
python-version: "3.9"
cloud-provider: gcp
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: aws
- os: ubuntu-latest-64-cores
python-version: "3.10"
cloud-provider: gcp
# only run windows with py3.10 on gcp
- os: windows-latest-64-cores
python-version: "3.9"
- os: windows-latest-64-cores
python-version: "3.10"
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: aws
- os: windows-latest-64-cores
python-version: "3.11"
cloud-provider: azure
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Add protoc to Windows path
if: runner.os == 'Windows'
run: |
echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install tox
run: python -m pip install tox
# only run doctest for macos on aws
- if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }}
name: Run Snowpark pandas API doctests
run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
shell: bash
# do not run other tests for macos on aws
- if: ${{ !(matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws') }}
name: Run Snowpark pandas API tests (excluding doctests)
run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasnotdoctest-modin-ci"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
env:
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
- uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-pandas-testing
path: |
.tox/.coverage
.tox/coverage.xml
test-modin-extra-without-pandas-extra:
name: Test modin extra without pandas extra py-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}.
needs: build
runs-on: ${{ matrix.os.image_name }}
strategy:
fail-fast: false
matrix:
os:
- image_name: macos-latest
download_name: macos
python-version: ["3.9"]
cloud-provider: [aws]
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Decrypt parameters.py
shell: bash
run: .github/scripts/decrypt_parameters.sh
env:
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
- name: Download wheel(s)
uses: actions/download-artifact@v4
with:
name: wheel
path: dist
- name: Show wheels downloaded
run: ls -lh dist
shell: bash
- name: Upgrade setuptools, pip and wheel
run: python -m pip install -U setuptools pip wheel
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Install tox
run: python -m pip install tox
- name: Run all tests
run: python -m tox -e "modin_extra_without_pandas_extra"
env:
PYTHON_VERSION: ${{ matrix.python-version }}
cloud_provider: ${{ matrix.cloud-provider }}
PYTEST_ADDOPTS: --color=yes --tb=short
TOX_PARALLEL_NO_SPINNER: 1
shell: bash
- name: Combine coverages
run: python -m tox -e coverage --skip-missing-interpreters false
shell: bash
- uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: coverage_modin_extra_without_pandas_extra
path: |
.tox/.coverage
.tox/coverage.xml
combine-coverage:
if: ${{ success() || failure() }}
name: Combine coverage
needs:
- test
- test-local-testing
- test-snowpark-pandas
- test-modin-extra-without-pandas-extra
- test-ast-encoding
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/download-artifact@v4
with:
path: artifacts
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Display Python version
run: python -c "import sys; print(sys.version)"
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Collect all coverages to one dir
run: |
python -c '
from pathlib import Path
import shutil
src_dir = Path("artifacts")
dst_dir = Path(".") / ".tox"
dst_dir.mkdir()
for src_file in src_dir.glob("*/.coverage"):
dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
print("{} copy to {}".format(src_file, dst_file))
shutil.copy(str(src_file), str(dst_file))'
- name: Combine coverages
run: python -m tox -e coverage
- name: Publish html coverage
uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: overall_cov_html
path: .tox/htmlcov
- name: Publish xml coverage
uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: overall_cov_xml
path: .tox/coverage.xml
- uses: codecov/codecov-action@v4
with:
file: .tox/coverage.xml
token: ${{ secrets.CODECOV_TOKEN }}
- name: Show coverage diff
run: |
pip install diff_cover
git log -n 5 --pretty=oneline
git fetch -f origin main:main || true
diff-cover .tox/coverage.xml --compare-branch=main --fail-under=95
doc:
needs: lint
name: Build Doc
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Upgrade setuptools and pip
run: python -m pip install -U setuptools pip
- name: Install tox
run: python -m pip install tox
- name: Install protoc
shell: bash
run: .github/scripts/install_protoc.sh
- name: Build document
run: python -m tox -e docs
- name: Upload html files
uses: actions/upload-artifact@v4
with:
include-hidden-files: true
name: docs
path: .tox/docs_out