Skip to content

Commit

Permalink
Cleanup Github Actions workflow for Unit Tests
Browse files Browse the repository at this point in the history
  • Loading branch information
narrieta@microsoft committed Jan 15, 2025
1 parent 1f3ead2 commit 3e92604
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 109 deletions.
163 changes: 62 additions & 101 deletions .github/workflows/ci_pr.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,92 +8,23 @@ on:
workflow_dispatch:

jobs:
test-python-2_6-and-3_4-versions:

strategy:
fail-fast: false
matrix:
include:
- python-version: "2.6"
- python-version: "3.4"

execute-tests:
name: "Python ${{ matrix.python-version }} Unit Tests"
runs-on: ubuntu-20.04
container:
image: ubuntu:24.04
volumes:
- /home/waagent:/home/waagent
defaults:
run:
shell: bash -l {0}

env:
NOSEOPTS: "--verbose"

steps:
- uses: actions/checkout@v3

- name: Install Python ${{ matrix.python-version }} Virtual Environment
run: |
apt-get update
apt-get install -y curl bzip2 sudo
curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
#
# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
#
if [[ "${{ matrix.python-version }}" == "2.6" ]]; then
apt-get -y install python3
fi
#
# The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
#
if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then
./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
fi
- name: Execute Tests
run: |
source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
./ci/nosetests.sh
exit $?
test-python-2_7:

strategy:
fail-fast: false

name: "Python 2.7 Unit Tests"
runs-on: ubuntu-20.04
defaults:
run:
shell: bash -l {0}

env:
NOSEOPTS: "--verbose"

steps:
- uses: actions/checkout@v3

- name: Install Python 2.7
run: |
apt-get update
apt-get install -y curl bzip2 sudo
curl https://dcrdata.blob.core.windows.net/python/python-2.7.tar.bz2 -o python-2.7.tar.bz2
sudo tar xjvf python-2.7.tar.bz2 --directory /
- name: Test with nosetests
run: |
source /home/waagent/virtualenv/python2.7.16/bin/activate
./ci/nosetests.sh
exit $?
test-current-python-versions:

strategy:
fail-fast: false
matrix:
include:
#
# Some of the Python versions we test are not supported by the setup-python Github Action. For those versions, we use a
# pre-built virtual environment.
#
- python-version: "2.6"
use_virtual_environment: true
- python-version: "2.7"
use_virtual_environment: true
- python-version: "3.4"
use_virtual_environment: true
- python-version: "3.5"
# workaround found in https://github.com/actions/setup-python/issues/866
# for issue "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:728)" on Python 3.5
Expand All @@ -106,33 +37,69 @@ jobs:
- python-version: "3.10"
- python-version: "3.11"

name: "Python ${{ matrix.python-version }} Unit Tests"
runs-on: ubuntu-20.04

env:
NOSEOPTS: "--with-timer ${{ matrix.additional-nose-opts }}"

steps:

- name: Checkout WALinuxAgent repo
- name: Checkout WALinuxAgent
uses: actions/checkout@v3

#
# We either install Python and the test dependencies, or download a pre-built virtual environment, depending on the
# use_virtual_environment flag.
#
- name: Setup Python ${{ matrix.python-version }}
if: (!matrix.use_virtual_environment)
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
env:
PIP_TRUSTED_HOST: ${{ matrix.pip_trusted_host }}

- name: Install dependencies
if: (!matrix.use_virtual_environment)
id: install-dependencies
run: |
sudo env "PATH=$PATH" python -m pip install --upgrade pip
sudo env "PATH=$PATH" pip install -r requirements.txt
sudo env "PATH=$PATH" pip install -r test-requirements.txt
sudo env "PATH=$PATH" pip install --upgrade pylint
- name: Setup Python ${{ matrix.python-version }} Virtual Environment
if: matrix.use_virtual_environment
id: install-venv
run: |
sudo apt-get update
sudo apt-get install -y curl bzip2 sudo
curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
#
# The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
#
if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then
sudo ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
fi
#
# Execute the tests
#
- name: Execute Unit Tests
run: |
if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then
./ci/pytest.sh
else
if [[ "${{ matrix.use_virtual_environment}}" == "true" ]]; then
export NOSEOPTS="--verbose ${{ matrix.additional-nose-opts }}" # the pytest version on the venvs does not support the --with-timer option.
source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
else
export NOSEOPTS="--verbose --with-timer ${{ matrix.additional-nose-opts }}"
fi
./ci/nosetests.sh
fi
#
# Execute pylint even when the tests fail (but only if the dependencies were installed successfully)
#
# The virtual environments do not include pylint, so we skip those Python versions.
#
- name: Run pylint
if: (!matrix.use_virtual_environment && (success() || (failure() && steps.install-dependencies.outcome == 'success')))
run: |
#
# List of files/directories to be checked by pylint.
Expand All @@ -153,7 +120,7 @@ jobs:
# * 'no-self-use' ("R0201: Method could be a function") was moved to an optional extension on 3.8 and is no longer used by default. It needs
# to be suppressed for previous versions (3.0-3.7), though.
# * 'contextmanager-generator-missing-cleanup' are false positives if yield is used inside an if-else block for contextmanager generator functions.
# (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html).
# (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html).
# This is not implemented on versions (3.0-3.7) Bad option value 'contextmanager-generator-missing-cleanup' (bad-option-value)
# * 3.9-3.11 will produce "too-many-positional-arguments" for several methods that are having more than 5 args, so we suppress that warning.
# (R0917: Too many positional arguments (8/5) (too-many-positional-arguments))
Expand All @@ -173,16 +140,10 @@ jobs:
pylint $PYLINT_OPTIONS $PYLINT_FILES
- name: Execute Unit Tests
if: success() || (failure() && steps.install-dependencies.outcome == 'success')
run: |
if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then
./ci/pytest.sh
else
./ci/nosetests.sh
fi
- name: Compile Coverage
#
# Lastly, compile code coverage
#
- name: Compile Code Coverage
if: matrix.python-version == '3.9'
run: |
echo looking for coverage files :
Expand All @@ -191,7 +152,7 @@ jobs:
sudo env "PATH=$PATH" coverage xml
sudo env "PATH=$PATH" coverage report
- name: Upload Coverage
- name: Upload Code Coverage
if: matrix.python-version == '3.9'
uses: codecov/codecov-action@v3
with:
Expand Down
6 changes: 3 additions & 3 deletions tests/python_eol/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,14 @@ function run-sudo-tests {
#
# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
#
if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7$ ]]; then
apt-get -y install python3
fi

#
# The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
# The virtual environments for 2.6, 2.7 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
#
if [[ "${PYTHON_VERSION}" =~ ^2\.6|3\.4$ ]]; then
if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7|3\.4$ ]]; then
/tmp/patch_python_venv.sh "${PYTHON_VERSION}"
fi
..
Expand Down
15 changes: 10 additions & 5 deletions tests/python_eol/patch_python_venv.sh
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
#
# The python 2.6 and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16.
# The python 2.6, 2.7, and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16.
# Modules like hashlib and ssl will fail to import on more recent versions of Ubuntu. The Agent uses classes HTTPSConnection and HTTPS, which depend on the ssl module. Those classes
# are added conditionally on the import of ssl on httplib.py and http/client.py with code similar to:
#
Expand All @@ -18,15 +18,20 @@
#
set -euo pipefail

if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|3\.4$ ]]; then
echo "Usage: patch_python_venv.sh 2.6|3.4"
if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|2\.7|3\.4$ ]]; then
echo "Usage: patch_python_venv.sh 2.6|2.7|3.4"
exit 1
fi

PYTHON_VERSION=$1

if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
cat >> /opt/python/2.6.9/lib/python2.6/httplib.py << ...
if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7$ ]]; then
if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
file_to_patch="/opt/python/2.6.9/lib/python2.6/httplib.py"
else
file_to_patch="/opt/python/2.7.16/lib/python2.7/httplib.py"
fi
cat >> "$file_to_patch" << ...
# Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries
class HTTPSConnection(HTTPConnection):
Expand Down

0 comments on commit 3e92604

Please sign in to comment.