From b4020b3fffee8ade515878e8766f9953d525dbad Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 14 Jan 2025 10:43:16 -0800 Subject: [PATCH] Enable unit tests for Python 2.6 & 3.4 on Github Actions (#3296) * Use Ubuntu 24 for unit tests runs [Python 2.6 & 3.4] --------- Co-authored-by: narrieta@microsoft (cherry picked from commit 1f3ead28533de79c787b998a2faf696318cc069d) --- .github/workflows/ci_pr.yml | 50 ++++++++++++++++ azurelinuxagent/common/utils/textutil.py | 14 ----- azurelinuxagent/ga/monitor.py | 7 +-- tests/common/osutil/test_default.py | 2 - tests/common/utils/test_text_util.py | 16 ----- tests/lib/tools.py | 8 ++- tests/python_eol/Dockerfile | 74 ++++++++++++++---------- tests/python_eol/README | 7 +++ tests/python_eol/execute_tests.sh | 4 +- tests/python_eol/patch_python_venv.sh | 68 ++++++++++++++++++++++ 10 files changed, 181 insertions(+), 69 deletions(-) create mode 100644 tests/python_eol/README create mode 100755 tests/python_eol/patch_python_venv.sh diff --git a/.github/workflows/ci_pr.yml b/.github/workflows/ci_pr.yml index 6be27f67e..a91c8efd6 100644 --- a/.github/workflows/ci_pr.yml +++ b/.github/workflows/ci_pr.yml @@ -8,6 +8,56 @@ on: workflow_dispatch: jobs: + test-python-2_6-and-3_4-versions: + + strategy: + fail-fast: false + matrix: + include: + - python-version: "2.6" + - python-version: "3.4" + + name: "Python ${{ matrix.python-version }} Unit Tests" + runs-on: ubuntu-20.04 + container: + image: ubuntu:24.04 + volumes: + - /home/waagent:/home/waagent + defaults: + run: + shell: bash -l {0} + + env: + NOSEOPTS: "--verbose" + + steps: + - uses: actions/checkout@v3 + + - name: Install Python ${{ matrix.python-version }} Virtual Environment + run: | + apt-get update + apt-get install -y curl bzip2 sudo + curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2 + sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory / + # + # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now. + # + if [[ "${{ matrix.python-version }}" == "2.6" ]]; then + apt-get -y install python3 + fi + # + # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments. + # + if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then + ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}" + fi + + - name: Execute Tests + run: | + source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate + ./ci/nosetests.sh + exit $? + test-python-2_7: strategy: diff --git a/azurelinuxagent/common/utils/textutil.py b/azurelinuxagent/common/utils/textutil.py index d83d58972..beebb9bc9 100644 --- a/azurelinuxagent/common/utils/textutil.py +++ b/azurelinuxagent/common/utils/textutil.py @@ -17,7 +17,6 @@ # Requires Python 2.6+ and Openssl 1.0+ import base64 -import hashlib import re import struct import sys @@ -385,19 +384,6 @@ def is_str_empty(s): return is_str_none_or_whitespace(s) or is_str_none_or_whitespace(s.rstrip(' \t\r\n\0')) -def hash_strings(string_list): - """ - Compute a cryptographic hash of a list of strings - - :param string_list: The strings to be hashed - :return: The cryptographic hash (digest) of the strings in the order provided - """ - sha1_hash = hashlib.sha1() - for item in string_list: - sha1_hash.update(item.encode()) - return sha1_hash.digest() - - def format_memory_value(unit, value): units = {'bytes': 1, 'kilobytes': 1024, 'megabytes': 1024*1024, 'gigabytes': 1024*1024*1024} diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py index c1340ed69..c71447a7d 100644 --- a/azurelinuxagent/ga/monitor.py +++ b/azurelinuxagent/ga/monitor.py @@ -34,7 +34,6 @@ from azurelinuxagent.common.protocol.imds import get_imds_client from azurelinuxagent.common.protocol.util import get_protocol_util from azurelinuxagent.common.utils.restutil import IOErrorCounter -from azurelinuxagent.common.utils.textutil import hash_strings from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION from azurelinuxagent.ga.periodic_operation import PeriodicOperation @@ -145,9 +144,9 @@ def log_network_configuration(self): def _operation(self): raw_route_list = self.osutil.read_route_table() - digest = hash_strings(raw_route_list) - if digest != self.last_route_table_hash: - self.last_route_table_hash = digest + route_table_hash = ":".join([str(hash(r)) for r in raw_route_list]) + if route_table_hash != self.last_route_table_hash: + self.last_route_table_hash = route_table_hash route_list = self.osutil.get_list_of_routes(raw_route_list) logger.info("Route table: [{0}]".format(",".join(map(networkutil.RouteEntry.to_json, route_list)))) diff --git a/tests/common/osutil/test_default.py b/tests/common/osutil/test_default.py index 5b4a2f307..93c314d32 100644 --- a/tests/common/osutil/test_default.py +++ b/tests/common/osutil/test_default.py @@ -132,14 +132,12 @@ def test_valid_routes(self): 'eth0\t10813FA8\tC1BB910A\t000F\t0\t0\t0\tFFFFFFFF\t0\t0\t0 \n' \ 'eth0\tFEA9FEA9\tC1BB910A\t0007\t0\t0\t0\tFFFFFFFF\t0\t0\t0 \n' \ 'docker0\t002BA8C0\t00000000\t0001\t0\t0\t10\t00FFFFFF\t0\t0\t0 \n' - known_sha1_hash = b'\x1e\xd1k\xae[\xf8\x9b\x1a\x13\xd0\xbbT\xa4\xe3Y\xa3\xdd\x0b\xbd\xa9' mo = mock.mock_open(read_data=routing_table) with patch(open_patch(), mo): raw_route_list = osutil.DefaultOSUtil().read_route_table() self.assertEqual(len(raw_route_list), 6) - self.assertEqual(textutil.hash_strings(raw_route_list), known_sha1_hash) route_list = osutil.DefaultOSUtil().get_list_of_routes(raw_route_list) diff --git a/tests/common/utils/test_text_util.py b/tests/common/utils/test_text_util.py index 531f03752..0c08a6e15 100644 --- a/tests/common/utils/test_text_util.py +++ b/tests/common/utils/test_text_util.py @@ -15,7 +15,6 @@ # Requires Python 2.6+ and Openssl 1.0+ # -import hashlib import unittest import azurelinuxagent.common.utils.textutil as textutil @@ -116,21 +115,6 @@ def test_compress(self): result = textutil.compress('[stdout]\nHello World\n\n[stderr]\n\n') self.assertEqual('eJyLLi5JyS8tieXySM3JyVcIzy/KSeHiigaKphYVxXJxAQDAYQr2', result) - def test_hash_empty_list(self): - result = textutil.hash_strings([]) - self.assertEqual(b'\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t', result) - - def test_hash_list(self): - test_list = ["abc", "123"] - result_from_list = textutil.hash_strings(test_list) - - test_string = "".join(test_list) - hash_from_string = hashlib.sha1() - hash_from_string.update(test_string.encode()) - - self.assertEqual(result_from_list, hash_from_string.digest()) - self.assertEqual(hash_from_string.hexdigest(), '6367c48dd193d56ea7b0baad25b19455e529f5ee') - def test_empty_strings(self): self.assertTrue(textutil.is_str_none_or_whitespace(None)) self.assertTrue(textutil.is_str_none_or_whitespace(' ')) diff --git a/tests/lib/tools.py b/tests/lib/tools.py index fc1f72150..127f7c796 100644 --- a/tests/lib/tools.py +++ b/tests/lib/tools.py @@ -411,8 +411,12 @@ def emulate_assertListEqual(self, seq1, seq2, msg=None, seq_type=None): diffMsg = '\n' + '\n'.join( difflib.ndiff(pprint.pformat(seq1).splitlines(), pprint.pformat(seq2).splitlines())) - standardMsg = self._truncateMessage(standardMsg, diffMsg) - msg = self._formatMessage(msg, standardMsg) + # _truncateMessage and _formatMessage are not defined on Python 2.6; output the entire diff in that case + if sys.version_info < (2, 7): + msg = standardMsg + "\n****************************************\n" + diffMsg + else: + standardMsg = self._truncateMessage(standardMsg, diffMsg) + msg = self._formatMessage(msg, standardMsg) self.fail(msg) def emulate_assertIsInstance(self, obj, object_type, msg=None): diff --git a/tests/python_eol/Dockerfile b/tests/python_eol/Dockerfile index 79adf8ab8..83cc7380e 100644 --- a/tests/python_eol/Dockerfile +++ b/tests/python_eol/Dockerfile @@ -18,41 +18,57 @@ # * Run unit tests: docker run --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-tests # * Run tests that require root: docker run --user root --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-sudo-tests # -FROM ubuntu:16.04 +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:24.04 ARG PYTHON_VERSION LABEL description="Test environment for WALinuxAgent" SHELL ["/bin/bash", "-c"] -RUN \ - apt-get update && \ - apt-get -y install curl bzip2 sudo && \ - groupadd waagent && \ - useradd --shell /bin/bash --create-home -g waagent waagent && \ - curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2 && \ - tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory / && \ - rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2 && \ - echo $'\ - \n\ - cd /home/waagent \n\ - source /home/waagent/virtualenv/python'${PYTHON_VERSION}/bin/activate$' \n\ - function run-tests { \n\ - nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests \n\ - } \n\ - function run-sudo-tests { \n\ - nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py \n\ - } \n\ - ' | tee -a /home/waagent/.profile >> ~/.profile && \ - sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile && \ - : - -# -# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now. -# -RUN \ - if [[ "${PYTHON_VERSION}" == "2.6" ]]; then \ - apt-get -y install python3; \ +COPY patch_python_venv.sh /tmp/patch_python_venv.sh + +RUN <<.. + # + # Install the Python venv + # + apt-get update + apt-get -y install curl bzip2 sudo + groupadd waagent + useradd --shell /bin/bash --create-home -g waagent waagent + curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2 + tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory / + chown -R waagent:waagent /home/waagent # The UID:GID in the tarball may not match those of the user, so we need to fix that. + rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2 + + # + # Add the convenience functions to the profiles of waagent and root + # + (cat << ... +cd /home/waagent +source /home/waagent/virtualenv/python${PYTHON_VERSION}/bin/activate +function run-tests { + nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests +} +function run-sudo-tests { + nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py +} +... + ) | tee -a /home/waagent/.profile >> ~/.profile + sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile + + # + # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now. + # + if [[ "${PYTHON_VERSION}" == "2.6" ]]; then + apt-get -y install python3 + fi + + # + # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments. + # + if [[ "${PYTHON_VERSION}" =~ ^2\.6|3\.4$ ]]; then + /tmp/patch_python_venv.sh "${PYTHON_VERSION}" fi +.. USER waagent:waagent diff --git a/tests/python_eol/README b/tests/python_eol/README new file mode 100644 index 000000000..d14b2deac --- /dev/null +++ b/tests/python_eol/README @@ -0,0 +1,7 @@ +This directory contains a DevOps Azure Pipeline to execute the unit tests for Python 2.6 and 3.4. + +Currently those tests are executed using Github Actions, using .github/workflows/ci_pr.yml. The +setup done by ci_pr.yml is practically identical to the setup done by this Dockerfile, with minor +differences to account for the behavior of Github Actions and Azure Pipelines. + +NOTE: ci_pr.yml also depends on the patch_python_venv.sh script. diff --git a/tests/python_eol/execute_tests.sh b/tests/python_eol/execute_tests.sh index 4cab4c220..5bf01080a 100755 --- a/tests/python_eol/execute_tests.sh +++ b/tests/python_eol/execute_tests.sh @@ -15,9 +15,9 @@ CONTAINER_SOURCES_DIRECTORY="/home/waagent/WALinuxAgent" NOSETESTS_OPTIONS="--verbose --with-xunit" # -# Give ownership of the logs directory to 'waagent' (UID 1000) +# Give ownership of the logs directory to 'waagent' (UID 1001) # -sudo chown 1000 "$LOGS_DIRECTORY" +sudo chown 1001 "$LOGS_DIRECTORY" # # Give the current user access to the Docker daemon diff --git a/tests/python_eol/patch_python_venv.sh b/tests/python_eol/patch_python_venv.sh new file mode 100755 index 000000000..16a585645 --- /dev/null +++ b/tests/python_eol/patch_python_venv.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# The python 2.6 and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16. +# Modules like hashlib and ssl will fail to import on more recent versions of Ubuntu. The Agent uses classes HTTPSConnection and HTTPS, which depend on the ssl module. Those classes +# are added conditionally on the import of ssl on httplib.py and http/client.py with code similar to: +# +# try: +# import ssl +# except ImportError: +# pass +# else: +# class HTTPSConnection(HTTPConnection):... +# class HTTPS(HTTP):... +# def FakeSocket (sock, sslobj):... +# +# Since the import fails, the classes will be undefined. To work around that, we define dummy items that raise NotImplementedError. The unit tests mock those classes anyway, so the +# actual implementation does not really matter. +# +set -euo pipefail + +if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|3\.4$ ]]; then + echo "Usage: patch_python_venv.sh 2.6|3.4" + exit 1 +fi + +PYTHON_VERSION=$1 + +if [[ "${PYTHON_VERSION}" == "2.6" ]]; then + cat >> /opt/python/2.6.9/lib/python2.6/httplib.py << ... + +# Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries +class HTTPSConnection(HTTPConnection): + default_port = HTTPS_PORT + + def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + raise NotImplementedError() + + def connect(self): + raise NotImplementedError() + +__all__.append("HTTPSConnection") + +class HTTPS(HTTP): + _connection_class = HTTPSConnection + + def __init__(self, host='', port=None, key_file=None, cert_file=None, strict=None): + raise NotImplementedError() + +def FakeSocket (sock, sslobj): + raise NotImplementedError() +... + +elif [[ "${PYTHON_VERSION}" == "3.4" ]]; then + cat >> /opt/python/3.4.8/lib/python3.4/http/client.py << ... + +# Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries +class HTTPSConnection(HTTPConnection): + default_port = HTTPS_PORT + + def __init__(self, host, port=None, key_file=None, cert_file=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, context=None, check_hostname=None): + raise NotImplementedError() + + def connect(self): + raise NotImplementedError() + +__all__.append("HTTPSConnection") +... +fi