Enable unit tests for Python 2.6 & 3.4 on Github Actions (#3296) (#3298)

* Use Ubuntu 24 for unit tests runs [Python 2.6 & 3.4]
---------

Co-authored-by: narrieta@microsoft <narrieta>
(cherry picked from commit 1f3ead2)

.github/workflows/ci_pr.yml

@@ -8,6 +8,56 @@ on:
   workflow_dispatch:
 
 jobs:
+  test-python-2_6-and-3_4-versions:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - python-version: "2.6"
+          - python-version: "3.4"
+
+    name: "Python ${{ matrix.python-version }} Unit Tests"
+    runs-on: ubuntu-20.04
+    container:
+      image: ubuntu:24.04
+      volumes:
+        - /home/waagent:/home/waagent
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      NOSEOPTS: "--verbose"
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Install Python ${{ matrix.python-version }} Virtual Environment
+      run: |
+        apt-get update
+        apt-get install -y curl bzip2 sudo
+        curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
+        sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
+        #
+        # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
+        #
+        if [[ "${{ matrix.python-version }}" == "2.6" ]]; then
+            apt-get -y install python3
+        fi
+        #
+        # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
+        #
+        if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then
+          ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
+        fi
+
+    - name: Execute Tests
+      run: |
+        source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
+        ./ci/nosetests.sh
+        exit $?
+
   test-python-2_7:
 
     strategy:

azurelinuxagent/common/utils/textutil.py

@@ -17,7 +17,6 @@
 # Requires Python 2.6+ and Openssl 1.0+
 
 import base64
-import hashlib
 import re
 import struct
 import sys
@@ -385,19 +384,6 @@ def is_str_empty(s):
     return is_str_none_or_whitespace(s) or is_str_none_or_whitespace(s.rstrip(' \t\r\n\0'))
 
 
-def hash_strings(string_list):
-    """
-    Compute a cryptographic hash of a list of strings
-
-    :param string_list: The strings to be hashed
-    :return: The cryptographic hash (digest) of the strings in the order provided
-    """
-    sha1_hash = hashlib.sha1()
-    for item in string_list:
-        sha1_hash.update(item.encode())
-    return sha1_hash.digest()
-
-
 def format_memory_value(unit, value):
     units = {'bytes': 1, 'kilobytes': 1024, 'megabytes': 1024*1024, 'gigabytes': 1024*1024*1024}
 

azurelinuxagent/ga/monitor.py

@@ -34,7 +34,6 @@
 from azurelinuxagent.common.protocol.imds import get_imds_client
 from azurelinuxagent.common.protocol.util import get_protocol_util
 from azurelinuxagent.common.utils.restutil import IOErrorCounter
-from azurelinuxagent.common.utils.textutil import hash_strings
 from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION
 from azurelinuxagent.ga.periodic_operation import PeriodicOperation
 
@@ -145,9 +144,9 @@ def log_network_configuration(self):
 
     def _operation(self):
         raw_route_list = self.osutil.read_route_table()
-        digest = hash_strings(raw_route_list)
-        if digest != self.last_route_table_hash:
-            self.last_route_table_hash = digest
+        route_table_hash = ":".join([str(hash(r)) for r in raw_route_list])
+        if route_table_hash != self.last_route_table_hash:
+            self.last_route_table_hash = route_table_hash
             route_list = self.osutil.get_list_of_routes(raw_route_list)
             logger.info("Route table: [{0}]".format(",".join(map(networkutil.RouteEntry.to_json, route_list))))
 

tests/common/osutil/test_default.py

@@ -132,14 +132,12 @@ def test_valid_routes(self):
             'eth0\t10813FA8\tC1BB910A\t000F\t0\t0\t0\tFFFFFFFF\t0\t0\t0    \n' \
             'eth0\tFEA9FEA9\tC1BB910A\t0007\t0\t0\t0\tFFFFFFFF\t0\t0\t0    \n' \
             'docker0\t002BA8C0\t00000000\t0001\t0\t0\t10\t00FFFFFF\t0\t0\t0    \n'
-        known_sha1_hash = b'\x1e\xd1k\xae[\xf8\x9b\x1a\x13\xd0\xbbT\xa4\xe3Y\xa3\xdd\x0b\xbd\xa9'
 
         mo = mock.mock_open(read_data=routing_table)
         with patch(open_patch(), mo):
             raw_route_list = osutil.DefaultOSUtil().read_route_table()
 
         self.assertEqual(len(raw_route_list), 6)
-        self.assertEqual(textutil.hash_strings(raw_route_list), known_sha1_hash)
 
         route_list = osutil.DefaultOSUtil().get_list_of_routes(raw_route_list)
 

tests/common/utils/test_text_util.py

@@ -15,7 +15,6 @@
 # Requires Python 2.6+ and Openssl 1.0+
 #
 
-import hashlib
 import unittest
 
 import azurelinuxagent.common.utils.textutil as textutil
@@ -116,21 +115,6 @@ def test_compress(self):
         result = textutil.compress('[stdout]\nHello World\n\n[stderr]\n\n')
         self.assertEqual('eJyLLi5JyS8tieXySM3JyVcIzy/KSeHiigaKphYVxXJxAQDAYQr2', result)
 
-    def test_hash_empty_list(self):
-        result = textutil.hash_strings([])
-        self.assertEqual(b'\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t', result)
-
-    def test_hash_list(self):
-        test_list = ["abc", "123"]
-        result_from_list = textutil.hash_strings(test_list)
-
-        test_string = "".join(test_list)
-        hash_from_string = hashlib.sha1()
-        hash_from_string.update(test_string.encode())
-
-        self.assertEqual(result_from_list, hash_from_string.digest())
-        self.assertEqual(hash_from_string.hexdigest(), '6367c48dd193d56ea7b0baad25b19455e529f5ee')
-
     def test_empty_strings(self):
         self.assertTrue(textutil.is_str_none_or_whitespace(None))
         self.assertTrue(textutil.is_str_none_or_whitespace(' '))

tests/lib/tools.py

@@ -411,8 +411,12 @@ def emulate_assertListEqual(self, seq1, seq2, msg=None, seq_type=None):
         diffMsg = '\n' + '\n'.join(
             difflib.ndiff(pprint.pformat(seq1).splitlines(),
                           pprint.pformat(seq2).splitlines()))
-        standardMsg = self._truncateMessage(standardMsg, diffMsg)
-        msg = self._formatMessage(msg, standardMsg)
+        # _truncateMessage and _formatMessage are not defined on Python 2.6; output the entire diff in that case
+        if sys.version_info < (2, 7):
+            msg = standardMsg + "\n****************************************\n" +  diffMsg
+        else:
+            standardMsg = self._truncateMessage(standardMsg, diffMsg)
+            msg = self._formatMessage(msg, standardMsg)
         self.fail(msg)
 
     def emulate_assertIsInstance(self, obj, object_type, msg=None):

tests/python_eol/Dockerfile

@@ -18,41 +18,57 @@
 #    * Run unit tests:               docker run --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-tests
 #    * Run tests that require root:  docker run --user root --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-sudo-tests
 #
-FROM ubuntu:16.04
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:24.04
 ARG PYTHON_VERSION
 LABEL description="Test environment for WALinuxAgent"
 
 SHELL ["/bin/bash", "-c"]
 
-RUN \
-    apt-get update && \
-    apt-get -y install curl bzip2 sudo && \
-    groupadd waagent && \
-    useradd --shell /bin/bash --create-home -g waagent waagent && \
-    curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2 && \
-    tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory / && \
-    rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2 && \
-    echo $'\
-        \n\
-        cd /home/waagent \n\
-        source /home/waagent/virtualenv/python'${PYTHON_VERSION}/bin/activate$' \n\
-        function run-tests { \n\
-            nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests \n\
-        } \n\
-        function run-sudo-tests { \n\
-            nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py \n\
-        } \n\
-        ' | tee -a /home/waagent/.profile >> ~/.profile && \
-    sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile && \
-    :
-
-#
-# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
-#
-RUN \
-    if [[ "${PYTHON_VERSION}" == "2.6" ]]; then \
-      apt-get -y install python3; \
+COPY patch_python_venv.sh /tmp/patch_python_venv.sh
+
+RUN <<..
+    #
+    # Install the Python venv
+    #
+    apt-get update
+    apt-get -y install curl bzip2 sudo
+    groupadd waagent
+    useradd --shell /bin/bash --create-home -g waagent waagent
+    curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2
+    tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory /
+    chown -R waagent:waagent /home/waagent  # The UID:GID in the tarball may not match those of the user, so we need to fix that.
+    rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2
+
+    #
+    # Add the convenience functions to the profiles of waagent and root
+    #
+    (cat << ...
+cd /home/waagent
+source /home/waagent/virtualenv/python${PYTHON_VERSION}/bin/activate
+function run-tests {
+    nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests
+}
+function run-sudo-tests {
+    nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py
+}
+...
+    ) | tee -a /home/waagent/.profile >> ~/.profile
+    sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile
+
+    #
+    # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
+    #
+    if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
+        apt-get -y install python3
+    fi
+
+    #
+    # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
+    #
+    if [[ "${PYTHON_VERSION}" =~ ^2\.6|3\.4$ ]]; then
+      /tmp/patch_python_venv.sh "${PYTHON_VERSION}"
     fi
+..
 
 USER waagent:waagent
 

tests/python_eol/README

@@ -0,0 +1,7 @@
+This directory contains a DevOps Azure Pipeline to execute the unit tests for Python 2.6 and 3.4.
+
+Currently those tests are executed using Github Actions, using .github/workflows/ci_pr.yml. The
+setup done by ci_pr.yml is practically identical to the setup done by this Dockerfile, with minor
+differences to account for the behavior of Github Actions and Azure Pipelines.
+
+NOTE: ci_pr.yml also depends on the patch_python_venv.sh script.

tests/python_eol/execute_tests.sh

@@ -15,9 +15,9 @@ CONTAINER_SOURCES_DIRECTORY="/home/waagent/WALinuxAgent"
 NOSETESTS_OPTIONS="--verbose --with-xunit"
 
 #
-# Give ownership of the logs directory to 'waagent' (UID 1000)
+# Give ownership of the logs directory to 'waagent' (UID 1001)
 #
-sudo chown 1000 "$LOGS_DIRECTORY"
+sudo chown 1001 "$LOGS_DIRECTORY"
 
 #
 # Give the current user access to the Docker daemon

tests/python_eol/patch_python_venv.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#
+# The python 2.6 and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16.
+# Modules like hashlib and ssl will fail to import on more recent versions of Ubuntu. The Agent uses classes HTTPSConnection and HTTPS, which depend on the ssl module. Those classes
+# are added conditionally on the import of ssl on httplib.py and http/client.py with code similar to:
+#
+#     try:
+#        import ssl
+#    except ImportError:
+#        pass
+#    else:
+#        class HTTPSConnection(HTTPConnection):...
+#        class HTTPS(HTTP):...
+#        def FakeSocket (sock, sslobj):...
+#
+# Since the import fails, the classes will be undefined. To work around that, we define dummy items that raise NotImplementedError. The unit tests mock those classes anyway, so the
+# actual implementation does not really matter.
+#
+set -euo pipefail
+
+if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|3\.4$ ]]; then
+  echo "Usage: patch_python_venv.sh 2.6|3.4"
+  exit 1
+fi
+
+PYTHON_VERSION=$1
+
+if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
+  cat >> /opt/python/2.6.9/lib/python2.6/httplib.py << ...
+
+# Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries
+class HTTPSConnection(HTTPConnection):
+    default_port = HTTPS_PORT
+
+    def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        raise NotImplementedError()
+
+    def connect(self):
+        raise NotImplementedError()
+
+__all__.append("HTTPSConnection")
+
+class HTTPS(HTTP):
+    _connection_class = HTTPSConnection
+
+    def __init__(self, host='', port=None, key_file=None, cert_file=None, strict=None):
+        raise NotImplementedError()
+
+def FakeSocket (sock, sslobj):
+    raise NotImplementedError()
+...
+
+elif [[ "${PYTHON_VERSION}" == "3.4" ]]; then
+  cat >> /opt/python/3.4.8/lib/python3.4/http/client.py << ...
+
+# Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries
+class HTTPSConnection(HTTPConnection):
+    default_port = HTTPS_PORT
+
+    def __init__(self, host, port=None, key_file=None, cert_file=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, context=None, check_hostname=None):
+        raise NotImplementedError()
+
+    def connect(self):
+        raise NotImplementedError()
+
+__all__.append("HTTPSConnection")
+...
+fi