diff --git a/.github/workflows/build-pkgs.yml b/.github/workflows/build-pkgs.yml index eb70d8d7..f0789df9 100644 --- a/.github/workflows/build-pkgs.yml +++ b/.github/workflows/build-pkgs.yml @@ -51,7 +51,7 @@ jobs: run: rpmlint ${{ steps.rpm.outputs.rpm_dir_path }} - name: Upload artifact - uses: actions/upload-artifact@v4.3.3 + uses: actions/upload-artifact@v4.4.3 with: name: Binary and Source RPMs path: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 522a4e5f..c6b5fc9b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -48,7 +48,7 @@ jobs: # Build and push Docker image # https://github.com/docker/build-push-action name: Build and push Docker image - uses: docker/build-push-action@v5.3.0 + uses: docker/build-push-action@v6.10.0 with: # Only push containers to the registry on GitHub pushes, # not pull requests. GitHub won't let a rogue PR create a container diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 2c9cdc98..bf073a8c 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -4,31 +4,37 @@ on: [push, pull_request] jobs: unit-test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 # 20.04 to allow for Py 3.6 strategy: fail-fast: false matrix: - python-version: ['3.x'] + # Python versions on Rocky 8, Ubuntu 20.04, Rocky 9 + python-version: ['3.6', '3.8', '3.9'] name: Python ${{ matrix.python-version }} test steps: - uses: actions/checkout@v4 - - name: Set up Python + + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Set up dependencies for python-ldap - run: sudo apt-get install libsasl2-dev libldap2-dev libssl-dev + cache: 'pip' + - name: Base requirements for SSM run: pip install -r requirements.txt + - name: Additional requirements for the unit and coverage tests run: pip install -r requirements-test.txt + - name: Pre-test set up run: | export TMPDIR=$PWD/tmp mkdir $TMPDIR export PYTHONPATH=$PYTHONPATH:`pwd -P` cd test + - name: Run unit tests run: coverage run --branch --source=ssm,bin -m unittest discover --buffer + - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4.3.1 + uses: codecov/codecov-action@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86c0ad80..de1e3c08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # See https://pre-commit.com for more information repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v4.1.0 # Python 3.6 compatible hooks: # Python related checks - id: check-ast @@ -13,9 +13,13 @@ repos: files: 'test/.*' # Other checks - id: check-added-large-files + - id: check-case-conflict - id: check-merge-conflict - id: check-yaml - id: debug-statements + - id: detect-private-key + # This file has a test cert and key + exclude: 'test_ssm.py' - id: end-of-file-fixer - id: mixed-line-ending name: Force line endings to LF diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 70e2d56c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -os: linux -language: python -python: - - "2.7" - - "3.8" - -# Cache the dependencies installed by pip -cache: pip -# Avoid pip log from affecting cache -before_cache: rm -fv ~/.cache/pip/log/debug.log - -install: - # Base requirements for ssm - - pip install -r requirements.txt - # Additional requirements for the unit and coverage tests - - pip install -r requirements-test.txt - -# Commands to prepare environment for the test -before_script: - - export TMPDIR=$PWD/tmp - - mkdir $TMPDIR - - export PYTHONPATH=$PYTHONPATH:`pwd -P` - - cd test - -script: coverage run --branch --source=ssm,bin -m unittest discover --buffer - -after_success: - - coveralls - - codecov diff --git a/CHANGELOG b/CHANGELOG index c35d5537..b4145013 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +* Fri Aug 30 2024 Adrian Coveney - 3.4.1-1 + - Improved error logging to store full traceback on unexpected exceptions. + - Changed more code to use pyOpenSSL to improve compatibility with newer OpenSSL versions. + - Added a check to prevent a host certificate being to used for target server encryption. + - Changed which version of exit function is used to avoid edge case. + - Various changes and improvements to build scripts and processes. + * Wed Feb 21 2024 Adrian Coveney - 3.4.0-1 - Fixed compatability with newer versions of OpenSSL that only provide comma separated DNs. - Fixed Python 3 compatability (indirectly fixing EL8+ compatability) by performing explicit diff --git a/Dockerfile b/Dockerfile index 8c6b5e8b..1d8623e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,6 @@ RUN yum -y install epel-release && yum clean all # Then get pip RUN yum -y install python3-pip && yum clean all -# Install the system requirements of python-ldap -RUN yum -y install gcc python3-devel openldap-devel && yum clean all - # Install libffi, a requirement of openssl RUN yum -y install libffi-devel && yum clean all @@ -25,7 +22,7 @@ RUN yum -y install libffi-devel && yum clean all RUN yum -y install openssl && yum clean all # Install the python requirements of SSM -RUN pip install -r requirements-docker.txt +RUN pip install -r requirements.txt # Then install the SSM RUN python3 setup.py install diff --git a/README.md b/README.md index 7cf98716..0764fd0e 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,15 @@ # Secure STOMP Messenger -[![Build Status](https://travis-ci.org/apel/ssm.svg?branch=dev)](https://travis-ci.org/apel/ssm) [![Coverage Status](https://coveralls.io/repos/github/apel/ssm/badge.svg?branch=dev)](https://coveralls.io/github/apel/ssm?branch=dev) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/9d2b1c88ab844f0081e5fafab49b269d)](https://www.codacy.com/gh/apel/ssm/dashboard) [![Maintainability](https://api.codeclimate.com/v1/badges/34aa04f3583afce2ceb2/maintainability)](https://codeclimate.com/github/apel/ssm/maintainability) Secure STOMP Messenger (SSM) is designed to simply send messages using the [STOMP protocol](http://stomp.github.io/) or via the ARGO Messaging Service (AMS). -Messages are signed and may be encrypted during transit. +Messages are signed and may additionally be encrypted during transit. Persistent queues should be used to guarantee delivery. -SSM is written in Python. Packages are available for RHEL 7, and Ubuntu Trusty. +SSM is written in Python 3. Packages are available for EL 8 and 9, and Ubuntu. For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL/SSM). @@ -31,15 +30,11 @@ the RPM for your version of SL, which is available on this page: http://fedoraproject.org/wiki/EPEL You will also need to have the OpenSSL library installed. Other prerequisites are listed below. -The Python STOMP library (N.B. versions between 3.1.1 (inclusive) and 5.0.0 -(exclusive) are currently supported) +The Python STOMP library * `yum install stomppy` The Python AMS library. This is only required if you want to use AMS. See here for details on obtaining an RPM: https://github.com/ARGOeu/argo-ams-library/ -The Python ldap library (N.B. versions before 3.4.0 (exclusive) are currently supported) -* `yum install python-ldap` - Optionally, the Python dirq library (N.B. this is only required if your messages are stored in a dirq structure) * `yum install python-dirq` @@ -99,7 +94,7 @@ Install any missing system packages needed for the SSM: * `apt-get -f install` Install any missing Python requirements that don't have system packages: -* `pip install "stomp.py<5.0.0" dirq` +* `pip install stomp.py dirq` If you wish to run the SSM as a receiver, you will also need to install the python-daemon system package: * `apt-get install python-daemon` @@ -239,7 +234,6 @@ add your messages using the `add` method. * `yum remove stomppy` * `yum remove python-daemon` -* `yum remove python-ldap` * `rm -rf /var/spool/apel` * `rm -rf /var/log/apel` diff --git a/apel-ssm.spec b/apel-ssm.spec index 82628a78..c217da04 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -1,10 +1,12 @@ +%define __python /usr/bin/python3 + # Conditionally define python_sitelib %if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} %endif Name: apel-ssm -Version: 3.4.0 +Version: 3.4.1 %define releasenumber 1 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger @@ -21,7 +23,7 @@ BuildArch: noarch BuildRequires: python-devel %endif -Requires: stomppy < 5.0.0, python-ldap < 3.4.0, python-setuptools, openssl +Requires: stomppy < 8.1.1, python-setuptools, openssl Requires(pre): shadow-utils %define ssmconf %_sysconfdir/apel @@ -100,6 +102,13 @@ rm -rf $RPM_BUILD_ROOT %doc %_defaultdocdir/%{name} %changelog +* Fri Aug 30 2024 Adrian Coveney - 3.4.1-1 + - Improved error logging to store full traceback on unexpected exceptions. + - Changed more code to use pyOpenSSL to improve compatibility with newer OpenSSL versions. + - Added a check to prevent a host certificate being to used for target server encryption. + - Changed which version of exit function is used to avoid edge case. + - Various changes and improvements to build scripts and processes. + * Wed Feb 21 2024 Adrian Coveney - 3.4.0-1 - Fixed compatability with newer versions of OpenSSL that only provide comma separated DNs. - Fixed Python 3 compatability (indirectly fixing EL8+ compatability) by performing explicit diff --git a/bin/receiver.py b/bin/receiver.py index dac81f3b..4a750383 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -21,15 +21,12 @@ import ssm.agents from ssm import __version__, LOG_BREAK +from argparse import ArgumentParser import logging import os import sys -from optparse import OptionParser -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser +import configparser def main(): @@ -37,20 +34,25 @@ def main(): ver = "SSM %s.%s.%s" % __version__ default_conf_location = '/etc/apel/receiver.cfg' default_dns_location = '/etc/apel/dns' - op = OptionParser(description=__doc__, version=ver) - op.add_option('-c', '--config', - help=('location of config file, ' - 'default path: ' + default_conf_location), - default=default_conf_location) - op.add_option('-l', '--log_config', - help='DEPRECATED - location of logging config file (optional)', - default=None) - op.add_option('-d', '--dn_file', - help=('location of the file containing valid DNs, ' - 'default path: ' + default_dns_location), - default=default_dns_location) - - options, unused_args = op.parse_args() + arg_parser = ArgumentParser(description=__doc__) + + arg_parser.add_argument('-c', '--config', + help='location of config file, default path: ' + '%s' % default_conf_location, + default=default_conf_location) + arg_parser.add_argument('-l', '--log_config', + help='DEPRECATED - location of logging config file', + default=None) + arg_parser.add_argument('-d', '--dn_file', + help='location of the file containing valid DNs, ' + 'default path: %s' % default_dns_location, + default=default_dns_location) + arg_parser.add_argument('-v', '--version', + action='version', + version=ver) + + # Parsing arguments into an argparse.Namespace object for structured access. + options = arg_parser.parse_args() # Deprecating functionality. old_log_config_default_path = '/etc/apel/logging.cfg' @@ -62,11 +64,11 @@ def main(): # Check if config file exists using os.path.isfile function. if os.path.isfile(options.config): - cp = ConfigParser.ConfigParser({'use_ssl': 'true'}) + cp = configparser.ConfigParser({'use_ssl': 'true'}) cp.read(options.config) else: print("Config file not found at", options.config) - exit(1) + sys.exit(1) # Check for pidfile pidfile = cp.get('daemon', 'pidfile') diff --git a/bin/sender.py b/bin/sender.py index f6d08e98..092269ec 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -21,30 +21,33 @@ import ssm.agents from ssm import __version__, LOG_BREAK +from argparse import ArgumentParser import logging -from optparse import OptionParser import os +import sys -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser +import configparser def main(): """Set up connection, send all messages and quit.""" ver = "SSM %s.%s.%s" % __version__ default_conf_location = '/etc/apel/sender.cfg' - op = OptionParser(description=__doc__, version=ver) - op.add_option('-c', '--config', - help=('location of config file, ' - 'default path: ' + default_conf_location), - default=default_conf_location) - op.add_option('-l', '--log_config', - help='DEPRECATED - location of logging config file (optional)', - default=None) - - options, unused_args = op.parse_args() + arg_parser = ArgumentParser(description=__doc__) + + arg_parser.add_argument('-c', '--config', + help='location of config file, default path: ' + '%s' % default_conf_location, + default=default_conf_location) + arg_parser.add_argument('-l', '--log_config', + help='DEPRECATED - location of logging config file', + default=None) + arg_parser.add_argument('-v', '--version', + action='version', + version=ver) + + # Parsing arguments into an argparse.Namespace object for structured access. + options = arg_parser.parse_args() # Deprecating functionality. old_log_config_default_path = '/etc/apel/logging.cfg' @@ -53,11 +56,11 @@ def main(): # Check if config file exists using os.path.isfile function. if os.path.isfile(options.config): - cp = ConfigParser.ConfigParser({'use_ssl': 'true'}) + cp = configparser.ConfigParser({'use_ssl': 'true'}) cp.read(options.config) else: print("Config file not found at", options.config) - exit(1) + sys.exit(1) ssm.agents.logging_helper(cp) diff --git a/conf/receiver.cfg b/conf/receiver.cfg index 84a935f4..096fbf81 100644 --- a/conf/receiver.cfg +++ b/conf/receiver.cfg @@ -3,16 +3,10 @@ protocol: AMS [broker] - -# The SSM will query a BDII to find brokers available. These details are for the -# EGI production broker network -#bdii: ldap://lcg-bdii.cern.ch:2170 -#network: PROD -# Alternatively, 'host' and 'port' can be set manually (with 'bdii' and -# 'network' commented out). The 'host' option MUST be used for AMS. +# 'host' and 'port' must be set manually as LDAP broker search is now removed. +# 'port' is not used with AMS. host: msg-devel.argo.grnet.gr -#host: msg.argo.grnet.gr -#port: +# port: 443 # broker authentication. If use_ssl is set, the certificates configured # in the mandatory [certificates] section will be used. diff --git a/conf/sender.cfg b/conf/sender.cfg index 843b8ef8..731699d8 100644 --- a/conf/sender.cfg +++ b/conf/sender.cfg @@ -3,16 +3,11 @@ protocol: AMS [broker] - -# The SSM will query a BDII to find brokers available. These details are for the -# EGI production broker network -#bdii: ldap://lcg-bdii.cern.ch:2170 -#network: PROD -# Alternatively, 'host' and 'port' can be set manually (with 'bdii' and -# 'network' commented out). The 'host' option MUST be used for AMS. -host: msg-devel.argo.grnet.gr -#host: msg.argo.grnet.gr -#port: +# msg-devel.argo.grnet.gr is only for test data +# msg.argo.grnet.gr is for production data +host: +# 'port' is only used for STOMP sending. +port: 443 # broker authentication. If use_ssl is set, the certificates configured # in the mandatory [certificates] section will be used. diff --git a/requirements-docker.txt b/requirements-docker.txt deleted file mode 100644 index 4765b7c5..00000000 --- a/requirements-docker.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Base requirements for ssm - -argo-ams-library -pyopenssl -cryptography -stomp.py -python-daemon -python-ldap -setuptools # Required for pkg_resources (also happens to be a dependency of python-ldap) - -# Dependencies for optional dirq based sending -dirq diff --git a/requirements-test.txt b/requirements-test.txt index 131ea0e8..9646490a 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ # Additional requirements for the unit and coverage tests -coveralls<=1.2.0 -mock<4.0.0 # Pinned because version 4 dropped support for Python 2.7 +coveralls<=3.3.1 # Last Python 3.6 version codecov pre-commit diff --git a/requirements.txt b/requirements.txt index 92e5691d..f4d3b1e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,11 @@ # Base requirements for ssm -argo-ams-library -certifi<2020.4.5.2 # Used by AMS (via requests), 2020.4.5.2 dropped support for Python 2 -pyopenssl >=19.1.0, <=21.0.0 # 22.0.0 dropped support for Python 2 -cryptography==3.3.2 # Crypto dropped support for Python 2 after 3.3 -stomp.py<5.0.0 -python-daemon<=2.3.0 # 2.3.1 dropped support for Python 2 -python-ldap<3.4.0 # python-ldap-3.4.0 dropped support for Python 2 -setuptools # Required for pkg_resources (also happens to be a dependency of python-ldap) +argo-ams-library>=0.5.1 # 0.4.x series no longer works with SSM +pyopenssl<23.3.0 # 23.3.0 dropped support for Python 3.6 +cryptography<41.0.0 # 41.0.0 dropped support for Python 3.6 +stomp.py<8.1.1 # 8.1.1 dropped suppot for Python 3.6 +python-daemon +setuptools # Required for pkg_resources # Dependencies for optional dirq based sending dirq diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index 707cc048..06e6c858 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=3.4.0-1 +TAG=3.4.1-1 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build @@ -55,9 +55,6 @@ fpm -s python -t deb \ --depends python2.7 \ --depends python-pip \ --depends 'python-stomp < 5.0.0' \ ---depends python-ldap \ ---depends libssl-dev \ ---depends libsasl2-dev \ --depends openssl \ --deb-changelog $SOURCE_DIR/ssm-$TAG/CHANGELOG \ --python-install-bin /usr/bin \ diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index e6d1502d..f0c37a5b 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=3.4.0-1 +VERSION=3.4.1-1 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS diff --git a/scripts/ssm-build.sh b/scripts/ssm-build.sh index e9641151..3b3d6209 100755 --- a/scripts/ssm-build.sh +++ b/scripts/ssm-build.sh @@ -67,7 +67,7 @@ fi PACK_TYPE=$1 VERSION=$2 ITERATION=$3 -PYTHON_ROOT_DIR=$4 # i.e. /usr/lib/python3.6 +PYTHON_ROOT_DIR=$4 # i.e. /usr/lib/python3.6 # Alter library, build and source directories depending on the package if [[ "$PACK_TYPE" = "deb" ]]; then @@ -86,7 +86,7 @@ elif [[ "$PACK_TYPE" = "rpm" ]]; then if [[ "$BUILD_ASSIGNED" = 0 ]]; then BUILD_DIR=~/rpmbuild/BUILD fi -else # If package type is neither deb nor rpm, show an error message and exit +else # If package type is neither deb nor rpm, show an error message and exit echo "$0 currently only supports 'deb' and 'rpm' packages." usage; fi @@ -126,20 +126,13 @@ FPM_CORE="fpm -s python \ # Simple Python filter for version specific FPM if [[ ${PY_NUM:0:1} == "3" ]]; then echo "Building $VERSION iteration $ITERATION for Python $PY_NUM as $PACK_TYPE." - # python-stomp < 5.0.0 to python-stomp, python to python3/pip3 - # edited python-pip3 to python-pip - # slight spelling inconsistencites betwixt OS's if [[ "$PACK_TYPE" = "deb" ]]; then FPM_PYTHON="--depends python3 \ --depends python3-pip \ --depends python3-cryptography \ --depends python3-openssl \ - --depends python3-daemon \ --depends 'python3-stomp' \ - --depends python3-ldap \ - --depends libssl-dev \ - --depends libsasl2-dev \ --depends openssl " # Currently builds for el8 @@ -149,36 +142,6 @@ if [[ ${PY_NUM:0:1} == "3" ]]; then --depends python3-pip \ --depends python3-cryptography \ --depends python3-pyOpenSSL \ - --depends python3-daemon \ - --depends python3-ldap \ - --depends openssl \ - --depends openssl-devel " - fi - -elif [[ ${PY_NUM:0:1} == "2" ]]; then - echo "Building $VERSION iteration $ITERATION for Python $PY_NUM as $PACK_TYPE." - - if [[ "$PACK_TYPE" = "deb" ]]; then - FPM_PYTHON="--depends python2.7 \ - --depends python-pip \ - --depends 'python-stomp < 5.0.0' \ - --depends python-ldap \ - --depends python-cryptography \ - --depends python-openssl \ - --depends python-daemon \ - --depends libssl-dev \ - --depends libsasl2-dev \ - --depends openssl " - - # el7 and below, due to yum package versions - elif [[ "$PACK_TYPE" = "rpm" ]]; then - FPM_PYTHON="--depends python2 \ - --depends python2-pip \ - --depends python2-cryptography \ - --depends python2-pyOpenSSL \ - --depends python2-daemon \ - --depends stomppy \ - --depends python-ldap \ --depends openssl \ --depends openssl-devel " fi @@ -198,6 +161,7 @@ PACKAGE_VERSION="--$PACK_TYPE-changelog $SOURCE_DIR/ssm-$VERSION-$ITERATION/CHAN BUILD_PACKAGE_COMMAND=${FPM_CORE}${FPM_PYTHON}${VERBOSE}${PACKAGE_VERSION} eval "$BUILD_PACKAGE_COMMAND" +echo echo "== Generating pleaserun package ==" # When installed, use pleaserun to perform system specific service setup @@ -211,10 +175,13 @@ fpm -s pleaserun -t "$PACK_TYPE" \ --architecture all \ --no-auto-depends \ --depends apel-ssm \ +--depends python3-daemon \ +--depends python3-dirq \ --package "$BUILD_DIR" \ /usr/bin/ssmreceive -echo "Possible Issues to Fix:" +echo +echo "== Possible Issues to Fix ==" if [ "$OS_EXTENSION" == "_all" ] then # Check the resultant debs for 'lint' diff --git a/setup.py b/setup.py index 950dd33e..e6f048ca 100644 --- a/setup.py +++ b/setup.py @@ -22,14 +22,30 @@ from ssm import __version__ +def setup_temp_files(): + """Create temporary files with deployment names. """ + copyfile('bin/receiver.py', 'bin/ssmreceive') + copyfile('bin/sender.py', 'bin/ssmsend') + copyfile('scripts/apel-ssm.logrotate', 'conf/apel-ssm') + copyfile('README.md', 'apel-ssm') + + def main(): """Called when run as script, e.g. 'python setup.py install'.""" - # Create temporary files with deployment names - if 'install' in sys.argv: - copyfile('bin/receiver.py', 'bin/ssmreceive') - copyfile('bin/sender.py', 'bin/ssmsend') - copyfile('scripts/apel-ssm.logrotate', 'conf/apel-ssm') - copyfile('README.md', 'apel-ssm') + supported_commands = { + "install", + "build", + "bdist", + "develop", + "build_scripts", + "install_scripts", + "install_data", + "bdist_dumb", + "bdist_egg", + } + + if supported_commands.intersection(sys.argv): + setup_temp_files() # conf_files will later be copied to conf_dir conf_dir = '/etc/apel/' @@ -53,12 +69,11 @@ def main(): install_requires=[ 'cryptography', 'stomp.py', - 'python-ldap', 'setuptools', 'pyopenssl', ], extras_require={ - 'AMS': ['argo-ams-library', ], + 'AMS': ['argo-ams-library>=0.5.1', ], 'daemon': ['python-daemon', ], 'dirq': ['dirq'], }, @@ -79,7 +94,7 @@ def main(): ) # Remove temporary files with deployment names - if 'install' in sys.argv: + if supported_commands.intersection(sys.argv): remove('bin/ssmreceive') remove('bin/ssmsend') remove('conf/apel-ssm') diff --git a/ssm/__init__.py b/ssm/__init__.py index 904c0c7d..79ecfe73 100644 --- a/ssm/__init__.py +++ b/ssm/__init__.py @@ -19,7 +19,7 @@ import logging import sys -__version__ = (3, 4, 0) +__version__ = (3, 4, 1) LOG_BREAK = '========================================' diff --git a/ssm/agents.py b/ssm/agents.py index 4726cbb2..bf859799 100644 --- a/ssm/agents.py +++ b/ssm/agents.py @@ -19,14 +19,10 @@ from __future__ import absolute_import, division, print_function, unicode_literals import logging -import ldap import sys import time -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser +import configparser try: from daemon import DaemonContext @@ -49,7 +45,6 @@ class AmsConnectionException(Exception): from ssm import set_up_logging, LOG_BREAK from ssm.ssm2 import Ssm2, Ssm2Exception from ssm.crypto import CryptoException, get_certificate_subject, _from_file -from ssm.brokers import StompBrokerGetter, STOMP_SERVICE, STOMP_SSL_SERVICE # How often (in seconds) to read the list of valid DNs. REFRESH_DNS = 600 @@ -63,7 +58,7 @@ def logging_helper(cp): cp.get('logging', 'level'), cp.getboolean('logging', 'console') ) - except (ConfigParser.Error, ValueError, IOError) as err: + except (configparser.Error, ValueError, IOError) as err: print('Error configuring logging: %s' % err) print('The system will exit.') sys.exit(1) @@ -77,12 +72,12 @@ def get_protocol(cp, log): elif 'receiver' in cp.sections(): protocol = cp.get('receiver', 'protocol') else: - raise ConfigParser.NoSectionError('sender or receiver') + raise configparser.NoSectionError('sender or receiver') if protocol not in (Ssm2.STOMP_MESSAGING, Ssm2.AMS_MESSAGING): raise ValueError - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + except (configparser.NoSectionError, configparser.NoOptionError): # If the newer configuration setting 'protocol' is not set, use 'STOMP' # for backwards compatability. protocol = Ssm2.STOMP_MESSAGING @@ -104,35 +99,13 @@ def get_ssm_args(protocol, cp, log): project = None token = '' - use_ssl = cp.getboolean('broker', 'use_ssl') - if use_ssl: - service = STOMP_SSL_SERVICE - else: - service = STOMP_SERVICE - - # If we can't get a broker to connect to, we have to give up. try: - bdii_url = cp.get('broker', 'bdii') - log.info('Retrieving broker details from %s ...', bdii_url) - bg = StompBrokerGetter(bdii_url) - brokers = bg.get_broker_hosts_and_ports(service, cp.get('broker', - 'network')) - log.info('Found %s brokers.', len(brokers)) - except ConfigParser.NoOptionError as e: - try: - host = cp.get('broker', 'host') - port = cp.get('broker', 'port') - brokers = [(host, int(port))] - except ConfigParser.NoOptionError: - log.error('Options incorrectly supplied for either single ' - 'broker or broker network. ' - 'Please check configuration') - log.error('System will exit.') - log.info(LOG_BREAK) - print('SSM failed to start. See log file for details.') - sys.exit(1) - except ldap.LDAPError as e: - log.error('Could not connect to LDAP server: %s', e) + host = cp.get('broker', 'host') + port = cp.get('broker', 'port') + brokers = [(host, int(port))] + except configparser.NoOptionError: + log.error('Host options incorrectly supplied for message broker ' + 'or AMS endpoint. Please check configuration.') log.error('System will exit.') log.info(LOG_BREAK) print('SSM failed to start. See log file for details.') @@ -148,7 +121,7 @@ def get_ssm_args(protocol, cp, log): # the exact destination type. brokers = [host] - except ConfigParser.NoOptionError: + except configparser.NoOptionError: log.error('The host must be specified when connecting to AMS, ' 'please check your configuration') log.error('System will exit.') @@ -160,7 +133,7 @@ def get_ssm_args(protocol, cp, log): try: project = cp.get('messaging', 'ams_project') - except (ConfigParser.Error, ValueError, IOError) as err: + except (configparser.Error, ValueError, IOError) as err: # A project is needed to successfully send to an # AMS instance, so log and then exit on an error. log.error('Error configuring AMS values: %s', err) @@ -170,7 +143,7 @@ def get_ssm_args(protocol, cp, log): try: token = cp.get('messaging', 'token') - except (ConfigParser.Error, ValueError, IOError) as err: + except (configparser.Error, ValueError, IOError) as err: # A token is not necessarily needed, if the cert and key can be # used by the underlying auth system to get a suitable token. log.info('No AMS token provided, using cert/key pair instead.') @@ -197,24 +170,24 @@ def run_sender(protocol, brokers, project, token, cp, log): log.info('Messages will be encrypted using %s', server_dn) try: verify_server_cert = cp.getboolean('certificates', 'verify_server_cert') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: # If option not set, resort to value of verify_server_cert set above. pass - except ConfigParser.NoOptionError: + except configparser.NoOptionError: log.info('No server certificate supplied. Will not encrypt messages.') try: destination = cp.get('messaging', 'destination') if destination == '': raise Ssm2Exception('No destination queue is configured.') - except ConfigParser.NoOptionError as e: + except configparser.NoOptionError as e: raise Ssm2Exception(e) # Determine what type of message store we are interacting with, # i.e. a dirq QueueSimple object or a plain MessageDirectory directory. try: path_type = cp.get('messaging', 'path_type') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: log.info('No path type defined, assuming dirq.') path_type = 'dirq' diff --git a/ssm/brokers.py b/ssm/brokers.py deleted file mode 100644 index 28826c34..00000000 --- a/ssm/brokers.py +++ /dev/null @@ -1,157 +0,0 @@ -""" - Copyright (C) 2012 STFC. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - @author: Will Rogers - -Class to interact with a BDII LDAP server to retrieve information about -the stomp brokers specified in a network. -""" -from __future__ import print_function - -import ldap -import logging - -log = logging.getLogger(__name__) - -# Constants used for specific LDAP queries -STOMP_SERVICE = 'msg.broker.stomp' -STOMP_SSL_SERVICE = 'msg.broker.stomp-ssl' - -STOMP_PREFIX = 'stomp' -STOMP_SSL_PREFIX = 'stomp+ssl' - - -class StompBrokerGetter(object): - """Class for seaching a BDII for message brokers. - - Given the URL of a BDII, searches for all the STOMP - brokers listed that are part of the specified network. - """ - - def __init__(self, bdii_url): - """Set up the LDAP connection and strings which are re-used.""" - # Set up the LDAP connection - logging.warning('LDAP is deprecated and will be removed in an upcoming version, ' - 'please set host locally in SSM config.') - log.debug('Connecting to %s...', bdii_url) - self._ldap_conn = ldap.initialize(bdii_url) - - self._base_dn = 'o=grid' - self._service_id_key = 'GlueServiceUniqueID' - self._endpoint_key = 'GlueServiceEndpoint' - self._service_data_value_key = 'GlueServiceDataValue' - - def get_broker_urls(self, service_type, network): - """Get a list stomp broker URLs in a specified network from a BDII. - - Checks them to see if they are part of the network. The network is - supplied as a string. Returns a list of URLs. - """ - prod_broker_urls = [] - - broker_details = self._get_broker_details(service_type) - - for broker_id, broker_url in broker_details: - if self._broker_in_network(broker_id, network): - prod_broker_urls.append(broker_url) - - return prod_broker_urls - - def get_broker_hosts_and_ports(self, service_type, network): - """Get a list of stomp broker (host, port) tuples from a BDII. - - Gets the list of all the stomp brokers in the BDII, then checks them to - see if they are part of the network. The network is supplied as a - string.Returns a list of (host, port) tuples. - """ - urls = self.get_broker_urls(service_type, network) - hosts_and_ports = [] - for url in urls: - hosts_and_ports.append(parse_stomp_url(url)) - return hosts_and_ports - - def _get_broker_details(self, service_type): - """Search the BDII for all STOMP message brokers. - - Returns a list of tuples: (, ). - """ - broker_details = [] - - ldap_filter = '(&(objectClass=GlueService)(GlueServiceType=%s))' % service_type - attrs = [self._service_id_key, self._endpoint_key] - - brokers = self._ldap_conn.search_s(self._base_dn, ldap.SCOPE_SUBTREE, ldap_filter, attrs) - - for unused_dn, attrs in brokers: - details = attrs[self._service_id_key][0], attrs[self._endpoint_key][0] - broker_details.append(details) - - return broker_details - - def _broker_in_network(self, broker_id, network): - """Check that a GlueServiceUniqueID is part of a specified network.""" - ldap_filter = '(&(GlueServiceDataKey=cluster)(GlueChunkKey=GlueServiceUniqueID=%s))' \ - % broker_id - attrs = [self._service_data_value_key] - results = self._ldap_conn.search_s(self._base_dn, ldap.SCOPE_SUBTREE, - ldap_filter, attrs) - - try: - unused_dn, attrs2 = results[0] - return network in attrs2[self._service_data_value_key] - except IndexError: # no results from the query - return False - - -def parse_stomp_url(stomp_url): - """Parse a stomp scheme URL. - - Given a URL of the form stomp://stomp.cern.ch:6262/, - return a tuple containing (stomp.cern.ch, 6262). - """ - parts = stomp_url.split(':') - - protocols = [STOMP_PREFIX, STOMP_SSL_PREFIX] - if not parts[0].lower() in protocols: - raise ValueError("URL %s does not begin 'stomp:'." % stomp_url) - - host = parts[1].strip('/') - port = parts[2].strip('/') - if not port.isdigit(): - raise ValueError('URL %s does not have an integer as its third part.') - - return host, int(port) - - -if __name__ == '__main__': - # BDII URL - BDII = 'ldap://lcg-bdii.cern.ch:2170' - BG = StompBrokerGetter(BDII) - - def print_brokers(text, service, network): - """Pretty print a list of brokers.""" - brokers = BG.get_broker_hosts_and_ports(service, network) - # Print section heading - print('==', text, '==') - # Print brokers in form 'host:port' - for broker in brokers: - print('%s:%i' % (broker[0], broker[1])) - # Leave space between sections - print() - - print_brokers('SSL production brokers', STOMP_SSL_SERVICE, 'PROD') - print_brokers('Production brokers', STOMP_SERVICE, 'PROD') - print_brokers('SSL test brokers', STOMP_SSL_SERVICE, 'TEST-NWOB') - print_brokers('Test brokers', STOMP_SERVICE, 'TEST-NWOB') diff --git a/ssm/ssm2.py b/ssm/ssm2.py index b7dc495d..fcff8e93 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -623,7 +623,6 @@ def start_connection(self): raise Ssm2Exception('Called start_connection() before a \ connection object was initialised.') - self._conn.start() self._conn.connect(wait=False) i = 0 diff --git a/test/test_agents.py b/test/test_agents.py index 7b33f9ee..db3f4cc3 100644 --- a/test/test_agents.py +++ b/test/test_agents.py @@ -4,8 +4,7 @@ import tempfile from textwrap import dedent import unittest - -import mock +import unittest.mock as mock import ssm.agents from ssm.ssm2 import Ssm2Exception diff --git a/test/test_brokers.py b/test/test_brokers.py deleted file mode 100644 index 93dc776a..00000000 --- a/test/test_brokers.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2019 UK Research and Innovation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function - -import unittest - -import mock - -from ssm import brokers - - -class Test(unittest.TestCase): - - def test_parse_stomp_url(self): - - wrong_url = 'this is not a correct url' - try: - brokers.parse_stomp_url(wrong_url) - self.fail('Appeared to parse a fake URL.') - except (IndexError, ValueError): - # Expected exception - pass - - http_url = 'http://not.a.stomp.url:8080' - - self.assertRaises(ValueError, brokers.parse_stomp_url, http_url) - - self.assertRaises(ValueError, brokers.parse_stomp_url, - 'stomp://invalid.port.number:abc') - - stomp_url = 'stomp://stomp.cern.ch:6262' - - try: - brokers.parse_stomp_url(stomp_url) - except Exception: - self.fail('Could not parse a valid stomp URL: %s' % stomp_url) - - stomp_ssl_url = 'stomp+ssl://stomp.cern.ch:61262' - - try: - brokers.parse_stomp_url(stomp_ssl_url) - except Exception: - self.fail('Could not parse a valid stomp+ssl URL: %s' % stomp_url) - - def test_fetch_brokers(self): - """Check the handling of responses from a mocked BDII.""" - bdii = 'ldap://no-bdii.utopia.ch:2170' - network = 'PROD' - - sbg = brokers.StompBrokerGetter(bdii) - - # So that there are no external LDAP calls, mock out the LDAP seach. - with mock.patch('ldap.ldapobject.SimpleLDAPObject.search_s', - side_effect=self._mocked_search): - bs = sbg.get_broker_hosts_and_ports(brokers.STOMP_SERVICE, network) - - if len(bs) < 1: - self.fail('No brokers found in the BDII.') - - host, port = bs[0] - if not str(port).isdigit(): - self.fail('Got a non-integer port from fetch_brokers()') - - if '.' not in host: - self.fail("Didn't get a hostname from fetch_brokers()") - - # Check that no brokers are returned from the TEST-NWOB network. - test_network = 'TEST-NWOB' - # So that there are no external LDAP calls, mock out the LDAP seach. - with mock.patch('ldap.ldapobject.SimpleLDAPObject.search_s', - side_effect=self._mocked_search): - test_bs = sbg.get_broker_hosts_and_ports(brokers.STOMP_SERVICE, - test_network) - self.assertEqual(len(test_bs), 0, "Test brokers found in error.") - - def _mocked_search(*args, **kwargs): - """Return values to mocked search call based on input.""" - - if ( - '(&(objectClass=GlueService)(GlueServiceType=msg.broker.stomp))' - ) in args: - return [( - 'GlueServiceUniqueID=mq.cro-ngi.hr_msg.broker.stomp_3523291347' - ',Mds-Vo-name=egee.srce.hr,Mds-Vo-name=local,o=grid', - {'GlueServiceUniqueID': - ['mq.cro-ngi.hr_msg.broker.stomp_3523291347'], - 'GlueServiceEndpoint': ['stomp://mq.cro-ngi.hr:6163/']}), - ( - 'GlueServiceUniqueID=broker-prod1.argo.grnet.gr_msg.broker.sto' - 'mp_175215210,Mds-Vo-name=HG-06-EKT,Mds-Vo-name=local,o=grid', - {'GlueServiceUniqueID': - ['broker-prod1.argo.grnet.gr_msg.broker.stomp_175215210'], - 'GlueServiceEndpoint': - ['stomp://broker-prod1.argo.grnet.gr:6163/']} - )] - elif ( - '(&(GlueServiceDataKey=cluster)(GlueChunkKey=GlueServiceUniqueID=' - 'mq.cro-ngi.hr_msg.broker.stomp_3523291347))' - ) in args: - return [( - 'GlueServiceDataKey=cluster,GlueServiceUniqueID=mq.cro-ngi.hr_' - 'msg.broker.stomp_3523291347,Mds-Vo-name=egee.srce.hr,Mds-Vo-n' - 'ame=local,o=grid', {'GlueServiceDataValue': ['PROD']} - )] - elif ( - '(&(GlueServiceDataKey=cluster)(GlueChunkKey=GlueServiceUniqueID=' - 'broker-prod1.argo.grnet.gr_msg.broker.stomp_175215210))' - ) in args: - return [( - 'GlueServiceDataKey=cluster,GlueServiceUniqueID=broker-prod1.a' - 'rgo.grnet.gr_msg.broker.stomp_175215210,Mds-Vo-name=HG-06-EKT' - ',Mds-Vo-name=local,o=grid', {'GlueServiceDataValue': ['PROD']} - )] - else: - # This will tell mock to use the normal return value - return mock.DEFAULT - - -if __name__ == '__main__': - unittest.main()