From fce07dca6954591d93fba4fbadefe89f2b6f86f5 Mon Sep 17 00:00:00 2001 From: Lars Hupfeldt Date: Tue, 1 Oct 2024 23:42:19 +0200 Subject: [PATCH] Fixes for newer Jenkins version. Require Python 3.9+. Workaround for https://issues.jenkins.io/browse/JENKINS-63845 Some timing issue fiexs in test. Major test update to move from tox to nox. Get rid of Tenjin to generate coverage settings, use environment variables in coverage_rc instead. TODO: More cleanup to get rid of tmp_intall file copying. --- .gitignore | 1 + .travis.yml | 31 +--- INSTALL.md | 125 +++++++------- README.rst | 5 +- cli/cli.py | 5 +- cli/set_build_description.py | 49 +++--- demo/basic.py | 2 +- demo/calculated_flow.py | 2 +- demo/errors.py | 2 +- demo/get_jenkins_api.py | 2 +- demo/hide_password.py | 2 +- demo/prefix.py | 3 +- doc/requirements.txt | 2 + doc/source/index.rst | 2 +- doc/source/jenkinsflow.rst | 2 +- jenkins_api.py | 2 +- noxfile.py | 102 +++++++++++ pytest.ini | 2 +- requirements.txt | 13 ++ rest_api_wrapper.py | 5 + script_api.py | 2 +- setup.py | 83 +++------ test/conftest.py | 115 +++++++++---- test/demos_test.py | 38 ++-- test/framework/abort_job.py | 36 ++-- test/framework/api_select.py | 17 +- test/framework/api_wrapper.py | 47 +++-- test/framework/cfg/__init__.py | 3 +- test/framework/cfg/all_cfg.py | 47 +++++ test/framework/cfg/api_type.py | 8 + test/framework/cfg/job_load.py | 21 +-- test/framework/cfg/os_env.py | 1 - test/framework/cfg/urls.py | 114 ++++++------ test/framework/coverage_rc | 4 +- test/framework/hyperspeed_test.py | 4 +- test/framework/job.xml.tenjin | 19 +- test/framework/job_script.py.tenjin | 3 + test/framework/killer.py | 14 +- test/framework/mock_api.py | 9 +- test/framework/nox_utils.py | 79 +++++++++ test/framework/pytest_options.py | 59 +++++++ test/framework/tmp_install.sh | 12 ++ test/{ => framework}/which_ci_server.html | 0 test/missing_jobs_test.py | 2 +- test/multiple_invocations_test.py | 2 +- test/no_running_jobs_test.py | 27 ++- test/requirements.txt | 9 + test/run.py | 200 ---------------------- test/set_build_description_test.py | 10 +- test/speed_test.py | 6 +- test/tmp_install.sh | 6 +- tox.ini | 16 -- 52 files changed, 782 insertions(+), 590 deletions(-) create mode 100644 doc/requirements.txt create mode 100644 noxfile.py create mode 100644 requirements.txt create mode 100644 test/framework/cfg/all_cfg.py delete mode 100644 test/framework/cfg/os_env.py create mode 100644 test/framework/nox_utils.py create mode 100644 test/framework/pytest_options.py create mode 100755 test/framework/tmp_install.sh rename test/{ => framework}/which_ci_server.html (100%) create mode 100644 test/requirements.txt delete mode 100755 test/run.py delete mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index a03e1cd..9799f64 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ pip-log.txt .coverage* coverage_rc.tenjin.cache* .tox +.nox nosetests.xml .pytest_cache/ diff --git a/.travis.yml b/.travis.yml index 5060093..ecbaabf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,24 +3,7 @@ python: - 3.6 - 3.7 - 3.8 - - pypy3.6-7.0.0 -matrix: - include: - - python: 3.7 - dist: xenial - env: REQUIREMENTS=lowest - - python: 3.7 - dist: xenial - env: REQUIREMENTS=release - - python: 3.8 - dist: xenial - env: REQUIREMENTS=lowest - - python: 3.8 - dist: xenial - env: REQUIREMENTS=release - allow_failures: - - python: pypy3.6-7.0.0 -script: tox + - 3.9 os: - linux before_install: @@ -28,14 +11,16 @@ before_install: - travis_retry pip install requirements-builder - requirements-builder --level=min setup.py > .travis-lowest-requirements.txt - requirements-builder --level=pypi setup.py > .travis-release-requirements.txt -install: - - pip install tox-travis - - pip install coveralls - - travis_retry pip install -r .travis-$REQUIREMENTS-requirements.txt - - pip install -e . env: - REQUIREMENTS=lowest - REQUIREMENTS=release +install: + - pip install --upgrade nox + - pip install --upgrade coveralls + - travis_retry pip install -r .travis-$REQUIREMENTS-requirements.txt + - pip install -e . +script: + - nox after_success: # rcfile name duplicated in test/run.py - coveralls --rcfile="./test/.coverage_rc_mock_script" diff --git a/INSTALL.md b/INSTALL.md index 3ec2897..4cbcfc1 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,118 +3,111 @@ Installation Python 3.6 or later is required. A recent Jenkins is required. -Hudson may be supported, but it is not longer tested. 1. The easy way: - Install python-devel (required by the psutil dependency of the script_api) + Install *python-devel* (required by the *psutil* dependency of the *script_api*) E.g on fedora: - sudo dnf install python-devel - pip install --user -U . + sudo dnf install python-devel + pip install --user --upgrade jenkinsflow To uninstall: - pip uninstall jenkinsflow - Read the file demo/demo_security.py if you have security enabled your Jenkins. + pip uninstall jenkinsflow -Jenkinsflow uses it's own specialized 'jenkins_api' python module to access Jenkins, using the Jenkins rest api. + Read the file *demo/demo_security.py* if you have security enabled your Jenkins. + +Jenkinsflow uses it's own specialized *jenkins_api* python module to access Jenkins, using the Jenkins rest api. 2. Manually: 2.1. Install dependencies: - pip install requests click atomicfile - optional: pip install tenjin (if you want to use the template based job loader) + pip install requests click atomicfile + + optional: `pip install tenjin` (if you want to use the template based job loader) 2.2. Install dependencies for experimental features: - To use the experimental script api: - Install python-devel (see above) - pip install psutil setproctitle + To use the experimental script API install python-devel (see above) + + pip install psutil setproctitle To use the experimental visualisation feature: - pip install bottle -2.3. Read the file demo/demo_security.py for notes about security, if you have enabled security on your Jenkins + pip install bottle + +2.3. Read the file *demo/demo_security.py* for notes about security, if you have enabled security on your Jenkins 2.4. All set! You can now create jobs that have a shell execution step, which will a use this library to control the running of other jobs. See the demo directory for example flows. The demo jobs can be loaded by running tests, see below. Note: I think jenkinsflow should work on Windows, but it has not been tested. - I'm SURE the test/run.py script will fail on Windows. There are a few Linux/Unix bits in the test setup. Check test/framework/config.py and - test/tmp_install.sh. Patches are welcome:) + I'm SURE the tests will fail on Windows. There are a few Linux/Unix bits in the test setup. Check *test/framework/cfg/* and + *test/framework/tmp_install.sh*. Patches are welcome:) Test ---- -0. The mocked and script_api test can be run using 'tox' - pip install tox +0. The mocked and script_api test can be run using `nox` -1. The tests which actually run Jenkins jobs currently do not run under tox. + pip install nox - Install pytest and tenjin template engine: - pip install --user -U -r test/requirements.txt - - # The test will also test the generation of documentation, for this you need: - pip install --user -U -r doc/requirements.txt - - -2. Important Jenkins setup and test preparation: +1. Important Jenkins setup and test preparation: Configure security - Some of the tests requires security to be enabled. You need to create two users in Jenkins: - Read the file demo/demo_security.py and create the user specified. - Create a user called 'jenkinsflow_authtest1', password 'abcæøåÆØÅ'. ( u'\u00e6\u00f8\u00e5\u00c6\u00d8\u00c5' ) + Read the file *demo/demo_security.py* and create the user specified. + Create a user called **jenkinsflow_authtest1**, password **abcæøåÆØÅ**. ( u'\u00e6\u00f8\u00e5\u00c6\u00d8\u00c5' ) Set the number of executers - - Jenkins is default configured with only two executors on master. To avoid timeouts in the test cases this must be raised to at least 32. + Jenkins is default configured with only two executors on 'built-in' node. To avoid time-outs in the test cases this must be raised to at least **32**. This is necessary because some of the test cases will execute a large amount of jobs in parallel. Change the 'Quite period' - - Jenkins is default configured with a 'Quiet period' of 5 seconds. To avoid timeouts in the test cases this must be set to 0. + Jenkins is default configured with a 'Quiet period' of 5 seconds. To avoid time-outs in the test cases this must be set to **0**. Set Jenkins URL - Jenkins: Manage Jenkins -> Configure System -> Jenkins Location -> Jenkins URL - The url should not use 'localhost' + The URL should **not** use **localhost**. Your Jenkins needs to be on the host where you are running the test. If it is not, you will need to make jenkinsflow available to Jenkins. See - test/tmp_install.sh + *test/framework/tmp_install.sh* 3. Run the tests: - Use tox - or - Use ./test/run.py to run the all tests. - JENKINS_URL= python ./test/run.py --mock-speedup=100 --direct-url + Use `nox` + + JENKINS_URL= nox --mock-speedup=100 --direct-url Note: you may omit JENKINS_URL if your Jenkins is on http://localhost:8080. - Note: you may omit --direct-url if your Jenkins is on http://localhost:8080 + Note: you may omit --direct-url if your Jenkins is on http://localhost:8080. - The test script will run the test suite with mocked jenkins api, script_api and jenkins_api in parallel. The mocked api is a very fast test of the flow logic. - Mocked tests and script_api tests do not require Jenkins. + Note: Run `nox -- --help` and look at *custom options:* to see special options. + + The test script will run the test suite with *mocked jenkins_api*, *script_api* and *jenkins_api* in parallel. The mocked api is a very fast test of the flow logic. + Mocked tests and script-api tests do not require Jenkins. The test jobs are automatically created in Jenkins. - It is possible to select a subset of the apis using the --apis option. + It is possible to select a subset of the apis using the `--api` option. - The value given to --mock-speedup is the time speedup for the mocked tests. If you have a reasonably fast computer, try 2000. - If you get FlowTimeoutException try a lower value. - If you get " is expected to be running, but state is IDLE" try a lower value. + The value given to `--mock-speedup` is the time speedup for the mocked tests. If you have a reasonably fast computer, try **2000**. + If you get `FlowTimeoutException` try a lower value. + If you get *\ is expected to be running, but state is IDLE* try a lower value. By default tests are run in parallel using xdist and jobs are not deleted (but will be updated) before each run. - You should have 32 executors or more for this, the cpu/disk load will be small, as the test jobs don't really do anything except sleep. - To disable the use of xdist use --job-delete. - - All jobs created by the test script are prefixed with 'jenkinsflow_', so they can easily be removed. + You should have **32** executors or more for this, the CPU/disk load will be small, as the test jobs don't really do anything except sleep. + To disable the use of xdist use `--job-delete`. - The test suite creates jobs called ..._0flow. These jobs are not executed by the test suite, by you can run them to see what the flows look like in a Jenkins job. - If your Jenkins is not secured, you must set username and password to '' in demo_security, in order to be able to run all the ..._0flow jobs. + All jobs created by the test script are prefixed with **jenkinsflow_**, so they can be easily removed. - To see more test options, run ./test/run.py ---help + The test suite creates jobs called *..._0flow*. These jobs are not executed by the test suite, by you can run them to see what the flows look like in a Jenkins job. + If your Jenkins is not secured, you must set username and password to '' in *demo_security.py*, in order to be able to run all the ..._0flow jobs. Demos ----- +----- 1. Run tests as described above to load jobs into Jenkins @@ -122,28 +115,34 @@ Demos python ./demo/.py 3. Demo scripts can be executed from the loaded 'jenkinsflow_demo____0flow' Jenkins jobs. - Jenkins needs to be able to find the scripts, the run.py script creates a test installation. + Jenkins needs to be able to find the scripts, executing `nox` creates a test installation. Flow Graph Visualisation ------------------------ +------------------------ + +1. To see a flow graph of the basic demo in your browser, execute: -1. To see a flow graph of the basic demo in your browser: - Start 'python ./visual/server.py' --json-dir '/tmp/jenkinsflow-test/graphs/jenkinsflow_demo__basic' before running ./demo/basic.py - Open localhost:9090 in your browser. + python ./visual/server.py' --json-dir '/tmp/jenkinsflow-test/graphs/jenkinsflow_demo__basic - The test suite also puts some other graps in subdirectories under '/tmp/jenkinsflow-test/graphs'. - The 'visual' feature is still experimental and does not yet show live info about the running flow/jobs. + before running *./demo/basic.py* - If you run ...0flow jobs that generate graphs from Jenkins the json graph file will be put in the workspace. + Open http://localhost:9090 in your browser. + + The test suite also puts some other graphs in subdirectories under */tmp/jenkinsflow-test/graphs*. + The *visual* feature is still experimental and does not yet show live info about the running flow/jobs. + + If you run *...0flow* jobs that generate graphs from Jenkins the json graph file will be put in the workspace. Documentation ---- 1. Install sphinx and extensions: - pip install 'sphinx>=1.6' sphinxcontrib-programoutput + + pip install 'sphinx>=1.6' sphinxcontrib-programoutput 2. Build documentation: - cd doc/source - make html (or some other format supported by sphinx) + + cd doc/source + make html (or some other format supported by sphinx) diff --git a/README.rst b/README.rst index bb2970e..0a2feb7 100644 --- a/README.rst +++ b/README.rst @@ -4,13 +4,12 @@ jenkinsflow =========== Python API with high level build flow constructs (parallel/serial) for -Jenkins (and Hudson). Allows full scriptable control over the execution +Jenkins. Allows full scriptable control over the execution of Jenkins jobs. Also allows running 'jobs' without using Jenkins (for testing without reloading Jenkins jobs). See INSTALL.md for installation and test setup. See demo/... for some -usage examples. I don't test continuously on Hudson, but patches are -welcome. +usage examples. Thanks to Aleksey Maksimov for contributing various bits, including the graph visualization. diff --git a/cli/cli.py b/cli/cli.py index 768b724..1f6f3b0 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -15,7 +15,7 @@ __package__ = "jenkinsflow.cli" -from .set_build_description import set_build_description +from .set_build_description import set_build_description, set_build_description_hidden @click.group() @@ -23,7 +23,8 @@ def cli(): """Commandline utilities for jenkinsflow""" -cli.add_command(set_build_description, name="set_build_description") +cli.add_command(set_build_description) +cli.add_command(set_build_description_hidden) # Backwards compatibility if __name__ == "__main__": diff --git a/cli/set_build_description.py b/cli/set_build_description.py index 4000f24..6eff391 100755 --- a/cli/set_build_description.py +++ b/cli/set_build_description.py @@ -6,25 +6,30 @@ from jenkinsflow.utils import set_build_description as usbd -@click.command() -@click.option('--description', help="The description to set on the build") -@click.option('--replace/--no-replace', default=False, help="Replace existing description, if any, instead of appending.") -@click.option('--separator', default='\n', help="A separator to insert between any existing description and the new 'description' if 'replace' is not specified.") -@click.option('--username', help="User Name for Jenkin authentication with secured Jenkins") -@click.option('--password', help="Password of Jenkins User") -@click.option('--build-url', help='Build URL', envvar='BUILD_URL') -@click.option('--job-name', help='Job Name', envvar='JOB_NAME') -@click.option('--build-number', help="Build Number", type=click.INT, envvar='BUILD_NUMBER') -@click.option( - '--direct-url', - default=None, - help="Jenkins URL - preferably non-proxied. If not specified, the value of JENKINS_URL or HUDSON_URL environment variables will be used.") -def set_build_description(description, replace, separator, username, password, build_url, job_name, build_number, direct_url): - """Utility to set/append build description on a job build. - - When called from a Jenkins job you can leave out the '--build-url', '--job-name' and '--build-number' arguments, the BUILD_URL env variable will be used. - """ - - # %(file)s --job-name --build-number --description [--direct-url ] [--replace | --separator ] [(--username --password )] - - usbd.set_build_description(description, replace, separator, username, password, build_url, job_name, build_number, direct_url) +# Generate twice for backwards compatibility, first (hidden) is the old name +for name, hidden in (("set_build_description", True), ("set-build-description", False)): + @click.command(name=name, hidden=hidden) + @click.option('--description', help="The description to set on the build") + @click.option('--replace/--no-replace', default=False, help="Replace existing description, if any, instead of appending.") + @click.option('--separator', default='\n', help="A separator to insert between any existing description and the new 'description' if 'replace' is not specified.") + @click.option('--username', help="User Name for Jenkin authentication with secured Jenkins") + @click.option('--password', help="Password of Jenkins User") + @click.option('--build-url', help='Build URL', envvar='BUILD_URL') + @click.option('--job-name', help='Job Name', envvar='JOB_NAME') + @click.option('--build-number', help="Build Number", type=click.INT, envvar='BUILD_NUMBER') + @click.option( + '--direct-url', + default=None, + help="Jenkins URL - preferably non-proxied. If not specified, the value of JENKINS_URL or HUDSON_URL environment variables will be used.") + def set_build_description(description, replace, separator, username, password, build_url, job_name, build_number, direct_url): + """Utility to set/append build description on a job build. + + When called from a Jenkins job you can leave out the '--build-url', '--job-name' and '--build-number' arguments, the BUILD_URL env variable will be used. + """ + + # %(file)s --job-name --build-number --description [--direct-url ] [--replace | --separator ] [(--username --password )] + + usbd.set_build_description(description, replace, separator, username, password, build_url, job_name, build_number, direct_url) + + if hidden: + set_build_description_hidden = set_build_description diff --git a/demo/basic.py b/demo/basic.py index d983a25..a072861 100755 --- a/demo/basic.py +++ b/demo/basic.py @@ -7,7 +7,7 @@ from jenkinsflow.flow import serial -import get_jenkins_api +from jenkinsflow.demo import get_jenkins_api def main(api, securitytoken): diff --git a/demo/calculated_flow.py b/demo/calculated_flow.py index d71f815..3bdc717 100755 --- a/demo/calculated_flow.py +++ b/demo/calculated_flow.py @@ -12,7 +12,7 @@ from jenkinsflow.flow import serial from jenkinsflow.unbuffered import UnBuffered -import get_jenkins_api +from jenkinsflow.demo import get_jenkins_api # Unbuffered output does not work well in Jenkins/Hudson, so in case diff --git a/demo/errors.py b/demo/errors.py index 8753c2b..ecd570b 100755 --- a/demo/errors.py +++ b/demo/errors.py @@ -5,7 +5,7 @@ from jenkinsflow.flow import serial -import get_jenkins_api +from jenkinsflow.demo import get_jenkins_api def main(api, securitytoken): diff --git a/demo/get_jenkins_api.py b/demo/get_jenkins_api.py index dba6531..60cce5d 100644 --- a/demo/get_jenkins_api.py +++ b/demo/get_jenkins_api.py @@ -2,7 +2,7 @@ from jenkinsflow.jenkins_api import Jenkins -import demo_security as security +from jenkinsflow.demo import demo_security as security def get_jenkins_api(): diff --git a/demo/hide_password.py b/demo/hide_password.py index e7815b6..85e56fd 100755 --- a/demo/hide_password.py +++ b/demo/hide_password.py @@ -5,7 +5,7 @@ from jenkinsflow.flow import serial -import get_jenkins_api +from jenkinsflow.demo import get_jenkins_api def main(api, securitytoken): diff --git a/demo/prefix.py b/demo/prefix.py index 29fde62..8b1d79b 100755 --- a/demo/prefix.py +++ b/demo/prefix.py @@ -4,7 +4,8 @@ # All rights reserved. This work is under a BSD license, see LICENSE.TXT. from jenkinsflow.flow import serial -import demo_security as security + +from jenkinsflow.demo import demo_security as security def main(api): diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..f71cd23 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,2 @@ +sphinx>=2.2.1 +sphinxcontrib-programoutput>=0.13 diff --git a/doc/source/index.rst b/doc/source/index.rst index a430d0e..faebc31 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,7 +9,7 @@ Welcome to jenkinsflow's documentation! The jenkinsflow package is used for controlling the invocation of `Jenkins `_ jobs in complex parallel and serial "flows". This effectively replaces the upstream/downstream dependencies in Jenkins with a fully scripted flow. Despite the name, this package may possibly also be used with `Hudson `_, although this is no longer being tested. -Note: this version requires Python 3.6.0 or newer. Use an older version for Python 2.7 support. +Note: this version requires Python 3.9.0 or newer. Use an older version for Python 3.6+ support. Package contents ---------------- diff --git a/doc/source/jenkinsflow.rst b/doc/source/jenkinsflow.rst index 600dbcf..92123ee 100644 --- a/doc/source/jenkinsflow.rst +++ b/doc/source/jenkinsflow.rst @@ -4,7 +4,7 @@ .. program-output:: cli/jenkinsflow --help :cwd: ../.. -.. program-output:: cli/jenkinsflow set_build_description --help +.. program-output:: cli/jenkinsflow set-build-description --help :cwd: ../.. You can also use :doc:`jenkinsflow.utils.set_build_description` in your python script. diff --git a/jenkins_api.py b/jenkins_api.py index 721b0e6..c65a5dc 100755 --- a/jenkins_api.py +++ b/jenkins_api.py @@ -28,7 +28,7 @@ class Jenkins(Speed, BaseApiMixin): """Optimized minimal set of methods needed for jenkinsflow to access Jenkins jobs. Args: - direct_uri (str): Should be a non-proxied uri if possible (e.g. http://localhost: if flow job is running on master) + direct_uri (str): Should be a non-proxied uri if possible (e.g. http://localhost: if flow job is running on 'built-in' node) The public URI will be retrieved from Jenkins and used in output. job_prefix_filter (str): Jobs with names that don't start with this string, will be skpped when polling Jenkins. If you are using Hudson and have many jobs, it might be a good idea to enable Team support and create a job-runner user, diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..c610822 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,102 @@ +import os +import sys +import argparse +import errno +from pathlib import Path +from os.path import join as jp +import subprocess + +import nox + +sys.path.append('.') + +from test.framework.cfg import ApiType, str_to_apis, dirs +from test.framework.nox_utils import cov_options_env, parallel_options +from test.framework.pytest_options import add_options + +_HERE = Path(__file__).resolve().parent +_TEST_DIR = _HERE/"test" +_DEMO_DIR = _HERE/"demo" +_DOC_DIR = _HERE/"doc" + +# Locally we have nox handle the different versions, but in each travis run there is only a single python which can always be found as just 'python' +_PY_VERSIONS = ["3.12", "3.11", "3.10", "3.9"] if not os.environ.get("TRAVIS_PYTHON_VERSION") else ["python"] +_IS_CI = os.environ.get("CI", "false").lower() == "true" + + +@nox.session(python=_PY_VERSIONS, reuse_venv=True) +def test(session): + """ + Test jenkinsflow. + Runs all tests mocked in hyperspeed, runs against Jenkins, using jenkins_api, and run script_api jobs. + + Normally jobs will be run in parallel, specifying 'job_delete' disables this. + The default options assumes that re-loading without deletions generates correct job config. + Tests that require jobs to be deleted/non-existing will delete the jobs, regardless of the 'job_delete' option. + + Will process some of the special pytest args: + direct_url: Direct Jenkins URL. Must be different from the URL set in Jenkins (and preferably non proxied). + apis: The apis totest, default all. + job_delete: Delete and re-load jobs into Jenkins. + job_load: Load jobs into Jenkins (skipping job load assumes all jobs already loaded and up to date). + """ + + session.install("--upgrade", ".", "-r", str(_TEST_DIR/"requirements.txt")) + + pytest_args = [] + if _IS_CI: + pytest_args.append("-vvv") + + parser = argparse.ArgumentParser(description="Process pytest options") + add_options(parser) + parsed_args = parser.parse_known_args(session.posargs)[0] + # print("parsed_args:", parsed_args) + apis = str_to_apis(parsed_args.api) + # print("noxfile, apis:", apis) + + parallel = parsed_args.job_load or parsed_args.job_delete + pytest_args.extend(parallel_options(parallel, apis)) + pytest_args.extend(["--capture=sys", "--instafail"]) + + pytest_args.extend(session.posargs) + + try: + os.makedirs(dirs.pseudo_install_dir) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + + env = {} + if apis != [ApiType.MOCK]: + print(f"Creating venv test installation in '{dirs.pseudo_install_dir}' to make files available to Jenkins.") + try: + session.run(f"{session.bin}/python", "-m", "venv", "--symlinks", "--prompt=jenkinsflow-test-venv", dirs.pseudo_install_dir) + python_executable = f"{dirs.pseudo_install_dir}/bin/python" + session.run(python_executable, "-m", "pip", "install", "--upgrade", ".") + env["JEKINSFLOW_TEST_JENKINS_API_PYTHON_EXECUTABLE"] = python_executable + subprocess.check_call([_HERE/"test/tmp_install.sh", _TEST_DIR, jp(dirs.test_tmp_dir, "test")]) + subprocess.check_call([_HERE/"test/tmp_install.sh", _DEMO_DIR, jp(dirs.test_tmp_dir, "demo")]) + except: + print(f"Failed venv test installation to '{dirs.pseudo_install_dir}'", file=sys.stderr) + raise + + cov_file = _HERE/".coverage" + if os.path.exists(cov_file): + os.remove(cov_file) + + cov_opts, cov_env = cov_options_env(apis, True) + env.update(cov_env) + # env["COVERAGE_DEBUG"] = "config" + session.run("pytest", "--capture=sys", *cov_opts, *pytest_args, env=env) + + +@nox.session(python=_PY_VERSIONS[0], reuse_venv=True) +def docs(session): + session.install("--upgrade", ".", "-r", str(_DOC_DIR/"requirements.txt")) + session.run("make", "-C", "doc/source", "html") + + +@nox.session(python=_PY_VERSIONS[0], reuse_venv=True) +def cli(session): + session.install("--upgrade", ".") + session.run("jenkinsflow", "set-build-description", "--help") diff --git a/pytest.ini b/pytest.ini index 32dfcd7..bb60f36 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,4 +4,4 @@ minversion = 2.9.2 testpaths = test norecursedirs = __pycache__ utils cli doc dist demo -addopts = -p no:warnings \ No newline at end of file +addopts = -p no:warnings --ff diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d550aea --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +setuptools>=69.0.3 +requests>=2.20,<=3.0 +atomicfile>=1.0,<=2.0 +click>=7.0 +tenjin>=1.1.1 + +# Required by the script API: +# You need to install python(3)-devel to be be able to install psutil, see INSTALL.md +psutil>=5.6.6 +setproctitle>=1.1.10 + +# Required by the job dependency graph visualisation +bottle>=0.12.1 diff --git a/rest_api_wrapper.py b/rest_api_wrapper.py index 4a4d70b..da3b68b 100644 --- a/rest_api_wrapper.py +++ b/rest_api_wrapper.py @@ -34,6 +34,11 @@ def _check_response(response, good_responses=(200,)): raise AuthError(ex) from ex if response.status_code == 403: raise ClientError(ex) from ex + if response.status_code == 500: + # TODO: Workaround for https://issues.jenkins.io/browse/JENKINS-63845 + # 500 Server Error: Server Error for url: http://localhost:8080/queue/cancelItem?id=14406 + if "Server Error for url:" in str(ex) and "queue/cancelItem" in str(ex): + raise ResourceNotFound(ex) from ex raise # TODO: This is dubious, maybe we should raise here instead. diff --git a/script_api.py b/script_api.py index 93289ff..ce2e628 100644 --- a/script_api.py +++ b/script_api.py @@ -232,7 +232,7 @@ def invoke(self, securitytoken, build_params, cause, description): BUILD_DISPLAY_NAME='#' + repr(build_number), JOB_NAME=self.name, BUILD_TAG='jenkinsflow-' + self.name + '-' + repr(build_number), - NODE_NAME='master', + NODE_NAME='built-in', NODE_LABELS='', WORKSPACE=self.workspace, JENKINS_HOME=self.jenkins.public_uri, diff --git a/setup.py b/setup.py index 8a20ab6..29c4d8a 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ -import sys, os +import os from setuptools import setup -from setuptools.command.test import test as TestCommand PROJECT_ROOT, _ = os.path.split(__file__) @@ -16,54 +15,12 @@ on_rtd = os.environ.get('READTHEDOCS') == 'True' is_ci = os.environ.get('CI', 'false').lower() == 'true' - _here = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(_here, 'py_version_check.py')) as ff: exec(ff.read()) - -class Test(TestCommand): - def initialize_options(self): - TestCommand.initialize_options(self) - self.pytest_args = [] - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - import test.run - if is_ci: - print("Running under CI") - # Note 'mock' is also hardcoded in .travis.yml - sys.exit(test.run.cli(apis='mock,script', mock_speedup=10)) - sys.exit(test.run.cli(apis='mock,script')) - - -flow_requires = ['atomicfile>=1.0,<=2.0'] -cli_requires = ['click>=6.0'] -job_load_requires = ['tenjin>=1.1.1'] -jenkins_api_requires = ['requests>=2.20,<=3.0'] -# You need to install python(3)-devel to be be able to install psutil, see INSTALL.md -script_api_requires = ['psutil>=5.6.6', 'setproctitle>=1.1.10'] -visual_requires = ['bottle>=0.12.1'] - -if not on_rtd: - install_requires = flow_requires + cli_requires + job_load_requires + jenkins_api_requires + script_api_requires + visual_requires -else: - install_requires = flow_requires + cli_requires + jenkins_api_requires + script_api_requires - -tests_require = [ - 'pytest>=5.3.5,<5.4', 'pytest-cov>=2.4.0,<3', 'pytest-instafail>=0.3.0', 'pytest-xdist>=1.16,<2', - 'click>=6.0', 'tenjin>=1.1.1', 'bottle>=0.12', 'objproxies>=0.9.4', - # The test also tests creation of the documentation - 'sphinx>=2.2.1', 'sphinxcontrib-programoutput>=0.13', -] - -extras = { - 'test': tests_require, -} +with open(os.path.join(_here, 'requirements.txt')) as ff: + install_requires=[req.strip() for req in ff.readlines() if req.strip() and req.strip()[0] != "#"] if __name__ == "__main__": setup( @@ -71,17 +28,32 @@ def run_tests(self): version_command=('git describe', 'pep440-git'), author=PROJECT_AUTHORS, author_email=PROJECT_EMAILS, - packages=['jenkinsflow', 'jenkinsflow.utils', 'jenkinsflow.cli'], - package_dir={'jenkinsflow':'.', 'jenkinsflow.utils': 'utils', 'jenkinsflow.cli': 'cli'}, + packages=[ + 'jenkinsflow', + 'jenkinsflow.utils', + 'jenkinsflow.cli', + 'jenkinsflow.demo', + 'jenkinsflow.demo.jobs', + 'jenkinsflow.test', + 'jenkinsflow.test.framework', + 'jenkinsflow.test.framework.cfg', + ], + package_dir={ + 'jenkinsflow': '.', + 'jenkinsflow.utils': 'utils', + 'jenkinsflow.cli': 'cli', + 'jenkinsflow.demo': 'demo', + 'jenkinsflow.demo.jobs': 'demo/jobs', + 'jenkinsflow.test': 'test', + 'jenkinsflow.test.framework': 'test/framework', + 'jenkinsflow.test.framework.cfg': 'test/framework/cfg', + }, zip_safe=True, include_package_data=False, - python_requires='>=3.6.0', + python_requires='>=3.9.0', install_requires=install_requires, setup_requires='setuptools-version-command>=2.2', test_suite='test', - tests_require=tests_require, - extras_require=extras, - cmdclass={'test': Test}, url=PROJECT_URL, description=SHORT_DESCRIPTION, long_description=LONG_DESCRIPTION, @@ -94,9 +66,10 @@ def run_tests(self): 'License :: OSI Approved :: BSD License', 'Natural Language :: English', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - # 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.9', 'Topic :: Software Development :: Testing', ], entry_points=''' diff --git a/test/conftest.py b/test/conftest.py index 3f407ed..f03e195 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,60 +3,111 @@ import os import sys -from pathlib import Path +import re +from itertools import chain +from typing import List import pytest from pytest import fixture # pylint: disable=no-name-in-module from click.testing import CliRunner -from .framework.cfg import ApiType, Urls +from .framework import pytest_options +from .framework.cfg import ApiType, opts_to_test_cfg +# Note: You can't (indirectly) import stuff from jenkinsflow here, it messes up the coverage -_HERE = Path(__file__).resolve().parent -_DEMO_DIR = (_HERE/'../demo').resolve() -sys.path.extend([str(_DEMO_DIR), str(_DEMO_DIR/"jobs")]) -# Note: You can't (indirectly) import stuff from jenkinsflow here, it messes up the coverage +# Singleton config +TEST_CFG = None + def pytest_addoption(parser): - parser.addoption( - "--api", - action="store", - metavar="NAME", - default=','.join(at.name for at in list(ApiType)), - help=f"Comma separated list of APIs to test. Default is all defined apis: {','.join([at.name for at in list(ApiType)])}", - ) + """pytest hook""" + pytest_options.add_options(parser) def pytest_configure(config): + global TEST_CFG + + """pytest hook""" # Register api marker config.addinivalue_line("markers", "apis(*ApiType): mark test to run only when using specified apis") config.addinivalue_line("markers", "not_apis(*ApiType): mark test NOT to run when using specified apis") - try: - apis = [ApiType[api.strip().upper()] for api in config.getoption('--api').split(',')] - except KeyError as ex: - print(ex, file=sys.stderr) - print(f"'{ex}' cannot be converted to ApiType. Should be one or more of: {','.join([at.name for at in list(ApiType)])}") - raise + TEST_CFG = opts_to_test_cfg( + config.getoption(pytest_options.OPT_DIRECT_URL), + config.getoption(pytest_options.OPT_JOB_LOAD), + config.getoption(pytest_options.OPT_JOB_DELETE), + config.getoption(pytest_options.OPT_MOCK_SPEEDUP), + config.getoption(pytest_options.OPT_API), + ) + config.cuctom_cfg = TEST_CFG + + +def pytest_collection_modifyitems(items: List[pytest.Item], config) -> None: + """pytest hook""" + selected_api_types = config.cuctom_cfg.apis + item_api_type_regex = re.compile(r'.*\[ApiType\.(.*)\]') + remaining = [] + deselected = [] + + def filter_items_by_api_type(item): + if "api_type" not in item.fixturenames: + for om in item.own_markers: + if om.name in ("apis", "not_apis"): + location = ':'.join(str(place) for place in item.location) + raise Exception(f"{location}: Error: A test using the 'apis' or 'not_apis' marker must also use the 'api_type' fixture.") + remaining.append(item) + return - print("APIs:", apis) + # We must have an ApiType in name now, since api_type fixture was used + current_item_api = ApiType[item_api_type_regex.match(item.name).groups()[0]] + if current_item_api not in selected_api_types: + print(f"{':'.join(str(place) for place in item.location)} DESELECTED ({current_item_api} not in {selected_api_types})") + deselected.append(item) + return + remaining.append(item) -@pytest.fixture(params=[ApiType.MOCK, ApiType.JENKINS, ApiType.SCRIPT]) -def api_type(request): + + print() + for item in items: + filter_items_by_api_type(item) + + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = remaining + + +@pytest.fixture() +def options(): + """Access to test configuration objects.""" + return TEST_CFG + + +@pytest.fixture(params=list(ApiType)) +def api_type(request, options): """ApiType fixture""" selected_api_type = request.param + assert isinstance(selected_api_type, ApiType) for apimarker in request.node.iter_markers("apis"): apis = apimarker.args + for allowed_api_type in apis: + assert isinstance(allowed_api_type, ApiType) + if selected_api_type not in apis: - pytest.skip("test requires one the following apis {apis}, current {api_type}".format(apis=apis, api_type=selected_api_type)) + pytest.skip(f"only for {apis} APIs, current {selected_api_type}") + return selected_api_type for not_apimarker in request.node.iter_markers("not_apis"): not_apis = not_apimarker.args + for not_api_type in not_apis: + assert isinstance(not_api_type, ApiType) + if selected_api_type in not_apis: - pytest.skip("test is not run for the following apis {apis}, current {api_type}".format(apis=not_apis, api_type=selected_api_type)) + pytest.skip(f"not for {not_apis} APIs, current {selected_api_type}") + return selected_api_type return selected_api_type @@ -117,21 +168,21 @@ def fin(): @fixture -def env_base_url(request, api_type): +def env_base_url(request, api_type, options): # Fake that we are running from inside jenkins job - public_url = Urls.public_url(api_type) + public_url = options.urls.public_url(api_type) _set_jenkins_url_env_if_not_set_fixture(public_url, request) return public_url @fixture -def env_base_url_trailing_slash(request, api_type): - _set_jenkins_url_env_if_not_set_fixture(Urls.public_url(api_type) + '/', request) +def env_base_url_trailing_slash(request, api_type, options): + _set_jenkins_url_env_if_not_set_fixture(options.urls.public_url(api_type) + '/', request) @fixture -def env_base_url_trailing_slashes(request, api_type): - _set_jenkins_url_env_if_not_set_fixture(Urls.public_url(api_type) + '//', request) +def env_base_url_trailing_slashes(request, api_type, options): + _set_jenkins_url_env_if_not_set_fixture(options.urls.public_url(api_type) + '//', request) @fixture @@ -142,10 +193,10 @@ def env_no_base_url(request): @fixture -def env_different_base_url(request): +def env_different_base_url(request, options): # Fake that we are running from inside jenkins job # This url is not used, but should simply be different fron direct_url used in test, to simulate proxied jenkins - _set_jenkins_url_env_fixture(Urls.proxied_public_url, request) + _set_jenkins_url_env_fixture(options.urls.proxied_public_url, request) @fixture diff --git a/test/demos_test.py b/test/demos_test.py index 0f8f562..88f2f66 100644 --- a/test/demos_test.py +++ b/test/demos_test.py @@ -1,7 +1,8 @@ # Copyright (c) 2012 - 2015 Lars Hupfeldt Nielsen, Hupfeldt IT # All rights reserved. This work is under a BSD license, see LICENSE.TXT. -import imp +import importlib.util +import importlib.machinery from pathlib import Path import pytest @@ -9,7 +10,7 @@ from jenkinsflow.flow import parallel, JobControlFailException -from demo import basic, calculated_flow, prefix, hide_password, errors +from jenkinsflow.demo import basic, calculated_flow, prefix, hide_password, errors from .framework import api_select from .framework.cfg import ApiType @@ -19,22 +20,36 @@ _DEMO_JOBS_DIR = (_HERE/"../demo/jobs").resolve() +def _load_source(modname, filename): + # https://docs.python.org/3/whatsnew/3.12.html#imp + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module + + def load_demo_jobs(demo, api_type): print("\nLoad jobs for demo:", demo.__name__) - job_load_module_name = demo.__name__.replace("demo.", "") + '_jobs' - job_load = imp.load_source(job_load_module_name, str(_DEMO_JOBS_DIR/(job_load_module_name + '.py'))) + simple_demo_name = demo.__name__.replace("jenkinsflow.", "").replace("demo.", "") + job_load_module_name = simple_demo_name + '_jobs' + job_load = _load_source(job_load_module_name, str(_DEMO_JOBS_DIR/(job_load_module_name + '.py'))) api = job_load.create_jobs(api_type) - return api + flow_job_name = simple_demo_name + "__0flow" + return flow_job_name def _test_demo(demo, api_type): - load_demo_jobs(demo, api_type) + flow_job_name = load_demo_jobs(demo, api_type) with api_select.api(__file__, api_type, fixed_prefix="jenkinsflow_demo__") as api: - api.job(demo.__name__ + "__0flow", max_fails=0, expect_invocations=1, expect_order=1) + api.job(flow_job_name, max_fails=0, expect_invocations=1, expect_order=1) with parallel(api, timeout=70, job_name_prefix=api.job_name_prefix) as ctrl1: - ctrl1.invoke(demo.__name__ + "__0flow") + ctrl1.invoke(flow_job_name) @pytest.mark.not_apis(ApiType.SCRIPT) # TODO: script api is not configured to run demos @@ -59,12 +74,11 @@ def test_demos_hide_password(api_type): @pytest.mark.not_apis(ApiType.SCRIPT) # TODO: script api is not configured to run demos def test_demos_with_errors(api_type): - demo = errors - load_demo_jobs(demo, api_type) + flow_job_name = load_demo_jobs(errors, api_type) with api_select.api(__file__, api_type, fixed_prefix="jenkinsflow_demo__") as api: - api.job(demo.__name__ + "__0flow", max_fails=1, expect_invocations=1, expect_order=1) + api.job(flow_job_name, max_fails=1, expect_invocations=1, expect_order=1) with raises(JobControlFailException): with parallel(api, timeout=70, job_name_prefix=api.job_name_prefix) as ctrl1: - ctrl1.invoke(demo.__name__ + "__0flow") + ctrl1.invoke(flow_job_name) diff --git a/test/framework/abort_job.py b/test/framework/abort_job.py index cb182bc..d4826b0 100644 --- a/test/framework/abort_job.py +++ b/test/framework/abort_job.py @@ -4,21 +4,28 @@ import sys import time import subprocess +from pathlib import Path +from jenkinsflow.test.conftest import TEST_CFG from . import api_select from .logger import log, logt -from .cfg import ApiType +from .cfg import ApiType, AllCfg, opt_strs_to_test_cfg, test_cfg_to_opt_strs -def _abort(log_file, test_file_name, api_type, fixed_prefix, job_name, sleep_time): +def _abort(log_file, test_file_name, api_type, fixed_prefix, job_name, sleep_time, test_cfg: AllCfg): log(log_file, '\n') - logt(log_file, "Waiting to abort job:", job_name) - logt(log_file, "args:", test_file_name, fixed_prefix, job_name, sleep_time) + logt(log_file, "Subprocess", __file__) + logt(log_file, f"Waiting {sleep_time} seconds to abort job:", job_name) + logt(log_file, "args:", test_file_name, api_type, fixed_prefix, job_name, sleep_time, test_cfg) time.sleep(sleep_time) - with api_select.api(test_file_name, api_type, fixed_prefix='jenkinsflow_test__' + fixed_prefix + '__', login=True) as api: + logt(log_file, "sleep", sleep_time, "finished") + with api_select.api(test_file_name, api_type, fixed_prefix='jenkinsflow_test__' + fixed_prefix + '__', login=True, options=test_cfg) as api: api.job(job_name, max_fails=0, expect_invocations=0, expect_order=None) + logt(log_file, "job defined", api) api.poll() + logt(log_file, "polled") api.quick_poll() + logt(log_file, "quick polled") abort_me = api.get_job(api.job_name_prefix + job_name) logt(log_file, "Abort job:", abort_me) @@ -28,8 +35,15 @@ def _abort(log_file, test_file_name, api_type, fixed_prefix, job_name, sleep_tim if __name__ == '__main__': job_name = sys.argv[4] - with open(job_name, 'a+') as log_file: - _abort(log_file, sys.argv[1], ApiType[sys.argv[2]], sys.argv[3], job_name, int(sys.argv[5])) + with open(job_name + '.log', 'a+') as log_file: + try: + _abort( + log_file, sys.argv[1], ApiType[sys.argv[2]], sys.argv[3], job_name, int(sys.argv[5]), + test_cfg = opt_strs_to_test_cfg( + direct_url=sys.argv[6], load_jobs=sys.argv[7], delete_jobs=sys.argv[8], mock_speedup=sys.argv[9], apis_str=sys.argv[10])) + except Exception as ex: + print(ex, file=log_file) + raise def abort(api, job_name, sleep_time): @@ -37,8 +51,10 @@ def abort(api, job_name, sleep_time): if api.api_type == ApiType.MOCK: return - ff = __file__.replace('.pyc', '.py') - args = [sys.executable, ff, api.file_name, api.api_type.name, api.func_name.replace('test_', ''), job_name, str(sleep_time)] - with open(job_name, 'w') as log_file: + args = [sys.executable, "-m", "jenkinsflow.test.framework.abort_job", + api.file_name, api.api_type.name, api.func_name.replace('test_', ''), job_name, str(sleep_time), + *test_cfg_to_opt_strs(TEST_CFG, api.api_type) + ] + with open(job_name + '.log', 'w') as log_file: logt(log_file, "Invoking abort subprocess.", args) subprocess.Popen(args) diff --git a/test/framework/api_select.py b/test/framework/api_select.py index 226e19d..a34c220 100644 --- a/test/framework/api_select.py +++ b/test/framework/api_select.py @@ -5,7 +5,9 @@ from jenkinsflow.unbuffered import UnBuffered -from .cfg import ApiType, Urls, JobLoad +from jenkinsflow.test.conftest import TEST_CFG + +from .cfg import ApiType, AllCfg from .cfg.speedup import speedup @@ -15,8 +17,9 @@ def api(file_name, api_type, login=None, fixed_prefix=None, url_or_dir=None, fake_public_uri=None, invocation_class=None, - username=None, password=None): + username=None, password=None, *, options: AllCfg = None): """Factory to create either Mock or Wrap api""" + options = options or TEST_CFG base_name = os.path.basename(file_name).replace('.pyc', '.py') job_name_prefix = _file_name_subst.sub('', base_name) func_name = None @@ -44,9 +47,9 @@ def api(file_name, api_type, login=None, fixed_prefix=None, url_or_dir=None, fak api_type = ApiType.JENKINS print('Using:', api_type) - url_or_dir = url_or_dir or Urls.direct_url(api_type) - reload_jobs = not JobLoad.skip_job_load() and not fixed_prefix - pre_delete_jobs = not JobLoad.skip_job_delete() + url_or_dir = url_or_dir or options.urls.direct_url(api_type) + reload_jobs = not options.job_load.skip_job_load() and not fixed_prefix + pre_delete_jobs = not options.job_load.skip_job_delete() from .cfg import jenkins_security if login is None: @@ -65,7 +68,7 @@ def api(file_name, api_type, login=None, fixed_prefix=None, url_or_dir=None, fak from .api_wrapper import JenkinsTestWrapperApi return JenkinsTestWrapperApi(file_name, func_name, func_num_params, job_name_prefix, reload_jobs, pre_delete_jobs, url_or_dir, fake_public_uri, username, password, jenkins_security.securitytoken, login=login, - invocation_class=invocation_class) + invocation_class=invocation_class, python_executable=os.environ["JEKINSFLOW_TEST_JENKINS_API_PYTHON_EXECUTABLE"]) if api_type == ApiType.SCRIPT: from .api_wrapper import ScriptTestWrapperApi return ScriptTestWrapperApi(file_name, func_name, func_num_params, job_name_prefix, reload_jobs, pre_delete_jobs, @@ -73,6 +76,6 @@ def api(file_name, api_type, login=None, fixed_prefix=None, url_or_dir=None, fak invocation_class=invocation_class) if api_type == ApiType.MOCK: from .mock_api import MockApi - return MockApi(job_name_prefix, speedup(), Urls.direct_url(api_type)) + return MockApi(job_name_prefix, speedup(), options.urls.direct_url(api_type), python_executable=sys.executable) raise Exception(f"Unhandled api_type: {repr(api_type)} - {api_type.__class__.__module__} was compared to {ApiType.MOCK.__class__.__module__}") diff --git a/test/framework/api_wrapper.py b/test/framework/api_wrapper.py index 9f30fab..ea7b328 100644 --- a/test/framework/api_wrapper.py +++ b/test/framework/api_wrapper.py @@ -3,6 +3,7 @@ import sys, time, os from os.path import join as jp +from pathlib import Path from objproxies import ObjectWrapper @@ -13,7 +14,7 @@ from .base_test_api import TestJob, TestJenkins, Jobs as TestJobs from .mock_api import MockJob -from .cfg import ApiType, JobLoad +from .cfg import ApiType from .cfg.dirs import test_tmp_dir, pseudo_install_dir from .cfg import jenkins_security @@ -86,7 +87,7 @@ def __exit__(self, exc_type, exc_value, traceback): class _TestWrapperApi(): - def __init__(self, file_name, func_name, func_num_params, reload_jobs, pre_delete_jobs, securitytoken, direct_url, fake_public_uri): + def __init__(self, file_name, func_name, func_num_params, reload_jobs, pre_delete_jobs, securitytoken, direct_url, fake_public_uri, python_executable): self.file_name = file_name self.func_name = func_name self.func_num_params = func_num_params @@ -96,12 +97,14 @@ def __init__(self, file_name, func_name, func_num_params, reload_jobs, pre_delet self.direct_url = direct_url self.fake_public_uri = fake_public_uri self.using_job_creator = False + self.python_executable = python_executable def _jenkins_job(self, name, exec_time, params, script, print_env, create_job, always_load, num_builds_to_keep, final_result_use_cli, set_build_descriptions): # Create job in Jenkins if self.reload_jobs or always_load: context = dict( + python_executable=self.python_executable, exec_time=exec_time, params=params or (), script=script, @@ -157,16 +160,17 @@ def job(self, name, max_fails, expect_invocations, expect_order, exec_time=None, initial_buildno=initial_buildno, invocation_delay=invocation_delay, unknown_result=unknown_result, final_result=final_result, serial=serial, params=params, flow_created=flow_created, create_job=create_job, disappearing=disappearing, non_existing=non_existing, kill=kill, allow_running=allow_running, api=self, final_result_use_cli=final_result_use_cli, - set_build_descriptions=set_build_descriptions) + set_build_descriptions=set_build_descriptions, python_executable=self.python_executable) self.test_jobs[job_name] = job def flow_job(self, name=None, params=None): - """ - Creates a flow job + """Creates a flow job. + For running demo/test flow script as jenkins job - Requires jenkinsflow to be copied to 'pseudo_install_dir' and all jobs to be loaded beforehand (e.g. test.py has been run) + Requires jenkinsflow to be installed in venv 'pseudo_install_dir' and all jobs to be loaded beforehand (e.g. test.py has been run) Returns job name """ + name = '0flow_' + name if name else '0flow' job_name = (self.job_name_prefix or '') + name # TODO Handle script api @@ -175,14 +179,18 @@ def flow_job(self, name=None, params=None): # Note: Use -B to avoid permission problems with .pyc files created from commandline test if self.func_name: - script = "export PYTHONPATH=" + test_tmp_dir + "\n" - script += JobLoad.skip_job_load_sh_export_str() + "\n" - # script += "export " + ApiType.JENKINS.env_name() + "=true\n" # pylint: disable=no-member # Supply dummy args for the py.test fixtures dummy_args = ','.join(['0' for _ in range(self.func_num_params)]) - script += sys.executable + " -Bc "import sys; from jenkinsflow.test." + self.file_name.replace('.py', '') + " import *; sys.exit(test_" + self.func_name + "(" + dummy_args + "))"" + script = ( + f"export PYTHONPATH={test_tmp_dir}:$PYTHONPATH\n" + f"python -Bc "" + f"import sys;" + f"from test.{self.file_name.replace('.py', '')} import *;" + f"sys.exit(test_{self.func_name}({dummy_args}))" + """ + ) else: - script = sys.executable + " -B " + jp(pseudo_install_dir, 'demo', self.file_name) + script = "python -B " + jp(test_tmp_dir, 'demo', self.file_name) self._jenkins_job(job_name, exec_time=0.5, params=params, script=script, print_env=False, create_job=None, always_load=False, num_builds_to_keep=4, final_result_use_cli=False, set_build_descriptions=()) return job_name @@ -239,15 +247,16 @@ class JenkinsTestWrapperApi(_TestWrapperApi, jenkins_api.Jenkins, TestJenkins): job_xml_template = jp(here, 'job.xml.tenjin') def __init__(self, file_name, func_name, func_num_params, job_name_prefix, reload_jobs, pre_delete_jobs, direct_url, fake_public_uri, - username, password, securitytoken, login, invocation_class): + username, password, securitytoken, login, invocation_class, python_executable): TestJenkins.__init__(self, job_name_prefix=job_name_prefix) if login: - jenkins_api.Jenkins.__init__(self, direct_uri=direct_url, job_prefix_filter=job_name_prefix, username=username, password=password, - invocation_class=invocation_class) + jenkins_api.Jenkins.__init__( + self, direct_uri=direct_url, job_prefix_filter=job_name_prefix, username=username, password=password, invocation_class=invocation_class) else: jenkins_api.Jenkins.__init__(self, direct_uri=direct_url, job_prefix_filter=job_name_prefix, invocation_class=invocation_class) self.job_loader_jenkins = jenkins_api.Jenkins(direct_uri=direct_url, job_prefix_filter=job_name_prefix, username=username, password=password) - _TestWrapperApi.__init__(self, file_name, func_name, func_num_params, reload_jobs, pre_delete_jobs, securitytoken, direct_url, fake_public_uri) + _TestWrapperApi.__init__( + self, file_name, func_name, func_num_params, reload_jobs, pre_delete_jobs, securitytoken, direct_url, fake_public_uri, python_executable) class ScriptTestWrapperApi(_TestWrapperApi, script_api.Jenkins, TestJenkins): @@ -258,9 +267,11 @@ def __init__(self, file_name, func_name, func_num_params, job_name_prefix, reloa username, password, securitytoken, login, invocation_class): TestJenkins.__init__(self, job_name_prefix=job_name_prefix) if login: - script_api.Jenkins.__init__(self, direct_uri=direct_url, job_prefix_filter=job_name_prefix, username=username, password=password, - invocation_class=invocation_class) + script_api.Jenkins.__init__( + self, direct_uri=direct_url, job_prefix_filter=job_name_prefix, username=username, password=password, invocation_class=invocation_class) else: script_api.Jenkins.__init__(self, direct_uri=direct_url, job_prefix_filter=job_name_prefix, invocation_class=invocation_class) self.job_loader_jenkins = script_api.Jenkins(direct_uri=direct_url, job_prefix_filter=job_name_prefix, username=username, password=password) - _TestWrapperApi.__init__(self, file_name, func_name, func_num_params, reload_jobs, pre_delete_jobs, securitytoken, direct_url, fake_public_uri) + _TestWrapperApi.__init__( + self, file_name, func_name, func_num_params, reload_jobs, pre_delete_jobs, securitytoken, direct_url, fake_public_uri, + str(Path(sys.executable).resolve())) diff --git a/test/framework/cfg/__init__.py b/test/framework/cfg/__init__.py index a3ca666..f31b59a 100644 --- a/test/framework/cfg/__init__.py +++ b/test/framework/cfg/__init__.py @@ -1,3 +1,4 @@ -from .api_type import ApiType +from .api_type import ApiType, str_to_apis, apis_to_str from .urls import Urls from .job_load import JobLoad +from .all_cfg import AllCfg, opts_to_test_cfg, opt_strs_to_test_cfg, test_cfg_to_opt_strs diff --git a/test/framework/cfg/all_cfg.py b/test/framework/cfg/all_cfg.py new file mode 100644 index 0000000..d2b9e5e --- /dev/null +++ b/test/framework/cfg/all_cfg.py @@ -0,0 +1,47 @@ +# Copyright (c) 2012 - 2024 Lars Hupfeldt Nielsen, Hupfeldt IT +# All rights reserved. This work is under a BSD license, see LICENSE.TXT. + +from . import ApiType, str_to_apis, apis_to_str, Urls, JobLoad, dirs, speedup + + +class AllCfg(): + def __init__(self, urls: Urls, job_load: JobLoad, apis: list[ApiType]): + self.urls = urls + self.job_load = job_load + self.apis = apis + + +def opts_to_test_cfg(direct_url: str, load_jobs: bool, delete_jobs: bool, mock_speedup: int, apis_str: str) -> AllCfg: + urls = Urls(direct_url, dirs.job_script_dir) + job_load = JobLoad(load_jobs, delete_jobs) + + speedup.select_speedup(mock_speedup) + + try: + apis = str_to_apis(apis_str) + print("APIs:", apis) + except KeyError as ex: + print(ex, file=sys.stderr) + print(f"'{ex}' cannot be converted to ApiType. Should be one or more of: {','.join([at.name for at in list(ApiType)])}") + raise + + return AllCfg(urls, job_load, apis) + + +def opt_strs_to_test_cfg(direct_url: str, load_jobs: str, delete_jobs: str, mock_speedup: str, apis_str: str) -> AllCfg: + return opts_to_test_cfg( + direct_url = direct_url, + load_jobs = True if load_jobs.lower() == "true" else False, + delete_jobs = True if delete_jobs.lower() == "true" else False, + mock_speedup = int(mock_speedup), + apis_str = apis_str) + + +def test_cfg_to_opt_strs(test_cfg: AllCfg, api_type: ApiType) -> tuple[str, str, str, str, str]: + return ( + test_cfg.urls.direct_url(api_type), + str(test_cfg.job_load.load_jobs).lower(), + str(test_cfg.job_load.delete_jobs).lower(), + str(speedup.speedup()), + apis_to_str(test_cfg.apis), + ) diff --git a/test/framework/cfg/api_type.py b/test/framework/cfg/api_type.py index 0c22569..9bbb94f 100644 --- a/test/framework/cfg/api_type.py +++ b/test/framework/cfg/api_type.py @@ -5,3 +5,11 @@ class ApiType(Enum): JENKINS = 0 SCRIPT = 1 MOCK = 2 + + +def str_to_apis(api_option: str) -> list[ApiType]: + return [ApiType[api.strip().upper()] for api in api_option.split(',')] + + +def apis_to_str(apis: list[ApiType]) -> str: + return ','.join([api.name for api in apis]) diff --git a/test/framework/cfg/job_load.py b/test/framework/cfg/job_load.py index 78401b9..7efdc9f 100644 --- a/test/framework/cfg/job_load.py +++ b/test/framework/cfg/job_load.py @@ -1,20 +1,15 @@ import os -from .os_env import ENV_VAR_PREFIX - - class JobLoad(): - skip_job_load_env_var_name = ENV_VAR_PREFIX + 'SKIP_JOB_LOAD' - skip_job_delete_env_var_name = ENV_VAR_PREFIX + 'SKIP_JOB_DELETE' + def __init__(self, load_jobs: bool, delete_jobs: bool): + self.load_jobs = load_jobs + self.delete_jobs = delete_jobs - @staticmethod - def skip_job_delete(): - return os.environ.get(JobLoad.skip_job_delete_env_var_name) == 'true' + def skip_job_delete(self): + return not self.delete_jobs - @staticmethod - def skip_job_load(): - return os.environ.get(JobLoad.skip_job_load_env_var_name) == 'true' + def skip_job_load(self): + return not self.load_jobs - @staticmethod - def skip_job_load_sh_export_str(): + def skip_job_load_sh_export_str(self): return 'export ' + JobLoad.skip_job_load_env_var_name + '=true' diff --git a/test/framework/cfg/os_env.py b/test/framework/cfg/os_env.py deleted file mode 100644 index 1f74da3..0000000 --- a/test/framework/cfg/os_env.py +++ /dev/null @@ -1 +0,0 @@ -ENV_VAR_PREFIX = "JENKINSFLOW_" diff --git a/test/framework/cfg/urls.py b/test/framework/cfg/urls.py index d67e545..7764991 100644 --- a/test/framework/cfg/urls.py +++ b/test/framework/cfg/urls.py @@ -2,70 +2,62 @@ from . import dirs from . import ApiType -from .os_env import ENV_VAR_PREFIX class Urls(): - direct_url_env_var_name = ENV_VAR_PREFIX + 'DIRECT_URL' - script_dir_env_var_name = ENV_VAR_PREFIX + 'SCRIPT_DIR' - + default_direct_url = "http://localhost:8080" proxied_public_url = "http://myproxy.mydom/jenkins" - @staticmethod - def direct_url(api_type): - if api_type != ApiType.SCRIPT: - durl = os.environ.get(Urls.direct_url_env_var_name) - return 'http://localhost:8080' if durl is None else durl.rstrip('/') - else: - return Urls._script_dir() - - @staticmethod - def public_url(api_type): - if api_type != ApiType.SCRIPT: - purl = os.environ.get('JENKINS_URL') or os.environ.get('HUDSON_URL') - return "http://" + socket.getfqdn() + ':' + repr(8080) + '/' if purl is None else purl - else: - return Urls._script_dir() - - @staticmethod - def direct_cli_url(api_type): - if api_type != ApiType.SCRIPT: - purl = os.environ.get('JENKINS_URL') - if purl: - return Urls.direct_url(api_type) + '/jnlpJars/jenkins-cli.jar' - purl = os.environ.get('HUDSON_URL') - if purl: - return Urls.direct_url(api_type) + '/jnlpJars/hudson-cli.jar' - # If neither JENKINS nor HUDSON URL is set, assume jenkins for testing - return Urls.direct_url(api_type) + '/jnlpJars/jenkins-cli.jar' - else: - return Urls._script_dir() - - @staticmethod - def public_cli_url(api_type): - if api_type != ApiType.SCRIPT: - purl = os.environ.get('JENKINS_URL') - if purl: - return purl.rstrip('/') + '/jnlpJars/jenkins-cli.jar' - purl = os.environ.get('HUDSON_URL') - if purl: - return purl.rstrip('/') + '/jnlpJars/hudson-cli.jar' - # If neither JENKINS nor HUDSON URL is set, assume jenkins for testing - return "http://" + socket.getfqdn() + ':' + repr(8080) + '/jnlpJars/jenkins-cli.jar' - else: - return Urls._script_dir() - - @staticmethod - def proxied_public_cli_url(api_type): + def __init__(self, direct_url, script_dir): + self._direct_url = direct_url.rstrip('/') if direct_url else self.default_direct_url + self._script_dir = script_dir.rstrip('/') if script_dir else dirs.job_script_dir + + def direct_url(self, api_type): + if api_type == ApiType.SCRIPT: + return self._script_dir + + return self._direct_url + + def public_url(self, api_type): + if api_type == ApiType.SCRIPT: + return self._script_dir + + purl = os.environ.get('JENKINS_URL') or os.environ.get('HUDSON_URL') + return "http://" + socket.getfqdn() + ':' + repr(8080) + '/' if purl is None else purl + + def direct_cli_url(self, api_type): + if api_type == ApiType.SCRIPT: + return self._script_dir + + purl = os.environ.get('JENKINS_URL') + if purl: + return self.direct_url(api_type) + '/jnlpJars/jenkins-cli.jar' + purl = os.environ.get('HUDSON_URL') + if purl: + return self.direct_url(api_type) + '/jnlpJars/hudson-cli.jar' + + # If neither JENKINS nor HUDSON URL is set, assume jenkins for testing + return self.direct_url(api_type) + '/jnlpJars/jenkins-cli.jar' + + def public_cli_url(self, api_type): + if api_type == ApiType.SCRIPT: + return self._script_dir + + purl = os.environ.get('JENKINS_URL') + if purl: + return purl.rstrip('/') + '/jnlpJars/jenkins-cli.jar' + purl = os.environ.get('HUDSON_URL') + if purl: + return purl.rstrip('/') + '/jnlpJars/hudson-cli.jar' + + # If neither JENKINS nor HUDSON URL is set, assume jenkins for testing + return "http://" + socket.getfqdn() + ':' + repr(8080) + '/jnlpJars/jenkins-cli.jar' + + def proxied_public_cli_url(self, api_type): # Not required to be a real url - if api_type != ApiType.SCRIPT: - # If neither JENKINS nor HUDSON URL is set, assume jenkins for testing - cli = '/jnlpJars/hudson-cli.jar' if os.environ.get('HUDSON_URL') else '/jnlpJars/jenkins-cli.jar' - return Urls.proxied_public_url + cli - else: - return Urls._script_dir() - - @staticmethod - def _script_dir(): - sdir = os.environ.get(Urls.script_dir_env_var_name) - return dirs.job_script_dir if sdir is None else sdir.rstrip('/') + if api_type == ApiType.SCRIPT: + return self._script_dir + + # If neither JENKINS nor HUDSON URL is set, assume jenkins for testing + cli = '/jnlpJars/hudson-cli.jar' if os.environ.get('HUDSON_URL') else '/jnlpJars/jenkins-cli.jar' + return self.proxied_public_url + cli diff --git a/test/framework/coverage_rc b/test/framework/coverage_rc index 0c5fc26..7f24433 100644 --- a/test/framework/coverage_rc +++ b/test/framework/coverage_rc @@ -13,7 +13,7 @@ exclude_lines = # The alternative (original) rest api is no longer tested def _check_restkit_response class RestkitRestApi - ${COV_API_EXCLUDE_LINES} + ${COV_API_EXCLUDE_LINES?} omit = @@ -26,4 +26,4 @@ omit = setup.py ordered_enum.py *_flymake.py - ${COV_API_EXCLUDE_FILES} + ${COV_API_EXCLUDE_FILES?} diff --git a/test/framework/hyperspeed_test.py b/test/framework/hyperspeed_test.py index d277d62..2f6564a 100644 --- a/test/framework/hyperspeed_test.py +++ b/test/framework/hyperspeed_test.py @@ -15,14 +15,14 @@ def test_hyperspeed_speedup(): @pytest.mark.apis(ApiType.MOCK) -def test_hyperspeed_mocked_time(): +def test_hyperspeed_mocked_time(api_type): hs = HyperSpeed(1000) time.sleep(0.001) assert hs.time() > time.time() @pytest.mark.apis(ApiType.MOCK) -def test_hyperspeed_mocked_sleep(): +def test_hyperspeed_mocked_sleep(api_type): hs = HyperSpeed(1000) before = time.time() hs.sleep(1) diff --git a/test/framework/job.xml.tenjin b/test/framework/job.xml.tenjin index 83a9732..abd523d 100644 --- a/test/framework/job.xml.tenjin +++ b/test/framework/job.xml.tenjin @@ -1,4 +1,5 @@ + @@ -73,15 +74,15 @@ - #!{==sys.executable==} -B + #!{==python_executable==} -B import sys sys.path.append("{==test_tmp_dir==}") from jenkinsflow.jobload import update_job_from_template -from .cfg import ApiType +from jenkinsflow.test.framework.cfg import ApiType - + from jenkinsflow import jenkins_api as jenkins @@ -98,13 +99,14 @@ mock_job = MockJob( initial_buildno=None, invocation_delay={==cj.invocation_delay==}, unknown_result={==cj.unknown_result==}, final_result="{==fr==}", serial={==cj.serial==}, params=(), flow_created={==cj.flow_created==}, create_job=None, disappearing=False, non_existing=False, kill=False, allow_running=False, api=None, - final_result_use_cli=False, set_build_descriptions=()) + final_result_use_cli=False, set_build_descriptions=(), python_executable="{==python_executable==}") mock_job = None -config_xml_template = "{==test_tmp_dir==}/jenkinsflow/test/framework/job.xml.tenjin" +config_xml_template = "{==test_tmp_dir==}/test/framework/job.xml.tenjin" context = dict( + python_executable="{==create_job.python_executable==}", exec_time={==create_job.exec_time==}, max_fails={==create_job.max_fails==}, expect_invocations={==create_job.expect_invocations==}, @@ -132,12 +134,9 @@ update_job_from_template(job_loader_jenkins, "{==create_job.name==}", config_xml - #!{==sys.executable==} -B -import sys + #!{==python_executable==} -B import time -sys.path.append("{==test_tmp_dir==}") - from jenkinsflow.utils.set_build_description import set_build_description @@ -155,6 +154,7 @@ set_build_description("{==description==}", {==replace==}, "{==separator==}", use set -u date --rfc-3339=ns +source {==pseudo_install_dir==}/bin/activate {==script==} date @@ -163,6 +163,7 @@ sleep {==exec_time==} date +echo "Force result '$force_result'" case $force_result in SUCCESS) exit 0;; FAILURE) exit 1;; diff --git a/test/framework/job_script.py.tenjin b/test/framework/job_script.py.tenjin index a4d3c9f..d4680fb 100644 --- a/test/framework/job_script.py.tenjin +++ b/test/framework/job_script.py.tenjin @@ -1,6 +1,7 @@ import sys import subprocess import time +import datetime from jenkinsflow.utils.set_build_description import set_build_description @@ -14,8 +15,10 @@ def run_job(job_name, job_prefix_filter, username, password, securitytoken, caus + print(datetime.datetime.now()) print('sleeping=', {==exec_time==}) time.sleep({==exec_time==}) + print(datetime.datetime.now()) set_build_description("{==description==}", {==replace==}, "{==separator==}") diff --git a/test/framework/killer.py b/test/framework/killer.py index 22d8d5b..69b5a5a 100755 --- a/test/framework/killer.py +++ b/test/framework/killer.py @@ -9,9 +9,10 @@ def _killer(log_file, pid, sleep_time, num_kills): log(log_file, '\n') + logt(log_file, "Subprocess", __file__) logt(log_file, "Killer going to sleep for", sleep_time, "seconds") time.sleep(sleep_time) - logt(log_file, "Killer woke up") + logt(log_file, "sleep", sleep_time, "finished") for ii in range(0, num_kills): logt(log_file, "Killer sending", ii + 1, "of", num_kills, "SIGTERM signals to ", pid) os.kill(pid, signal.SIGTERM) @@ -22,15 +23,18 @@ def _killer(log_file, pid, sleep_time, num_kills): if __name__ == '__main__': log_file_name = sys.argv[4] with open(log_file_name, 'a+') as log_file: - _killer(log_file, int(sys.argv[1]), float(sys.argv[2]), int(sys.argv[3])) + try: + _killer(log_file, int(sys.argv[1]), float(sys.argv[2]), int(sys.argv[3])) + except Exception as ex: + print(ex, file=log_file) + raise def kill(api, sleep_time, num_kills): """Kill this process""" pid = os.getpid() - ff = __file__.replace('.pyc', '.py') - log_file_name = api.func_name.replace('test_', '') - args = [sys.executable, ff, repr(pid), repr(sleep_time), repr(num_kills), log_file_name] + log_file_name = api.func_name.replace('test_', '') + ".log" + args = [sys.executable, "-m", "jenkinsflow.test.framework.killer", repr(pid), repr(sleep_time), repr(num_kills), log_file_name] with open(log_file_name, 'w') as log_file: logt(log_file, "Invoking kill subprocess.", args) diff --git a/test/framework/mock_api.py b/test/framework/mock_api.py index c3d04a4..6081a3c 100644 --- a/test/framework/mock_api.py +++ b/test/framework/mock_api.py @@ -18,11 +18,12 @@ class MockApi(TestJenkins, HyperSpeed): job_xml_template = jp(here, 'job.xml.tenjin') api_type = ApiType.MOCK - def __init__(self, job_name_prefix, speedup, public_uri): + def __init__(self, job_name_prefix, speedup, public_uri, python_executable=None): super().__init__(job_name_prefix=job_name_prefix, speedup=speedup) self.public_uri = public_uri self.username = 'dummy' self.password = 'dummy' + self.python_executable = python_executable or sys.executable def job(self, name, max_fails, expect_invocations, expect_order, exec_time=None, initial_buildno=None, invocation_delay=0.1, params=None, script=None, unknown_result=False, final_result=None, serial=False, print_env=False, flow_created=False, create_job=None, @@ -39,7 +40,8 @@ def job(self, name, max_fails, expect_invocations, expect_order, exec_time=None, initial_buildno=initial_buildno, invocation_delay=invocation_delay, unknown_result=unknown_result, final_result=final_result, serial=serial, params=params, flow_created=flow_created, create_job=create_job, disappearing=disappearing, non_existing=non_existing, kill=kill, allow_running=allow_running, api=self, - final_result_use_cli=final_result_use_cli, set_build_descriptions=set_build_descriptions) + final_result_use_cli=final_result_use_cli, set_build_descriptions=set_build_descriptions, + python_executable=self.python_executable) self.test_jobs[job_name] = job def flow_job(self, name=None, params=None): @@ -73,7 +75,7 @@ def set_build_description(self, description, replace=False, separator='\n', job_ class MockJob(TestJob): def __init__(self, name, exec_time, max_fails, expect_invocations, expect_order, initial_buildno, invocation_delay, unknown_result, final_result, serial, params, flow_created, create_job, disappearing, non_existing, kill, allow_running, api, final_result_use_cli, - set_build_descriptions): + set_build_descriptions, python_executable): super().__init__(exec_time=exec_time, max_fails=max_fails, expect_invocations=expect_invocations, expect_order=expect_order, initial_buildno=initial_buildno, invocation_delay=invocation_delay, unknown_result=unknown_result, final_result=final_result, serial=serial, print_env=False, flow_created=flow_created, create_job=create_job, disappearing=disappearing, @@ -95,6 +97,7 @@ def __init__(self, name, exec_time, max_fails, expect_invocations, expect_order, self.queued_why = "Why am I queued?" self._killed = False self._just_killed = False + self.python_executable = python_executable def job_status(self): latest_build_number = self._get_last_build_number_or_none() diff --git a/test/framework/nox_utils.py b/test/framework/nox_utils.py new file mode 100644 index 0000000..58fb0bd --- /dev/null +++ b/test/framework/nox_utils.py @@ -0,0 +1,79 @@ +""" +Utility library for noxfile. +""" + +import os +from pathlib import Path +from typing import Sequence, Tuple, Dict + +_HERE = Path(__file__).resolve().parent +_TOP_DIR = _HERE.parent.parent + +from .cfg import ApiType, speedup + + +def cov_options_env(api_types: Sequence[str], coverage=True) -> Tuple[Sequence[str], Dict[str, str]]: + """Setup coverage options. + + Return pytest coverage options, and env variables dict. + """ + + if not coverage: + return () + + if len(api_types) == 3: + fail_under = 95 + elif ApiType.JENKINS in api_types: + fail_under = 94 + elif ApiType.MOCK in api_types and ApiType.SCRIPT in api_types: + fail_under = 90 + elif ApiType.MOCK in api_types: + fail_under = 88 + else: + fail_under = 85 + + # Set coverage exclude lines based on selected API types + api_exclude_lines = [] + if ApiType.JENKINS in api_types: + if os.environ.get('HUDSON_URL'): + # Parts of jenkins_api not used when hudson + api_exclude_lines.append("if self.jenkins.is_jenkins") + api_exclude_lines.append(r'if head_response.get\("X-Jenkins"\)') + else: + # Parts of jenkins_api not used when jenkins + api_exclude_lines.append("else: # Hudson") + api_exclude_lines.append("self.is_jenkins = False") + api_exclude_lines.append(r'if head_response.get\("X-Hudson"\)') + + if api_types == [ApiType.SCRIPT]: + # Parts of api_base not used in script_api (overridden methods) + api_exclude_lines.append(r"return (self.job.public_uri + '/' + repr(self.build_number) + '/console')") + + # Set coverage exclude files based on selected API type + api_exclude_files = [] + if ApiType.JENKINS not in api_types: + api_exclude_files.append("jenkins_api.py") + if ApiType.SCRIPT not in api_types: + api_exclude_files.append("script_api.py") + + return ( + [f'--cov={_TOP_DIR}', '--cov-report=term-missing', f'--cov-fail-under={fail_under}', f'--cov-config={_HERE/"coverage_rc"}'], + { + "COV_API_EXCLUDE_LINES": "\n".join(api_exclude_lines), + "COV_API_EXCLUDE_FILES": "\n".join(api_exclude_files), + } + ) + + +def parallel_options(parallel, api_types): + args = [] + + if api_types != [ApiType.MOCK]: + # Note: 'forked' is required for the kill/abort_current test not to abort other tests + args.append('--forked') + + # Note: parallel actually significantly slows down the test when only running mock + if parallel: + args.extend(['-n', '16']) + + return args diff --git a/test/framework/pytest_options.py b/test/framework/pytest_options.py new file mode 100644 index 0000000..c6e9e70 --- /dev/null +++ b/test/framework/pytest_options.py @@ -0,0 +1,59 @@ +"""Utilities used by both nox and pytest invocation""" + +import argparse + +from .cfg import ApiType, Urls + + +OPT_API = "--api" +OPT_JOB_LOAD = "--job-load" +OPT_JOB_DELETE = "--job-delete" +OPT_DIRECT_URL = "--direct-url" +OPT_MOCK_SPEEDUP = "--mock-speedup" + + +def add_options(parser): + def _add_opt(opt, **kwargs): + """Pytest uses it's own parser which works similar to ArgParse""" + if isinstance(parser, argparse.ArgumentParser): + parser.add_argument(opt, **kwargs) + return + + # metavar="NAME", + parser.addoption(opt, **kwargs) + + _add_opt( + OPT_API, + action="store", + default=','.join(at.name for at in list(ApiType)), + help=f"Comma separated list of APIs to test. Default is all defined apis: {','.join([at.name for at in list(ApiType)])}", + ) + + _add_opt( + OPT_JOB_LOAD, + action="store_true", + default=True, + help="Load Jenkins jobs - can be skipped to speed up testing if there are no changes to jobs and they are already loaded.", + ) + + _add_opt( + OPT_JOB_DELETE, + action="store_true", + default=False, + help="Delete Jenkins jobs before loading. May be necessary depending the changes to jobs (or for cleanup of obsolete jobs).", + ) + + _add_opt( + OPT_DIRECT_URL, + action="store", + default=Urls.default_direct_url, + help="Direct URL (i.e. non-proxied) of Jenkins server.", + ) + + _add_opt( + OPT_MOCK_SPEEDUP, + action="store", + type=int, + default=1000, + help="Time speedup for mock API tests. If set too high test may fail. If set low it just takes longer to run tests.", + ) diff --git a/test/framework/tmp_install.sh b/test/framework/tmp_install.sh new file mode 100755 index 0000000..7ac047c --- /dev/null +++ b/test/framework/tmp_install.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -u + +source_dir=$(cd $(dirname $0)/../.. && pwd) +target_dir=$1 + +rsync -a --delete --delete-excluded --exclude .git --exclude '*~' --exclude '*.py[cod]' --exclude '__pycache__' --exclude '*.cache' $source_dir/ $target_dir/ +mkdir -p $target_dir/.cache +chmod -R a+r $target_dir +chmod a+wrx $(find $target_dir -type d) +chmod a+x $target_dir/{test,demo}/*.py diff --git a/test/which_ci_server.html b/test/framework/which_ci_server.html similarity index 100% rename from test/which_ci_server.html rename to test/framework/which_ci_server.html diff --git a/test/missing_jobs_test.py b/test/missing_jobs_test.py index 07e497f..2526b1b 100644 --- a/test/missing_jobs_test.py +++ b/test/missing_jobs_test.py @@ -93,7 +93,7 @@ def test_missing_jobs_allowed_created_serial_parallel(api_type): api.job('j2', max_fails=0, expect_invocations=1, expect_order=3, create_job='missingC') api.job('missingC', max_fails=0, expect_invocations=1, expect_order=4, flow_created=True) - with serial(api, 20, job_name_prefix=api.job_name_prefix, allow_missing_jobs=True) as ctrl1: + with serial(api, 60, job_name_prefix=api.job_name_prefix, allow_missing_jobs=True) as ctrl1: ctrl1.invoke('j1') ctrl1.invoke('missingA') with ctrl1.parallel() as ctrl2: diff --git a/test/multiple_invocations_test.py b/test/multiple_invocations_test.py index fe4fbc0..d445980 100644 --- a/test/multiple_invocations_test.py +++ b/test/multiple_invocations_test.py @@ -123,7 +123,7 @@ def test_multiple_invocations_parallel_same_flow_no_args_singlequeued(api_type, api.flow_job() num_inv = 20 - api.job('j1', max_fails=0, expect_invocations=num_inv, expect_order=1, exec_time=15) + api.job('j1', max_fails=0, expect_invocations=num_inv, expect_order=1, exec_time=30) with parallel(api, timeout=70, job_name_prefix=api.job_name_prefix, report_interval=0.1, poll_interval=0.1) as ctrl1: for _ in range(0, num_inv): diff --git a/test/no_running_jobs_test.py b/test/no_running_jobs_test.py index 8bd7252..f4348cf 100644 --- a/test/no_running_jobs_test.py +++ b/test/no_running_jobs_test.py @@ -1,7 +1,9 @@ # Copyright (c) 2012 - 2015 Lars Hupfeldt Nielsen, Hupfeldt IT # All rights reserved. This work is under a BSD license, see LICENSE.TXT. +from datetime import datetime from pytest import raises +import psutil from jenkinsflow.flow import serial, JobNotIdleException @@ -10,23 +12,37 @@ from .framework.cfg import ApiType +def _wait_for_job_to_start(api, name): + if api.api_type == ApiType.SCRIPT: + return + + # Make sure job has started. Because it may be queued? + # TODO: Wait until it is started, no 'hard' sleep. + api.sleep(1) + + def test_no_running_jobs(api_type, capsys): with api_select.api(__file__, api_type, login=True) as api: api.flow_job() api.job('j1', max_fails=0, expect_invocations=1, expect_order=None, exec_time=50, invocation_delay=0, unknown_result=True) + print("Starting first flow:", datetime.now()) with serial(api, timeout=70, job_name_prefix=api.job_name_prefix) as ctrl1: ctrl1.invoke_unchecked('j1') + print("Finished first flow:", datetime.now()) - sout, _ = capsys.readouterr() + sout, serr = capsys.readouterr() assert lines_in(api_type, sout, "unchecked job: 'jenkinsflow_test__no_running_jobs__j1' UNKNOWN - RUNNING") + print(sout) + print(serr) - # Make sure job has actually started before entering new flow - api.sleep(1) + _wait_for_job_to_start(api, "j1") + print("Starting second flow:", datetime.now()) with raises(JobNotIdleException) as exinfo: with serial(api, timeout=70, job_name_prefix=api.job_name_prefix) as ctrl1: ctrl1.invoke('j1') + print("Finished second flow:", datetime.now()) assert "job: 'jenkinsflow_test__no_running_jobs__j1' is in state RUNNING. It must be IDLE." in str(exinfo.value) @@ -42,8 +58,9 @@ def test_no_running_jobs_unchecked(api_type, capsys): sout, _ = capsys.readouterr() assert lines_in(api_type, sout, "unchecked job: 'jenkinsflow_test__no_running_jobs_unchecked__j1' UNKNOWN - RUNNING") - api.sleep(1) + _wait_for_job_to_start(api, "j1") + print("second run") with raises(JobNotIdleException) as exinfo: with serial(api, timeout=70, job_name_prefix=api.job_name_prefix) as ctrl1: ctrl1.invoke_unchecked('j1') @@ -62,7 +79,7 @@ def test_no_running_jobs_jobs_allowed(api_type): with serial(api, timeout=70, job_name_prefix=api.job_name_prefix) as ctrl1: ctrl1.invoke_unchecked('j1') - api.sleep(1) + _wait_for_job_to_start(api, "j1") # TODO if api.api_type != ApiType.MOCK: diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000..eba2320 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,9 @@ +pytest>=6.0.2 +pytest-cov>=2.4.0 +pytest-instafail>=0.3.0 +pytest-xdist>=1.16 +pytest-forked>=1.6.0 +click>=7.0 +tenjin>=1.1.1 +bottle>=0.12 +objproxies>=0.9.4 diff --git a/test/run.py b/test/run.py deleted file mode 100755 index 3286bc7..0000000 --- a/test/run.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python - -""" -Test jenkinsflow. -""" - -import sys, copy, errno, os -from os.path import join as jp -import shutil -import subprocess - -import click - -try: - import pytest -except ImportError: - print("See setup.py for test requirements, or use 'python setup.py test'", file=sys.stderr) - raise - -here = os.path.abspath(os.path.dirname(__file__)) -top_dir = os.path.dirname(here) - -extra_sys_path = [os.path.normpath(path) for path in [jp(top_dir, '..'), jp(top_dir, 'demo'), jp(top_dir, 'demo/jobs')]] -sys.path = extra_sys_path + sys.path -os.environ['PYTHONPATH'] = ':'.join(extra_sys_path) - -from jenkinsflow.test.framework.cfg import ApiType, Urls, JobLoad -from jenkinsflow.test.framework.cfg.speedup import select_speedup -from jenkinsflow.test.framework.cfg import dirs - - -def run_tests(parallel, api_types, args, coverage=True, mock_speedup=1): - args = copy.copy(args) - - select_speedup(mock_speedup) - - if coverage: - # Set coverage fail-under based on selected API type - if len(api_types) == 3: - fail_under = 95 - elif ApiType.JENKINS in api_types: - fail_under = 94 - elif ApiType.MOCK in api_types and ApiType.SCRIPT in api_types: - fail_under = 90 - elif ApiType.MOCK in api_types: - fail_under = 88 - else: - fail_under = 86 - - # Set coverage exclude lines based on selected API types - cov_api_exclude_lines = [] - if ApiType.JENKINS in api_types: - if os.environ.get('HUDSON_URL'): - # Parts of jenkins_api not used when hudson - cov_api_exclude_lines.append("if self.jenkins.is_jenkins") - cov_api_exclude_lines.append(r'if head_response.get\("X-Jenkins"\)') - else: - # Parts of jenkins_api not used when jenkins - cov_api_exclude_lines.append("else: # Hudson") - cov_api_exclude_lines.append("self.is_jenkins = False") - cov_api_exclude_lines.append(r'if head_response.get\("X-Hudson"\)') - - if api_types == [ApiType.SCRIPT]: - # Parts of api_base not used in script_api (overridden methods) - cov_api_exclude_lines.append(r"return (self.job.public_uri + '/' + repr(self.build_number) + '/console')") - - # Set coverage exclude files based on selected API type - cov_api_exclude_files = [] - if ApiType.JENKINS not in api_types: - cov_api_exclude_files.append("jenkins_api.py") - if ApiType.SCRIPT not in api_types: - cov_api_exclude_files.append("script_api.py") - - os.environ["COV_API_EXCLUDE_LINES"] = " ".join(cov_api_exclude_lines) - os.environ["COV_API_EXCLUDE_FILES"] = " ".join(cov_api_exclude_files) - - # Note: cov_rc_file_name hardcoded in .travis.yml - cov_rc_file_name = jp(here, '.coverage_rc_' + '_'.join(api_type.name.lower() for api_type in api_types)) - - args.extend([f'--cov={top_dir}', '--cov-report=term-missing', f'--cov-config={cov_rc_file_name}', f'--cov-fail-under={fail_under}']) - - shutil.copy2(jp(here, "framework/coverage_rc"), cov_rc_file_name) - - if api_types != [ApiType.MOCK]: - # Note: 'boxed' is required for the kill/abort_current test not to abort other tests - args.append('--boxed') - - if parallel: - args.extend(['-n', '16']) - - print('pytest.main', args) - rc = pytest.main(args) - if rc: - raise Exception("pytest {args} failed with code {rc}".format(args=args, rc=rc)) - - -def start_msg(*msg): - print("\n++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", *msg) - - -def cli(mock_speedup=1000, - direct_url=Urls.direct_url(ApiType.JENKINS), - apis=None, - pytest_args=None, - job_delete=False, - job_load=True, - testfile=None): - """ - Test jenkinsflow. - First runs all tests mocked in hyperspeed, then runs against Jenkins, using jenkins_api, then run script_api jobs. - - Normally jobs will be run in parallel, specifying --job-delete disables this. - The default options assumes that re-loading without deletions generates correct job config. - Tests that require jobs to be deleted/non-existing will delete the jobs, regardless of the --job-delete option. - - [TESTFILE]... File names to pass to py.test - """ - - os.environ[Urls.direct_url_env_var_name] = direct_url - os.environ[JobLoad.skip_job_delete_env_var_name] = 'false' if job_delete else 'true' - os.environ[JobLoad.skip_job_load_env_var_name] = 'false' if job_load else 'true' - - is_ci = os.environ.get('CI', 'false').lower() == 'true' - - args = ['--capture=sys', '--instafail'] - - if apis is None: - api_types = [ApiType.MOCK, ApiType.SCRIPT, ApiType.JENKINS] - else: - api_types = [ApiType[api_name.strip().upper()] for api_name in apis.split(',')] - args.extend(['-k', ' or '.join([apit.name for apit in api_types])]) - - rc = 0 - target_dir = "/tmp/jenkinsflow-test/jenkinsflow" - try: - os.makedirs(target_dir) - except OSError as ex: - if ex.errno != errno.EEXIST: - raise - - if api_types != [ApiType.MOCK]: - print("Creating temporary test installation in", repr(dirs.pseudo_install_dir), "to make files available to Jenkins.") - install_script = jp(here, 'tmp_install.sh') - rc = subprocess.call([install_script, target_dir]) - if rc: - print("Failed test installation to", repr(dirs.pseudo_install_dir), "Install script is:", repr(install_script), file=sys.stderr) - print("Warning: Some tests will fail!", file=sys.stderr) - - cov_file = ".coverage" - for cov_file in jp(here, cov_file), jp(top_dir, cov_file): - if os.path.exists(cov_file): - os.remove(cov_file) - - print("\nRunning tests") - try: - if pytest_args or testfile: - coverage = False - args.extend(pytest_args.split(' ') + list(testfile) if pytest_args else list(testfile)) - else: - coverage = True - args.append('--ff') - - if is_ci: - args.append('-vvv') - - hudson = os.environ.get('HUDSON_URL') - if hudson: - print("Disabling parallel run, Hudson can't handle it :(") - parallel = JobLoad.skip_job_load() or JobLoad.skip_job_delete() and not hudson - run_tests(parallel, api_types, args, coverage, mock_speedup) - - if not testfile and not is_ci: - # This is automatically tested by readdthedocs, so no need to test on Travis - start_msg("Testing documentation generation") - - os.chdir(jp(top_dir, 'doc/source')) - del os.environ['PYTHONPATH'] - subprocess.check_call(['make', 'html']) - except Exception as ex: - print('*** ERROR: There were errors! Check output! ***', repr(ex), file=sys.stderr) - raise - - sys.exit(rc) - - -@click.command() -@click.option('--mock-speedup', '-s', help="Time speedup when running mocked tests.", default=1000) -@click.option('--direct-url', help="Direct Jenkins URL. Must be different from the URL set in Jenkins (and preferably non proxied)", - default=Urls.direct_url(ApiType.JENKINS)) -@click.option('--apis', help="Select which api(s) to use/test. Comma separated list. Possible values: 'jenkins, 'script', 'mock'. Default is all.", default=None) -@click.option('--pytest-args', help="py.test arguments.") -@click.option('--job-delete/--no-job-delete', help="Delete and re-load jobs into Jenkins. Default is --no-job-delete.", default=False) -@click.option('--job-load/--no-job-load', help="Load jobs into Jenkins (skipping job load assumes all jobs already loaded and up to date). Deafult is --job-load.", default=True) -@click.argument('testfile', nargs=-1, type=click.Path(exists=True, readable=True)) -def _cli(mock_speedup, direct_url, apis, pytest_args, job_delete, job_load, testfile): - cli(mock_speedup, direct_url, apis, pytest_args, job_delete, job_load, testfile) - - -if __name__ == '__main__': - _cli() # pylint: disable=no-value-for-parameter diff --git a/test/set_build_description_test.py b/test/set_build_description_test.py index eedfc20..fc96da3 100644 --- a/test/set_build_description_test.py +++ b/test/set_build_description_test.py @@ -14,7 +14,7 @@ from .framework import api_select from .framework.cfg import jenkins_security -from .framework.cfg import ApiType, Urls +from .framework.cfg import ApiType _here = os.path.dirname(os.path.abspath(__file__)) @@ -113,7 +113,7 @@ def test_set_build_description_api(api_type): @pytest.mark.not_apis(ApiType.MOCK) -def test_set_build_description_utils(api_type): +def test_set_build_description_utils(api_type, options): with api_select.api(__file__, api_type, login=True) as api: api.flow_job() job_name = 'job-1' @@ -131,7 +131,7 @@ def test_set_build_description_utils(api_type): if api.api_type != ApiType.SCRIPT: job = api.get_job(api.job_name_prefix + job_name) _, _, build_num = job.job_status() - direct_url = Urls.direct_url(api_type) + direct_url = options.urls.direct_url(api_type) if jenkins_security.set_build_description_must_authenticate: set_build_description( @@ -243,7 +243,7 @@ def test_set_build_description_unknown_job(api_type): @pytest.mark.not_apis(ApiType.MOCK) -def test_set_build_description_cli(api_type, cli_runner): +def test_set_build_description_cli(api_type, cli_runner, options): with api_select.api(__file__, api_type, login=True) as api: api.flow_job() job_name = 'job-1' @@ -255,7 +255,7 @@ def test_set_build_description_cli(api_type, cli_runner): # Need to read the build number job = api.get_job(api.job_name_prefix + job_name) _, _, build_num = job.job_status() - base_url = Urls.direct_url(api_type) + '/' + base_url = options.urls.direct_url(api_type) + '/' _clear_description(api, job) diff --git a/test/speed_test.py b/test/speed_test.py index 745d826..8848231 100644 --- a/test/speed_test.py +++ b/test/speed_test.py @@ -11,19 +11,19 @@ @pytest.mark.not_apis(ApiType.MOCK) -def test_hyperspeed_speedup(): +def test_hyperspeed_speedup(api_type): hs = Speed() assert hs.speedup == 1 @pytest.mark.not_apis(ApiType.MOCK) -def test_hyperspeed_real_time(): +def test_hyperspeed_real_time(api_type): hs = Speed() assert hs.time() <= time.time() @pytest.mark.not_apis(ApiType.MOCK) -def test_hyperspeed_real_sleep(): +def test_hyperspeed_real_sleep(api_type): hs = Speed() before = time.time() hs.sleep(1) diff --git a/test/tmp_install.sh b/test/tmp_install.sh index 8e79030..9bf5ee6 100755 --- a/test/tmp_install.sh +++ b/test/tmp_install.sh @@ -1,11 +1,9 @@ #!/bin/bash set -u -source_dir=$(cd $(dirname $0)/.. && pwd) -target_dir=$1 +source_dir=$1 +target_dir=$2 rsync -a --delete --delete-excluded --exclude .git --exclude '*~' --exclude '*.py[cod]' --exclude '__pycache__' --exclude '*.cache' $source_dir/ $target_dir/ -mkdir -p $target_dir/.cache chmod -R a+r $target_dir chmod a+wrx $(find $target_dir -type d) -chmod a+x $target_dir/{test,demo}/*.py diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 0dded99..0000000 --- a/tox.ini +++ /dev/null @@ -1,16 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -#envlist = py26, py27, py32, py33, py34, py35, pypy, jython -envlist = py36, py37, py38 - -[testenv] -passenv = CI -deps = - .[test] -commands = - {envpython} setup.py test - jenkinsflow set_build_description --help