diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6ab4fc7d..ed8a771a 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -13,10 +13,10 @@ ### Build / Run Issue [//]: # (If you are migrating from catkin_make, please follow the migration guide:) -[//]: # (http://catkin-tools.readthedocs.org/en/latest/migration.html) +[//]: # (https://catkin-tools.readthedocs.org/en/latest/migration.html) [//]: # (Please also check for solved issues here:) -[//]: # (http://catkin-tools.readthedocs.org/en/latest/troubleshooting.html) +[//]: # (https://catkin-tools.readthedocs.org/en/latest/troubleshooting.html) [//]: # (And check for open issues here:) [//]: # (https://github.com/catkin/catkin_tools/issues?q=is%3Aopen+is%3Aissue+label%3Abug) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 00000000..2aceb242 --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,51 @@ +name: build + +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + versions: + - dist: ubuntu-18.04 + python: 3.6 + catkin: indigo-devel + - dist: ubuntu-18.04 + python: 3.7 + catkin: indigo-devel + - dist: ubuntu-18.04 + python: 3.8 + catkin: indigo-devel + - dist: ubuntu-20.04 + python: 3.9 + catkin: indigo-devel + + runs-on: ${{ matrix.versions.dist }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.versions.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.versions.python }} + - name: Install package and dependencies + run: | + python -m pip install --upgrade pip + pip install . + pip install --upgrade empy sphinx_rtd_theme sphinxcontrib-spelling nose coverage flake8 mock + - name: Set up catkin + run: | + git clone https://github.com/ros/catkin.git -b ${{ matrix.versions.catkin }} /tmp/catkin_source + mkdir /tmp/catkin_source/build + pushd /tmp/catkin_source/build + cmake .. && make + popd + - name: Test catkin_tools + run: | + source /tmp/catkin_source/build/devel/setup.bash + nosetests -s tests + - name: Build documentation + run: | + pushd docs + make html + sphinx-build -b spelling . build -t use_spelling + popd diff --git a/.travis.before_install.bash b/.travis.before_install.bash deleted file mode 100755 index e0c39d22..00000000 --- a/.travis.before_install.bash +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -if [ "$TRAVIS_OS_NAME" == "linux" ]; then - echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main universe restricted" > /etc/apt/sources.list - echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main universe restricted" >> /etc/apt/sources.list - sudo apt update - sudo apt install enchant -y - sudo apt install python2 -y || true -elif [ "$TRAVIS_OS_NAME" == "osx" ]; then - brew upgrade python - $PYTHON -m pip install virtualenv - $PYTHON -m virtualenv -p $PYTHON venv - brew install enchant - source venv/bin/activate -fi diff --git a/.travis.before_script.bash b/.travis.before_script.bash deleted file mode 100755 index 7f217730..00000000 --- a/.travis.before_script.bash +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -if [ "$TRAVIS_OS_NAME" == "linux" ]; then - sudo apt-get install cmake build-essential -elif [ "$TRAVIS_OS_NAME" == "osx" ]; then - echo "No OS X-specific before_script steps." -fi diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9957e88a..00000000 --- a/.travis.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Travis auto-virtualenv isn't supported on OS X -language: generic -matrix: - include: - - dist: bionic - language: python - python: "3.6" - os: linux - env: PYTHON=/usr/bin/python3.6 CATKIN_VERSION=indigo-devel - - dist: bionic - language: python - python: "3.7" - os: linux - env: PYTHON=/usr/bin/python3.7 CATKIN_VERSION=indigo-devel - - dist: bionic - language: python - python: "3.8" - os: linux - env: PYTHON=/usr/bin/python3.8 CATKIN_VERSION=indigo-devel - - dist: focal - language: python - python: "3.9" - os: linux - env: PYTHON=/usr/bin/python3.9 CATKIN_VERSION=indigo-devel -before_install: - # Install catkin_tools dependencies - - source .travis.before_install.bash - - pip install setuptools argparse catkin-pkg PyYAML psutil osrf_pycommon pyenchant sphinxcontrib-spelling -install: - # Install catkin_tools - - python setup.py install -before_script: - # Install catkin_tools test harness dependencies - - ./.travis.before_script.bash - - pip install empy sphinx_rtd_theme nose coverage flake8 mock --upgrade - - git clone https://github.com/ros/catkin.git /tmp/catkin_source -b $CATKIN_VERSION --depth 1 - - mkdir /tmp/catkin_source/build - - pushd /tmp/catkin_source/build - - cmake .. && make - - source devel/setup.bash - - popd -script: - # Run catkin_tools test harness - - python setup.py nosetests -s - # Build documentation - - pushd docs - - make html - - sphinx-build -b spelling . build -t use_spelling - - popd -notifications: - email: false diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c186b896..a3d520d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,88 @@ Changelog for package catkin_tools ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Upcoming +-------- + +0.8.2 (2021-12-01) +------------------ + +* Add back flags that were removed in #691 (#702) +* Documentation: Update CLI documentation + +0.8.1 (2021-11-30) +------------------ + +* catkin test: add --limit-status-rate option (#699) + +0.8.0 (2021-11-30) +------------------ + +* Add catkin test verb (#676) +* Code cleanup (#692) +* Documentation: https in docs, properties over vars, sphinx cleanup (#690) +* Documentation: Improve wording (#694) +* Documentation: Add custom build, devel, install dirs (#697) +* Contributors: Balint, Jim Aldon D'Souza, Matthijs van der Burgh, mobangjack, pseyfert, Timon Engelke + +0.7.2 (2021-11-18) +------------------ + +* Update installation docs for python 3 (#687) +* Fix environment variable parsing (#686) +* Switch from Travis CI to GitHub actions (#684) +* Regenerate setup files when the install space was cleaned (#682) +* Add possibility to clean individual package with isolated devel space (#683) +* Fix regeneration of setup file when the install space was cleaned (#682) +* Fix workspace generation with catkin build --this and --start-with-this (#685) + +0.7.1 (2021-07-17) +------------------ +* Fixes in the build system requiring a version increase + +0.7.0 (2021-03-01) +------------------ +* Fix placeholders for cmake warning and error coloring (#678) +* Fix catkin clean --all-profiles when not at workspace root (#673) +* Fix `catkin create pkg` without license parameter (#671) +* Support building from a symlinked workspace (#669) +* Use loadavg over the last 1 minute (#668) +* Fix shell completion install locations (#652) +* Fix blank lines in build output (#666) +* Use standard python function to determine terminal width (#653) +* Fix handling of invalid package.xml files (#660) +* Fixes for extending profiles (#658) +* escape whitespaces in `catkin config` printout (#657) +* updates to zsh completion (#609) +* Ignore catkin_tools_prebuild package in build space (#650) +* fix 'catkin locate' for symlinked pkgs inside workspace (#624) +* Report circular dependencies detected by topological_order_packages() (#617) +* Add `--this` option to `clean` verb (#623) +* In catkin build, preserve original job list topological ordering (#626) +* Fail build if jobs were abandoned (#644) +* Fix installation of new cmake files (#615) +* Abort with error message on circular dependency. (#641) +* Changed yield from lock to await for Python 3.9 compat (#635) +* Remove older py35+xenial config and add py39+focal (#637) +* Install python2 before travis runs on Focal. (#639) +* Bump cmake min ver to 2.8.12 (#634) +* Fix byte decoding for python 3 (Issue #625) (#627) +* Cleanup of jobs flag parsing (#610, #656, #655) +* Fix get_python_install_dir for Python 2/3 (#601) +* Minor cleanup: + - import cleanup (#651) + - remove hack (#659) + - Add missing space in devel layout error message + - fix TypeError on executing catkin env (#649) + - Put a space between 'workspace' and 'and' (#619) + - Remove redundant 'configuration' in mechanics.rst (#646) + - Use PYTHONASYNCIODEBUG instead of TROLLIUSDEBUG (#661) +* Contributors: Akash Patel, Guglielmo Gemignani, Ivor Wanders, Kevin Jaget, Lucas Walter, Mathias Lüdtke, Matthijs van der Burgh, Mike Purvis, Robert Haschke, Simon Schmeisser, Tim Rakowski, Timon Engelke, Vojtech Spurny, ckurtz22, mobangjack, pseyfert, xiaxi, zig-for + +0.6.0 (2020-06-03) +------------------ +* This release restores the 0.4.5 state due to an accident with the 0.5.0 release where we pushed it to Python2 users which it doesn't support. + 0.5.0 (2020-06-02) ------------------ * Revert "jobs: Fixing environment required to run catkin test targets on pre-indigo catkin" (`#600 `_) diff --git a/README.md b/README.md index fc14a36f..0d049321 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# catkin_tools [![Build Status](https://travis-ci.org/catkin/catkin_tools.svg?branch=master)](https://travis-ci.org/catkin/catkin_tools) +# catkin_tools [![Build Status](https://github.com/catkin/catkin_tools/actions/workflows/workflow.yml/badge.svg)](https://github.com/catkin/catkin_tools/actions/workflows/workflow.yml) Command line tools for working with [catkin](https://github.com/ros/catkin) -Documentation: http://catkin-tools.readthedocs.org/ +Documentation: https://catkin-tools.readthedocs.org/ diff --git a/catkin_tools/argument_parsing.py b/catkin_tools/argument_parsing.py index 6273530f..feb28455 100644 --- a/catkin_tools/argument_parsing.py +++ b/catkin_tools/argument_parsing.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import argparse import os import re @@ -254,7 +252,7 @@ def extract_jobs_flags(mflags): :rtype: list """ if not mflags: - return [] + return None # Each line matches a flag type, i.e. -j, -l, --jobs, --load-average # (?:^|\s) and (?=$|\s) make sure that the flag is surrounded by whitespace @@ -324,6 +322,8 @@ def configure_make_args(make_args, jobs_args, use_internal_make_jobserver): :param make_args: arguments to be passed to GNU Make :type make_args: list + :param jobs_args: job arguments overriding make flags + :type jobs_args: list :param use_internal_make_jobserver: if true, use the internal jobserver :type make_args: bool :rtype: tuple (final make_args, using makeflags, using cliflags, using jobserver) @@ -331,7 +331,7 @@ def configure_make_args(make_args, jobs_args, use_internal_make_jobserver): # Configure default jobs options: use all CPUs in each package try: - # NOTE: this will yeild greater than 100% CPU utilization + # NOTE: this will yield greater than 100% CPU utilization n_cpus = cpu_count() jobs_flags = { 'jobs': n_cpus, @@ -378,7 +378,7 @@ def argument_preprocessor(args): :param args: system arguments from which special arguments need to be extracted :type args: list - :returns: a tuple contianing a list of the arguments which can be handled + :returns: a tuple containing a list of the arguments which can be handled by argparse and a dict of the extra arguments which this function has extracted :rtype: tuple @@ -394,12 +394,12 @@ def argument_preprocessor(args): jobs_args = extract_jobs_flags(' '.join(args)) if jobs_args: # Remove jobs flags from cli args if they're present - args = re.sub(' '.join(jobs_args), '', ' '.join(args)).split() + args = [arg for arg in args if arg not in jobs_args] elif make_args is not None: jobs_args = extract_jobs_flags(' '.join(make_args)) if jobs_args: # Remove jobs flags from cli args if they're present - make_args = re.sub(' '.join(jobs_args), '', ' '.join(make_args)).split() + make_args = [arg for arg in make_args if arg not in jobs_args] extras = { 'cmake_args': cmake_args, diff --git a/catkin_tools/commands/catkin.py b/catkin_tools/commands/catkin.py index 1f09f1a9..d1a7a41f 100644 --- a/catkin_tools/commands/catkin.py +++ b/catkin_tools/commands/catkin.py @@ -12,18 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import argparse -from datetime import date import os import pkg_resources import sys - -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote +from datetime import date +from shlex import quote as cmd_quote from catkin_tools.common import is_tty @@ -195,7 +189,7 @@ def catkin_main(sysargs): date.today().year) ) print('catkin_tools is released under the Apache License,' - ' Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)') + ' Version 2.0 (https://www.apache.org/licenses/LICENSE-2.0)') print('---') print('Using Python {}'.format(''.join(sys.version.split('\n')))) sys.exit(0) diff --git a/catkin_tools/common.py b/catkin_tools/common.py index 9e934257..981da43c 100644 --- a/catkin_tools/common.py +++ b/catkin_tools/common.py @@ -12,23 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - +import asyncio import datetime import errno import os import re +import shutil import sys from fnmatch import fnmatch - -import asyncio - -from shlex import split as cmd_split - -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote +from itertools import chain from catkin_pkg.packages import find_packages @@ -37,16 +29,6 @@ color_mapper = ColorMapper() clr = color_mapper.clr -try: - string_type = basestring -except NameError: - string_type = str - -try: - unicode_type = unicode -except NameError: - unicode_type = str - class FakeLock(asyncio.locks.Lock): @@ -55,9 +37,8 @@ class FakeLock(asyncio.locks.Lock): def locked(self): return False - @asyncio.coroutine - def acquire(self): - return(True) + async def acquire(self): + return True def release(self): pass @@ -79,7 +60,7 @@ def getcwd(symlinks=True): realpath = os.getcwd() # The `PWD` environment variable should contain the path that we took to - # get here, includng symlinks + # get here, including symlinks if symlinks: cwd = os.environ.get('PWD', '') @@ -157,7 +138,7 @@ def format_time_delta_short(delta): def get_cached_recursive_build_depends_in_workspace(package, workspace_packages): - """Returns cached or calculated recursive build dependes for a given package + """Returns cached or calculated recursive build depends for a given package If the recursive build depends for this package and this set of workspace packages has already been calculated, the cached results are returned. @@ -185,18 +166,15 @@ def get_cached_recursive_build_depends_in_workspace(package, workspace_packages) def get_recursive_depends_in_workspace( packages, ordered_packages, - root_include_function, include_function, exclude_function): """Computes the recursive dependencies of a package in a workspace based on include and exclude functions of each package's dependencies. - :param package: package for which the recursive depends should be calculated - :type package: :py:class:`catkin_pkg.package.Package` + :param packages: package for which the recursive depends should be calculated + :type packages: list(:py:class:`catkin_pkg.package.Package`) :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) - :param root_include_function: a function which take a package and returns a list of root packages to include - :type root_include_function: callable :param include_function: a function which take a package and returns a list of packages to include :type include_function: callable :param exclude_function: a function which take a package and returns a list of packages to exclude @@ -215,9 +193,12 @@ def get_recursive_depends_in_workspace( } # Initialize working sets - pkgs_to_check = set([ - pkg.name for pkg in sum([root_include_function(p) for p in packages], []) - ]) + pkgs_to_check = set( + pkg.name + # Only include the packages where the condition has evaluated to true + for pkg in chain(*(filter(lambda pkg: pkg.evaluated_condition, include_function(p)) for p in packages)) + ) + checked_pkgs = set() recursive_deps = set() @@ -229,16 +210,16 @@ def get_recursive_depends_in_workspace( continue # Add this package's dependencies which should be checked _, pkg = workspace_packages_by_name[pkg_name] - pkgs_to_check.update([ + pkgs_to_check.update( d.name - for d in include_function(pkg) + for d in filter(lambda pkg: pkg.evaluated_condition, include_function(pkg)) if d.name not in checked_pkgs - ]) + ) # Add this package's dependencies which shouldn't be checked - checked_pkgs.update([ + checked_pkgs.update( d.name for d in exclude_function(pkg) - ]) + ) # Add the package itself in case we have a circular dependency checked_pkgs.add(pkg.name) # Add this package to the list of recursive dependencies for this package @@ -270,11 +251,6 @@ def get_recursive_build_depends_in_workspace(package, ordered_packages): return get_recursive_depends_in_workspace( [package], ordered_packages, - root_include_function=lambda p: ( - p.build_depends + - p.buildtool_depends + - p.test_depends + - p.run_depends), include_function=lambda p: ( p.build_depends + p.buildtool_depends + @@ -289,7 +265,7 @@ def get_recursive_run_depends_in_workspace(packages, ordered_packages): but excluding packages which are build depended on by another package in the list :param packages: packages for which the recursive depends should be calculated - :type packages: list of :py:class:`catkin_pkg.package.Package` + :type packages: list(:py:class:`catkin_pkg.package.Package`) :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) @@ -301,7 +277,6 @@ def get_recursive_run_depends_in_workspace(packages, ordered_packages): return get_recursive_depends_in_workspace( packages, ordered_packages, - root_include_function=lambda p: p.run_depends, include_function=lambda p: p.run_depends, exclude_function=lambda p: p.buildtool_depends + p.build_depends ) @@ -311,8 +286,8 @@ def get_recursive_build_dependents_in_workspace(package_name, ordered_packages): """Calculates the recursive build dependents of a package which are also in the ordered_packages - :param package: package for which the recursive depends should be calculated - :type package: :py:class:`catkin_pkg.package.Package` + :param package_name: name of the package for which the recursive depends should be calculated + :type package_name: str :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) @@ -340,8 +315,8 @@ def get_recursive_run_dependents_in_workspace(package_name, ordered_packages): """Calculates the recursive run dependents of a package which are also in the ordered_packages - :param package: package for which the recursive depends should be calculated - :type package: :py:class:`catkin_pkg.package.Package` + :param package_name: name of the package for which the recursive depends should be calculated + :type package_name: str :param ordered_packages: packages in the workspace, ordered topologically :type ordered_packages: list(tuple(package path, :py:class:`catkin_pkg.package.Package`)) @@ -382,7 +357,7 @@ def log(*args, **kwargs): except UnicodeEncodeError: # Strip unicode characters from string args sanitized_args = [unicode_sanitizer.sub('?', a) - if type(a) in [str, unicode_type] + if isinstance(a, str) else a for a in args] print(*sanitized_args, **kwargs) @@ -396,63 +371,9 @@ def log(*args, **kwargs): unicode_error_printed = True -__warn_terminal_width_once_has_printed = False -__default_terminal_width = 80 - - -def __warn_terminal_width_once(): - global __warn_terminal_width_once_has_printed - if __warn_terminal_width_once_has_printed: - return - __warn_terminal_width_once_has_printed = True - print('NOTICE: Could not determine the width of the terminal. ' - 'A default width of {} will be used. ' - 'This warning will only be printed once.'.format(__default_terminal_width), - file=sys.stderr) - - -def terminal_width_windows(): - """Returns the estimated width of the terminal on Windows""" - from ctypes import windll, create_string_buffer - h = windll.kernel32.GetStdHandle(-12) - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) - - # return default size if actual size can't be determined - if not res: - __warn_terminal_width_once() - return __default_terminal_width - - import struct - (bufx, bufy, curx, cury, wattr, left, top, right, bottom, maxx, maxy)\ - = struct.unpack("hhhhHhhhhhh", csbi.raw) - width = right - left + 1 - - return width - - -def terminal_width_linux(): - """Returns the estimated width of the terminal on linux""" - from fcntl import ioctl - from termios import TIOCGWINSZ - import struct - try: - with open(os.ctermid(), "rb") as f: - height, width = struct.unpack("hh", ioctl(f.fileno(), TIOCGWINSZ, "1234")) - except (IOError, OSError, struct.error): - # return default size if actual size can't be determined - __warn_terminal_width_once() - return __default_terminal_width - return width - - def terminal_width(): """Returns the estimated width of the terminal""" - try: - return terminal_width_windows() if os.name == 'nt' else terminal_width_linux() - except ValueError: - # Failed to get the width, use the default 80 - return __default_terminal_width + return shutil.get_terminal_size().columns _ansi_escape = re.compile(r'\x1b[^m]*m') @@ -502,7 +423,7 @@ def slice_to_printed_length(string, length): def printed_fill(string, length): - """Textwrapping for strings with esacpe characters.""" + """Textwrapping for strings with escape characters.""" splat = string.replace('\\n', ' \\n ').split() count = 0 @@ -607,7 +528,15 @@ def find_enclosing_package(search_start_path=None, ws_path=None, warnings=None, """Get the package containing a specific directory. :param search_start_path: The path to crawl upward to find a package, CWD if None + :type search_start_path: str :param ws_path: The path at which the search should stop + :type ws_path: str + :param warnings: Print warnings if None or return them in the given list + :type warnings: list + :param symlinks: If True, then get the path considering symlinks. If false, + resolve the path to the actual path. + :type symlinks: bool + :returns: """ search_path = search_start_path or getcwd(symlinks=symlinks) @@ -649,11 +578,15 @@ def mkdir_p(path): raise -def format_env_dict(environ): +def format_env_dict(environ, human_readable=True): """Format an environment dict for printing to console similarly to `typeset` builtin.""" - return '\n'.join([ - 'typeset -x {}={}'.format(k, cmd_quote(v)) + if human_readable: + separator = '\n' + else: + separator = '\x00' + return separator.join([ + '{}={}'.format(k, v) for k, v in environ.items() ]) @@ -663,18 +596,16 @@ def parse_env_str(environ_str): environ_str must be encoded in utf-8 """ - - try: - split_envs = [e.split('=', 1) for e in cmd_split(environ_str.decode())] - return { - e[0]: e[1] for e - in split_envs - if len(e) == 2 - } - except ValueError: - print('WARNING: Could not parse env string: `{}`'.format(environ_str), - file=sys.stderr) - raise + variables = environ_str.decode().rstrip('\x00').split('\x00') + environment = {} + for v in variables: + try: + key, value = v.split('=', 1) + environment[key] = value + except ValueError: + print('WARNING: Could not parse env string: `{}`'.format(v), + file=sys.stderr) + return environment def expand_glob_package(pattern, all_workspace_packages): diff --git a/catkin_tools/config.py b/catkin_tools/config.py index bae5aede..ac923900 100644 --- a/catkin_tools/config.py +++ b/catkin_tools/config.py @@ -12,13 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import yaml -import shlex - -from .common import string_type +from shlex import split as cmd_split catkin_config_path = os.path.join(os.path.expanduser('~'), '.config', 'catkin') @@ -35,8 +31,7 @@ ls: list install: config --install p: create pkg -test: build --verbose --skip-install --make-args test -- -run_tests: build --verbose --skip-install --catkin-make-args run_tests -- +run_tests: test """ @@ -98,13 +93,13 @@ def get_verb_aliases(path=catkin_config_path): raise RuntimeError("Invalid alias file ('{0}'), expected a dict but got a {1}" .format(full_path, type(yaml_dict))) for key, value in yaml_dict.items(): - if not isinstance(key, string_type): + if not isinstance(key, str): raise RuntimeError("Invalid alias in file ('{0}'), expected a string but got '{1}' of type {2}" .format(full_path, key, type(key))) parsed_value = None - if isinstance(value, string_type): + if isinstance(value, str): # Parse using shlex - parsed_value = shlex.split(value) + parsed_value = cmd_split(value) elif isinstance(value, list) or isinstance(value, type(None)): # Take plainly parsed_value = value diff --git a/catkin_tools/context.py b/catkin_tools/context.py index 1044f5c3..3737c972 100644 --- a/catkin_tools/context.py +++ b/catkin_tools/context.py @@ -14,8 +14,6 @@ """This module implements a class for representing a catkin workspace context""" -from __future__ import print_function - import os import re import sys @@ -92,7 +90,7 @@ def space_setter(self, value): if self.__locked: raise RuntimeError("Setting of context members is not allowed while locked.") setattr(self, '__%s_space' % space, value) - setattr(self, '__%s_space_abs' % space, os.path.join(self.__workspace, value)) + setattr(self, '__%s_space_abs' % space, os.path.realpath(os.path.join(self.workspace, value))) def space_exists(self): """ @@ -101,10 +99,21 @@ def space_exists(self): space_abs = getattr(self, '__%s_space_abs' % space) return os.path.exists(space_abs) and os.path.isdir(space_abs) + def package_space(self, package): + """ + Get the package specific path in a space + """ + space_abs = getattr(self, '__%s_space_abs' % space) + return os.path.join(space_abs, package.name) + setattr(cls, '%s_space' % space, property(space_getter, space_setter)) setattr(cls, '%s_space_abs' % space, property(space_abs_getter)) setattr(cls, '%s_space_exists' % space, space_exists) + package_space_name = "package_%s_space" % space + if not hasattr(cls, package_space_name): + setattr(cls, package_space_name, package_space) + @classmethod def setup_space_keys(cls): """ @@ -244,6 +253,10 @@ def get_metadata_recursive(profile): if workspace: key_origins[k] = profile + # When --no-make-args and no other jobs args are given, clean jobs args too + if 'make_args' in opts_vars and opts_vars['make_args'] == [] and opts_vars['jobs_args'] is None: + context_args['jobs_args'] = [] + context_args["key_origins"] = key_origins # Create the build context @@ -269,7 +282,7 @@ def save_in_file(file, key, value): files[file] = {key: value} for key, val in data.items(): - if key in context.key_origins: + if context.extends is not None and key in context.key_origins: save_in_file(context.key_origins[key], key, val) else: save_in_file(context.profile, key, val) @@ -321,7 +334,7 @@ def __init__( :type source_space: str :param log_space: relative location of log space, defaults to '/logs' :type log_space: str - :param build_space: relativetarget location of build space, defaults to '/build' + :param build_space: relative target location of build space, defaults to '/build' :type build_space: str :param devel_space: relative target location of devel space, defaults to '/devel' :type devel_space: str @@ -339,7 +352,7 @@ def __init__( :type make_args: list :param jobs_args: -j and -l jobs args :type jobs_args: list - :param use_internal_make_jobserver: true if this configuration should use an internal make jobserv + :param use_internal_make_jobserver: true if this configuration should use an internal make jobserver :type use_internal_make_jobserver: bool :param use_env_cache: true if this configuration should cache job environments loaded from resultspaces :type use_env_cache: bool @@ -384,7 +397,7 @@ def __init__( if len(kwargs) > 0: print('Warning: Unhandled config context options: {}'.format(kwargs), file=sys.stderr) - self.destdir = os.environ['DESTDIR'] if 'DESTDIR' in os.environ else None + self.destdir = os.environ.get('DESTDIR', None) # Handle package whitelist/blacklist self.whitelist = whitelist or [] @@ -393,7 +406,7 @@ def __init__( # Handle default authors/maintainers self.authors = authors or [] self.maintainers = maintainers or [] - self.licenses = licenses or 'TODO' + self.licenses = licenses or ['TODO'] # Handle build options self.devel_layout = devel_layout if devel_layout else 'linked' @@ -476,7 +489,7 @@ def load_env(self): "requires the `catkin` CMake package in your source space " "in order to be built.")] - # Add warnings based on conflicing CMAKE_PREFIX_PATH + # Add warnings based on conflicting CMAKE_PREFIX_PATH elif self.cached_cmake_prefix_path and self.extend_path: ep_not_in_lcpp = any([self.extend_path in p for p in self.cached_cmake_prefix_path.split(':')]) if not ep_not_in_lcpp: @@ -581,8 +594,20 @@ def existence_str(path, used=True): return clr(' @{bf}[unused]@|') install_layout = 'None' - if self.__install: - install_layout = 'merged' if not self.__isolate_install else 'isolated' + if self.install: + install_layout = 'merged' if not self.isolate_install else 'isolated' + + def quote(argument): + # Distinguish in the printout if space separates two arguments or if we + # print an argument with a space. + # e.g. -DCMAKE_C_FLAGS="-g -O3" -DCMAKE_C_COMPILER=clang + if ' ' in argument: + if "=" in argument: + key, value = argument.split("=", 1) + if ' ' not in key: + return key + '="' + value + '"' + return '"' + argument + '"' + return argument subs = { 'profile': self.profile, @@ -590,18 +615,21 @@ def existence_str(path, used=True): 'extend': extend_value, 'install_layout': install_layout, 'cmake_prefix_path': (self.cmake_prefix_path or ['Empty']), - 'cmake_args': ' '.join(self.__cmake_args or ['None']), - 'make_args': ' '.join(self.__make_args + self.__jobs_args or ['None']), - 'catkin_make_args': ', '.join(self.__catkin_make_args or ['None']), - 'source_missing': existence_str(self.source_space_abs), - 'log_missing': existence_str(self.log_space_abs), - 'build_missing': existence_str(self.build_space_abs), - 'devel_missing': existence_str(self.devel_space_abs), - 'install_missing': existence_str(self.install_space_abs, used=self.__install), + 'cmake_args': ' '.join([quote(a) for a in self.cmake_args or ['None']]), + 'make_args': ' '.join(self.make_args + self.jobs_args or ['None']), + 'catkin_make_args': ', '.join(self.catkin_make_args or ['None']), 'destdir_missing': existence_str(self.destdir, used=self.destdir), - 'whitelisted_packages': ' '.join(self.__whitelist or ['None']), - 'blacklisted_packages': ' '.join(self.__blacklist or ['None']), + 'whitelisted_packages': ' '.join(self.whitelist or ['None']), + 'blacklisted_packages': ' '.join(self.blacklist or ['None']), } + for space, space_dict in sorted(Context.SPACES.items()): + key_missing = '{}_missing'.format(space) + space_abs = getattr(self, '{}_space_abs'.format(space)) + if hasattr(self, space): + subs[key_missing] = existence_str(space_abs, used=getattr(self, space)) + else: + subs[key_missing] = existence_str(space_abs) + subs.update(**self.__dict__) # Get the width of the shell width = terminal_width() @@ -630,7 +658,7 @@ def existence_str(path, used=True): return (divider + "\n" + ("\n" + divider + "\n").join(groups) + "\n" + divider + "\n" + - ((("\n\n").join(notes) + "\n" + divider) if notes else '') + + (("\n\n".join(notes) + "\n" + divider) if notes else '') + warnings_joined) @property @@ -644,7 +672,7 @@ def workspace(self, value): # Validate Workspace if not os.path.exists(value): raise ValueError("Workspace path '{0}' does not exist.".format(value)) - self.__workspace = os.path.abspath(value) + self.__workspace = os.path.realpath(os.path.abspath(value)) @property def extend_path(self): @@ -842,16 +870,23 @@ def extends(self, value): @property def private_devel_path(self): """The path to the hidden directory in the develspace that - contains the symbollically-linked isolated develspaces.""" + contains the symbolically-linked isolated develspaces.""" return os.path.join(self.devel_space_abs, '.private') def package_private_devel_path(self, package): """The path to the linked devel space for a given package.""" return os.path.join(self.private_devel_path, package.name) - def package_build_space(self, package): - """Get the build directory for a specific package.""" - return os.path.join(self.build_space_abs, package.name) + def package_source_space(self, package): + """Get the source directory of a specific package.""" + for pkg_name, pkg in self.packages: + if pkg_name == package.name: + pkg_dir = os.path.dirname(pkg.filename) + # Need to check if the pkg_dir is the source space as it can also be loaded from the metadata + if os.path.commonpath([self.source_space_abs, pkg_dir]) == self.source_space_abs: + return pkg_dir + + return None def package_devel_space(self, package): """Get the devel directory for a specific package. @@ -864,7 +899,7 @@ def package_devel_space(self, package): elif self.link_devel: return os.path.join(self.private_devel_path, package.name) else: - raise ValueError('Unkown devel space layout: {}'.format(self.devel_layout)) + raise ValueError('Unknown devel space layout: {}'.format(self.devel_layout)) def package_install_space(self, package): """Get the install directory for a specific package. @@ -876,7 +911,7 @@ def package_install_space(self, package): elif self.isolate_install: return os.path.join(self.install_space_abs, package.name) else: - raise ValueError('Unkown install space layout: {}'.format(self.devel_layout)) + raise ValueError('Unknown install space layout: {}'.format(self.devel_layout)) def package_dest_path(self, package): """Get the intermediate destination into which a specific package is built.""" diff --git a/catkin_tools/execution/controllers.py b/catkin_tools/execution/controllers.py index 4506ab2b..55b61412 100644 --- a/catkin_tools/execution/controllers.py +++ b/catkin_tools/execution/controllers.py @@ -12,18 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - # Python3 - from queue import Empty -except ImportError: - # Python2 - from Queue import Empty - import math import itertools import sys import threading import time +from queue import Empty from catkin_tools.common import disable_wide_log from catkin_tools.common import format_time_delta @@ -40,7 +34,7 @@ from catkin_tools.execution import job_server -# This map translates more human reable format strings into colorized versions +# This map translates more human readable format strings into colorized versions _color_translation_map = { # 'output': 'colorized_output' @@ -695,7 +689,8 @@ def run(self): if self.show_buffered_stdout: if len(event.data['interleaved']) > 0: lines = [ - line for line in event.data['interleaved'].splitlines(True) + line + '\n' + for line in event.data['interleaved'].splitlines() if (self.show_compact_io is False or len(line.strip()) > 0) ] else: @@ -705,7 +700,8 @@ def run(self): elif self.show_buffered_stderr: if len(event.data['stderr']) > 0: lines = [ - line for line in event.data['stderr'].splitlines(True) + line + '\n' + for line in event.data['stderr'].splitlines() if (self.show_compact_io is False or len(line.strip()) > 0) ] lines_target = sys.stderr @@ -725,7 +721,7 @@ def run(self): if header_title: wide_log(header_title, file=header_title_file) if len(lines) > 0: - wide_log(''.join(lines), end='\r', file=lines_target) + wide_log(''.join(lines), end='\n', file=lines_target) if footer_border: wide_log(footer_border, file=footer_border_file) if footer_title: @@ -733,11 +729,11 @@ def run(self): elif 'STDERR' == eid: if self.show_live_stderr and len(event.data['data']) > 0: - wide_log(self.format_interleaved_lines(event.data), end='\r', file=sys.stderr) + wide_log(self.format_interleaved_lines(event.data), file=sys.stderr) elif 'STDOUT' == eid: if self.show_live_stdout and len(event.data['data']) > 0: - wide_log(self.format_interleaved_lines(event.data), end='\r') + wide_log(self.format_interleaved_lines(event.data)) elif 'MESSAGE' == eid: wide_log(event.data['msg']) @@ -763,7 +759,8 @@ def format_interleaved_lines(self, data): else: prefix = '' - template = '\r{}\r{}'.format(' ' * terminal_width(), prefix) + # This is used to clear the status bar that is printed in the current line + clear_line = '\r{}\r'.format(' ' * terminal_width()) suffix = clr('@|') - - return ''.join(template + line + suffix for line in data['data'].splitlines(True)) + lines = data['data'].splitlines() + return clear_line + '\n'.join(prefix + line + suffix for line in lines) diff --git a/catkin_tools/execution/events.py b/catkin_tools/execution/events.py index afedce83..16320ce4 100644 --- a/catkin_tools/execution/events.py +++ b/catkin_tools/execution/events.py @@ -21,7 +21,7 @@ class ExecutionEvent(object): Events can be jobs starting/finishing, commands starting/failing/finishing, commands producing output (each line is an event), or when the executor - quits or failes. + quits or fails. """ # TODO: Make this a map of ID -> fields diff --git a/catkin_tools/execution/executor.py b/catkin_tools/execution/executor.py index bbceb3be..89088a64 100644 --- a/catkin_tools/execution/executor.py +++ b/catkin_tools/execution/executor.py @@ -12,17 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -import traceback - -from itertools import tee - import asyncio - +import traceback from concurrent.futures import ThreadPoolExecutor from concurrent.futures import FIRST_COMPLETED - +from itertools import tee from osrf_pycommon.process_utils import async_execute_process from osrf_pycommon.process_utils import get_loop @@ -46,9 +40,12 @@ def split(values, cond): async def async_job(verb, job, threadpool, locks, event_queue, log_path): """Run a sequence of Stages from a Job and collect their output. + :param verb: Current command verb :param job: A Job instance - :threadpool: A thread pool executor for blocking stages - :event_queue: A queue for asynchronous events + :param threadpool: A thread pool executor for blocking stages + :param locks: Dict containing the locks to acquire + :param event_queue: A queue for asynchronous events + :param log_path: The path in which logfiles can be written """ # Initialize success flag @@ -101,7 +98,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): # Get the logger protocol_type = stage.logger_factory(verb, job.jid, stage.label, event_queue, log_path) - # Start asynchroonous execution + # Start asynchronous execution transport, logger = await ( async_execute_process( protocol_type, @@ -126,7 +123,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): # Asynchronously yield until this command is completed retcode = await logger.complete except: # noqa: E722 - # Bare except is permissable here because the set of errors which the CommandState might raise + # Bare except is permissible here because the set of errors which the CommandState might raise # is unbounded. We capture the traceback here and save it to the build's log files. logger = IOBufferLogger(verb, job.jid, stage.label, event_queue, log_path) logger.err(str(traceback.format_exc())) @@ -142,7 +139,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): logger, event_queue) except: # noqa: E722 - # Bare except is permissable here because the set of errors which the FunctionStage might raise + # Bare except is permissible here because the set of errors which the FunctionStage might raise # is unbounded. We capture the traceback here and save it to the build's log files. logger.err('Stage `{}` failed with arguments:'.format(stage.label)) for arg_val in stage.args: @@ -154,7 +151,7 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): raise TypeError("Bad Job Stage: {}".format(stage)) # Set whether this stage succeeded - stage_succeeded = (retcode == 0) + stage_succeeded = (retcode in stage.success_retcodes) # Update success tracker from this stage all_stages_succeeded = all_stages_succeeded and stage_succeeded @@ -172,13 +169,17 @@ async def async_job(verb, job, threadpool, locks, event_queue, log_path): repro=stage.get_reproduction_cmd(verb, job.jid), retcode=retcode)) + # Early termination of the whole job + if retcode == stage.early_termination_retcode: + break + # Close logger logger.close() finally: lock.release() # Finally, return whether all stages of the job completed - return (job.jid, all_stages_succeeded) + return job.jid, all_stages_succeeded async def execute_jobs( @@ -192,7 +193,9 @@ async def execute_jobs( continue_without_deps=False): """Process a number of jobs asynchronously. + :param verb: Current command verb :param jobs: A list of topologically-sorted Jobs with no circular dependencies. + :param locks: Dict containing the locks to acquire :param event_queue: A python queue for reporting events. :param log_path: The path in which logfiles can be written :param max_toplevel_jobs: Max number of top-level jobs diff --git a/catkin_tools/execution/io.py b/catkin_tools/execution/io.py index 0e725cb5..ecb26ad1 100644 --- a/catkin_tools/execution/io.py +++ b/catkin_tools/execution/io.py @@ -13,14 +13,16 @@ # limitations under the License. import os +import re import shutil - from glob import glob from osrf_pycommon.process_utils import AsyncSubprocessProtocol from catkin_tools.common import mkdir_p +from catkin_tools.terminal_color import fmt + from .events import ExecutionEvent MAX_LOGFILE_HISTORY = 10 @@ -127,13 +129,15 @@ def get_stderr_log(self): except UnicodeDecodeError: return "stderr_log: some output cannot be displayed.\n" - def _encode(self, data): + @staticmethod + def _encode(data): """Encode a Python str into bytes. :type data: str """ return _encode(data) - def _decode(self, data): + @staticmethod + def _decode(data): """Decode bytes into Python str. :type data: bytes """ @@ -167,6 +171,7 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k def out(self, data, end='\n'): """ :type data: str + :type end: str """ # Buffer the encoded data data += end @@ -187,6 +192,7 @@ def out(self, data, end='\n'): def err(self, data, end='\n'): """ :type data: str + :type end: str """ # Buffer the encoded data data += end @@ -223,7 +229,8 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k self.intermediate_stdout_buffer = b'' self.intermediate_stderr_buffer = b'' - def _split(self, data): + @staticmethod + def _split(data): try: last_break = data.rindex(b'\n') + 1 return data[0:last_break], data[last_break:] @@ -281,3 +288,20 @@ def on_process_exited2(self, returncode): self.on_stdout_received(self.intermediate_stdout_buffer + b'\n') if len(self.intermediate_stderr_buffer) > 0: self.on_stderr_received(self.intermediate_stderr_buffer + b'\n') + + +class CatkinTestResultsIOBufferProtocol(IOBufferProtocol): + """An IOBufferProtocol which parses the output of catkin_test_results""" + def on_stdout_received(self, data): + lines = data.decode().splitlines() + clines = [] + for line in lines: + match = re.match(r'(.*): (\d+) tests, (\d+) errors, (\d+) failures, (\d+) skipped', line) + if match: + line = fmt('@!{}@|: {} tests, @{rf}{} errors@|, @{rf}{} failures@|, @{kf}{} skipped@|') + line = line.format(*match.groups()) + clines.append(line) + + cdata = '\n'.join(clines) + '\n' + + super(CatkinTestResultsIOBufferProtocol, self).on_stdout_received(cdata.encode()) diff --git a/catkin_tools/execution/job_server.py b/catkin_tools/execution/job_server.py index 3185be90..8e14b604 100644 --- a/catkin_tools/execution/job_server.py +++ b/catkin_tools/execution/job_server.py @@ -12,18 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -from multiprocessing import cpu_count -from tempfile import mkstemp -from termios import FIONREAD - import array import fcntl import os import re import subprocess import time +from multiprocessing import cpu_count +from tempfile import mkstemp +from termios import FIONREAD from catkin_tools.common import log from catkin_tools.common import version_tuple @@ -75,7 +72,7 @@ def memory_usage(): def test_gnu_make_support_common(makefile_content): """ - Test if "make -f MAKEFILE -j2" runs successfullyn when MAKEFILE + Test if "make -f MAKEFILE -j2" runs successfully when MAKEFILE contains makefile_content. """ @@ -86,7 +83,7 @@ def test_gnu_make_support_common(makefile_content): ret = subprocess.call(['make', '-f', makefile, '-j2'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) os.unlink(makefile) - return (ret == 0) + return ret == 0 def test_gnu_make_support_old(): @@ -222,7 +219,7 @@ def _check_load(cls): if cls._max_load is not None: try: load = os.getloadavg() - if load[1] < cls._max_load: + if load[0] < cls._max_load: cls._load_ok = True else: cls._load_ok = False @@ -297,6 +294,7 @@ def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=Fals :param max_mem: do not dispatch additional jobs if system physical memory usage exceeds this value (see _set_max_mem for additional documentation) + :param gnu_make_enabled: Set gnu make compatibility enabled """ # Check initialization @@ -311,7 +309,7 @@ def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=Fals log(clr('@!@{yf}WARNING:@| Make job server not supported. The number of Make ' 'jobs may exceed the number of CPU cores.@|')) - # Set gnu make compatibilty enabled + # Set gnu make compatibility enabled JobServer._gnu_make_enabled = gnu_make_enabled # Set the maximum number of jobs diff --git a/catkin_tools/execution/jobs.py b/catkin_tools/execution/jobs.py index 1826926e..7bb5bc2b 100644 --- a/catkin_tools/execution/jobs.py +++ b/catkin_tools/execution/jobs.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from catkin_tools.terminal_color import ColorMapper mapper = ColorMapper() diff --git a/catkin_tools/execution/stages.py b/catkin_tools/execution/stages.py index 5d4e9dee..a63fdc68 100644 --- a/catkin_tools/execution/stages.py +++ b/catkin_tools/execution/stages.py @@ -14,13 +14,7 @@ import os import traceback - -from catkin_tools.common import string_type - -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote +from shlex import quote as cmd_quote from .io import IOBufferLogger from .io import IOBufferProtocol @@ -40,12 +34,16 @@ def __init__( logger_factory=IOBufferProtocol.factory, occupy_job=True, locked_resource=None, + early_termination_retcode=None, + success_retcodes=(0,), repro=None): self.label = str(label) self.logger_factory = logger_factory self.occupy_job = occupy_job self.repro = repro self.locked_resource = locked_resource + self.early_termination_retcode = early_termination_retcode + self.success_retcodes = success_retcodes def get_reproduction_cmd(self, verb, jid): """Get a command line to reproduce this stage with the proper environment.""" @@ -69,13 +67,12 @@ def __init__( stderr_to_stdout=False, occupy_job=True, locked_resource=None, - logger_factory=IOBufferProtocol.factory): + logger_factory=IOBufferProtocol.factory, + early_termination_retcode=None, + success_retcodes=(0,)): """ :param label: The label for the stage - :param command: A list of strings composing a system command - :param protocol: A protocol class to use for this stage - - :parma cmd: The command to run + :param cmd: The command to run :param cwd: The directory in which to run the command (default: os.getcwd()) :param env: The base environment. (default: {}) :param env_overrides: The variables that override the base environment. (default: {}) @@ -85,11 +82,14 @@ def __init__( :param occupy_job: Whether this stage should wait for a worker from the job server (default: True) :param logger_factory: The factory to use to construct a logger (default: IOBufferProtocol.factory) + :param early_termination_retcode: A return code that ends the whole job successfully (default: None) + :param success_retcodes: A tuple of return codes that mean successful termination of the command (default: (0,)) """ - if not type(cmd) in [list, tuple] or not all([isinstance(s, string_type) for s in cmd]): + if not type(cmd) in [list, tuple] or not all([isinstance(s, str) for s in cmd]): raise ValueError('Command stage must be a list of strings: {}'.format(cmd)) - super(CommandStage, self).__init__(label, logger_factory, occupy_job, locked_resource) + super(CommandStage, self).__init__(label, logger_factory, occupy_job, locked_resource, + early_termination_retcode, success_retcodes) # Store environment overrides self.env_overrides = env_overrides or {} @@ -160,11 +160,14 @@ def __init__( logger_factory=IOBufferLogger.factory, occupy_job=True, locked_resource=None, + early_termination_retcode=None, + success_retcodes=(0,), *args, **kwargs): if not callable(function): raise ValueError('Function stage must be callable.') - super(FunctionStage, self).__init__(label, logger_factory, occupy_job, locked_resource) + super(FunctionStage, self).__init__(label, logger_factory, occupy_job, locked_resource, + early_termination_retcode, success_retcodes) self.args = args self.kwargs = kwargs diff --git a/catkin_tools/jobs/catkin.py b/catkin_tools/jobs/catkin.py index fee4a133..8236e651 100644 --- a/catkin_tools/jobs/catkin.py +++ b/catkin_tools/jobs/catkin.py @@ -14,11 +14,7 @@ import csv import os - -try: - from md5 import md5 -except ImportError: - from hashlib import md5 +from hashlib import md5 from catkin_tools.argument_parsing import handle_make_arguments @@ -27,10 +23,12 @@ from catkin_tools.execution.jobs import Job from catkin_tools.execution.stages import CommandStage from catkin_tools.execution.stages import FunctionStage +from catkin_tools.execution.io import CatkinTestResultsIOBufferProtocol from .commands.cmake import CMAKE_EXEC from .commands.cmake import CMakeIOBufferProtocol from .commands.cmake import CMakeMakeIOBufferProtocol +from .commands.cmake import CMakeMakeRunTestsIOBufferProtocol from .commands.cmake import get_installed_files from .commands.make import MAKE_EXEC @@ -86,13 +84,14 @@ def clean_linked_files( files_that_collide, files_to_clean, dry_run): - """Removes a list of files and adjusts collison counts for colliding files. + """Removes a list of files and adjusts collision counts for colliding files. This function synchronizes access to the devel collisions file. - :param devel_space_abs: absolute path to merged devel space + :param metadata_path: absolute path to the general metadata directory :param files_that_collide: list of absolute paths to files that collide :param files_to_clean: list of absolute paths to files to clean + :param dry_run: Perform a dry-run """ # Get paths @@ -168,8 +167,10 @@ def unlink_devel_products( :param devel_space_abs: Path to a merged devel space. :param private_devel_path: Path to the private devel space - :param devel_manifest_path: Path to the directory containing the package's + :param metadata_path: Path to the directory containing the general metadata + :param package_metadata_path: Path to the directory containing the package's catkin_tools metadata + :param dry_run: Perform a dry-run """ # Check paths @@ -204,7 +205,7 @@ def unlink_devel_products( # Clean the file or decrement the collision count files_to_clean.append(dest_file) - # Remove all listed symli and empty directories which have been removed + # Remove all listed symlinks and empty directories which have been removed # after this build, and update the collision file clean_linked_files(logger, event_queue, metadata_path, [], files_to_clean, dry_run) @@ -263,7 +264,7 @@ def link_devel_products( logger.out('Linked: ({}, {})'.format(source_dir, dest_dir)) else: # Create a symlink - logger.out('Symlinking %s' % (dest_dir)) + logger.out('Symlinking %s' % dest_dir) try: os.symlink(source_dir, dest_dir) except OSError: @@ -308,7 +309,7 @@ def link_devel_products( logger.out('Linked: ({}, {})'.format(source_file, dest_file)) else: # Create the symlink - logger.out('Symlinking %s' % (dest_file)) + logger.out('Symlinking %s' % dest_file) try: os.symlink(source_file, dest_file) except OSError: @@ -589,11 +590,89 @@ def create_catkin_clean_job( stages=stages) +def create_catkin_test_job( + context, + package, + package_path, + test_target, + verbose, +): + """Generate a job that tests a package""" + + # Package source space path + pkg_dir = os.path.join(context.source_space_abs, package_path) + # Package build space path + build_space = context.package_build_space(package) + # Environment dictionary for the job, which will be built + # up by the executions in the loadenv stage. + job_env = dict(os.environ) + + # Create job stages + stages = [] + + # Load environment for job + stages.append(FunctionStage( + 'loadenv', + loadenv, + locked_resource=None, + job_env=job_env, + package=package, + context=context, + verbose=False, + )) + + # Check buildsystem command + # The stdout is suppressed here instead of globally because for the actual tests, + # stdout contains important information, but for cmake it is only relevant when verbose + stages.append(CommandStage( + 'check', + [MAKE_EXEC, 'cmake_check_build_system'], + cwd=build_space, + logger_factory=CMakeIOBufferProtocol.factory_factory(pkg_dir, suppress_stdout=not verbose), + occupy_job=True + )) + + # Check if the test target exists + # make -q target_name returns 2 if the target does not exist, in that case we want to terminate this test job + # the other cases (0=target is up-to-date, 1=target exists but is not up-to-date) can be ignored + stages.append(CommandStage( + 'findtest', + [MAKE_EXEC, '-q', test_target], + cwd=build_space, + early_termination_retcode=2, + success_retcodes=(0, 1, 2), + )) + + # Make command + stages.append(CommandStage( + 'make', + [MAKE_EXEC, test_target] + context.make_args, + cwd=build_space, + logger_factory=CMakeMakeRunTestsIOBufferProtocol.factory_factory(verbose), + )) + + # catkin_test_results + stages.append(CommandStage( + 'results', + ['catkin_test_results'], + cwd=build_space, + logger_factory=CatkinTestResultsIOBufferProtocol.factory, + )) + + return Job( + jid=package.name, + deps=[], + env=job_env, + stages=stages, + ) + + description = dict( build_type='catkin', description="Builds a catkin package.", create_build_job=create_catkin_build_job, - create_clean_job=create_catkin_clean_job + create_clean_job=create_catkin_clean_job, + create_test_job=create_catkin_test_job, ) diff --git a/catkin_tools/jobs/cmake.py b/catkin_tools/jobs/cmake.py index 0c943a40..d90ff634 100644 --- a/catkin_tools/jobs/cmake.py +++ b/catkin_tools/jobs/cmake.py @@ -36,6 +36,7 @@ from .utils import require_command from .utils import rmfiles +from catkin_tools.execution.io import IOBufferProtocol from catkin_tools.execution.jobs import Job from catkin_tools.execution.stages import CommandStage from catkin_tools.execution.stages import FunctionStage @@ -45,13 +46,6 @@ mapper = ColorMapper() clr = mapper.clr -# FileNotFoundError from Python3 -try: - FileNotFoundError -except NameError: - class FileNotFoundError(OSError): - pass - def copy_install_manifest( logger, event_queue, @@ -413,11 +407,67 @@ def create_cmake_clean_job( stages=stages) +def create_cmake_test_job( + context, + package, + package_path, + test_target, + verbose, +): + """Generate a job to test a cmake package""" + # Package build space path + build_space = context.package_build_space(package) + # Environment dictionary for the job, which will be built + # up by the executions in the loadenv stage. + job_env = dict(os.environ) + + # Create job stages + stages = [] + + # Load environment for job + stages.append(FunctionStage( + 'loadenv', + loadenv, + locked_resource=None, + job_env=job_env, + package=package, + context=context, + verbose=False, + )) + + # Check if the test target exists + # make -q target_name returns 2 if the target does not exist, in that case we want to terminate this test job + # the other cases (0=target is up-to-date, 1=target exists but is not up-to-date) can be ignored + stages.append(CommandStage( + 'findtest', + [MAKE_EXEC, '-q', test_target], + cwd=build_space, + early_termination_retcode=2, + success_retcodes=(0, 1, 2), + )) + + # Make command + stages.append(CommandStage( + 'make', + [MAKE_EXEC, test_target] + context.make_args, + cwd=build_space, + logger_factory=IOBufferProtocol.factory, + )) + + return Job( + jid=package.name, + deps=[], + env=job_env, + stages=stages, + ) + + description = dict( build_type='cmake', description="Builds a plain CMake package.", create_build_job=create_cmake_build_job, - create_clean_job=create_cmake_clean_job + create_clean_job=create_cmake_clean_job, + create_test_job=create_cmake_test_job, ) diff --git a/catkin_tools/jobs/commands/cmake.py b/catkin_tools/jobs/commands/cmake.py index caff7311..981f8d45 100644 --- a/catkin_tools/jobs/commands/cmake.py +++ b/catkin_tools/jobs/commands/cmake.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import re @@ -52,9 +50,11 @@ def abspath(self, groups): """Group filter that turns source-relative paths into absolute paths.""" return (groups[0] if groups[0].startswith(os.sep) else os.path.join(self.source_path, groups[0]),) + groups[1:] - def __init__(self, label, job_id, stage_label, event_queue, log_path, source_path, *args, **kwargs): + def __init__(self, label, job_id, stage_label, event_queue, log_path, source_path, suppress_stdout, *args, + **kwargs): super(CMakeIOBufferProtocol, self).__init__(label, job_id, stage_label, event_queue, log_path, *args, **kwargs) self.source_path = source_path + self.suppress_stdout = suppress_stdout # These are buffers for incomplete lines that we want to wait to parse # until we have received them completely @@ -72,17 +72,18 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, source_pat (r'CMake Error at (.+):(.+)', '@{rf}@!CMake Error@| at {}:{}', self.abspath), (r'CMake Warning at (.+):(.+)', '@{yf}@!CMake Warning@| at {}:{}', self.abspath), (r'CMake Warning (dev) at (.+):(.+)', '@{yf}@!CMake Warning (dev)@| at {}:{}', self.abspath), - (r'(?i)(warning.*)', '@(yf){}@|', None), - (r'(?i)ERROR:(.*)', '@!@(rf)ERROR:@|{}@|', None), + (r'(?i)(warning.*)', '@{yf}{}@|', None), + (r'(?i)ERROR:(.*)', '@!@{rf}ERROR:@|{}@|', None), (r'Call Stack \(most recent call first\):(.*)', '@{cf}Call Stack (most recent call first):@|{}', None), ] self.filters = [(re.compile(p), r, f) for (p, r, f) in filters] def on_stdout_received(self, data): - data_head, self.stdout_tail = split_to_last_line_break(self.stdout_tail + data) - colored = self.color_lines(data_head) - super(CMakeIOBufferProtocol, self).on_stdout_received(colored) + if not self.suppress_stdout: + data_head, self.stdout_tail = split_to_last_line_break(self.stdout_tail + data) + colored = self.color_lines(data_head) + super(CMakeIOBufferProtocol, self).on_stdout_received(colored) def on_stderr_received(self, data): data_head, self.stderr_tail = split_to_last_line_break(self.stderr_tail + data) @@ -109,7 +110,7 @@ def color_lines(self, data): """Apply colorization rules to each line in data""" decoded_data = self._decode(data) # TODO: This will only work if all lines are received at once. Instead - # of direclty splitting lines, we should buffer the data lines until + # of directly splitting lines, we should buffer the data lines until # the last character is a line break lines = decoded_data.splitlines(True) # Keep line breaks colored_lines = [self.colorize_cmake(line) for line in lines] @@ -118,13 +119,14 @@ def color_lines(self, data): return encoded_data @classmethod - def factory_factory(cls, source_path): + def factory_factory(cls, source_path, suppress_stdout=False): """Factory factory for constructing protocols that know the source path for this CMake package.""" def factory(label, job_id, stage_label, event_queue, log_path): - # factory is called by caktin_tools executor + # factory is called by catkin_tools executor def init_proxy(*args, **kwargs): # init_proxy is called by asyncio - return cls(label, job_id, stage_label, event_queue, log_path, source_path, *args, **kwargs) + return cls(label, job_id, stage_label, event_queue, log_path, source_path, suppress_stdout, *args, + **kwargs) return init_proxy return factory @@ -150,12 +152,12 @@ def colorize_cmake(self, line): cline = cline.format(*match.groups()) break - return cline + '\r\n' + return cline + '\n' class CMakeMakeIOBufferProtocol(IOBufferProtocol): - """An IOBufferProtocol which parses CMake's progree prefixes and emits corresponding STAGE_PROGRESS events.""" + """An IOBufferProtocol which parses CMake's progress prefixes and emits corresponding STAGE_PROGRESS events.""" def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **kwargs): super(CMakeMakeIOBufferProtocol, self).__init__( @@ -163,8 +165,10 @@ def __init__(self, label, job_id, stage_label, event_queue, log_path, *args, **k def on_stdout_received(self, data): super(CMakeMakeIOBufferProtocol, self).on_stdout_received(data) + self.send_progress(data) - # Parse CMake Make completion progress + def send_progress(self, data): + """Parse CMake Make completion progress""" progress_matches = re.match(r'\[\s*([0-9]+)%\]', self._decode(data)) if progress_matches is not None: self.event_queue.put(ExecutionEvent( @@ -174,6 +178,56 @@ def on_stdout_received(self, data): percent=str(progress_matches.groups()[0]))) +class CMakeMakeRunTestsIOBufferProtocol(CMakeMakeIOBufferProtocol): + """An IOBufferProtocol which parses the output of `make run_tests`.""" + def __init__(self, label, job_id, stage_label, event_queue, log_path, verbose, *args, **kwargs): + super(CMakeMakeRunTestsIOBufferProtocol, self).__init__( + label, job_id, stage_label, event_queue, log_path, *args, **kwargs) + + # Line formatting filters + # Each is a 2-tuple: + # - regular expression + # - output formatting line + self.filters = [ + (re.compile(r'^-- run_tests.py:'), '@!@{kf}{}@|'), + ] + + self.in_test_output = False + self.verbose = verbose + + def on_stdout_received(self, data): + self.send_progress(data) + + data = self._decode(data) + if data.startswith('-- run_tests.py: execute command'): + self.in_test_output = True + elif data.startswith('-- run_tests.py: verify result'): + self.in_test_output = False + + if self.verbose or self.in_test_output: + colored = self.colorize_run_tests(data) + super(CMakeMakeRunTestsIOBufferProtocol, self).on_stdout_received(colored.encode()) + + def colorize_run_tests(self, line): + cline = sanitize(line).rstrip() + for p, r in self.filters: + if p.match(cline): + lines = [fmt(r).format(line) for line in cline.splitlines()] + cline = '\n'.join(lines) + return cline + '\n' + + @classmethod + def factory_factory(cls, verbose): + """Factory factory for constructing protocols that know the verbosity.""" + def factory(label, job_id, stage_label, event_queue, log_path): + # factory is called by catkin_tools executor + def init_proxy(*args, **kwargs): + # init_proxy is called by asyncio + return cls(label, job_id, stage_label, event_queue, log_path, verbose, *args, **kwargs) + return init_proxy + return factory + + def get_installed_files(path): """Get a set of files installed by a CMake package as specified by an install_manifest.txt in a given directory.""" diff --git a/catkin_tools/jobs/output.py b/catkin_tools/jobs/output.py index 429c6a61..f97fc197 100644 --- a/catkin_tools/jobs/output.py +++ b/catkin_tools/jobs/output.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.terminal_color import ansi diff --git a/catkin_tools/jobs/utils.py b/catkin_tools/jobs/utils.py index d9cb8b36..431c218f 100644 --- a/catkin_tools/jobs/utils.py +++ b/catkin_tools/jobs/utils.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import shutil @@ -26,7 +24,7 @@ class CommandMissing(Exception): - '''A required command is missing.''' + """A required command is missing.""" def __init__(self, name): super(CommandMissing, self).__init__( @@ -46,7 +44,7 @@ def get_env_loaders(package, context): if (context.install and context.isolate_install) or (not context.install and context.isolate_devel): # Source each package's install or devel space space = context.install_space_abs if context.install else context.devel_space_abs - # Get the recursive dependcies + # Get the recursive dependencies depends = get_cached_recursive_build_depends_in_workspace(package, context.packages) # For each dep add a line to source its setup file for dep_pth, dep in depends: @@ -64,12 +62,12 @@ def get_env_loaders(package, context): def merge_envs(job_env, overlay_envs): - ''' + """ In the merged/linked case of single env, this function amounts to a straight assignment, but a more complex merge is required with isolated result spaces, since a package's build environment may require extending that of multiple other result spaces. - ''' + """ merge_path_values = {} for overlay_env in overlay_envs: @@ -103,7 +101,7 @@ def merge_envs(job_env, overlay_envs): job_env[key] = os.pathsep.join(reversed(new_values_list)) -def loadenv(logger, event_queue, job_env, package, context): +def loadenv(logger, event_queue, job_env, package, context, verbose=True): # Get the paths to the env loaders env_loader_paths = get_env_loaders(package, context) # If DESTDIR is set, set _CATKIN_SETUP_DIR as well @@ -112,7 +110,7 @@ def loadenv(logger, event_queue, job_env, package, context): envs = [] for env_loader_path in env_loader_paths: - if logger: + if logger and verbose: logger.out('Loading environment from: {}'.format(env_loader_path)) envs.append(get_resultspace_environment( os.path.split(env_loader_path)[0], @@ -151,16 +149,16 @@ def rmfile(logger, event_queue, path): return 0 -def rmdirs(logger, event_queue, paths): +def rmdirs(logger, event_queue, paths, dry_run): """FunctionStage functor that removes a directory tree.""" - return rmfiles(logger, event_queue, paths, remove_empty=False) + return rmfiles(logger, event_queue, paths, dry_run, remove_empty=False) def rmfiles(logger, event_queue, paths, dry_run, remove_empty=False, empty_root='/'): """FunctionStage functor that removes a list of files and directories. If remove_empty is True, then this will also remove directories which - become emprt after deleting the files in `paths`. It will delete files up + become empty after deleting the files in `paths`. It will delete files up to the path specified by `empty_root`. """ @@ -203,7 +201,7 @@ def rmfiles(logger, event_queue, paths, dry_run, remove_empty=False, empty_root= if dir_descendants[path] == 0: paths.append(path) - # REmove the paths + # Remove the paths for index, path in enumerate(paths): # Remove the path diff --git a/catkin_tools/metadata.py b/catkin_tools/metadata.py index 981ba51e..a89fb28c 100644 --- a/catkin_tools/metadata.py +++ b/catkin_tools/metadata.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import pkg_resources import shutil @@ -102,7 +100,7 @@ def get_paths(workspace_path, profile_name, verb=None): # Get the metadata for this verb metadata_file_path = os.path.join(metadata_path, '%s.yaml' % verb) if profile_name and verb else None - return (metadata_path, metadata_file_path) + return metadata_path, metadata_file_path def find_enclosing_workspaces(search_start_path): """Find a catkin workspaces based on the existence of a catkin_tools @@ -230,7 +228,7 @@ def init_metadata_root(workspace_path, reset=False): if not os.path.exists(workspace_path): raise IOError( "Can't initialize Catkin workspace in path %s because it does " - "not exist." % (workspace_path)) + "not exist." % workspace_path) # Construct the full path to the metadata directory metadata_root_path = get_metadata_root_path(workspace_path) @@ -239,7 +237,7 @@ def init_metadata_root(workspace_path, reset=False): if os.path.exists(metadata_root_path): # Reset the directory if requested if reset: - print("Deleting existing metadata from catkin_tools metadata directory: %s" % (metadata_root_path)) + print("Deleting existing metadata from catkin_tools metadata directory: %s" % metadata_root_path) shutil.rmtree(metadata_root_path) os.mkdir(metadata_root_path) else: @@ -265,6 +263,8 @@ def init_profile(workspace_path, profile_name, reset=False): :type workspace_path: str :param profile_name: The catkin_tools metadata profile name to initialize :type profile_name: str + :param reset: Delete profile with the same name if existing + :type reset: bool """ init_metadata_root(workspace_path) @@ -275,7 +275,7 @@ def init_profile(workspace_path, profile_name, reset=False): if os.path.exists(profile_path): # Reset the directory if requested if reset: - print("Deleting existing profile from catkin_tools profile directory: %s" % (profile_path)) + print("Deleting existing profile from catkin_tools profile directory: %s" % profile_path) shutil.rmtree(profile_path) os.mkdir(profile_path) else: @@ -407,7 +407,7 @@ def get_metadata(workspace_path, profile, verb): return yaml.safe_load(metadata_file) -def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, merge=True): +def update_metadata(workspace_path, profile, verb, new_data=None, no_init=False, merge=True): """Update the catkin_tools verb metadata for a given profile. :param workspace_path: The path to the root of a catkin workspace @@ -418,7 +418,13 @@ def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, m :type verb: str :param new_data: A python dictionary or array to write to the metadata file :type new_data: dict + :param no_init: Do not init metadata root and/or profile folder (default: False) + :type no_init: bool + :param merge: Merge new data with current data or ignore current data (default: True) + :type merge: bool """ + if new_data is None: + new_data = {} migrate_metadata(workspace_path) @@ -429,7 +435,7 @@ def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, m init_metadata_root(workspace_path) init_profile(workspace_path, profile) - # Get the curent metadata for this verb + # Get the current metadata for this verb if merge: data = get_metadata(workspace_path, profile, verb) else: @@ -441,7 +447,7 @@ def update_metadata(workspace_path, profile, verb, new_data={}, no_init=False, m with open(metadata_file_path, 'w') as metadata_file: yaml.dump(data, metadata_file, default_flow_style=False) except PermissionError: - print("Could not write to metadata file '%s'!" % (metadata_file_path)) + print("Could not write to metadata file '%s'!" % metadata_file_path) return data @@ -462,7 +468,7 @@ def get_active_metadata(workspace_path, verb): get_metadata(workspace_path, active_profile, verb) -def update_active_metadata(workspace_path, verb, new_data={}): +def update_active_metadata(workspace_path, verb, new_data=None): """Update the catkin_tools verb metadata for the active profile. :param workspace_path: The path to the root of a catkin workspace @@ -472,6 +478,8 @@ def update_active_metadata(workspace_path, verb, new_data={}): :param new_data: A python dictionary or array to write to the metadata file :type new_data: dict """ + if new_data is None: + new_data = {} active_profile = get_active_profile(workspace_path) - update_active_metadata(workspace_path, active_profile, verb, new_data) + update_metadata(workspace_path, active_profile, verb, new_data) diff --git a/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown b/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown index b3ef53cf..521f625e 100644 --- a/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown +++ b/catkin_tools/notifications/resources/osx/catkin_build_notifier_src/README.markdown @@ -74,7 +74,7 @@ Some examples are: ``` $ echo 'Piped Message Data!' | terminal-notifier -sound default -$ terminal-notifier -title '💰' -message 'Check your Apple stock!' -open 'http://finance.yahoo.com/q?s=AAPL' +$ terminal-notifier -title '💰' -message 'Check your Apple stock!' -open 'https://finance.yahoo.com/q?s=AAPL' $ terminal-notifier -group 'address-book-sync' -title 'Address Book Sync' -subtitle 'Finished' -message 'Imported 42 contacts.' -activate 'com.apple.AddressBook' ``` diff --git a/catkin_tools/resultspace.py b/catkin_tools/resultspace.py index 0f0ad9c3..6242e780 100644 --- a/catkin_tools/resultspace.py +++ b/catkin_tools/resultspace.py @@ -12,25 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -try: - from md5 import md5 -except ImportError: - from hashlib import md5 import os import subprocess import sys - -try: - from shlex import quote as cmd_quote -except ImportError: - from pipes import quote as cmd_quote - -from osrf_pycommon.process_utils import execute_process +from hashlib import md5 +from shlex import quote as cmd_quote from .common import parse_env_str -from .common import string_type DEFAULT_SHELL = '/bin/bash' @@ -40,13 +28,15 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, cached=True, strict=True): - """Get the environemt variables which result from sourcing another catkin + """Get the environment variables which result from sourcing another catkin workspace's setup files as the string output of `cmake -E environment`. This cmake command is used to be as portable as possible. :param result_space_path: path to a Catkin result-space whose environment should be loaded, ``str`` :type result_space_path: str - :param quiet: don't throw exceptions, ``bool`` + :param base_env: Base environment dictionary (default: os.environ) + :type base_env: dict + :param quiet: don't throw exceptions :type quiet: bool :param cached: use the cached environment :type cached: bool @@ -124,17 +114,11 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c ) # Construct a command list which sources the setup file and prints the env to stdout - norc_flags = { - 'bash': '--norc', - 'zsh': '-f' - } - command = ' '.join([ cmd_quote(setup_file_path), shell_path, - norc_flags[shell_name], '-c', - '"typeset -px"' + '"env --null"', ]) # Define some "blacklisted" environment variables which shouldn't be copied @@ -142,18 +126,8 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c env_dict = {} try: - # Run the command synchronously to get the resultspace environmnet - if 0: - # NOTE: This sometimes fails to get all output (returns prematurely) - lines = '' - for ret in execute_process(command, cwd=os.getcwd(), env=base_env, emulate_tty=False, shell=True): - if type(ret) is bytes: - ret = ret.decode() - if isinstance(ret, string_type): - lines += ret - else: - p = subprocess.Popen(command, cwd=os.getcwd(), env=base_env, shell=True, stdout=subprocess.PIPE) - lines, _ = p.communicate() + # Run the command synchronously to get the resultspace environment + lines = subprocess.check_output(command, cwd=os.getcwd(), env=base_env, shell=True) # Extract the environment variables env_dict = { @@ -179,11 +153,13 @@ def get_resultspace_environment(result_space_path, base_env=None, quiet=False, c def load_resultspace_environment(result_space_path, base_env=None, cached=True): - """Load the environemt variables which result from sourcing another + """Load the environment variables which result from sourcing another workspace path into this process's environment. :param result_space_path: path to a Catkin result-space whose environment should be loaded, ``str`` :type result_space_path: str + :param base_env: Base environment dictionary (default: os.environ) + :type base_env: dict :param cached: use the cached environment :type cached: bool """ diff --git a/catkin_tools/terminal_color.py b/catkin_tools/terminal_color.py index 1851ee41..6eb139ed 100644 --- a/catkin_tools/terminal_color.py +++ b/catkin_tools/terminal_color.py @@ -16,8 +16,6 @@ Module to enable color terminal output """ -from __future__ import print_function - import string import os @@ -155,7 +153,7 @@ class ColorMapper(object): functionality to convert them into colorized version. """ - # This map translates more human reable format strings into colorized versions + # This map translates more human readable format strings into colorized versions default_color_translation_map = { # 'output': 'colorized_output' '': fmt('@!' + sanitize('') + '@|') @@ -164,7 +162,7 @@ class ColorMapper(object): def __init__(self, color_map={}): """Create a color mapper with a given map. - :param color_map: A dictionary of format strings and colorized verisons + :param color_map: A dictionary of format strings and colorized versions :type color_map: dict """ self.color_map = ColorMapper.default_color_translation_map @@ -173,7 +171,7 @@ def __init__(self, color_map={}): def clr(self, key): """Returns a colorized version of the string given. - This is occomplished by either returning a hit from the color translation + This is accomplished by either returning a hit from the color translation map or by calling :py:func:`fmt` on the string and returning it. :param key: string to be colorized diff --git a/catkin_tools/utils.py b/catkin_tools/utils.py index 91c6854c..af229c05 100644 --- a/catkin_tools/utils.py +++ b/catkin_tools/utils.py @@ -12,15 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os def which(program): """Custom version of the ``which`` built-in shell command. - Searches the pathes in the ``PATH`` environment variable for a given + Searches the paths in the ``PATH`` environment variable for a given executable name. It returns the full path to the first instance of the executable found or None if it was not found. diff --git a/catkin_tools/verbs/catkin_build/build.py b/catkin_tools/verbs/catkin_build/build.py index d935e588..4d5e8316 100644 --- a/catkin_tools/verbs/catkin_build/build.py +++ b/catkin_tools/verbs/catkin_build/build.py @@ -16,21 +16,16 @@ import os import pkg_resources +from queue import Queue import sys import time import traceback import yaml - import asyncio try: - # Python3 - from queue import Queue -except ImportError: - # Python2 - from Queue import Queue - -try: + from catkin_pkg.package import parse_package + from catkin_pkg.package import InvalidPackage from catkin_pkg.packages import find_packages from catkin_pkg.topological_order import topological_order_packages except ImportError as e: @@ -39,8 +34,6 @@ '"catkin_pkg", and that it is up to date and on the PYTHONPATH.' % e ) -from catkin_pkg.package import parse_package - from catkin_tools.common import FakeLock, expand_glob_package from catkin_tools.common import format_time_delta from catkin_tools.common import get_cached_recursive_build_depends_in_workspace @@ -58,7 +51,6 @@ from .color import clr - BUILDSPACE_MARKER_FILE = '.catkin_tools.yaml' BUILDSPACE_IGNORE_FILE = 'CATKIN_IGNORE' DEVELSPACE_MARKER_FILE = '.catkin_tools.yaml' @@ -71,6 +63,8 @@ def determine_packages_to_be_built(packages, context, workspace_packages): :type packages: list :param context: Workspace context :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` + :param workspace_packages: list of all packages in the workspace + :type workspace_packages: list :returns: tuple of packages to be built and those package's deps :rtype: tuple """ @@ -215,10 +209,14 @@ def build_isolated_workspace( :type start_with: str :param no_deps: If True, the dependencies of packages will not be built first :type no_deps: bool + :param unbuilt: Handle unbuilt packages + :type unbuilt: bool :param n_jobs: number of parallel package build n_jobs :type n_jobs: int :param force_cmake: forces invocation of CMake if True, default is False :type force_cmake: bool + :param pre_clean: Clean current build before building + :type pre_clean: bool :param force_color: forces colored output even if terminal does not support it :type force_color: bool :param quiet: suppresses the output of commands unless there is an error @@ -248,7 +246,7 @@ def build_isolated_workspace( # Assert that the limit_status_rate is valid if limit_status_rate < 0: - sys.exit("[build] @!@{rf}Error:@| The value of --status-rate must be greater than or equal to zero.") + sys.exit("[build] @!@{rf}Error:@| The value of --limit-status-rate must be greater than or equal to zero.") # Declare a buildspace marker describing the build config for error checking buildspace_marker_data = { @@ -305,7 +303,11 @@ def build_isolated_workspace( # Get all the packages in the context source space # Suppress warnings since this is a utility function - workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) + try: + workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Get packages which have not been built yet built_packages, unbuilt_pkgs = get_built_unbuilt_packages(context, workspace_packages) @@ -400,6 +402,8 @@ def build_isolated_workspace( # Generate prebuild and prebuild clean jobs, if necessary prebuild_jobs = {} setup_util_present = os.path.exists(os.path.join(context.devel_space_abs, '_setup_util.py')) + if context.install: + setup_util_present &= os.path.exists(os.path.join(context.install_space_abs, '_setup_util.py')) catkin_present = 'catkin' in (packages_to_be_built_names + packages_to_be_built_deps_names) catkin_built = 'catkin' in built_packages prebuild_built = 'catkin_tools_prebuild' in built_packages @@ -680,7 +684,7 @@ def _create_unmerged_devel_setup_for_install(context): # This file is aggregates the many setup.sh files in the various # unmerged devel spaces in this folder. -# This is occomplished by sourcing each leaf package and all the +# This is accomplished by sourcing each leaf package and all the # recursive run dependencies of those leaf packages # Source the first package's setup.sh without the --extend option diff --git a/catkin_tools/verbs/catkin_build/cli.py b/catkin_tools/verbs/catkin_build/cli.py index 369ca665..8d20fa24 100644 --- a/catkin_tools/verbs/catkin_build/cli.py +++ b/catkin_tools/verbs/catkin_build/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import argparse import logging import os @@ -64,32 +62,6 @@ from .build import determine_packages_to_be_built from .build import verify_start_with_option -# -# Begin Hack -# - -# TODO(wjwwood): remove this, once it is no longer needed. -# argparse may not support mutually exclusive groups inside other groups, see: -# http://bugs.python.org/issue10680 - -# Backup the original constructor -backup__ArgumentGroup___init__ = argparse._ArgumentGroup.__init__ - - -# Make a new constructor with the fix -def fixed__ArgumentGroup___init__(self, container, title=None, description=None, **kwargs): - backup__ArgumentGroup___init__(self, container, title, description, **kwargs) - # Make sure this line is run, maybe redundant on versions which already have it - self._mutually_exclusive_groups = container._mutually_exclusive_groups - - -# Monkey patch in the fixed constructor -argparse._ArgumentGroup.__init__ = fixed__ArgumentGroup___init__ - -# -# End Hack -# - def prepare_arguments(parser): parser.description = """\ @@ -169,7 +141,7 @@ def prepare_arguments(parser): add('--no-summarize', '--no-summary', action='store_false', dest='summarize', help='Explicitly disable the end of build summary') add('--override-build-tool-check', action='store_true', default=False, - help='use to override failure due to using differnt build tools on the same workspace.') + help='use to override failure due to using different build tools on the same workspace.') # Deprecated args now handled by main catkin command add('--no-color', action='store_true', help=argparse.SUPPRESS) @@ -234,7 +206,7 @@ def print_build_env(context, package_name): if pkg.name == package_name: environ = dict(os.environ) loadenv(None, None, environ, pkg, context) - print(format_env_dict(environ)) + print(format_env_dict(environ, human_readable=sys.stdout.isatty())) return 0 print('[build] Error: Package `{}` not in workspace.'.format(package_name), file=sys.stderr) @@ -245,7 +217,7 @@ def main(opts): # Check for develdebug mode if opts.develdebug is not None: - os.environ['TROLLIUSDEBUG'] = opts.develdebug.lower() + os.environ['PYTHONASYNCIODEBUG'] = opts.develdebug.lower() logging.basicConfig(level=opts.develdebug.upper()) # Set color options @@ -260,15 +232,16 @@ def main(opts): # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) - # Suppress warnings since this won't necessaraly find all packages + # Suppress warnings since this won't necessarily find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package( search_start_path=getcwd(), ws_path=ws_path, warnings=[]) - except (InvalidPackage, RuntimeError): - this_package = None + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Handle context-based package building if opts.build_this: @@ -290,7 +263,14 @@ def main(opts): sys.exit(clr("[build] @!@{rf}Error:@| With --no-deps, you must specify packages to build.")) # Load the context - ctx = Context.load(opts.workspace, opts.profile, opts, append=True) + if opts.build_this or opts.start_with_this: + ctx = Context.load(opts.workspace, opts.profile, opts, append=True, strict=True) + else: + ctx = Context.load(opts.workspace, opts.profile, opts, append=True) + + # Handle no workspace + if ctx is None: + sys.exit(clr("[build] @!@{rf}Error:@| The current folder is not part of a catkin workspace.")) # Initialize the build configuration make_args, makeflags, cli_flags, jobserver = configure_make_args( diff --git a/catkin_tools/verbs/catkin_build/color.py b/catkin_tools/verbs/catkin_build/color.py index 60382031..0c2d1a9c 100644 --- a/catkin_tools/verbs/catkin_build/color.py +++ b/catkin_tools/verbs/catkin_build/color.py @@ -19,7 +19,7 @@ from catkin_tools.terminal_color import sanitize from catkin_tools.terminal_color import ColorMapper -# This map translates more human reable format strings into colorized versions +# This map translates more human readable format strings into colorized versions _color_translation_map = { # 'output': 'colorized_output' '': fmt('@!' + sanitize('') + '@|'), diff --git a/catkin_tools/verbs/catkin_clean/clean.py b/catkin_tools/verbs/catkin_clean/clean.py index f4073cd8..996a2332 100644 --- a/catkin_tools/verbs/catkin_clean/clean.py +++ b/catkin_tools/verbs/catkin_clean/clean.py @@ -18,13 +18,8 @@ import sys import time import traceback +from queue import Queue -try: - # Python3 - from queue import Queue -except ImportError: - # Python2 - from Queue import Queue try: from catkin_pkg.packages import find_packages @@ -50,6 +45,8 @@ def determine_packages_to_be_cleaned(context, include_dependents, packages): :param context: Workspace context :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` + :param include_dependents: Also clean dependents of the packages to be cleaned + :type include_dependents: bool :param packages: list of package names to be cleaned :type packages: list :returns: full list of package names to be cleaned @@ -60,6 +57,8 @@ def determine_packages_to_be_cleaned(context, include_dependents, packages): workspace_packages = find_packages(context.package_metadata_path(), exclude_subspaces=True, warnings=[]) # Order the packages by topology ordered_packages = topological_order_packages(workspace_packages) + # Set the packages in the workspace for the context + context.packages = ordered_packages # Create a dict of all packages in the workspace by name workspace_packages_by_name = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) @@ -131,7 +130,7 @@ def clean_packages( # It's a problem if there aren't any build types available if len(clean_job_creators) == 0: - sys.exit('Error: No build types availalbe. Please check your catkin_tools installation.') + sys.exit('Error: No build types available. Please check your catkin_tools installation.') # Determine the job parameters clean_job_kwargs = dict( diff --git a/catkin_tools/verbs/catkin_clean/cli.py b/catkin_tools/verbs/catkin_clean/cli.py index cbd41d6d..44c0b653 100644 --- a/catkin_tools/verbs/catkin_clean/cli.py +++ b/catkin_tools/verbs/catkin_clean/cli.py @@ -12,13 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -try: - raw_input -except NameError: - raw_input = input - import os import shutil import sys @@ -38,10 +31,10 @@ import catkin_tools.execution.job_server as job_server from catkin_tools.metadata import find_enclosing_workspace +from catkin_tools.metadata import get_metadata_root_path from catkin_tools.metadata import get_paths as get_metadata_paths from catkin_tools.metadata import get_profile_names from catkin_tools.metadata import update_metadata -from catkin_tools.metadata import METADATA_DIR_NAME from catkin_tools.terminal_color import ColorMapper @@ -59,7 +52,7 @@ def yes_no_loop(question): while True: - resp = str(raw_input(question + " [yN]: ")) + resp = str(input(question + " [yN]: ")) if resp.lower() in ['n', 'no'] or len(resp) == 0: return False elif resp.lower() in ['y', 'yes']: @@ -111,18 +104,19 @@ def prepare_arguments(parser): ' the workspace.') # Basic group - basic_group = parser.add_argument_group( + spaces_group = parser.add_argument_group( 'Spaces', 'Clean workspace subdirectories for the selected profile.') - add = basic_group.add_argument - add('-l', '--logs', action='store_true', default=False, - help='Remove the entire log space.') - add('-b', '--build', action='store_true', default=False, - help='Remove the entire build space.') - add('-d', '--devel', action='store_true', default=False, - help='Remove the entire devel space.') - add('-i', '--install', action='store_true', default=False, - help='Remove the entire install space.') + Context.setup_space_keys() + add = spaces_group.add_argument + for space, space_dict in Context.SPACES.items(): + if space == 'source': + continue + flags = [space_dict['short_flag']] if 'short_flag' in space_dict else [] + flags.append('--{}'.format(space_dict['default'])) + flags.append('--{}-space'.format(space)) + add(*flags, dest='spaces', action='append_const', const=space, + help='Remove the entire {} space.'.format(space)) # Packages group packages_group = parser.add_argument_group( @@ -175,20 +169,22 @@ def clean_profile(opts, profile): profile = ctx.profile # Check if the user wants to do something explicit - actions = [ - 'build', 'devel', 'install', 'logs', - 'packages', 'clean_this', 'orphans', - 'deinit', 'setup_files'] + actions = ['spaces', 'packages', 'clean_this', 'orphans', 'deinit', 'setup_files'] - logs_exists = os.path.exists(ctx.log_space_abs) - build_exists = os.path.exists(ctx.build_space_abs) - devel_exists = os.path.exists(ctx.devel_space_abs) + paths = {} # noqa + paths_exists = {} # noqa - install_path = ( + paths['install'] = ( os.path.join(ctx.destdir, ctx.install_space_abs.lstrip(os.sep)) if ctx.destdir else ctx.install_space_abs) - install_exists = os.path.exists(install_path) + paths_exists['install'] = os.path.exists(paths['install']) and os.path.isdir(paths['install']) + + for space in Context.SPACES.keys(): + if space in paths: + continue + paths[space] = getattr(ctx, '{}_space_abs'.format(space)) + paths_exists[space] = getattr(ctx, '{}_space_exists'.format(space))() # Default is to clean all products for this profile no_specific_action = not any([ @@ -198,21 +194,17 @@ def clean_profile(opts, profile): # Initialize action options if clean_all: - opts.logs = opts.build = opts.devel = opts.install = True + opts.spaces = [k for k in Context.SPACES.keys() if k != 'source'] - # Make sure the user intends to clena everything - spaces_to_clean = (opts.logs or opts.build or opts.devel or opts.install) + # Make sure the user intends to clean everything spaces_to_clean_msgs = [] - if spaces_to_clean and not (opts.yes or opts.dry_run): - if opts.logs and logs_exists: - spaces_to_clean_msgs.append(clr("[clean] Log Space: @{yf}{}").format(ctx.log_space_abs)) - if opts.build and build_exists: - spaces_to_clean_msgs.append(clr("[clean] Build Space: @{yf}{}").format(ctx.build_space_abs)) - if opts.devel and devel_exists: - spaces_to_clean_msgs.append(clr("[clean] Devel Space: @{yf}{}").format(ctx.devel_space_abs)) - if opts.install and install_exists: - spaces_to_clean_msgs.append(clr("[clean] Install Space: @{yf}{}").format(install_path)) + if opts.spaces and not (opts.yes or opts.dry_run): + for space in opts.spaces: + if getattr(ctx, '{}_space_exists'.format(space))(): + space_name = Context.SPACES[space]['space'] + space_abs = getattr(ctx, '{}_space_abs'.format(space)) + spaces_to_clean_msgs.append(clr("[clean] {:14} @{yf}{}").format(space_name + ':', space_abs)) if len(spaces_to_clean_msgs) == 0 and not opts.deinit: log("[clean] Nothing to be cleaned for profile: `{}`".format(profile)) @@ -239,50 +231,41 @@ def clean_profile(opts, profile): needs_force = False try: - # Remove all installspace files - if opts.install and install_exists: - log("[clean] Removing installspace: %s" % install_path) - if not opts.dry_run: - safe_rmtree(install_path, ctx.workspace, opts.force) - - # Remove all develspace files - if opts.devel: - if devel_exists: - log("[clean] Removing develspace: %s" % ctx.devel_space_abs) - if not opts.dry_run: - safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force) - # Clear the cached metadata from the last build run - _, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build') - if os.path.exists(build_metadata_file): - os.unlink(build_metadata_file) - # Clear the cached packages data, if it exists - packages_metadata_path = ctx.package_metadata_path() - if os.path.exists(packages_metadata_path): - safe_rmtree(packages_metadata_path, ctx.workspace, opts.force) - - # Remove all buildspace files - if opts.build and build_exists: - log("[clean] Removing buildspace: %s" % ctx.build_space_abs) - if not opts.dry_run: - safe_rmtree(ctx.build_space_abs, ctx.workspace, opts.force) + for space in opts.spaces: + if space == 'devel': + # Remove all develspace files + if paths_exists['devel']: + log("[clean] Removing {}: {}".format(Context.SPACES['devel']['space'], ctx.devel_space_abs)) + if not opts.dry_run: + safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force) + # Clear the cached metadata from the last build run + _, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build') + if os.path.exists(build_metadata_file): + os.unlink(build_metadata_file) + # Clear the cached packages data, if it exists + packages_metadata_path = ctx.package_metadata_path() + if os.path.exists(packages_metadata_path): + safe_rmtree(packages_metadata_path, ctx.workspace, opts.force) + + else: + if paths_exists[space]: + space_name = Context.SPACES[space]['space'] + space_path = paths[space] + log("[clean] Removing {}: {}".format(space_name, space_path)) + if not opts.dry_run: + safe_rmtree(space_path, ctx.workspace, opts.force) # Setup file removal if opts.setup_files: - if devel_exists: - log("[clean] Removing setup files from develspace: %s" % ctx.devel_space_abs) + if paths_exists['devel']: + log("[clean] Removing setup files from {}: {}".format(Context.SPACES['devel']['space'], paths['devel'])) opts.packages.append('catkin') opts.packages.append('catkin_tools_prebuild') else: - log("[clean] No develspace exists, no setup files to clean.") - - # Clean log files - if opts.logs and logs_exists: - log("[clean] Removing log space: {}".format(ctx.log_space_abs)) - if not opts.dry_run: - safe_rmtree(ctx.log_space_abs, ctx.workspace, opts.force) + log("[clean] No {} exists, no setup files to clean.".format(Context.SPACES['devel']['space'])) # Find orphaned packages - if ctx.link_devel and not any([opts.build, opts.devel]): + if ctx.link_devel or ctx.isolate_devel and not ('devel' in opts.spaces or 'build' in opts.spaces): if opts.orphans: if os.path.exists(ctx.build_space_abs): log("[clean] Determining orphaned packages...") @@ -306,22 +289,23 @@ def clean_profile(opts, profile): else: log("[clean] No orphans in the workspace.") else: - log("[clean] No buildspace exists, no potential for orphans.") + log("[clean] No {} exists, no potential for orphans.".format(Context.SPACES['build']['space'])) # Remove specific packages if len(opts.packages) > 0 or opts.clean_this: # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) - # Suppress warnings since this won't necessaraly find all packages + # Suppress warnings since this won't necessarily find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package( search_start_path=getcwd(), ws_path=ws_path, warnings=[]) - except (InvalidPackage, RuntimeError): - this_package = None + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Handle context-based package cleaning if opts.clean_this: @@ -344,9 +328,9 @@ def clean_profile(opts, profile): return False elif opts.orphans or len(opts.packages) > 0 or opts.clean_this: - log("[clean] Error: Individual packages can only be cleaned from " - "workspaces with symbolically-linked develspaces (`catkin " - "config --link-devel`).") + log("[clean] Error: Individual packages cannot be cleaned from " + "workspaces with merged develspaces, use a symbolically-linked " + "or isolated develspace instead.") except: # noqa: E722 # Silencing E722 here since we immediately re-raise the exception. @@ -367,25 +351,27 @@ def clean_profile(opts, profile): def main(opts): # Check for exclusivity full_options = opts.deinit - space_options = opts.logs or opts.build or opts.devel or opts.install package_options = len(opts.packages) > 0 or opts.orphans or opts.clean_this advanced_options = opts.setup_files + if opts.spaces is None: + opts.spaces = [] + if full_options: - if space_options or package_options or advanced_options: + if opts.spaces or package_options or advanced_options: log("[clean] Error: Using `--deinit` will remove all spaces, so" " additional partial cleaning options will be ignored.") - elif space_options: + elif opts.spaces: if package_options: log("[clean] Error: Package arguments are not allowed with space" - " arguments (--build, --devel, --install, --logs). See usage.") + " arguments. See usage.") elif advanced_options: log("[clean] Error: Advanced arguments are not allowed with space" - " arguments (--build, --devel, --install, --logs). See usage.") + " arguments. See usage.") # Check for all profiles option if opts.all_profiles: - profiles = get_profile_names(opts.workspace or os.getcwd()) + profiles = get_profile_names(opts.workspace or find_enclosing_workspace(getcwd())) else: profiles = [opts.profile] @@ -419,7 +405,7 @@ def main(opts): # Nuke .catkin_tools if opts.deinit: ctx = Context.load(opts.workspace, profile, opts, strict=True, load_env=False) - metadata_dir = os.path.join(ctx.workspace, METADATA_DIR_NAME) + metadata_dir = get_metadata_root_path(ctx.workspace) log("[clean] Deinitializing workspace by removing catkin_tools config: %s" % metadata_dir) if not opts.dry_run: safe_rmtree(metadata_dir, ctx.workspace, opts.force) diff --git a/catkin_tools/verbs/catkin_clean/color.py b/catkin_tools/verbs/catkin_clean/color.py index 3ab39dbc..d00ffa46 100644 --- a/catkin_tools/verbs/catkin_clean/color.py +++ b/catkin_tools/verbs/catkin_clean/color.py @@ -19,7 +19,7 @@ from catkin_tools.terminal_color import sanitize from catkin_tools.terminal_color import ColorMapper -# This map translates more human reable format strings into colorized versions +# This map translates more human readable format strings into colorized versions _color_translation_map = { # 'output': 'colorized_output' '': fmt('@!' + sanitize('') + '@|'), diff --git a/catkin_tools/verbs/catkin_config/cli.py b/catkin_tools/verbs/catkin_config/cli.py index 5b0889ba..68305f1a 100644 --- a/catkin_tools/verbs/catkin_config/cli.py +++ b/catkin_tools/verbs/catkin_config/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.argument_parsing import add_cmake_and_make_and_catkin_make_args @@ -66,7 +64,7 @@ def prepare_arguments(parser): add('--maintainers', metavar=('NAME', 'EMAIL'), dest='maintainers', nargs='+', required=False, type=str, default=None, help='Set the default maintainers of created packages') - add('--licenses', metavar=('LICENSE'), dest='licenses', nargs='+', required=False, type=str, default=None, + add('--licenses', metavar='LICENSE', dest='licenses', nargs='+', required=False, type=str, default=None, help='Set the default licenses of created packages') lists_group = parser.add_argument_group( diff --git a/catkin_tools/verbs/catkin_create/cli.py b/catkin_tools/verbs/catkin_create/cli.py index 483de8b7..e35d09c9 100644 --- a/catkin_tools/verbs/catkin_create/cli.py +++ b/catkin_tools/verbs/catkin_create/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.argument_parsing import add_context_args @@ -35,14 +33,14 @@ def prepare_arguments(parser): parser_pkg.description = ( "Create a new Catkin package. Note that while the " "default options used by this command are sufficient for prototyping and " - "local usage, it is important that any publically-available packages have " + "local usage, it is important that any publicly-available packages have " "a valid license and a valid maintainer e-mail address.") add = parser_pkg.add_argument add('name', metavar='PKG_NAME', nargs='+', help='The name of one or more packages to create. This name should be ' - 'completely lower-case with individual words separated by undercores.') + 'completely lower-case with individual words separated by underscores.') add('-p', '--path', action='store', default=os.getcwd(), help='The path into which the package should be generated.') @@ -57,7 +55,7 @@ def prepare_arguments(parser): # default='catkin', # help='The buildtool to use to build the package. (default: catkin)') - rosdistro_name = os.environ['ROS_DISTRO'] if 'ROS_DISTRO' in os.environ else None + rosdistro_name = os.environ.get('ROS_DISTRO', None) add('--rosdistro', required=rosdistro_name is None, default=rosdistro_name, help='The ROS distro (default: environment variable ROS_DISTRO if defined)') diff --git a/catkin_tools/verbs/catkin_env/cli.py b/catkin_tools/verbs/catkin_env/cli.py index 08992ea5..402c78c5 100644 --- a/catkin_tools/verbs/catkin_env/cli.py +++ b/catkin_tools/verbs/catkin_env/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import re import sys @@ -32,7 +30,7 @@ def prepare_arguments(parser): help='Start with an empty environment.') add('-s', '--stdin', default=False, action='store_true', help='Read environment variable definitions from stdin. ' - 'Variables should be given in NAME=VALUE format. ') + 'Variables should be given in NAME=VALUE format, separated by null-bytes.') add('envs_', metavar='NAME=VALUE', nargs='*', type=str, default=[], help='Explicitly set environment variables for the subcommand. ' @@ -54,7 +52,7 @@ def argument_preprocessor(args): :param args: system arguments from which special arguments need to be extracted :type args: list - :returns: a tuple contianing a list of the arguments which can be handled + :returns: a tuple containing a list of the arguments which can be handled by argparse and a dict of the extra arguments which this function has extracted :rtype: tuple @@ -104,7 +102,7 @@ def main(opts): # Update environment from stdin if opts.stdin: - input_env_str = sys.stdin.read() + input_env_str = sys.stdin.read().strip() environ.update(parse_env_str(input_env_str.encode())) # Finally, update with explicit vars diff --git a/catkin_tools/verbs/catkin_init/cli.py b/catkin_tools/verbs/catkin_init/cli.py index b61146a7..db3878e8 100644 --- a/catkin_tools/verbs/catkin_init/cli.py +++ b/catkin_tools/verbs/catkin_init/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os from catkin_tools.argument_parsing import add_workspace_arg @@ -42,7 +40,7 @@ def main(opts): # Initialize the workspace if necessary if ctx: - print('Catkin workspace `%s` is already initialized. No action taken.' % (ctx.workspace)) + print('Catkin workspace `%s` is already initialized. No action taken.' % ctx.workspace) else: print('Initializing catkin workspace in `%s`.' % (opts.workspace or os.getcwd())) # initialize the workspace diff --git a/catkin_tools/verbs/catkin_list/__init__.py b/catkin_tools/verbs/catkin_list/__init__.py index 74d360ed..98cff3dc 100644 --- a/catkin_tools/verbs/catkin_list/__init__.py +++ b/catkin_tools/verbs/catkin_list/__init__.py @@ -18,7 +18,7 @@ # This describes this command to the loader description = dict( verb='list', - description="Lists catkin packages in the workspace or other arbitray folders.", + description="Lists catkin packages in the workspace or other arbitrary folders.", main=main, prepare_arguments=prepare_arguments, ) diff --git a/catkin_tools/verbs/catkin_list/cli.py b/catkin_tools/verbs/catkin_list/cli.py index dac82ace..b20e5301 100644 --- a/catkin_tools/verbs/catkin_list/cli.py +++ b/catkin_tools/verbs/catkin_list/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import sys from catkin_tools.argument_parsing import add_context_args @@ -77,8 +75,7 @@ def main(opts): ctx = Context.load(opts.workspace, opts.profile, load_env=False) if not ctx: - print(clr("@{rf}ERROR: Could not determine workspace.@|"), file=sys.stderr) - sys.exit(1) + sys.exit(clr("@{rf}ERROR: Could not determine workspace.@|")) if opts.directory: folders = opts.directory @@ -94,9 +91,8 @@ def main(opts): packages = find_packages(folder, warnings=warnings) ordered_packages = topological_order_packages(packages) if ordered_packages and ordered_packages[-1][0] is None: - print(clr("@{rf}ERROR: Circular dependency within packages:@| " - + ordered_packages[-1][1]), file=sys.stderr) - sys.exit(1) + sys.exit(clr("@{rf}ERROR: Circular dependency within packages:@| " + + ordered_packages[-1][1])) packages_by_name = {pkg.name: (pth, pkg) for pth, pkg in ordered_packages} if opts.depends_on or opts.rdepends_on: @@ -141,8 +137,8 @@ def main(opts): build_deps = [p for dp, p in get_recursive_build_depends_in_workspace(pkg, ordered_packages)] run_deps = [p for dp, p in get_recursive_run_depends_in_workspace([pkg], ordered_packages)] else: - build_deps = pkg.build_depends - run_deps = pkg.run_depends + build_deps = [dep for dep in pkg.build_depends if dep.evaluated_condition] + run_deps = [dep for dep in pkg.run_depends if dep.evaluated_condition] if opts.deps or opts.rdeps: if len(build_deps) > 0: @@ -154,9 +150,8 @@ def main(opts): for dep in run_deps: print(clr(' @{pf}-@| %s' % dep.name)) except InvalidPackage as ex: - message = '\n'.join(ex.args) - print(clr("@{rf}Error:@| The directory %s contains an invalid package." - " See below for details:\n\n%s" % (folder, message))) + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) # Print out warnings if not opts.quiet: diff --git a/catkin_tools/verbs/catkin_locate/cli.py b/catkin_tools/verbs/catkin_locate/cli.py index 7a676cc9..091b3c81 100644 --- a/catkin_tools/verbs/catkin_locate/cli.py +++ b/catkin_tools/verbs/catkin_locate/cli.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import sys +from catkin_pkg.package import InvalidPackage + from catkin_tools.common import find_enclosing_package from catkin_tools.common import getcwd @@ -45,20 +45,18 @@ def prepare_arguments(parser): help="Suppress warning output.") # Path options - dir_group = parser.add_argument_group( + spaces_group = parser.add_argument_group( 'Sub-Space Options', 'Get the absolute path to one of the following locations in the given ' 'workspace with the given profile.') - dir_group_mut = dir_group.add_mutually_exclusive_group() - add = dir_group_mut.add_argument - add('-s', '--src', dest='space', action='store_const', const='src', - help="Get the path to the source space.") - add('-b', '--build', dest='space', action='store_const', const='build', - help="Get the path to the build space.") - add('-d', '--devel', dest='space', action='store_const', const='devel', - help="Get the path to the devel space.") - add('-i', '--install', dest='space', action='store_const', const='install', - help="Get the path to the install space.") + Context.setup_space_keys() + add = spaces_group.add_mutually_exclusive_group().add_argument + for space, space_dict in Context.SPACES.items(): + flags = [space_dict['short_flag']] if 'short_flag' in space_dict else [] + flags.append('--{}'.format(space_dict['default'])) + flags.append('--{}-space'.format(space)) + add(*flags, dest='space', action='store_const', const=space, + help='Get the path to the {} space.'.format(space)) pkg_group = parser.add_argument_group( 'Package Directories', @@ -120,48 +118,41 @@ def main(opts): path = None if opts.space: - # Get the subspace - if opts.space == 'src': - path = ctx.source_space_abs - elif opts.space == 'build': - path = ctx.build_space_abs - elif opts.space == 'devel': - path = ctx.devel_space_abs - elif opts.space == 'install': - path = ctx.install_space_abs + path = getattr(ctx, "{}_space_abs".format(opts.space)) package = None if opts.package or opts.this: if opts.this: - package = find_enclosing_package( - search_start_path=getcwd(), - ws_path=ctx.workspace, - warnings=[]) - if package is None: - print(clr("@{rf}ERROR: Passed '--this' but could not determine enclosing package. " - "Is '%s' in a package in '%s' workspace?@|" % (getcwd(), ctx.workspace)), file=sys.stderr) - sys.exit(2) + try: + package = find_enclosing_package( + search_start_path=getcwd(), + ws_path=ctx.workspace, + warnings=[]) + if package is None: + sys.exit(clr("@{rf}ERROR: Passed '--this' but could not determine enclosing package. " + "Is '%s' in a package in '%s' workspace?@|" % (getcwd(), ctx.workspace))) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) else: package = opts.package # Get the path to the given package path = path or ctx.source_space_abs - if opts.space == 'build': - path = os.path.join(path, package) - elif opts.space in ['devel', 'install']: - path = os.path.join(path, 'share', package) - else: + if not opts.space or opts.space == 'source': try: packages = find_packages(path, warnings=[]) catkin_package = [pkg_path for pkg_path, p in packages.items() if p.name == package] if catkin_package: path = os.path.join(path, catkin_package[0]) else: - print(clr("@{rf}ERROR: Could not locate a package named '%s' in path '%s'@|" % - (package, path)), file=sys.stderr) - sys.exit(2) + sys.exit(clr("@{rf}ERROR: Could not locate a package named '%s' in path '%s'@|" % + (package, path))) except RuntimeError as e: - print(clr('@{rf}ERROR: %s@|' % str(e)), file=sys.stderr) - sys.exit(1) + sys.exit(clr('@{rf}ERROR: %s@|' % str(e))) + elif opts.space in ['devel', 'install']: + path = os.path.join(path, 'share', package) + else: + path = os.path.join(path, package) if not opts.space and package is None: # Get the path to the workspace root @@ -169,8 +160,7 @@ def main(opts): # Check if the path exists if opts.existing_only and not os.path.exists(path): - print(clr("@{rf}ERROR: Requested path '%s' does not exist.@|" % path), file=sys.stderr) - sys.exit(1) + sys.exit(clr("@{rf}ERROR: Requested path '%s' does not exist.@|" % path)) # Make the path relative if desired if opts.relative: diff --git a/catkin_tools/verbs/catkin_profile/cli.py b/catkin_tools/verbs/catkin_profile/cli.py index 79b14d6d..180157be 100644 --- a/catkin_tools/verbs/catkin_profile/cli.py +++ b/catkin_tools/verbs/catkin_profile/cli.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from catkin_tools.context import Context from catkin_tools.metadata import get_active_profile @@ -64,6 +62,8 @@ def prepare_arguments(parser): help="Copy the settings from an existing profile. (default: None)") add('--copy-active', action='store_true', default=False, help="Copy the settings from the active profile.") + add('--extend', metavar='PARENT_PROFILE', type=str, + help="Extend another profile") add = parser_rename.add_argument add('current_name', type=str, @@ -128,11 +128,11 @@ def main(opts): elif opts.subcommand == 'add': if opts.name in profiles: if opts.force: - print(clr('[profile] @{yf}Warning:@| Overwriting existing profile named @{cf}%s@|' % (opts.name))) + print(clr('[profile] @{yf}Warning:@| Overwriting existing profile named @{cf}%s@|' % opts.name)) else: print(clr('catkin profile: error: A profile named ' '@{cf}%s@| already exists. Use `--force` to ' - 'overwrite.' % (opts.name))) + 'overwrite.' % opts.name)) return 1 if opts.copy_active: ctx.profile = opts.name @@ -148,10 +148,16 @@ def main(opts): 'based on profile @{cf}%s@|' % (opts.name, opts.copy))) else: print(clr('[profile] @{rf}A profile with this name does not exist: %s@|' % opts.copy)) + elif opts.extend: + if opts.extend in profiles: + new_ctx = Context(workspace=ctx.workspace, profile=opts.name, extends=opts.extend) + Context.save(new_ctx) + print(clr('[profile] Created a new profile named @{cf}%s@| ' + 'extending profile @{cf}%s@|' % (opts.name, opts.extend))) else: new_ctx = Context(workspace=ctx.workspace, profile=opts.name) Context.save(new_ctx) - print(clr('[profile] Created a new profile named @{cf}%s@| with default settings.' % (opts.name))) + print(clr('[profile] Created a new profile named @{cf}%s@| with default settings.' % opts.name)) profiles = get_profile_names(ctx.workspace) active_profile = get_active_profile(ctx.workspace) @@ -177,11 +183,11 @@ def main(opts): if opts.new_name in profiles: if opts.force: print(clr('[profile] @{yf}Warning:@| Overwriting ' - 'existing profile named @{cf}%s@|' % (opts.new_name))) + 'existing profile named @{cf}%s@|' % opts.new_name)) else: print(clr('catkin profile: error: A profile named ' '@{cf}%s@| already exists. Use `--force` to ' - 'overwrite.' % (opts.new_name))) + 'overwrite.' % opts.new_name)) return 1 ctx.profile = opts.new_name Context.save(ctx) diff --git a/catkin_tools/verbs/catkin_test/__init__.py b/catkin_tools/verbs/catkin_test/__init__.py new file mode 100644 index 00000000..62dbed3d --- /dev/null +++ b/catkin_tools/verbs/catkin_test/__init__.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from catkin_tools.argument_parsing import argument_preprocessor + +from .cli import main +from .cli import prepare_arguments + +# This describes this command to the loader +description = dict( + verb='test', + description="Tests a catkin workspace.", + main=main, + prepare_arguments=prepare_arguments, + argument_preprocessor=argument_preprocessor, +) diff --git a/catkin_tools/verbs/catkin_test/cli.py b/catkin_tools/verbs/catkin_test/cli.py new file mode 100644 index 00000000..a9554439 --- /dev/null +++ b/catkin_tools/verbs/catkin_test/cli.py @@ -0,0 +1,179 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import sys + +from catkin_pkg.package import InvalidPackage + +from catkin_tools.argument_parsing import add_context_args +from catkin_tools.argument_parsing import configure_make_args +from catkin_tools.common import is_tty +from catkin_tools.common import getcwd +from catkin_tools.common import find_enclosing_package + +from catkin_tools.context import Context +from catkin_tools.context import clr + +from catkin_tools.metadata import find_enclosing_workspace + +from catkin_tools.resultspace import load_resultspace_environment + +from catkin_tools.terminal_color import set_color + +from .test import test_workspace + + +def prepare_arguments(parser): + parser.description = """\ +Test one or more packages in a catkin workspace. +This invokes `make run_tests` or `make test` for either all or the specified +packages in a catkin workspace.\ +""" + + # Workspace / profile args + add_context_args(parser) + # Sub-commands + # What packages to test + pkg_group = parser.add_argument_group('Packages', 'Control which packages get tested.') + add = pkg_group.add_argument + add('packages', metavar='PKGNAME', nargs='*', + help='Workspace packages to test. If no packages are given, then all the packages are tested.') + add('--this', dest='build_this', action='store_true', default=False, + help='Test the package containing the current working directory.') + add('--continue-on-failure', '-c', action='store_true', default=False, + help='Continue testing packages even if the tests for other requested packages fail.') + + config_group = parser.add_argument_group('Config', 'Parameters for the underlying build system.') + add = config_group.add_argument + add('-p', '--parallel-packages', metavar='PACKAGE_JOBS', dest='parallel_jobs', default=None, type=int, + help='Maximum number of packages allowed to be built in parallel (default is cpu count)') + add('-t', '--test-target', metavar='TARGET', default=None, type=str, + help='Make target to run for tests (default is "run_tests" for catkin and "test" for cmake)') + add('--catkin-test-target', metavar='TARGET', default=None, type=str, + help='Make target to run for tests for catkin packages, overwrites --test-target (default is "run_tests")') + add('--make-args', metavar='ARG', dest='make_args', nargs='+', required=False, type=str, default=None, + help='Arbitrary arguments which are passed to make. ' + 'It collects all of following arguments until a "--" is read.') + + interface_group = parser.add_argument_group('Interface', 'The behavior of the command-line interface.') + add = interface_group.add_argument + add('--verbose', '-v', action='store_true', default=False, + help='Print output from commands in ordered blocks once the command finishes.') + add('--interleave-output', '-i', action='store_true', default=False, + help='Prevents ordering of command output when multiple commands are running at the same time.') + add('--summarize', '--summary', '-s', action='store_true', default=None, + help='Adds a summary to the end of the log') + add('--no-status', action='store_true', default=False, + help='Suppresses status line, useful in situations where carriage return is not properly supported.') + + def status_rate_type(rate): + rate = float(rate) + if rate < 0: + raise argparse.ArgumentTypeError("must be greater than or equal to zero.") + return rate + + add('--limit-status-rate', '--status-rate', type=status_rate_type, default=10.0, + help='Limit the update rate of the status bar to this frequency. Zero means unlimited. ' + 'Must be positive, default is 10 Hz.') + add('--no-notify', action='store_true', default=False, + help='Suppresses system pop-up notification.') + + # Deprecated arguments + # colors are handled by the main catkin command + add('--no-color', action='store_true', help=argparse.SUPPRESS) + add('--force-color', action='store_true', help=argparse.SUPPRESS) + # no-deps became the default + add('--no-deps', action='store_true', help=argparse.SUPPRESS) + + return parser + + +def main(opts): + # Set color options + opts.force_color = os.environ.get('CATKIN_TOOLS_FORCE_COLOR', opts.force_color) + if (opts.force_color or is_tty(sys.stdout)) and not opts.no_color: + set_color(True) + else: + set_color(False) + + # Context-aware args + if opts.build_this: + # Determine the enclosing package + try: + ws_path = find_enclosing_workspace(getcwd()) + # Suppress warnings since this won't necessarily find all packages + # in the workspace (it stops when it finds one package), and + # relying on it for warnings could mislead people. + this_package = find_enclosing_package( + search_start_path=getcwd(), + ws_path=ws_path, + warnings=[]) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) + + # Handle context-based package building + if this_package: + opts.packages += [this_package] + else: + sys.exit( + "[build] Error: In order to use --this, the current directory must be part of a catkin package.") + + # Load the context + ctx = Context.load(opts.workspace, opts.profile, opts, append=True) + + # Load the environment of the workspace to extend + if ctx.extend_path is not None: + try: + load_resultspace_environment(ctx.extend_path) + except IOError as exc: + sys.exit(clr("[build] @!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % + (ctx.extend_path, exc.message))) + + # Extract make arguments + make_args, _, _, _ = configure_make_args(ctx.make_args, ctx.jobs_args, ctx.use_internal_make_jobserver) + ctx.make_args = make_args + + # Get parallel toplevel jobs + try: + parallel_jobs = int(opts.parallel_jobs) + except TypeError: + parallel_jobs = None + + # Set VERBOSE environment variable + if opts.verbose and 'VERBOSE' not in os.environ: + os.environ['VERBOSE'] = '1' + + # Get test targets + catkin_test_target = 'run_tests' + cmake_test_target = 'test' + if opts.test_target: + catkin_test_target = opts.test_target + cmake_test_target = opts.test_target + if opts.catkin_test_target: + catkin_test_target = opts.catkin_test_target + + return test_workspace( + ctx, + packages=opts.packages, + n_jobs=parallel_jobs, + quiet=not opts.verbose, + interleave_output=opts.interleave_output, + no_status=opts.no_status, + limit_status_rate=opts.limit_status_rate, + no_notify=opts.no_notify, + continue_on_failure=opts.continue_on_failure, + summarize_build=opts.summarize, + catkin_test_target=catkin_test_target, + cmake_test_target=cmake_test_target, + ) diff --git a/catkin_tools/verbs/catkin_test/test.py b/catkin_tools/verbs/catkin_test/test.py new file mode 100644 index 00000000..87a3f006 --- /dev/null +++ b/catkin_tools/verbs/catkin_test/test.py @@ -0,0 +1,238 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import traceback +import time +from queue import Queue + +import pkg_resources +from catkin_pkg.package import InvalidPackage +from catkin_pkg.packages import find_packages +from catkin_pkg.topological_order import topological_order_packages + +from catkin_tools.common import clr, wide_log, expand_glob_package +from catkin_tools.execution import job_server +from catkin_tools.execution.controllers import ConsoleStatusController +from catkin_tools.execution.executor import run_until_complete, execute_jobs + + +def test_workspace( + context, + packages=None, + n_jobs=None, + quiet=False, + interleave_output=False, + no_status=False, + limit_status_rate=10.0, + no_notify=False, + continue_on_failure=False, + summarize_build=False, + catkin_test_target='run_tests', + cmake_test_target='test', +): + """Tests a catkin workspace + + :param context: context in which to test the catkin workspace + :type context: :py:class:`catkin_tools.context.Context` + :param packages: list of packages to test + :type packages: list + :param n_jobs: number of parallel package test jobs + :type n_jobs: int + :param quiet: suppresses verbose build or test information + :type quiet: bool + :param interleave_output: prints the output of commands as they are received + :type interleave_output: bool + :param no_status: suppresses the bottom status line + :type no_status: bool + :param limit_status_rate: rate to which status updates are limited; the default 0, places no limit. + :type limit_status_rate: float + :param no_notify: suppresses system notifications + :type no_notify: bool + :param continue_on_failure: do not stop testing other packages on error + :type continue_on_failure: bool + :param summarize_build: summarizes the build at the end + :type summarize_build: bool + :param catkin_test_target: make target for tests in catkin packages + :type catkin_test_target: str + :param cmake_test_target: make target for tests in cmake packages + :type cmake_test_target: str + """ + pre_start_time = time.time() + + # Assert that the limit_status_rate is valid + if limit_status_rate < 0: + sys.exit("[test] @!@{rf}Error:@| The value of --limit-status-rate must be greater than or equal to zero.") + + # Get all the packages in the context source space + # Suppress warnings since this is a utility function + try: + workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) + except InvalidPackage as ex: + sys.exit(clr("@{rf}Error:@| The file %s is an invalid package.xml file." + " See below for details:\n\n%s" % (ex.package_path, ex.msg))) + + # Get all build type plugins + test_job_creators = { + ep.name: ep.load()['create_test_job'] + for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs') + } + + # It's a problem if there aren't any build types available + if len(test_job_creators) == 0: + sys.exit('Error: No build types available. Please check your catkin_tools installation.') + + # Get list of packages to test + ordered_packages = topological_order_packages(workspace_packages) + + # Check if topological_order_packages determined any circular dependencies, if so print an error and fail. + # If this is the case, the last entry of ordered packages is a tuple that starts with nil. + if ordered_packages and ordered_packages[-1][0] is None: + guilty_packages = ", ".join(ordered_packages[-1][1:]) + sys.exit("[test] Circular dependency detected in the following packages: {}".format(guilty_packages)) + + workspace_packages = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) + packages_to_test = [] + if packages: + for package in packages: + if package not in workspace_packages: + # Try whether package is a pattern and matches + glob_packages = expand_glob_package(package, workspace_packages) + if len(glob_packages) > 0: + packages.extend(glob_packages) + else: + sys.exit("[test] Given packages '{}' is not in the workspace " + "and pattern does not match any package".format(package)) + for pkg_path, package in ordered_packages: + if package.name in packages: + packages_to_test.append((pkg_path, package)) + else: + # Only use whitelist when no other packages are specified + if len(context.whitelist) > 0: + # Expand glob patterns in whitelist + whitelist = [] + for whitelisted_package in context.whitelist: + whitelist.extend(expand_glob_package(whitelisted_package, workspace_packages)) + packages_to_test = [p for p in ordered_packages if (p[1].name in whitelist)] + else: + packages_to_test = ordered_packages + + # Filter packages on blacklist + if len(context.blacklist) > 0: + # Expand glob patterns in blacklist + blacklist = [] + for blacklisted_package in context.blacklist: + blacklist.extend(expand_glob_package(blacklisted_package, workspace_packages)) + # Apply blacklist to packages and dependencies + packages_to_test = [ + (path, pkg) for path, pkg in packages_to_test + if (pkg.name not in blacklist or pkg.name in packages)] + + # Check if all packages to test are already built + built_packages = set([ + pkg.name for (path, pkg) in + find_packages(context.package_metadata_path(), warnings=[]).items()]) + + packages_to_test_names = set(pkg.name for path, pkg in packages_to_test) + if not built_packages.issuperset(packages_to_test_names): + wide_log(clr("@{rf}Error: Packages have to be built before they can be tested.@|")) + wide_log(clr("The following requested packages are not built yet:")) + for package_name in packages_to_test_names.difference(built_packages): + wide_log(' - ' + package_name) + sys.exit(1) + + # Construct jobs + jobs = [] + for pkg_path, pkg in packages_to_test: + # Determine the job parameters + test_job_kwargs = dict( + context=context, + package=pkg, + package_path=pkg_path, + verbose=not quiet) + + # Create the job based on the build type + build_type = pkg.get_build_type() + + if build_type == 'catkin': + test_job_kwargs['test_target'] = catkin_test_target + elif build_type == 'cmake': + test_job_kwargs['test_target'] = cmake_test_target + + if build_type in test_job_creators: + jobs.append(test_job_creators[build_type](**test_job_kwargs)) + + # Queue for communicating status + event_queue = Queue() + + # Initialize job server + job_server.initialize( + max_jobs=n_jobs, + max_load=None, + gnu_make_enabled=context.use_internal_make_jobserver, + ) + + try: + # Spin up status output thread + status_thread = ConsoleStatusController( + 'test', + ['package', 'packages'], + jobs, + n_jobs, + [pkg.name for path, pkg in packages_to_test], + [p for p in context.whitelist], + [p for p in context.blacklist], + event_queue, + show_notifications=not no_notify, + show_active_status=not no_status, + show_buffered_stdout=not interleave_output, + show_buffered_stderr=not interleave_output, + show_live_stdout=interleave_output, + show_live_stderr=interleave_output, + show_full_summary=summarize_build, + show_stage_events=not quiet, + pre_start_time=pre_start_time, + active_status_rate=limit_status_rate, + ) + + status_thread.start() + + locks = {} + + # Block while running N jobs asynchronously + try: + all_succeeded = run_until_complete(execute_jobs( + 'test', + jobs, + locks, + event_queue, + context.log_space_abs, + max_toplevel_jobs=n_jobs, + continue_on_failure=continue_on_failure, + continue_without_deps=False)) + except Exception: + status_thread.keep_running = False + all_succeeded = False + status_thread.join(1.0) + wide_log(str(traceback.format_exc())) + + status_thread.join(1.0) + + if all_succeeded: + return 0 + else: + return 1 + + except KeyboardInterrupt: + wide_log("[test] Interrupted by user!") + event_queue.put(None) + + return 130 diff --git a/completion/_catkin b/completion/_catkin index aa35d240..146b6204 100644 --- a/completion/_catkin +++ b/completion/_catkin @@ -205,6 +205,7 @@ _catkin_verbs_complete() { 'list:List workspace components' 'locate:Locate workspace components' 'profile:Switch between configurations' + 'test:Test packages' ) _describe -t verbs 'catkin verb' verbs_array -V verbs && return 0 } @@ -468,6 +469,28 @@ _catkin_locate_complete() { } +_catkin_test_complete() { + local test_opts + + test_opts=( + {-h,--help}'[Show usage help]'\ + {-w,--workspace}'[The workspace to test]:workspace:_files'\ + '--profile[Which configuration profile to use]:profile:_catkin_get_profiles'\ + {-v,--verbose}'[Print all output from test commands]'\ + {-i,--interleave-output}"[Print output from test commands as it's generated]"\ + '--this[Test the current package]'\ + {-c,--continue-on-failure}"[Continue testing after failed tests]"\ + {-p,--parallel-packages}"[Number of packages to build in parallel]:number"\ + {-t,--test-target}"[make target to run for tests]:target"\ + '--catkin-test-target[make target to run for tests in catkin packages]:target'\ + '--no-status[Suppress status line]'\ + {-s,--summarize}"[Add a summary at the end]"\ + '*:package:_catkin_get_packages') + + _arguments -C $test_opts && return 0 +} + + local curcontext="$curcontext" state line ret ret=1 @@ -580,6 +603,8 @@ case "$state" in ;; (profile) _catkin_profile_complete && ret=0 ;; + (test) _catkin_test_complete && ret=0 + ;; esac ret=0 ;; diff --git a/completion/catkin_tools-completion.bash b/completion/catkin.bash similarity index 95% rename from completion/catkin_tools-completion.bash rename to completion/catkin.bash index 09537e87..297379dc 100644 --- a/completion/catkin_tools-completion.bash +++ b/completion/catkin.bash @@ -55,7 +55,7 @@ _catkin() _init_completion || return # this handles default completion (variables, redirection) # complete to the following verbs - local catkin_verbs="build cd clean config create init list profile run_tests" + local catkin_verbs="build clean config create init list profile test" # filter for long options (from bash_completion) local OPTS_FILTER='s/.*\(--[-A-Za-z0-9]\{1,\}=\{0,1\}\).*/\1/p' @@ -136,9 +136,9 @@ _catkin() local catkin_list_opts=$(catkin list --help 2>&1 | sed -ne $OPTS_FILTER | sort -u) COMPREPLY=($(compgen -W "${catkin_list_opts}" -- ${cur})) ;; - run_tests) + test) if [[ ${cur} == -* ]]; then - local catkin_run_tests_opts=$(catkin run_tests --help 2>&1 | sed -ne $OPTS_FILTER | sort -u) + local catkin_run_tests_opts=$(catkin test --help 2>&1 | sed -ne $OPTS_FILTER | sort -u) COMPREPLY=($(compgen -W "${catkin_run_tests_opts}" -- ${cur})) else COMPREPLY=($(compgen -W "$(_catkin_pkgs)" -- ${cur})) diff --git a/docs/cheat_sheet.rst b/docs/cheat_sheet.rst index aef7e846..6d054015 100644 --- a/docs/cheat_sheet.rst +++ b/docs/cheat_sheet.rst @@ -75,6 +75,20 @@ Build all packages in a given directory: ... or in the current folder: - ``catkin build $(catkin list -u -d .)`` +Testing Packages +^^^^^^^^^^^^^^^^ + +Test all the packages: + - ``catkin test`` + +... one at a time, with live output: + - ``catkin build -p 1 -i`` + +Test a specific package: + - ``catkin test my_package`` + +... or a specific test target of a package + - ``catkin test -t my_target my_package`` Cleaning Build Products ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/conf.py b/docs/conf.py index d42392d0..2106a275 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,8 +34,6 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.viewcode', - #'sphinxcontrib.programoutput', - #'sphinxcontrib.ansi', ] # Conditionally add spelling diff --git a/docs/installing.rst b/docs/installing.rst index 5b5244ac..3fded3ff 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -26,7 +26,7 @@ Once you have added that repository, run these commands to install ``catkin_tool .. code-block:: bash $ sudo apt-get update - $ sudo apt-get install python-catkin-tools + $ sudo apt-get install python3-catkin-tools Installing on other platforms with pip ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -35,7 +35,7 @@ Simply install it with ``pip``: .. code-block:: bash - $ sudo pip install -U catkin_tools + $ sudo pip3 install -U catkin_tools Installing from source ^^^^^^^^^^^^^^^^^^^^^^ @@ -51,13 +51,13 @@ Then install the dependencies with ``pip``: .. code-block:: bash - $ pip install -r requirements.txt --upgrade + $ pip3 install -r requirements.txt --upgrade Then install with the ``setup.py`` file: .. code-block:: bash - $ python setup.py install --record install_manifest.txt + $ python3 setup.py install --record install_manifest.txt .. note:: @@ -74,14 +74,14 @@ To setup ``catkin_tools`` for fast iteration during development, use the ``devel .. code-block:: bash - $ python setup.py develop + $ python3 setup.py develop Now the commands, like ``catkin``, will be in the system path and the local source files located in the ``catkin_tools`` folder will be on the ``PYTHONPATH``. When you are done with your development, undo this by running this command: .. code-block:: bash - $ python setup.py develop -u + $ python3 setup.py develop -u Uninstalling from Source diff --git a/docs/migration.rst b/docs/migration.rst index d972444a..3a15a66a 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -52,7 +52,7 @@ To make iterating easier, use ``catkin_make`` with build and devel spaces with t .. code-block:: bash cd /path/to/ws - catkin_make --cmake-args [CMAKE_ARGS...] --make-args [MAKE_ARGS...] + catkin_make --build build_cm --cmake-args -DCATKIN_DEVEL_PREFIX=devel_cm -DCMAKE_INSTALL_PREFIX=install_cm [CMAKE_ARGS...] --make-args [MAKE_ARGS...] If your packages build and other appropriate tests pass, continue to the next step. diff --git a/docs/quick_start.rst b/docs/quick_start.rst index 7531d5ae..9dc073b7 100644 --- a/docs/quick_start.rst +++ b/docs/quick_start.rst @@ -80,7 +80,7 @@ Now that there are some packages in the workspace, Catkin has something to build .. note:: Catkin utilizes an "out-of-source" and "aggregated" build pattern. - This means that temporary or final build will products never be placed in a package's source directory (or anywhere in the **source space**. + This means that temporary or final build products will never be placed in a package's source directory (or anywhere in the **source space**. Instead all build directories are aggregated in the **build space** and all final build products like executables, libraries, etc., will be put in the **devel space**. Building the Workspace diff --git a/docs/verbs/catkin_build.rst b/docs/verbs/catkin_build.rst index 404cd995..30c6d0cd 100644 --- a/docs/verbs/catkin_build.rst +++ b/docs/verbs/catkin_build.rst @@ -232,59 +232,6 @@ This will skip all of the package's dependencies, build the given package, and t
-Building and Running Tests -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Running tests for a given package typically is done by invoking a special ``make`` target like ``test`` or ``run_tests``. -catkin packages all define the ``run_tests`` target which aggregates all types of tests and runs them together. -So in order to get tests to build and run for your packages you need to pass them this additional ``run_tests`` or ``test`` target as a command line option to ``make``. - -To run catkin tests for all catkin packages in the workspace, use the following: - -.. code-block:: bash - - $ catkin run_tests - -Or the longer version: - -.. code-block:: bash - - $ catkin build [...] --catkin-make-args run_tests - -To run a catkin test for a specific catkin package, from a directory within that package: - -.. code-block:: bash - - $ catkin run_tests --no-deps --this - -For non-catkin packages which define a ``test`` target, you can do this: - -.. code-block:: bash - - $ catkin build [...] --make-args test - -If you want to run tests for just one package, then you should build that package and this narrow down the build to just that package with the additional make argument: - -.. code-block:: bash - - $ # First build the package - $ catkin build package - ... - $ # Then run its tests - $ catkin build package --no-deps --catkin-make-args run_tests - $ # Or for non-catkin packages - $ catkin build package --no-deps --make-args test - -For catkin packages and the ``run_tests`` target, failing tests will not result in an non-zero exit code. -So if you want to check for failing tests, use the ``catkin_test_results`` command like this: - -.. code-block:: bash - - $ catkin_test_results build/ - -The result code will be non-zero unless all tests passed. - - Advanced Options ^^^^^^^^^^^^^^^^ diff --git a/docs/verbs/catkin_test.rst b/docs/verbs/catkin_test.rst new file mode 100644 index 00000000..14aab526 --- /dev/null +++ b/docs/verbs/catkin_test.rst @@ -0,0 +1,55 @@ +``catkin test`` -- Test Packages +================================== + +The ``test`` verb is used to test one or more packages in a catkin workspace. +Like most verbs, ``test`` is context-aware and can be executed from within any directory contained by an initialized workspace. +Specific workspaces can also be built from arbitrary working directories with the ``--workspace`` option. + +Basic Usage +^^^^^^^^^^^ + +Before running tests for packages in the workspace, they have to be built with ``catkin build``. +Then, to run the tests, use the following: + +.. code-block:: bash + + $ catkin test + +Under the hood, this invokes the ``make`` targets ``run_tests`` or ``test``, depending on the package. +catkin packages all define the ``run_tests`` target which aggregates all types of tests and runs them together. +For cmake packages that do not use catkin, the ``test`` target is invoked. +This target is usually populated by cmake when the ``enable_testing()`` command is used in the ``CMakeLists.txt``. +If it does not exist, a warning is printed. + +To run a catkin test for a specific catkin package, from a directory within that package: + +.. code-block:: bash + + $ catkin test --this + +Advanced Options +^^^^^^^^^^^^^^^^ + +To manually specify a different ``make`` target, use ``--test-target``: + +.. code-block:: bash + + $ catkin test --test-target gtest + +It is also possible to use ``--catkin-test-target`` to change the target only for catkin packages. + +Normally, the tests are run in parallel, similar to the build jobs of ``catkin build``. +To avoid building packages in parallel or to reduce the amount of parallel jobs, use ``-p``: + +.. code-block:: bash + + $ catkin test -p 1 + +Sometimes, it can be helpful to see the output of tests while they are still running. +This can be achieved using ``--interleave-output``. + +Full Command-Line Interface +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. literalinclude:: cli/catkin_test.txt + :language: text diff --git a/docs/verbs/cli/catkin_build.txt b/docs/verbs/cli/catkin_build.txt index bfa4943e..67e05020 100644 --- a/docs/verbs/cli/catkin_build.txt +++ b/docs/verbs/cli/catkin_build.txt @@ -3,7 +3,8 @@ usage: catkin build [-h] [--workspace WORKSPACE] [--profile PROFILE] [--unbuilt] [--start-with PKGNAME | --start-with-this] [--continue-on-failure] [--force-cmake] [--pre-clean] [--skip-install] [--no-install-lock] [--save-config] - [-j JOBS] [-p PACKAGE_JOBS] [--jobserver | --no-jobserver] + [-j JOBS] [-p PACKAGE_JOBS] [-l LOAD_AVERAGE] + [--jobserver | --no-jobserver] [--env-cache | --no-env-cache] [--cmake-args ARG [ARG ...] | --no-cmake-args] [--make-args ARG [ARG ...] | --no-make-args] [--catkin-make-args ARG [ARG ...] | @@ -72,6 +73,9 @@ Config: -p PACKAGE_JOBS, --parallel-packages PACKAGE_JOBS Maximum number of packages allowed to be built in parallel (default is cpu count) + -l LOAD_AVERAGE, --load-average LOAD_AVERAGE + Maximum load average before no new build jobs are + scheduled --jobserver Use the internal GNU Make job server which will limit the number of Make jobs across all active packages. --no-jobserver Disable the internal GNU Make job server, and use an @@ -116,7 +120,7 @@ Interface: --no-summarize, --no-summary Explicitly disable the end of build summary --override-build-tool-check - use to override failure due to using differnt build + use to override failure due to using different build tools on the same workspace. --limit-status-rate LIMIT_STATUS_RATE, --status-rate LIMIT_STATUS_RATE Limit the update rate of the status bar to this diff --git a/docs/verbs/cli/catkin_clean.txt b/docs/verbs/cli/catkin_clean.txt index c27f4fdd..804146ee 100644 --- a/docs/verbs/cli/catkin_clean.txt +++ b/docs/verbs/cli/catkin_clean.txt @@ -1,8 +1,8 @@ usage: catkin clean [-h] [--workspace WORKSPACE] [--profile PROFILE] [--dry-run] [--verbose] [--yes] [--force] [--all-profiles] - [--deinit] [-l] [-b] [-d] [-i] [--this] [--dependents] + [--deinit] [-b] [-d] [-i] [-L] [--this] [--dependents] [--orphans] [--setup-files] - [PKGNAME [PKGNAME ...]] + [PKGNAME ...] Deletes various products of the build verb. @@ -31,10 +31,14 @@ Full: Spaces: Clean workspace subdirectories for the selected profile. - -l, --logs Remove the entire log space. - -b, --build Remove the entire build space. - -d, --devel Remove the entire devel space. - -i, --install Remove the entire install space. + -b, --build, --build-space + Remove the entire build space. + -d, --devel, --devel-space + Remove the entire devel space. + -i, --install, --install-space + Remove the entire install space. + -L, --logs, --log-space + Remove the entire log space. Packages: Clean products from specific packages in the workspace. Note that these diff --git a/docs/verbs/cli/catkin_config.txt b/docs/verbs/cli/catkin_config.txt index a30f47bf..9aacd495 100644 --- a/docs/verbs/cli/catkin_config.txt +++ b/docs/verbs/cli/catkin_config.txt @@ -6,15 +6,16 @@ usage: catkin config [-h] [--workspace WORKSPACE] [--profile PROFILE] [--whitelist PKG [PKG ...] | --no-whitelist] [--blacklist PKG [PKG ...] | --no-blacklist] [--build-space BUILD_SPACE | --default-build-space] - [--log-space LOG_SPACE | --default-log-space] - [--install-space INSTALL_SPACE | --default-install-space] [--devel-space DEVEL_SPACE | --default-devel-space] + [--install-space INSTALL_SPACE | --default-install-space] + [--log-space LOG_SPACE | --default-log-space] [--source-space SOURCE_SPACE | --default-source-space] [-x SPACE_SUFFIX] [--link-devel | --merge-devel | --isolate-devel] [--install | --no-install] [--isolate-install | --merge-install] [-j JOBS] - [-p PACKAGE_JOBS] [--jobserver | --no-jobserver] + [-p PACKAGE_JOBS] [-l LOAD_AVERAGE] + [--jobserver | --no-jobserver] [--env-cache | --no-env-cache] [--cmake-args ARG [ARG ...] | --no-cmake-args] [--make-args ARG [ARG ...] | --no-make-args] @@ -83,17 +84,17 @@ Spaces: The path to the build space. --default-build-space Use the default path to the build space ("build") - --log-space LOG_SPACE, -l LOG_SPACE - The path to the log space. - --default-log-space Use the default path to the log space ("logs") - --install-space INSTALL_SPACE, -i INSTALL_SPACE - The path to the install space. - --default-install-space - Use the default path to the install space ("install") --devel-space DEVEL_SPACE, -d DEVEL_SPACE The path to the devel space. --default-devel-space Use the default path to the devel space ("devel") + --install-space INSTALL_SPACE, -i INSTALL_SPACE + The path to the install space. + --default-install-space + Use the default path to the install space ("install") + --log-space LOG_SPACE, -L LOG_SPACE + The path to the log space. + --default-log-space Use the default path to the log space ("logs") --source-space SOURCE_SPACE, -s SOURCE_SPACE The path to the source space. --default-source-space @@ -133,6 +134,9 @@ Build Options: -p PACKAGE_JOBS, --parallel-packages PACKAGE_JOBS Maximum number of packages allowed to be built in parallel (default is cpu count) + -l LOAD_AVERAGE, --load-average LOAD_AVERAGE + Maximum load average before no new build jobs are + scheduled --jobserver Use the internal GNU Make job server which will limit the number of Make jobs across all active packages. --no-jobserver Disable the internal GNU Make job server, and use an diff --git a/docs/verbs/cli/catkin_create_pkg.txt b/docs/verbs/cli/catkin_create_pkg.txt index 94cd79fc..0a12ce8b 100644 --- a/docs/verbs/cli/catkin_create_pkg.txt +++ b/docs/verbs/cli/catkin_create_pkg.txt @@ -1,20 +1,19 @@ -usage: catkin create pkg [-h] [-p PATH] [--rosdistro ROSDISTRO] +usage: catkin create pkg [-h] [-p PATH] --rosdistro ROSDISTRO [-v MAJOR.MINOR.PATCH] [-l LICENSE] [-m NAME EMAIL] [-a NAME EMAIL] [-d DESCRIPTION] - [--catkin-deps [DEP [DEP ...]]] - [--system-deps [DEP [DEP ...]]] - [--boost-components [COMP [COMP ...]]] + [--catkin-deps [DEP ...]] [--system-deps [DEP ...]] + [--boost-components [COMP ...]] PKG_NAME [PKG_NAME ...] Create a new Catkin package. Note that while the default options used by this command are sufficient for prototyping and local usage, it is important that -any publically-available packages have a valid license and a valid maintainer +any publicly-available packages have a valid license and a valid maintainer e-mail address. positional arguments: PKG_NAME The name of one or more packages to create. This name should be completely lower-case with individual words - separated by undercores. + separated by underscores. optional arguments: -h, --help show this help message and exit @@ -41,15 +40,15 @@ Package Metadata: Description of the package. (default: empty) Package Dependencies: - --catkin-deps [DEP [DEP ...]], -c [DEP [DEP ...]] + --catkin-deps [DEP ...], -c [DEP ...] The names of one or more Catkin dependencies. These are Catkin-based packages which are either built as source or installed by your system's package manager. - --system-deps [DEP [DEP ...]], -s [DEP [DEP ...]] + --system-deps [DEP ...], -s [DEP ...] The names of one or more system dependencies. These are other packages installed by your operating system's package manager. C++ Options: - --boost-components [COMP [COMP ...]] + --boost-components [COMP ...] One or more boost components used by the package. diff --git a/docs/verbs/cli/catkin_env.txt b/docs/verbs/cli/catkin_env.txt index 9387d5a7..48f43d32 100644 --- a/docs/verbs/cli/catkin_env.txt +++ b/docs/verbs/cli/catkin_env.txt @@ -1,5 +1,4 @@ -usage: catkin env [-h] [-i] [-s] - [NAME=VALUE [NAME=VALUE ...]] [COMMAND] [ARG [ARG ...]] +usage: catkin env [-h] [-i] [-s] [NAME=VALUE ...] [COMMAND] [ARG ...] Run an arbitrary command in a modified environment. @@ -12,7 +11,8 @@ optional arguments: -i, --ignore-environment Start with an empty environment. -s, --stdin Read environment variable definitions from stdin. - Variables should be given in NAME=VALUE format. + Variables should be given in NAME=VALUE format, + separated by null-bytes. command: COMMAND Command to run. If omitted, the environment is printed diff --git a/docs/verbs/cli/catkin_list.txt b/docs/verbs/cli/catkin_list.txt index b3a1708b..87c379f0 100644 --- a/docs/verbs/cli/catkin_list.txt +++ b/docs/verbs/cli/catkin_list.txt @@ -1,10 +1,9 @@ usage: catkin list [-h] [--workspace WORKSPACE] [--profile PROFILE] - [--deps | --rdeps] [--depends-on [PKG [PKG ...]]] - [--rdepends-on [PKG [PKG ...]]] [--this] - [--directory [DIRECTORY [DIRECTORY ...]]] [--quiet] - [--unformatted] + [--deps | --rdeps] [--depends-on [PKG ...]] + [--rdepends-on [PKG ...]] [--this] + [--directory [DIRECTORY ...]] [--quiet] [--unformatted] -Lists catkin packages in the workspace or other arbitray folders. +Lists catkin packages in the workspace or other arbitrary folders. optional arguments: -h, --help show this help message and exit @@ -25,15 +24,15 @@ Information: Packages: Control which packages are listed. - --depends-on [PKG [PKG ...]] + --depends-on [PKG ...] Only show packages that directly depend on specific package(s). - --rdepends-on [PKG [PKG ...]], --recursive-depends-on [PKG [PKG ...]] + --rdepends-on [PKG ...], --recursive-depends-on [PKG ...] Only show packages that recursively depend on specific package(s). --this Show the package which contains the current working directory. - --directory [DIRECTORY [DIRECTORY ...]], -d [DIRECTORY [DIRECTORY ...]] + --directory [DIRECTORY ...], -d [DIRECTORY ...] Pass list of directories process all packages in directory diff --git a/docs/verbs/cli/catkin_locate.txt b/docs/verbs/cli/catkin_locate.txt index e7bb40d6..cd1a3795 100644 --- a/docs/verbs/cli/catkin_locate.txt +++ b/docs/verbs/cli/catkin_locate.txt @@ -1,6 +1,6 @@ usage: catkin locate [-h] [--workspace WORKSPACE] [--profile PROFILE] [-e] - [-r] [-q] [-s | -b | -d | -i] [--this] [--shell-verbs] - [--examples] + [-r] [-q] [-b | -d | -i | -L | -s] [--this] + [--shell-verbs] [--examples] [PACKAGE] Get the paths to various locations in a workspace. @@ -22,10 +22,16 @@ Sub-Space Options: Get the absolute path to one of the following locations in the given workspace with the given profile. - -s, --src Get the path to the source space. - -b, --build Get the path to the build space. - -d, --devel Get the path to the devel space. - -i, --install Get the path to the install space. + -b, --build, --build-space + Get the path to the build space. + -d, --devel, --devel-space + Get the path to the devel space. + -i, --install, --install-space + Get the path to the install space. + -L, --logs, --log-space + Get the path to the log space. + -s, --src, --source-space + Get the path to the source space. Package Directories: Get the absolute path to package directories in the given workspace and diff --git a/docs/verbs/cli/catkin_profile_add.txt b/docs/verbs/cli/catkin_profile_add.txt index f674b89e..284f4bdc 100644 --- a/docs/verbs/cli/catkin_profile_add.txt +++ b/docs/verbs/cli/catkin_profile_add.txt @@ -1,11 +1,15 @@ -usage: catkin profile add [-h] [-f] [--copy BASE_PROFILE | --copy-active] name +usage: catkin profile add [-h] [-f] + [--copy BASE_PROFILE | --copy-active | --extend PARENT_PROFILE] + name positional arguments: - name The new profile name. + name The new profile name. optional arguments: - -h, --help show this help message and exit - -f, --force Overwrite an existing profile. - --copy BASE_PROFILE Copy the settings from an existing profile. (default: - None) - --copy-active Copy the settings from the active profile. + -h, --help show this help message and exit + -f, --force Overwrite an existing profile. + --copy BASE_PROFILE Copy the settings from an existing profile. (default: + None) + --copy-active Copy the settings from the active profile. + --extend PARENT_PROFILE + Extend another profile diff --git a/docs/verbs/cli/catkin_profile_remove.txt b/docs/verbs/cli/catkin_profile_remove.txt index 0fd0b59f..3e9600d9 100644 --- a/docs/verbs/cli/catkin_profile_remove.txt +++ b/docs/verbs/cli/catkin_profile_remove.txt @@ -1,4 +1,4 @@ -usage: catkin profile remove [-h] [name [name ...]] +usage: catkin profile remove [-h] [name ...] positional arguments: name One or more profile names to remove. diff --git a/docs/verbs/cli/catkin_test.txt b/docs/verbs/cli/catkin_test.txt new file mode 100644 index 00000000..2963497a --- /dev/null +++ b/docs/verbs/cli/catkin_test.txt @@ -0,0 +1,64 @@ +usage: catkin test [-h] [--workspace WORKSPACE] [--profile PROFILE] [--this] + [--continue-on-failure] [-p PACKAGE_JOBS] [-t TARGET] + [--catkin-test-target TARGET] [--make-args ARG [ARG ...]] + [--verbose] [--interleave-output] [--summarize] + [--no-status] [--limit-status-rate LIMIT_STATUS_RATE] + [--no-notify] + [PKGNAME ...] + +Test one or more packages in a catkin workspace. This invokes `make run_tests` +or `make test` for either all or the specified packages in a catkin workspace. + +optional arguments: + -h, --help show this help message and exit + --workspace WORKSPACE, -w WORKSPACE + The path to the catkin_tools workspace or a directory + contained within it (default: ".") + --profile PROFILE The name of a config profile to use (default: active + profile) + +Packages: + Control which packages get tested. + + PKGNAME Workspace packages to test. If no packages are given, + then all the packages are tested. + --this Test the package containing the current working + directory. + --continue-on-failure, -c + Continue testing packages even if the tests for other + requested packages fail. + +Config: + Parameters for the underlying build system. + + -p PACKAGE_JOBS, --parallel-packages PACKAGE_JOBS + Maximum number of packages allowed to be built in + parallel (default is cpu count) + -t TARGET, --test-target TARGET + Make target to run for tests (default is "run_tests" + for catkin and "test" for cmake) + --catkin-test-target TARGET + Make target to run for tests for catkin packages, + overwrites --test-target (default is "run_tests") + --make-args ARG [ARG ...] + Arbitrary arguments which are passed to make. It + collects all of following arguments until a "--" is + read. + +Interface: + The behavior of the command-line interface. + + --verbose, -v Print output from commands in ordered blocks once the + command finishes. + --interleave-output, -i + Prevents ordering of command output when multiple + commands are running at the same time. + --summarize, --summary, -s + Adds a summary to the end of the log + --no-status Suppresses status line, useful in situations where + carriage return is not properly supported. + --limit-status-rate LIMIT_STATUS_RATE, --status-rate LIMIT_STATUS_RATE + Limit the update rate of the status bar to this + frequency. Zero means unlimited. Must be positive, + default is 10 Hz. + --no-notify Suppresses system pop-up notification. diff --git a/docs/verbs/cli/dump_cli b/docs/verbs/cli/dump_cli index 1143b024..c69c828a 100755 --- a/docs/verbs/cli/dump_cli +++ b/docs/verbs/cli/dump_cli @@ -15,3 +15,4 @@ catkin profile set -h > catkin_profile_set.txt catkin profile add -h > catkin_profile_add.txt catkin profile rename -h > catkin_profile_rename.txt catkin profile remove -h > catkin_profile_remove.txt +catkin test -h > catkin_test.txt diff --git a/requirements.txt b/requirements.txt index 06ceec8a..59061505 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ catkin_pkg osrf_pycommon pyyaml setuptools -sphinxcontrib-programoutput diff --git a/setup.py b/setup.py index 6d0bcdee..a9876cb9 100644 --- a/setup.py +++ b/setup.py @@ -11,12 +11,16 @@ # Setup installation dependencies install_requires = [ - 'catkin-pkg > 0.2.9', 'setuptools', 'PyYAML', 'osrf-pycommon > 0.1.1', ] +# When building the deb, do not require catkin_pkg +if 'DEB_BUILD' not in os.environ: + install_requires += ['catkin_pkg >= 0.3.0'] + + # Figure out the resources that need to be installed this_dir = os.path.abspath(os.path.dirname(__file__)) osx_resources_path = os.path.join( @@ -34,38 +38,33 @@ for x in osx_notification_resources] -def _resolve_prefix(prefix, type): +def _resolve_prefix(type): osx_system_prefix = '/System/Library/Frameworks/Python.framework/Versions' - if type == 'man': - if prefix == '/usr': - return '/usr/share' + if type == 'bash_comp': if sys.prefix.startswith(osx_system_prefix): - return '/usr/share' - elif type == 'bash_comp': - if prefix == '/usr': - return '/' - if sys.prefix.startswith(osx_system_prefix): - return '/' + return '/usr' elif type == 'zsh_comp': if sys.prefix.startswith(osx_system_prefix): return '/usr/local' else: raise ValueError('not supported type') - return prefix + return '' -def get_data_files(prefix): +def get_data_files(): data_files = [] # Bash completion - bash_comp_dest = os.path.join(_resolve_prefix(prefix, 'bash_comp'), - 'etc/bash_completion.d') - data_files.append((bash_comp_dest, - ['completion/catkin_tools-completion.bash'])) + bash_comp_dest = os.path.join(_resolve_prefix('bash_comp'), + 'share/bash-completion/completions') + data_files.append((bash_comp_dest, ['completion/catkin.bash'])) # Zsh completion - zsh_comp_dest = os.path.join(_resolve_prefix(prefix, 'zsh_comp'), - 'share/zsh/site-functions') + if 'DEB_BUILD' in os.environ: + dirname = 'share/zsh/vendor-completions' + else: + dirname = 'share/zsh/site-functions' + zsh_comp_dest = os.path.join(_resolve_prefix('zsh_comp'), dirname) data_files.append((zsh_comp_dest, ['completion/_catkin'])) return data_files @@ -81,30 +80,10 @@ def run(self): log.info("changing permissions of %s to %o" % (file, mode)) os.chmod(file, mode) - # Provide information about bash completion after default install. - if (sys.platform.startswith("linux") and - self.install_data == "/usr/local"): - log.info(""" ----------------------------------------------------------------- -To enable tab completion, add the following to your '~/.bashrc': - - source {0} - ----------------------------------------------------------------- -""".format(os.path.join(self.install_data, - 'etc/bash_completion.d', - 'catkin_tools-completion.bash'))) - -from distutils.core import setup -from distutils.dist import Distribution -dist = Distribution() -dist.parse_config_files() -dist.parse_command_line() -prefix = dist.get_option_dict('install').get('prefix',("default", sys.prefix))[1] setup( name='catkin_tools', - version='0.5.0', + version='0.8.2', python_requires='>=3.5', packages=find_packages(exclude=['tests*', 'docs']), package_data={ @@ -117,13 +96,13 @@ def run(self): 'docs/examples', ] + osx_notification_resources }, - data_files=get_data_files(prefix), + data_files=get_data_files(), install_requires=install_requires, author='William Woodall', author_email='william@osrfoundation.org', maintainer='William Woodall', maintainer_email='william@osrfoundation.org', - url='http://catkin-tools.readthedocs.org/', + url='https://catkin-tools.readthedocs.org/', keywords=['catkin'], classifiers=[ 'Environment :: Console', @@ -149,6 +128,7 @@ def run(self): 'list = catkin_tools.verbs.catkin_list:description', 'locate = catkin_tools.verbs.catkin_locate:description', 'profile = catkin_tools.verbs.catkin_profile:description', + 'test = catkin_tools.verbs.catkin_test:description', ], 'catkin_tools.jobs': [ 'catkin = catkin_tools.jobs.catkin:description', diff --git a/stdeb.cfg b/stdeb.cfg index 23de0f22..8dc9259f 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,7 +1,7 @@ [DEFAULT] -Depends: python-argparse, python-setuptools, python-catkin-pkg (> 0.2.9), python-yaml -Depends3: python3-setuptools, python3-catkin-pkg (> 0.2.9), python3-yaml -Conflicts: python3-catkin-tools +Depends3: python3-setuptools, python3-catkin-pkg-modules, python3-yaml, python3-osrf-pycommon Conflicts3: python-catkin-tools Suite: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster X-Python3-Version: >= 3.5 +Setup-Env-Vars: DEB_BUILD=1 +No-Python2: diff --git a/tests/system/resources/catkin_pkgs/depend_condition/CMakeLists.txt b/tests/system/resources/catkin_pkgs/depend_condition/CMakeLists.txt new file mode 100644 index 00000000..a96ca4af --- /dev/null +++ b/tests/system/resources/catkin_pkgs/depend_condition/CMakeLists.txt @@ -0,0 +1,2 @@ +cmake_minimum_required(VERSION 2.8.12) +project(build_type_condition) diff --git a/tests/system/resources/catkin_pkgs/depend_condition/package.xml b/tests/system/resources/catkin_pkgs/depend_condition/package.xml new file mode 100644 index 00000000..279bbc93 --- /dev/null +++ b/tests/system/resources/catkin_pkgs/depend_condition/package.xml @@ -0,0 +1,11 @@ + + depend_condition + Package with conditional depend element. + 0.1.0 + BSD + todo + + catkin + ros1_pkg + ros2_pkg + diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/CMakeLists.txt b/tests/system/resources/catkin_pkgs/python_tests_targets/CMakeLists.txt new file mode 100644 index 00000000..4e349acd --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 2.8.12) +project(python_tests_targets) +find_package(catkin REQUIRED) + +catkin_package() + +if(CATKIN_ENABLE_TESTING) + catkin_add_nosetests(test_good.py) + catkin_add_nosetests(test_bad.py) +endif() diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/package.xml b/tests/system/resources/catkin_pkgs/python_tests_targets/package.xml new file mode 100644 index 00000000..9b4b2e6b --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/package.xml @@ -0,0 +1,11 @@ + + python_tests_targets + 0.1.0 + BSD + todo + This package contains two python tests. + + catkin + unittest + + diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/setup.py b/tests/system/resources/catkin_pkgs/python_tests_targets/setup.py new file mode 100644 index 00000000..fb1311ce --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/setup.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +from distutils.core import setup +from catkin_pkg.python_setup import generate_distutils_setup + +d = generate_distutils_setup() +setup(**d) diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/test_bad.py b/tests/system/resources/catkin_pkgs/python_tests_targets/test_bad.py new file mode 100644 index 00000000..f46637e8 --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/test_bad.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +import unittest + + +class TestBad(unittest.TestCase): + + def test_zero(self): + self.assertEqual(0, 1) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/system/resources/catkin_pkgs/python_tests_targets/test_good.py b/tests/system/resources/catkin_pkgs/python_tests_targets/test_good.py new file mode 100644 index 00000000..0ae625a2 --- /dev/null +++ b/tests/system/resources/catkin_pkgs/python_tests_targets/test_good.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +import unittest + + +class TestGood(unittest.TestCase): + + def test_zero(self): + self.assertEqual(0, 0) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/system/resources/cmake_pkgs/test_err_pkg/CMakeLists.txt b/tests/system/resources/cmake_pkgs/test_err_pkg/CMakeLists.txt new file mode 100644 index 00000000..b68158d5 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_err_pkg/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 2.8.12) +project(test_err_pkg) + +include(CTest) +add_executable(MyTest test.cpp) +add_test(NAME MyTest COMMAND MyTest) +enable_testing() + +install(TARGETS MyTest + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib/static) diff --git a/tests/system/resources/cmake_pkgs/test_err_pkg/package.xml b/tests/system/resources/cmake_pkgs/test_err_pkg/package.xml new file mode 100644 index 00000000..b4d9610a --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_err_pkg/package.xml @@ -0,0 +1,10 @@ + + test_err_pkg + This package has a failing ctest test + 0.1.0 + BSD + todo + + cmake + + diff --git a/tests/system/resources/cmake_pkgs/test_err_pkg/test.cpp b/tests/system/resources/cmake_pkgs/test_err_pkg/test.cpp new file mode 100644 index 00000000..b830dcd3 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_err_pkg/test.cpp @@ -0,0 +1,6 @@ +#include + +int main() { + std::cout << "Test failure" << std::endl; + return 1; +} diff --git a/tests/system/resources/cmake_pkgs/test_pkg/CMakeLists.txt b/tests/system/resources/cmake_pkgs/test_pkg/CMakeLists.txt new file mode 100644 index 00000000..b55a2b25 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_pkg/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 2.8.12) +project(test_pkg) + +include(CTest) +add_executable(MyTest test.cpp) +add_test(NAME MyTest COMMAND MyTest) +enable_testing() + +install(TARGETS MyTest + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib/static) diff --git a/tests/system/resources/cmake_pkgs/test_pkg/package.xml b/tests/system/resources/cmake_pkgs/test_pkg/package.xml new file mode 100644 index 00000000..f9501c7f --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_pkg/package.xml @@ -0,0 +1,10 @@ + + test_pkg + This package has a passing ctest test + 0.1.0 + BSD + todo + + cmake + + diff --git a/tests/system/resources/cmake_pkgs/test_pkg/test.cpp b/tests/system/resources/cmake_pkgs/test_pkg/test.cpp new file mode 100644 index 00000000..1e5d1669 --- /dev/null +++ b/tests/system/resources/cmake_pkgs/test_pkg/test.cpp @@ -0,0 +1,6 @@ +#include + +int main() { + std::cout << "Test success" << std::endl; + return 0; +} diff --git a/tests/system/verbs/catkin_build/test_args.py b/tests/system/verbs/catkin_build/test_args.py index 27627283..74a15946 100644 --- a/tests/system/verbs/catkin_build/test_args.py +++ b/tests/system/verbs/catkin_build/test_args.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import shutil diff --git a/tests/system/verbs/catkin_build/test_build.py b/tests/system/verbs/catkin_build/test_build.py index ae3c7645..71fb95dd 100644 --- a/tests/system/verbs/catkin_build/test_build.py +++ b/tests/system/verbs/catkin_build/test_build.py @@ -1,12 +1,10 @@ -from __future__ import print_function - import os import re import shutil from ...workspace_factory import workspace_factory -from ....utils import in_temporary_directory +from ....utils import in_temporary_directory, temporary_directory from ....utils import assert_cmd_success from ....utils import assert_cmd_failure from ....utils import assert_files_exist @@ -380,3 +378,56 @@ def test_pkg_with_conditional_build_type(): # So we have to infer this skipping by checking the build directory. msg = "Package with ROS 2 conditional build_type was skipped." assert os.path.exists(os.path.join('build', 'build_type_condition')), msg + + +def test_pkg_with_conditional_depend(): + """Test building a package with a condition attribute in the depend tag""" + with redirected_stdio() as (out, err): + with workspace_factory() as wf: + wf.create_package('ros1_pkg') + wf.create_package('ros2_pkg') + wf.build() + shutil.copytree( + os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'depend_condition'), + os.path.join('src/depend_condition')) + assert catkin_success(BUILD + ['depend_condition'], env={'ROS_VERSION': '1'}) + assert os.path.exists(os.path.join('build', 'depend_condition')) + assert os.path.exists(os.path.join('build', 'ros1_pkg')) + assert not os.path.exists(os.path.join('build', 'ros2_pkg')) + + +def test_symlinked_workspace(): + """Test building from a symlinked workspace""" + with redirected_stdio() as (out, err): + with workspace_factory() as wf: + wf.create_package('pkg') + wf.build() + assert catkin_success(BUILD) + with temporary_directory() as t: + os.symlink(wf.workspace, os.path.join(t, 'ws')) + assert catkin_success(BUILD + ['-w', os.path.join(t, 'ws')]) + + +def test_generate_setup_util(): + """Test generation of setup utilities in a linked devel space""" + with redirected_stdio() as (out, err): + with workspace_factory() as wf: + wf.create_package('pkg') + wf.build() + # Test that the files are generated in a clean workspace + assert catkin_success(['config', '--install']) + assert catkin_success(BUILD) + assert os.path.exists(os.path.join(wf.workspace, 'devel', '_setup_util.py')) + assert os.path.exists(os.path.join(wf.workspace, 'install', '_setup_util.py')) + + # Test that the files are regenerated after clean + assert catkin_success(['clean', '--yes']) + assert catkin_success(BUILD) + assert os.path.exists(os.path.join(wf.workspace, 'devel', '_setup_util.py')) + assert os.path.exists(os.path.join(wf.workspace, 'install', '_setup_util.py')) + + # Test that the files are regenerated after cleaning the install space + assert catkin_success(['clean', '--yes', '--install']) + assert catkin_success(BUILD) + assert os.path.exists(os.path.join(wf.workspace, 'devel', '_setup_util.py')) + assert os.path.exists(os.path.join(wf.workspace, 'install', '_setup_util.py')) diff --git a/tests/system/verbs/catkin_build/test_bwlists.py b/tests/system/verbs/catkin_build/test_bwlists.py index ea4e276c..234c28e7 100644 --- a/tests/system/verbs/catkin_build/test_bwlists.py +++ b/tests/system/verbs/catkin_build/test_bwlists.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os TEST_DIR = os.path.dirname(__file__) diff --git a/tests/system/verbs/catkin_build/test_context.py b/tests/system/verbs/catkin_build/test_context.py index 740d36ac..7701911f 100644 --- a/tests/system/verbs/catkin_build/test_context.py +++ b/tests/system/verbs/catkin_build/test_context.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os TEST_DIR = os.path.dirname(__file__) diff --git a/tests/system/verbs/catkin_build/test_eclipse.py b/tests/system/verbs/catkin_build/test_eclipse.py index c0f6a3e4..f8ed1237 100644 --- a/tests/system/verbs/catkin_build/test_eclipse.py +++ b/tests/system/verbs/catkin_build/test_eclipse.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os from ....utils import in_temporary_directory diff --git a/tests/system/verbs/catkin_build/test_modify_ws.py b/tests/system/verbs/catkin_build/test_modify_ws.py index 6168972e..14097135 100644 --- a/tests/system/verbs/catkin_build/test_modify_ws.py +++ b/tests/system/verbs/catkin_build/test_modify_ws.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os TEST_DIR = os.path.dirname(__file__) diff --git a/tests/system/verbs/catkin_build/test_unit_tests.py b/tests/system/verbs/catkin_build/test_unit_tests.py deleted file mode 100644 index 33e83784..00000000 --- a/tests/system/verbs/catkin_build/test_unit_tests.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import print_function - -import os -import shutil - -from ....utils import in_temporary_directory -from ....utils import assert_cmd_success -from ....utils import assert_cmd_failure -from ....utils import catkin_success -from ....utils import redirected_stdio - -TEST_DIR = os.path.dirname(__file__) -RESOURCES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'resources') - - -@in_temporary_directory -def test_build_pkg_unit_tests(): - """Test running working unit tests""" - cwd = os.getcwd() - source_space = os.path.join(cwd, 'src') - shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests'), source_space) - with redirected_stdio() as (out, err): - assert catkin_success( - ['build', '--no-notify', '--no-status', '--verbose', '--no-deps', - 'python_tests', '--make-args', 'run_tests']) - assert_cmd_success(['catkin_test_results', 'build/python_tests']) - - assert catkin_success( - ['run_tests', 'python_tests', '--no-deps', '--no-notify', '--no-status']) - assert_cmd_success(['catkin_test_results', 'build/python_tests']) - - -@in_temporary_directory -def test_build_pkg_unit_tests_broken(): - """Test running broken unit tests""" - cwd = os.getcwd() - source_space = os.path.join(cwd, 'src') - shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests_err'), source_space) - - with redirected_stdio() as (out, err): - assert catkin_success( - ['build', '--no-notify', '--no-status', '--verbose', '--no-deps', - 'python_tests_err', '--make-args', 'run_tests']) - assert_cmd_failure(['catkin_test_results', 'build/python_tests_err']) - - assert catkin_success( - ['run_tests', 'python_tests_err', '--no-deps', '--no-notify', '--no-status']) - assert_cmd_failure(['catkin_test_results', 'build/python_tests_err']) diff --git a/tests/system/verbs/catkin_config/test_config.py b/tests/system/verbs/catkin_config/test_config.py index 98886c50..9811c31c 100644 --- a/tests/system/verbs/catkin_config/test_config.py +++ b/tests/system/verbs/catkin_config/test_config.py @@ -1,11 +1,13 @@ import os +from ...workspace_factory import workspace_factory from ....utils import in_temporary_directory from ....utils import assert_cmd_success from ....workspace_assertions import assert_workspace_initialized from ....workspace_assertions import assert_warning_message from ....workspace_assertions import assert_no_warnings +from ....workspace_assertions import assert_in_config @in_temporary_directory @@ -28,3 +30,24 @@ def test_config_non_bare(): out = assert_cmd_success(['catkin', 'config', '--install']) assert_workspace_initialized('.') assert_warning_message(out, 'Source space .+ does not yet exist') + + +@in_temporary_directory +def test_config_unchanged(): + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', '-j6']) + assert_cmd_success(['catkin', 'config']) + assert_in_config('.', 'default', 'jobs_args', ['-j6']) + + +def test_config_no_args_flags(): + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', '-j6', 'test']) + assert_cmd_success(['catkin', 'config', '--cmake-args', '-DCMAKE_BUILD_TYPE=Release']) + assert_cmd_success(['catkin', 'config', '--catkin-make-args', 'run_tests']) + assert_cmd_success(['catkin', 'config', '--no-make-args', '--no-cmake-args', '--no-catkin-make-args']) + assert_in_config('.', 'default', 'jobs_args', []) + assert_in_config('.', 'default', 'make_args', []) + assert_in_config('.', 'default', 'cmake_args', []) diff --git a/tests/system/verbs/catkin_profile/test_profile.py b/tests/system/verbs/catkin_profile/test_profile.py index ea08c93e..f18c9b31 100644 --- a/tests/system/verbs/catkin_profile/test_profile.py +++ b/tests/system/verbs/catkin_profile/test_profile.py @@ -1,11 +1,10 @@ import os +from ...workspace_factory import workspace_factory from ....utils import in_temporary_directory from ....utils import assert_cmd_success -from ....workspace_assertions import assert_workspace_initialized -from ....workspace_assertions import assert_warning_message -from ....workspace_assertions import assert_no_warnings +from ....workspace_assertions import assert_in_config @in_temporary_directory @@ -22,3 +21,21 @@ def test_profile_set(): assert_cmd_success(['catkin', 'init']) assert_cmd_success(['catkin', 'build']) assert_cmd_success(['catkin', 'profile', 'set', 'default']) + + +def test_profile_copy(): + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', 'test']) + assert_cmd_success(['catkin', 'profile', 'add', '--copy', 'default', 'mycopy']) + assert_in_config('.', 'mycopy', 'make_args', ['test']) + + +def test_profile_extend(): + with workspace_factory() as wf: + wf.build() + assert_cmd_success(['catkin', 'config', '--make-args', 'test']) + assert_cmd_success(['catkin', 'profile', 'add', '--extend', 'default', 'myextend']) + assert_cmd_success(['catkin', 'config', '--profile', 'myextend', '--blacklist', 'mypackage']) + assert_in_config('.', 'default', 'make_args', ['test']) + assert_in_config('.', 'myextend', 'blacklist', ['mypackage']) diff --git a/tests/system/verbs/catkin_test/__init__.py b/tests/system/verbs/catkin_test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/system/verbs/catkin_test/test_unit_tests.py b/tests/system/verbs/catkin_test/test_unit_tests.py new file mode 100644 index 00000000..dd3fcd7a --- /dev/null +++ b/tests/system/verbs/catkin_test/test_unit_tests.py @@ -0,0 +1,86 @@ +import os +import shutil + +from ....utils import in_temporary_directory +from ....utils import assert_cmd_success +from ....utils import assert_cmd_failure +from ....utils import catkin_success +from ....utils import catkin_failure +from ....utils import redirected_stdio + +TEST_DIR = os.path.dirname(__file__) +RESOURCES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'resources') + + +@in_temporary_directory +def test_catkin_success(): + """Test running working unit tests""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests'), source_space) + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'python_tests', '--no-notify', '--no-status']) + assert catkin_success(['test', 'python_tests', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_catkin_failure(): + """Test running broken unit tests""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests_err'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'python_tests_err', '--no-notify', '--no-status']) + assert catkin_failure(['test', 'python_tests_err', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_cmake_success(): + """Test vanilla cmake package""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'test_pkg'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'test_pkg', '--no-notify', '--no-status']) + assert catkin_success(['test', 'test_pkg', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_cmake_failure(): + """Test failing vanilla cmake package""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'test_err_pkg'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'test_err_pkg', '--no-notify', '--no-status']) + assert catkin_failure(['test', 'test_err_pkg', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_skip_missing_test(): + """Test to skip packages without tests""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'cmake_pkg'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'cmake_pkg', '--no-notify', '--no-status']) + assert catkin_success(['test', '--no-notify', '--no-status']) + + +@in_temporary_directory +def test_other_target(): + """Test with a manually specified target""" + cwd = os.getcwd() + source_space = os.path.join(cwd, 'src') + shutil.copytree(os.path.join(RESOURCES_DIR, 'catkin_pkgs', 'python_tests_targets'), source_space) + + with redirected_stdio() as (out, err): + assert catkin_success(['build', 'python_tests_targets', '--no-notify', '--no-status']) + assert catkin_success(['test', '--test-target', 'run_tests_python_tests_targets_nosetests_test_good.py', + '--no-notify', '--no-status']) + assert catkin_failure(['test', '--test-target', 'run_tests_python_tests_targets_nosetests_test_bad.py', + '--no-notify', '--no-status']) diff --git a/tests/unit/test_common.py b/tests/unit/test_common.py index 03a12aac..a277ff2f 100644 --- a/tests/unit/test_common.py +++ b/tests/unit/test_common.py @@ -13,6 +13,8 @@ def __init__(self, name): self.run_depends = [] self.exec_depends = [] self.build_export_depends = [] + self.evaluated_condition = True + def test_get_recursive_build_depends_in_workspace_with_test_depend(): pkg1 = MockPackage('pkg1') @@ -28,6 +30,23 @@ def test_get_recursive_build_depends_in_workspace_with_test_depend(): assert r == ordered_packages[1:], r +def test_get_recursive_build_depends_in_workspace_with_condition(): + pkg = MockPackage('pkg') + cond_false_pkg = MockPackage('cond_false_pkg') + cond_false_pkg.evaluated_condition = False + cond_true_pkg = MockPackage('cond_true_pkg') + pkg.build_depends = [cond_true_pkg, cond_false_pkg] + + ordered_packages = [ + ('/path/to/pkg', pkg), + ('/path/to/cond_false_pkg', cond_false_pkg), + ('/path/to/cond_true_pkg', cond_true_pkg), + ] + + r = common.get_recursive_build_depends_in_workspace(pkg, ordered_packages) + assert r == ordered_packages[2:], r + + def test_get_recursive_build_depends_in_workspace_circular_run_depend(): pkg1 = MockPackage('pkg1') pkg2 = MockPackage('pkg2') diff --git a/tests/unit/test_io.py b/tests/unit/test_io.py index ff03da80..00a2df87 100644 --- a/tests/unit/test_io.py +++ b/tests/unit/test_io.py @@ -1,11 +1,6 @@ -try: - # Python3 - from queue import Queue -except ImportError: - # Python2 - from Queue import Queue - import binascii +from queue import Queue + from catkin_tools.execution import io as io diff --git a/tests/unit/test_jobs.py b/tests/unit/test_jobs.py index 94b5e760..4d99342a 100644 --- a/tests/unit/test_jobs.py +++ b/tests/unit/test_jobs.py @@ -2,11 +2,11 @@ def test_merge_envs_basic(): - job_env = { 'PATH': '/usr/local/bin:/usr/bin', 'FOO': 'foo' } + job_env = {'PATH': '/usr/local/bin:/usr/bin', 'FOO': 'foo'} merge_envs(job_env, [ - { 'PATH': '/usr/local/bin:/bar/baz/bin' }, - { 'BAR': 'bar' } ]) + {'PATH': '/usr/local/bin:/bar/baz/bin'}, + {'BAR': 'bar'}]) # Validate that the known path was not moved from the existing order, and the unfamiliar # path was correctly prepended. @@ -20,14 +20,14 @@ def test_merge_envs_basic(): def test_merge_envs_complex(): - ''' Confirm that merged paths are deduplicated and that order is maintained. ''' - job_env = { 'PATH': 'C:B:A' } - merge_envs(job_env, [{ 'PATH': 'D:C' }, { 'PATH': 'E:A:C' }]) + """Confirm that merged paths are deduplicated and that order is maintained.""" + job_env = {'PATH': 'C:B:A'} + merge_envs(job_env, [{'PATH': 'D:C'}, {'PATH': 'E:A:C'}]) assert job_env['PATH'] == 'E:D:C:B:A', job_env['PATH'] def test_merge_envs_nonpaths(): - ''' Confirm that non-path vars are simply overwritten on a last-wins policy. ''' - job_env = { 'FOO': 'foo:bar' } - merge_envs(job_env, [{ 'FOO': 'bar:baz' }, { 'FOO': 'baz:bar' }]) + """Confirm that non-path vars are simply overwritten on a last-wins policy.""" + job_env = {'FOO': 'foo:bar'} + merge_envs(job_env, [{'FOO': 'bar:baz'}, {'FOO': 'baz:bar'}]) assert job_env['FOO'] == 'baz:bar' diff --git a/tests/utils.py b/tests/utils.py index ab9080f3..21781bc3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,28 +1,15 @@ -from __future__ import print_function - import functools import os import re import shutil +import subprocess import sys import tempfile - -import subprocess +from io import StringIO +from subprocess import TimeoutExpired from catkin_tools.commands.catkin import main as catkin_main -try: - # Python2 - from StringIO import StringIO -except ImportError: - # Python3 - from io import StringIO - -try: - from subprocess import TimeoutExpired -except ImportError: - class TimeoutExpired(object): - pass TESTS_DIR = os.path.dirname(__file__) MOCK_DIR = os.path.join(TESTS_DIR, 'mock_resources') diff --git a/tests/workspace_assertions.py b/tests/workspace_assertions.py index 2ae912fe..3c190a91 100644 --- a/tests/workspace_assertions.py +++ b/tests/workspace_assertions.py @@ -1,6 +1,6 @@ -from __future__ import print_function - +import os import re +import yaml from .utils import assert_files_exist @@ -31,3 +31,10 @@ def assert_no_warnings(out_str): out_str_stripped = ' '.join(str(out_str).splitlines()) found = re.findall('WARNING:', out_str_stripped) assert len(found) == 0 + + +def assert_in_config(workspace, profile, key, value): + with open(os.path.join(workspace, '.catkin_tools', 'profiles', profile, 'config.yaml')) as f: + config = yaml.safe_load(f) + + assert config.get(key, None) == value