diff --git a/CHANGELOG b/CHANGELOG index e5c940d..13335fd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,8 @@ -21f000c (HEAD, tag: 0.2.1, origin/master, origin/HEAD, master, logger-fix) Update README.md +f67b4bf (HEAD, tag: 0.2.3, origin/master, origin/HEAD, master, allow-passing-arbitrary-pip-arguments) Merge pull request #9 from cloudify-cosmo/fix-requirement-files-not-being-dealt-with-correctly +8d87e55 (origin/fix-requirement-files-not-being-dealt-with-correctly, fix-requirement-files-not-being-dealt-with-correctly) fixed failure when trying to wheel a package that has dependencies in a requirements file +5e92335 (tag: 0.2.2) Merge pull request #7 from cloudify-cosmo/logger-fix +7d60140 (origin/logger-fix, logger-fix) changed module to package and now no longer overriding user's logger +21f000c (tag: 0.2.1) Update README.md e8b78eb Update README.md 0a8a55d Update README.md 95b384b Merge pull request #6 from cloudify-cosmo/CFY-3709-add-module-exclusion diff --git a/README.md b/README.md index d98a1f7..d95600f 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ wagon create -s cloudify-script-plugin==1.2 --keep-wheels -v --exclude cloudify- wagon create -s http://github.com/cloudify-cosmo/cloudify-script-plugin/archive/1.2.tar.gz -r . --validate # create an archive by retrieving the source from a local path and output the tar.gz file to /tmp/.tar.gz (defaults to /.tar.gz) and provides explicit Python versions supported by the package (which usually defaults to the first two digits of the Python version used to create the archive.) wagon create -s ~/packages/cloudify-script-plugin/ -o /tmp/ --pyver 33 --pyver 26 --pyver 27 +# pass additional args to `pip wheel` (NOTE that conflicting arguments are not handled by wagon.) +wagon create -s cloudify-script-plugin==1.2 -a '--retries 5' ``` Regarding exclusions, note that excluding packages can result in an archive being non-installable. The user will be warned about this but creation will succeed. Creation validation, though (i.e. using the `--validate` flag), will fail and show an error incase the archive cannot be installed. @@ -57,6 +59,8 @@ wagon install --help wagon install -s ~/tars/cloudify_script_plugin-1.2-py27-none-any.tar.gz --upgrade --ignore-platform # install a package from a url into an existing virtualenv. wagon install -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.tar.gz --virtualenv my_venv -v +# pass additional args to `pip install` (NOTE that conflicting arguments are not handled by wagon.) +wagon create -s cloudify-script-plugin==1.2 -a '--no-cache-dir' ``` Note that `--pre` is appended to the installation command to enable installation of prerelease versions. diff --git a/README.rst b/README.rst index a313831..27f5a01 100644 --- a/README.rst +++ b/README.rst @@ -43,19 +43,29 @@ Examples .. code:: shell - # create an archive by retrieving the source from PyPI and keep the downloaded wheels (kept under /plugin) and exclude the cloudify-plugins-common and cloudify-rest-client modules from the archive. + # create an archive by retrieving the source from PyPI and keep the downloaded wheels (kept under /plugin) and exclude the cloudify-plugins-common and cloudify-rest-client packages from the archive. wagon create -s cloudify-script-plugin==1.2 --keep-wheels -v --exclude cloudify-plugins-common --exclude cloudify-rest-client # create an archive by retrieving the source from a URL and creating wheels from requirement files found within the archive. Then, validation of the archive takes place. wagon create -s http://github.com/cloudify-cosmo/cloudify-script-plugin/archive/1.2.tar.gz -r . --validate - # create an archive by retrieving the source from a local path and output the tar.gz file to /tmp/.tar.gz (defaults to /.tar.gz) and provides explicit Python versions supported by the module (which usually defaults to the first two digits of the Python version used to create the archive.) - wagon create -s ~/modules/cloudify-script-plugin/ -o /tmp/ --pyver 33 --pyver 26 --pyver 27 + # create an archive by retrieving the source from a local path and output the tar.gz file to /tmp/.tar.gz (defaults to /.tar.gz) and provides explicit Python versions supported by the package (which usually defaults to the first two digits of the Python version used to create the archive.) + wagon create -s ~/packages/cloudify-script-plugin/ -o /tmp/ --pyver 33 --pyver 26 --pyver 27 + # pass additional args to `pip wheel` + wagon create -s cloudify-script-plugin==1.2 -a '--retries 5' -Regarding exclusions, note that excluding modules can result in an +Regarding exclusions, note that excluding packages can result in an archive being non-installable. The user will be warned about this but creation will succeed. Creation validation, though (i.e. using the ``--validate`` flag), will fail and show an error incase the archive cannot be installed. +Also note that Wagon doesn't currently provide a way for packaging +packages that are in editable mode. So, for instance, providing a +dev-requirements file which contains a ``-e DEPENDENCY`` requirement +will not be taken into consideration. This is not related to wagon but +rather to the default ``pip wheel`` implementation stating that it will +be "Skipping bdist\_wheel for #PACKAGE#, due to being editable". We +might allow processing editable provided dependencies in the future. + Install Packages ~~~~~~~~~~~~~~~~ @@ -68,17 +78,22 @@ Examples .. code:: shell - # install a module from a local archive tar file and upgrade if already installed. Also, ignore the platform check which would force a module (whether it is or isn't compiled for a specific platform) to be installed. + # install a package from a local archive tar file and upgrade if already installed. Also, ignore the platform check which would force a package (whether it is or isn't compiled for a specific platform) to be installed. wagon install -s ~/tars/cloudify_script_plugin-1.2-py27-none-any.tar.gz --upgrade --ignore-platform - # install a module from a url into an existing virtualenv. + # install a package from a url into an existing virtualenv. wagon install -s http://me.com/cloudify_script_plugin-1.2-py27-none-any-none-none.tar.gz --virtualenv my_venv -v + # pass additional args to `pip install` + wagon create -s cloudify-script-plugin==1.2 -a '--no-cache-dir' + +Note that ``--pre`` is appended to the installation command to enable +installation of prerelease versions. Installing Manually ^^^^^^^^^^^^^^^^^^^ While wagon provides a generic way of installing wagon created archives, you might not want to use the installer as you might not wish to install -wagon on your application servers. Installing the module manually via +wagon on your application servers. Installing the package manually via pip is as easy as running (for example): .. code:: shell @@ -95,8 +110,9 @@ Validate Packages The ``validate`` function provides shallow validation of a Wagon archive. Basically, it checks that some keys in the metadata are found, -that all required wheels for a module are present and that the module is -installable. It obviously does not check for a module's functionality. +that all required wheels for a package are present and that the package +is installable. It obviously does not check for a package's +functionality. This shallow validation should, at the very least, allow a user to be sure that a Wagon archive is not corrupted. @@ -137,7 +153,7 @@ Source: PyPI ~~~~~~~~~~~~ When providing a PyPI source, it must be supplied as -MODULE\_NAME==MODULE\_VERSION. wagon then applies the correct name and +PACKAGE\_NAME==PACKAGE\_VERSION. wagon then applies the correct name and version to the archive according to the two parameters. Source: Else @@ -162,15 +178,15 @@ this: :: { - "archive_name": "cloudify_script_plugin-1.2-py27-none-any-ubuntu-trusty.tar.gz", + "archive_name": "cloudify_script_plugin-1.2-py27-none-linux_x86_64-ubuntu-trusty.tar.gz", "build_server_os_properties": { "distribution": "ubuntu", "distribution_release": "trusty", "distribution_version": "14.04" }, - "module_name": "cloudify-script-plugin", - "module_source": "cloudify-script-plugin==1.2", - "module_version": "1.2", + "package_name": "cloudify-script-plugin", + "package_source": "cloudify-script-plugin==1.2", + "package_version": "1.2", "supported_platform": "any", "supported_python_versions": [ "py26", @@ -178,22 +194,28 @@ this: ], "wheels": [ "proxy_tools-0.1.0-py2-none-any.whl", + "pyzmq-14.7.0-cp27-none-linux_x86_64.whl", "bottle-0.12.7-py2-none-any.whl", "networkx-1.8.1-py2-none-any.whl", + "requests-2.5.1-py2.py3-none-any.whl", + "PyYAML-3.10-cp27-none-linux_x86_64.whl", "pika-0.9.13-py2-none-any.whl", - "cloudify_plugins_common-3.2.1-py2-none-any.whl", - "requests-2.7.0-py2.py3-none-any.whl", - "cloudify_rest_client-3.2.1-py2-none-any.whl", + "jsonschema-2.3.0-py2.py3-none-any.whl", + "cloudify_dsl_parser-3.2-py2-none-any.whl", + "cloudify_rest_client-3.2-py2-none-any.whl", "cloudify_script_plugin-1.2-py2-none-any.whl" + ], + "excluded_wheels": [ + "cloudify_plugins_common-3.2-py2-none-any.whl" ] } - The wheels to be installed reside in the tar.gz file under 'wheels/\*.whl'. -- The Metadata file resides in the tar.gz file under 'module.json'. +- The Metadata file resides in the tar.gz file under 'package.json'. - The installer uses the metadata file to check that the platform fits - the machine the module is being installed on. -- OS Properties only appear when creating compiled Linux modules (see + the machine the package is being installed on. +- OS Properties only appear when creating compiled Linux packages (see Linux Distributions section). In case of a non-linux platform (e.g. win32, any), null values will be supplied for OS properties. @@ -208,18 +230,21 @@ Example Output Archive: ``cloudify_fabric_plugin-1.2.1-py27-none-any-none-none.tar.gz`` - ``{python tag}``: The Python version is set by the Python running the - packaging process. That means that while a module might run on both + packaging process. That means that while a package might run on both py27 and py33 (for example), since the packaging process took place using Python 2.7, only py27 will be appended to the name. A user can - also explicitly provide the supported Python versions for the module + also explicitly provide the supported Python versions for the package via the ``pyver`` flag. - ``{platform tag}``: The platform (e.g. ``linux_x86_64``, ``win32``) - is set each specific wheel. To know which platform the module with + is set each specific wheel. To know which platform the package with its dependencies can be installed on, all wheels are checked. If a specific wheel has a platform property other than ``any``, that platform will be used as the platform of the package. Of course, we assume that there can't be wheels downloaded or created on a specific machine platform that belongs to two different platforms. +- ``{abi tag}``: Note that the ABI tag is currently ignored and will + always be ``none``. This might be changed in the future to support + providing an ABI tag. - For Linux (see below), two additional tags are added: ``{distribution tag}`` and ``{release tag}``. Note that these tags are NOT a part of the PEP. @@ -235,12 +260,12 @@ Linux are not uploaded to PyPI due to variations between compilation environments on different distributions and links to varying system libraries. -To overcome that (partially), if running Wagon on Linux and the module +To overcome that (partially), if running Wagon on Linux and the package requires compilation, the metadata and archive name both provide the distribution and release of the OS that the archive was created on (via platform.linux\_distribution()). Statistically speaking, this should provide the user with the information they need to know which OS the -module can be installed on. Obviously, this is not true for cases where +package can be installed on. Obviously, this is not true for cases where non-generic compilation methods are used on the creating OS but otherwise should work, and should specifically always work when both compilation environment and Python version are similar on the creating @@ -248,13 +273,13 @@ and installing OS - which, we generally recommend. What this practically means, is that in most cases, using the metadata to compare the distro, release and the Python version under which the -module is installed would allow a user to use Wagon rather safely. Of +package is installed would allow a user to use Wagon rather safely. Of course, Wagon provides no guarantee whatsoever as to whether this will actually work or not and users must test their archives. That being said, Wagon is completely safe for creating and installing -Pure Python module archives for any platform, and, due to the nature of -Wheels, modules compiled for OS X or Windows. +Pure Python package archives for any platform, and, due to the nature of +Wheels, packages compiled for OS X or Windows. Testing ------- diff --git a/TODO.md b/TODO.md index 832890a..bca32e5 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,5 @@ -- wagon/wagon.py:360: # TODO: maybe we don't want to be that explicit and allow using >= -- wagon/wagon.py:361: # TODO: or just a package name... -- wagon/wagon.py:436: # TODO: Let the user provide supported Python versions. -- wagon/wagon.py:437: # TODO: Let the user provide supported Architectures. +- wagon/wagon.py:361: # TODO: maybe we don't want to be that explicit and allow using >= +- wagon/wagon.py:362: # TODO: or just a package name... +- wagon/wagon.py:440: # TODO: Let the user provide supported Python versions. +- wagon/wagon.py:441: # TODO: Let the user provide supported Architectures. - wagon/utils.py:48: # TODO: implement using sh diff --git a/setup.py b/setup.py index ac3e555..c321ff2 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def read(*parts): setup( name='wagon', - version='0.2.3', + version='0.2.4', url='https://github.com/cloudify-cosmo/wagon', author='Gigaspaces', author_email='cosmo-admin@gigaspaces.com', diff --git a/wagon/tests/test_wagon.py b/wagon/tests/test_wagon.py index 47f9cdf..9d397dd 100644 --- a/wagon/tests/test_wagon.py +++ b/wagon/tests/test_wagon.py @@ -211,6 +211,22 @@ def test_create_archive_from_pypi(self): m = self._test() self.assertEqual(m['package_source'], TEST_PACKAGE) + def test_create_archive_from_pypi_with_additional_wheel_args(self): + fd, reqs_file_path = tempfile.mkstemp() + os.write(fd, 'virtualenv==13.1.2') + params = { + '-s': TEST_PACKAGE, + '-v': None, + '-f': None, + '-a': '-r {0}'.format(reqs_file_path) + } + result = _invoke_click('create', params) + self.assertEqual(str(result), '') + m = self._test() + self.assertEqual(m['package_source'], TEST_PACKAGE) + self.assertIn('virtualenv-13.1.2-py2.py3-none-any.whl', m['wheels']) + os.close(fd) + def test_create_archive_from_url_with_requirements(self): self.wagon.platform = utils.get_machine_platform() self.archive_name = self.wagon.set_archive_name( diff --git a/wagon/utils.py b/wagon/utils.py index 129f9d9..5267b17 100644 --- a/wagon/utils.py +++ b/wagon/utils.py @@ -75,7 +75,9 @@ def run(cmd, suppress_errors=False, suppress_output=False): def wheel(package, requirement_files=False, wheels_path='package', - excluded_packages=None): + excluded_packages=None, wheel_args=None): + # wheel_args = wheel_args or [] + lgr.info('Downloading Wheels for {0}...'.format(package)) wheel_cmd = ['pip', 'wheel'] wheel_cmd.append('--wheel-dir={0}'.format(wheels_path)) @@ -89,6 +91,8 @@ def wheel(package, requirement_files=False, wheels_path='package', lgr.error('Could not download wheels for: {0}. ' 'Please verify that the file you are trying ' 'to wheel is wheelable.'.format(req_file)) + if wheel_args: + wheel_cmd.append(wheel_args) wheel_cmd.append(package) p = run(' '.join(wheel_cmd)) if not p.returncode == 0: @@ -118,7 +122,8 @@ def get_wheel_for_package(wheels_path, package): def install_package(package, wheels_path, virtualenv_path=None, - requirements_file=None, upgrade=False): + requirements_file=None, upgrade=False, + install_args=None): """This will install a Python package. Can specify a specific version. @@ -128,6 +133,8 @@ def install_package(package, wheels_path, virtualenv_path=None, Can specify a local wheels_path to use for offline installation. Can request an upgrade. """ + # install_args = install_args or [] + lgr.info('Installing {0}...'.format(package)) pip_cmd = ['pip', 'install'] @@ -136,6 +143,8 @@ def install_package(package, wheels_path, virtualenv_path=None, _get_env_bin_path(virtualenv_path), pip_cmd[0]) if requirements_file: pip_cmd.extend(['-r', requirements_file]) + if install_args: + pip_cmd.append(install_args) pip_cmd.append(package) pip_cmd.extend(['--use-wheel', '--no-index', '--find-links', wheels_path]) # pre allows installing both prereleases and regular releases depending diff --git a/wagon/wagon.py b/wagon/wagon.py index b7e9df0..ee21d13 100644 --- a/wagon/wagon.py +++ b/wagon/wagon.py @@ -47,7 +47,7 @@ def __init__(self, source, verbose=False): def create(self, with_requirements=None, force=False, keep_wheels=False, excluded_packages=None, archive_destination_dir='.', python_versions=None, - validate=False): + validate=False, wheel_args=None): """Creates a Wagon archive and returns its path. This currently only creates tar.gz archives. The `install` @@ -93,7 +93,8 @@ def create(self, with_requirements=None, force=False, with_requirements = [with_requirements] wheels, excluded_wheels = utils.wheel( - source, with_requirements, wheels_path, excluded_packages) + source, with_requirements, wheels_path, excluded_packages, + wheel_args) self.platform = utils.get_platform_for_set_of_wheels(wheels_path) if python_versions: self.python_versions = ['py{0}'.format(v) for v in python_versions] @@ -119,7 +120,7 @@ def create(self, with_requirements=None, force=False, return archive_path def install(self, virtualenv=None, requirements_file=None, upgrade=False, - ignore_platform=False): + ignore_platform=False, install_args=None): """Installs a Wagon archive. This can install in a provided `virtualenv` or in the current @@ -149,7 +150,7 @@ def install(self, virtualenv=None, requirements_file=None, upgrade=False, wheels_path = os.path.join(source, DEFAULT_WHEELS_PATH) utils.install_package( metadata['package_name'], wheels_path, virtualenv, - requirements_file, upgrade) + requirements_file, upgrade, install_args) def validate(self): """Validates a Wagon archive. Return True if succeeds, False otherwise. @@ -414,9 +415,12 @@ def main(): 'This argument can be provided multiple times.') @click.option('--validate', default=False, is_flag=True, help='Runs a postcreation validation on the archive.') +@click.option('-a', '--wheel-args', required=False, + help='Allows to pass additional arguments to `pip wheel`. ' + '(e.g. --no-cache-dir -c constains.txt') @click.option('-v', '--verbose', default=False, is_flag=True) def create(source, with_requirements, force, keep_wheels, exclude, - output_directory, pyver, validate, verbose): + output_directory, pyver, validate, wheel_args, verbose): """Creates a Python package's wheel base archive. \b @@ -439,7 +443,7 @@ def create(source, with_requirements, force, keep_wheels, exclude, packager = Wagon(source, verbose) packager.create( with_requirements, force, keep_wheels, exclude, output_directory, - pyver, validate) + pyver, validate, wheel_args) @click.command() @@ -453,14 +457,18 @@ def create(source, with_requirements, force, keep_wheels, exclude, help='Upgrades the package if it is already installed.') @click.option('--ignore-platform', required=False, is_flag=True, help='Ignores supported platform check.') +@click.option('-a', '--install-args', required=False, + help='Allows to pass additional arguments to `pip install`. ' + '(e.g. -i my_pypi_index --retries 5') @click.option('-v', '--verbose', default=False, is_flag=True) def install(source, virtualenv, requirements_file, upgrade, - ignore_platform, verbose): + ignore_platform, install_args, verbose): """Installs a Wagon archive. """ logger.configure() installer = Wagon(source, verbose) - installer.install(virtualenv, requirements_file, upgrade, ignore_platform) + installer.install( + virtualenv, requirements_file, upgrade, ignore_platform, install_args) @click.command()