Merge branch 'release/0.3.0'

praekeltfoundation · Mar 22, 2017 · 5e8b9bc · 5e8b9bc
2 parents bc19f75 + cb9c8ba
commit 5e8b9bc
Show file tree

Hide file tree

Showing 3 changed files with 153 additions and 38 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,28 +1,43 @@
 language: python
-env:
-  global:
-    - SETUP_SCRIPT=setup-pyenv.sh
 matrix:
   include:
-    # Without PyPy
+    # Normal method of specifying Python version
     - python: '2.7'
-    # Basic options
+    # Use PYENV_VERSION with CPython, specific major.minor.change version
+    - env: PYENV_VERSION=3.6.0 PYENV_VERSION_STRING='Python 3.6.0'
+    # Use PYENV_VERSION with PyPy
     - python: pypy
-      env: PYENV_VERSION=pypy-5.4.1 PYENV_VERSION_STRING='PyPy 5.4.1'
-    # Custom options
+      env: PYENV_VERSION=pypy-portable-5.7.0 PYENV_VERSION_STRING='PyPy 5.7.0'
+    # Use PYENV_VERSION with PyPy and Custom options
     - python: pypy
-      env: PYENV_VERSION=pypy-5.4.1 PYENV_VERSION_STRING='PyPy 5.4.1' PYENV_ROOT=$HOME/.pyenv-pypy PYENV_RELEASE=v1.0.2 PYTHON_BUILD_CACHE=$HOME/.pyenv-pypy-cache
+      env: PYENV_VERSION=pypy-portable-5.6.0 PYENV_VERSION_STRING='PyPy 5.6.0' PYENV_ROOT=$HOME/.pyenv-pypy PYENV_RELEASE=v1.0.8 PYENV_CACHE_PATH=$HOME/.pyenv-pypy-cache
     # Legacy setup-pypy.sh
     - python: pypy
-      env: PYPY_VERSION=5.4.1 SETUP_SCRIPT=setup-pypy.sh
+      env: PYPY_VERSION=5.6.0 SETUP_SCRIPT=setup-pypy.sh
+    # macOS
+    # Note that the `python` key is *not* set and the language is overridden to
+    # *not* be `python`. If set to `python`, the build will fail, complaining
+    # that there's no Python tarball to download and install on macOS.
+    # See https://github.com/travis-ci/travis-ci/issues/2312 for details.
+    # So, it shows up as Ruby in the Travis UI but as the PYENV_VERSION_STRING
+    # check can attest, we are still fine.
+    - os: osx
+      language: ruby
+      env: PYENV_VERSION=3.6.0 PYENV_VERSION_STRING='Python 3.6.0'
+    # macOS Framework Build
+    # Useful for PyInstaller
+    # Example issue: https://github.com/pyenv/pyenv/issues/443
+    - os: osx
+      language: ruby
+      env: PYENV_VERSION=3.6.0 PYENV_VERSION_STRING='Python 3.6.0' PYTHON_CONFIGURE_OPTS="--enable-framework"
 
 cache:
   - pip
   - directories:
-    - $HOME/.pyenv_cache
+    - "${PYENV_CACHE_PATH:-$HOME/.pyenv_cache}"
 
 script:
-  - source $SETUP_SCRIPT
+  - source "${SETUP_SCRIPT:-setup-pyenv.sh}"
   - python --version
 
 deploy:

diff --git a/README.md b/README.md
@@ -1,14 +1,23 @@
 # travis-pyenv
+
+[![GitHub release](https://img.shields.io/github/release/praekeltfoundation/travis-pyenv.svg?style=flat-square)](https://github.com/praekeltfoundation/travis-pyenv/releases/latest)
+[![Build status](https://img.shields.io/travis/praekeltfoundation/travis-pyenv/develop.svg?style=flat-square)](https://travis-ci.org/praekeltfoundation/travis-pyenv)
+
 Set up [pyenv](https://github.com/yyuu/pyenv) to use in [Travis CI](https://travis-ci.org) builds.
 
 Setting up pyenv properly in a Travis CI build environment can be quite tricky. This repo contains a script ([`setup-pyenv.sh`](setup-pyenv.sh)) that makes this process much simpler.
 
-A common use case for this is to install an up-to-date version of [PyPy](http://pypy.org). The Travis CI build images currently contain a very old version of PyPy which breaks some common Python modules.
+Use cases for this include:
+
+* Install an up-to-date version of [PyPy](http://pypy.org). The Travis CI build images currently contain a very old version of PyPy which breaks some common Python modules.
+* Install an exact version of [CPython](http://www.python.org) or some other lesser-known distribution that Travis CI doesn't support.
+* Install Python on macOS builds.
 
 ## Usage
 1. Set the `$PYENV_VERSION` environment variable to the Python to install.
 2. Tell Travis to cache the `$HOME/.pyenv_cache` directory.
-3. Download and source the script in `before_install`.
+3. Download and source the `setup-pyenv.sh` script in `before_install`.
+4. Build your project and run your tests as usual.
 
 There are a few install options that can be set via environment variables:
 * `PYENV_VERSION`
@@ -19,28 +28,27 @@ There are a few install options that can be set via environment variables:
     Directory in which to install pyenv [default: `~/.pyenv`]
 * `PYENV_RELEASE`
     Release tag of pyenv to download [default: clone from master]
-* `PYTHON_BUILD_CACHE_PATH`
-    Directory in which to cache PyPy builds [default: `~/.pyenv_cache`]
+* `PYENV_CACHE_PATH`
+    Directory in which to cache pyenv's Python builds [default: `~/.pyenv_cache`]
 
 
 ### Example `travis.yml`
 ```yaml
 language: python
 matrix:
   include:
-    - python: '2.7'
+    - env: PYENV_VERSION='2.7.13' PYENV_VERSION_STRING='Python 2.7.13'
     - python: '3.5'
-    - python: pypy
-      env: PYENV_VERSION=pypy-5.4.1 PYENV_VERSION_STRING='PyPy 5.4.1'
+    - env: PYENV_VERSION=pypy-portable-5.7.0 PYENV_VERSION_STRING='PyPy 5.7.0'
 cache:
   - pip
   - directories:
-    - ~/.pyenv_cache
+    - $HOME/.pyenv_cache
 
 before_install:
   - |
       if [[ -n "$PYENV_VERSION" ]]; then
-        wget https://github.com/praekeltfoundation/travis-pyenv/releases/download/0.2.0/setup-pyenv.sh
+        wget https://github.com/praekeltfoundation/travis-pyenv/releases/download/0.3.0/setup-pyenv.sh
         source setup-pyenv.sh
       fi
 
@@ -50,4 +58,5 @@ script:
 
 ## Notes
 * Installing pyenv by downloading a release tag rather than cloning the git repo can make your builds a bit faster in some cases. Set the `PYENV_RELEASE` environment variable to achieve that.
-* pyenv fails to install properly if `~/.pyenv` is present, even if the directory is empty. So if you cache any directories within `~/.pyenv` then you will probably break pyenv.
+* If you want to use `$PYENV_CACHE_PATH`, you must also set up Travis to cache this directory in your Travis configuration. Using the cache is optional, but it can greatly speed up subsequent builds.
+* pyenv fails to install properly if the `$PYENV_ROOT` is already present, even if the directory is empty. So if you set Travis to cache any directories within the pyenv root, then you will probably break pyenv. For this reason, Python builds are cached outside the pyenv root and then linked after pyenv is installed.
diff --git a/setup-pyenv.sh b/setup-pyenv.sh
@@ -10,19 +10,79 @@
 #     Directory in which to install pyenv [default: ~/.pyenv]
 # - PYENV_RELEASE
 #     Release tag of pyenv to download [default: clone from master]
-# - PYTHON_BUILD_CACHE_PATH:
-#     Directory in which to cache PyPy builds [default: ~/.pyenv_cache]
+# - PYENV_CACHE_PATH
+#     Directory where full Python builds are cached (i.e., for Travis)
+
+PYENV_ROOT="${PYENV_ROOT:-$HOME/.pyenv}"
+PYENV_CACHE_PATH="${PYENV_CACHE_PATH:-$HOME/.pyenv_cache}"
+version_cache_path="$PYENV_CACHE_PATH/$PYENV_VERSION"
+version_pyenv_path="$PYENV_ROOT/versions/$PYENV_VERSION"
+
+# Functions
+#
+# verify_python -- attempts to call the Python command or binary
+# supplied in the first argument with the --version flag. If
+# PYENV_VERSION_STRING is set, then it validates the returned version string
+# as well (using fgrep). Returns whatever status code the command returns.
+verify_python() {
+  local python_bin="$1"; shift
+
+  if [[ -n "$PYENV_VERSION_STRING" ]]; then
+    "$python_bin" --version 2>&1 | fgrep "$PYENV_VERSION_STRING" &>/dev/null
+  else
+    "$python_bin" --version &>/dev/null
+  fi
+}
+
+# use_cached_python -- Tries symlinking to the cached PYENV_VERSION and
+# verifying that it's a working build. Returns 0 if it's found and it
+# verifies, otherwise returns 1.
+use_cached_python() {
+  if [[ -d "$version_cache_path" ]]; then
+    printf "Cached python found, $PYENV_VERSION. Verifying..."
+    ln -s "$version_cache_path" "$version_pyenv_path"
+    if verify_python "$version_pyenv_path/bin/python"; then
+      printf "success!\n"
+      return 0
+    else
+      printf "FAILED.\nClearing cached version..."
+      rm -f "$version_pyenv_path"
+      rm -rf "$version_cache_path"
+      printf "done.\n"
+      return 1
+    fi
+  else
+    echo "No cached python found."
+    return 1
+  fi
+}
+
+# output_debugging_info -- Outputs useful debugging information
+output_debugging_info() {
+  echo "**** Debugging information"
+  printf "PYENV_VERSION\n$PYENV_VERSION\n"
+  printf "PYENV_VERSION_STRING\n$PYENV_VERSION_STRING\n"
+  printf "PYENV_CACHE_PATH\n$PYENV_CACHE_PATH\n"
+  set -x
+  python --version
+  $version_cache_path/bin/python --version
+  which python
+  pyenv which python
+  set +x
+}
+
+# Main script begins.
 
 if [[ -z "$PYENV_VERSION" ]]; then
-  echo "\$PYENV_VERSION is not set. Not installing a pyenv."
+  echo "PYENV_VERSION is not set. Not installing a pyenv."
   return 0
 fi
 
-# Get out of the virtualenv we're in.
-deactivate
+# Get out of the virtualenv we're in (if we're in one).
+[[ -z "$VIRTUAL_ENV" ]] || deactivate
 
 # Install pyenv
-PYENV_ROOT="${PYENV_ROOT:-$HOME/.pyenv}"
+echo "**** Installing pyenv."
 if [[ -n "$PYENV_RELEASE" ]]; then
   # Fetch the release archive from Github (slightly faster than cloning)
   mkdir "$PYENV_ROOT"
@@ -37,21 +97,52 @@ export PATH="$PYENV_ROOT/bin:$PATH"
 eval "$(pyenv init -)"
 
 # Make sure the cache directory exists
-PYTHON_BUILD_CACHE_PATH="${PYTHON_BUILD_CACHE_PATH:-$HOME/.pyenv_cache}"
-mkdir -p "$PYTHON_BUILD_CACHE_PATH"
+mkdir -p "$PYENV_CACHE_PATH"
+
+# Try using an already cached PYENV_VERSION. If it fails or is not found,
+# then install from scratch.
+echo "**** Trying to find and use cached python $PYENV_VERSION."
+if ! use_cached_python; then
+  echo "**** Installing python $PYENV_VERSION with pyenv now."
+  if pyenv install "$PYENV_VERSION"; then
+    if mv "$version_pyenv_path" "$PYENV_CACHE_PATH"; then
+      echo "Python was successfully built and moved to cache."
+      echo "**** Trying to find and use cached python $PYENV_VERSION."
+      if ! use_cached_python; then
+        echo "Python version $PYENV_VERSION was apparently successfully built"
+        echo "with pyenv, but, once cached, it could not be verified."
+        output_debugging_info
+        return 1
+      fi
+    else
+      echo "**** Warning: Python was succesfully built, but moving to cache"
+      echo "failed. Proceeding anyway without caching."
+    fi
+  else
+    echo "Python version $PYENV_VERSION build FAILED."
+    return 1
+  fi
+fi
 
-# Install the pyenv
-pyenv install "$PYENV_VERSION"
+# Now we have to reinitialize pyenv, as we need the shims etc to be created so
+# the pyenv activates correctly.
+echo "**** Activating python $PYENV_VERSION and generating new virtualenv."
+eval "$(pyenv init -)"
 pyenv global "$PYENV_VERSION"
 
-# Make and source a new virtualenv
-VIRTUAL_ENV="$HOME/ve-pyenv-$PYENV_PYTHON"
+# Make sure virtualenv is installed and up-to-date...
+pip install -U virtualenv
+
+# Then make and source a new virtualenv
+VIRTUAL_ENV="$HOME/ve-pyenv-$PYENV_VERSION"
 virtualenv -p "$(which python)" "$VIRTUAL_ENV"
 source "$VIRTUAL_ENV/bin/activate"
 
-if [[ -n "$PYENV_VERSION_STRING" ]]; then
-  if ! python --version 2>&1 | fgrep "$PYENV_VERSION_STRING"; then
-    echo "Failed to verify that the pyenv was properly installed."
-    return 1
-  fi
+printf "One final verification that the virtualenv is working..."
+if verify_python "python"; then
+  printf "success!\n"
+else
+  printf "FAILED!\n"
+  output_debugging_info
+  return 1
 fi