Update dependency pinnings to latest versions and enable circleci (#253)

* Update pinnings to latest versions.

* Update conda deps.

* Fix conda dep.

* Make test cases agnostic of used python version.

* Fix version formatting.

* Add circleci configuration.

* Trigger build.

* Trigger build

* Include checkout in cache key.

* Simplify test running.

* Fix conda version.

* Use latest available conda version (osx backlog).

* Restore cache after checkout.

* dito

* Minor.

* Index local channel.

* Fix osx detection.

* Remove travis-specific code.

* Fix osx detection. Refactor output.

* Debugging.

* Always build the noarch index.

* Use VM.

* Fix osx test.

* Fix env variable.

* Capture output.

* Increase no_output_timeout

* Use SSH configured by circleci.

* Fix docsource.

* Refactoring.

* Also install bioconda-utils for doc building.

* Cleanup

* Install bioconda-utils itself into dockerfile.

* Cleanup conda caches.

* Fixes.

* Hide possible token exposures.

* Trigger rebuild.

build-docs.sh → .circleci/build-docs.sh

@@ -19,28 +19,16 @@ BRANCH="master"
 ORIGIN="bioconda.github.io"
 GITHUB_USERNAME="bioconda"
 
-HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
 # DOCSOURCE is directory containing the Makefile, relative to the directory
 # containing this bash script.
-DOCSOURCE=${HERE}/docs
+DOCSOURCE=`pwd`/docs
 
 # DOCHTML is where sphinx is configured to save the output HTML
-DOCHTML=${HERE}/docs/build/html
+DOCHTML=$DOCSOURCE/build/html
 
 # tmpdir to which built docs will be copied
 STAGING=/tmp/${GITHUB_USERNAME}-docs
 
-# The public key should have been added to the repo's settings in github; the
-# private key should have been encrypted using `travis encrypt-file` and the
-# encrypted version committed to the repo under $ENCRYPTED_FILE.
-#
-# ENCRYPTION_LABEL is from .travis.yml, and should have been edited to match
-# the hash value reported by `travis encrypt-file`.
-#
-# See https://gist.github.com/domenic/ec8b0fc8ab45f39403dd for details
-ENCRYPTED_FILE=${HERE}/docs/key.enc
-
 # Build docs only if travis-ci is testing this branch:
 BUILD_DOCS_FROM_BRANCH="master"
 
@@ -51,29 +39,12 @@ BUILD_DOCS_FROM_BRANCH="master"
 #
 # ----------------------------------------------------------------------------
 
-# Stop early (and descriptively)
-if [[ $TRAVIS_OS_NAME != "linux" ]]; then
-    echo "OS is not Linux, so not building docs"
+if [[ $CIRCLE_PROJECT_USERNAME != bioconda ]]; then
+    # exit if not in bioconda repo
     exit 0
 fi
 
-
-if [[ $TRAVIS_REPO_SLUG == "bioconda/bioconda-utils" ]] \
-&& [[ $TRAVIS_PULL_REQUEST == "false" ]] \
-&& [[ $TRAVIS_BRANCH == $BUILD_DOCS_FROM_BRANCH ]]; then
-    # Decrypt and ssh-add key.
-    ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
-    ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv"
-    ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
-    ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
-    openssl aes-256-cbc -K $ENCRYPTED_KEY -iv $ENCRYPTED_IV -in $ENCRYPTED_FILE -out key -d
-    chmod 600 key
-    eval `ssh-agent -s`
-    ssh-add key
-    REPO="git@github.com:${GITHUB_USERNAME}/${ORIGIN}.git"
-else
-    REPO="https://github.com/${GITHUB_USERNAME}/${ORIGIN}.git"
-fi
+REPO="git@github.com:${GITHUB_USERNAME}/${ORIGIN}.git"
 
 # clone the branch to tmpdir, clean out contents
 rm -rf $STAGING
@@ -101,21 +72,11 @@ if git diff --quiet; then
     exit 0
 fi
 
-if [[ $TRAVIS_BRANCH != $BUILD_DOCS_FROM_BRANCH ]]; then
+if [[ $CIRCLE_BRANCH != master ]]; then
     echo "Not pushing docs because not on branch '$BUILD_DOCS_FROM_BRANCH'"
     exit 0
 fi
 
-if [[ $TRAVIS_PULL_REQUEST != "false" ]]; then
-    echo "This is a pull request, so not pushing docs"
-    exit 0
-fi
-
-if [[ $TRAVIS_REPO_SLUG != "bioconda/bioconda-utils" ]]; then
-    echo "On a fork of the main bioconda-utils repo, so not pushing docs"
-    exit 0
-fi
-
 
 # Add, commit, and push
 echo ".*" >> .gitignore

.circleci/config.yml

@@ -0,0 +1,95 @@
+version: 2
+
+variables:
+  restore_cache: &restore_cache
+    restore_cache:
+      keys:
+        - bioconda-utils-{{ .Environment.MINICONDA_VER }}-{{ checksum ".circleci/setup.sh" }}-{{ arch }}
+  save_cache: &save_cache
+    save_cache:
+      key: bioconda-utils-{{ .Environment.MINICONDA_VER }}-{{ checksum ".circleci/setup.sh" }}-{{ arch }}
+      paths:
+        - /tmp/workspace/miniconda
+  setup: &setup
+    run:
+      name: Setup dependencies
+      command: .circleci/setup.sh
+  run: &run
+    run:
+      name: Run tests
+      command: .circleci/run.sh
+  macos: &macos
+    macos:
+      xcode: "8.3.3"
+  linux: &linux
+    machine: true
+  install_bioconda_utils: &install_bioconda_utils
+    run:
+      name: Install bioconda-utils
+      command: python setup.py install
+
+
+jobs:
+  test-linux:
+    <<: *linux
+    steps:
+      - checkout
+      - *restore_cache
+      - *setup
+      - *save_cache
+      - *install_bioconda_utils
+      - run:
+          name: Testing
+          # --tb=line is important because py.test otherwise exposes tokens in case of errors
+          command: py.test test/ -v --tb=line
+          no_output_timeout: 1200
+  test-macos:
+    <<: *macos
+    steps:
+      - checkout
+      - *setup
+      - *install_bioconda_utils
+      - run:
+          name: Testing
+          # --tb=line is important because py.test otherwise exposes tokens in case of errors
+          command: py.test test/ -v -k "not docker" --tb=line
+          no_output_timeout: 1200
+  build-docs:
+    <<: *linux
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - f8:26:86:86:f8:d3:a5:66:ea:7d:f6:42:2e:5c:7a:82
+      - checkout
+      - *restore_cache
+      - *setup
+      - *save_cache
+      - *install_bioconda_utils
+      - run:
+          name: Build and upload docs
+          command: .circleci/build-docs.sh
+
+
+
+workflows:
+  version: 2
+  # workflow for testing pushes and PRs
+  bioconda-utils-test:
+    jobs:
+      - test-linux:
+          context: org-global
+      - test-macos:
+          context: org-global
+      - build-docs:
+          context: org-global
+  bioconda-utils-nightly-docs:
+     triggers:
+       - schedule:
+           cron: "0 0 * * *"
+           filters:
+             branches:
+               only:
+                 - master
+     jobs:
+       - build-docs:
+           context: org-global

.circleci/setup.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -eu
+
+WORKSPACE=/tmp/workspace
+
+# Set path
+echo "export PATH=$WORKSPACE/miniconda/bin:$PATH" >> $BASH_ENV
+
+if [[ ! -d $WORKSPACE/miniconda ]]; then
+    # setup conda and bioconda-utils if not loaded from cache
+    mkdir -p $WORKSPACE
+
+    # step 1: download and install miniconda
+    if [[ $OSTYPE == darwin* ]]; then
+        tag="MacOSX"
+    elif [[ $OSTYPE == linux* ]]; then
+	tag="Linux"
+    else
+	echo "Unsupported OS: $OSTYPE"
+	exit 1
+    fi
+    curl -L -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-$MINICONDA_VER-$tag-x86_64.sh
+    bash miniconda.sh -b -p $WORKSPACE/miniconda
+    source $BASH_ENV
+
+    # step 2: setup channels
+    conda config --add channels defaults
+    conda config --add channels conda-forge
+    conda config --add channels bioconda
+
+    # step 3: install bioconda-utils
+    conda install -y --file bioconda_utils/bioconda_utils-requirements.txt
+
+    # step 4: cleanup
+    conda clean -y --all
+
+    # Add local channel as highest priority
+    conda index $WORKSPACE/miniconda/conda-bld/linux-64 $WORKSPACE/miniconda/conda-bld/osx-64 $WORKSPACE/miniconda/conda-bld/noarch
+    conda config --add channels file://anaconda/conda-bld
+
+    pip install -r pip-test-requirements.txt
+    pip install -r pip-requirements.txt
+fi
+
+source $BASH_ENV
+conda config --get
+
+ls $WORKSPACE/miniconda/conda-bld
+ls $WORKSPACE/miniconda/conda-bld/noarch

Dockerfile

@@ -1,6 +1,10 @@
 FROM condaforge/linux-anvil
-RUN /usr/bin/sudo -n yum install -y openssh-clients
-COPY bioconda_utils/bioconda_utils-requirements.txt /tmp/requirements.txt
-COPY setup-channels.sh /tmp/setup-channels.sh
-RUN CONDA_ROOT=/opt/conda bash tmp/setup-channels.sh
-RUN /opt/conda/bin/conda install --file /tmp/requirements.txt
+ENV PATH="/opt/conda/bin:${PATH}"
+RUN sudo -n yum install -y openssh-clients
+ADD . /tmp/repo
+RUN conda config --add channels defaults; \
+    conda config --add channels conda-forge; \
+    conda config --add channels bioconda
+RUN conda install --file /tmp/repo/bioconda_utils/bioconda_utils-requirements.txt; \
+    conda clean -y --all
+RUN pip install /tmp/repo

bioconda_utils/bioconda_utils-requirements.txt

@@ -1,23 +1,22 @@
-anaconda-client=1.6.3
-argh=0.26.2
-beautifulsoup4=4.6.0
-conda=4.3.21
-conda-build=2.1.16
-galaxy-lib=17.9.7
-jinja2=2.9.6
-jsonschema=2.5.1
+anaconda-client=1.6.*
+argh=0.26.*
+beautifulsoup4=4.6.*
+conda=4.3.29
+conda-build=2.1.18
+galaxy-lib=17.9.*
+jinja2=2.10.*
+jsonschema=2.6.*
 networkx=1.11
-pyaml=15.8.2
-pydotplus=2.0.2
-python=3.5.2
-requests=2.12.4
+pyaml=17.12.*
+pydotplus=2.0.*
+requests=2.18.*
 sphinx
 docutils
 sphinx_rtd_theme
-involucro=1.1.2
-pandas=0.19.2
-numpy=1.12.1
-pygithub=1.29
-colorlog=2.10.*
-six=1.10.0
-alabaster=0.7.9
+involucro=1.1.*
+pandas=0.22.*
+numpy=1.14.*
+pygithub=1.34.*
+colorlog=3.1.*
+six=1.11.*
+alabaster=0.7.*

bioconda_utils/build.py

@@ -15,12 +15,12 @@
 
 
 def purge():
-    utils.run(["conda", "build", "purge"])
+    utils.run(["conda", "build", "purge"], mask=False)
 
     free = utils.get_free_space()
     if free < 10:
         logger.info("CLEANING UP PACKAGE CACHE (free space: %iMB).", free)
-        utils.run(["conda", "clean", "--all"])
+        utils.run(["conda", "clean", "--all"], mask=False)
         logger.info("CLEANED UP PACKAGE CACHE (free space: %iMB).",
                     utils.get_free_space())
 
@@ -140,7 +140,7 @@ def build(
                       [os.path.join(recipe, 'meta.yaml')]
                 logger.debug('command: %s', cmd)
                 with utils.Progress():
-                    p = utils.run(cmd, env=os.environ)
+                    p = utils.run(cmd, env=os.environ, mask=False)
 
             build_success = True
 

bioconda_utils/cli.py

@@ -130,7 +130,7 @@ def remove_package(spec):
                 token = []
             else:
                 token = ['-t', token]
-            print(utils.run([utils.bin_for('anaconda')] + token + subcmd).stdout)
+            print(utils.run([utils.bin_for('anaconda')] + token + subcmd, mask=[token]).stdout)
 
     def get_packages(channel):
         return {get_spec(pkg)
@@ -184,7 +184,7 @@ def get_packages(channel):
      be sent to the current commit on github. Also needs --user and --repo to
      be set. Requires the env var GITHUB_TOKEN to be set. Note that pull
      requests from forks will not have access to encrypted variables on
-     travis-ci, so this feature may be of limited use.''')
+     ci, so this feature may be of limited use.''')
 @arg('--commit', help='Commit on github on which to update status')
 @arg('--push-comment', action='store_true', help='''If set, the lint status
      will be posted as a comment in the corresponding pull request (given by

bioconda_utils/docker_utils.py

@@ -391,7 +391,7 @@ def _build_image(self):
 
         try:
             with utils.Progress():
-                p = utils.run(cmd)
+                p = utils.run(cmd, mask=False)
         except sp.CalledProcessError as e:
             logger.error(
                 'DOCKER FAILED: Error building docker container %s. ',
@@ -467,9 +467,9 @@ def build_recipe(self, recipe_dir, build_args, env, pkg, noarch=False):
 
         logger.debug('DOCKER: cmd: %s', cmd)
         with utils.Progress():
-            p = utils.run(cmd)
+            p = utils.run(cmd, mask=False)
         return p
 
     def cleanup(self):
         cmd = ['docker', 'rmi', self.tag]
-        utils.run(cmd)
+        utils.run(cmd, mask=False)

bioconda_utils/linting.py

@@ -170,7 +170,7 @@ def lint(recipes, config, df, exclude=None, registry=None):
 
     exclude : list
         List of function names in `registry` to skip globally. When running on
-        travis, this will be merged with anything else detected from the commit
+        CI, this will be merged with anything else detected from the commit
         message or LINT_SKIP environment variable using the special string
         "[skip lint <function name> for <recipe name>]". While those other
         mechanisms define skipping on a recipe-specific basis, this argument
@@ -186,28 +186,15 @@ def lint(recipes, config, df, exclude=None, registry=None):
 
     skip_dict = defaultdict(list)
 
-    # We want to get the commit message of HEAD to see if we should skip any
-    # linting tests. However, for PRs, travis tests the merge from PR to
-    # master. This means that we can't rely on "TRAVIS_COMMIT_MESSAGE" env var
-    # since, for PRs, it will be "merge $hash into $hash".
-    #
-    # For PRs, we need TRAVIS_PULL_REQUEST_SHA
-    #
-    # If not on travis, then don't look for any commit messages.
     commit_message = ""
-
-    on_travis = os.environ.get('TRAVIS') == 'true'
-    pull_request = os.environ.get('TRAVIS_PULL_REQUEST', 'false') != 'false'
-
-    if not on_travis and 'LINT_SKIP' in os.environ:
+    if 'LINT_SKIP' in os.environ:
+        # Allow overwriting of commit message
         commit_message = os.environ['LINT_SKIP']
-
-    if on_travis and pull_request:
-        p = utils.run(
-            ['git', 'log', '--format=%B', '-n', '1',
-             os.environ['TRAVIS_PULL_REQUEST_SHA']]
-        )
-        commit_message = p.stdout
+    else:
+        # Obtain commit message from last commit.
+        commit_message = utils.run(
+            ['git', 'log', '--format=%B', '-n', '1'], mask=False
+        ).stdout
 
     # For example the following text in the commit message will skip
     # lint_functions.uses_setuptools for recipe argparse: