From 6179ca17c42dc97927e3ced60b7f3907c993ee9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Thu, 25 Jan 2018 18:43:57 +0100 Subject: [PATCH] Update dependency pinnings to latest versions and enable circleci (#253) * Update pinnings to latest versions. * Update conda deps. * Fix conda dep. * Make test cases agnostic of used python version. * Fix version formatting. * Add circleci configuration. * Trigger build. * Trigger build * Include checkout in cache key. * Simplify test running. * Fix conda version. * Use latest available conda version (osx backlog). * Restore cache after checkout. * dito * Minor. * Index local channel. * Fix osx detection. * Remove travis-specific code. * Fix osx detection. Refactor output. * Debugging. * Always build the noarch index. * Use VM. * Fix osx test. * Fix env variable. * Capture output. * Increase no_output_timeout * Use SSH configured by circleci. * Fix docsource. * Refactoring. * Also install bioconda-utils for doc building. * Cleanup * Install bioconda-utils itself into dockerfile. * Cleanup conda caches. * Fixes. * Hide possible token exposures. * Trigger rebuild. --- build-docs.sh => .circleci/build-docs.sh | 51 ++-------- .circleci/config.yml | 95 ++++++++++++++++++ .circleci/setup.sh | 49 +++++++++ Dockerfile | 14 ++- .../bioconda_utils-requirements.txt | 37 ++++--- bioconda_utils/build.py | 6 +- bioconda_utils/cli.py | 4 +- bioconda_utils/docker_utils.py | 6 +- bioconda_utils/linting.py | 29 ++---- bioconda_utils/pkg_test.py | 4 +- bioconda_utils/upload.py | 11 +- bioconda_utils/utils.py | 30 ++++-- docs/key.enc | Bin 3248 -> 0 bytes setup-channels.sh | 21 ---- test/test_pkg_test.py | 4 +- test/test_utils.py | 14 +-- travis-run.sh | 25 ----- travis-setup.sh | 41 -------- 18 files changed, 230 insertions(+), 211 deletions(-) rename build-docs.sh => .circleci/build-docs.sh (57%) create mode 100644 .circleci/config.yml create mode 100755 .circleci/setup.sh delete mode 100644 docs/key.enc delete mode 100755 setup-channels.sh delete mode 100755 travis-run.sh delete mode 100755 travis-setup.sh diff --git a/build-docs.sh b/.circleci/build-docs.sh similarity index 57% rename from build-docs.sh rename to .circleci/build-docs.sh index 97e0cdf246..d06b93d9da 100755 --- a/build-docs.sh +++ b/.circleci/build-docs.sh @@ -19,28 +19,16 @@ BRANCH="master" ORIGIN="bioconda.github.io" GITHUB_USERNAME="bioconda" -HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - # DOCSOURCE is directory containing the Makefile, relative to the directory # containing this bash script. -DOCSOURCE=${HERE}/docs +DOCSOURCE=`pwd`/docs # DOCHTML is where sphinx is configured to save the output HTML -DOCHTML=${HERE}/docs/build/html +DOCHTML=$DOCSOURCE/build/html # tmpdir to which built docs will be copied STAGING=/tmp/${GITHUB_USERNAME}-docs -# The public key should have been added to the repo's settings in github; the -# private key should have been encrypted using `travis encrypt-file` and the -# encrypted version committed to the repo under $ENCRYPTED_FILE. -# -# ENCRYPTION_LABEL is from .travis.yml, and should have been edited to match -# the hash value reported by `travis encrypt-file`. -# -# See https://gist.github.com/domenic/ec8b0fc8ab45f39403dd for details -ENCRYPTED_FILE=${HERE}/docs/key.enc - # Build docs only if travis-ci is testing this branch: BUILD_DOCS_FROM_BRANCH="master" @@ -51,29 +39,12 @@ BUILD_DOCS_FROM_BRANCH="master" # # ---------------------------------------------------------------------------- -# Stop early (and descriptively) -if [[ $TRAVIS_OS_NAME != "linux" ]]; then - echo "OS is not Linux, so not building docs" +if [[ $CIRCLE_PROJECT_USERNAME != bioconda ]]; then + # exit if not in bioconda repo exit 0 fi - -if [[ $TRAVIS_REPO_SLUG == "bioconda/bioconda-utils" ]] \ -&& [[ $TRAVIS_PULL_REQUEST == "false" ]] \ -&& [[ $TRAVIS_BRANCH == $BUILD_DOCS_FROM_BRANCH ]]; then - # Decrypt and ssh-add key. - ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key" - ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv" - ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR} - ENCRYPTED_IV=${!ENCRYPTED_IV_VAR} - openssl aes-256-cbc -K $ENCRYPTED_KEY -iv $ENCRYPTED_IV -in $ENCRYPTED_FILE -out key -d - chmod 600 key - eval `ssh-agent -s` - ssh-add key - REPO="git@github.com:${GITHUB_USERNAME}/${ORIGIN}.git" -else - REPO="https://github.com/${GITHUB_USERNAME}/${ORIGIN}.git" -fi +REPO="git@github.com:${GITHUB_USERNAME}/${ORIGIN}.git" # clone the branch to tmpdir, clean out contents rm -rf $STAGING @@ -101,21 +72,11 @@ if git diff --quiet; then exit 0 fi -if [[ $TRAVIS_BRANCH != $BUILD_DOCS_FROM_BRANCH ]]; then +if [[ $CIRCLE_BRANCH != master ]]; then echo "Not pushing docs because not on branch '$BUILD_DOCS_FROM_BRANCH'" exit 0 fi -if [[ $TRAVIS_PULL_REQUEST != "false" ]]; then - echo "This is a pull request, so not pushing docs" - exit 0 -fi - -if [[ $TRAVIS_REPO_SLUG != "bioconda/bioconda-utils" ]]; then - echo "On a fork of the main bioconda-utils repo, so not pushing docs" - exit 0 -fi - # Add, commit, and push echo ".*" >> .gitignore diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..912fd507f8 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,95 @@ +version: 2 + +variables: + restore_cache: &restore_cache + restore_cache: + keys: + - bioconda-utils-{{ .Environment.MINICONDA_VER }}-{{ checksum ".circleci/setup.sh" }}-{{ arch }} + save_cache: &save_cache + save_cache: + key: bioconda-utils-{{ .Environment.MINICONDA_VER }}-{{ checksum ".circleci/setup.sh" }}-{{ arch }} + paths: + - /tmp/workspace/miniconda + setup: &setup + run: + name: Setup dependencies + command: .circleci/setup.sh + run: &run + run: + name: Run tests + command: .circleci/run.sh + macos: &macos + macos: + xcode: "8.3.3" + linux: &linux + machine: true + install_bioconda_utils: &install_bioconda_utils + run: + name: Install bioconda-utils + command: python setup.py install + + +jobs: + test-linux: + <<: *linux + steps: + - checkout + - *restore_cache + - *setup + - *save_cache + - *install_bioconda_utils + - run: + name: Testing + # --tb=line is important because py.test otherwise exposes tokens in case of errors + command: py.test test/ -v --tb=line + no_output_timeout: 1200 + test-macos: + <<: *macos + steps: + - checkout + - *setup + - *install_bioconda_utils + - run: + name: Testing + # --tb=line is important because py.test otherwise exposes tokens in case of errors + command: py.test test/ -v -k "not docker" --tb=line + no_output_timeout: 1200 + build-docs: + <<: *linux + steps: + - add_ssh_keys: + fingerprints: + - f8:26:86:86:f8:d3:a5:66:ea:7d:f6:42:2e:5c:7a:82 + - checkout + - *restore_cache + - *setup + - *save_cache + - *install_bioconda_utils + - run: + name: Build and upload docs + command: .circleci/build-docs.sh + + + +workflows: + version: 2 + # workflow for testing pushes and PRs + bioconda-utils-test: + jobs: + - test-linux: + context: org-global + - test-macos: + context: org-global + - build-docs: + context: org-global + bioconda-utils-nightly-docs: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + jobs: + - build-docs: + context: org-global diff --git a/.circleci/setup.sh b/.circleci/setup.sh new file mode 100755 index 0000000000..3d2158487d --- /dev/null +++ b/.circleci/setup.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -eu + +WORKSPACE=/tmp/workspace + +# Set path +echo "export PATH=$WORKSPACE/miniconda/bin:$PATH" >> $BASH_ENV + +if [[ ! -d $WORKSPACE/miniconda ]]; then + # setup conda and bioconda-utils if not loaded from cache + mkdir -p $WORKSPACE + + # step 1: download and install miniconda + if [[ $OSTYPE == darwin* ]]; then + tag="MacOSX" + elif [[ $OSTYPE == linux* ]]; then + tag="Linux" + else + echo "Unsupported OS: $OSTYPE" + exit 1 + fi + curl -L -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-$MINICONDA_VER-$tag-x86_64.sh + bash miniconda.sh -b -p $WORKSPACE/miniconda + source $BASH_ENV + + # step 2: setup channels + conda config --add channels defaults + conda config --add channels conda-forge + conda config --add channels bioconda + + # step 3: install bioconda-utils + conda install -y --file bioconda_utils/bioconda_utils-requirements.txt + + # step 4: cleanup + conda clean -y --all + + # Add local channel as highest priority + conda index $WORKSPACE/miniconda/conda-bld/linux-64 $WORKSPACE/miniconda/conda-bld/osx-64 $WORKSPACE/miniconda/conda-bld/noarch + conda config --add channels file://anaconda/conda-bld + + pip install -r pip-test-requirements.txt + pip install -r pip-requirements.txt +fi + +source $BASH_ENV +conda config --get + +ls $WORKSPACE/miniconda/conda-bld +ls $WORKSPACE/miniconda/conda-bld/noarch diff --git a/Dockerfile b/Dockerfile index 2aeb1ff062..fc7d72eed0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ FROM condaforge/linux-anvil -RUN /usr/bin/sudo -n yum install -y openssh-clients -COPY bioconda_utils/bioconda_utils-requirements.txt /tmp/requirements.txt -COPY setup-channels.sh /tmp/setup-channels.sh -RUN CONDA_ROOT=/opt/conda bash tmp/setup-channels.sh -RUN /opt/conda/bin/conda install --file /tmp/requirements.txt +ENV PATH="/opt/conda/bin:${PATH}" +RUN sudo -n yum install -y openssh-clients +ADD . /tmp/repo +RUN conda config --add channels defaults; \ + conda config --add channels conda-forge; \ + conda config --add channels bioconda +RUN conda install --file /tmp/repo/bioconda_utils/bioconda_utils-requirements.txt; \ + conda clean -y --all +RUN pip install /tmp/repo diff --git a/bioconda_utils/bioconda_utils-requirements.txt b/bioconda_utils/bioconda_utils-requirements.txt index 4feb63d53f..63a3bdee80 100644 --- a/bioconda_utils/bioconda_utils-requirements.txt +++ b/bioconda_utils/bioconda_utils-requirements.txt @@ -1,23 +1,22 @@ -anaconda-client=1.6.3 -argh=0.26.2 -beautifulsoup4=4.6.0 -conda=4.3.21 -conda-build=2.1.16 -galaxy-lib=17.9.7 -jinja2=2.9.6 -jsonschema=2.5.1 +anaconda-client=1.6.* +argh=0.26.* +beautifulsoup4=4.6.* +conda=4.3.29 +conda-build=2.1.18 +galaxy-lib=17.9.* +jinja2=2.10.* +jsonschema=2.6.* networkx=1.11 -pyaml=15.8.2 -pydotplus=2.0.2 -python=3.5.2 -requests=2.12.4 +pyaml=17.12.* +pydotplus=2.0.* +requests=2.18.* sphinx docutils sphinx_rtd_theme -involucro=1.1.2 -pandas=0.19.2 -numpy=1.12.1 -pygithub=1.29 -colorlog=2.10.* -six=1.10.0 -alabaster=0.7.9 +involucro=1.1.* +pandas=0.22.* +numpy=1.14.* +pygithub=1.34.* +colorlog=3.1.* +six=1.11.* +alabaster=0.7.* diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 188511f120..5c672ffed0 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -15,12 +15,12 @@ def purge(): - utils.run(["conda", "build", "purge"]) + utils.run(["conda", "build", "purge"], mask=False) free = utils.get_free_space() if free < 10: logger.info("CLEANING UP PACKAGE CACHE (free space: %iMB).", free) - utils.run(["conda", "clean", "--all"]) + utils.run(["conda", "clean", "--all"], mask=False) logger.info("CLEANED UP PACKAGE CACHE (free space: %iMB).", utils.get_free_space()) @@ -140,7 +140,7 @@ def build( [os.path.join(recipe, 'meta.yaml')] logger.debug('command: %s', cmd) with utils.Progress(): - p = utils.run(cmd, env=os.environ) + p = utils.run(cmd, env=os.environ, mask=False) build_success = True diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 49b72f2f4a..20c354db86 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -130,7 +130,7 @@ def remove_package(spec): token = [] else: token = ['-t', token] - print(utils.run([utils.bin_for('anaconda')] + token + subcmd).stdout) + print(utils.run([utils.bin_for('anaconda')] + token + subcmd, mask=[token]).stdout) def get_packages(channel): return {get_spec(pkg) @@ -184,7 +184,7 @@ def get_packages(channel): be sent to the current commit on github. Also needs --user and --repo to be set. Requires the env var GITHUB_TOKEN to be set. Note that pull requests from forks will not have access to encrypted variables on - travis-ci, so this feature may be of limited use.''') + ci, so this feature may be of limited use.''') @arg('--commit', help='Commit on github on which to update status') @arg('--push-comment', action='store_true', help='''If set, the lint status will be posted as a comment in the corresponding pull request (given by diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index ccea2f4c22..4e79710ed9 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -391,7 +391,7 @@ def _build_image(self): try: with utils.Progress(): - p = utils.run(cmd) + p = utils.run(cmd, mask=False) except sp.CalledProcessError as e: logger.error( 'DOCKER FAILED: Error building docker container %s. ', @@ -467,9 +467,9 @@ def build_recipe(self, recipe_dir, build_args, env, pkg, noarch=False): logger.debug('DOCKER: cmd: %s', cmd) with utils.Progress(): - p = utils.run(cmd) + p = utils.run(cmd, mask=False) return p def cleanup(self): cmd = ['docker', 'rmi', self.tag] - utils.run(cmd) + utils.run(cmd, mask=False) diff --git a/bioconda_utils/linting.py b/bioconda_utils/linting.py index 5227706daf..d15f76dc7a 100644 --- a/bioconda_utils/linting.py +++ b/bioconda_utils/linting.py @@ -170,7 +170,7 @@ def lint(recipes, config, df, exclude=None, registry=None): exclude : list List of function names in `registry` to skip globally. When running on - travis, this will be merged with anything else detected from the commit + CI, this will be merged with anything else detected from the commit message or LINT_SKIP environment variable using the special string "[skip lint for ]". While those other mechanisms define skipping on a recipe-specific basis, this argument @@ -186,28 +186,15 @@ def lint(recipes, config, df, exclude=None, registry=None): skip_dict = defaultdict(list) - # We want to get the commit message of HEAD to see if we should skip any - # linting tests. However, for PRs, travis tests the merge from PR to - # master. This means that we can't rely on "TRAVIS_COMMIT_MESSAGE" env var - # since, for PRs, it will be "merge $hash into $hash". - # - # For PRs, we need TRAVIS_PULL_REQUEST_SHA - # - # If not on travis, then don't look for any commit messages. commit_message = "" - - on_travis = os.environ.get('TRAVIS') == 'true' - pull_request = os.environ.get('TRAVIS_PULL_REQUEST', 'false') != 'false' - - if not on_travis and 'LINT_SKIP' in os.environ: + if 'LINT_SKIP' in os.environ: + # Allow overwriting of commit message commit_message = os.environ['LINT_SKIP'] - - if on_travis and pull_request: - p = utils.run( - ['git', 'log', '--format=%B', '-n', '1', - os.environ['TRAVIS_PULL_REQUEST_SHA']] - ) - commit_message = p.stdout + else: + # Obtain commit message from last commit. + commit_message = utils.run( + ['git', 'log', '--format=%B', '-n', '1'], mask=False + ).stdout # For example the following text in the commit message will skip # lint_functions.uses_setuptools for recipe argparse: diff --git a/bioconda_utils/pkg_test.py b/bioconda_utils/pkg_test.py index 18af573b33..83e7b83442 100644 --- a/bioconda_utils/pkg_test.py +++ b/bioconda_utils/pkg_test.py @@ -112,6 +112,8 @@ def test_package( conda_bld_dir = os.path.abspath(os.path.dirname(os.path.dirname(path))) sp.check_call([utils.bin_for('conda'), 'index', os.path.dirname(path)]) + # always build noarch index to make conda happy + sp.check_call([utils.bin_for('conda'), 'index', os.path.join(conda_bld_dir, "noarch")]) spec = get_image_name(path) @@ -144,6 +146,6 @@ def test_package( env["DEST_BASE_IMAGE"] = base_image with tempfile.TemporaryDirectory() as d: with utils.Progress(): - p = utils.run(cmd, env=env, cwd=d) + p = utils.run(cmd, env=env, cwd=d, mask=False) return p diff --git a/bioconda_utils/upload.py b/bioconda_utils/upload.py index 1249fc5415..6e52e26416 100644 --- a/bioconda_utils/upload.py +++ b/bioconda_utils/upload.py @@ -40,7 +40,7 @@ def anaconda_upload(package, token=None, label=None): logger.info("UPLOAD uploading package %s", package) try: cmds = ["anaconda", "-t", token, 'upload', package] + label_arg - p = utils.run(cmds) + p = utils.run(cmds, mask=[token]) logger.info("UPLOAD SUCCESS: uploaded package %s", package) return True @@ -53,8 +53,6 @@ def anaconda_upload(package, token=None, label=None): "%s", e.stdout) return True else: - # to avoid broadcasting the token in logs - e.cmd = ' '.join(cmds).replace(token, '') logger.error('UPLOAD ERROR: command: %s', e.cmd) logger.error('UPLOAD ERROR: stdout+stderr: %s', e.stdout) raise e @@ -65,6 +63,9 @@ def mulled_upload(image, quay_target): Upload the build Docker images to quay.io with 'mulled-build push' """ cmd = ['mulled-build', 'push', image, '-n', quay_target] + mask = [] if os.environ.get('QUAY_OAUTH_TOKEN', False): - cmd.extend(['--oauth-token', os.environ['QUAY_OAUTH_TOKEN']]) - return utils.run(cmd) + token = os.environ['QUAY_OAUTH_TOKEN'] + cmd.extend(['--oauth-token', token]) + mask = [token] + return utils.run(cmd, mask=mask) diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index 60189b8a77..a8ea9f4c13 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -202,7 +202,7 @@ def temp_os(platform): sys.platform = original -def run(cmds, env=None, **kwargs): +def run(cmds, env=None, mask=None, **kwargs): """ Wrapper around subprocess.run() @@ -221,8 +221,21 @@ def run(cmds, env=None, **kwargs): p.stdout = p.stdout.decode(errors='replace') except sp.CalledProcessError as e: e.stdout = e.stdout.decode(errors='replace') + # mask command arguments + def do_mask(arg): + if mask is None: + # caller has not considered masking, hide the entire command + # for security reasons + return '' + elif mask == False: + # masking has been deactivated + return arg + for m in mask: + arg = arg.replace(m, '') + return arg + e.cmd = [do_mask(c) for c in e.cmd] logger.error('COMMAND FAILED: %s', ' '.join(e.cmd)) - logger.error('STDOUT+STDERR:\n%s', e.stdout) + logger.error('STDOUT+STDERR:\n%s', do_mask(e.stdout)) raise e return p @@ -755,12 +768,11 @@ def tobuild(recipe, env): with temp_env(env): # with temp_os, we can fool the MetaData if needed. - platform = os.environ.get('TRAVIS_OS_NAME', sys.platform) - - # TRAVIS_OS_NAME uses 'osx', but sys.platform uses 'darwin', and - # that's what conda will be looking for. - if platform == 'osx': + platform = os.environ.get('OSTYPE', sys.platform) + if platform.startswith("darwin"): platform = 'darwin' + elif platform == "linux-gnu": + platform = "linux" with temp_os(platform): meta = MetaData(recipe) @@ -770,8 +782,8 @@ def tobuild(recipe, env): 'it defines skip for this env', pkg) return False - # If on travis, handle noarch. - if os.environ.get('TRAVIS', None) == 'true': + # If on CI, handle noarch. + if os.environ.get('CI', None) == 'true': if meta.get_value('build/noarch'): if platform != 'linux': logger.debug('FILTER: only building %s on ' diff --git a/docs/key.enc b/docs/key.enc deleted file mode 100644 index c9a5d89777989ea625df27d4e29c7601aaa861b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3248 zcmV;h3{Ug$%|rZpGM1i##EVJkhIGshz@7DV)IWixQ4>uxuvv8zhT&Ru^rx?mvBODE z)DfzO0^Mx&`DgwtGm7S9EIc1fo?h@b$?hub+m~GV#I0xD7Sv&L?#Gqs|8SR4elWlx zc5!#)Rt5*0s<&b`Mzh_i{V|8ITg+6ps%CHc{fK?V5Q`txvSa3c$Xnnq1;{js zR~<^r@5?!{-sGR<>AT&~vVmesuq+k~tO~*AzT@{qEUXv5(dbQBE8bZ?m+-aZzW#4H zfj_0O!Hw08&4LNI44^qc<1w=$*F6*aS{NuOJ_;9>Gh0P|r}hqLfXsa@PF7a|OuP>~ z;E}L%k~y7N5ON)92?7V3{e-kUgT}XrQ+Zk_zFTzPA_{y*peT6v1?3VSRxOKu+&WXB zafBYXyOuq&U!0cRurw~9H^pzZMsw*`gP^ePepn2MbZLv3pLpfZAx=~ff=D9l4(bf+ zEFr1tVuOHq86U^U-DnU!ekDM$FKRf+@RoX-@?i2PLRNG^EwBL9uZ(lp6 zbT&}3*RwVTZG2ZNwp<0y<@{#aSxYoS@%Xr~sj+9DR)Y#`_w2?lvn)lq;nDzr1yeXF zqS#1aNVh+ppda#El4X9Bd{HobTt6D5pc)FeE8E#u`;NIPx`mZIX%91a+BYj;*`nc~9$BI;=B7E=-SPQc-iMa!Gi zd*SG0nKAGb+lLkBh@zQD+~JHBk!EyktdqZnnB2tMo=f=0ef=x7m9N@{MdB*pHP2Yu zw#p>)xjM8-K*X-9)n$}lX92;Z&zCC;7&Ji(q`R;46*ldvpJu7^L2gt82^VZeS_QC- z(W8rt?Lyy43$!Br_t6Fwb0oWnJl_GB!7X=;A@8yDh3F41RBLR{H>bJ(=C({`Pa6@~ zNA5BgZ2Iq$cd-utNTwD1%$(>^u|6RBq#4&#a^nM`oQFQYq*%QwQV~t%2@3%g;j>TK z_CP7ADuF~u-d1c*OTJp@s5#QSlmr;byWofcub>G&>-G$g=rs<4Mpj*$K!vUphix-$ zIbyfK?x+}Ne86w%$+GOSP5jfgXqiKgp+_kMlb|1O&pfIr8NP?9C_pG|zyH9R2%AdP z!JwHd5={!e)^3;z>D(?W8GEQ<#x9RE#9vg=cuD-~`1H0YyEQ7Xi?%VyW8kb0g_{bO z0;Uy{64?1QFJ!Dl9!a0mvsYUr*alm4%*TwdI`>S>(s7iRY@NHyZth1W#M7*+OBOB7 z05AV6Fid5^YFOLaG5qMtDn>^u8s6Y+15Ak|1GumlXQVL3H(WRCD9^R)kW>l7z>T5!+_PWf zjjCQxC|A9?Gw$}v-WG+Xsy`k1u-{{ zD_)7>_;lPZL+%8Pj(oMX?L`jH0dRq44LI|3Q-I6*1o*sHXl@<6mKhwky}Z~$L_2;O z4z&ljO#y~Z*OoXxtzw0qL-_7n1zueH@0O1Eg~@OKK7^P(hElLc#Mdef{BjaTnxc=tn@ z^cYfhSotkn;-0?pTH5?Y7MSr5EeTBDx&k z=`AL9lkz?(wvpriWz7S0%U55`-?QFRM9f^IGBcaa)w0yD�>wgvkAO)`YRD*PSuO z#Aw01qR{e9PZ5lDsH%rkft0jfrl$n?VpjshF)c3{KrWW`CzdujG0z+q8n=j_!TgNy z8s5ge2%WIKU=kB&`wf=7`>v?Z_`=6iV+a5c%~EtM{EJ5^9YQhKU%?}D-ktu*@7x{U zmBR?HE2pSEjEWIE=05x~ctel_$b!dj1F~`kCc1}>ro-pg_YS`*yKK*x8FABvQOvpo2 z5H?}}?z>qBCqQe{tC<{PQkpkLd(W>zES&Y^7?R!T!LD7;*Z$HxPhwBgQKr7+v^I6r;}2n;}U!Z>iufU9m5>J zz^}wgG25OD$|7ks^|$aclc^{cprook)V?09zXFr3s3arF4~Vx;Mcq*o@tvfz2$uF3-6_Y{1!gU$aFn_QPnE8^+pEA@zFQC8^}6*A(_c7A3~t z|9i8}ukk;oEcG7eL)mhRvLKu)=2_AY?Nb~BK=^-%st5nxS(5A3FmkkT~X`-EIGlW(gT{V47peHL}L&|wBv|{jH>38^(WNrAV#Va;7MzQrEXo)-{aRg zw&=U?CU6(Krzr_l9eFDXOidB|VySEjplz+OKHx^Mv2hm-mc)Sztm9Xttgckwshxigf5kXRJZt~%{v=!D*$Qw3wxEI z7fnodpEi>6n#ktE{gD=x~9{N~v%{ic0EU!8HDBFsM}k8owBXxTR3PIG}XR4}lCglX!c zult`oR)U7>MX%0*o{oA@pS`>u_76Hnc<;pbdpJ3exwS*?Tg^Bb<65Nq zO~%UolVV%WpMx%Qu6@tUkqm~XOcbuv@oz%p3z@~ood%4xuEcf1BBKs%40SBzHh*tJ z@d$OitI5`xvZliN8WZpJ7f5E3l8KEa`Uv>HEIEG$ZDMX;faoD;;Jj<5s7KQm_tbka zyEADB;AkeN7KUQeC37}512^Ob#!u- z8yAax#PZ(TbL0+eu)tal+`b3>u|d7khc06{#&BWXidY2{dai8%)}ZMi&sx^fFOIp_ zNhCkgt~|lH*5Tbc1D2We1-9T~dKN51XYo0;oz#t-2l-rZbhV_F1_DMmN*C&+Tzv6A ixDw|J4`_QFu@MW}O!XOa?)^P3Ce??k{Vm0>eEV~!$z~-0 diff --git a/setup-channels.sh b/setup-channels.sh deleted file mode 100755 index 9fea50199d..0000000000 --- a/setup-channels.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -if [[ $1 == "-h" ]]; then - echo - echo " Sets the correct channel order required by bioconda." - echo - echo " Usage: $0 [optional path to conda]" - echo - exit 0 -fi - -CONDA_BIN=conda -if [[ ! -z $CONDA_ROOT ]]; then - CONDA_BIN=$CONDA_ROOT/bin/conda -fi - -echo "NOTE: messages like 'Warning: defaults already in channels list, moving to the top' are normal and can be ignored" -set -x -$CONDA_BIN config --add channels defaults -$CONDA_BIN config --add channels conda-forge -$CONDA_BIN config --add channels bioconda diff --git a/test/test_pkg_test.py b/test/test_pkg_test.py index d121a176ec..5c935196c4 100644 --- a/test/test_pkg_test.py +++ b/test/test_pkg_test.py @@ -1,4 +1,4 @@ -import os +import sys from textwrap import dedent import subprocess as sp @@ -15,7 +15,7 @@ # -SKIP_OSX = os.environ.get('TRAVIS_OS_NAME') == 'osx' +SKIP_OSX = sys.platform.startswith('darwin') RECIPE_ONE = dedent(""" diff --git a/test/test_utils.py b/test/test_utils.py index da6ff58ddd..1ae6bc9f7d 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,4 +1,5 @@ import os +import sys import subprocess as sp import pytest import yaml @@ -25,7 +26,7 @@ # Label that will be used for uploading test packages to anaconda/binstar TEST_LABEL = 'bioconda-utils-test' -SKIP_DOCKER_TESTS = os.environ.get('TRAVIS_OS_NAME') == 'osx' +SKIP_DOCKER_TESTS = sys.platform.startswith('darwin') if SKIP_DOCKER_TESTS: PARAMS = [False] @@ -155,11 +156,7 @@ def single_upload(): pkg = utils.built_package_path(r.recipe_dirs[name]) - with utils.temp_env(dict( - TRAVIS_BRANCH='master', - TRAVIS_PULL_REQUEST='false') - ): - upload.anaconda_upload(pkg, label=TEST_LABEL) + upload.anaconda_upload(pkg, label=TEST_LABEL) yield (name, pkg, r.recipe_dirs[name]) @@ -612,10 +609,9 @@ def test_built_package_path(): """, from_string=True) r.write_recipes() - # assumes we're running on py35 assert os.path.basename( utils.built_package_path(r.recipe_dirs['one']) - ) == 'one-0.1-py35_0.tar.bz2' + ) == 'one-0.1-py{ver.major}{ver.minor}_0.tar.bz2'.format(ver=sys.version_info) # resetting with a different CONDA_PY passed as env dict assert os.path.basename( @@ -692,7 +688,7 @@ def test_pkgname_with_numpy_x_x(): os.environ['CONDA_NPY'] = '1.9' assert os.path.basename( utils.built_package_path(r.recipe_dirs['one'], env=os.environ) - ) == 'one-0.1-np19py35_0.tar.bz2' + ) == 'one-0.1-np19py{ver.major}{ver.minor}_0.tar.bz2'.format(ver=sys.version_info) def test_string_or_float_to_integer_python(): diff --git a/travis-run.sh b/travis-run.sh deleted file mode 100755 index f29323f44e..0000000000 --- a/travis-run.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -set +u -[[ -z $BIOCONDA_UTILS_TEST_TYPE ]] && BIOCONDA_UTILS_TEST_TYPE="all" -set -u - -if [ $BIOCONDA_UTILS_TEST_TYPE == 'pytest_docker' ] || [ $BIOCONDA_UTILS_TEST_TYPE == 'all' ]; then - - py.test test/ -v -k "docker" -s - -fi - -if [ $BIOCONDA_UTILS_TEST_TYPE == 'pytest_not_docker' ] || [ $BIOCONDA_UTILS_TEST_TYPE == 'all' ]; then - - py.test test/ -v -k "not docker" - -fi - -if [ $BIOCONDA_UTILS_TEST_TYPE == 'docs' ] || [ $BIOCONDA_UTILS_TEST_TYPE == 'all' ]; then - - ./build-docs.sh 2>&1 | grep -v "nonlocal image URI found" | grep -v "reading sources..." | grep -v "writing output..." - -fi diff --git a/travis-setup.sh b/travis-setup.sh deleted file mode 100755 index 48b230f7ac..0000000000 --- a/travis-setup.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Sets up travis-ci environment for testing bioconda-utils. - -if [[ $TRAVIS_OS_NAME = "linux" ]] -then - tag=Linux -else - tag=MacOSX -fi - -curl -O https://repo.continuum.io/miniconda/Miniconda3-$MINICONDA_VER-$tag-x86_64.sh -sudo bash Miniconda3-$MINICONDA_VER-$tag-x86_64.sh -b -p /anaconda -sudo chown -R $USER /anaconda -export PATH=/anaconda/bin:$PATH - -# TODO: it would be nice to have a single location where channels are -# configured across bioconda-recipes and bioconda-utils. -conda config --add channels defaults -conda config --add channels conda-forge -conda config --add channels bioconda - -conda config --get -conda install -y --file bioconda_utils/bioconda_utils-requirements.txt -conda clean -y -tp - -python setup.py install - -pip install -r pip-test-requirements.txt -pip install -r pip-requirements.txt - -# Add local channel as highest priority -conda config --add channels file://anaconda/conda-bld -conda config --get - -# involucro used for mulled-build -curl -O https://github.com/involucro/involucro/releases/download/v1.1.2/involucro -sudo mv involucro /opt/involucro -sudo chmod +x /opt/involucro -export PATH=/opt/involucro:$PATH