Merge pull request #629 from bioconda/build-container-if-conda-dependent

build_recipes: enable container tests for conda-dependent recipes

Author: mbargull

.circleci/config.yml

@@ -41,9 +41,10 @@ variables:
     run:
       name: Build the updated docker container
       command: |
-        docker build -t bioconda-utils-build-env:latest ./
-        docker history bioconda-utils-build-env:latest
-        docker run --rm -t bioconda-utils-build-env:latest sh -lec 'type -t conda && conda info -a && conda list'
+        docker build -t bioconda/bioconda-utils-build-env:latest ./
+        docker history bioconda/bioconda-utils-build-env:latest
+        docker run --rm -t bioconda/bioconda-utils-build-env:latest sh -lec 'type -t conda && conda info -a && conda list'
+        docker build -t bioconda/bioconda-utils-test-env:latest -f ./Dockerfile.test ./
   autobump_run: &autobump_run
     name: Check recipes for new upstream releases
     command: |
@@ -77,7 +78,7 @@ variables:
           name: Testing
           command: |
             if git diff --name-only origin/master...$CIRCLE_SHA1 | grep -vE ^docs; then
-              py.test --durations=0 test/ -v --tb=native -m "${PY_TEST_MARKER}"
+              py.test --durations=0 test/ -v --log-level=DEBUG --tb=native -m "${PY_TEST_MARKER}"
             else
               echo "Skipping pytest - only docs modified"
             fi
@@ -110,7 +111,7 @@ jobs:
           name: Testing
           command: |
             if git diff --name-only origin/master...$CIRCLE_SHA1 | grep -vE ^docs; then
-              py.test --durations=0 test/ -v -k "not docker" --tb=native
+              py.test --durations=0 test/ -v --log-level=DEBUG -k "not docker" --tb=native
             else
               echo "Skipping pytest - only docs modified"
             fi

.github/workflows/GithubActionTests.yml

@@ -19,13 +19,14 @@ jobs:
         python setup.py install
     - name: Build docker container
       run: |
-        docker build -t bioconda-utils-build-env:latest ./
-        docker history bioconda-utils-build-env:latest
-        docker run --rm -t bioconda-utils-build-env:latest sh -lec 'type -t conda && conda info -a && conda list'
+        docker build -t bioconda/bioconda-utils-build-env:latest ./
+        docker history bioconda/bioconda-utils-build-env:latest
+        docker run --rm -t bioconda/bioconda-utils-build-env:latest sh -lec 'type -t conda && conda info -a && conda list'
+        docker build -t bioconda/bioconda-utils-test-env:latest -f ./Dockerfile.test ./
     - name: Run tests '${{ matrix.py_test_marker }}'
       run: |
         if git diff --name-only origin/master...HEAD | grep -vE ^docs; then
-            py.test --durations=0 test/ -v --tb=native -m '${{ matrix.py_test_marker }}'
+            py.test --durations=0 test/ -v --log-level=DEBUG --tb=native -m '${{ matrix.py_test_marker }}'
         else
             echo "Skipping pytest - only docs modified"
         fi
@@ -42,7 +43,7 @@ jobs:
     - name: Run tests
       run: |
         if git diff --name-only origin/master...HEAD | grep -vE ^docs; then
-            py.test --durations=0 test/ -v -k "not docker" --tb=native
+            py.test --durations=0 test/ -v --log-level=DEBUG -k "not docker" --tb=native
         else
             echo "Skipping pytest - only docs modified"
         fi

Dockerfile.test

@@ -0,0 +1,5 @@
+FROM bioconda/bioconda-utils-build-env
+# Retrieve index and set TTL to make it last over the duration of the test.
+RUN \
+    conda search bioconda-utils > /dev/null && \
+    conda config --set local_repodata_ttl 3600

bioconda_utils/bioconductor_skeleton.py

@@ -137,7 +137,7 @@ def bioconductor_versions():
     """
     url = "https://bioconductor.org/config.yaml"
     response = requests.get(url)
-    bioc_config = yaml.load(response.text)
+    bioc_config = yaml.safe_load(response.text)
     versions = list(bioc_config["r_ver_for_bioc_ver"].keys())
     # Handle semantic version sorting like 3.10 and 3.9
     versions = sorted(versions, key=lambda v: list(map(int, v.split('.'))), reverse=True)
@@ -174,7 +174,7 @@ def bioconductor_tarball_url(package, pkg_version, bioc_version):
         Bioconductor release version
     """
     return (
-        'http://bioconductor.org/packages/{bioc_version}'
+        'https://bioconductor.org/packages/{bioc_version}'
         '/bioc/src/contrib/{package}_{pkg_version}.tar.gz'.format(**locals())
     )
 
@@ -195,7 +195,7 @@ def bioconductor_annotation_data_url(package, pkg_version, bioc_version):
         Bioconductor release version
     """
     return (
-        'http://bioconductor.org/packages/{bioc_version}'
+        'https://bioconductor.org/packages/{bioc_version}'
         '/data/annotation/src/contrib/{package}_{pkg_version}.tar.gz'.format(**locals())
     )
 
@@ -216,7 +216,7 @@ def bioconductor_experiment_data_url(package, pkg_version, bioc_version):
         Bioconductor release version
     """
     return (
-        'http://bioconductor.org/packages/{bioc_version}'
+        'https://bioconductor.org/packages/{bioc_version}'
         '/data/experiment/src/contrib/{package}_{pkg_version}.tar.gz'.format(**locals())
     )
 
@@ -376,7 +376,7 @@ def __init__(self, package, bioc_version=None, pkg_version=None, packages=None):
         scraped data.
         >>> x = BioCProjectPage('DESeq2')
         >>> x.tarball_url
-        'http://bioconductor.org/packages/release/bioc/src/contrib/DESeq2_1.8.2.tar.gz'
+        'https://bioconductor.org/packages/release/bioc/src/contrib/DESeq2_1.8.2.tar.gz'
         """
         self.base_url = base_url
         self.package = package

bioconda_utils/build.py

@@ -46,13 +46,6 @@ def conda_build_purge() -> None:
                     utils.get_free_space())
 
 
-def has_conda_in_build(recipe):
-    """Checks if recipe has conda or conda build in deps"""
-    deps = utils.get_deps(recipe, build=True)
-    deps.update(utils.get_deps(recipe, build=False))
-    return any(pkg in deps for pkg in ('conda', 'conda-build'))
-
-
 def build(recipe: str, pkg_paths: List[str] = None,
           testonly: bool = False, mulled_test: bool = True,
           channels: List[str] = None,
@@ -342,19 +335,10 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
             logger.info("Nothing to be done for recipe %s", recipe)
             continue
 
-        # If a recipe depends on conda, it means it must be installed in
-        # the root env, which is not compatible with mulled-build tests. In
-        # that case, we temporarily disable the mulled-build tests for the
-        # recipe.
-        keep_mulled_test = not has_conda_in_build(recipe)
-        if mulled_test and not keep_mulled_test:
-            logger.info('TEST SKIP: skipping mulled-build test for %s because it '
-                        'depends on conda or conda-build', recipe)
-
         res = build(recipe=recipe,
                     pkg_paths=pkg_paths,
                     testonly=testonly,
-                    mulled_test=mulled_test and keep_mulled_test,
+                    mulled_test=mulled_test,
                     channels=config['channels'],
                     docker_builder=docker_builder,
                     linter=linter)
@@ -370,7 +354,7 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
                     for pkg in pkg_paths:
                         if not upload.anaconda_upload(pkg, label=label):
                             failed_uploads.append(pkg)
-                if mulled_upload_target and keep_mulled_test:
+                if mulled_upload_target:
                     for img in res.mulled_images:
                         upload.mulled_upload(img, mulled_upload_target)
 

bioconda_utils/cran_skeleton.py

@@ -24,20 +24,19 @@
 # Raw strings needed to support the awkward backslashes needed when adding the
 # command to yaml
 gpl2_short = r"  license_family: GPL2"
-gpl2_long = (
-    "  license_family: GPL2\n  license_file: '{{ environ[\"PREFIX\"] }}"
-    "\/lib\/R\/share\/licenses\/GPL-2'  # [unix]\n  "
-    "license_file: '{{ environ[\"PREFIX\"] }}"
-    "\\\R\\\share\\\licenses\\\GPL-2'  # [win]"
-)
+gpl2_long = r"""
+  license_family: GPL2
+  license_file: '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2'  # [unix]
+  license_file: '{{ environ["PREFIX"] }}\\R\\share\\licenses\\GPL-2'  # [win]
+""".strip('\n')
 
 gpl3_short = r"  license_family: GPL3"
-gpl3_long = (
-    "  license_family: GPL3\n  license_file: '{{ environ[\"PREFIX\"] }}"
-    "\/lib\/R\/share\/licenses\/GPL-3'  # [unix]\n  "
-    "license_file: '{{ environ[\"PREFIX\"] }}"
-    "\\\R\\\share\\\licenses\\\GPL-3'  # [win]"
-)
+gpl3_long = r"""
+  license_family: GPL3
+  license_file: '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-3'  # [unix]
+  license_file: '{{ environ["PREFIX"] }}\\R\\share\\licenses\\GPL-3'  # [win]
+""".strip('\n')
+
 win32_string = 'number: 0\n  skip: true  # [win32]'
 
 

bioconda_utils/docker_utils.py

@@ -359,7 +359,7 @@ def _build_image(self):
         except sp.CalledProcessError:
             logger.error('DOCKER FAILED: Error checking docker version.')
             raise
-        p = re.compile("\d+\.\d+\.\d+")  # three groups of at least on digit separated by dots
+        p = re.compile(r"\d+\.\d+\.\d+")  # three groups of at least on digit separated by dots
         version_string = re.search(p, s).group(0)
         if LooseVersion(version_string) >= LooseVersion("1.13.0"):
             cmd = [

test/test_utils.py

@@ -34,7 +34,7 @@
 # docker, once without). On OSX, only the non-docker runs.
 
 # Docker ref for build container
-DOCKER_BASE_IMAGE = "bioconda/bioconda-utils-build-env:latest"
+DOCKER_BASE_IMAGE = "bioconda/bioconda-utils-test-env:latest"
 
 SKIP_DOCKER_TESTS = sys.platform.startswith('darwin')
 SKIP_NOT_OSX = not sys.platform.startswith('darwin')
@@ -303,26 +303,47 @@ def test_get_deps():
     assert list(utils.get_deps(r.recipe_dirs['three'], build=False)) == ['two']
 
 
-def test_conda_as_dep(config_fixture):
+@pytest.mark.long_running_1
+@pytest.mark.parametrize('mulled_test', PARAMS, ids=IDS)
+def test_conda_as_dep(config_fixture, mulled_test):
+    docker_builder = None
+    if mulled_test:
+        docker_builder = docker_utils.RecipeBuilder(
+            use_host_conda_bld=True,
+            docker_base_image=DOCKER_BASE_IMAGE,
+        )
     r = Recipes(
         """
         one:
           meta.yaml: |
             package:
-              name: one
+              name: bioconda_utils_test_conda_as_dep
               version: 0.1
             requirements:
+              host:
+                - conda
               run:
                 - conda
+            test:
+              commands:
+                - test -e "${PREFIX}/bin/conda"
         """, from_string=True)
     r.write_recipes()
-    build_result = build.build_recipes(r.basedir, config_fixture,
-                                       r.recipe_dirnames,
-                                       testonly=False,
-                                       force=False,
-                                       mulled_test=True)
+    build_result = build.build_recipes(
+        r.basedir, config_fixture,
+        r.recipe_dirnames,
+        testonly=False,
+        force=False,
+        docker_builder=docker_builder,
+        mulled_test=mulled_test,
+    )
     assert build_result
 
+    for k, v in r.recipe_dirs.items():
+        for i in utils.built_package_paths(v):
+            assert os.path.exists(i)
+            ensure_missing(i)
+
 # TODO replace the filter tests with tests for utils.get_package_paths()
 # def test_filter_recipes_no_skipping():
 #     """
@@ -581,7 +602,7 @@ def test_sandboxed():
 
 def test_env_sandboxing():
     r = Recipes(
-        """
+        r"""
         one:
           meta.yaml: |
             package:
@@ -596,7 +617,7 @@ def test_env_sandboxing():
                 echo "\$GITHUB_TOKEN has leaked into the build environment!"
                 exit 1
             fi
-    """, from_string=True)
+        """, from_string=True)
     r.write_recipes()
     pkg_paths = utils.built_package_paths(r.recipe_dirs['one'])
 
@@ -749,6 +770,11 @@ def test_build_container_no_default_gcc(tmpdir):
     )
     assert build_result.success
 
+    for k, v in r.recipe_dirs.items():
+        for i in utils.built_package_paths(v):
+            assert os.path.exists(i)
+            ensure_missing(i)
+
 
 # FIXME: This test fails erraticaly. Both in built_package_paths
 # and in build_recipes, the generated name can be either