Skip to content

Commit 6b31b8c

Browse files
epruesseepruesse
authored
Merge branch 'master' into epruesse-patch-1
2 parents 4898bfe + 12ef6d3 commit 6b31b8c

20 files changed

+188
-74
lines changed

‎.circleci/config.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ variables:
4141
run:
4242
name: Build the updated docker container
4343
command: |
44-
docker build -t bioconda/bioconda-utils-build-env:latest ./
45-
docker history bioconda/bioconda-utils-build-env:latest
46-
docker run --rm -t bioconda/bioconda-utils-build-env:latest sh -lec 'type -t conda && conda info -a && conda list'
47-
docker build -t bioconda/bioconda-utils-test-env:latest -f ./Dockerfile.test ./
44+
docker build -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest ./
45+
docker history quay.io/bioconda/bioconda-utils-build-env-cos7:latest
46+
docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info -a && conda list'
47+
docker build -t quay.io/bioconda/bioconda-utils-test-env-cos7:latest -f ./Dockerfile.test ./
4848
autobump_run: &autobump_run
4949
name: Check recipes for new upstream releases
5050
command: |

‎.dockerignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
1-
.git
21
miniconda

‎.github/workflows/GithubActionTests.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ jobs:
1919
python setup.py install
2020
- name: Build docker container
2121
run: |
22-
docker build -t bioconda/bioconda-utils-build-env:latest ./
23-
docker history bioconda/bioconda-utils-build-env:latest
24-
docker run --rm -t bioconda/bioconda-utils-build-env:latest sh -lec 'type -t conda && conda info -a && conda list'
25-
docker build -t bioconda/bioconda-utils-test-env:latest -f ./Dockerfile.test ./
22+
docker build -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest ./
23+
docker history quay.io/bioconda/bioconda-utils-build-env-cos7:latest
24+
docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info -a && conda list'
25+
docker build -t quay.io/bioconda/bioconda-utils-test-env-cos7:latest -f ./Dockerfile.test ./
2626
- name: Run tests '${{ matrix.py_test_marker }}'
2727
run: |
2828
if git diff --name-only origin/master...HEAD | grep -vE ^docs; then

‎.github/workflows/build-and-push-image.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Build and Push Image
2+
on:
3+
release:
4+
types:
5+
- published
6+
pull_request:
7+
paths-ignore:
8+
- '.circleci/**'
9+
- 'docs/**'
10+
- 'test/**'
11+
12+
jobs:
13+
build:
14+
name: Build and Push Image
15+
runs-on: ubuntu-20.04
16+
17+
steps:
18+
- uses: actions/checkout@v2
19+
20+
- name: Build Image
21+
id: buildah-build
22+
uses: redhat-actions/buildah-build@v2
23+
with:
24+
image: bioconda-utils-build-env-cos7
25+
tags: >-
26+
latest
27+
${{ github.event.release && github.event.release.tag_name || github.sha }}
28+
dockerfiles: |
29+
./Dockerfile
30+
31+
- name: Test Built Image
32+
run: |
33+
image='${{ steps.buildah-build.outputs.image }}'
34+
for tag in ${{ steps.buildah-build.outputs.tags }} ; do
35+
podman run --rm "${image}:${tag}" bioconda-utils --version
36+
done
37+
38+
- if: ${{ github.event.release }}
39+
name: Push To Quay
40+
uses: redhat-actions/push-to-registry@v2
41+
with:
42+
image: ${{ steps.buildah-build.outputs.image }}
43+
tags: ${{ steps.buildah-build.outputs.tags }}
44+
registry: ${{ secrets.QUAY_BIOCONDA_REPO }}
45+
username: ${{ secrets.QUAY_BIOCONDA_USERNAME }}
46+
password: ${{ secrets.QUAY_BIOCONDA_TOKEN }}

‎Dockerfile

Lines changed: 53 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,56 @@
1-
FROM condaforge/linux-anvil-comp7
2-
# Pretend we'd have C.UTF-8: Just copy en_US.UTF-8
3-
RUN localedef -i en_US -f UTF-8 C.UTF-8
4-
RUN sudo -n yum install -y openssh-clients && \
5-
sudo -n yum clean all && \
6-
sudo -n rm -rf /var/cache/yum/*
7-
RUN mkdir -p /tmp/repo/bioconda_utils/
8-
COPY ./bioconda_utils/bioconda_utils-requirements.txt /tmp/repo/bioconda_utils/
9-
ENV PATH="/opt/conda/bin:${PATH}"
10-
RUN conda config --add channels defaults && \
11-
conda config --add channels bioconda && \
12-
conda config --add channels conda-forge && \
13-
{ conda config --remove repodata_fns current_repodata.json || true ; } && \
1+
FROM quay.io/condaforge/linux-anvil-cos7-x86_64 as base
2+
3+
# Copy over C.UTF-8 locale from our base image to make it consistently available during build.
4+
COPY --from=quay.io/bioconda/base-glibc-busybox-bash /usr/lib/locale/C.UTF-8 /usr/lib/locale/C.UTF-8
5+
6+
# Provide system deps unconditionally until we are able to offer per-recipe installs.
7+
# (Addresses, e.g., "ImportError: libGL.so.1" in tests directly invoked by conda-build.)
8+
# Also install packages that have been installed historically (openssh-client).
9+
RUN yum install -y \
10+
mesa-libGL-devel \
11+
openssh-client \
12+
&& \
13+
yum clean all && \
14+
rm -rf /var/cache/yum/*
15+
16+
# This changes root's .condarc which ENTRYPOINT copies to /home/conda/.condarc later.
17+
RUN . /opt/conda/etc/profile.d/conda.sh && \
18+
conda config \
19+
--prepend channels defaults \
20+
--prepend channels bioconda \
21+
--prepend channels conda-forge \
22+
&& \
23+
{ conda config --remove repodata_fns current_repodata.json 2> /dev/null || true ; } && \
1424
conda config --prepend repodata_fns repodata.json && \
1525
conda config --set auto_update_conda False
16-
RUN : 'Make sure we get the (working) conda we want before installing the rest.' && \
17-
sed -nE \
18-
-e 's/\s*#.*$//' \
19-
-e 's/^(conda([><!=~ ].+)?)$/\1/p' \
20-
/tmp/repo/bioconda_utils/bioconda_utils-requirements.txt \
21-
| xargs -r conda install -y
22-
RUN conda install -y --file /tmp/repo/bioconda_utils/bioconda_utils-requirements.txt
23-
RUN conda clean -y -it
24-
COPY . /tmp/repo
25-
RUN pip install /tmp/repo
2626

27-
ENTRYPOINT [ "/opt/conda/bin/tini", "--", "/tmp/repo/docker-entrypoint" ]
27+
FROM base as build
28+
WORKDIR /tmp/repo
29+
COPY . ./
30+
RUN . /opt/conda/etc/profile.d/conda.sh && conda activate base && \
31+
pip wheel . && \
32+
mkdir - /opt/bioconda-utils && \
33+
cp ./bioconda_utils-*.whl \
34+
./bioconda_utils/bioconda_utils-requirements.txt \
35+
/opt/bioconda-utils/ \
36+
&& \
37+
chgrp -R lucky /opt/bioconda-utils && \
38+
chmod -R g=u /opt/bioconda-utils
39+
40+
FROM base
41+
COPY --from=build /opt/bioconda-utils /opt/bioconda-utils
42+
RUN . /opt/conda/etc/profile.d/conda.sh && conda activate base && \
43+
# Make sure we get the (working) conda we want before installing the rest.
44+
sed -nE \
45+
'/^conda([><!=~ ].+)?$/p' \
46+
/opt/bioconda-utils/bioconda_utils-requirements.txt \
47+
| xargs -r conda install --yes && \
48+
conda install --yes --file /opt/bioconda-utils/bioconda_utils-requirements.txt && \
49+
pip install --no-deps --find-links /opt/bioconda-utils bioconda_utils && \
50+
conda clean --yes --index --tarballs && \
51+
# Find files that are not already in group "lucky" and change their group and mode.
52+
find /opt/conda \
53+
\! -group lucky \
54+
-exec chgrp --no-dereference lucky {} + \
55+
\! -type l \
56+
-exec chmod g=u {} +

‎Dockerfile.test

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1-
FROM bioconda/bioconda-utils-build-env
1+
FROM quay.io/bioconda/bioconda-utils-build-env-cos7
22
# Retrieve index and set TTL to make it last over the duration of the test.
3-
RUN \
3+
RUN . /opt/conda/etc/profile.d/conda.sh && \
44
conda search bioconda-utils > /dev/null && \
5-
conda config --set local_repodata_ttl 3600
5+
conda config --set local_repodata_ttl 3600 && \
6+
find /opt/conda \
7+
\! -group lucky \
8+
-exec chgrp --no-dereference lucky {} + \
9+
\! -type l \
10+
-exec chmod g=u {} +

‎bioconda_utils/autobump.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ async def apply(self, recipe: Recipe) -> None:
423423
@classmethod
424424
def _sp_apply(cls, data) -> None:
425425
config, recipe = data
426-
status = update_pinnings.check(recipe, build_config=config, keep_metas=True)
426+
status, recipe = update_pinnings.check(recipe, build_config=config, keep_metas=True)
427427
if status.needs_bump():
428428
metas = recipe.conda_render(config=config)
429429
reason = cls.find_reason(recipe, metas)

‎bioconda_utils/build.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,10 @@ def build(recipe: str, pkg_paths: List[str] = None,
104104
meta = utils.load_first_metadata(recipe, finalize=False)
105105
is_noarch = bool(meta.get_value('build/noarch', default=False))
106106
use_base_image = meta.get_value('extra/container', {}).get('extended-base', False)
107-
base_image = 'bioconda/extended-base-image' if use_base_image else None
107+
if use_base_image:
108+
base_image = 'quay.io/bioconda/base-glibc-debian-bash:2.0.0'
109+
else:
110+
base_image = 'quay.io/bioconda/base-glibc-busybox-bash:2.0.0'
108111

109112
try:
110113
if docker_builder is not None:

‎bioconda_utils/cli.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -843,7 +843,7 @@ def autobump(recipe_folder, config, packages='*', exclude=None, cache=None,
843843
no_skip_pending_deps = True
844844
else:
845845
recipe_source = autobump.RecipeGraphSource(
846-
recipe_folder, packages, exclude, not no_shuffle,
846+
recipe_folder, packages, exclude or [], not no_shuffle,
847847
config_dict, cache_fn=cache and cache + "_dag.pkl")
848848

849849
# Setup scanning pipeline

‎bioconda_utils/docker_utils.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import conda_build
6262

6363
from . import utils
64+
from . import __version__
6465

6566
import logging
6667
logger = logging.getLogger(__name__)
@@ -117,14 +118,21 @@
117118
# This template can be used for last-minute changes to the docker image, such
118119
# as adding proxies.
119120
#
120-
# The default image is created automatically on DockerHub using the Dockerfile
121+
# The default image is created automatically for releases using the Dockerfile
121122
# in the bioconda-utils repo.
122123

123124
DOCKERFILE_TEMPLATE = \
124-
"""
125+
r"""
125126
FROM {docker_base_image}
126127
{proxies}
127-
RUN /opt/conda/bin/conda install -y conda={conda_ver} conda-build={conda_build_ver}
128+
RUN \
129+
/opt/conda/bin/conda install -y conda={conda_ver} conda-build={conda_build_ver} \
130+
&& \
131+
find /opt/conda \
132+
\! -group lucky \
133+
-exec chgrp --no-dereference lucky {{}} + \
134+
\! -type l \
135+
-exec chmod g=u {{}} +
128136
""" # noqa: E122 continuation line missing indentation or outdented
129137

130138

@@ -163,7 +171,7 @@ def __init__(
163171
keep_image=False,
164172
build_image=False,
165173
image_build_dir=None,
166-
docker_base_image='bioconda/bioconda-utils-build-env:latest'
174+
docker_base_image='quay.io/bioconda/bioconda-utils-build-env-cos7:{}'.format(__version__.replace('+', '_'))
167175
):
168176
"""
169177
Class to handle building a custom docker container that can be used for
@@ -245,7 +253,7 @@ def __init__(
245253
246254
docker_base_image : str or None
247255
Name of base image that can be used in **dockerfile_template**.
248-
Defaults to 'bioconda/bioconda-utils-build-env:latest'
256+
Defaults to 'quay.io/bioconda/bioconda-utils-build-env-cos7:bioconda-utils-version'
249257
"""
250258
self.requirements = requirements
251259
self.conda_build_args = ""

‎bioconda_utils/hosters.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ class GithubRelease(GithubBase):
330330

331331
class GithubTag(GithubBase):
332332
"""Matches GitHub repository archives created automatically from tags"""
333-
link_pattern = r"/{account}/{project}/archive/{tag}{ext}"
333+
link_pattern = r"/{account}/{project}/archive(/refs/tags)?/{tag}{ext}"
334334
releases_formats = ["https://github.com/{account}/{project}/tags"]
335335

336336

‎bioconda_utils/involucro

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#! /bin/sh
2+
3+
exec involucro \
4+
-set POSTINSTALL='create-env --conda=: /usr/local' \
5+
"${@}"

‎bioconda_utils/pkg_test.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
logger = logging.getLogger(__name__)
1919

2020
# TODO: Make this configurable in bioconda_utils.build and bioconda_utils.cli.
21-
MULLED_CONDA_IMAGE = "quay.io/dpryan79/mulled_container:latest"
21+
MULLED_CONDA_IMAGE = "quay.io/bioconda/create-env:1.0.2"
2222

2323

2424
def get_tests(path):
@@ -154,10 +154,10 @@ def test_package(
154154
cmd += shlex.split(mulled_args)
155155

156156
# galaxy-lib always downloads involucro, unless it's in cwd or its path is explicitly given.
157-
# TODO: This should go into galaxy-lib. Once it is fixed upstream, remove this here.
158-
involucro_path = which('involucro')
159-
if involucro_path:
160-
cmd += ['--involucro-path', involucro_path]
157+
# We inject a POSTINSTALL to the involucro command with a small wrapper to
158+
# create activation / entrypoint scripts for the container.
159+
involucro_path = os.path.join(os.path.dirname(__file__), 'involucro')
160+
cmd += ['--involucro-path', involucro_path]
161161

162162
logger.debug('mulled-build command: %s' % cmd)
163163

‎docker-entrypoint

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎docker-entrypoint-source

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎docs/source/contributor/build-system.rst

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ need to create the entire bioconda-build system environment from scratch for
2929
each build.
3030

3131
When testing locally with ``circleci build``, we use the
32-
``bioconda/bioconda-utils-build-env`` Docker container to avoid changing the
32+
``quay.io/bioconda/bioconda-utils-build-env-cos7`` Docker container to avoid changing the
3333
local system. This container is defined by `this Dockerfile
3434
<https://github.com/bioconda/bioconda-utils/blob/master/Dockerfile>`_.
3535

@@ -69,7 +69,7 @@ Configure the environment
6969
- package with that version number and build number does not exist in
7070
bioconda channel (we check the channel for each of the changed recipes)
7171

72-
- Download the configured Docker container (currently based on CentOS 6)
72+
- Download the configured Docker container (currently based on CentOS 7)
7373
- default configured in ``bioconda-utils: docker_utils.py``
7474

7575
- Build a new, temporary Docker container
@@ -97,8 +97,8 @@ Configure the environment
9797
`galaxy-lib <https://github.com/galaxyproject/galaxy-lib>`_). This acts as
9898
a more stringent test than ``conda-build`` alone. The BusyBox container
9999
purposefully is missing many system libraries (like libgcc) that may be
100-
present in the CentOS 6 container. Note that it is common for a package to
101-
build in the CentOS 6 container but fail in the BusyBox container. When this
100+
present in the CentOS 7 container. Note that it is common for a package to
101+
build in the CentOS 7 container but fail in the BusyBox container. When this
102102
happens, it is often because a dependency needs to be added to the recipe.
103103

104104
- Upon successfully testing the package in the BusyBox container, we have a branch point:

‎docs/source/contributor/building-locally.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ You can execute an almost exact copy of our Linux build pipeline by
2020
folder where your copy of ``bioconda-recipes`` resides::
2121

2222
# Ensure the build container is up-to-date
23-
docker pull bioconda/bioconda-utils-build-env:latest
23+
docker pull quay.io/bioconda/bioconda-utils-build-env:latest
2424

2525
# Run the build locally
2626
circleci build

‎test/test_case.yaml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ one:
55
version: "0.1"
66
build.sh: |
77
#!/bin/bash
8-
ls ${PREFIX}
8+
touch "${PREFIX}/file-one"
9+
ls "${PREFIX}"
910
two:
1011
meta.yaml: |
1112
package:
@@ -20,10 +21,11 @@ three:
2021
name: three
2122
version: "0.1"
2223
requirements:
23-
build:
24+
host:
2425
- "one"
2526
build.sh: |
2627
#!/bin/bash
27-
ls ${PREFIX}
28+
ls "${PREFIX}"
29+
ls "${PREFIX}/file-one"
2830
2931
# vim: ts=2 sw=2

‎test/test_pkg_test.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -101,22 +101,23 @@ def test_pkg_test_conda_image():
101101
# Inspects the installing conda version by writing $PREFIX/conda-version as
102102
# a post-link step -- but only if we are actually doing mulled tests, i.e.,
103103
# when $PREFIX == /usr/local.
104-
recipe = dedent("""
104+
conda_version = '4.9.2'
105+
recipe = dedent(f"""
105106
one:
106107
meta.yaml: |
107108
package:
108109
name: one
109110
version: 0.1
110111
test:
111112
commands:
112-
- '[ "${PREFIX}" != /usr/local ] || cat /usr/local/conda-version'
113-
- '[ "${PREFIX}" != /usr/local ] || grep -F ''conda 4.3.11'' /usr/local/conda-version'
113+
- '[[ "${{PREFIX}}" != /usr/local ]] || cat /usr/local/conda-version'
114+
- '[[ "${{PREFIX}}" != /usr/local ]] || grep -F ''conda {conda_version}'' /usr/local/conda-version'
114115
post-link.sh: |
115116
#!/bin/bash
116-
if [ "${PREFIX}" == /usr/local ] ; then
117-
/opt/conda/bin/conda --version > /usr/local/conda-version
117+
if [[ "${{PREFIX}}" == /usr/local ]] ; then
118+
/opt/*/bin/conda --version > /usr/local/conda-version
118119
fi
119120
""") # noqa: E501: line too long
120121
built_packages = _build_pkg(recipe)
121122
for pkg in built_packages:
122-
pkg_test.test_package(pkg, conda_image="continuumio/miniconda3:4.3.11")
123+
pkg_test.test_package(pkg, conda_image=f"quay.io/bioconda/create-env:1.0.1-{conda_version}")

0 commit comments

Comments
 (0)