From f3ed46e3629fa7ca1325a1fed85f7106f1f274e1 Mon Sep 17 00:00:00 2001 From: Peter Krusche Date: Sat, 3 Jun 2017 10:48:43 +0100 Subject: [PATCH 1/6] HAP-318 change version of rtgtools to 3.8.1 --- external/make_dependencies.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/external/make_dependencies.sh b/external/make_dependencies.sh index 455a4c4..d129738 100755 --- a/external/make_dependencies.sh +++ b/external/make_dependencies.sh @@ -115,9 +115,9 @@ if [[ ! -z $BUILD_VCFEVAL ]]; then cd ${TLD} mkdir -p ${TLD}/rtg-tools cd rtg-tools - wget http://github.com/RealTimeGenomics/rtg-tools/archive/3.7.1.tar.gz -O ${TLD}/rtg-tools/rtg-tools.tar.gz + wget http://github.com/RealTimeGenomics/rtg-tools/archive/3.8.1.tar.gz -O ${TLD}/rtg-tools/rtg-tools.tar.gz tar xvf rtg-tools.tar.gz - cd rtg-tools-3.7.1 + cd rtg-tools-3.8.1 if [[ ! -z ${ANT_HOME} ]]; then $ANT_HOME/bin/ant zip-nojre @@ -126,7 +126,7 @@ if [[ ! -z $BUILD_VCFEVAL ]]; then fi cd .. - RTG_ZIPFILE=$(ls rtg-tools-3.7.1/dist/*-nojre.zip | head -1) + RTG_ZIPFILE=$(ls rtg-tools-3.8.1/dist/*-nojre.zip | head -1) RTG_BASE=$(basename $RTG_ZIPFILE -nojre.zip) jar xvf $RTG_ZIPFILE mv $RTG_BASE ${ISD}/libexec/rtg-tools-install From b6844386b822e8c353c47b5680c7f518ea4a3dcc Mon Sep 17 00:00:00 2001 From: Peter Krusche Date: Sat, 3 Jun 2017 10:57:55 +0100 Subject: [PATCH 2/6] =?UTF-8?q?HAP-319=20don=E2=80=99t=20download=20hg19?= =?UTF-8?q?=20in=20default=20Docker=20image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 24 ++++------ Dockerfile.ubuntu-with-tests | 90 ++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 16 deletions(-) create mode 100755 Dockerfile.ubuntu-with-tests diff --git a/Dockerfile b/Dockerfile index 13b7873..94d3f90 100755 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,6 @@ RUN apt-get install bzip2 -y RUN apt-get install wget -y RUN apt-get install libbz2-dev -y - RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ apt-get update && \ @@ -67,14 +66,6 @@ RUN mkdir -p /opt/hap.py-data # download HG19 reference data WORKDIR /opt/hap.py-data -# This downloads from UCSC -RUN /opt/hap.py-source/src/sh/make_hg19.sh - -# this downloads our copy from box.com (once available) -# RUN wget https://illumina.app.box.com/s/1vwpu1w23p2qupukjjlrenq81mrtth1z/hg19.fa.bz2 -O /opt/hap.py-source/src/data/hg19.fa.bz2 -# RUN bzcat /opt/hap.py-source/src/data/hg19.fa.bz2 > /opt/hap.py-data/hg19.fa -RUN samtools faidx /opt/hap.py-data/hg19.fa - # get + install ant WORKDIR /opt RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.7-bin.tar.gz && \ @@ -82,14 +73,15 @@ RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.7-bin.tar.gz rm apache-ant-1.9.7-bin.tar.gz ENV PATH $PATH:/opt/apache-ant-1.9.7/bin -# run hap.py installer in the image +# run hap.py installer in the image, don't run tests since we don't have a reference file WORKDIR /opt/hap.py-source -RUN HG19=/opt/hap.py-data/hg19.fa python install.py /opt/hap.py --with-rtgtools +RUN python install.py /opt/hap.py --with-rtgtools --no-tests +WORKDIR /opt/hap.py + +# run basic tests +RUN bin/test_haplotypes + +# remove source folder WORKDIR / RUN rm -rf /opt/hap.py-source -# Optionally set up I/O folders -# RUN mkdir /data && chgrp users /data && chmod a+rwx /data -# VOLUME /data -# RUN useradd happy -m -s /bin/bash -G users -# USER happy diff --git a/Dockerfile.ubuntu-with-tests b/Dockerfile.ubuntu-with-tests new file mode 100755 index 0000000..ef26835 --- /dev/null +++ b/Dockerfile.ubuntu-with-tests @@ -0,0 +1,90 @@ +FROM ubuntu:14.04 + +RUN apt-get update +RUN apt-get upgrade -y + +# install python and PIP +RUN apt-get install python2.7 python2.7-dev -y +RUN apt-get install cython -y +RUN apt-get install python-setuptools -y +RUN apt-get install python-pip -y +RUN apt-get install python-numpy -y +RUN apt-get install zlib1g-dev -y +# ideally, we don't want to use the ubuntu version +# here because this bloats the Docker image +# RUN apt-get install python-pandas -y +RUN easy_install -U distribute + +# numpy and such +RUN apt-get install build-essential -y +RUN apt-get install gfortran -y +RUN apt-get install -y libatlas-base-dev +RUN apt-get install pkg-config -y +RUN apt-get install software-properties-common python-software-properties -y +RUN apt-get install cmake -y + +# hap.py v0.3.7+ requires g++ 4.9 +RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y +RUN apt-get update +RUN apt-get install g++-4.9 -y +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 50 +RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 50 + +# upgrade to newest versions / install +RUN pip install --upgrade cython +RUN pip install --upgrade numpy +RUN pip install --upgrade pandas +RUN pip install pybedtools +RUN pip install pysam +RUN pip install bx-python +RUN pip install scipy + +# boost and cmake for hap.py +RUN apt-get install libncurses5-dev -y +RUN apt-get install git -y +RUN apt-get install samtools -y +RUN apt-get install bzip2 -y +RUN apt-get install wget -y +RUN apt-get install libbz2-dev -y + + +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + add-apt-repository -y ppa:webupd8team/java && \ + apt-get update && \ + apt-get install -y oracle-java8-installer && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/cache/oracle-jdk8-installer + +RUN apt-get clean -y + +# copy git repository into the image +RUN mkdir -p /opt/hap.py-source +COPY . /opt/hap.py-source/ + +# make minimal HG19 reference sequence +RUN mkdir -p /opt/hap.py-data + +# download HG19 reference data +WORKDIR /opt/hap.py-data + +# This downloads from UCSC +RUN /opt/hap.py-source/src/sh/make_hg19.sh + +# this downloads our copy from box.com (once available) +# RUN wget https://illumina.app.box.com/s/1vwpu1w23p2qupukjjlrenq81mrtth1z/hg19.fa.bz2 -O /opt/hap.py-source/src/data/hg19.fa.bz2 +# RUN bzcat /opt/hap.py-source/src/data/hg19.fa.bz2 > /opt/hap.py-data/hg19.fa +RUN samtools faidx /opt/hap.py-data/hg19.fa + +# get + install ant +WORKDIR /opt +RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.7-bin.tar.gz && \ + tar xzf apache-ant-1.9.7-bin.tar.gz && \ + rm apache-ant-1.9.7-bin.tar.gz +ENV PATH $PATH:/opt/apache-ant-1.9.7/bin + +# run hap.py installer in the image +WORKDIR /opt/hap.py-source +RUN HG19=/opt/hap.py-data/hg19.fa python install.py /opt/hap.py --with-rtgtools +WORKDIR / +RUN rm -rf /opt/hap.py-source + From e074716aa2477a3f1277969256b61abe7f5dfec8 Mon Sep 17 00:00:00 2001 From: Peter Krusche Date: Sat, 3 Jun 2017 11:44:26 +0100 Subject: [PATCH 3/6] HAP-319 slim Dockerfiles --- Dockerfile | 66 +++++++-------------------- Dockerfile.ubuntu-with-tests | 88 +++++++++++------------------------- happy.requirements.txt | 3 +- 3 files changed, 46 insertions(+), 111 deletions(-) diff --git a/Dockerfile b/Dockerfile index 94d3f90..5ee8df5 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,51 +1,21 @@ -FROM ubuntu:14.04 - -RUN apt-get update -RUN apt-get upgrade -y - -# install python and PIP -RUN apt-get install python2.7 python2.7-dev -y -RUN apt-get install cython -y -RUN apt-get install python-setuptools -y -RUN apt-get install python-pip -y -RUN apt-get install python-numpy -y -RUN apt-get install zlib1g-dev -y -# ideally, we don't want to use the ubuntu version -# here because this bloats the Docker image -# RUN apt-get install python-pandas -y -RUN easy_install -U distribute - -# numpy and such -RUN apt-get install build-essential -y -RUN apt-get install gfortran -y -RUN apt-get install -y libatlas-base-dev -RUN apt-get install pkg-config -y +FROM ubuntu:16.04 + +RUN apt-get update && apt-get upgrade -y +RUN apt-get install build-essential zlib1g-dev libbz2-dev pkg-config cmake libncurses5-dev autoconf -y +RUN apt-get install git bzip2 wget -y +RUN apt-get install python2.7 python2.7-dev python \ + python-setuptools \ + python-pip \ + python-psutil \ + cython \ + python-numpy \ + python-pandas \ + python-distribute \ + python-pysam \ + python-scipy \ + -y RUN apt-get install software-properties-common python-software-properties -y -RUN apt-get install cmake -y - -# hap.py v0.3.7+ requires g++ 4.9 -RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y -RUN apt-get update -RUN apt-get install g++-4.9 -y -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 50 -RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 50 - -# upgrade to newest versions / install -RUN pip install --upgrade cython -RUN pip install --upgrade numpy -RUN pip install --upgrade pandas -RUN pip install pybedtools -RUN pip install pysam -RUN pip install bx-python -RUN pip install scipy - -# boost and cmake for hap.py -RUN apt-get install libncurses5-dev -y -RUN apt-get install git -y -RUN apt-get install samtools -y -RUN apt-get install bzip2 -y -RUN apt-get install wget -y -RUN apt-get install libbz2-dev -y +RUN apt-get clean -y RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ @@ -54,8 +24,6 @@ RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true rm -rf /var/lib/apt/lists/* && \ rm -rf /var/cache/oracle-jdk8-installer -RUN apt-get clean -y - # copy git repository into the image RUN mkdir -p /opt/hap.py-source COPY . /opt/hap.py-source/ diff --git a/Dockerfile.ubuntu-with-tests b/Dockerfile.ubuntu-with-tests index ef26835..f414772 100755 --- a/Dockerfile.ubuntu-with-tests +++ b/Dockerfile.ubuntu-with-tests @@ -1,52 +1,21 @@ -FROM ubuntu:14.04 - -RUN apt-get update -RUN apt-get upgrade -y - -# install python and PIP -RUN apt-get install python2.7 python2.7-dev -y -RUN apt-get install cython -y -RUN apt-get install python-setuptools -y -RUN apt-get install python-pip -y -RUN apt-get install python-numpy -y -RUN apt-get install zlib1g-dev -y -# ideally, we don't want to use the ubuntu version -# here because this bloats the Docker image -# RUN apt-get install python-pandas -y -RUN easy_install -U distribute - -# numpy and such -RUN apt-get install build-essential -y -RUN apt-get install gfortran -y -RUN apt-get install -y libatlas-base-dev -RUN apt-get install pkg-config -y +FROM ubuntu:16.04 + +RUN apt-get update && apt-get upgrade -y +RUN apt-get install build-essential zlib1g-dev libbz2-dev pkg-config cmake libncurses5-dev autoconf -y +RUN apt-get install git bzip2 wget -y +RUN apt-get install python2.7 python2.7-dev python \ + python-setuptools \ + python-pip \ + python-psutil \ + cython \ + python-numpy \ + python-pandas \ + python-distribute \ + python-pysam \ + python-scipy \ + -y RUN apt-get install software-properties-common python-software-properties -y -RUN apt-get install cmake -y - -# hap.py v0.3.7+ requires g++ 4.9 -RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y -RUN apt-get update -RUN apt-get install g++-4.9 -y -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 50 -RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 50 - -# upgrade to newest versions / install -RUN pip install --upgrade cython -RUN pip install --upgrade numpy -RUN pip install --upgrade pandas -RUN pip install pybedtools -RUN pip install pysam -RUN pip install bx-python -RUN pip install scipy - -# boost and cmake for hap.py -RUN apt-get install libncurses5-dev -y -RUN apt-get install git -y -RUN apt-get install samtools -y -RUN apt-get install bzip2 -y -RUN apt-get install wget -y -RUN apt-get install libbz2-dev -y - +RUN apt-get clean -y RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ @@ -55,8 +24,6 @@ RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true rm -rf /var/lib/apt/lists/* && \ rm -rf /var/cache/oracle-jdk8-installer -RUN apt-get clean -y - # copy git repository into the image RUN mkdir -p /opt/hap.py-source COPY . /opt/hap.py-source/ @@ -67,14 +34,6 @@ RUN mkdir -p /opt/hap.py-data # download HG19 reference data WORKDIR /opt/hap.py-data -# This downloads from UCSC -RUN /opt/hap.py-source/src/sh/make_hg19.sh - -# this downloads our copy from box.com (once available) -# RUN wget https://illumina.app.box.com/s/1vwpu1w23p2qupukjjlrenq81mrtth1z/hg19.fa.bz2 -O /opt/hap.py-source/src/data/hg19.fa.bz2 -# RUN bzcat /opt/hap.py-source/src/data/hg19.fa.bz2 > /opt/hap.py-data/hg19.fa -RUN samtools faidx /opt/hap.py-data/hg19.fa - # get + install ant WORKDIR /opt RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.7-bin.tar.gz && \ @@ -82,9 +41,16 @@ RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.7-bin.tar.gz rm apache-ant-1.9.7-bin.tar.gz ENV PATH $PATH:/opt/apache-ant-1.9.7/bin -# run hap.py installer in the image +# run hap.py installer in the image, don't run tests since we don't have a reference file WORKDIR /opt/hap.py-source -RUN HG19=/opt/hap.py-data/hg19.fa python install.py /opt/hap.py --with-rtgtools +RUN python install.py /opt/hap.py --with-rtgtools --no-tests +WORKDIR /opt/hap.py + +# This downloads from UCSC +RUN /opt/hap.py-source/src/sh/make_hg19.sh && samtools faidx /opt/hap.py-data/hg19.fa +# Run tests +RUN /opt/hap.py-source/src/sh/run_tests.sh + +# remove source folder WORKDIR / RUN rm -rf /opt/hap.py-source - diff --git a/happy.requirements.txt b/happy.requirements.txt index ad76a18..4f64c87 100755 --- a/happy.requirements.txt +++ b/happy.requirements.txt @@ -1,7 +1,8 @@ Cython numpy pandas -pybedtools +psutil +pip pysam nose scipy From 2101539212caa5dd59e77ac97d28d45f41424eb6 Mon Sep 17 00:00:00 2001 From: Peter Krusche Date: Sat, 3 Jun 2017 12:16:22 +0100 Subject: [PATCH 4/6] HAP-318 update release notes --- RELEASES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 020b723..7360d7f 100755 --- a/RELEASES.md +++ b/RELEASES.md @@ -5,6 +5,8 @@ | Ticket | Description | |---------|------------------------------------------------------------------------------------| | HAP-317 | Update htslib to 1.4.1 and always check for BCF conversion errors | +| HAP-318 | Update rtgtools to 3.8.1 | +| HAP-319 | Slimmer docker image without hg19 reference | ## v0.3.8 From 8bc369f446a008449487c60e063db0c0b379c280 Mon Sep 17 00:00:00 2001 From: Peter Krusche Date: Sat, 3 Jun 2017 14:47:39 +0100 Subject: [PATCH 5/6] HAP-319 fix Dockerfile with tests --- Dockerfile.ubuntu-with-tests | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile.ubuntu-with-tests b/Dockerfile.ubuntu-with-tests index f414772..74ea227 100755 --- a/Dockerfile.ubuntu-with-tests +++ b/Dockerfile.ubuntu-with-tests @@ -31,9 +31,6 @@ COPY . /opt/hap.py-source/ # make minimal HG19 reference sequence RUN mkdir -p /opt/hap.py-data -# download HG19 reference data -WORKDIR /opt/hap.py-data - # get + install ant WORKDIR /opt RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.7-bin.tar.gz && \ @@ -44,11 +41,15 @@ ENV PATH $PATH:/opt/apache-ant-1.9.7/bin # run hap.py installer in the image, don't run tests since we don't have a reference file WORKDIR /opt/hap.py-source RUN python install.py /opt/hap.py --with-rtgtools --no-tests -WORKDIR /opt/hap.py +# download HG19 reference data # This downloads from UCSC +WORKDIR /opt/hap.py-data +ENV PATH $PATH:/opt/hap.py/bin RUN /opt/hap.py-source/src/sh/make_hg19.sh && samtools faidx /opt/hap.py-data/hg19.fa # Run tests +ENV HG19 /opt/hap.py-data/hg19.fa +WORKDIR /opt/hap.py RUN /opt/hap.py-source/src/sh/run_tests.sh # remove source folder From 0f5fe578f6e30cb09169e6469bfe87a79e9432e2 Mon Sep 17 00:00:00 2001 From: Peter Krusche Date: Sat, 3 Jun 2017 18:23:59 +0100 Subject: [PATCH 6/6] HAP-319 add bx-python dependency --- Dockerfile | 2 ++ Dockerfile.ubuntu-with-tests | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 5ee8df5..0685cbc 100755 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,8 @@ RUN apt-get install python2.7 python2.7-dev python \ RUN apt-get install software-properties-common python-software-properties -y RUN apt-get clean -y +RUN pip install bx-python + RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ apt-get update && \ diff --git a/Dockerfile.ubuntu-with-tests b/Dockerfile.ubuntu-with-tests index 74ea227..e23ce80 100755 --- a/Dockerfile.ubuntu-with-tests +++ b/Dockerfile.ubuntu-with-tests @@ -17,6 +17,8 @@ RUN apt-get install python2.7 python2.7-dev python \ RUN apt-get install software-properties-common python-software-properties -y RUN apt-get clean -y +RUN pip install bx-python + RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ apt-get update && \