From 2b2e8e45556a1a91c278f92bf01d46dd668d34cd Mon Sep 17 00:00:00 2001 From: Ola Tarkowska Date: Thu, 25 Jan 2018 15:40:37 +0000 Subject: [PATCH 1/6] update dockerfile --- docker/Dockerfile | 6 ++- docker/docker-compose.yml | 13 ++--- docker/run.sh | 111 ++++++++------------------------------ 3 files changed, 33 insertions(+), 97 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7fe4c4a1f..f17705dfc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,6 +6,10 @@ RUN yum -y update; yum clean all RUN yum -y install epel-release gcc bzip2 git wget; yum clean all +RUN yum -y install python34 python34-devel python34-setuptools +RUN easy_install-3.4 pip +RUN pip3 install -U pip virtualenv setuptools + # Install mysql tools RUN yum -y install mysql mysql-devel @@ -23,4 +27,4 @@ WORKDIR /home/emg RUN mkdir -p $HOME/emg -ENTRYPOINT ["/tmp/docker/run.sh", "docker"] +ENTRYPOINT ["/tmp/docker/run.sh"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2d870a889..6fb71f2ee 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,7 +5,7 @@ services: image: mysql:5.6 volumes: # - ./mysql:/var/lib/mysql:rw - - ../database/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro + - ../schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro environment: - MYSQL_ALLOW_EMPTY_PASSWORD=yes - MYSQL_DATABASE=emg @@ -19,18 +19,19 @@ services: - ../docker/config.yaml:/home/emg/emg/config.yaml:ro links: - mysql - - redis + - mongodb environment: - PYTHONUNBUFFERED=0 nginx: - image: nginx:1.10 + image: nginx:1.12 links: - app volumes: - - ../nginx/conf.d:/etc/nginx/conf.d + - ../nginx/conf.d:/etc/nginx/conf.d:ro + - ../results:/results:ro ports: - "80:80" - redis: - image: redis + mongodb: + image: mongo:3.4 diff --git a/docker/run.sh b/docker/run.sh index a591489ec..3189d4729 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -2,102 +2,33 @@ set -eux -entryPoint=${1:-} srcDir=${SRCDIR:-"${HOME}/src"} -condaDir=${CONDADIR:-"${HOME}/miniconda"} -pythonVer=${PYTHON_VERSION:-"3.6"} +venvDir=${VENVDIR:-"${HOME}/venv"} -check_conda() { - if [[ -f "/etc/profile.d/conda.sh" ]]; then - command -v conda >/dev/null && echo "conda detected in $PATH" - fi; - conda info - conda search "^python$" -} - -create_venv() { - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - /bin/bash miniconda.sh -b -p $condaDir - export PATH=$condaDir/bin:$PATH - rm -rf miniconda.sh - hash -r - conda install python=${pythonVer} -y - # conda update -q conda - conda info -a - python -V - - # TODO: wait until 4.4 https://github.com/conda/conda/issues/3200 - #conda create -q -n test-environment python=$pythonVer - #source activate test-environment - -} - -is_db_running() { - local conn=${1:-}; - - echo "DB startup..." - until mysql $conn -e 'show databases;'; do - >&2 echo "MySQL is unavailable - sleeping" - sleep 5 - done - >&2 echo "MySQL now accepts connections, creating database..." -} - -install_src() { - echo "Installing EMG API..." - pip install "git+git://github.com/ola-t/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi" - pip install -U $srcDir -} - -start() { - echo "EMG API Start up..." - - # emgcli check --deploy - emgcli migrate --fake-initial - emgcli collectstatic --noinput - - # development server - # emgcli runserver 0.0.0.0:8000 - emgdeploy -p ${HOME}/emg/emg.pid --bind 0.0.0.0:8000 --workers 5 --reload emgcli.wsgi:application - -} - - -# helpers -install() { - create_venv - python -V -} +rm -f $srcDir/var/django.pid -install_docker() { - install - pip install -U "django-redis>=4.4" - is_db_running "-u root -h mysql -P 3306" -} +python3 -V -docker() { - install_docker - install_src - start -} +virtualenv -p /usr/bin/python3 $venvDir --system-site-packages +# $HOME/venv/bin/pip install -U "django-redis>=4.4" -travis() { - # Install source, otherwise OSError: [Errno 2] No such file or directory. - install_src - cd $srcDir - python -V - python setup.py test -} +echo "Installing EMG API..." + $venvDir/bin/pip install -U "git+git://github.com/ola-t/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi" +$venvDir/bin/pip install -U $srcDir -travis_conda() { - export PATH=$condaDir/bin:$PATH - # Install source, otherwise OSError: [Errno 2] No such file or directory. - travis -} +echo "DB startup..." +until mysql -u root -h mysql -P 3306 -e 'show databases;'; do + >&2 echo "MySQL is unavailable - sleeping" + sleep 5 +done +>&2 echo "MySQL now accepts connections, creating database..." +echo "EMG API Start up..." -rm -f $srcDir/var/django.pid +# $HOME/venv/bin/emgcli check --deploy +# $venvDir/bin/emgcli migrate --fake-initial +$venvDir/bin/emgcli collectstatic --noinput -if [ ! -z ${entryPoint} ] ; then - eval $entryPoint -fi +# development server +# $HOME/venv/bin/emgcli runserver 0.0.0.0:8000 +$venvDir/bin/emgdeploy -p $HOME/emg/emg.pid --bind 0.0.0.0:8000 --workers 5 --reload emgcli.wsgi:application From de5d84cc727bce94ebf1bcdf2660652dc51043b7 Mon Sep 17 00:00:00 2001 From: Ola Tarkowska Date: Thu, 25 Jan 2018 16:31:48 +0000 Subject: [PATCH 2/6] update travis --- .travis.yml | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index c3c53802d..824a7525d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,45 +7,28 @@ python: - '3.6' env: - - VENV=miniconda - - VENV=virtualenv + - EMG_CONFIG=$TRAVIS_BUILD_DIR/travis/config.yaml services: - - docker + - mysql - mongodb before_install: - - pip install "flake8==3.4" pycodestyle pep8-naming - - python setup.py sdist + - mysql -e 'CREATE DATABASE emg;' + - python -V install: - # UnsatisfiableError: The following specifications were found to be in conflict: - # - python 3.4* - # - urllib3 -> ipaddress -> python 2.7* - - if [[ "$VENV" == "miniconda" && "$TRAVIS_PYTHON_VERSION" != "3.4" ]]; then - SRCDIR=$(pwd) PYTHON_VERSION=$TRAVIS_PYTHON_VERSION docker/run.sh install; - fi + - pip install "flake8==3.4" pycodestyle pep8-naming + - python setup.py sdist + - pip install "git+git://github.com/ola-t/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi" before_script: - - docker --version - # travis by default comes with MySQL 5.5 - - sudo service mysql stop - - docker run --rm --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql:5.6 - # optional redis - # - docker run --rm --name redis -p 6379:6379 -d redis - flake8 --version - flake8 -v . - - mkdir -p ~/emg && cp travis/config.yaml ~/emg/ script: - - if [[ "$VENV" == "miniconda" ]]; then - SRCDIR=$(pwd) docker/run.sh travis_conda; - else - SRCDIR=$(pwd) docker/run.sh travis; - fi - -after_script: - - docker stop mysql #redis + - pip install -U . + - python setup.py test notifications: slack: From 65cfe02dd6fe5f699d5b9782d5d2560f848be10f Mon Sep 17 00:00:00 2001 From: Ola Tarkowska Date: Fri, 23 Feb 2018 14:11:23 +0000 Subject: [PATCH 3/6] fix study filter on runs --- emgapi/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emgapi/filters.py b/emgapi/filters.py index b04690d7c..e84fb9626 100644 --- a/emgapi/filters.py +++ b/emgapi/filters.py @@ -558,7 +558,7 @@ def filter_sample_accession(self, qs, name, value): help_text='Study accession') def filter_study_accession(self, qs, name, value): - return qs.filter(sample__studies__accession=value) + return qs.filter(study__accession=value) # include include = django_filters.CharFilter( From 0c5701074f8f36a9b5742b0ee8a631f28f13f735 Mon Sep 17 00:00:00 2001 From: Ola Tarkowska Date: Fri, 23 Feb 2018 14:14:42 +0000 Subject: [PATCH 4/6] bump version 1.0.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9566b02e8..7ac952099 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ _requirements = os.path.join(_base, 'requirements.txt') _requirements_test = os.path.join(_base, 'requirements-test.txt') -version = "1.0.6" +version = "1.0.7" install_requirements = [] with open(_requirements) as f: From 931d2683cd45392e2fe49c764b15ca4650ca9f15 Mon Sep 17 00:00:00 2001 From: Ola Tarkowska Date: Fri, 23 Feb 2018 14:25:55 +0000 Subject: [PATCH 5/6] update README --- README.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.rst b/README.rst index e3c593483..3d54a74a1 100644 --- a/README.rst +++ b/README.rst @@ -20,14 +20,6 @@ Create configuration file in `~/path/to/config.yaml `_. Install ------- -Install Miniconda:: - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh - - export PATH=~/miniconda/bin:$PATH - - Install application:: pip install "git+git://github.com/ola-t/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi" From da403df99220907867b26a1e57638c2c58c52827 Mon Sep 17 00:00:00 2001 From: Ola Tarkowska Date: Fri, 23 Feb 2018 14:37:27 +0000 Subject: [PATCH 6/6] don't display functional analysis for amplicon runs --- emgapianns/views.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/emgapianns/views.py b/emgapianns/views.py index 7eb2644ba..1b98eb9af 100644 --- a/emgapianns/views.py +++ b/emgapianns/views.py @@ -260,9 +260,11 @@ class AnalysisGoTermRelationshipViewSet( # NOQA def get_queryset(self): accession = self.kwargs['accession'] release_version = self.kwargs['release_version'] + qs = emg_models.AnalysisJob.objects.exclude( + experiment_type__experiment_type='amplicon') job = get_object_or_404( - emg_models.AnalysisJob, accession=accession, - pipeline__release_version=release_version + qs, accession=accession, + pipeline__release_version=release_version, ) analysis = None @@ -297,9 +299,11 @@ class AnalysisGoSlimRelationshipViewSet( # NOQA def get_queryset(self): accession = self.kwargs['accession'] release_version = self.kwargs['release_version'] + qs = emg_models.AnalysisJob.objects.exclude( + experiment_type__experiment_type='amplicon') job = get_object_or_404( - emg_models.AnalysisJob, accession=accession, - pipeline__release_version=release_version + qs, accession=accession, + pipeline__release_version=release_version, ) analysis = None @@ -334,9 +338,11 @@ class AnalysisInterproIdentifierRelationshipViewSet( # NOQA def get_queryset(self): accession = self.kwargs['accession'] release_version = self.kwargs['release_version'] + qs = emg_models.AnalysisJob.objects.exclude( + experiment_type__experiment_type='amplicon') job = get_object_or_404( - emg_models.AnalysisJob, accession=accession, - pipeline__release_version=release_version + qs, accession=accession, + pipeline__release_version=release_version, ) analysis = None