Merge pull request #314 from bcgsc/release/v3.0.0

Release/v3.0.0

.github/CONTRIBUTING.md

@@ -1,4 +1,3 @@
-
 ## Getting Started
 
 If you are new to the project a good way to get started is by adding to the documentation, or adding unit tests where
@@ -47,7 +46,6 @@ mkdocs build
 The contents of the user manual can then be viewed by opening the build-docs/index.html
 in any available web browser (i.e. google-chrome, firefox, etc.)
 
-
 ## Deploy to PyPi
 
 Install deployment dependencies
@@ -68,21 +66,19 @@ Use twine to upload
 twine upload -r pypi dist/*
 ```
 
-
 ## Reporting a Bug
 
 Please make sure to search through the issues before reporting a bug to ensure there isn't
 already an open issue.
 
-
 ## Conventions
 
 ### Linting
 
 Use [black](https://github.com/psf/black) with strings off and line length 100
 
 ```bash
-black mavis -S -l 100
+black src/mavis -S -l 100
 ```
 
 ### Docstrings
@@ -112,7 +108,6 @@ def some_function(some_arg: List[str]) -> None:
 
 any column name which may appear in any of the intermediate or final output files must be defined in `mavis.constants.COLUMNS` as well as added to the [columns glossary](../outputs/columns)
 
-
 ### Tests
 
 - all new code must have unit tests in the tests subdirectory

.github/workflows/build.yml

@@ -3,15 +3,20 @@
 
 name: build
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
 
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
-
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+    name: python-${{ matrix.python-version }}
     steps:
     - uses: actions/checkout@v2
     - name: install machine dependencies
@@ -22,18 +27,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip "setuptools<58" wheel
-        pip install .[test]
-    - name: Lint with flake8
-      run: |
-        pip install flake8
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 mavis --count --select=E9,F63,F7,F82 --show-source --statistics
-    - name: Lint with black
-      run: |
-        pip install black
-        # stop the build if black needs to be run
-        black mavis -S -l 100 --check
+        python -m pip install --upgrade pip setuptools
+        pip install -e .[test]  # need editable to make sure the coverage reports correctly
     - name: install bwa
       run: |
         git clone https://github.com/lh3/bwa.git
@@ -45,20 +40,24 @@ jobs:
       run: |
         wget http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat
         chmod a+x blat
-    - name: run short tests with pytest
+    - name: set up .pth file
       run: |
-        export PATH=$PATH:$(pwd):$(pwd)/bwa
-        pytest tests -v --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov mavis --cov-report term --cov-report xml --durations=10
-      env:
-        RUN_FULL: 0
-      if: github.event_name != 'pull_request'
+        python tests/setup_subprocess_cov.py
     - name: run full tests with pytest
       run: |
         export PATH=$PATH:$(pwd):$(pwd)/bwa
-        pytest tests -v --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov mavis --cov-report term --cov-report xml --durations=10
+        export COVERAGE_PROCESS_START=$(pwd)/.coveragerc
+
+        pytest tests -v \
+          --junitxml=junit/test-results-${{ matrix.python-version }}.xml \
+          --cov mavis \
+          --cov tools.convert_annotations_format \
+          --cov-report term-missing \
+          --cov-report xml \
+          --durations=10 \
+          --cov-branch
       env:
         RUN_FULL: 1
-      if: github.event_name == 'pull_request'
     - name: Upload pytest test results
       uses: actions/upload-artifact@master
       with:
@@ -76,3 +75,34 @@ jobs:
         name: codecov-umbrella
         fail_ci_if_error: true
       if: matrix.python-version == 3.8
+  docker:
+    runs-on: ubuntu-latest
+    name: docker build
+    steps:
+      - uses: actions/checkout@v2
+      - name: build the docker container
+        run: |
+          docker build --file Dockerfile --tag bcgsc/mavis:latest .
+      - name: test the help menu
+        run: |
+          docker run bcgsc/mavis -h
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install workflow dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install mavis_config pandas snakemake
+      - uses: eWaterCycle/setup-singularity@v6
+        with:
+          singularity-version: 3.6.4
+      - name: docker2singularity
+        run:
+          docker run --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock --mount type=bind,source="$(pwd)",target=/output --privileged -t --rm singularityware/docker2singularity bcgsc/mavis:latest
+      - name: Run analysis with snakemake & singularity
+        run: |
+          # get the SIMG filename
+          export SNAKEMAKE_CONTAINER=$(ls *mavis*.simg)
+          snakemake -j 2 --configfile tests/mini-tutorial.config.json --use-singularity
+        if: always()

.github/workflows/publish.yml

@@ -8,10 +8,8 @@ on:
     types: [created]
 
 jobs:
-  deploy:
-
+  pypi:
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
@@ -30,3 +28,15 @@ jobs:
         python setup.py sdist bdist_wheel install
         twine check dist/*
         twine upload dist/*
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      - run: |
+          docker build --file Dockerfile --tag bcgsc/mavis:latest --tag bcgsc/mavis:${{ github.event.release.tag_name }} .
+      - run: docker push bcgsc/mavis:latest
+      - run: docker push bcgsc/mavis:${{ github.event.release.tag_name }}

.github/workflows/quick-tests.yml

@@ -0,0 +1,53 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: quick-tests
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+    name: python-${{ matrix.python-version }} quick
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        pip install .[test]
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 src/mavis --count --select=E9,F63,F7,F82 --show-source --statistics
+    - name: Lint with black
+      run: |
+        pip install black
+        # stop the build if black needs to be run
+        black src/mavis -S -l 100 --check
+    - name: install bwa
+      run: |
+        git clone https://github.com/lh3/bwa.git
+        cd bwa
+        git checkout v0.7.17
+        make
+        cd ..
+    - name: install blat
+      run: |
+        wget http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat
+        chmod a+x blat
+    - name: run short tests with pytest
+      run: |
+        export PATH=$PATH:$(pwd):$(pwd)/bwa
+        pytest tests -v \
+          --junitxml=junit/test-results-${{ matrix.python-version }}.xml \
+          --durations=10
+      env:
+        RUN_FULL: 0

.gitignore

@@ -1,7 +1,7 @@
 # python generated files
 /.eggs
 /coverage
-/venv
+/venv*
 /.coverage
 *.pyc
 *__pycache__
@@ -15,6 +15,7 @@ junit
 .tox
 *eggs/
 .mypy_cache
+.snakemake
 
 # aligners
 blat
@@ -32,3 +33,12 @@ junit
 /docs/package/mavis/*/*.md
 # don't ignore subpackage summary files
 !/docs/package/mavis/*/index.md
+docs/configuration/settings.md
+
+.snakemake
+output_dir*
+bin
+dag*
+tutorial_data
+reference_inputs
+tmp

.readthedocs.yml

@@ -20,4 +20,4 @@ python:
     -   method: pip
         path: .
         extra_requirements:
-            - docs
+            - doc

Dockerfile

@@ -0,0 +1,55 @@
+FROM python:3.7-slim-buster
+
+WORKDIR /app
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y git wget make gcc libz-dev
+
+# pysam dependencies
+RUN apt-get install -y libncurses5-dev zlib1g-dev libbz2-dev libncursesw5-dev liblzma-dev
+
+# install BWA
+RUN git clone https://github.com/lh3/bwa.git && \
+    cd bwa && \
+    git checkout v0.7.17 && \
+    make && \
+    cd .. && \
+    mv bwa/bwa /usr/local/bin
+
+# install minimap2
+RUN git clone https://github.com/lh3/minimap2.git && \
+    cd minimap2 && \
+    git checkout v2.24 && \
+    make && \
+    cd .. && \
+    mv minimap2/minimap2.1 /usr/local/bin
+
+# install blat dependencies
+RUN apt-get install -y libcurl4
+
+# install blat
+RUN wget http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat && \
+    chmod a+x blat && \
+    mv blat /usr/local/bin
+
+# install wtdbg2
+RUN git clone https://github.com/ruanjue/wtdbg2.git && \
+    cd wtdbg2 && \
+    make && \
+    cd .. && \
+    mv wtdbg2/wtdbg2 /usr/local/bin
+
+COPY setup.py setup.py
+COPY setup.cfg setup.cfg
+COPY MANIFEST.in MANIFEST.in
+COPY pyproject.toml pyproject.toml
+COPY src src
+COPY LICENSE LICENSE
+COPY README.md README.md
+
+# install python package
+RUN pip install -U setuptools pip wheel
+RUN pip install .
+RUN which mavis
+ENTRYPOINT [ "mavis" ]

MANIFEST.in

@@ -1,10 +1,6 @@
-recursive-include docs *
-recursive-include tests *.py 
-include tests/*/data/*
-recursive-include mavis *.py
-recursive-include tools *.pl *.py *.pm
-recursive-include tab *.py
+recursive-include src *.py *.json
+include src/mavis/py.typed
 include README.md
 include LICENSE
-prune docs/build
-prune docs/source/auto
+prune docs
+prune tests