From 74687f8022089e23572948418dd12a9dad9ebd0e Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 19 Apr 2021 09:24:50 -0500 Subject: [PATCH 01/44] Update gitignore --- .gitignore | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 95493ef..7b75539 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -.deps/ -docker-requirements.txt - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -11,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python +env/ build/ develop-eggs/ dist/ @@ -26,7 +24,7 @@ wheels/ *.egg-info/ .installed.cfg *.egg -MANIFEST +_version.py # PyInstaller # Usually these files are written by a python script from a template @@ -48,7 +46,6 @@ nosetests.xml coverage.xml *.cover .hypothesis/ -.pytest_cache/ # Translations *.mo @@ -57,7 +54,6 @@ coverage.xml # Django stuff: *.log local_settings.py -db.sqlite3 # Flask stuff: instance/ @@ -84,14 +80,17 @@ celerybeat-schedule # SageMath parsed files *.sage.py -# Environments +# dotenv .env + +# virtualenv .venv -env/ venv/ ENV/ -env.bak/ -venv.bak/ + +# direnv +.envrc +.direnv/ # Spyder project settings .spyderproject @@ -105,3 +104,6 @@ venv.bak/ # mypy .mypy_cache/ + +# VSCode +.vscode/ From f22fcccd62b9f12afa616598e736d821f91f099f Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 19 Apr 2021 09:40:43 -0500 Subject: [PATCH 02/44] Setup update for setuptools_scm --- setup.py | 163 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 144 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 152e737..2ea4ed6 100644 --- a/setup.py +++ b/setup.py @@ -1,28 +1,153 @@ from setuptools import setup, find_packages +#!/usr/bin/env python + +import importlib +import os +import subprocess +from textwrap import dedent +from types import SimpleNamespace + +from setuptools import Command, find_packages, setup + +GIT_REPO = "aliquot-maf-tools" +PACKAGE = "aliquotmaf" + +PYPI_REPO = "bioinf-{}".format(PACKAGE) +GIT_REPO_URL = "https://github.com/NCI-GDC/{}".format(GIT_REPO) + +INSTALL_REQUIRES = [] + +TESTS_REQUIRES = [ + 'mock', + 'pytest', + 'pytest-cov', +] + +DEV_REQUIRES = [ + 'detect-secrets==0.13.1', + 'isort', + 'flake8', + 'pre-commit', +] + + +GIT_COMMANDS = SimpleNamespace( + branch=["git", "rev-parse", "--abbrev-ref", "HEAD"], + commit=["git", "rev-list", "--count", "HEAD"], + hash=["git", "rev-parse", "HEAD"], + shorthash=["git", "rev-parse", "--short", "HEAD"], +) + + + +try: + # Set versions if version file exists + mod = importlib.import_module("{}".format(PACKAGE)) + __pypi_version__ = mod.__version__ +except Exception: + # Set defaults otherwise + __pypi_version__ = '0.0.0' + + +class PrintVersion(Command): + description = "Print out specified version, default long version." + user_options = [ + ("pypi", None, "Print package version."), + ("short", None, "Print semantic version."), + ("hash", None, "Print commit hash."), + ] + + def initialize_options(self): + self.pypi = False + self.short = False + self.hash = False + + def finalize_options(self): + pass + + def run(self): + if self.pypi: + print(__pypi_version__) + elif self.short: + print(__short_version__) + elif self.hash: + try: + commit_hash = call_subprocess(GIT_COMMANDS.hash) + except Exception: + print('') + else: + print(commit_hash) + else: + print(__long_version__) + + + + +def call_subprocess(cmd: list): + """Return stdout of given command.""" + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + stdout, _ = p.communicate() + return stdout.decode().strip() + + +class Requirements(Command): + description = "foobar" + user_options = [ + ("install", None, "Bundles only install requirements"), + ("test", None, "Bundles only install requirements"), + ("dev", None, "Bundles all requirements"), + ] + + def initialize_options(self): + self.install = False + self.test = False + self.dev = False + + def finalize_options(self): + assert self.install + self.test + self.dev == 1, "Provide only one arg" + + def run(self): + path = os.path.join(".", "requirements.in") + if self.dev: + reqs = INSTALL_REQUIRES + TESTS_REQUIRES + DEV_REQUIRES + elif self.test: + reqs = INSTALL_REQUIRES + TESTS_REQUIRES + elif self.install: + reqs = INSTALL_REQUIRES + self.write_requirements(path, reqs) + return + + def write_requirements(self, path, reqs): + with open(path, "w") as fh: + fh.write("\n".join(reqs) + "\n") + + setup( - name = "aliquot-maf-tools", - author = "Kyle Hernandez", - version = 0.1, - description = "Tools for creating and filtering aliquot-level MAFs", - license = "Apache 2.0", - packages = find_packages(), - setup_requires = [ - "pytest-runner" - ], - tests_require = [ - "pytest" - ], - classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - ], + name=PYPI_REPO, + description="Mutation Annotation Format (MAF) library", + version=__pypi_version__, + url=GIT_REPO_URL, + python_requires=">=3.6", + setup_requires=['setuptools_scm'], + use_scm_version={ + "write_to": os.path.join(f"{PACKAGE}", "_version.py"), + "fallback_version": __pypi_version__, + }, + packages=find_packages(), + package_data={'maflib': ['schemas/*.json']}, + install_requires=INSTALL_REQUIRES, + tests_require=TESTS_REQUIRES, + cmdclass={ + "capture_requirements": Requirements, + "print_version": PrintVersion, + }, + include_package_data=True, entry_points= { 'console_scripts': [ 'aliquot-maf-tools = aliquotmaf.__main__:main' ] } ) + +# __END__ From d7783907db3bd0665e74bc6a281501a0a15c84d5 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 19 Apr 2021 09:41:01 -0500 Subject: [PATCH 03/44] Import version --- aliquotmaf/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aliquotmaf/__init__.py b/aliquotmaf/__init__.py index e69de29..8b93748 100644 --- a/aliquotmaf/__init__.py +++ b/aliquotmaf/__init__.py @@ -0,0 +1,5 @@ +try: + from aliquotmaf._version import version + __version__ = version +except ImportError: + __version__ = '0' From eec34914e2002954f00957703cb0fcb59c121364 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 19 Apr 2021 09:41:15 -0500 Subject: [PATCH 04/44] Build tool scripts --- Jenkinsfile | 104 +++++++++++++++++++++++++++++++++++++++++++ Makefile | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 Jenkinsfile create mode 100644 Makefile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..6fe2292 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,104 @@ +#!groovy + +PROJECT_NAME = "aliquot-maf-tools" +PYENV_VERSION = "3.6-dev" + +BRANCH_NAME = env.BRANCH_NAME +GIT_HASH = "" +VERSION = "" + +PROXY = "http://cloud-proxy:3128" + +pipeline { + agent any + environment { + TWINE_REPOSITORY_URL = credentials("${BRANCH_NAME == 'main' ? 'twine_repository_url_prod' : 'twine_repository_url'}") + TWINE_USERNAME = credentials('twine_username') + TWINE_PASSWORD = credentials('twine_password') + QUAY_USERNAME = credentials('QUAY_USERNAME') + QUAY_PASSWORD = credentials('QUAY_PASSWORD') + } + options { + disableConcurrentBuilds() + skipStagesAfterUnstable() + } + + stages { + stage('Init') { + steps { + vbash 'make version' + + script { + GIT_HASH = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() + VERSION = sh(script: "make print-version BRANCH_NAME=${BRANCH_NAME}", returnStdout: true).trim() + PYPI_VERSION = sh(script: "make print-pypi", returnStdout: true).trim() + currentBuild.displayName = "#${currentBuild.number} - ${VERSION}" + } + + echo "Version: ${VERSION}" + } + } + stage('Docker Build') { + steps { + vbash "make build-docker PROXY=${PROXY}" + } + } + stage('Docker Test') { + steps { + sh 'make test-docker' + } + } + stage('Docker Publish Staging') { + when { + anyOf { + branch 'feat/*' + branch 'feature/*' + branch 'develop' + branch 'hotfix/*' + branch 'release/*' + } + } + steps { + sh 'make publish-staging' + } + } + stage('Docker Publish Release') { + when { + anyOf { + branch 'main' + } + } + steps { + sh 'make publish-release' + } + } + stage('PyPI Publish Branch') { + when { + anyOf { + branch 'main' + branch 'develop' + branch 'hotfix/*' + branch 'release/*' + } + } + steps { + echo "Building PyPI Version: ${PYPI_VERSION}" + sh "pip install --user twine wheel" + vbash "make build-pypi" + vbash "make publish-pypi" + } + } + } +} + +def vbash(command) { + sh """#!/bin/bash + eval \"\$(pyenv init -)\" + eval \"\$(pyenv virtualenv-init -)\" + + pyenv virtualenv ${PYENV_VERSION} ${PROJECT_NAME}-venv || true + pyenv activate ${PROJECT_NAME}-venv + + ${command} + """ +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3bbfbf2 --- /dev/null +++ b/Makefile @@ -0,0 +1,125 @@ +# UPDATE ME +REPO = aliquot-maf-tools + +# UPDATE ME +MODULE = aliquotmaf + +BRANCH_NAME?=unknown +GIT_SHORT_HASH:=$(shell git rev-parse --short HEAD) + +PYPI_VERSION:=$(shell python3 setup.py -q print_version --pypi) +COMMIT_HASH:=$(shell python3 setup.py -q print_version --hash) + +DOCKER_REPO := quay.io/ncigdc +DOCKER_IMAGE := ${DOCKER_REPO}/${REPO}:${LONG_VERSION} +DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH} +DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest + +TWINE_REPOSITORY_URL?="" + +.PHONY: version version-* print-* +version: + @echo --- VERSION: ${PYPI_VERSION} --- + +version-docker: + @echo ${DOCKER_IMAGE} + @echo ${DOCKER_IMAGE_COMMIT} + +.PHONY: docker-login +docker-login: + docker login -u="${QUAY_USERNAME}" -p="${QUAY_PASSWORD}" quay.io + + +.PHONY: build build-* clean init init-* lint requirements run version +init: init-pip init-hooks + +# Include next line if publishing to Jenkins +# --extra-index-url ${TWINE_REPOSITORY_URL} +init-pip: + @echo + @echo -- Installing pip packages -- + pip3 install \ + --no-cache-dir \ + -r requirements.txt + python3 setup.py develop + +init-hooks: + @echo + @echo -- Installing Precommit Hooks -- + pre-commit install + +init-venv: + @echo + PIP_REQUIRE_VIRTUALENV=true pip3 install --upgrade pip-tools + +clean: + rm -rf ./build/ + rm -rf ./dist/ + rm -rf ./*.egg-info/ + +lint: + @echo + @echo -- Lint -- + python3 -m flake8 \ + --ignore=E501,F401,E302,E502,E126,E731,W503,W605,F841,C901 \ + ${MODULE}/ + +run: + bin/run + +# Include next line if publishing to Jenkins +# --extra-index-url ${TWINE_REPOSITORY_URL} +requirements: init-venv + python3 setup.py -q capture_requirements --dev + pip-compile -o requirements.txt requirements.in + +.PHONY: build build-* + +build: build-docker + +build-docker: + @echo + @echo -- Building docker -- + python3 setup.py build + mkdir -p dist + cp -r build/lib/* dist/ + cp -r bin/ dist/ + cp -f Makefile requirements.txt README.md setup.py dist/ + docker build . \ + --file ./Dockerfile \ + --build-arg http_proxy=${PROXY} \ + --build-arg https_proxy=${PROXY} \ + -t "${DOCKER_IMAGE_COMMIT}" \ + -t "${DOCKER_IMAGE}" \ + -t "${DOCKER_IMAGE_LATEST}" + +build-pypi: + @echo + @echo Building wheel - ${PYPI_VERSION} + python3 setup.py bdist_wheel -b ${MODULE}.egg-info + +.PHONY: test test-* +test: lint test-unit + +test-unit: + @echo + @echo -- Unit Test -- + python3 -m pytest --cov-report term-missing \ + --junitxml=build/unit-test.xml \ + --cov=${MODULE} \ + tests/ + +test-docker: + @echo + @echo -- Running Docker Test -- + docker run --rm ${DOCKER_IMAGE_LATEST} test + +.PHONY: publish publish-* +publish: docker-login + docker push ${DOCKER_IMAGE_COMMIT} + docker push ${DOCKER_IMAGE} + +publish-pypi: dist/*.whl + @echo + @echo Publishing wheel + python3 -m twine upload $(shell ls -1 dist/*.whl | head -1) From 3b8ab903b3ee9bf7f9458b665076b22bed18916c Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 20 Apr 2021 14:56:06 -0500 Subject: [PATCH 05/44] Bump to py36. use scm --- setup.py | 70 ++++++++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/setup.py b/setup.py index 2ea4ed6..c2fcdde 100644 --- a/setup.py +++ b/setup.py @@ -16,19 +16,22 @@ PYPI_REPO = "bioinf-{}".format(PACKAGE) GIT_REPO_URL = "https://github.com/NCI-GDC/{}".format(GIT_REPO) -INSTALL_REQUIRES = [] +INSTALL_REQUIRES = [ + "bioinf-maflib", + "pysam", +] -TESTS_REQUIRES = [ - 'mock', - 'pytest', - 'pytest-cov', +TESTS_REQUIRE = [ + "mock", + "pytest", + "pytest-cov", ] DEV_REQUIRES = [ - 'detect-secrets==0.13.1', - 'isort', - 'flake8', - 'pre-commit', + "detect-secrets==0.13.1", + "isort", + "flake8", + "pre-commit", ] @@ -40,27 +43,26 @@ ) - try: # Set versions if version file exists mod = importlib.import_module("{}".format(PACKAGE)) __pypi_version__ = mod.__version__ except Exception: # Set defaults otherwise - __pypi_version__ = '0.0.0' + __pypi_version__ = "0.0.0" class PrintVersion(Command): description = "Print out specified version, default long version." user_options = [ ("pypi", None, "Print package version."), - ("short", None, "Print semantic version."), + ("docker", None, "Print Docker-friendly package version."), ("hash", None, "Print commit hash."), ] def initialize_options(self): self.pypi = False - self.short = False + self.docker = False self.hash = False def finalize_options(self): @@ -69,19 +71,17 @@ def finalize_options(self): def run(self): if self.pypi: print(__pypi_version__) - elif self.short: - print(__short_version__) + elif self.docker: + print(__pypi_version__.replace("+", ".")) elif self.hash: try: commit_hash = call_subprocess(GIT_COMMANDS.hash) except Exception: - print('') + print("") else: print(commit_hash) else: - print(__long_version__) - - + print(__pypi_version__) def call_subprocess(cmd: list): @@ -92,29 +92,24 @@ def call_subprocess(cmd: list): class Requirements(Command): - description = "foobar" + description = "Write out dev-requirements.in file." user_options = [ - ("install", None, "Bundles only install requirements"), - ("test", None, "Bundles only install requirements"), ("dev", None, "Bundles all requirements"), ] def initialize_options(self): - self.install = False - self.test = False self.dev = False def finalize_options(self): - assert self.install + self.test + self.dev == 1, "Provide only one arg" + pass def run(self): - path = os.path.join(".", "requirements.in") + REQUIREMENT = ["-c requirements.txt"] if self.dev: - reqs = INSTALL_REQUIRES + TESTS_REQUIRES + DEV_REQUIRES - elif self.test: - reqs = INSTALL_REQUIRES + TESTS_REQUIRES - elif self.install: - reqs = INSTALL_REQUIRES + reqs = REQUIREMENT + DEV_REQUIRES + TESTS_REQUIRE + path = "dev-requirements.in" + else: + raise ValueError("Choose one of install, test, or dev") self.write_requirements(path, reqs) return @@ -125,29 +120,24 @@ def write_requirements(self, path, reqs): setup( name=PYPI_REPO, - description="Mutation Annotation Format (MAF) library", + description="Tools for creating and filtering aliquot-level MAFs", version=__pypi_version__, url=GIT_REPO_URL, python_requires=">=3.6", - setup_requires=['setuptools_scm'], + setup_requires=["setuptools_scm"], use_scm_version={ "write_to": os.path.join(f"{PACKAGE}", "_version.py"), "fallback_version": __pypi_version__, }, packages=find_packages(), - package_data={'maflib': ['schemas/*.json']}, install_requires=INSTALL_REQUIRES, - tests_require=TESTS_REQUIRES, + tests_require=TESTS_REQUIRE, cmdclass={ "capture_requirements": Requirements, "print_version": PrintVersion, }, include_package_data=True, - entry_points= { - 'console_scripts': [ - 'aliquot-maf-tools = aliquotmaf.__main__:main' - ] - } + entry_points={"console_scripts": ["aliquot-maf-tools = aliquotmaf.__main__:main"]}, ) # __END__ From 254fdad842426a6da0e6b8778c680fc151ee318f Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 20 Apr 2021 14:56:20 -0500 Subject: [PATCH 06/44] Dev requirements --- dev-requirements.in | 8 ++++ dev-requirements.txt | 95 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 dev-requirements.in create mode 100644 dev-requirements.txt diff --git a/dev-requirements.in b/dev-requirements.in new file mode 100644 index 0000000..a42ee28 --- /dev/null +++ b/dev-requirements.in @@ -0,0 +1,8 @@ +-c requirements.txt +detect-secrets==0.13.1 +isort +flake8 +pre-commit +mock +pytest +pytest-cov diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..313bd61 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,95 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=dev-requirements.txt dev-requirements.in +# +--extra-index-url https://nexus.osdc.io/repository/pypi-all/simple + +appdirs==1.4.4 + # via virtualenv +attrs==20.3.0 + # via pytest +certifi==2020.12.5 + # via requests +cfgv==3.2.0 + # via pre-commit +chardet==4.0.0 + # via requests +coverage==5.5 + # via pytest-cov +detect-secrets==0.13.1 + # via -r dev-requirements.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +flake8==3.9.1 + # via -r dev-requirements.in +identify==2.2.3 + # via pre-commit +idna==2.10 + # via requests +importlib-metadata==4.0.0 + # via + # flake8 + # pluggy + # pre-commit + # pytest + # virtualenv +importlib-resources==5.1.2 + # via + # pre-commit + # virtualenv +iniconfig==1.1.1 + # via pytest +isort==5.8.0 + # via -r dev-requirements.in +mccabe==0.6.1 + # via flake8 +mock==4.0.3 + # via -r dev-requirements.in +nodeenv==1.6.0 + # via pre-commit +packaging==20.9 + # via pytest +pluggy==0.13.1 + # via pytest +pre-commit==2.12.1 + # via -r dev-requirements.in +py==1.10.0 + # via pytest +pycodestyle==2.7.0 + # via flake8 +pyflakes==2.3.1 + # via flake8 +pyparsing==2.4.7 + # via packaging +pytest-cov==2.11.1 + # via -r dev-requirements.in +pytest==6.2.3 + # via + # -r dev-requirements.in + # pytest-cov +pyyaml==5.4.1 + # via + # detect-secrets + # pre-commit +requests==2.25.1 + # via detect-secrets +six==1.15.0 + # via virtualenv +toml==0.10.2 + # via + # pre-commit + # pytest +typing-extensions==3.7.4.3 + # via importlib-metadata +urllib3==1.26.4 + # via requests +virtualenv==20.4.4 + # via pre-commit +zipp==3.4.1 + # via + # importlib-metadata + # importlib-resources From 7c0ac65ddf1c3e0939d423115fdeec73f8a2c16e Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 20 Apr 2021 14:56:28 -0500 Subject: [PATCH 07/44] Py36 Docker --- Dockerfile | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7de3835..979cb4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,15 @@ -FROM python:3.5 +FROM quay.io/ncigdc/bio-python:3.6 -# Copy over source -COPY . aliquot-maf-tools/ -WORKDIR /aliquot-maf-tools +ENV BINARY=maflib -# Installing dependencies -RUN bash -c "./repo-install.sh" && \ - pip install -r docker-requirements.txt +COPY ./dist/ /opt -# Install bio-submitter-qc -RUN pip install . +WORKDIR /opt + +RUN make init-pip \ + && ln -s /opt/bin/${BINARY} /bin/${BINARY} \ + && chmod +x /bin/${BINARY} + +ENTRYPOINT ["/bin/aliquotmaf"] + +CMD ["--help"] From 000dc3d5233992bd27d4f4e9c315684e7219e9e6 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 20 Apr 2021 14:56:36 -0500 Subject: [PATCH 08/44] Makefile updates --- Makefile | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3bbfbf2..26c3be3 100644 --- a/Makefile +++ b/Makefile @@ -4,18 +4,19 @@ REPO = aliquot-maf-tools # UPDATE ME MODULE = aliquotmaf -BRANCH_NAME?=unknown GIT_SHORT_HASH:=$(shell git rev-parse --short HEAD) PYPI_VERSION:=$(shell python3 setup.py -q print_version --pypi) +DOCKER_VERSION:=$(shell python3 setup.py -q print_version --docker) COMMIT_HASH:=$(shell python3 setup.py -q print_version --hash) DOCKER_REPO := quay.io/ncigdc -DOCKER_IMAGE := ${DOCKER_REPO}/${REPO}:${LONG_VERSION} +DOCKER_IMAGE := ${DOCKER_REPO}/${REPO}:${DOCKER_VERSION} DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH} DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest TWINE_REPOSITORY_URL?="" +PIP_EXTRA_INDEX_URL?= .PHONY: version version-* print-* version: @@ -40,6 +41,7 @@ init-pip: @echo -- Installing pip packages -- pip3 install \ --no-cache-dir \ + -r dev-requirements.txt \ -r requirements.txt python3 setup.py develop @@ -67,11 +69,14 @@ lint: run: bin/run -# Include next line if publishing to Jenkins -# --extra-index-url ${TWINE_REPOSITORY_URL} -requirements: init-venv +requirements: init-venv requirements-prod requirements-dev + +requirements-dev: python3 setup.py -q capture_requirements --dev - pip-compile -o requirements.txt requirements.in + pip-compile -o dev-requirements.txt dev-requirements.in + +requirements-prod: + pip-compile -o requirements.txt .PHONY: build build-* @@ -84,7 +89,7 @@ build-docker: mkdir -p dist cp -r build/lib/* dist/ cp -r bin/ dist/ - cp -f Makefile requirements.txt README.md setup.py dist/ + cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/ docker build . \ --file ./Dockerfile \ --build-arg http_proxy=${PROXY} \ From e588ab9d1bf4120bfc0452536d2fe098f58016a7 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 20 Apr 2021 14:56:44 -0500 Subject: [PATCH 09/44] Update requirements --- requirements.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9593b02..63271ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,12 @@ -pysam==0.14.1 -git+https://github.com/NCI-GDC/maf-lib.git@775ceca93b431d6a2e211b095fda51b8eca59bac#egg=maf-lib +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements.txt +# +--extra-index-url https://nexus.osdc.io/repository/pypi-all/simple + +bioinf-maflib==1.0.0 + # via bioinf-aliquotmaf (setup.py) +pysam==0.16.0.1 + # via bioinf-aliquotmaf (setup.py) From f111391b7461bf4088cbbf741cdca89989dabf24 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 09:00:33 -0500 Subject: [PATCH 10/44] Remove travis CI --- .travis.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9ed55b5..0000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: python - -python: - - "3.5.2" - - "3.6.9" - -before_install: - - pip freeze - -install: - - pip install -r requirements.txt - - pip install -r test-requirements.txt - - python setup.py install - - -before_script: - - pip freeze - -script: "python -m pytest -v tests/" From c81d5b1ecd2c09adda12409a63f8328358e5f45f Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 09:00:49 -0500 Subject: [PATCH 11/44] Python config files --- pyproject.toml | 7 +++++++ tox.ini | 4 ++++ 2 files changed, 11 insertions(+) create mode 100644 pyproject.toml create mode 100644 tox.ini diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..be6dc15 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.black] +line-length = 88 +skip-string-normalization = true +target-version = ['py36'] + +[tool.flake8] +max-line-length = 88 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..eb12fa5 --- /dev/null +++ b/tox.ini @@ -0,0 +1,4 @@ +[flake8] +ignore = E501,F401,E302,E502,E126,E731,W503,W605,F841,C901 +exclude = .git,__pycache__,build,dist +max-complexity = 10 From 0cb45e461cd0d1ced34f268e682414b602b083d0 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 09:10:59 -0500 Subject: [PATCH 12/44] Linting --- aliquotmaf/__init__.py | 1 + aliquotmaf/annotators/hotspot.py | 4 ++-- aliquotmaf/annotators/nontcga_exac.py | 2 +- aliquotmaf/converters/builder.py | 6 +++--- aliquotmaf/converters/formatters.py | 3 +-- aliquotmaf/filters/gdc_blacklist.py | 6 +++--- aliquotmaf/filters/gdc_pon.py | 2 +- aliquotmaf/filters/multiallelic.py | 2 +- aliquotmaf/filters/nonexonic.py | 2 +- aliquotmaf/filters/normal_depth.py | 2 +- aliquotmaf/filters/offtarget.py | 2 +- aliquotmaf/merging/overlap_set.py | 4 ++-- aliquotmaf/merging/record_merger/base.py | 12 ++++++------ aliquotmaf/merging/record_merger/impl/v1_0.py | 8 ++++---- .../subcommands/mask_merged_aliquot/__main__.py | 2 +- aliquotmaf/subcommands/merge_aliquot/__main__.py | 2 +- aliquotmaf/subcommands/vcf_to_aliquot/__main__.py | 3 +-- .../vcf_to_aliquot/extractors/effects.py | 2 +- .../vcf_to_aliquot/extractors/genotypes.py | 14 +++++++------- .../vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py | 6 +++--- 20 files changed, 42 insertions(+), 43 deletions(-) diff --git a/aliquotmaf/__init__.py b/aliquotmaf/__init__.py index 8b93748..b5cb43f 100644 --- a/aliquotmaf/__init__.py +++ b/aliquotmaf/__init__.py @@ -1,5 +1,6 @@ try: from aliquotmaf._version import version + __version__ = version except ImportError: __version__ = '0' diff --git a/aliquotmaf/annotators/hotspot.py b/aliquotmaf/annotators/hotspot.py index 80337b8..ca1c40c 100644 --- a/aliquotmaf/annotators/hotspot.py +++ b/aliquotmaf/annotators/hotspot.py @@ -1,5 +1,5 @@ """ -Implements the hotspots annotation. +Implements the hotspots annotation. """ from __future__ import absolute_import @@ -25,7 +25,7 @@ def setup(cls, scheme, source): head = line.rstrip("\r\n").lower().split("\t") assert all( [i in head for i in ["hugo_symbol", "change", "type"]] - ), self.logger.error("Unexpected header {0} found!".format(head)) + ), cls.logger.error("Unexpected header {0} found!".format(head)) else: dat = dict(zip(head, line.rstrip("\r\n").split("\t"))) if dat["hugo_symbol"] not in hsdic: diff --git a/aliquotmaf/annotators/nontcga_exac.py b/aliquotmaf/annotators/nontcga_exac.py index e96bfae..95f13bd 100644 --- a/aliquotmaf/annotators/nontcga_exac.py +++ b/aliquotmaf/annotators/nontcga_exac.py @@ -1,5 +1,5 @@ """ -Annotates the population frequency from the non-TCGA ExAC file. +Annotates the population frequency from the non-TCGA ExAC file. """ from __future__ import absolute_import diff --git a/aliquotmaf/converters/builder.py b/aliquotmaf/converters/builder.py index ed5e415..49c3dd7 100644 --- a/aliquotmaf/converters/builder.py +++ b/aliquotmaf/converters/builder.py @@ -10,7 +10,7 @@ class Builder(metaclass=abc.ABCMeta): """ - Specify an abstract interface for creating column builder + Specify an abstract interface for creating column builder objects. """ @@ -68,7 +68,7 @@ def build(cls, column, value=None, default=None, scheme=None): class GenericColumnBuilder(Builder): """ - Generic builder class that should handle the majority of the cases. + Generic builder class that should handle the majority of the cases. """ @classmethod @@ -85,7 +85,7 @@ def build(cls, key, value, scheme, default=None, fn=None, **kwargs): class GenericSequenceBuilder(Builder): """ - Generic sequence builder class that should handle the majority of the cases. + Generic sequence builder class that should handle the majority of the cases. """ @classmethod diff --git a/aliquotmaf/converters/formatters.py b/aliquotmaf/converters/formatters.py index be54bdf..887bd49 100644 --- a/aliquotmaf/converters/formatters.py +++ b/aliquotmaf/converters/formatters.py @@ -1,6 +1,5 @@ """ -A module containing formatting utility functions needed when converting from a -VCF to a MAF. +A module containing formatting utility functions needed when converting from a VCF to a MAF. """ diff --git a/aliquotmaf/filters/gdc_blacklist.py b/aliquotmaf/filters/gdc_blacklist.py index b5fbeb5..f381aba 100644 --- a/aliquotmaf/filters/gdc_blacklist.py +++ b/aliquotmaf/filters/gdc_blacklist.py @@ -1,5 +1,5 @@ """ -Applies the GDC Blacklist filter. +Applies the GDC Blacklist filter. """ from __future__ import absolute_import @@ -29,12 +29,12 @@ def setup(cls, source): assert ( "tumor_aliquot_id" in head ), 'Required column "tumor_aliquot_id" missing from blacklist file {0}'.format( - self.source + cls.source ) assert ( "tag" in head ), 'Required column "tag" missing from blacklist file {0}'.format( - self.source + cls.source ) else: # Parse row into dict diff --git a/aliquotmaf/filters/gdc_pon.py b/aliquotmaf/filters/gdc_pon.py index 7904b8b..1db9f9b 100644 --- a/aliquotmaf/filters/gdc_pon.py +++ b/aliquotmaf/filters/gdc_pon.py @@ -1,5 +1,5 @@ """ -Applies the GDC PON filter. We don't care about alleles, just positions. +Applies the GDC PON filter. We don't care about alleles, just positions. """ from __future__ import absolute_import diff --git a/aliquotmaf/filters/multiallelic.py b/aliquotmaf/filters/multiallelic.py index efb8b57..3b2ecb3 100644 --- a/aliquotmaf/filters/multiallelic.py +++ b/aliquotmaf/filters/multiallelic.py @@ -1,5 +1,5 @@ """ -Applies the multiallelic filter. +Applies the multiallelic filter. """ from __future__ import absolute_import diff --git a/aliquotmaf/filters/nonexonic.py b/aliquotmaf/filters/nonexonic.py index c988293..dd24526 100644 --- a/aliquotmaf/filters/nonexonic.py +++ b/aliquotmaf/filters/nonexonic.py @@ -1,5 +1,5 @@ """ -Applies the NonExonic filter for regions outside the gencode intervals provided. +Applies the NonExonic filter for regions outside the gencode intervals provided. """ from __future__ import absolute_import diff --git a/aliquotmaf/filters/normal_depth.py b/aliquotmaf/filters/normal_depth.py index 608a406..0daf93e 100644 --- a/aliquotmaf/filters/normal_depth.py +++ b/aliquotmaf/filters/normal_depth.py @@ -1,5 +1,5 @@ """ -Applies the normal depth filter +Applies the normal depth filter """ from __future__ import absolute_import diff --git a/aliquotmaf/filters/offtarget.py b/aliquotmaf/filters/offtarget.py index cee7962..222e314 100644 --- a/aliquotmaf/filters/offtarget.py +++ b/aliquotmaf/filters/offtarget.py @@ -1,5 +1,5 @@ """ -Applies the off_target filter for regions outside the provided intervals. +Applies the off_target filter for regions outside the provided intervals. """ from __future__ import absolute_import diff --git a/aliquotmaf/merging/overlap_set.py b/aliquotmaf/merging/overlap_set.py index c74d18f..993308c 100644 --- a/aliquotmaf/merging/overlap_set.py +++ b/aliquotmaf/merging/overlap_set.py @@ -8,7 +8,7 @@ def __init__(self, result, maf_keys): """ Container for the results of an iteration of the overlap iterator. The results are converted to a dictionary where the caller is the key and the - overlaps for that caller are the values. + overlaps for that caller are the values. :param result: result from an iteration of `maflib.overlap_iter.LocatableOverlapIterator` :param maf_keys: list of callers in same order as results @@ -116,7 +116,7 @@ def locus_allele_map(self): @property def caller_type_map(self): """ - Dictionary where the keys are a ``tuple`` of caller and variant types + Dictionary where the keys are a ``tuple`` of caller and variant types mapped using self._type_dic and values are a list of records. """ if self._caller_type_map is None: diff --git a/aliquotmaf/merging/record_merger/base.py b/aliquotmaf/merging/record_merger/base.py index 6b12951..b4fe451 100644 --- a/aliquotmaf/merging/record_merger/base.py +++ b/aliquotmaf/merging/record_merger/base.py @@ -11,7 +11,7 @@ class BaseMafRecordMerger(metaclass=ABCMeta): def __init__(self, scheme): """ Initialize the MAF record merging object which has the main `merge_records` - function to take an `aliquotmaf.merging.overlap_set.OverlapSet` + function to take an `aliquotmaf.merging.overlap_set.OverlapSet` instance and performs merging. """ @@ -31,12 +31,12 @@ def caller_order(self): def caller_type_order(self): """ :return: a ``list`` of ``tuples`` of the format (caller, variant type) - in their order of priority. + in their order of priority. """ def allele_columns(self): """ - :return: a ``tuple`` of column names that contain allele information + :return: a ``tuple`` of column names that contain allele information """ return ( "Tumor_Seq_Allele1", @@ -48,7 +48,7 @@ def allele_columns(self): @abstractmethod def merge_records(self, results, tumor_only=False): """ - The main function for merging a MAF recrods from an + The main function for merging a MAF recrods from an `aliquotmaf.merging.overlap_set.OverlapSet` instance. The idea is to create a dictionary of column names as keys and the merged values as the values and then convert this dictionary to a `maflib.maf_record.MafRecord` @@ -95,12 +95,12 @@ def standardize_alleles(self, maf_dic, tumor_only=False): def fix_depths(self, maf_dic, tumor_only=False): """ Sets the total depths of tumor/normal to be the sum of the ref and alt - count columns if the sum of the ref and alt count columns is less than + count columns if the sum of the ref and alt count columns is less than the dp. :param maf_dic: ``dict`` of the maf record to format :param tumor_only: ``True`` if there is no matched normal else ``False`` - :return: updated maf_dic with formatted depths + :return: updated maf_dic with formatted depths """ # fix depths tsum = maf_dic["t_ref_count"].value + maf_dic["t_alt_count"].value diff --git a/aliquotmaf/merging/record_merger/impl/v1_0.py b/aliquotmaf/merging/record_merger/impl/v1_0.py index fa07d42..364bde9 100644 --- a/aliquotmaf/merging/record_merger/impl/v1_0.py +++ b/aliquotmaf/merging/record_merger/impl/v1_0.py @@ -33,7 +33,7 @@ def average_columns(self, tumor_only=False): def combine_columns(self): """ :return: a ``tuple`` of column names that should be combined into - a unique set. + a unique set. """ return "GDC_FILTER" @@ -46,7 +46,7 @@ def caller_order(self): def caller_type_order(self): """ :return: a ``list`` of ``tuples`` of the format (caller, variant type) - in their order of priority. + in their order of priority. """ return [ ("mutect2", "MNP"), @@ -74,11 +74,11 @@ def merge_records(self, results, tumor_only=False): 1. If singleton -> format and write 2. elif only one caller has overlaps -> loop over all records, format, and write - 3. elif there are multiple callers with only a single allele annotated + 3. elif there are multiple callers with only a single allele annotated -> merge into single record with most columns copied from caller based on self.caller_order() 4. elif there are multiple variant types -> - if there is a majority vote -> merge and write + if there is a majority vote -> merge and write else -> collapse by self.caller_type_order (see self.collapse_by_caller_type) 5. else -> collapse by self.caller_type_order (see self.collapse_by_caller_type) """ diff --git a/aliquotmaf/subcommands/mask_merged_aliquot/__main__.py b/aliquotmaf/subcommands/mask_merged_aliquot/__main__.py index 7386e77..ed3d6ac 100644 --- a/aliquotmaf/subcommands/mask_merged_aliquot/__main__.py +++ b/aliquotmaf/subcommands/mask_merged_aliquot/__main__.py @@ -1,5 +1,5 @@ """ -Subcommand for masking/filtering a merged aliquot MAF. +Subcommand for masking/filtering a merged aliquot MAF. """ from aliquotmaf.subcommands.base import Subcommand from aliquotmaf.subcommands.mask_merged_aliquot.runners import ( diff --git a/aliquotmaf/subcommands/merge_aliquot/__main__.py b/aliquotmaf/subcommands/merge_aliquot/__main__.py index 03c5b8f..e3064d0 100644 --- a/aliquotmaf/subcommands/merge_aliquot/__main__.py +++ b/aliquotmaf/subcommands/merge_aliquot/__main__.py @@ -1,5 +1,5 @@ """ -Subcommand for merging single caller raw aliquot MAFs to merged raw MAFs. +Subcommand for merging single caller raw aliquot MAFs to merged raw MAFs. """ from aliquotmaf.subcommands.base import Subcommand from aliquotmaf.subcommands.merge_aliquot.runners import GDC_1_0_0_Aliquot_Merged diff --git a/aliquotmaf/subcommands/vcf_to_aliquot/__main__.py b/aliquotmaf/subcommands/vcf_to_aliquot/__main__.py index 10d6308..7b8dee8 100644 --- a/aliquotmaf/subcommands/vcf_to_aliquot/__main__.py +++ b/aliquotmaf/subcommands/vcf_to_aliquot/__main__.py @@ -1,6 +1,5 @@ """ -Subcommand for converting a VEP annotated VCF to a raw -aliquot MAF. +Subcommand for converting a VEP annotated VCF to a raw aliquot MAF. """ from aliquotmaf.subcommands.base import Subcommand from aliquotmaf.subcommands.vcf_to_aliquot.runners import GDC_1_0_0_Aliquot diff --git a/aliquotmaf/subcommands/vcf_to_aliquot/extractors/effects.py b/aliquotmaf/subcommands/vcf_to_aliquot/extractors/effects.py index 2db0b2f..07ca558 100644 --- a/aliquotmaf/subcommands/vcf_to_aliquot/extractors/effects.py +++ b/aliquotmaf/subcommands/vcf_to_aliquot/extractors/effects.py @@ -69,7 +69,7 @@ def extract( effect[v] = edat[i].replace("&", ";") else: effect[v] = None - except: + except Exception: effect[v] = None # Skip effects on other ALT alleles. If ALLELE_NUM is undefined (e.g. diff --git a/aliquotmaf/subcommands/vcf_to_aliquot/extractors/genotypes.py b/aliquotmaf/subcommands/vcf_to_aliquot/extractors/genotypes.py index 7586e1d..52bc6bb 100644 --- a/aliquotmaf/subcommands/vcf_to_aliquot/extractors/genotypes.py +++ b/aliquotmaf/subcommands/vcf_to_aliquot/extractors/genotypes.py @@ -35,7 +35,7 @@ def extract(cls, tumor_genotype): class GenotypeAndDepthsExtractor(Extractor): """Extractor class for extracting the genotype and depths based on the - variant allele index. + variant allele index. """ logger = Logger.get_logger("GenotypeAndDepthsExtractor") @@ -43,14 +43,14 @@ class GenotypeAndDepthsExtractor(Extractor): @classmethod def extract(cls, var_allele_idx, genotype, alleles): """ - Extracts the information for the variant alleles based on the + Extracts the information for the variant alleles based on the variant allele index. Creates a new, updated genotype record and depths list. - + :param var_allele_idx: the variant allele index :param genotype: a dictionary or dictionary-like object containing various possible keys like AD, DP, etc. - :param alleles: an ordered list or tuple of the possible alleles + :param alleles: an ordered list or tuple of the possible alleles at the locus :returns: an updated genotype record and depths list """ @@ -113,18 +113,18 @@ def extract(cls, var_allele_idx, genotype, alleles): if i and i != ".": new_gt["DP"] += i - ## If depths is empty, just set to 0, 0 + # If depths is empty, just set to 0, 0 if not depths: depths = [0, 0] - ## If we have REF/ALT allele depths but not DP, then set DP equal to sum of all ADs + # If we have REF/ALT allele depths but not DP, then set DP equal to sum of all ADs if (depths[0] is not None and depths[var_allele_idx] is not None) and ( "DP" not in genotype or genotype["DP"] is None or genotype["DP"] == "." ): # cls.logger.warn('Missing DP field. setting DP equal to sum of ADs!!') new_gt["DP"] = sum([i for i in depths if i and i != "."]) - ## Set the formatted AD and alleles + # Set the formatted AD and alleles new_gt["AD"] = tuple([i if i != "" and i is not None else "." for i in depths]) new_gt["GT"] = genotype["GT"] depths = [i if i != "." and i is not None else 0 for i in new_gt["AD"]] diff --git a/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py b/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py index 78ad2a5..7afcdc0 100644 --- a/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py +++ b/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py @@ -600,12 +600,12 @@ def transform(self, vcf_record, data, is_tumor_only, line_number=None): collection.add(column=i, value=None) collection.transform(self._scheme) - ## Generate maf record + # Generate maf record maf_record = init_empty_maf_record(line_number=line_number) for i in collection: maf_record += i.transformed - ## Annotations + # Annotations if self.annotators["dbsnp_priority_db"]: maf_record = self.annotators["dbsnp_priority_db"].annotate(maf_record) else: @@ -635,7 +635,7 @@ def transform(self, vcf_record, data, is_tumor_only, line_number=None): maf_record, vcf_record, self.options["tumor_vcf_id"] ) - ## Filters + # Filters gdc_filters = [] for filt_key in self.filters: filt_obj = self.filters[filt_key] From 1466f1b99c7ee25d38a4da1c798f39fbf412f163 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:01:32 -0500 Subject: [PATCH 13/44] Linting --- .isort.cfg | 7 +++++ .pre-commit-config.yaml | 29 +++++++++++++++-- aliquotmaf/__main__.py | 9 ++---- aliquotmaf/annotators/__init__.py | 8 ++--- aliquotmaf/annotators/cosmic.py | 4 +-- aliquotmaf/annotators/dbsnp_validation.py | 4 +-- aliquotmaf/annotators/hotspot.py | 3 +- aliquotmaf/annotators/mutation_status.py | 4 +-- aliquotmaf/annotators/nontcga_exac.py | 4 +-- aliquotmaf/annotators/reference_context.py | 3 +- aliquotmaf/converters/utils.py | 2 +- aliquotmaf/filters/__init__.py | 2 +- aliquotmaf/filters/gdc_pon.py | 4 +-- aliquotmaf/filters/nonexonic.py | 4 +-- aliquotmaf/filters/offtarget.py | 4 +-- aliquotmaf/merging/record_merger/impl/v1_0.py | 3 +- .../mask_merged_aliquot/runners/base.py | 1 - .../gdc_1_0_0_aliquot_merged_masked.py | 8 ++--- .../subcommands/merge_aliquot/runners/base.py | 1 - .../runners/gdc_1_0_0_aliquot_merged.py | 13 +++----- aliquotmaf/subcommands/utils.py | 2 +- .../vcf_to_aliquot/extractors/__init__.py | 4 +-- .../vcf_to_aliquot/runners/base.py | 1 - .../runners/gdc_1_0_0_aliquot.py | 31 ++++++++----------- tests/annotators/test_cosmic.py | 6 ++-- tests/annotators/test_dbsnp_validation.py | 6 ++-- tests/annotators/test_exac.py | 4 +-- tests/annotators/test_hotspot.py | 4 +-- tests/annotators/test_mutation_status.py | 5 +-- tests/annotators/test_reference_context.py | 4 +-- tests/conftest.py | 6 ++-- tests/extractors/test_genotypes.py | 2 +- tests/filters/test_exac_freq.py | 4 +-- tests/filters/test_gdc_blacklist.py | 4 +-- tests/filters/test_gdc_pon.py | 4 +-- tests/filters/test_multiallelic.py | 4 +-- tests/filters/test_nonexonic.py | 6 ++-- tests/filters/test_normal_depth.py | 4 +-- tests/filters/test_offtarget.py | 6 ++-- tests/merging/conftest.py | 14 ++++----- tests/subcommands/test_subcommand.py | 2 +- tests/subcommands/test_vcf_to_aliquot_maf.py | 3 +- 42 files changed, 132 insertions(+), 111 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..c438575 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,7 @@ +[settings] +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=88 +known_third_party = maflib,pysam,pytest,setuptools diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26ab308..20245af 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,31 @@ +default_stages: [commit, push] +default_language_version: + python_venv: python3.6 + repos: -- repo: git@github.com:Yelp/detect-secrets + - repo: git@github.com:Yelp/detect-secrets rev: v0.13.1 hooks: - - id: detect-secrets + - id: detect-secrets args: ['--baseline', '.secrets.baseline'] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: check-yaml + - id: check-toml + - id: check-json + - id: detect-aws-credentials + args: ["--allow-missing-credentials"] + - id: detect-private-key + - repo: https://github.com/asottile/seed-isort-config + rev: v1.9.3 + hooks: + - id: seed-isort-config + - repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.21 + hooks: + - id: isort + - repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black diff --git a/aliquotmaf/__main__.py b/aliquotmaf/__main__.py index 95cf5ef..7e936f9 100644 --- a/aliquotmaf/__main__.py +++ b/aliquotmaf/__main__.py @@ -4,15 +4,12 @@ import argparse import datetime import sys - -from signal import signal, SIGPIPE, SIG_DFL +from signal import SIG_DFL, SIGPIPE, signal from aliquotmaf.logger import Logger - -from aliquotmaf.subcommands.vcf_to_aliquot.__main__ import VcfToAliquotMaf -from aliquotmaf.subcommands.merge_aliquot.__main__ import MergeAliquotMafs from aliquotmaf.subcommands.mask_merged_aliquot.__main__ import MaskMergedAliquotMaf - +from aliquotmaf.subcommands.merge_aliquot.__main__ import MergeAliquotMafs +from aliquotmaf.subcommands.vcf_to_aliquot.__main__ import VcfToAliquotMaf signal(SIGPIPE, SIG_DFL) diff --git a/aliquotmaf/annotators/__init__.py b/aliquotmaf/annotators/__init__.py index 2292818..fa7b1c3 100644 --- a/aliquotmaf/annotators/__init__.py +++ b/aliquotmaf/annotators/__init__.py @@ -1,11 +1,11 @@ from __future__ import absolute_import -from .mutation_status import MutationStatus -from .dbsnp_validation import DbSnpValidation -from .reference_context import ReferenceContext from .cosmic import CosmicID -from .nontcga_exac import NonTcgaExac +from .dbsnp_validation import DbSnpValidation from .hotspot import Hotspot +from .mutation_status import MutationStatus +from .nontcga_exac import NonTcgaExac +from .reference_context import ReferenceContext __all__ = [ DbSnpValidation, diff --git a/aliquotmaf/annotators/cosmic.py b/aliquotmaf/annotators/cosmic.py index e8b529d..47b8cbd 100644 --- a/aliquotmaf/annotators/cosmic.py +++ b/aliquotmaf/annotators/cosmic.py @@ -5,10 +5,10 @@ import pysam -from .annotator import Annotator - from aliquotmaf.converters.builder import get_builder +from .annotator import Annotator + class CosmicID(Annotator): def __init__(self, scheme, source): diff --git a/aliquotmaf/annotators/dbsnp_validation.py b/aliquotmaf/annotators/dbsnp_validation.py index 12defc4..053fe42 100644 --- a/aliquotmaf/annotators/dbsnp_validation.py +++ b/aliquotmaf/annotators/dbsnp_validation.py @@ -5,10 +5,10 @@ import sqlite3 as lite -from .annotator import Annotator - from aliquotmaf.converters.builder import get_builder +from .annotator import Annotator + class DbSnpValidation(Annotator): def __init__(self, scheme, source): diff --git a/aliquotmaf/annotators/hotspot.py b/aliquotmaf/annotators/hotspot.py index ca1c40c..7957778 100644 --- a/aliquotmaf/annotators/hotspot.py +++ b/aliquotmaf/annotators/hotspot.py @@ -3,9 +3,10 @@ """ from __future__ import absolute_import -from .annotator import Annotator from aliquotmaf.converters.builder import get_builder +from .annotator import Annotator + class Hotspot(Annotator): def __init__(self, source, scheme, data): diff --git a/aliquotmaf/annotators/mutation_status.py b/aliquotmaf/annotators/mutation_status.py index b351f77..7668d2a 100644 --- a/aliquotmaf/annotators/mutation_status.py +++ b/aliquotmaf/annotators/mutation_status.py @@ -5,10 +5,10 @@ import pysam -from .annotator import Annotator - from aliquotmaf.converters.builder import get_builder +from .annotator import Annotator + class MutationStatus(Annotator): def __init__(self, scheme, caller): diff --git a/aliquotmaf/annotators/nontcga_exac.py b/aliquotmaf/annotators/nontcga_exac.py index 95f13bd..2d22262 100644 --- a/aliquotmaf/annotators/nontcga_exac.py +++ b/aliquotmaf/annotators/nontcga_exac.py @@ -5,10 +5,10 @@ import pysam -from .annotator import Annotator - from aliquotmaf.converters.builder import get_builder +from .annotator import Annotator + class NonTcgaExac(Annotator): def __init__(self, scheme, source): diff --git a/aliquotmaf/annotators/reference_context.py b/aliquotmaf/annotators/reference_context.py index 7c984c9..65f5372 100644 --- a/aliquotmaf/annotators/reference_context.py +++ b/aliquotmaf/annotators/reference_context.py @@ -5,9 +5,10 @@ import pysam -from .annotator import Annotator from aliquotmaf.converters.builder import get_builder +from .annotator import Annotator + class ReferenceContext(Annotator): def __init__(self, source, scheme, context_size=5): diff --git a/aliquotmaf/converters/utils.py b/aliquotmaf/converters/utils.py index 073a62f..4ab9504 100644 --- a/aliquotmaf/converters/utils.py +++ b/aliquotmaf/converters/utils.py @@ -1,6 +1,6 @@ """Utilities for converters""" -from maflib.validation import ValidationStringency from maflib.record import MafRecord +from maflib.validation import ValidationStringency def get_columns_from_header(header): diff --git a/aliquotmaf/filters/__init__.py b/aliquotmaf/filters/__init__.py index bab8b8a..2ce2850 100644 --- a/aliquotmaf/filters/__init__.py +++ b/aliquotmaf/filters/__init__.py @@ -2,10 +2,10 @@ from .exac import ExAC from .gdc_blacklist import GdcBlacklist -from .normal_depth import NormalDepth from .gdc_pon import GdcPon from .multiallelic import Multiallelic from .nonexonic import NonExonic +from .normal_depth import NormalDepth from .offtarget import OffTarget __all__ = [ExAC, GdcBlacklist, NormalDepth, GdcPon, Multiallelic, NonExonic, OffTarget] diff --git a/aliquotmaf/filters/gdc_pon.py b/aliquotmaf/filters/gdc_pon.py index 1db9f9b..bd8c36a 100644 --- a/aliquotmaf/filters/gdc_pon.py +++ b/aliquotmaf/filters/gdc_pon.py @@ -3,10 +3,10 @@ """ from __future__ import absolute_import -from .filter_base import Filter - from pysam import VariantFile +from .filter_base import Filter + class GdcPon(Filter): def __init__(self, source): diff --git a/aliquotmaf/filters/nonexonic.py b/aliquotmaf/filters/nonexonic.py index dd24526..a95facc 100644 --- a/aliquotmaf/filters/nonexonic.py +++ b/aliquotmaf/filters/nonexonic.py @@ -3,10 +3,10 @@ """ from __future__ import absolute_import -from .filter_base import Filter - from pysam import TabixFile, asBed +from .filter_base import Filter + class NonExonic(Filter): def __init__(self, source): diff --git a/aliquotmaf/filters/offtarget.py b/aliquotmaf/filters/offtarget.py index 222e314..ee505c8 100644 --- a/aliquotmaf/filters/offtarget.py +++ b/aliquotmaf/filters/offtarget.py @@ -3,10 +3,10 @@ """ from __future__ import absolute_import -from .filter_base import Filter - from pysam import TabixFile, asBed +from .filter_base import Filter + class OffTarget(Filter): def __init__(self, source): diff --git a/aliquotmaf/merging/record_merger/impl/v1_0.py b/aliquotmaf/merging/record_merger/impl/v1_0.py index 364bde9..2771486 100644 --- a/aliquotmaf/merging/record_merger/impl/v1_0.py +++ b/aliquotmaf/merging/record_merger/impl/v1_0.py @@ -1,9 +1,8 @@ """ Maf recorder merger implementation v1.0 """ -from aliquotmaf.converters.utils import init_empty_maf_record from aliquotmaf.converters.builder import get_builder - +from aliquotmaf.converters.utils import init_empty_maf_record from aliquotmaf.merging.record_merger.base import BaseMafRecordMerger from aliquotmaf.merging.record_merger.mixins import ( MafMergingAverageColumnsMixin, diff --git a/aliquotmaf/subcommands/mask_merged_aliquot/runners/base.py b/aliquotmaf/subcommands/mask_merged_aliquot/runners/base.py index 3dc30a7..d6477e8 100644 --- a/aliquotmaf/subcommands/mask_merged_aliquot/runners/base.py +++ b/aliquotmaf/subcommands/mask_merged_aliquot/runners/base.py @@ -2,7 +2,6 @@ Base class for all protected -> public MAF runners. """ import datetime - from abc import ABCMeta, abstractmethod from maflib.header import MafHeaderRecord diff --git a/aliquotmaf/subcommands/mask_merged_aliquot/runners/gdc_1_0_0_aliquot_merged_masked.py b/aliquotmaf/subcommands/mask_merged_aliquot/runners/gdc_1_0_0_aliquot_merged_masked.py index cdfef46..78d87c3 100644 --- a/aliquotmaf/subcommands/mask_merged_aliquot/runners/gdc_1_0_0_aliquot_merged_masked.py +++ b/aliquotmaf/subcommands/mask_merged_aliquot/runners/gdc_1_0_0_aliquot_merged_masked.py @@ -4,15 +4,15 @@ """ import json -from maflib.reader import MafReader from maflib.header import MafHeader -from maflib.writer import MafWriter +from maflib.reader import MafReader from maflib.sort_order import BarcodesAndCoordinate from maflib.validation import ValidationStringency +from maflib.writer import MafWriter -from aliquotmaf.subcommands.mask_merged_aliquot.runners import BaseRunner -from aliquotmaf.converters.utils import init_empty_maf_record, get_columns_from_header from aliquotmaf.converters.builder import get_builder +from aliquotmaf.converters.utils import get_columns_from_header, init_empty_maf_record +from aliquotmaf.subcommands.mask_merged_aliquot.runners import BaseRunner class GDC_1_0_0_Aliquot_Merged_Masked(BaseRunner): diff --git a/aliquotmaf/subcommands/merge_aliquot/runners/base.py b/aliquotmaf/subcommands/merge_aliquot/runners/base.py index d966ec7..ebfe4e2 100644 --- a/aliquotmaf/subcommands/merge_aliquot/runners/base.py +++ b/aliquotmaf/subcommands/merge_aliquot/runners/base.py @@ -2,7 +2,6 @@ Base class for all aliquot-level per-caller protected MAFs -> merged protected MAF runners. """ import datetime - from abc import ABCMeta, abstractmethod from maflib.header import MafHeaderRecord diff --git a/aliquotmaf/subcommands/merge_aliquot/runners/gdc_1_0_0_aliquot_merged.py b/aliquotmaf/subcommands/merge_aliquot/runners/gdc_1_0_0_aliquot_merged.py index 804326c..99c7cd7 100644 --- a/aliquotmaf/subcommands/merge_aliquot/runners/gdc_1_0_0_aliquot_merged.py +++ b/aliquotmaf/subcommands/merge_aliquot/runners/gdc_1_0_0_aliquot_merged.py @@ -3,25 +3,22 @@ """ import json -from maflib.reader import MafReader from maflib.header import MafHeader -from maflib.writer import MafWriter +from maflib.overlap_iter import LocatableOverlapIterator +from maflib.reader import MafReader from maflib.sort_order import BarcodesAndCoordinate from maflib.sorter import MafSorter from maflib.validation import ValidationStringency -from maflib.overlap_iter import LocatableOverlapIterator +from maflib.writer import MafWriter import aliquotmaf.filters as Filters - +from aliquotmaf.converters.builder import get_builder +from aliquotmaf.converters.utils import get_columns_from_header from aliquotmaf.merging.filtering_iterator import FilteringPeekableIterator from aliquotmaf.merging.overlap_set import OverlapSet from aliquotmaf.merging.record_merger.impl.v1_0 import MafRecordMerger_1_0_0 - from aliquotmaf.subcommands.merge_aliquot.runners import BaseRunner -from aliquotmaf.converters.utils import get_columns_from_header -from aliquotmaf.converters.builder import get_builder - class GDC_1_0_0_Aliquot_Merged(BaseRunner): def __init__(self, options=dict()): diff --git a/aliquotmaf/subcommands/utils.py b/aliquotmaf/subcommands/utils.py index 288edb1..c83584d 100644 --- a/aliquotmaf/subcommands/utils.py +++ b/aliquotmaf/subcommands/utils.py @@ -1,7 +1,7 @@ """Utility functions for subcommands""" import gzip -import re import json +import re def get_open_function(fil): diff --git a/aliquotmaf/subcommands/vcf_to_aliquot/extractors/__init__.py b/aliquotmaf/subcommands/vcf_to_aliquot/extractors/__init__.py index 81b2864..5485c25 100644 --- a/aliquotmaf/subcommands/vcf_to_aliquot/extractors/__init__.py +++ b/aliquotmaf/subcommands/vcf_to_aliquot/extractors/__init__.py @@ -1,9 +1,9 @@ from __future__ import absolute_import from .base import Extractor -from .genotypes import VariantAlleleIndexExtractor, GenotypeAndDepthsExtractor -from .location import LocationDataExtractor from .effects import EffectsExtractor, SelectOneEffectExtractor +from .genotypes import GenotypeAndDepthsExtractor, VariantAlleleIndexExtractor +from .location import LocationDataExtractor from .population_frequency import PopulationFrequencyExtractor from .variant_class import VariantClassExtractor diff --git a/aliquotmaf/subcommands/vcf_to_aliquot/runners/base.py b/aliquotmaf/subcommands/vcf_to_aliquot/runners/base.py index 98200c9..d469c8b 100644 --- a/aliquotmaf/subcommands/vcf_to_aliquot/runners/base.py +++ b/aliquotmaf/subcommands/vcf_to_aliquot/runners/base.py @@ -1,6 +1,5 @@ """Base class for all vcf2maf runners""" import datetime - from abc import ABCMeta, abstractmethod from maflib.header import MafHeaderRecord diff --git a/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py b/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py index 7afcdc0..942f69f 100644 --- a/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py +++ b/aliquotmaf/subcommands/vcf_to_aliquot/runners/gdc_1_0_0_aliquot.py @@ -1,38 +1,33 @@ """Main vcf2maf logic for spec gdc-1.0.0-aliquot""" -import pysam import urllib.parse - from operator import itemgetter -from maflib.header import MafHeader -from maflib.writer import MafWriter +import pysam +from maflib.header import MafHeader, MafHeaderRecord from maflib.sort_order import BarcodesAndCoordinate from maflib.sorter import MafSorter -from maflib.header import MafHeaderRecord from maflib.validation import ValidationStringency +from maflib.writer import MafWriter import aliquotmaf.annotators as Annotators import aliquotmaf.filters as Filters import aliquotmaf.subcommands.vcf_to_aliquot.extractors as Extractors - -from aliquotmaf.subcommands.vcf_to_aliquot.runners import BaseRunner -from aliquotmaf.converters.utils import init_empty_maf_record, get_columns_from_header -from aliquotmaf.converters.collection import InputCollection from aliquotmaf.converters.builder import get_builder - -from aliquotmaf.subcommands.utils import ( - extract_annotation_from_header, - assert_sample_in_header, - load_json, - load_enst, -) - +from aliquotmaf.converters.collection import InputCollection from aliquotmaf.converters.formatters import ( + format_all_effects, format_alleles, format_depths, - format_all_effects, format_vcf_columns, ) +from aliquotmaf.converters.utils import get_columns_from_header, init_empty_maf_record +from aliquotmaf.subcommands.utils import ( + assert_sample_in_header, + extract_annotation_from_header, + load_enst, + load_json, +) +from aliquotmaf.subcommands.vcf_to_aliquot.runners import BaseRunner class GDC_1_0_0_Aliquot(BaseRunner): diff --git a/tests/annotators/test_cosmic.py b/tests/annotators/test_cosmic.py index cba1b4d..4624788 100644 --- a/tests/annotators/test_cosmic.py +++ b/tests/annotators/test_cosmic.py @@ -1,12 +1,12 @@ """ Tests for the ``aliquotmaf.annotators.Cosmic`` class. """ -import pysam -import pytest from collections import OrderedDict, namedtuple -from maflib.record import MafColumnRecord +import pysam +import pytest from maflib.column_types import SequenceOfStrings +from maflib.record import MafColumnRecord from aliquotmaf.annotators import CosmicID from aliquotmaf.converters.builder import get_builder diff --git a/tests/annotators/test_dbsnp_validation.py b/tests/annotators/test_dbsnp_validation.py index ec51662..42df4b5 100644 --- a/tests/annotators/test_dbsnp_validation.py +++ b/tests/annotators/test_dbsnp_validation.py @@ -1,12 +1,12 @@ """ Tests for the ``aliquotmaf.annotators.DbSnpValidation`` class. """ -import pysam -import pytest from collections import OrderedDict -from maflib.record import MafColumnRecord +import pysam +import pytest from maflib.column_types import SequenceOfStrings +from maflib.record import MafColumnRecord from aliquotmaf.annotators import DbSnpValidation from aliquotmaf.converters.builder import get_builder diff --git a/tests/annotators/test_exac.py b/tests/annotators/test_exac.py index 02b2c44..ac5facc 100644 --- a/tests/annotators/test_exac.py +++ b/tests/annotators/test_exac.py @@ -1,11 +1,11 @@ """ Tests for the ``aliquotmaf.annotators.NonTcgaExac`` class. """ -import pytest from collections import OrderedDict -from maflib.record import MafColumnRecord +import pytest from maflib.column_types import NullableFloatColumn +from maflib.record import MafColumnRecord from aliquotmaf.annotators import NonTcgaExac from aliquotmaf.converters.builder import get_builder diff --git a/tests/annotators/test_hotspot.py b/tests/annotators/test_hotspot.py index 0234f95..0820d75 100644 --- a/tests/annotators/test_hotspot.py +++ b/tests/annotators/test_hotspot.py @@ -1,11 +1,11 @@ """ Tests for the ``aliquotmaf.annotators.Hotpot`` class. """ -import pytest from collections import OrderedDict +import pytest +from maflib.column_types import NullableStringColumn, NullableYOrN, StringColumn from maflib.record import MafColumnRecord -from maflib.column_types import StringColumn, NullableStringColumn, NullableYOrN from aliquotmaf.annotators import Hotspot from aliquotmaf.converters.builder import get_builder diff --git a/tests/annotators/test_mutation_status.py b/tests/annotators/test_mutation_status.py index 371e924..f5de29d 100644 --- a/tests/annotators/test_mutation_status.py +++ b/tests/annotators/test_mutation_status.py @@ -1,11 +1,12 @@ """ Tests for the ``aliquotmaf.annotators.MutationStatus`` class. """ -import pytest from collections import OrderedDict -from maflib.column_values import MutationStatusEnum +import pytest from maflib.column_types import MutationStatus +from maflib.column_values import MutationStatusEnum + from aliquotmaf.annotators import MutationStatus as MutationStatusAnnotator diff --git a/tests/annotators/test_reference_context.py b/tests/annotators/test_reference_context.py index 9bb0a56..74188f1 100644 --- a/tests/annotators/test_reference_context.py +++ b/tests/annotators/test_reference_context.py @@ -1,10 +1,10 @@ """ Tests for the ``aliquotmaf.annotators.ReferenceContext`` class. """ -import pysam -import pytest from collections import OrderedDict, namedtuple +import pysam +import pytest from maflib.column_types import StringColumn from aliquotmaf.annotators import ReferenceContext diff --git a/tests/conftest.py b/tests/conftest.py index 36bcc52..6409c1f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,12 @@ import logging -import pytest -import pysam import os from collections import OrderedDict, namedtuple +import pysam +import pytest +from maflib.record import MafRecord from maflib.schemes import MafScheme from maflib.validation import ValidationStringency -from maflib.record import MafRecord log = logging.getLogger() log.setLevel(logging.INFO) diff --git a/tests/extractors/test_genotypes.py b/tests/extractors/test_genotypes.py index 497514c..83aacb0 100644 --- a/tests/extractors/test_genotypes.py +++ b/tests/extractors/test_genotypes.py @@ -4,8 +4,8 @@ import pytest from aliquotmaf.subcommands.vcf_to_aliquot.extractors.genotypes import ( - VariantAlleleIndexExtractor, GenotypeAndDepthsExtractor, + VariantAlleleIndexExtractor, ) diff --git a/tests/filters/test_exac_freq.py b/tests/filters/test_exac_freq.py index 1af9579..fc67b78 100644 --- a/tests/filters/test_exac_freq.py +++ b/tests/filters/test_exac_freq.py @@ -1,13 +1,13 @@ """ Tests for the ``aliquotmaf.filters.ExAC`` class. """ -import pytest from collections import OrderedDict +import pytest from maflib.column_types import NullableFloatColumn -from aliquotmaf.filters import ExAC from aliquotmaf.converters.builder import get_builder +from aliquotmaf.filters import ExAC subpops = [ "nontcga_ExAC_AF_Adj", diff --git a/tests/filters/test_gdc_blacklist.py b/tests/filters/test_gdc_blacklist.py index 316901b..ec58dca 100644 --- a/tests/filters/test_gdc_blacklist.py +++ b/tests/filters/test_gdc_blacklist.py @@ -1,13 +1,13 @@ """ Tests for the ``aliquotmaf.filters.GdcBlacklist`` class. """ -import pytest from collections import OrderedDict +import pytest from maflib.column_types import UUIDColumn -from aliquotmaf.filters import GdcBlacklist from aliquotmaf.converters.builder import get_builder +from aliquotmaf.filters import GdcBlacklist @pytest.fixture diff --git a/tests/filters/test_gdc_pon.py b/tests/filters/test_gdc_pon.py index 14545b2..6fe8a63 100644 --- a/tests/filters/test_gdc_pon.py +++ b/tests/filters/test_gdc_pon.py @@ -1,13 +1,13 @@ """ Tests for the ``aliquotmaf.filters.GdcPon`` class. """ -import pytest from collections import OrderedDict +import pytest from maflib.column_types import StringColumn -from aliquotmaf.filters import GdcPon from aliquotmaf.converters.builder import get_builder +from aliquotmaf.filters import GdcPon @pytest.fixture diff --git a/tests/filters/test_multiallelic.py b/tests/filters/test_multiallelic.py index ba5320e..bad77b4 100644 --- a/tests/filters/test_multiallelic.py +++ b/tests/filters/test_multiallelic.py @@ -1,13 +1,13 @@ """ Tests for the ``aliquotmaf.filters.Multiallelic`` class. """ -import pytest from collections import OrderedDict +import pytest from maflib.column_types import StringColumn -from aliquotmaf.filters import Multiallelic from aliquotmaf.converters.builder import get_builder +from aliquotmaf.filters import Multiallelic @pytest.fixture diff --git a/tests/filters/test_nonexonic.py b/tests/filters/test_nonexonic.py index cfd1dfb..6df407d 100644 --- a/tests/filters/test_nonexonic.py +++ b/tests/filters/test_nonexonic.py @@ -1,13 +1,13 @@ """ Tests for the ``aliquotmaf.filters.NonExonic`` class. """ -import pytest from collections import OrderedDict -from maflib.column_types import StringColumn, OneBasedIntegerColumn +import pytest +from maflib.column_types import OneBasedIntegerColumn, StringColumn -from aliquotmaf.filters import NonExonic from aliquotmaf.converters.builder import get_builder +from aliquotmaf.filters import NonExonic @pytest.fixture diff --git a/tests/filters/test_normal_depth.py b/tests/filters/test_normal_depth.py index e306c80..e309980 100644 --- a/tests/filters/test_normal_depth.py +++ b/tests/filters/test_normal_depth.py @@ -1,13 +1,13 @@ """ Tests for the ``aliquotmaf.filters.NormalDepth`` class. """ -import pytest from collections import OrderedDict +import pytest from maflib.column_types import NullableZeroBasedIntegerColumn -from aliquotmaf.filters import NormalDepth from aliquotmaf.converters.builder import get_builder +from aliquotmaf.filters import NormalDepth @pytest.fixture diff --git a/tests/filters/test_offtarget.py b/tests/filters/test_offtarget.py index ec907da..07ec5dd 100644 --- a/tests/filters/test_offtarget.py +++ b/tests/filters/test_offtarget.py @@ -1,13 +1,13 @@ """ Tests for the ``aliquotmaf.filters.OffTarget`` class. """ -import pytest from collections import OrderedDict -from maflib.column_types import StringColumn, OneBasedIntegerColumn +import pytest +from maflib.column_types import OneBasedIntegerColumn, StringColumn -from aliquotmaf.filters import OffTarget from aliquotmaf.converters.builder import get_builder +from aliquotmaf.filters import OffTarget @pytest.fixture diff --git a/tests/merging/conftest.py b/tests/merging/conftest.py index b902d67..e9a2a60 100644 --- a/tests/merging/conftest.py +++ b/tests/merging/conftest.py @@ -1,19 +1,19 @@ -import pytest from collections import OrderedDict -from maflib.record import MafRecord -from maflib.validation import ValidationStringency +import pytest from maflib.column_types import ( - OneBasedIntegerColumn, DnaString, NullableDnaString, - VariantType, - ZeroBasedIntegerColumn, + NullableStringColumn, NullableZeroBasedIntegerColumn, + OneBasedIntegerColumn, SequenceOfStrings, StringColumn, - NullableStringColumn, + VariantType, + ZeroBasedIntegerColumn, ) +from maflib.record import MafRecord +from maflib.validation import ValidationStringency from aliquotmaf.merging.overlap_set import OverlapSet diff --git a/tests/subcommands/test_subcommand.py b/tests/subcommands/test_subcommand.py index 26849f6..b5eaaaa 100644 --- a/tests/subcommands/test_subcommand.py +++ b/tests/subcommands/test_subcommand.py @@ -3,8 +3,8 @@ """ import pytest -from aliquotmaf.subcommands.base import Subcommand from aliquotmaf.__main__ import main +from aliquotmaf.subcommands.base import Subcommand class Example(Subcommand): diff --git a/tests/subcommands/test_vcf_to_aliquot_maf.py b/tests/subcommands/test_vcf_to_aliquot_maf.py index 2b90ddb..9073265 100644 --- a/tests/subcommands/test_vcf_to_aliquot_maf.py +++ b/tests/subcommands/test_vcf_to_aliquot_maf.py @@ -1,9 +1,10 @@ """ Tests for the ``aliquotmaf.subcommands.vcf_to_aliquot`` subcommands. """ -import pytest import uuid +import pytest + from aliquotmaf.__main__ import main From de577414891b3a53c856637530945cab7fa5b8c7 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:01:51 -0500 Subject: [PATCH 14/44] Update linting target --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 26c3be3..3532f1e 100644 --- a/Makefile +++ b/Makefile @@ -62,9 +62,7 @@ clean: lint: @echo @echo -- Lint -- - python3 -m flake8 \ - --ignore=E501,F401,E302,E502,E126,E731,W503,W605,F841,C901 \ - ${MODULE}/ + python3 -m flake8 ${MODULE}/ run: bin/run From 9dd5dccd0d75eebccad094ce5b0b626b1e7d3f96 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:02:25 -0500 Subject: [PATCH 15/44] Linting --- setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index c2fcdde..d0d051a 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,3 @@ -from setuptools import setup, find_packages - -#!/usr/bin/env python - import importlib import os import subprocess @@ -10,6 +6,9 @@ from setuptools import Command, find_packages, setup +#!/usr/bin/env python + + GIT_REPO = "aliquot-maf-tools" PACKAGE = "aliquotmaf" @@ -132,10 +131,7 @@ def write_requirements(self, path, reqs): packages=find_packages(), install_requires=INSTALL_REQUIRES, tests_require=TESTS_REQUIRE, - cmdclass={ - "capture_requirements": Requirements, - "print_version": PrintVersion, - }, + cmdclass={"capture_requirements": Requirements, "print_version": PrintVersion}, include_package_data=True, entry_points={"console_scripts": ["aliquot-maf-tools = aliquotmaf.__main__:main"]}, ) From ded13f255c7d4be20f99e9cd55ea1f9f921bce4d Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:02:52 -0500 Subject: [PATCH 16/44] Remove setup cfg --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b7e4789..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[aliases] -test=pytest From c03694828cc2ae791a5ad712c6acce51af9d5fc5 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:13:49 -0500 Subject: [PATCH 17/44] Add --version arg --- aliquotmaf/__main__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aliquotmaf/__main__.py b/aliquotmaf/__main__.py index 7e936f9..743f38a 100644 --- a/aliquotmaf/__main__.py +++ b/aliquotmaf/__main__.py @@ -13,6 +13,11 @@ signal(SIGPIPE, SIG_DFL) +try: + from aliquotmaf import __version__ +except ImportError: + __version__ = '0' + def main(args=None): """ @@ -32,6 +37,7 @@ def main(args=None): # Get args p = argparse.ArgumentParser("GDC Aliquot MAF Tools") + p.add_argument("--version", action="version", version=__version__) subparsers = p.add_subparsers(dest="subcommand") subparsers.required = True From 30230e8dd205846eb78da15499022426f7dc75a2 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:14:03 -0500 Subject: [PATCH 18/44] Add entrypoint script --- bin/aliquotmaf | 7 +++++++ setup.py | 1 + 2 files changed, 8 insertions(+) create mode 100755 bin/aliquotmaf diff --git a/bin/aliquotmaf b/bin/aliquotmaf new file mode 100755 index 0000000..b6da7a8 --- /dev/null +++ b/bin/aliquotmaf @@ -0,0 +1,7 @@ +#!/bin/sh + +case "$1" in + test) python -m pytest tests;; + *version) python -m aliquotmaf --version;; + *) python -m aliquotmaf $@;; +esac diff --git a/setup.py b/setup.py index d0d051a..19535b9 100644 --- a/setup.py +++ b/setup.py @@ -133,6 +133,7 @@ def write_requirements(self, path, reqs): tests_require=TESTS_REQUIRE, cmdclass={"capture_requirements": Requirements, "print_version": PrintVersion}, include_package_data=True, + scripts=[os.path.join(os.path.dirname(__file__), 'bin', f"{PACKAGE}")], entry_points={"console_scripts": ["aliquot-maf-tools = aliquotmaf.__main__:main"]}, ) From cde5be88549fd2d54b98f9214cee78be02ffbde8 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:14:12 -0500 Subject: [PATCH 19/44] Fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 979cb4e..496fd0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM quay.io/ncigdc/bio-python:3.6 -ENV BINARY=maflib +ENV BINARY=aliquotmaf COPY ./dist/ /opt From f6ea54960b8022e7bc241ab5540f1d051325fc15 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:14:22 -0500 Subject: [PATCH 20/44] Added missing init files --- tests/__init__.py | 0 tests/aliquotmaf/__init__.py | 0 tests/aliquotmaf/merge_aliquot/__init__.py | 0 tests/aliquotmaf/vcf_to_aliquot/__init__.py | 0 tests/aliquotmaf/vcf_to_aliquot/aliquot/__init__.py | 0 tests/aliquotmaf/vcf_to_aliquot/annotators/__init__.py | 0 tests/aliquotmaf/vcf_to_aliquot/converters/__init__.py | 0 tests/aliquotmaf/vcf_to_aliquot/extractors/__init__.py | 0 tests/aliquotmaf/vcf_to_aliquot/filters/__init__.py | 0 tests/annotators/__init__.py | 0 tests/data/__init__.py | 0 tests/extractors/__init__.py | 0 tests/filters/__init__.py | 0 tests/formatters/__init__.py | 0 tests/merging/__init__.py | 0 tests/subcommands/__init__.py | 0 16 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/aliquotmaf/__init__.py create mode 100644 tests/aliquotmaf/merge_aliquot/__init__.py create mode 100644 tests/aliquotmaf/vcf_to_aliquot/__init__.py create mode 100644 tests/aliquotmaf/vcf_to_aliquot/aliquot/__init__.py create mode 100644 tests/aliquotmaf/vcf_to_aliquot/annotators/__init__.py create mode 100644 tests/aliquotmaf/vcf_to_aliquot/converters/__init__.py create mode 100644 tests/aliquotmaf/vcf_to_aliquot/extractors/__init__.py create mode 100644 tests/aliquotmaf/vcf_to_aliquot/filters/__init__.py create mode 100644 tests/annotators/__init__.py create mode 100644 tests/data/__init__.py create mode 100644 tests/extractors/__init__.py create mode 100644 tests/filters/__init__.py create mode 100644 tests/formatters/__init__.py create mode 100644 tests/merging/__init__.py create mode 100644 tests/subcommands/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/__init__.py b/tests/aliquotmaf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/merge_aliquot/__init__.py b/tests/aliquotmaf/merge_aliquot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/vcf_to_aliquot/__init__.py b/tests/aliquotmaf/vcf_to_aliquot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/vcf_to_aliquot/aliquot/__init__.py b/tests/aliquotmaf/vcf_to_aliquot/aliquot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/vcf_to_aliquot/annotators/__init__.py b/tests/aliquotmaf/vcf_to_aliquot/annotators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/vcf_to_aliquot/converters/__init__.py b/tests/aliquotmaf/vcf_to_aliquot/converters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/vcf_to_aliquot/extractors/__init__.py b/tests/aliquotmaf/vcf_to_aliquot/extractors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/aliquotmaf/vcf_to_aliquot/filters/__init__.py b/tests/aliquotmaf/vcf_to_aliquot/filters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/annotators/__init__.py b/tests/annotators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/__init__.py b/tests/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/extractors/__init__.py b/tests/extractors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/filters/__init__.py b/tests/filters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/formatters/__init__.py b/tests/formatters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/merging/__init__.py b/tests/merging/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/subcommands/__init__.py b/tests/subcommands/__init__.py new file mode 100644 index 0000000..e69de29 From a36da0e69c8dd73e4535fb14d808d31cddaad72a Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:24:06 -0500 Subject: [PATCH 21/44] Remove old requirements file --- test-requirements.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test-requirements.txt diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index e079f8a..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pytest From f6cd13c8a6908f25fbe8c9737ddf2c9e9233cb83 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:24:21 -0500 Subject: [PATCH 22/44] Remove old build script --- docker-build.sh | 54 ------------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100755 docker-build.sh diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index 7b1378e..0000000 --- a/docker-build.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# This script does the following: -# - Overwrites .deps dir TODO: Only over write required repos only if not on hash -# - Parses requirements.txt and clones all private repos in dependencies -# - Creates docker.requirements.txt with remaining modules - -# Overwriting .deps dir -D_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.deps" -[[ -d ${D_DIR} ]] && rm -rf ${D_DIR}/* || mkdir -p ${D_DIR} - -pushd () { - command pushd "$@" > /dev/null -} - -popd () { - command popd "$@" > /dev/null -} - -docker_build () { - command docker build \ - --build-arg http_proxy=$http_proxy \ - --build-arg https_proxy=$https_proxy \ - --build-arg no_proxy=$no_proxy \ - "$@" -} - -# Parses requirements.txt and clones all private repos in dependencies to a specific hash -for i in $(grep '^-e\|\.git' requirements.txt | sed -n 's/.*git+\([^ ]*\)#egg.*/\1/p') -do - echo "Git Repo:${i%@*} Git Hash:${i##*@}" - pushd ${D_DIR} - repo=${i%@*} - git clone ${repo} - pushd `__=${repo%.git}&&echo ${__##*/}` - echo $PWD - git reset --hard ${i##*@} - popd - popd -done - -# Creating requirements file with remaining modules -grep -v "^-e\|\.git" requirements.txt > docker-requirements.txt - -# tag -quay="quay.io/ncigdc/aliquot-maf-tools" -version=$(git log --first-parent --max-count=1 --format=format:%H) -imagetag="${quay}:${version}" - -echo "Building tag: $imagetag" -docker_build -t $imagetag . - -# Cleanup -rm docker-requirements.txt -if [[ -d ${D_DIR} ]]; then rm -rf ${D_DIR}; fi From c4b1179b6a8fb54d065afe572849ec3fcf9b0a39 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 10:24:36 -0500 Subject: [PATCH 23/44] Remove old build script --- repo-install.sh | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100755 repo-install.sh diff --git a/repo-install.sh b/repo-install.sh deleted file mode 100755 index 6a1b971..0000000 --- a/repo-install.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# This script installs python directories in the order -# of dependencies. - -# Example: There are for python repos A to D in .deps dir -# if A Depends on A, B, C -# B -> C, D -# C -> D -# Then the script will install in the following order: -# D, C, B, A - -pushd ${1:-.deps} > /dev/null #Getting inside .deps - -# Getting list of repos inside .deps -__=($(ls -d */)) -GLIST=(${__[*]%/}) - -# Function: install_module -# Arguments: -# 1. list of repos -# 2. repo to install -install_module () { - declare -a list=("${!1}") - local mod=${2} - declare -a rem_list=(${list[@]/$mod}) - for r_mod in "${rem_list[@]}" - do - if grep -q ${r_mod} ${mod}/setup.py - then - install_module rem_list[@] ${r_mod} - fi - done - if [[ " ${GLIST[@]} " =~ " ${mod} " ]] - then - pushd $mod > /dev/null - pip install . - popd > /dev/null - GLIST=(${GLIST[@]/$mod}) - fi - [[ ${#GLIST[@]} -ne 0 ]] && install_module GLIST[@] ${GLIST[0]} -} - -# Calling function to install repos in order. -install_module GLIST[@] ${GLIST[0]} - -popd > /dev/null #leaving .deps From 895092d00a39941bad395f0a239f6fa82c39cb6b Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Wed, 21 Apr 2021 12:41:07 -0500 Subject: [PATCH 24/44] Do not docker login --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3532f1e..f1a1bb1 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ test-docker: docker run --rm ${DOCKER_IMAGE_LATEST} test .PHONY: publish publish-* -publish: docker-login +publish: docker push ${DOCKER_IMAGE_COMMIT} docker push ${DOCKER_IMAGE} From 3cc10551053cf81f844d61d3d088e96bfd76f3f7 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Fri, 23 Apr 2021 14:55:16 -0500 Subject: [PATCH 25/44] Test new docker pipeline --- Jenkinsfile | 103 +--------------------------------------------------- 1 file changed, 2 insertions(+), 101 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6fe2292..2b96245 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,104 +1,5 @@ #!groovy -PROJECT_NAME = "aliquot-maf-tools" -PYENV_VERSION = "3.6-dev" - -BRANCH_NAME = env.BRANCH_NAME -GIT_HASH = "" -VERSION = "" - -PROXY = "http://cloud-proxy:3128" - -pipeline { - agent any - environment { - TWINE_REPOSITORY_URL = credentials("${BRANCH_NAME == 'main' ? 'twine_repository_url_prod' : 'twine_repository_url'}") - TWINE_USERNAME = credentials('twine_username') - TWINE_PASSWORD = credentials('twine_password') - QUAY_USERNAME = credentials('QUAY_USERNAME') - QUAY_PASSWORD = credentials('QUAY_PASSWORD') - } - options { - disableConcurrentBuilds() - skipStagesAfterUnstable() - } - - stages { - stage('Init') { - steps { - vbash 'make version' - - script { - GIT_HASH = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() - VERSION = sh(script: "make print-version BRANCH_NAME=${BRANCH_NAME}", returnStdout: true).trim() - PYPI_VERSION = sh(script: "make print-pypi", returnStdout: true).trim() - currentBuild.displayName = "#${currentBuild.number} - ${VERSION}" - } - - echo "Version: ${VERSION}" - } - } - stage('Docker Build') { - steps { - vbash "make build-docker PROXY=${PROXY}" - } - } - stage('Docker Test') { - steps { - sh 'make test-docker' - } - } - stage('Docker Publish Staging') { - when { - anyOf { - branch 'feat/*' - branch 'feature/*' - branch 'develop' - branch 'hotfix/*' - branch 'release/*' - } - } - steps { - sh 'make publish-staging' - } - } - stage('Docker Publish Release') { - when { - anyOf { - branch 'main' - } - } - steps { - sh 'make publish-release' - } - } - stage('PyPI Publish Branch') { - when { - anyOf { - branch 'main' - branch 'develop' - branch 'hotfix/*' - branch 'release/*' - } - } - steps { - echo "Building PyPI Version: ${PYPI_VERSION}" - sh "pip install --user twine wheel" - vbash "make build-pypi" - vbash "make publish-pypi" - } - } - } -} - -def vbash(command) { - sh """#!/bin/bash - eval \"\$(pyenv init -)\" - eval \"\$(pyenv virtualenv-init -)\" - - pyenv virtualenv ${PYENV_VERSION} ${PROJECT_NAME}-venv || true - pyenv activate ${PROJECT_NAME}-venv - - ${command} - """ +library identifier: "jenkins-lib@feat/bio-docker-pipeline" +bioDockerPipeline { } From 38d05f73ddcd2c306d48551971906ce56598c7ef Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 10:09:55 -0500 Subject: [PATCH 26/44] Revert jenkins pipeline --- Jenkinsfile | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++-- setup.py | 4 +-- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2b96245..8b4b0a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,103 @@ #!groovy -library identifier: "jenkins-lib@feat/bio-docker-pipeline" -bioDockerPipeline { +PROJECT_NAME = "aliquot-maf-tools" +PYENV_VERSION = "3.6-dev" + +BRANCH_NAME = env.BRANCH_NAME +GIT_HASH = "" +VERSION = "" + +PROXY = "http://cloud-proxy:3128" + +pipeline { + agent any + environment { + TWINE_REPOSITORY_URL = credentials("${BRANCH_NAME == 'main' ? 'twine_repository_url_prod' : 'twine_repository_url'}") + TWINE_USERNAME = credentials('twine_username') + TWINE_PASSWORD = credentials('twine_password') + QUAY_USERNAME = credentials('QUAY_USERNAME') + QUAY_PASSWORD = credentials('QUAY_PASSWORD') + } + options { + disableConcurrentBuilds() + skipStagesAfterUnstable() + } + + stages { + stage('Init') { + steps { + vbash 'make version' + + script { + GIT_HASH = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() + VERSION = sh(script: "make print-version BRANCH_NAME=${BRANCH_NAME}", returnStdout: true).trim() + PYPI_VERSION = sh(script: "make print-pypi", returnStdout: true).trim() + currentBuild.displayName = "#${currentBuild.number} - ${VERSION}" + } + + echo "Version: ${VERSION}" + } + } + stage('Docker Build') { + steps { + vbash "make build-docker PROXY=${PROXY}" + } + } + stage('Docker Test') { + steps { + sh 'make test-docker' + } + } + stage('Docker Publish Staging') { + when { + anyOf { + branch 'feat*' + branch 'develop' + branch 'hotfix/*' + branch 'release/*' + } + } + steps { + sh 'make publish-staging' + } + } + stage('Docker Publish Release') { + when { + anyOf { + branch 'main' + } + } + steps { + sh 'make publish-release' + } + } + stage('PyPI Publish Branch') { + when { + anyOf { + branch 'main' + branch 'develop' + branch 'hotfix/*' + branch 'release/*' + } + } + steps { + echo "Building PyPI Version: ${PYPI_VERSION}" + sh "pip install --user twine wheel" + vbash "make build-pypi" + vbash "make publish-pypi" + } + } + } +} + +def vbash(command) { + sh """#!/bin/bash + eval \"\$(pyenv init -)\" + eval \"\$(pyenv virtualenv-init -)\" + + pyenv virtualenv ${PYENV_VERSION} ${PROJECT_NAME}-venv || true + pyenv activate ${PROJECT_NAME}-venv + + ${command} + """ } diff --git a/setup.py b/setup.py index 19535b9..1f0a5ba 100644 --- a/setup.py +++ b/setup.py @@ -125,7 +125,7 @@ def write_requirements(self, path, reqs): python_requires=">=3.6", setup_requires=["setuptools_scm"], use_scm_version={ - "write_to": os.path.join(f"{PACKAGE}", "_version.py"), + "write_to": os.path.join(PACKAGE, "_version.py"), "fallback_version": __pypi_version__, }, packages=find_packages(), @@ -133,7 +133,7 @@ def write_requirements(self, path, reqs): tests_require=TESTS_REQUIRE, cmdclass={"capture_requirements": Requirements, "print_version": PrintVersion}, include_package_data=True, - scripts=[os.path.join(os.path.dirname(__file__), 'bin', f"{PACKAGE}")], + scripts=[os.path.join(os.path.dirname(__file__), 'bin', PACKAGE)], entry_points={"console_scripts": ["aliquot-maf-tools = aliquotmaf.__main__:main"]}, ) From 787b667ca8ce510a0634f0204f6b26dbf8b89b1a Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 12:19:52 -0500 Subject: [PATCH 27/44] Docker image cleanup target --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f1a1bb1..c8ca8e0 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ docker-login: docker login -u="${QUAY_USERNAME}" -p="${QUAY_PASSWORD}" quay.io -.PHONY: build build-* clean init init-* lint requirements run version +.PHONY: build build-* clean clean-* init init-* lint requirements run version init: init-pip init-hooks # Include next line if publishing to Jenkins @@ -59,6 +59,10 @@ clean: rm -rf ./dist/ rm -rf ./*.egg-info/ +clean-docker: + @echo "Removing latest image: ${DOCKER_IMAGE_LATEST}" + docker rmi -f ${DOCKER_IMAGE_LATEST} + lint: @echo @echo -- Lint -- From f5c6a4205962bd854bbb3d855df6438840e2e264 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 12:21:07 -0500 Subject: [PATCH 28/44] Update requirements --- requirements.txt | 2 +- setup.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 63271ec..af33478 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # --extra-index-url https://nexus.osdc.io/repository/pypi-all/simple -bioinf-maflib==1.0.0 +bioinf-maflib==1.1.0 # via bioinf-aliquotmaf (setup.py) pysam==0.16.0.1 # via bioinf-aliquotmaf (setup.py) diff --git a/setup.py b/setup.py index 1f0a5ba..26ab90a 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import importlib import os import subprocess @@ -6,9 +8,6 @@ from setuptools import Command, find_packages, setup -#!/usr/bin/env python - - GIT_REPO = "aliquot-maf-tools" PACKAGE = "aliquotmaf" @@ -16,7 +15,7 @@ GIT_REPO_URL = "https://github.com/NCI-GDC/{}".format(GIT_REPO) INSTALL_REQUIRES = [ - "bioinf-maflib", + "bioinf-maflib>=1.1.0", "pysam", ] From 0762f53f080ae6f165d1ed3e39dc7ae506366ba0 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 12:38:17 -0500 Subject: [PATCH 29/44] set proxy --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 8b4b0a6..6f7cc5e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,6 +17,8 @@ pipeline { TWINE_PASSWORD = credentials('twine_password') QUAY_USERNAME = credentials('QUAY_USERNAME') QUAY_PASSWORD = credentials('QUAY_PASSWORD') + http_proxy = "$PROXY" + https_proxy = "$PROXY" } options { disableConcurrentBuilds() From bc067d7324a5d74a11dcdd385d7fd137d269765e Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 14:16:56 -0500 Subject: [PATCH 30/44] Test multistage build --- Dockerfile.multistage | 28 ++++++++++++++++++++++++++++ Makefile | 12 ++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 Dockerfile.multistage diff --git a/Dockerfile.multistage b/Dockerfile.multistage new file mode 100644 index 0000000..bd0fbeb --- /dev/null +++ b/Dockerfile.multistage @@ -0,0 +1,28 @@ +FROM quay.io/ncigdc/bio-python:3.6 as builder + +COPY ./ /opt + +WORKDIR /opt + +RUN python3 setup.py build \ + && mkdir -p dist/ \ + && cp -r build/lib/* dist/ \ + && cp -r bin/ dist/ \ + && cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/ \ + && ls -l /opt/dist/* + +FROM quay.io/ncigdc/bio-python:3.6 + +COPY --from=builder /opt /opt + +ENV BINARY=aliquotmaf + +WORKDIR /opt + +RUN make init-pip \ + && ln -s /opt/bin/${BINARY} /bin/${BINARY} \ + && chmod +x /bin/${BINARY} + +ENTRYPOINT ["/bin/aliquotmaf"] + +CMD ["--help"] diff --git a/Makefile b/Makefile index c8ca8e0..68f5dc8 100644 --- a/Makefile +++ b/Makefile @@ -87,13 +87,13 @@ build: build-docker build-docker: @echo @echo -- Building docker -- - python3 setup.py build - mkdir -p dist - cp -r build/lib/* dist/ - cp -r bin/ dist/ - cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/ + # python3 setup.py build + # mkdir -p dist + # cp -r build/lib/* dist/ + # cp -r bin/ dist/ + # cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/ docker build . \ - --file ./Dockerfile \ + --file ./Dockerfile.multistage \ --build-arg http_proxy=${PROXY} \ --build-arg https_proxy=${PROXY} \ -t "${DOCKER_IMAGE_COMMIT}" \ From 787a64e1239f396aab8747d34ca3c42e73ca6347 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 14:18:13 -0500 Subject: [PATCH 31/44] Update pipeline --- Jenkinsfile | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6f7cc5e..45f329f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,20 +26,6 @@ pipeline { } stages { - stage('Init') { - steps { - vbash 'make version' - - script { - GIT_HASH = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() - VERSION = sh(script: "make print-version BRANCH_NAME=${BRANCH_NAME}", returnStdout: true).trim() - PYPI_VERSION = sh(script: "make print-pypi", returnStdout: true).trim() - currentBuild.displayName = "#${currentBuild.number} - ${VERSION}" - } - - echo "Version: ${VERSION}" - } - } stage('Docker Build') { steps { vbash "make build-docker PROXY=${PROXY}" @@ -73,22 +59,6 @@ pipeline { sh 'make publish-release' } } - stage('PyPI Publish Branch') { - when { - anyOf { - branch 'main' - branch 'develop' - branch 'hotfix/*' - branch 'release/*' - } - } - steps { - echo "Building PyPI Version: ${PYPI_VERSION}" - sh "pip install --user twine wheel" - vbash "make build-pypi" - vbash "make publish-pypi" - } - } } } From b047f686ac93c2dec5e0130ecc5e734b855c4725 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 15:22:40 -0500 Subject: [PATCH 32/44] Test --- Dockerfile.multistage | 5 ++++- Makefile | 12 ++++++------ setup.py | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Dockerfile.multistage b/Dockerfile.multistage index bd0fbeb..9afb88b 100644 --- a/Dockerfile.multistage +++ b/Dockerfile.multistage @@ -1,5 +1,7 @@ FROM quay.io/ncigdc/bio-python:3.6 as builder +ARG PIP_EXTRA_INDEX_URL + COPY ./ /opt WORKDIR /opt @@ -19,7 +21,8 @@ ENV BINARY=aliquotmaf WORKDIR /opt -RUN make init-pip \ +RUN python3 -m pip install -r requirements.txt -r dev-requirements.txt \ + && python3 setup.py develop \ && ln -s /opt/bin/${BINARY} /bin/${BINARY} \ && chmod +x /bin/${BINARY} diff --git a/Makefile b/Makefile index 68f5dc8..05c53a4 100644 --- a/Makefile +++ b/Makefile @@ -6,16 +6,17 @@ MODULE = aliquotmaf GIT_SHORT_HASH:=$(shell git rev-parse --short HEAD) -PYPI_VERSION:=$(shell python3 setup.py -q print_version --pypi) -DOCKER_VERSION:=$(shell python3 setup.py -q print_version --docker) -COMMIT_HASH:=$(shell python3 setup.py -q print_version --hash) +# PYPI_VERSION:=$(shell python3 setup.py -q print_version --pypi) +# DOCKER_VERSION:=$(shell python3 setup.py -q print_version --docker) +# COMMIT_HASH:=$(shell python3 setup.py -q print_version --hash) +COMMIT_HASH:=$(shell git rev-parse HEAD) DOCKER_REPO := quay.io/ncigdc -DOCKER_IMAGE := ${DOCKER_REPO}/${REPO}:${DOCKER_VERSION} +# DOCKER_IMAGE := ${DOCKER_REPO}/${REPO}:${DOCKER_VERSION} DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH} DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest -TWINE_REPOSITORY_URL?="" +TWINE_REPOSITORY_URL?= PIP_EXTRA_INDEX_URL?= .PHONY: version version-* print-* @@ -97,7 +98,6 @@ build-docker: --build-arg http_proxy=${PROXY} \ --build-arg https_proxy=${PROXY} \ -t "${DOCKER_IMAGE_COMMIT}" \ - -t "${DOCKER_IMAGE}" \ -t "${DOCKER_IMAGE_LATEST}" build-pypi: diff --git a/setup.py b/setup.py index 26ab90a..4c43586 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ GIT_REPO_URL = "https://github.com/NCI-GDC/{}".format(GIT_REPO) INSTALL_REQUIRES = [ - "bioinf-maflib>=1.1.0", + "bioinf-maflib", "pysam", ] From 8f981bc7cd52ca7fe68b1c69a3b235d0b0b7a186 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Mon, 26 Apr 2021 15:43:56 -0500 Subject: [PATCH 33/44] Get pypi version from docker --- Jenkinsfile | 22 ++-------------------- Makefile | 13 ++++--------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 45f329f..b9dcfa7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,27 +36,9 @@ pipeline { sh 'make test-docker' } } - stage('Docker Publish Staging') { - when { - anyOf { - branch 'feat*' - branch 'develop' - branch 'hotfix/*' - branch 'release/*' - } - } - steps { - sh 'make publish-staging' - } - } - stage('Docker Publish Release') { - when { - anyOf { - branch 'main' - } - } + stage('Docker Publish') { steps { - sh 'make publish-release' + sh 'make publish-docker DOCKER_IMAGE=$(make version-docker)' } } } diff --git a/Makefile b/Makefile index 05c53a4..409e423 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,7 @@ version: @echo --- VERSION: ${PYPI_VERSION} --- version-docker: - @echo ${DOCKER_IMAGE} - @echo ${DOCKER_IMAGE_COMMIT} + @docker run --rm ${DOCKER_IMAGE_LATEST} --version .PHONY: docker-login docker-login: @@ -88,11 +87,6 @@ build: build-docker build-docker: @echo @echo -- Building docker -- - # python3 setup.py build - # mkdir -p dist - # cp -r build/lib/* dist/ - # cp -r bin/ dist/ - # cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/ docker build . \ --file ./Dockerfile.multistage \ --build-arg http_proxy=${PROXY} \ @@ -121,8 +115,9 @@ test-docker: @echo -- Running Docker Test -- docker run --rm ${DOCKER_IMAGE_LATEST} test -.PHONY: publish publish-* -publish: +.PHONY: publish-* +publish-docker: + docker tag ${DOCKER_IMAGE_COMMIT} ${DOCKER_IMAGE} docker push ${DOCKER_IMAGE_COMMIT} docker push ${DOCKER_IMAGE} From 8d369cd5d986e678168380cfcaa7bba4de6f69c7 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 09:20:52 -0500 Subject: [PATCH 34/44] Update to grab tag from docker --- Dockerfile.multistage | 7 ++----- Jenkinsfile | 7 +++---- Makefile | 13 ++++--------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Dockerfile.multistage b/Dockerfile.multistage index 9afb88b..30d6246 100644 --- a/Dockerfile.multistage +++ b/Dockerfile.multistage @@ -1,7 +1,5 @@ FROM quay.io/ncigdc/bio-python:3.6 as builder -ARG PIP_EXTRA_INDEX_URL - COPY ./ /opt WORKDIR /opt @@ -10,12 +8,11 @@ RUN python3 setup.py build \ && mkdir -p dist/ \ && cp -r build/lib/* dist/ \ && cp -r bin/ dist/ \ - && cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/ \ - && ls -l /opt/dist/* + && cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/ FROM quay.io/ncigdc/bio-python:3.6 -COPY --from=builder /opt /opt +COPY --from=builder /opt/dist/ /opt ENV BINARY=aliquotmaf diff --git a/Jenkinsfile b/Jenkinsfile index b9dcfa7..1f13a58 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,8 +17,6 @@ pipeline { TWINE_PASSWORD = credentials('twine_password') QUAY_USERNAME = credentials('QUAY_USERNAME') QUAY_PASSWORD = credentials('QUAY_PASSWORD') - http_proxy = "$PROXY" - https_proxy = "$PROXY" } options { disableConcurrentBuilds() @@ -28,7 +26,7 @@ pipeline { stages { stage('Docker Build') { steps { - vbash "make build-docker PROXY=${PROXY}" + vbash "make build-docker" } } stage('Docker Test') { @@ -38,7 +36,8 @@ pipeline { } stage('Docker Publish') { steps { - sh 'make publish-docker DOCKER_IMAGE=$(make version-docker)' + DOCKER_IMAGE = sh([script: "make version-docker-tag", returnStdout: true]).trim() + sh 'make publish-docker DOCKER_IMAGE=${DOCKER_IMAGE}' } } } diff --git a/Makefile b/Makefile index 409e423..e050eae 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,10 @@ -# UPDATE ME REPO = aliquot-maf-tools -# UPDATE ME MODULE = aliquotmaf -GIT_SHORT_HASH:=$(shell git rev-parse --short HEAD) - -# PYPI_VERSION:=$(shell python3 setup.py -q print_version --pypi) -# DOCKER_VERSION:=$(shell python3 setup.py -q print_version --docker) -# COMMIT_HASH:=$(shell python3 setup.py -q print_version --hash) COMMIT_HASH:=$(shell git rev-parse HEAD) DOCKER_REPO := quay.io/ncigdc -# DOCKER_IMAGE := ${DOCKER_REPO}/${REPO}:${DOCKER_VERSION} DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH} DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest @@ -24,7 +16,10 @@ version: @echo --- VERSION: ${PYPI_VERSION} --- version-docker: - @docker run --rm ${DOCKER_IMAGE_LATEST} --version + @python setup.py -q print_version --docker + +version-docker-tag: + @docker run --rm --entrypoint="make -C /opt" ${DOCKER_IMAGE_LATEST} version-docker .PHONY: docker-login docker-login: From 3c25c51847f51b919d26e2d650066fa8af591c21 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 09:28:28 -0500 Subject: [PATCH 35/44] Fix --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1f13a58..1e9750c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,7 +36,9 @@ pipeline { } stage('Docker Publish') { steps { - DOCKER_IMAGE = sh([script: "make version-docker-tag", returnStdout: true]).trim() + script { + DOCKER_IMAGE = sh(script: "make version-docker-tag", returnStdout: true).trim() + } sh 'make publish-docker DOCKER_IMAGE=${DOCKER_IMAGE}' } } From 4b44e7e02cd49123733eeb6e5ae43ca6de5ab1c6 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 09:36:41 -0500 Subject: [PATCH 36/44] Pass in proxy --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1e9750c..6202230 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,7 +26,7 @@ pipeline { stages { stage('Docker Build') { steps { - vbash "make build-docker" + vbash "make build-docker PROXY=${PROXY}" } } stage('Docker Test') { From 0d647dcc98fbe24ed4d7136a5db36725fbbd37de Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 09:39:57 -0500 Subject: [PATCH 37/44] Clean in post --- Jenkinsfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 6202230..75f40e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,6 +43,11 @@ pipeline { } } } + post { + always { + sh 'make clean' + } + } } def vbash(command) { From dd5d0bd01213fdd056754b9e2297ffb10323799e Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 09:54:27 -0500 Subject: [PATCH 38/44] Fix getting version via setuptools_scm --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e050eae..75a28b4 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ REPO = aliquot-maf-tools MODULE = aliquotmaf -COMMIT_HASH:=$(shell git rev-parse HEAD) +COMMIT_HASH:=$(shell git rev-parse HEAD 2>/dev/null) DOCKER_REPO := quay.io/ncigdc DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH} @@ -19,7 +19,7 @@ version-docker: @python setup.py -q print_version --docker version-docker-tag: - @docker run --rm --entrypoint="make -C /opt" ${DOCKER_IMAGE_LATEST} version-docker + @docker run --rm --entrypoint="make" ${DOCKER_IMAGE_LATEST} "version-docker" .PHONY: docker-login docker-login: From 1bf57371ef5622f307f37e2a38c98e176a9e3cef Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 10:00:54 -0500 Subject: [PATCH 39/44] Typo --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 75f40e5..130fb80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,7 @@ pipeline { script { DOCKER_IMAGE = sh(script: "make version-docker-tag", returnStdout: true).trim() } - sh 'make publish-docker DOCKER_IMAGE=${DOCKER_IMAGE}' + sh "make publish-docker DOCKER_IMAGE=${DOCKER_IMAGE}" } } } From a461279b8ef696b86c9e4f49165465a45f5aa993 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 10:10:27 -0500 Subject: [PATCH 40/44] Fix docker tag --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 75a28b4..ada6944 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,7 @@ test-docker: .PHONY: publish-* publish-docker: + DOCKER_IMAGE:=${DOCKER_REPO}/${REPO}:{DOCKER_TAG} docker tag ${DOCKER_IMAGE_COMMIT} ${DOCKER_IMAGE} docker push ${DOCKER_IMAGE_COMMIT} docker push ${DOCKER_IMAGE} From a333b21ccd8861031031290067a7acbebc119b71 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 10:16:57 -0500 Subject: [PATCH 41/44] Typo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ada6944..94289d6 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ test-docker: .PHONY: publish-* publish-docker: - DOCKER_IMAGE:=${DOCKER_REPO}/${REPO}:{DOCKER_TAG} + DOCKER_IMAGE:=${DOCKER_REPO}/${REPO}:${DOCKER_TAG} docker tag ${DOCKER_IMAGE_COMMIT} ${DOCKER_IMAGE} docker push ${DOCKER_IMAGE_COMMIT} docker push ${DOCKER_IMAGE} From 058ab476362646aedc468576e72d2a466e7c0e28 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 10:24:31 -0500 Subject: [PATCH 42/44] Correct variable name --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 130fb80..241decf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,9 +37,9 @@ pipeline { stage('Docker Publish') { steps { script { - DOCKER_IMAGE = sh(script: "make version-docker-tag", returnStdout: true).trim() + DOCKER_TAG = sh(script: "make version-docker-tag", returnStdout: true).trim() } - sh "make publish-docker DOCKER_IMAGE=${DOCKER_IMAGE}" + sh "make publish-docker DOCKER_TAG=${DOCKER_TAG}" } } } From cee793e942925db9162c146767d19644a6f01442 Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 10:34:10 -0500 Subject: [PATCH 43/44] Fix --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 94289d6..55287b1 100644 --- a/Makefile +++ b/Makefile @@ -112,8 +112,7 @@ test-docker: .PHONY: publish-* publish-docker: - DOCKER_IMAGE:=${DOCKER_REPO}/${REPO}:${DOCKER_TAG} - docker tag ${DOCKER_IMAGE_COMMIT} ${DOCKER_IMAGE} + docker tag ${DOCKER_IMAGE_COMMIT} ${DOCKER_REPO}/${REPO}:${DOCKER_TAG} docker push ${DOCKER_IMAGE_COMMIT} docker push ${DOCKER_IMAGE} From 06c225375cdf8e16fc02977278c233a02ec3006d Mon Sep 17 00:00:00 2001 From: Charles Czysz Date: Tue, 27 Apr 2021 10:43:44 -0500 Subject: [PATCH 44/44] Fix push tag --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 55287b1..f8e99ff 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ test-docker: publish-docker: docker tag ${DOCKER_IMAGE_COMMIT} ${DOCKER_REPO}/${REPO}:${DOCKER_TAG} docker push ${DOCKER_IMAGE_COMMIT} - docker push ${DOCKER_IMAGE} + docker push ${DOCKER_REPO}/${REPO}:${DOCKER_TAG} publish-pypi: dist/*.whl @echo