Merge pull request #13 from NCI-GDC/feature/maf-lib-pypi

Feature/maf lib pypi

.gitignore

@@ -1,6 +1,3 @@
-.deps/
-docker-requirements.txt
-
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -11,6 +8,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+env/
 build/
 develop-eggs/
 dist/
@@ -26,7 +24,7 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
+_version.py
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -48,7 +46,6 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
-.pytest_cache/
 
 # Translations
 *.mo
@@ -57,7 +54,6 @@ coverage.xml
 # Django stuff:
 *.log
 local_settings.py
-db.sqlite3
 
 # Flask stuff:
 instance/
@@ -84,14 +80,17 @@ celerybeat-schedule
 # SageMath parsed files
 *.sage.py
 
-# Environments
+# dotenv
 .env
+
+# virtualenv
 .venv
-env/
 venv/
 ENV/
-env.bak/
-venv.bak/
+
+# direnv
+.envrc
+.direnv/
 
 # Spyder project settings
 .spyderproject
@@ -105,3 +104,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+# VSCode
+.vscode/

.isort.cfg

@@ -0,0 +1,7 @@
+[settings]
+multi_line_output=3
+include_trailing_comma=True
+force_grid_wrap=0
+use_parentheses=True
+line_length=88
+known_third_party = maflib,pysam,pytest,setuptools

.pre-commit-config.yaml

@@ -1,6 +1,31 @@
+default_stages: [commit, push]
+default_language_version:
+  python_venv: python3.6
+
 repos:
--   repo: git@github.com:Yelp/detect-secrets
+  - repo: git@github.com:Yelp/detect-secrets
     rev: v0.13.1
     hooks:
-    -   id: detect-secrets
+      - id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+      - id: check-yaml
+      - id: check-toml
+      - id: check-json
+      - id: detect-aws-credentials
+        args: ["--allow-missing-credentials"]
+      - id: detect-private-key
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v1.9.3
+    hooks:
+      - id: seed-isort-config
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v4.3.21
+    hooks:
+      - id: isort
+  - repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+      - id: black

Dockerfile

@@ -1,12 +1,15 @@
-FROM python:3.5
+FROM quay.io/ncigdc/bio-python:3.6
 
-# Copy over source
-COPY . aliquot-maf-tools/
-WORKDIR /aliquot-maf-tools
+ENV BINARY=aliquotmaf
 
-# Installing dependencies
-RUN bash -c "./repo-install.sh" && \
-    pip install -r docker-requirements.txt
+COPY ./dist/ /opt
 
-# Install bio-submitter-qc
-RUN pip install .
+WORKDIR /opt
+
+RUN make init-pip \
+  && ln -s /opt/bin/${BINARY} /bin/${BINARY} \
+  && chmod +x /bin/${BINARY}
+
+ENTRYPOINT ["/bin/aliquotmaf"]
+
+CMD ["--help"]

Dockerfile.multistage

@@ -0,0 +1,28 @@
+FROM quay.io/ncigdc/bio-python:3.6 as builder
+
+COPY ./ /opt
+
+WORKDIR /opt
+
+RUN python3 setup.py build \
+	&& mkdir -p dist/ \
+	&& cp -r build/lib/* dist/ \
+	&& cp -r bin/ dist/ \
+	&& cp -f Makefile requirements.txt dev-requirements.txt README.md setup.py dist/
+
+FROM quay.io/ncigdc/bio-python:3.6
+
+COPY --from=builder /opt/dist/ /opt
+
+ENV BINARY=aliquotmaf
+
+WORKDIR /opt
+
+RUN python3 -m pip install -r requirements.txt -r dev-requirements.txt \
+	&& python3 setup.py develop \
+	&& ln -s /opt/bin/${BINARY} /bin/${BINARY} \
+	&& chmod +x /bin/${BINARY}
+
+ENTRYPOINT ["/bin/aliquotmaf"]
+
+CMD ["--help"]

Jenkinsfile

@@ -0,0 +1,63 @@
+#!groovy
+
+PROJECT_NAME = "aliquot-maf-tools"
+PYENV_VERSION = "3.6-dev"
+
+BRANCH_NAME = env.BRANCH_NAME
+GIT_HASH = ""
+VERSION = ""
+
+PROXY = "http://cloud-proxy:3128"
+
+pipeline {
+  agent any
+  environment {
+        TWINE_REPOSITORY_URL = credentials("${BRANCH_NAME == 'main' ? 'twine_repository_url_prod' : 'twine_repository_url'}")
+	TWINE_USERNAME = credentials('twine_username')
+	TWINE_PASSWORD = credentials('twine_password')
+	QUAY_USERNAME = credentials('QUAY_USERNAME')
+	QUAY_PASSWORD = credentials('QUAY_PASSWORD')
+  }
+  options {
+    disableConcurrentBuilds()
+    skipStagesAfterUnstable()
+  }
+
+  stages {
+    stage('Docker Build') {
+      steps {
+        vbash "make build-docker PROXY=${PROXY}"
+      }
+    }
+    stage('Docker Test') {
+      steps {
+        sh 'make test-docker'
+      }
+    }
+    stage('Docker Publish') {
+      steps {
+        script {
+          DOCKER_TAG = sh(script: "make version-docker-tag", returnStdout: true).trim()
+	}
+        sh "make publish-docker DOCKER_TAG=${DOCKER_TAG}"
+      }
+    }
+  }
+  post {
+    always {
+      sh 'make clean'
+    }
+  }
+}
+
+def vbash(command) {
+  sh """#!/bin/bash
+        eval \"\$(pyenv init -)\"
+	eval \"\$(pyenv virtualenv-init -)\"
+
+	pyenv virtualenv ${PYENV_VERSION} ${PROJECT_NAME}-venv || true
+	pyenv activate ${PROJECT_NAME}-venv
+
+	${command}
+  """
+}

Makefile

@@ -0,0 +1,122 @@
+REPO = aliquot-maf-tools
+
+MODULE = aliquotmaf
+
+COMMIT_HASH:=$(shell git rev-parse HEAD 2>/dev/null)
+
+DOCKER_REPO := quay.io/ncigdc
+DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH}
+DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest
+
+TWINE_REPOSITORY_URL?=
+PIP_EXTRA_INDEX_URL?=
+
+.PHONY: version version-* print-*
+version:
+	@echo --- VERSION: ${PYPI_VERSION} ---
+
+version-docker:
+	@python setup.py -q print_version --docker
+
+version-docker-tag:
+	@docker run --rm --entrypoint="make" ${DOCKER_IMAGE_LATEST} "version-docker"
+
+.PHONY: docker-login
+docker-login:
+	docker login -u="${QUAY_USERNAME}" -p="${QUAY_PASSWORD}" quay.io
+
+
+.PHONY: build build-* clean clean-* init init-* lint requirements run version
+init: init-pip init-hooks
+
+# Include next line if publishing to Jenkins
+# --extra-index-url ${TWINE_REPOSITORY_URL}
+init-pip:
+	@echo
+	@echo -- Installing pip packages --
+	pip3 install \
+		--no-cache-dir \
+		-r dev-requirements.txt \
+		-r requirements.txt
+	python3 setup.py develop
+
+init-hooks:
+	@echo
+	@echo -- Installing Precommit Hooks --
+	pre-commit install
+
+init-venv:
+	@echo
+	PIP_REQUIRE_VIRTUALENV=true pip3 install --upgrade pip-tools
+
+clean:
+	rm -rf ./build/
+	rm -rf ./dist/
+	rm -rf ./*.egg-info/
+
+clean-docker:
+	@echo "Removing latest image: ${DOCKER_IMAGE_LATEST}"
+	docker rmi -f ${DOCKER_IMAGE_LATEST}
+
+lint:
+	@echo
+	@echo -- Lint --
+	python3 -m flake8 ${MODULE}/
+
+run:
+	bin/run
+
+requirements: init-venv requirements-prod requirements-dev
+
+requirements-dev:
+	python3 setup.py -q capture_requirements --dev
+	pip-compile -o dev-requirements.txt dev-requirements.in
+
+requirements-prod:
+	pip-compile -o requirements.txt
+
+.PHONY: build build-*
+
+build: build-docker
+
+build-docker:
+	@echo
+	@echo -- Building docker --
+	docker build . \
+		--file ./Dockerfile.multistage \
+		--build-arg http_proxy=${PROXY} \
+		--build-arg https_proxy=${PROXY} \
+		-t "${DOCKER_IMAGE_COMMIT}" \
+		-t "${DOCKER_IMAGE_LATEST}"
+
+build-pypi:
+	@echo
+	@echo Building wheel - ${PYPI_VERSION}
+	python3 setup.py bdist_wheel -b ${MODULE}.egg-info
+
+.PHONY: test test-*
+test: lint test-unit
+
+test-unit:
+	@echo
+	@echo -- Unit Test --
+	python3 -m pytest --cov-report term-missing \
+		--junitxml=build/unit-test.xml \
+		--cov=${MODULE} \
+		tests/
+
+test-docker:
+	@echo
+	@echo -- Running Docker Test --
+	docker run --rm ${DOCKER_IMAGE_LATEST} test
+
+.PHONY: publish-*
+publish-docker:
+	docker tag ${DOCKER_IMAGE_COMMIT} ${DOCKER_REPO}/${REPO}:${DOCKER_TAG}
+	docker push ${DOCKER_IMAGE_COMMIT}
+	docker push ${DOCKER_REPO}/${REPO}:${DOCKER_TAG}
+
+publish-pypi: dist/*.whl
+	@echo
+	@echo Publishing wheel
+	python3 -m twine upload $(shell ls -1 dist/*.whl | head -1)

aliquotmaf/__init__.py

@@ -0,0 +1,6 @@
+try:
+    from aliquotmaf._version import version
+
+    __version__ = version
+except ImportError:
+    __version__ = '0'