From f5057ffaa367542808b9f5f8ce1538f7b00d1572 Mon Sep 17 00:00:00 2001 From: adamstruck Date: Tue, 3 Sep 2019 09:28:30 -0700 Subject: [PATCH] fix linting and test errors --- .travis.yml | 7 ++----- README.md | 2 +- cwl_tes/ftp.py | 1 + cwl_tes/main.py | 16 ++++++++++++---- cwl_tes/tes.py | 27 ++++++++++++++++++--------- requirements.txt | 2 +- setup.py | 7 ++++--- tests/schemas | 2 +- tox.ini | 18 +++++++++--------- 9 files changed, 49 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68e771f..ad0016a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: xenial sudo: required language: python cache: pip @@ -5,10 +6,9 @@ python: - 2.7 - 3.5 - 3.6 + - 3.7 services: - docker -os: - - linux install: - pip install tox-travis script: @@ -16,8 +16,5 @@ script: after_failure: - echo "FUNNEL LOGS ----------" - cat ./tests/test_tmp/conformance_test_v1.0_*/funnel_log.txt -branches: - only: - - master notifications: email: false diff --git a/README.md b/README.md index 371507a..06ebd02 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/common-workflow-language/cwl-tes.svg?branch=master)](https://travis-ci.org/common-workflow-language/cwl-tes) +[![Build Status](https://travis-ci.org/ohsu-comp-bio/cwl-tes.svg?branch=master)](https://travis-ci.org/ohsu-comp-bio/cwl-tes) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) # GA4GH CWL Task Execution diff --git a/cwl_tes/ftp.py b/cwl_tes/ftp.py index 0787ce4..4f1adbe 100644 --- a/cwl_tes/ftp.py +++ b/cwl_tes/ftp.py @@ -13,6 +13,7 @@ from six import PY2 from six.moves import urllib from schema_salad.ref_resolver import uri_file_path +from typing import Tuple, Optional from cwltool.stdfsaccess import StdFsAccess from cwltool.loghandler import _logger diff --git a/cwl_tes/main.py b/cwl_tes/main.py index c2f099c..fb0ff64 100644 --- a/cwl_tes/main.py +++ b/cwl_tes/main.py @@ -19,12 +19,16 @@ from ruamel import yaml from schema_salad.sourceline import cmap +from typing import Any, Dict, Tuple, Optional import cwltool.main from cwltool.builder import substitute +from cwltool.context import LoadingContext, RuntimeContext from cwltool.process import scandeps, shortname -from cwltool.executors import MultithreadedJobExecutor, SingleJobExecutor +from cwltool.executors import (MultithreadedJobExecutor, SingleJobExecutor, + JobExecutor) from cwltool.resolver import ga4gh_tool_registries from cwltool.pathmapper import visit_class +from cwltool.process import Process from .tes import make_tes_tool, TESPathMapper from .__init__ import __version__ @@ -119,9 +123,13 @@ def main(args=None): if parsed_args.token: try: - jwt.decode(parsed_args.token, - parsed_args.token_public_key.encode('utf-8').decode('unicode_escape'), algorithms=['RS256']) - except Exception as e: + jwt.decode( + parsed_args.token, + parsed_args.token_public_key.encode('utf-8') + .decode('unicode_escape'), + algorithms=['RS256'] + ) + except Exception: raise Exception('Token is not valid') if parsed_args.quiet: diff --git a/cwl_tes/tes.py b/cwl_tes/tes.py index 5544c2e..9e27a76 100644 --- a/cwl_tes/tes.py +++ b/cwl_tes/tes.py @@ -4,6 +4,7 @@ import os import random import time +import threading import stat from builtins import str import shutil @@ -11,7 +12,8 @@ import uuid from tempfile import NamedTemporaryFile from pprint import pformat -from typing import Any, Dict, List, MutableMapping, MutableSequence, Union +from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence, + Optional, Union) from typing_extensions import Text import tes @@ -20,8 +22,11 @@ from schema_salad.ref_resolver import file_uri from schema_salad.sourceline import SourceLine from schema_salad import validate +from cwltool.builder import Builder from cwltool.command_line_tool import CommandLineTool +from cwltool.context import RuntimeContext from cwltool.errors import WorkflowException, UnsupportedRequirement +from cwltool.expression import JSON from cwltool.job import JobBase from cwltool.stdfsaccess import StdFsAccess from cwltool.pathmapper import (PathMapper, uri_file_path, MapperEnt, @@ -146,7 +151,7 @@ class TESTask(JobBase): def __init__(self, builder, # type: Builder - joborder, # type: JobOrderType + joborder, # type: JSON make_path_mapper, # type: Callable[..., PathMapper] requirements, # type: List[Dict[Text, Text]] hints, # type: List[Dict[Text, Text]] @@ -180,10 +185,10 @@ def get_container(self): docker_req, _ = self.get_requirement("DockerRequirement") if docker_req: - container = docker_req.get( - "dockerPull", - docker_req.get("dockerImageId", default) - ) + container = docker_req.get( + "dockerPull", + docker_req.get("dockerImageId", default) + ) return container def create_input(self, name, d): @@ -364,7 +369,10 @@ def create_task_msg(self): return create_body - def run(self, runtimeContext): + def run(self, + runtimeContext, # type: RuntimeContext + tmpdir_lock=None # type: Optional[threading.Lock] + ): # type: (...) -> None log.debug( "[job %s] self.__dict__ in run() ----------------------", self.name @@ -444,10 +452,11 @@ def run(self, runtimeContext): original_outdir = self.builder.outdir if not remote_cwl_output_json: self.builder.outdir = self.remote_storage_url - outputs = self.collect_outputs(self.remote_storage_url) + outputs = self.collect_outputs(self.remote_storage_url, + self.exit_code) self.builder.outdir = original_outdir else: - outputs = self.collect_outputs(self.outdir) + outputs = self.collect_outputs(self.outdir, self.exit_code) cleaned_outputs = {} for k, v in outputs.items(): if isinstance(k, bytes): diff --git a/requirements.txt b/requirements.txt index a844e90..96917d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ future>=0.16.0 requests>=2.18.2 py-tes>=0.3.0 PyJWT>=1.6.4 -typing_extensions +typing_extensions>=3.7.4 diff --git a/setup.py b/setup.py index 60fe95c..60a8f96 100644 --- a/setup.py +++ b/setup.py @@ -45,13 +45,13 @@ def find_version(*file_paths): "py-tes>=0.3.0", "PyJWT>=1.6.4", "requests>=2.14.2", - "typing_extensions" + "typing_extensions>=3.7.4" ], extras_require={ "test": [ - "cwltool>=1.0.20181201184214", + "cwltest>=1.0.20190228134645", "nose>=1.3.7", - "flake8>=3.5.0", + "flake8>=3.7.0", "PyYAML>=3.12" ] }, @@ -65,5 +65,6 @@ def find_version(*file_paths): "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", ], ) diff --git a/tests/schemas b/tests/schemas index 8721d40..ffd2dbf 160000 --- a/tests/schemas +++ b/tests/schemas @@ -1 +1 @@ -Subproject commit 8721d406301873e443d410d5140981312764920f +Subproject commit ffd2dbfc4a73937c91cfa05afa079f1609eff583 diff --git a/tox.ini b/tox.ini index 2fd7778..b79e9bd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,21 @@ [tox] envlist = - py{27,35,36}-lint, - py{27,35,36}-unit + py{27,35,36,37}-lint, + py{27,35,36,37}-unit skip_missing_interpreters = True [testenv] passenv = CI TRAVIS TRAVIS_* deps = -rrequirements.txt - py{27,35,36}-unit: .[test] - py{27,35,36}-lint: flake8>=3.5.0 + py{27,35,36,37}-unit: .[test] + py{27,35,36,37}-lint: flake8>=3.7.0 commands = - py{27,35,36}-unit: - wget -nc -O /tmp/funnel.tar.gz https://github.com/ohsu-comp-bio/funnel/releases/download/0.7.0/funnel-linux-amd64-0.7.0.tar.gz - py{27,35,36}-unit: tar -zxvf /tmp/funnel.tar.gz -C {envbindir} - py{27,35,36}-unit: git submodule update --init --recursive - py{27,35,36}-unit: python -m nose tests {posargs} - py{27,35,36}-lint: flake8 cwl_tes + py{27,35,36,37}-unit: - wget -nc -O /tmp/funnel.tar.gz https://github.com/ohsu-comp-bio/funnel/releases/download/0.7.0/funnel-linux-amd64-0.7.0.tar.gz + py{27,35,36,37}-unit: tar -zxvf /tmp/funnel.tar.gz -C {envbindir} + py{27,35,36,37}-unit: git submodule update --init --recursive + py{27,35,36,37}-unit: python -m nose tests {posargs} + py{27,35,36,37}-lint: flake8 cwl_tes whitelist_externals = wget tar