fix linting and test errors

.travis.yml

@@ -1,23 +1,20 @@
+dist: xenial
 sudo: required
 language: python
 cache: pip
 python:
   - 2.7
   - 3.5
   - 3.6
+  - 3.7
 services:
   - docker
-os:
-  - linux
 install:
   - pip install tox-travis
 script:
   - tox
 after_failure:
   - echo "FUNNEL LOGS ----------"
   - cat ./tests/test_tmp/conformance_test_v1.0_*/funnel_log.txt
-branches:
-  only:
-    - master
 notifications:
   email: false

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/common-workflow-language/cwl-tes.svg?branch=master)](https://travis-ci.org/common-workflow-language/cwl-tes)
+[![Build Status](https://travis-ci.org/ohsu-comp-bio/cwl-tes.svg?branch=master)](https://travis-ci.org/ohsu-comp-bio/cwl-tes)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 # GA4GH CWL Task Execution

cwl_tes/ftp.py

@@ -13,6 +13,7 @@
 from six import PY2
 from six.moves import urllib
 from schema_salad.ref_resolver import uri_file_path
+from typing import Tuple, Optional
 
 from cwltool.stdfsaccess import StdFsAccess
 from cwltool.loghandler import _logger

cwl_tes/main.py

@@ -19,12 +19,16 @@
 
 from ruamel import yaml
 from schema_salad.sourceline import cmap
+from typing import Any, Dict, Tuple, Optional
 import cwltool.main
 from cwltool.builder import substitute
+from cwltool.context import LoadingContext, RuntimeContext
 from cwltool.process import scandeps, shortname
-from cwltool.executors import MultithreadedJobExecutor, SingleJobExecutor
+from cwltool.executors import (MultithreadedJobExecutor, SingleJobExecutor,
+                               JobExecutor)
 from cwltool.resolver import ga4gh_tool_registries
 from cwltool.pathmapper import visit_class
+from cwltool.process import Process
 
 from .tes import make_tes_tool, TESPathMapper
 from .__init__ import __version__
@@ -119,9 +123,13 @@ def main(args=None):
 
     if parsed_args.token:
         try:
-            jwt.decode(parsed_args.token,
-                       parsed_args.token_public_key.encode('utf-8').decode('unicode_escape'), algorithms=['RS256'])
-        except Exception as e:
+            jwt.decode(
+                parsed_args.token,
+                parsed_args.token_public_key.encode('utf-8')
+                .decode('unicode_escape'),
+                algorithms=['RS256']
+            )
+        except Exception:
             raise Exception('Token is not valid')
 
     if parsed_args.quiet:

cwl_tes/tes.py

@@ -4,14 +4,16 @@
 import os
 import random
 import time
+import threading
 import stat
 from builtins import str
 import shutil
 import functools
 import uuid
 from tempfile import NamedTemporaryFile
 from pprint import pformat
-from typing import Any, Dict, List, MutableMapping, MutableSequence, Union
+from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
+                    Optional, Union)
 from typing_extensions import Text
 
 import tes
@@ -20,8 +22,11 @@
 from schema_salad.ref_resolver import file_uri
 from schema_salad.sourceline import SourceLine
 from schema_salad import validate
+from cwltool.builder import Builder
 from cwltool.command_line_tool import CommandLineTool
+from cwltool.context import RuntimeContext
 from cwltool.errors import WorkflowException, UnsupportedRequirement
+from cwltool.expression import JSON
 from cwltool.job import JobBase
 from cwltool.stdfsaccess import StdFsAccess
 from cwltool.pathmapper import (PathMapper, uri_file_path, MapperEnt,
@@ -146,7 +151,7 @@ class TESTask(JobBase):
 
     def __init__(self,
                  builder,   # type: Builder
-                 joborder,  # type: JobOrderType
+                 joborder,  # type: JSON
                  make_path_mapper,  # type: Callable[..., PathMapper]
                  requirements,  # type: List[Dict[Text, Text]]
                  hints,  # type: List[Dict[Text, Text]]
@@ -180,10 +185,10 @@ def get_container(self):
 
         docker_req, _ = self.get_requirement("DockerRequirement")
         if docker_req:
-                container = docker_req.get(
-                    "dockerPull",
-                    docker_req.get("dockerImageId", default)
-                )
+            container = docker_req.get(
+                "dockerPull",
+                docker_req.get("dockerImageId", default)
+            )
         return container
 
     def create_input(self, name, d):
@@ -364,7 +369,10 @@ def create_task_msg(self):
 
         return create_body
 
-    def run(self, runtimeContext):
+    def run(self,
+            runtimeContext,   # type: RuntimeContext
+            tmpdir_lock=None  # type: Optional[threading.Lock]
+            ):  # type: (...) -> None
         log.debug(
             "[job %s] self.__dict__ in run() ----------------------",
             self.name
@@ -444,10 +452,11 @@ def run(self, runtimeContext):
                 original_outdir = self.builder.outdir
                 if not remote_cwl_output_json:
                     self.builder.outdir = self.remote_storage_url
-                outputs = self.collect_outputs(self.remote_storage_url)
+                outputs = self.collect_outputs(self.remote_storage_url,
+                                               self.exit_code)
                 self.builder.outdir = original_outdir
             else:
-                outputs = self.collect_outputs(self.outdir)
+                outputs = self.collect_outputs(self.outdir, self.exit_code)
             cleaned_outputs = {}
             for k, v in outputs.items():
                 if isinstance(k, bytes):

requirements.txt

@@ -3,4 +3,4 @@ future>=0.16.0
 requests>=2.18.2
 py-tes>=0.3.0
 PyJWT>=1.6.4
-typing_extensions
+typing_extensions>=3.7.4

setup.py

@@ -45,13 +45,13 @@ def find_version(*file_paths):
         "py-tes>=0.3.0",
         "PyJWT>=1.6.4",
         "requests>=2.14.2",
-        "typing_extensions"
+        "typing_extensions>=3.7.4"
     ],
     extras_require={
         "test": [
-            "cwltool>=1.0.20181201184214",
+            "cwltest>=1.0.20190228134645",
             "nose>=1.3.7",
-            "flake8>=3.5.0",
+            "flake8>=3.7.0",
             "PyYAML>=3.12"
         ]
     },
@@ -65,5 +65,6 @@ def find_version(*file_paths):
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
     ],
 )

tox.ini

@@ -1,21 +1,21 @@
 [tox]
 envlist =
-  py{27,35,36}-lint,
-  py{27,35,36}-unit
+  py{27,35,36,37}-lint,
+  py{27,35,36,37}-unit
 skip_missing_interpreters = True
 
 [testenv]
 passenv = CI TRAVIS TRAVIS_*
 deps =
   -rrequirements.txt
-  py{27,35,36}-unit: .[test]
-  py{27,35,36}-lint: flake8>=3.5.0
+  py{27,35,36,37}-unit: .[test]
+  py{27,35,36,37}-lint: flake8>=3.7.0
 commands =
-  py{27,35,36}-unit: - wget -nc -O /tmp/funnel.tar.gz https://github.com/ohsu-comp-bio/funnel/releases/download/0.7.0/funnel-linux-amd64-0.7.0.tar.gz
-  py{27,35,36}-unit: tar -zxvf /tmp/funnel.tar.gz -C {envbindir}
-  py{27,35,36}-unit: git submodule update --init --recursive
-  py{27,35,36}-unit: python -m nose tests {posargs}
-  py{27,35,36}-lint: flake8 cwl_tes
+  py{27,35,36,37}-unit: - wget -nc -O /tmp/funnel.tar.gz https://github.com/ohsu-comp-bio/funnel/releases/download/0.7.0/funnel-linux-amd64-0.7.0.tar.gz
+  py{27,35,36,37}-unit: tar -zxvf /tmp/funnel.tar.gz -C {envbindir}
+  py{27,35,36,37}-unit: git submodule update --init --recursive
+  py{27,35,36,37}-unit: python -m nose tests {posargs}
+  py{27,35,36,37}-lint: flake8 cwl_tes
 whitelist_externals =
   wget
   tar