Merge pull request #12 from wtsi-hgi/dev2

Merge latest development into master

Author: ThomasHickman

CHANGELOG

@@ -1,3 +1,9 @@
+v1.4
+- Added no_docker command line option, changed docker_container_name to docker_image_name
+- Fixed beheviour of array and tag types
+- Rewrote gen_cwl_arg to be clearer
+- Reduced the dependance on the command line for the tests
+
 v1.3
 - Changed the docker container to be broad institute's official docker container, not wtsi-hgi own container
 

README.md

@@ -4,7 +4,7 @@ Generates [CWL](http://www.commonwl.org/v1.0/) files from the [GATK documentatio
 
 ## Installation
 
-First, install the module 
+First, install the module
 ```bash
 git clone https://github.com/wtsi-hgi/gatk-cwl-generator
 cd gatk-cwl-generator
@@ -13,38 +13,44 @@ python setup.py install
 
 You may also want to install [cwltool](https://github.com/common-workflow-language/cwltool) to run the generated CWL files
 
+## Requirements
+
+- Python 3
+- Docker and node.js for the tests
+
 ## Usage
 
 ```
-usage: gatkcwlgenerator [-h] [--version GATKVERSION] [--out OUTPUTDIR]
-                        [--include INCLUDE_FILE] [--dev]
-                        [--docker_container_name DOCKER_CONTAINER_NAME]
-                        [--gatk_location GATK_LOCATION]
+usage: gatk_cwl_generator [-h] [--version VERSION] [--out OUTPUT_DIR]
+                          [--include INCLUDE] [--dev] [--no_docker]
+                          [--docker_image_name DOCKER_IMAGE_NAME]
+                          [--gatk_command GATK_COMMAND]
 
 Generates CWL files from the GATK documentation
 
 optional arguments:
   -h, --help            show this help message and exit
-  --version GATKVERSION, -v GATKVERSION
+  --version VERSION, -v VERSION
                         Sets the version of GATK to parse documentation for.
-                        Default is 3.5
-  --out OUTPUTDIR, -o OUTPUTDIR
+                        Default is 3.5-0
+  --out OUTPUT_DIR, -o OUTPUT_DIR
                         Sets the output directory for generated files. Default
                         is ./gatk_cmdline_tools/<VERSION>/
-  --include INCLUDE_FILE
-                        Only generate this file (note, CommandLinkGATK has to
+  --include INCLUDE     Only generate this file (note, CommandLinkGATK has to
                         be generated for v3.x)
   --dev                 Enable network caching and overwriting of the
                         generated files (for development purposes). Requires
                         requests_cache to be installed
-  --docker_container_name DOCKER_CONTAINER_NAME, -c DOCKER_CONTAINER_NAME
-                        Docker container name for generated cwl files. Default
-                        is 'broadinstitute/gatk3:<VERSION>' for version 3.x and
+  --no_docker           Make the generated CWL files not use docker
+                        containers. Default is False.
+  --docker_image_name DOCKER_IMAGE_NAME, -c DOCKER_IMAGE_NAME
+                        Docker image name for generated cwl files. Default is
+                        'broadinstitute/gatk3:<VERSION>' for version 3.x and
                         'broadinstitute/gatk:<VERSION>' for 4.x
-  --gatk_location GATK_LOCATION, -l GATK_LOCATION
-                        Location of the gatk jar file. Default is
-                        '/usr/GenomeAnalysisTK.jar' for gatk 3.x and
-                        '/gatk/gatk.jar' for gatk 4.x
+  --gatk_command GATK_COMMAND, -l GATK_COMMAND
+                        Command to launch GATK. Default is 'java -jar
+                        /usr/GenomeAnalysisTK.jar' for gatk 3.x and 'java -jar
+                        /gatk/gatk.jar' for gatk 4.x
 ```
 
 This has been tested on versions 3.5-3.8 and generates files for version 4 (though some parameters are unknown and default to outputting a string).
@@ -82,14 +88,10 @@ The generated CWL files can also be found in the [releases](https://github.com/w
 
 ## Tests
 
-First install the test requirements
-```
-pip install -r test_requirements.txt
-```
-Then add example data to `cwl-example-data` such that `examples/HaplotypeCaller_inputs.yml` will run, then:
-
+Install the tests requirements, then run the tests. Note: docker must be installed in order to run the tests (the cwl files are tested during the tests):
 ```bash
-py.test -v gatkcwlgenerator/tests/test.py
+pip install -r test_requirements.txt
+pytest gatkcwlgenerator
 ```
 
 You can also run the tests in parallel with `-n` to improve performance

build.sh

@@ -10,7 +10,7 @@ VERSIONS=( 3.5-0 3.6-0 3.7-0 3.8-0 4.beta.6 )
 tarbase="gatk-cwl-generator-${generator_version}-gatk_cmdline_tools"
 
 tmpdir=$(mktemp -d)
-python_bin=$(which python)
+python_bin=$(which python3)
 echo "Using ${python_bin} to generate temporary virtualenv ${tmpdir}/venv"
 set -x
 ${python_bin} -m virtualenv "${tmpdir}/venv"
@@ -33,7 +33,7 @@ for ver in ${VERSIONS[@]}
 do
     echo "Generating CWL for GATK version ${ver}"
     set -x
-    PYTHONPATH=. python gatkcwlgenerator -v ${ver} -o "${builddir}/${ver}" "$@"
+    PYTHONPATH=. python -m gatkcwlgenerator -v ${ver} -o "${builddir}/${ver}" "$@"
     set +x
 done
 

examples/HaplotypeCaller_inputs.yml

@@ -1,18 +1,18 @@
 reference_sequence:
    class: File
    #path: /path/to/fasta/ref/file
-   path: ../cwl-example-data/chr22_cwl_test.fa 
+   path: ../cwl-example-data/chr22_cwl_test.fa
 refIndex:
    class: File
    #path: /path/to/index/file
-   path: ../cwl-example-data/chr22_cwl_test.fa.fai 
+   path: ../cwl-example-data/chr22_cwl_test.fa.fai
 refDict:
    class: File
    #path: /path/to/dict/file
-   path: ../cwl-example-data/chr22_cwl_test.fa.dict 
+   path: ../cwl-example-data/chr22_cwl_test.fa.dict
 input_file: #must be BAM or CRAM
    class: File
-   #path: /path/to/input/file 
+   #path: /path/to/input/file
    path: ../cwl-example-data/chr22_cwl_test.cram
 out: out.gvcf.gz
 intervals: [chr22:10591400-10591600]

gatkcwlgenerator/VERSION

@@ -0,0 +1 @@
+1.4.0

gatkcwlgenerator/__init__.py

@@ -1,3 +1,8 @@
 from .main import *
 import gatkcwlgenerator.json2cwl
-import gatkcwlgenerator.gen_cwl_arg
+import gatkcwlgenerator.gen_cwl_arg
+
+import os.path as path
+
+with open(path.join(path.dirname(__file__), "VERSION"), "r") as _version_file:
+    __version__ = _version_file.read()

gatkcwlgenerator/__main__.py

@@ -1,3 +1,3 @@
-import main
+from .main import cmdline_main
 
-main.main()
+cmdline_main()

gatkcwlgenerator/cwl_ast.py

@@ -0,0 +1,121 @@
+"""
+Classes to make an AST for CWL types
+"""
+import abc
+
+class CWLType:
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def get_cwl_object(self):
+        pass
+
+    def is_array_type(self):
+        return False
+
+    @property
+    def children(self):
+        if hasattr(self, "inner_type"):
+            return [getattr(self, "inner_type")]
+        else:
+            raise AttributeError(repr(self) + " has no children")
+
+    def find_node(self, predicate):
+        """
+        Traverses the AST to find a node that satifies the given predicate.
+        If the no node is found, returns None
+        """
+        if predicate(self):
+            return self
+        else:
+            try:
+                return next(filter(None, (child.find_node(predicate) for child in self.children)))
+            except (AttributeError, StopIteration):
+                return None
+
+
+class CWLBasicType(CWLType):
+    def __init__(self, name):
+        self.name = name
+
+    def get_cwl_object(self):
+        return self.name
+
+
+class CWLArrayType(CWLType):
+    def __init__(self, inner_type):
+        self.inner_type = inner_type
+        self._input_binding = None
+
+    def is_array_type(self):
+        return True
+
+    def add_input_binding(self, inputBinding):
+        self._input_binding = inputBinding
+
+    def get_cwl_object(self):
+        inner_cwl_object = self.inner_type.get_cwl_object()
+
+        if isinstance(inner_cwl_object, str) and self._input_binding is None:
+            return inner_cwl_object + "[]"
+        else:
+            cwl_object = {
+                "type": "array",
+                "items": self.inner_type.get_cwl_object()
+            }
+
+            if self._input_binding is not None:
+                cwl_object["inputBinding"] = self._input_binding
+
+            return cwl_object
+
+
+class CWLUnionType(CWLType):
+    def __init__(self, *items):
+        self.items = items
+
+    def get_cwl_object(self):
+        cwl_object = []
+
+        for item in self.items:
+            if isinstance(item, CWLUnionType):
+                cwl_object.extend(item.get_cwl_object())
+            else:
+                cwl_object.append(item.get_cwl_object())
+
+        return cwl_object
+
+    @property
+    def children(self):
+        return self.items
+
+
+class CWLEnumType(CWLType):
+    def __init__(self, symbols):
+        self.symbols = symbols
+
+    def get_cwl_object(self):
+        return {
+            "type": "enum",
+            "symbols": self.symbols
+        }
+
+class CWLOptionalType(CWLType):
+    def __init__(self, inner_type):
+        self.inner_type = inner_type
+
+    def is_array_type(self):
+        return self.inner_type.is_array_type()
+
+    def get_cwl_object(self):
+        inner_cwl_object = self.inner_type.get_cwl_object()
+
+        if isinstance(inner_cwl_object, str):
+            return inner_cwl_object + "?"
+        elif isinstance(inner_cwl_object, list):
+            return ["null"] + inner_cwl_object
+        else:
+            return [
+                "null",
+                inner_cwl_object
+            ]