Merge branch 'master' of github.com:DataBiosphere/toil into issues/45…

…54-string-to-file-wdl

docker/Dockerfile.py

@@ -102,16 +102,13 @@ def heredoc(s):
     RUN add-apt-repository -y ppa:deadsnakes/ppa
 
     # Find a repo with a Mesos build.
-    # See https://rpm.aventer.biz/README.txt
-    # A working snapshot is https://ipfs.io/ipfs/QmfTy9sXhHsgyWwosCJDfYR4fChTosA8HhoaMgmeJ5LSmS/ for https://rpm.aventer.biz/Ubuntu
-    # And one that works with https://rpm.aventer.biz/Ubuntu/focal (the new URL) is at https://ipfs.io/ipfs/Qmcrmx7T1YkEnyexMXdd7QjoBZxf7DMDrQ5ErUKi9mDRw6/
-    # As archived with:
+    # This one was archived like:
     # mkdir mesos-repo && cd mesos-repo
     # wget --recursive --restrict-file-names=windows -k --convert-links --no-parent --page-requisites https://rpm.aventer.biz/Ubuntu/ https://www.aventer.biz/assets/support_aventer.asc https://rpm.aventer.biz/README.txt
     # ipfs add -r .
-    RUN echo "deb https://rpm.aventer.biz/Ubuntu/focal focal main" \
+    RUN echo "deb https://courtyard.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmeaErHzK4Dajz2mCMd36eUDQp7GX2bSECVRpGfrqdragR/rpm.aventer.biz/Ubuntu/focal focal main" \
         > /etc/apt/sources.list.d/mesos.list \
-        && curl https://www.aventer.biz/assets/support_aventer.asc | apt-key add -
+        && curl https://courtyard.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmeaErHzK4Dajz2mCMd36eUDQp7GX2bSECVRpGfrqdragR/www.aventer.biz/assets/support_aventer.asc | apt-key add -
 
     RUN apt-get -y update --fix-missing && \
         DEBIAN_FRONTEND=noninteractive apt-get -y upgrade && \

docs/running/wdl.rst

@@ -100,11 +100,12 @@ Toil WDL Runner Options
 ``--jobStore``: Specifies where to keep the Toil state information while
 running the workflow. Must be accessible from all machines.
 
-``-o`` or ``--outputDirectory``: Specifies the output folder to save
-workflow output files in. Defaults to a new directory in the current directory.
+``-o`` or ``--outputDirectory``: Specifies the output folder or URI prefix to
+save workflow output files in. Defaults to a new directory in the current
+directory.
 
-``-m`` or ``--outputFile``: Specifies a JSON file to save workflow output
-values to. Defaults to standard output.
+``-m`` or ``--outputFile``: Specifies a JSON file name or URI to save workflow
+output values at. Defaults to standard output.
 
 ``-i`` or ``--input``: Alternative to the positional argument for the
 input JSON file, for compatibility with other WDL runners.

src/toil/jobStores/fileJobStore.py

@@ -342,6 +342,9 @@ def _export_file(self, otherCls, file_id, uri):
         if issubclass(otherCls, FileJobStore):
             srcPath = self._get_file_path_from_id(file_id)
             destPath = self._extract_path_from_url(uri)
+            # Make sure we don't need to worry about directories when exporting
+            # to local files, just like for cloud storage.
+            os.makedirs(os.path.dirname(destPath), exist_ok=True)
             executable = getattr(file_id, 'executable', False)
             if self.moveExports:
                 self._move_and_linkback(srcPath, destPath, executable=executable)

src/toil/wdl/wdltoil.py

@@ -2388,9 +2388,9 @@ def main() -> None:
                               "values as JSON, while 'miniwdl' nests that under an 'outputs' key, and "
                               "includes a 'dir' key where files are written."))
     parser.add_argument("--outputDirectory", "-o", dest="output_directory", type=str, default=None,
-                        help=("Directory in which to save output files. By default a new directory is created in the current directory."))
-    parser.add_argument("--outputFile", "-m", dest="output_file", type=argparse.FileType('w'), default=sys.stdout,
-                        help="File to save output JSON to.")
+                        help=("Directory or URI prefix to save output files at. By default a new directory is created in the current directory."))
+    parser.add_argument("--outputFile", "-m", dest="output_file", type=str, default=None,
+                        help="File or URI to save output JSON to.")
 
     options = parser.parse_args(sys.argv[1:])
 
@@ -2399,14 +2399,10 @@ def main() -> None:
         # TODO: Move cwltoil's generate_default_job_store where we can use it
         options.jobStore = os.path.join(tempfile.mkdtemp(), 'tree')
 
-    # Make sure we have an output directory and we don't need to ever worry
-    # about a None, and MyPy knows it.
+    # Make sure we have an output directory (or URL prefix) and we don't need
+    # to ever worry about a None, and MyPy knows it.
     # If we don't have a directory assigned, make one in the current directory.
     output_directory: str = options.output_directory if options.output_directory else tempfile.mkdtemp(prefix='wdl-out-', dir=os.getcwd())
-    if not os.path.isdir(output_directory):
-        # Make sure it exists
-        os.mkdir(output_directory)
-
 
     with Toil(options) as toil:
         if options.restart:
@@ -2484,7 +2480,7 @@ def devirtualize_output(filename: str) -> str:
                 # TODO: Deal with name collisions
                 dest_name = os.path.join(output_directory, file_basename)
                 # Export the file
-                toil.exportFile(file_id, dest_name)
+                toil.export_file(file_id, dest_name)
                 # And return where we put it
                 return dest_name
             elif filename.startswith('http:') or filename.startswith('https:') or filename.startswith('s3:') or filename.startswith('gs:'):
@@ -2497,7 +2493,7 @@ def devirtualize_output(filename: str) -> str:
                 file_basename = os.path.basename(urlsplit(filename).path)
                 # Do the same as we do for files we actually made.
                 dest_name = os.path.join(output_directory, file_basename)
-                toil.exportFile(imported, dest_name)
+                toil.export_file(imported, dest_name)
                 return dest_name
             else:
                 # Not a fancy file
@@ -2510,8 +2506,24 @@ def devirtualize_output(filename: str) -> str:
         outputs = WDL.values_to_json(output_bindings)
         if options.output_dialect == 'miniwdl':
             outputs = {'dir': output_directory, 'outputs': outputs}
-        options.output_file.write(json.dumps(outputs))
-        options.output_file.write('\n')
+        if options.output_file is None:
+            # Send outputs to standard out
+            print(json.dumps(outputs))
+        else:
+            # Export output to path or URL.
+            # So we need to import and then export.
+            fd, filename = tempfile.mkstemp()
+            with open(fd, 'w') as handle:
+                # Populate the file
+                handle.write(json.dumps(outputs))
+                handle.write('\n')
+            # Import it. Don't link because the temp file will go away.
+            file_id = toil.import_file(filename, symlink=False)
+            # Delete the temp file
+            os.remove(filename)
+            # Export it into place
+            toil.export_file(file_id, options.output_file)
+
 
 
 if __name__ == "__main__":