moved *.txt config files to global ~/wic

api.py

@@ -68,26 +68,28 @@ async def compile_wf(request: Request) -> Json:
     print('----------Run Workflow!---------')
     root_yaml_tree = await request.json()
 
-    tools_cwl = plugins.get_tools_cwl(Path('cwl_dirs.txt'))
-    yml_paths = plugins.get_yml_paths(Path('yml_dirs.txt'))
+    yaml_path = "workflow.json"
+    args = get_args(yaml_path)
+
+    tools_cwl = plugins.get_tools_cwl(args.homedir)
+    yml_paths = plugins.get_yml_paths(args.homedir)
 
     # Perform initialization via mutating global variables (This is not ideal)
-    compiler.inference_rules = dict(utils.read_lines_pairs(Path('inference_rules.txt')))
-    inference.renaming_conventions = utils.read_lines_pairs(Path('renaming_conventions.txt'))
+    wicdir = Path(args.homedir) / 'wic'
+    compiler.inference_rules = dict(utils.read_lines_pairs(wicdir / 'inference_rules.txt'))
+    inference.renaming_conventions = utils.read_lines_pairs(wicdir / 'renaming_conventions.txt')
 
     # Generate schemas for validation and vscode IntelliSense code completion
     yaml_stems = utils.flatten([list(p) for p in yml_paths.values()])
     validator = wic_schema.get_validator(tools_cwl, yaml_stems, write_to_disk=True)
 
-    yaml_path = "workflow.json"
-
     # Load the high-level yaml root workflow file.
     Path('autogenerated/').mkdir(parents=True, exist_ok=True)
     wic_obj = {'wic': root_yaml_tree.get('wic', {})}
     plugin_ns = wic_obj['wic'].get('namespace', 'global')
     step_id = StepId(yaml_path, plugin_ns)
     y_t = YamlTree(step_id, root_yaml_tree)
-    yaml_tree_raw = ast.read_ast_from_disk(y_t, yml_paths, tools_cwl, validator)
+    yaml_tree_raw = ast.read_ast_from_disk(args.homedir, y_t, yml_paths, tools_cwl, validator)
     yaml_tree = ast.merge_yml_trees(yaml_tree_raw, {}, tools_cwl)
 
     rootgraph = graphviz.Digraph(name=yaml_path)
@@ -96,7 +98,7 @@ async def compile_wf(request: Request) -> Json:
         graphdata = GraphData(yaml_path)
         subgraph = GraphReps(subgraph_gv, subgraph_nx, graphdata)
         try:
-            compiler_info = compiler.compile_workflow(yaml_tree, get_args(yaml_path), [], [subgraph], {}, {}, {}, {},
+            compiler_info = compiler.compile_workflow(yaml_tree, args, [], [subgraph], {}, {}, {}, {},
                                                       tools_cwl, True, relative_run_path=True, testing=False)
         except Exception as e:
             # Certain constraints are conditionally dependent on values and are

cwl_adapters/file_watchers/cwl_watcher.cwl

@@ -66,38 +66,27 @@ inputs:
       position: 5
       prefix: --config
 
-  cwl_dirs_file:
-    label: Configuration file which lists the directories which contains the CWL CommandLineTools
+  root_workflow_yml_path:
+    label: The full absolute path to the root workflow yml file.
     doc: |
-      Configuration file which lists the directories which contains the CWL CommandLineTools
+      The full absolute path to the root workflow yml file.
     type: string
     format:
     - edam:format_2330 # 'Textual format'
     inputBinding:
       position: 6
-      prefix: --cwl_dirs_file
+      prefix: --root_workflow_yml_path
 
-  yml_dirs_file:
-    label: Configuration file which lists the directories which contains the YAML Workflows
+  homedir:
+    label: The full absolute path to the uers home directory.
     doc: |
-      Configuration file which lists the directories which contains the YAML Workflows
+      The full absolute path to the root users home directory.
     type: string
     format:
     - edam:format_2330 # 'Textual format'
     inputBinding:
       position: 7
-      prefix: --yml_dirs_file
-
-  root_workflow_yml_path:
-    label: The full absolute path to the root workflow yml file.
-    doc: |
-      The full absolute path to the root workflow yml file.
-    type: string
-    format:
-    - edam:format_2330 # 'Textual format'
-    inputBinding:
-      position: 8
-      prefix: --root_workflow_yml_path
+      prefix: --homedir
 
 outputs:
   output_log_path:

ipycytoscape.ipynb

@@ -27,8 +27,8 @@
     "with patch.object(sys, 'argv', testargs):\n",
     "    args = wic.cli.parser.parse_args()\n",
     "\n",
-    "tools_cwl = wic.plugins.get_tools_cwl(Path(args.cwl_dirs_file))\n",
-    "yml_paths = wic.plugins.get_yml_paths(Path(args.yml_dirs_file))"
+    "tools_cwl = wic.plugins.get_tools_cwl(args.homedir)\n",
+    "yml_paths = wic.plugins.get_yml_paths(args.homedir)"
    ]
   },
   {
@@ -61,7 +61,7 @@
     "plugin_ns = wic_tag['wic'].get('namespace', 'global')\n",
     "step_id = StepId(yml_path_str, plugin_ns)\n",
     "y_t = YamlTree(step_id, root_yaml_tree)\n",
-    "yaml_tree_raw = wic.ast.read_ast_from_disk(y_t, yml_paths, tools_cwl, validator)\n",
+    "yaml_tree_raw = wic.ast.read_ast_from_disk(args.homedir, y_t, yml_paths, tools_cwl, validator)\n",
     "with open(f'autogenerated/{Path(yml_path).stem}_tree_raw.yml', mode='w', encoding='utf-8') as f:\n",
     "    f.write(yaml.dump(yaml_tree_raw.yml))\n",
     "yaml_tree = wic.ast.merge_yml_trees(yaml_tree_raw, {}, tools_cwl)\n",

src/wic/ast.py

@@ -19,13 +19,15 @@
 # That way, we should be able to serialize back to disk without duplication.
 
 
-def read_ast_from_disk(yaml_tree_tuple: YamlTree,
+def read_ast_from_disk(homedir: str,
+                       yaml_tree_tuple: YamlTree,
                        yml_paths: Dict[str, Dict[str, Path]],
                        tools: Tools,
                        validator: Draft202012Validator) -> YamlTree:
     """Reads the yml workflow definition files from disk (recursively) and inlines them into an AST
 
     Args:
+        homedir (str): The users home directory
         yaml_tree_tuple (YamlTree): A tuple of a filepath and its Yaml file contents.
         yml_paths (Dict[str, Dict[str, Path]]): The yml workflow definitions found using get_yml_paths()
         tools (Tools): The CWL CommandLineTool definitions found using get_tools_cwl()
@@ -47,7 +49,7 @@ def read_ast_from_disk(yaml_tree_tuple: YamlTree,
         for back_name, back in wic['wic']['backends'].items():
             plugin_ns = wic['wic'].get('namespace', 'global')
             stepid = StepId(back_name, plugin_ns)
-            backends_tree = read_ast_from_disk(YamlTree(stepid, back), yml_paths, tools, validator)
+            backends_tree = read_ast_from_disk(homedir, YamlTree(stepid, back), yml_paths, tools, validator)
             backends_trees.append(backends_tree)
         yaml_tree['wic']['backends'] = dict(backends_trees)
         return YamlTree(step_id, yaml_tree)
@@ -74,7 +76,9 @@ def read_ast_from_disk(yaml_tree_tuple: YamlTree,
 
             paths_ns_i = yml_paths.get(plugin_ns, {})
             if paths_ns_i == {}:
-                raise Exception(f'Error! namespace {plugin_ns} not found in yaml paths. Check yml_dirs.txt')
+                wicdir = Path(homedir) / 'wic'
+                raise Exception(
+                    f'Error! namespace {plugin_ns} not found in yaml paths. Check {wicdir / "yml_dirs.txt"}')
             if stem not in paths_ns_i:
                 msg = f'Error! {stem} not found in namespace {plugin_ns} when attempting to read {step_id.stem}.yml'
                 if stem == 'in':
@@ -100,7 +104,7 @@ def read_ast_from_disk(yaml_tree_tuple: YamlTree,
                 sys.exit(1)
 
             y_t = YamlTree(StepId(step_key, plugin_ns), sub_yaml_tree_raw)
-            (step_id_, sub_yml_tree) = read_ast_from_disk(y_t, yml_paths, tools, validator)
+            (step_id_, sub_yml_tree) = read_ast_from_disk(homedir, y_t, yml_paths, tools, validator)
 
             step_i_dict = {} if steps[i][step_key] is None else steps[i][step_key]
             # Do not merge these two dicts; use subtree and parentargs so we can

src/wic/cli.py

@@ -1,16 +1,15 @@
 import argparse
 import sys
+from pathlib import Path
 
 parser = argparse.ArgumentParser(prog='main', description='Convert a high-level yaml workflow file to CWL.')
 parser.add_argument('--yaml', type=str, required=('--generate_schemas_only' not in sys.argv),
                     help='Yaml workflow file')
 
 parser.add_argument('--generate_schemas_only', default=False, action="store_true",
-                    help='Generate schemas for the files in --cwl_dirs_file and --yml_dirs_file.')
-parser.add_argument('--cwl_dirs_file', type=str, required=False, default='cwl_dirs.txt',
-                    help='Configuration file which lists the directories which contains the CWL CommandLineTools')
-parser.add_argument('--yml_dirs_file', type=str, required=False, default='yml_dirs.txt',
-                    help='Configuration file which lists the directories which contains the YAML Workflows')
+                    help='Generate schemas for the files in ~/wic/cwl_dirs.txt and ~/wic/yml_dirs.txt')
+parser.add_argument('--homedir', type=str, required=False, default=str(Path().home()),
+                    help='The users home directory. This is necessary because CWL clears environment variables (e.g. HOME)')
 # Change default to True for now. See comment in compiler.py
 parser.add_argument('--cwl_output_intermediate_files', type=bool, required=False, default=True,
                     help='Enable output files which are used between steps (for debugging).')

src/wic/compiler.py

@@ -1,7 +1,7 @@
 import argparse
 import copy
 import json
-import subprocess as sub
+import os
 from pathlib import Path
 from typing import Dict, List
 
@@ -360,15 +360,21 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
         # NOTE: run: path issues were causing test_cwl_embedding_independence()
         # to fail, so I simply ignore the run tag in that test.
         run_path = tool_i.run_path
+        # NOTE: run_path is always relative; relative_run_path should probably
+        # be called use_subdirs, because it simply determines if subworkflows
+        # should be written to subdirectories or if everything should be
+        # written to autogenerated/
         if relative_run_path:
             if step_key in subkeys:
                 run_path = step_name_i + '/' + run_path
             else:
+                run_path = os.path.relpath(run_path, 'autogenerated/')
                 run_path = ('../' * len(namespaces)) + run_path
-                run_path = '../' + run_path  # Relative to autogenerated/
         else:
             if step_key in subkeys:
                 run_path = '___'.join(namespaces + [step_name_i, run_path])
+            else:
+                run_path = os.path.relpath(run_path, 'autogenerated/')
 
         if steps[i][step_key]:
             if not 'run' in steps[i][step_key]:
@@ -382,7 +388,7 @@ def compile_workflow_once(yaml_tree_ast: YamlTree,
 
         if 'cwl_watcher' == step_key:
             in_dict_in = steps[i][step_key]['in']  # NOTE: Mutates in_dict_in
-            utils.write_absolute_config_files(args, in_dict_in, namespaces, step_name_i, explicit_edge_calls_copy)
+            utils.write_absolute_yaml_tags(args, in_dict_in, namespaces, step_name_i, explicit_edge_calls_copy)
 
         args_provided = []
         if steps[i][step_key] and 'in' in steps[i][step_key]:

src/wic/cwl_watcher.py

@@ -59,13 +59,14 @@ def absolute_paths(config: Json, cachedir_path: Path) -> Json:
     return new_json
 
 
-def rerun_cwltool(_directory_realtime: Path, cachedir_path: Path, cwl_tool: str,
+def rerun_cwltool(homedir: str, _directory_realtime: Path, cachedir_path: Path, cwl_tool: str,
                   args_vals: Json, tools_cwl: Tools, yml_paths: Dict[str, Dict[str, Path]],
                   validator: Draft202012Validator, root_workflow_yml_path: Path) -> None:
     """This will speculatively execute cwltool for real-time analysis purposes.\n
     It will NOT check for return code 0. See docs/userguide.md
 
     Args:
+        homedir (str): The users home directory
         _directory_realtime (Path): The working directory of the main workflow.\n
         Currently unused to avoid this workflow from overwriting files from the main\n
         workflow (which by design will likely be running concurrently with this code).
@@ -93,7 +94,7 @@ def rerun_cwltool(_directory_realtime: Path, cachedir_path: Path, cwl_tool: str,
             plugin_ns = 'global'  # wic['wic'].get('namespace', 'global')
             step_id = StepId(yaml_path, plugin_ns)
             y_t = YamlTree(step_id, root_yaml_tree)
-            yaml_tree_raw = ast.read_ast_from_disk(y_t, yml_paths, tools_cwl, validator)
+            yaml_tree_raw = ast.read_ast_from_disk(homedir, y_t, yml_paths, tools_cwl, validator)
             yaml_tree = ast.merge_yml_trees(yaml_tree_raw, {}, tools_cwl)
             yml = yaml_tree.yml
         else:
@@ -230,10 +231,8 @@ def cli_watcher() -> argparse.Namespace:
                         help='--cwl_tool will be speculatively executed at most max_times')
     parser.add_argument('--config', type=str, required=True,
                         help='This should be a json-encoded representation of the config: YAML subtag of --cwl_tool')
-    parser.add_argument('--cwl_dirs_file', type=str, required=True,
-                        help='Configuration file which lists the directories which contains the CWL CommandLineTools')
-    parser.add_argument('--yml_dirs_file', type=str, required=True,
-                        help='Configuration file which lists the directories which contains the YAML Workflows')
+    parser.add_argument('--homedir', type=str, required=False,  # default=str(Path().home()), # NOTE: no default!
+                        help='The users home directory. This is necessary because CWL clears environment variables (e.g. HOME)')
     parser.add_argument('--root_workflow_yml_path', type=str, required=True,
                         help='The full absolute path to the root workflow yml file.')
     return parser.parse_args()
@@ -248,8 +247,6 @@ def main() -> None:
     file_pattern = args.file_pattern
     cwl_tool = args.cwl_tool
     max_times = int(args.max_times)
-    cwl_dirs_file = Path(args.cwl_dirs_file)
-    yml_dirs_file = Path(args.yml_dirs_file)
     root_workflow_yml_path = Path(args.root_workflow_yml_path)
 
     # Create an empty 'logfile' so that cwl_watcher.cwl succeeds.
@@ -259,12 +256,13 @@ def main() -> None:
 
     args_vals = json.loads(args.config)
 
-    tools_cwl = get_tools_cwl(cwl_dirs_file)
-    yml_paths = get_yml_paths(yml_dirs_file)
+    tools_cwl = get_tools_cwl(args.homedir)
+    yml_paths = get_yml_paths(args.homedir)
 
     # Perform initialization via mutating global variables (This is not ideal)
-    compiler.inference_rules = dict(utils.read_lines_pairs(Path('inference_rules.txt')))
-    inference.renaming_conventions = utils.read_lines_pairs(Path('renaming_conventions.txt'))
+    wicdir = Path(args.homedir) / 'wic'
+    compiler.inference_rules = dict(utils.read_lines_pairs(wicdir / 'inference_rules.txt'))
+    inference.renaming_conventions = utils.read_lines_pairs(wicdir / 'renaming_conventions.txt')
 
     # Generate schemas for validation
     yaml_stems = utils.flatten([list(p) for p in yml_paths.values()])
@@ -296,7 +294,7 @@ def main() -> None:
             for file in changed_files:
                 if file_pattern[1:] in file:
                     print(file)
-                    rerun_cwltool(Path(file).parent, cachedir_path, cwl_tool,
+                    rerun_cwltool(args.homedir, Path(file).parent, cachedir_path, cwl_tool,
                                   args_vals, tools_cwl, yml_paths, validator,
                                   root_workflow_yml_path)
             prev_files = {**prev_files, **changed_files}

src/wic/main.py

@@ -16,15 +16,17 @@ def main() -> None:
     """See docs/userguide.md"""
     args = cli.parser.parse_args()
 
-    tools_cwl = plugins.get_tools_cwl(Path(args.cwl_dirs_file), args.validate_plugins,
+    tools_cwl = plugins.get_tools_cwl(args.homedir,
+                                      args.validate_plugins,
                                       not args.no_skip_dollar_schemas)
     # This takes ~1 second but it is not really necessary.
     # utils_graphs.make_plugins_dag(tools_cwl, args.graph_dark_theme)
-    yml_paths = plugins.get_yml_paths(Path(args.yml_dirs_file))
+    yml_paths = plugins.get_yml_paths(args.homedir)
 
     # Perform initialization via mutating global variables (This is not ideal)
-    compiler.inference_rules = dict(utils.read_lines_pairs(Path('inference_rules.txt')))
-    inference.renaming_conventions = utils.read_lines_pairs(Path('renaming_conventions.txt'))
+    wicdir = Path(args.homedir) / 'wic'
+    compiler.inference_rules = dict(utils.read_lines_pairs(wicdir / 'inference_rules.txt'))
+    inference.renaming_conventions = utils.read_lines_pairs(wicdir / 'renaming_conventions.txt')
 
     # Generate schemas for validation and vscode IntelliSense code completion
     yaml_stems = utils.flatten([list(p) for p in yml_paths.values()])
@@ -43,7 +45,7 @@ def main() -> None:
                             for yml_path_str, yml_path in yml_paths_dict.items()]
 
         for yml_path_str, yml_path in yml_paths_tuples:
-            schema = wic_schema.compile_workflow_generate_schema(yml_path_str, yml_path,
+            schema = wic_schema.compile_workflow_generate_schema(args.homedir, yml_path_str, yml_path,
                                                                  tools_cwl, yml_paths, validator)
             # overwrite placeholders in schema_store. See comment in get_validator()
             schema_store[schema['$id']] = schema
@@ -67,7 +69,7 @@ def main() -> None:
     plugin_ns = wic['wic'].get('namespace', 'global')
     step_id = StepId(yaml_path, plugin_ns)
     y_t = YamlTree(step_id, root_yaml_tree)
-    yaml_tree_raw = ast.read_ast_from_disk(y_t, yml_paths, tools_cwl, validator)
+    yaml_tree_raw = ast.read_ast_from_disk(args.homedir, y_t, yml_paths, tools_cwl, validator)
     # Write the combined workflow (with all subworkflows as children) to disk.
     with open(f'autogenerated/{Path(yaml_path).stem}_tree_raw.yml', mode='w', encoding='utf-8') as f:
         f.write(yaml.dump(yaml_tree_raw.yml))