diff --git a/README.md b/README.md
index 91d9246..681b2dd 100644
--- a/README.md
+++ b/README.md
@@ -78,10 +78,10 @@ You will need to have an account on Nextflow Tower (see [Plans and pricing](http
twkit hello-world-config.yml
```
- Note: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. For example:
+ Note: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with `--cli` followed by the `--insecure` flag. For example:
```
- twkit hello-world-config.yml --insecure
+ twkit hello-world-config.yml --cli --insecure
```
3. Login to your Tower instance and check the Runs page in the appropriate Workspace for the pipeline you just launched!
diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py
new file mode 100644
index 0000000..fd6bb13
--- /dev/null
+++ b/tests/unit/test_helper.py
@@ -0,0 +1,42 @@
+import unittest
+from unittest.mock import mock_open, patch
+from twkit import helper
+
+mocked_yaml = """
+datasets:
+ - name: 'test_dataset1'
+ description: 'My test dataset 1'
+ header: true
+ workspace: 'my_organization/my_workspace'
+ file-path: './examples/yaml/datasets/samples.csv'
+ overwrite: True
+"""
+
+mocked_file = mock_open(read_data=mocked_yaml)
+
+
+class TestYamlParserFunctions(unittest.TestCase):
+ @patch("builtins.open", mocked_file)
+ def test_parse_datasets_yaml(self):
+ result = helper.parse_all_yaml(["test_path.yaml"])
+ expected_block_output = [
+ {
+ "cmd_args": [
+ "--header",
+ "./examples/yaml/datasets/samples.csv",
+ "--name",
+ "test_dataset1",
+ "--workspace",
+ "my_organization/my_workspace",
+ "--description",
+ "My test dataset 1",
+ ],
+ "overwrite": True,
+ }
+ ]
+
+ self.assertIn("datasets", result)
+ self.assertEqual(result["datasets"], expected_block_output)
+
+
+# TODO: add more tests for other functions in helper.py
diff --git a/twkit/cli.py b/twkit/cli.py
index e86d6c3..0ac5753 100644
--- a/twkit/cli.py
+++ b/twkit/cli.py
@@ -5,12 +5,10 @@
"""
import argparse
import logging
-import time
-import yaml
from pathlib import Path
from twkit import tower, helper, overwrite
-from twkit.tower import ResourceCreationError, ResourceExistsError
+from twkit.tower import ResourceExistsError
logger = logging.getLogger(__name__)
@@ -32,12 +30,21 @@ def parse_args():
help="Print the commands that would be executed without running them.",
)
parser.add_argument(
- "yaml", type=Path, help="Config file with Tower resources to create"
+ "yaml",
+ type=Path,
+ nargs="+", # allow multiple YAML paths
+ help="One or more YAML files with Tower resources to create",
)
parser.add_argument(
- "cli_args",
+ "--delete",
+ action="store_true",
+ help="Recursively delete all resources defined in the YAML file(s)",
+ )
+ parser.add_argument(
+ "--cli",
+ dest="cli_args",
nargs=argparse.REMAINDER,
- help="Additional arguments to pass to the Tower CLI",
+ help="Additional arguments to pass to the Tower CLI (e.g. '--insecure')",
)
return parser.parse_args()
@@ -62,7 +69,15 @@ def __init__(self, tw, list_for_add_method):
# Create an instance of Overwrite class
self.overwrite_method = overwrite.Overwrite(self.tw)
- def handle_block(self, block, args):
+ def handle_block(self, block, args, destroy=False):
+ # Check if delete is set to True, and call delete handler
+ if destroy:
+ logging.debug(" The '--delete' flag has been specified.\n")
+ self.overwrite_method.handle_overwrite(
+ block, args["cmd_args"], overwrite=False, destroy=True
+ )
+ return
+
# Handles a block of commands by calling the appropriate function.
block_handler_map = {
"teams": (helper.handle_teams),
@@ -108,24 +123,19 @@ def main():
"datasets",
],
)
- try:
- with open(options.yaml, "r") as f:
- data = yaml.safe_load(f)
-
- # Returns a dict that maps block names to lists of command line arguments.
- cmd_args_dict = helper.parse_all_yaml(options.yaml, list(data.keys()))
-
- for block, args_list in cmd_args_dict.items():
- for args in args_list:
- try:
- # Run the 'tw' methods for each block
- block_manager.handle_block(block, args)
- time.sleep(3)
- except ResourceExistsError as e:
- logging.error(e)
- continue
- except ResourceCreationError as e:
- logging.error(e)
+
+ # Parse the YAML file(s) by blocks
+ # and get a dictionary of command line arguments
+ cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete)
+
+ for block, args_list in cmd_args_dict.items():
+ for args in args_list:
+ try:
+ # Run the 'tw' methods for each block
+ block_manager.handle_block(block, args, destroy=options.delete)
+ except ResourceExistsError as e:
+ logging.error(e)
+ continue
if __name__ == "__main__":
diff --git a/twkit/helper.py b/twkit/helper.py
index 42addd6..aaf76c1 100644
--- a/twkit/helper.py
+++ b/twkit/helper.py
@@ -7,13 +7,13 @@
from twkit import utils
-def parse_yaml_block(file_path, block_name):
- # Load the YAML file.
- with open(file_path, "r") as f:
- data = yaml.safe_load(f)
+def parse_yaml_block(yaml_data, block_name):
+ # Get the name of the specified block/resource.
+ block = yaml_data.get(block_name)
- # Get the specified block.
- block = data.get(block_name)
+ # If block is not found in the YAML, return an empty list.
+ if not block:
+ return block_name, []
# Initialize an empty list to hold the lists of command line arguments.
cmd_args_list = []
@@ -38,7 +38,20 @@ def parse_yaml_block(file_path, block_name):
return block_name, cmd_args_list
-def parse_all_yaml(file_path, block_names):
+def parse_all_yaml(file_paths, destroy=False):
+ # If multiple yamls, merge them into one dictionary
+ merged_data = {}
+
+ for file_path in file_paths:
+ with open(file_path, "r") as f:
+ data = yaml.safe_load(f)
+ # Update merged_data with the content of this file
+ merged_data.update(data)
+
+ # Get the names of all the blocks/resources to create in the merged data.
+ block_names = list(merged_data.keys())
+
+ # Define the order in which the resources should be created.
resource_order = [
"organizations",
"teams",
@@ -52,15 +65,19 @@ def parse_all_yaml(file_path, block_names):
"pipelines",
"launch",
]
+
+ # Reverse the order of resources if destroy is True
+ if destroy:
+ resource_order = resource_order[:-1][::-1]
+
# Initialize an empty dictionary to hold all the command arguments.
cmd_args_dict = {}
# Iterate over each block name in the desired order.
for block_name in resource_order:
- # Check if the block name is present in the provided block_names list
if block_name in block_names:
- # Parse the block and add its command arguments to the dictionary.
- block_name, cmd_args_list = parse_yaml_block(file_path, block_name)
+ # Parse the block and add its command line arguments to the dictionary.
+ block_name, cmd_args_list = parse_yaml_block(merged_data, block_name)
cmd_args_dict[block_name] = cmd_args_list
# Return the dictionary of command arguments.
diff --git a/twkit/overwrite.py b/twkit/overwrite.py
index c2cab78..34853c6 100644
--- a/twkit/overwrite.py
+++ b/twkit/overwrite.py
@@ -63,7 +63,7 @@ def __init__(self, tw):
},
}
- def handle_overwrite(self, block, args, overwrite=False):
+ def handle_overwrite(self, block, args, overwrite=False, destroy=False):
"""
Handles overwrite functionality for Tower resources and
calling the 'tw delete' method with the correct args.
@@ -89,13 +89,16 @@ def handle_overwrite(self, block, args, overwrite=False):
self.block_operations["participants"]["name_key"] = "email"
if self.check_resource_exists(operation["name_key"], tw_args):
- # if resource exists, delete
+ # if resource exists and overwrite is true, delete
if overwrite:
logging.debug(
f" The attempted {block} resource already exists."
" Overwriting.\n"
)
- self._delete_resource(block, operation, tw_args)
+ self.delete_resource(block, operation, tw_args)
+ elif destroy:
+ logging.debug(f" Deleting the {block} resource.")
+ self.delete_resource(block, operation, tw_args)
else: # return an error if resource exists, overwrite=False
raise ResourceExistsError(
f" The {block} resource already exists and"
@@ -217,7 +220,7 @@ def check_resource_exists(self, name_key, tw_args):
"""
return utils.check_if_exists(self.cached_jsondata, name_key, tw_args["name"])
- def _delete_resource(self, block, operation, tw_args):
+ def delete_resource(self, block, operation, tw_args):
"""
Delete a resource in Tower by calling the delete() method and
arguments defined in the operation dictionary.