Recursively delete from yaml

README.md

@@ -78,10 +78,10 @@ You will need to have an account on Nextflow Tower (see [Plans and pricing](http
    twkit hello-world-config.yml
    ```
 
-   <b>Note</b>: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. For example:
+   <b>Note</b>: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with `--cli` followed by the `--insecure` flag. For example:
 
    ```
-   twkit hello-world-config.yml --insecure
+   twkit hello-world-config.yml --cli --insecure
    ```
 
 3. Login to your Tower instance and check the Runs page in the appropriate Workspace for the pipeline you just launched!

tests/unit/test_helper.py

@@ -0,0 +1,42 @@
+import unittest
+from unittest.mock import mock_open, patch
+from twkit import helper
+
+mocked_yaml = """
+datasets:
+  - name: 'test_dataset1'
+    description: 'My test dataset 1'
+    header: true
+    workspace: 'my_organization/my_workspace'
+    file-path: './examples/yaml/datasets/samples.csv'
+    overwrite: True
+"""
+
+mocked_file = mock_open(read_data=mocked_yaml)
+
+
+class TestYamlParserFunctions(unittest.TestCase):
+    @patch("builtins.open", mocked_file)
+    def test_parse_datasets_yaml(self):
+        result = helper.parse_all_yaml(["test_path.yaml"])
+        expected_block_output = [
+            {
+                "cmd_args": [
+                    "--header",
+                    "./examples/yaml/datasets/samples.csv",
+                    "--name",
+                    "test_dataset1",
+                    "--workspace",
+                    "my_organization/my_workspace",
+                    "--description",
+                    "My test dataset 1",
+                ],
+                "overwrite": True,
+            }
+        ]
+
+        self.assertIn("datasets", result)
+        self.assertEqual(result["datasets"], expected_block_output)
+
+
+# TODO: add more tests for other functions in helper.py

twkit/cli.py

@@ -5,12 +5,10 @@
 """
 import argparse
 import logging
-import time
-import yaml
 
 from pathlib import Path
 from twkit import tower, helper, overwrite
-from twkit.tower import ResourceCreationError, ResourceExistsError
+from twkit.tower import ResourceExistsError
 
 
 logger = logging.getLogger(__name__)
@@ -32,12 +30,21 @@ def parse_args():
         help="Print the commands that would be executed without running them.",
     )
     parser.add_argument(
-        "yaml", type=Path, help="Config file with Tower resources to create"
+        "yaml",
+        type=Path,
+        nargs="+",  # allow multiple YAML paths
+        help="One or more YAML files with Tower resources to create",
     )
     parser.add_argument(
-        "cli_args",
+        "--delete",
+        action="store_true",
+        help="Recursively delete all resources defined in the YAML file(s)",
+    )
+    parser.add_argument(
+        "--cli",
+        dest="cli_args",
         nargs=argparse.REMAINDER,
-        help="Additional arguments to pass to the Tower CLI",
+        help="Additional arguments to pass to the Tower CLI (e.g. '--insecure')",
     )
     return parser.parse_args()
 
@@ -62,7 +69,15 @@ def __init__(self, tw, list_for_add_method):
         # Create an instance of Overwrite class
         self.overwrite_method = overwrite.Overwrite(self.tw)
 
-    def handle_block(self, block, args):
+    def handle_block(self, block, args, destroy=False):
+        # Check if delete is set to True, and call delete handler
+        if destroy:
+            logging.debug(" The '--delete' flag has been specified.\n")
+            self.overwrite_method.handle_overwrite(
+                block, args["cmd_args"], overwrite=False, destroy=True
+            )
+            return
+
         # Handles a block of commands by calling the appropriate function.
         block_handler_map = {
             "teams": (helper.handle_teams),
@@ -108,24 +123,19 @@ def main():
             "datasets",
         ],
     )
-    try:
-        with open(options.yaml, "r") as f:
-            data = yaml.safe_load(f)
-
-        # Returns a dict that maps block names to lists of command line arguments.
-        cmd_args_dict = helper.parse_all_yaml(options.yaml, list(data.keys()))
-
-        for block, args_list in cmd_args_dict.items():
-            for args in args_list:
-                try:
-                    # Run the 'tw' methods for each block
-                    block_manager.handle_block(block, args)
-                    time.sleep(3)
-                except ResourceExistsError as e:
-                    logging.error(e)
-                    continue
-    except ResourceCreationError as e:
-        logging.error(e)
+
+    # Parse the YAML file(s) by blocks
+    # and get a dictionary of command line arguments
+    cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete)
+
+    for block, args_list in cmd_args_dict.items():
+        for args in args_list:
+            try:
+                # Run the 'tw' methods for each block
+                block_manager.handle_block(block, args, destroy=options.delete)
+            except ResourceExistsError as e:
+                logging.error(e)
+                continue
 
 
 if __name__ == "__main__":

twkit/helper.py

@@ -7,13 +7,13 @@
 from twkit import utils
 
 
-def parse_yaml_block(file_path, block_name):
-    # Load the YAML file.
-    with open(file_path, "r") as f:
-        data = yaml.safe_load(f)
+def parse_yaml_block(yaml_data, block_name):
+    # Get the name of the specified block/resource.
+    block = yaml_data.get(block_name)
 
-    # Get the specified block.
-    block = data.get(block_name)
+    # If block is not found in the YAML, return an empty list.
+    if not block:
+        return block_name, []
 
     # Initialize an empty list to hold the lists of command line arguments.
     cmd_args_list = []
@@ -38,7 +38,20 @@ def parse_yaml_block(file_path, block_name):
     return block_name, cmd_args_list
 
 
-def parse_all_yaml(file_path, block_names):
+def parse_all_yaml(file_paths, destroy=False):
+    # If multiple yamls, merge them into one dictionary
+    merged_data = {}
+
+    for file_path in file_paths:
+        with open(file_path, "r") as f:
+            data = yaml.safe_load(f)
+            # Update merged_data with the content of this file
+            merged_data.update(data)
+
+    # Get the names of all the blocks/resources to create in the merged data.
+    block_names = list(merged_data.keys())
+
+    # Define the order in which the resources should be created.
     resource_order = [
         "organizations",
         "teams",
@@ -52,15 +65,19 @@ def parse_all_yaml(file_path, block_names):
         "pipelines",
         "launch",
     ]
+
+    # Reverse the order of resources if destroy is True
+    if destroy:
+        resource_order = resource_order[:-1][::-1]
+
     # Initialize an empty dictionary to hold all the command arguments.
     cmd_args_dict = {}
 
     # Iterate over each block name in the desired order.
     for block_name in resource_order:
-        # Check if the block name is present in the provided block_names list
         if block_name in block_names:
-            # Parse the block and add its command arguments to the dictionary.
-            block_name, cmd_args_list = parse_yaml_block(file_path, block_name)
+            # Parse the block and add its command line arguments to the dictionary.
+            block_name, cmd_args_list = parse_yaml_block(merged_data, block_name)
             cmd_args_dict[block_name] = cmd_args_list
 
     # Return the dictionary of command arguments.

twkit/overwrite.py

@@ -63,7 +63,7 @@ def __init__(self, tw):
             },
         }
 
-    def handle_overwrite(self, block, args, overwrite=False):
+    def handle_overwrite(self, block, args, overwrite=False, destroy=False):
         """
         Handles overwrite functionality for Tower resources and
         calling the 'tw delete' method with the correct args.
@@ -89,13 +89,16 @@ def handle_overwrite(self, block, args, overwrite=False):
                     self.block_operations["participants"]["name_key"] = "email"
 
             if self.check_resource_exists(operation["name_key"], tw_args):
-                # if resource exists, delete
+                # if resource exists and overwrite is true, delete
                 if overwrite:
                     logging.debug(
                         f" The attempted {block} resource already exists."
                         " Overwriting.\n"
                     )
-                    self._delete_resource(block, operation, tw_args)
+                    self.delete_resource(block, operation, tw_args)
+                elif destroy:
+                    logging.debug(f" Deleting the {block} resource.")
+                    self.delete_resource(block, operation, tw_args)
                 else:  # return an error if resource exists, overwrite=False
                     raise ResourceExistsError(
                         f" The {block} resource already exists and"
@@ -217,7 +220,7 @@ def check_resource_exists(self, name_key, tw_args):
         """
         return utils.check_if_exists(self.cached_jsondata, name_key, tw_args["name"])
 
-    def _delete_resource(self, block, operation, tw_args):
+    def delete_resource(self, block, operation, tw_args):
         """
         Delete a resource in Tower by calling the delete() method and
         arguments defined in the operation dictionary.