diff --git a/README.md b/README.md index 91d9246..681b2dd 100644 --- a/README.md +++ b/README.md @@ -78,10 +78,10 @@ You will need to have an account on Nextflow Tower (see [Plans and pricing](http twkit hello-world-config.yml ``` - Note: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. For example: + Note: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with `--cli` followed by the `--insecure` flag. For example: ``` - twkit hello-world-config.yml --insecure + twkit hello-world-config.yml --cli --insecure ``` 3. Login to your Tower instance and check the Runs page in the appropriate Workspace for the pipeline you just launched! diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py new file mode 100644 index 0000000..fd6bb13 --- /dev/null +++ b/tests/unit/test_helper.py @@ -0,0 +1,42 @@ +import unittest +from unittest.mock import mock_open, patch +from twkit import helper + +mocked_yaml = """ +datasets: + - name: 'test_dataset1' + description: 'My test dataset 1' + header: true + workspace: 'my_organization/my_workspace' + file-path: './examples/yaml/datasets/samples.csv' + overwrite: True +""" + +mocked_file = mock_open(read_data=mocked_yaml) + + +class TestYamlParserFunctions(unittest.TestCase): + @patch("builtins.open", mocked_file) + def test_parse_datasets_yaml(self): + result = helper.parse_all_yaml(["test_path.yaml"]) + expected_block_output = [ + { + "cmd_args": [ + "--header", + "./examples/yaml/datasets/samples.csv", + "--name", + "test_dataset1", + "--workspace", + "my_organization/my_workspace", + "--description", + "My test dataset 1", + ], + "overwrite": True, + } + ] + + self.assertIn("datasets", result) + self.assertEqual(result["datasets"], expected_block_output) + + +# TODO: add more tests for other functions in helper.py diff --git a/twkit/cli.py b/twkit/cli.py index e86d6c3..0ac5753 100644 --- a/twkit/cli.py +++ b/twkit/cli.py @@ -5,12 +5,10 @@ """ import argparse import logging -import time -import yaml from pathlib import Path from twkit import tower, helper, overwrite -from twkit.tower import ResourceCreationError, ResourceExistsError +from twkit.tower import ResourceExistsError logger = logging.getLogger(__name__) @@ -32,12 +30,21 @@ def parse_args(): help="Print the commands that would be executed without running them.", ) parser.add_argument( - "yaml", type=Path, help="Config file with Tower resources to create" + "yaml", + type=Path, + nargs="+", # allow multiple YAML paths + help="One or more YAML files with Tower resources to create", ) parser.add_argument( - "cli_args", + "--delete", + action="store_true", + help="Recursively delete all resources defined in the YAML file(s)", + ) + parser.add_argument( + "--cli", + dest="cli_args", nargs=argparse.REMAINDER, - help="Additional arguments to pass to the Tower CLI", + help="Additional arguments to pass to the Tower CLI (e.g. '--insecure')", ) return parser.parse_args() @@ -62,7 +69,15 @@ def __init__(self, tw, list_for_add_method): # Create an instance of Overwrite class self.overwrite_method = overwrite.Overwrite(self.tw) - def handle_block(self, block, args): + def handle_block(self, block, args, destroy=False): + # Check if delete is set to True, and call delete handler + if destroy: + logging.debug(" The '--delete' flag has been specified.\n") + self.overwrite_method.handle_overwrite( + block, args["cmd_args"], overwrite=False, destroy=True + ) + return + # Handles a block of commands by calling the appropriate function. block_handler_map = { "teams": (helper.handle_teams), @@ -108,24 +123,19 @@ def main(): "datasets", ], ) - try: - with open(options.yaml, "r") as f: - data = yaml.safe_load(f) - - # Returns a dict that maps block names to lists of command line arguments. - cmd_args_dict = helper.parse_all_yaml(options.yaml, list(data.keys())) - - for block, args_list in cmd_args_dict.items(): - for args in args_list: - try: - # Run the 'tw' methods for each block - block_manager.handle_block(block, args) - time.sleep(3) - except ResourceExistsError as e: - logging.error(e) - continue - except ResourceCreationError as e: - logging.error(e) + + # Parse the YAML file(s) by blocks + # and get a dictionary of command line arguments + cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) + + for block, args_list in cmd_args_dict.items(): + for args in args_list: + try: + # Run the 'tw' methods for each block + block_manager.handle_block(block, args, destroy=options.delete) + except ResourceExistsError as e: + logging.error(e) + continue if __name__ == "__main__": diff --git a/twkit/helper.py b/twkit/helper.py index 42addd6..aaf76c1 100644 --- a/twkit/helper.py +++ b/twkit/helper.py @@ -7,13 +7,13 @@ from twkit import utils -def parse_yaml_block(file_path, block_name): - # Load the YAML file. - with open(file_path, "r") as f: - data = yaml.safe_load(f) +def parse_yaml_block(yaml_data, block_name): + # Get the name of the specified block/resource. + block = yaml_data.get(block_name) - # Get the specified block. - block = data.get(block_name) + # If block is not found in the YAML, return an empty list. + if not block: + return block_name, [] # Initialize an empty list to hold the lists of command line arguments. cmd_args_list = [] @@ -38,7 +38,20 @@ def parse_yaml_block(file_path, block_name): return block_name, cmd_args_list -def parse_all_yaml(file_path, block_names): +def parse_all_yaml(file_paths, destroy=False): + # If multiple yamls, merge them into one dictionary + merged_data = {} + + for file_path in file_paths: + with open(file_path, "r") as f: + data = yaml.safe_load(f) + # Update merged_data with the content of this file + merged_data.update(data) + + # Get the names of all the blocks/resources to create in the merged data. + block_names = list(merged_data.keys()) + + # Define the order in which the resources should be created. resource_order = [ "organizations", "teams", @@ -52,15 +65,19 @@ def parse_all_yaml(file_path, block_names): "pipelines", "launch", ] + + # Reverse the order of resources if destroy is True + if destroy: + resource_order = resource_order[:-1][::-1] + # Initialize an empty dictionary to hold all the command arguments. cmd_args_dict = {} # Iterate over each block name in the desired order. for block_name in resource_order: - # Check if the block name is present in the provided block_names list if block_name in block_names: - # Parse the block and add its command arguments to the dictionary. - block_name, cmd_args_list = parse_yaml_block(file_path, block_name) + # Parse the block and add its command line arguments to the dictionary. + block_name, cmd_args_list = parse_yaml_block(merged_data, block_name) cmd_args_dict[block_name] = cmd_args_list # Return the dictionary of command arguments. diff --git a/twkit/overwrite.py b/twkit/overwrite.py index c2cab78..34853c6 100644 --- a/twkit/overwrite.py +++ b/twkit/overwrite.py @@ -63,7 +63,7 @@ def __init__(self, tw): }, } - def handle_overwrite(self, block, args, overwrite=False): + def handle_overwrite(self, block, args, overwrite=False, destroy=False): """ Handles overwrite functionality for Tower resources and calling the 'tw delete' method with the correct args. @@ -89,13 +89,16 @@ def handle_overwrite(self, block, args, overwrite=False): self.block_operations["participants"]["name_key"] = "email" if self.check_resource_exists(operation["name_key"], tw_args): - # if resource exists, delete + # if resource exists and overwrite is true, delete if overwrite: logging.debug( f" The attempted {block} resource already exists." " Overwriting.\n" ) - self._delete_resource(block, operation, tw_args) + self.delete_resource(block, operation, tw_args) + elif destroy: + logging.debug(f" Deleting the {block} resource.") + self.delete_resource(block, operation, tw_args) else: # return an error if resource exists, overwrite=False raise ResourceExistsError( f" The {block} resource already exists and" @@ -217,7 +220,7 @@ def check_resource_exists(self, name_key, tw_args): """ return utils.check_if_exists(self.cached_jsondata, name_key, tw_args["name"]) - def _delete_resource(self, block, operation, tw_args): + def delete_resource(self, block, operation, tw_args): """ Delete a resource in Tower by calling the delete() method and arguments defined in the operation dictionary.