From fed7bee866e0d2bac8ae6a3be8c8c0c4dd9ee49a Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 5 Oct 2023 20:53:02 -0400 Subject: [PATCH 1/5] feat: add support for parsing multiple yaml inputs Re: Issue #3 --- twkit/cli.py | 40 +++++++++++++++++++--------------------- twkit/helper.py | 33 +++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/twkit/cli.py b/twkit/cli.py index e86d6c3..99660a5 100644 --- a/twkit/cli.py +++ b/twkit/cli.py @@ -6,11 +6,10 @@ import argparse import logging import time -import yaml from pathlib import Path from twkit import tower, helper, overwrite -from twkit.tower import ResourceCreationError, ResourceExistsError +from twkit.tower import ResourceExistsError logger = logging.getLogger(__name__) @@ -32,7 +31,10 @@ def parse_args(): help="Print the commands that would be executed without running them.", ) parser.add_argument( - "yaml", type=Path, help="Config file with Tower resources to create" + "yaml", + type=Path, + nargs="+", # allow multiple YAML paths + help="One or more YAML files with Tower resources to create", ) parser.add_argument( "cli_args", @@ -108,24 +110,20 @@ def main(): "datasets", ], ) - try: - with open(options.yaml, "r") as f: - data = yaml.safe_load(f) - - # Returns a dict that maps block names to lists of command line arguments. - cmd_args_dict = helper.parse_all_yaml(options.yaml, list(data.keys())) - - for block, args_list in cmd_args_dict.items(): - for args in args_list: - try: - # Run the 'tw' methods for each block - block_manager.handle_block(block, args) - time.sleep(3) - except ResourceExistsError as e: - logging.error(e) - continue - except ResourceCreationError as e: - logging.error(e) + + # Parse the YAML file(s) by blocks + # and get a dictionary of command line arguments + cmd_args_dict = helper.parse_all_yaml(options.yaml) + + for block, args_list in cmd_args_dict.items(): + for args in args_list: + try: + # Run the 'tw' methods for each block + block_manager.handle_block(block, args) + time.sleep(3) + except ResourceExistsError as e: + logging.error(e) + continue if __name__ == "__main__": diff --git a/twkit/helper.py b/twkit/helper.py index 42addd6..0d3be6f 100644 --- a/twkit/helper.py +++ b/twkit/helper.py @@ -7,13 +7,14 @@ from twkit import utils -def parse_yaml_block(file_path, block_name): - # Load the YAML file. - with open(file_path, "r") as f: - data = yaml.safe_load(f) +def parse_yaml_block(yaml_data, block_name): - # Get the specified block. - block = data.get(block_name) + # Get the name of the specified block/resource. + block = yaml_data.get(block_name) + + # If block is not found in the YAML, return an empty list. + if not block: + return block_name, [] # Initialize an empty list to hold the lists of command line arguments. cmd_args_list = [] @@ -38,7 +39,20 @@ def parse_yaml_block(file_path, block_name): return block_name, cmd_args_list -def parse_all_yaml(file_path, block_names): +def parse_all_yaml(file_paths): + # If multiple yamls, merge them into one dictionary + merged_data = {} + + for file_path in file_paths: + with open(file_path, "r") as f: + data = yaml.safe_load(f) + # Update merged_data with the content of this file + merged_data.update(data) + + # Get the names of all the blocks/resources to create in the merged data. + block_names = list(merged_data.keys()) + + # Define the order in which the resources should be created. resource_order = [ "organizations", "teams", @@ -57,10 +71,9 @@ def parse_all_yaml(file_path, block_names): # Iterate over each block name in the desired order. for block_name in resource_order: - # Check if the block name is present in the provided block_names list if block_name in block_names: - # Parse the block and add its command arguments to the dictionary. - block_name, cmd_args_list = parse_yaml_block(file_path, block_name) + # Parse the block and add its command line arguments to the dictionary. + block_name, cmd_args_list = parse_yaml_block(merged_data, block_name) cmd_args_dict[block_name] = cmd_args_list # Return the dictionary of command arguments. From 5deec1fa0bdf2cc51ea7519026c9d01a9be57042 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 5 Oct 2023 22:15:39 -0400 Subject: [PATCH 2/5] test: add initial tests for helper parsing functions --- tests/unit/test_helper.py | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/unit/test_helper.py diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py new file mode 100644 index 0000000..fd6bb13 --- /dev/null +++ b/tests/unit/test_helper.py @@ -0,0 +1,42 @@ +import unittest +from unittest.mock import mock_open, patch +from twkit import helper + +mocked_yaml = """ +datasets: + - name: 'test_dataset1' + description: 'My test dataset 1' + header: true + workspace: 'my_organization/my_workspace' + file-path: './examples/yaml/datasets/samples.csv' + overwrite: True +""" + +mocked_file = mock_open(read_data=mocked_yaml) + + +class TestYamlParserFunctions(unittest.TestCase): + @patch("builtins.open", mocked_file) + def test_parse_datasets_yaml(self): + result = helper.parse_all_yaml(["test_path.yaml"]) + expected_block_output = [ + { + "cmd_args": [ + "--header", + "./examples/yaml/datasets/samples.csv", + "--name", + "test_dataset1", + "--workspace", + "my_organization/my_workspace", + "--description", + "My test dataset 1", + ], + "overwrite": True, + } + ] + + self.assertIn("datasets", result) + self.assertEqual(result["datasets"], expected_block_output) + + +# TODO: add more tests for other functions in helper.py From 2e781600f8daae07766b8cea18f60de6da26766e Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 5 Oct 2023 23:18:36 -0400 Subject: [PATCH 3/5] feat: add support for --delete to recursively destroy Re: Issue 47 --- twkit/cli.py | 22 ++++++++++++++++++---- twkit/helper.py | 8 ++++++-- twkit/overwrite.py | 11 +++++++---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/twkit/cli.py b/twkit/cli.py index 99660a5..1f410de 100644 --- a/twkit/cli.py +++ b/twkit/cli.py @@ -37,7 +37,13 @@ def parse_args(): help="One or more YAML files with Tower resources to create", ) parser.add_argument( - "cli_args", + "--delete", + action="store_true", + help="Recursively delete all resources defined in the YAML file(s)", + ) + parser.add_argument( + "--cli", + dest="cli_args", nargs=argparse.REMAINDER, help="Additional arguments to pass to the Tower CLI", ) @@ -64,7 +70,15 @@ def __init__(self, tw, list_for_add_method): # Create an instance of Overwrite class self.overwrite_method = overwrite.Overwrite(self.tw) - def handle_block(self, block, args): + def handle_block(self, block, args, destroy=False): + # Check if delete is set to True, and call delete handler + if destroy: + logging.debug(" The '--delete' flag has been specified.\n") + self.overwrite_method.handle_overwrite( + block, args["cmd_args"], overwrite=False, destroy=True + ) + return + # Handles a block of commands by calling the appropriate function. block_handler_map = { "teams": (helper.handle_teams), @@ -113,13 +127,13 @@ def main(): # Parse the YAML file(s) by blocks # and get a dictionary of command line arguments - cmd_args_dict = helper.parse_all_yaml(options.yaml) + cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) for block, args_list in cmd_args_dict.items(): for args in args_list: try: # Run the 'tw' methods for each block - block_manager.handle_block(block, args) + block_manager.handle_block(block, args, destroy=options.delete) time.sleep(3) except ResourceExistsError as e: logging.error(e) diff --git a/twkit/helper.py b/twkit/helper.py index 0d3be6f..aaf76c1 100644 --- a/twkit/helper.py +++ b/twkit/helper.py @@ -8,7 +8,6 @@ def parse_yaml_block(yaml_data, block_name): - # Get the name of the specified block/resource. block = yaml_data.get(block_name) @@ -39,7 +38,7 @@ def parse_yaml_block(yaml_data, block_name): return block_name, cmd_args_list -def parse_all_yaml(file_paths): +def parse_all_yaml(file_paths, destroy=False): # If multiple yamls, merge them into one dictionary merged_data = {} @@ -66,6 +65,11 @@ def parse_all_yaml(file_paths): "pipelines", "launch", ] + + # Reverse the order of resources if destroy is True + if destroy: + resource_order = resource_order[:-1][::-1] + # Initialize an empty dictionary to hold all the command arguments. cmd_args_dict = {} diff --git a/twkit/overwrite.py b/twkit/overwrite.py index c2cab78..34853c6 100644 --- a/twkit/overwrite.py +++ b/twkit/overwrite.py @@ -63,7 +63,7 @@ def __init__(self, tw): }, } - def handle_overwrite(self, block, args, overwrite=False): + def handle_overwrite(self, block, args, overwrite=False, destroy=False): """ Handles overwrite functionality for Tower resources and calling the 'tw delete' method with the correct args. @@ -89,13 +89,16 @@ def handle_overwrite(self, block, args, overwrite=False): self.block_operations["participants"]["name_key"] = "email" if self.check_resource_exists(operation["name_key"], tw_args): - # if resource exists, delete + # if resource exists and overwrite is true, delete if overwrite: logging.debug( f" The attempted {block} resource already exists." " Overwriting.\n" ) - self._delete_resource(block, operation, tw_args) + self.delete_resource(block, operation, tw_args) + elif destroy: + logging.debug(f" Deleting the {block} resource.") + self.delete_resource(block, operation, tw_args) else: # return an error if resource exists, overwrite=False raise ResourceExistsError( f" The {block} resource already exists and" @@ -217,7 +220,7 @@ def check_resource_exists(self, name_key, tw_args): """ return utils.check_if_exists(self.cached_jsondata, name_key, tw_args["name"]) - def _delete_resource(self, block, operation, tw_args): + def delete_resource(self, block, operation, tw_args): """ Delete a resource in Tower by calling the delete() method and arguments defined in the operation dictionary. From fb99d4517ca9447ba59b0965c5af44218908492e Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 5 Oct 2023 23:19:40 -0400 Subject: [PATCH 4/5] fix: remove sleep in cli interface --- twkit/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/twkit/cli.py b/twkit/cli.py index 1f410de..09f86ae 100644 --- a/twkit/cli.py +++ b/twkit/cli.py @@ -5,7 +5,6 @@ """ import argparse import logging -import time from pathlib import Path from twkit import tower, helper, overwrite @@ -134,7 +133,6 @@ def main(): try: # Run the 'tw' methods for each block block_manager.handle_block(block, args, destroy=options.delete) - time.sleep(3) except ResourceExistsError as e: logging.error(e) continue From fb6409eb1597dfbce223e2850820cf30123d4ec7 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 5 Oct 2023 23:21:50 -0400 Subject: [PATCH 5/5] docs: add clarification on using additional tw cli args --- README.md | 4 ++-- twkit/cli.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 91d9246..681b2dd 100644 --- a/README.md +++ b/README.md @@ -78,10 +78,10 @@ You will need to have an account on Nextflow Tower (see [Plans and pricing](http twkit hello-world-config.yml ``` - Note: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. For example: + Note: The Tower CLI expects to connect to a Tower instance that is secured by a TLS certificate. If your Tower instance does not present a certificate, you will need to qualify and run your `tw` commands with `--cli` followed by the `--insecure` flag. For example: ``` - twkit hello-world-config.yml --insecure + twkit hello-world-config.yml --cli --insecure ``` 3. Login to your Tower instance and check the Runs page in the appropriate Workspace for the pipeline you just launched! diff --git a/twkit/cli.py b/twkit/cli.py index 09f86ae..0ac5753 100644 --- a/twkit/cli.py +++ b/twkit/cli.py @@ -44,7 +44,7 @@ def parse_args(): "--cli", dest="cli_args", nargs=argparse.REMAINDER, - help="Additional arguments to pass to the Tower CLI", + help="Additional arguments to pass to the Tower CLI (e.g. '--insecure')", ) return parser.parse_args()