Merge branch 'dev' into json_output_to_stdout

adamrtalbot · adamrtalbot · commit 0271c38efe18 · 2024-08-29T09:19:03.000+01:00
diff --git a/.github/workflows/teardown.yml b/.github/workflows/teardown.yml
@@ -36,7 +36,7 @@ jobs:
           environment-file: environment.yml
           python-version: '3.12'
           mamba-version: '*'
-          channels: conda-forge,bioconda,defaults
+          channels: conda-forge,bioconda
           activate-environment: seqerakit
           use-mamba: true
 
diff --git a/README.md b/README.md
@@ -27,7 +27,6 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt
 You can install `seqerakit` and its dependencies via Conda. Ensure that you have the correct channels configured:
 
 ```console
-conda config --add channels defaults
 conda config --add channels bioconda
 conda config --add channels conda-forge
 conda config --set channel_priority strict
@@ -170,6 +169,43 @@ seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/pat
 
 <b>Note</b>: Use of `--verbose` option for the `tw` CLI is currently not supported by `seqerakit`. Supplying `--cli="--verbose"` will raise an error.
 
+## Specify targets
+When using a YAML file as input that defines multiple resources, you can use the `--targets` flag to specify which resources to create. This flag takes a comma-separated list of resource names.
+
+For example, given a YAML file that defines the following resources:
+
+```yaml
+workspaces:
+  - name: 'showcase'
+    organization: 'seqerakit_automation'
+...
+compute-envs:
+  - name: 'compute-env'
+    type: 'aws-batch forge'
+    workspace: 'seqerakit/test'
+...
+pipelines:
+  - name: "hello-world-test-seqerakit"
+    url: "https://github.com/nextflow-io/hello"
+    workspace: 'seqerakit/test'
+    compute-env: "compute-env"
+...
+```
+
+You can target the creation of `pipelines` only by running:
+
+```bash
+seqerakit test.yml --targets pipelines
+```
+This will process only the pipelines block from the YAML file and ignore other blocks such as `workspaces` and `compute-envs`.
+
+### Multiple Targets
+You can also specify multiple resources to create by separating them with commas. For example, to create both workspaces and pipelines, run:
+
+```bash
+seqerakit test.yml --targets workspaces,pipelines
+```
+
 ## YAML Configuration Options
 
 There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not exposed as `tw` CLI options.
diff --git a/environment.yaml b/environment.yaml
@@ -0,0 +1,8 @@
+name: seqerakitdev
+channels:
+  - bioconda
+  - conda-forge
+dependencies:
+  - conda-forge::python=3.10.9
+  - conda-forge::pyyaml=6.0
+  - bioconda::tower-cli=0.9.2
diff --git a/seqerakit/cli.py b/seqerakit/cli.py
@@ -83,6 +83,13 @@ def parse_args(args=None):
         help="Additional Seqera Platform CLI specific options to be passed,"
         " enclosed in double quotes (e.g. '--cli=\"--insecure\"').",
     )
+    yaml_processing.add_argument(
+        "--targets",
+        dest="targets",
+        type=str,
+        help="Specify the resources to be targeted for creation in a YAML file through "
+        "a comma-separated list (e.g. '--targets=teams,participants').",
+    )
     return parser.parse_args(args)
 
 
@@ -146,12 +153,20 @@ def handle_block(self, block, args, destroy=False, dryrun=False):
 
 def main(args=None):
     options = parse_args(args if args is not None else sys.argv[1:])
-    logging.basicConfig(level=options.log_level)
+    logging.basicConfig(level=getattr(logging, options.log_level.upper()))
+
+    # Parse CLI arguments into a list
+    cli_args_list = options.cli_args.split() if options.cli_args else []
+
+    sp = seqeraplatform.SeqeraPlatform(
+        cli_args=cli_args_list, dryrun=options.dryrun, json=options.json
+    )
 
     # If the info flag is set, run 'tw info'
     if options.info:
-        sp = seqeraplatform.SeqeraPlatform()
-        print(sp.info())
+        result = sp.info()
+        if not options.dryrun:
+            print(result)
         return
 
     if not options.yaml:
@@ -164,13 +179,6 @@ def main(args=None):
         else:
             options.yaml = [sys.stdin]
 
-    # Parse CLI arguments into a list
-    cli_args_list = options.cli_args.split() if options.cli_args else []
-
-    sp = seqeraplatform.SeqeraPlatform(
-        cli_args=cli_args_list, dryrun=options.dryrun, json=options.json
-    )
-
     block_manager = BlockParser(
         sp,
         [
@@ -188,18 +196,15 @@ def main(args=None):
     # Parse the YAML file(s) by blocks
     # and get a dictionary of command line arguments
     try:
-        cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete)
+        cmd_args_dict = helper.parse_all_yaml(
+            options.yaml, destroy=options.delete, targets=options.targets
+        )
         for block, args_list in cmd_args_dict.items():
             for args in args_list:
-                try:
-                    # Run the 'tw' methods for each block
-                    block_manager.handle_block(
-                        block, args, destroy=options.delete, dryrun=options.dryrun
-                    )
-                except (ResourceExistsError, ResourceCreationError) as e:
-                    logging.error(e)
-                    sys.exit(1)
-    except ValueError as e:
+                block_manager.handle_block(
+                    block, args, destroy=options.delete, dryrun=options.dryrun
+                )
+    except (ResourceExistsError, ResourceCreationError, ValueError) as e:
         logging.error(e)
         sys.exit(1)
 
diff --git a/seqerakit/helper.py b/seqerakit/helper.py
@@ -55,7 +55,7 @@ def parse_yaml_block(yaml_data, block_name):
     return block_name, cmd_args_list
 
 
-def parse_all_yaml(file_paths, destroy=False):
+def parse_all_yaml(file_paths, destroy=False, targets=None):
     # If multiple yamls, merge them into one dictionary
     merged_data = {}
 
@@ -108,6 +108,11 @@ def parse_all_yaml(file_paths, destroy=False):
 
     block_names = list(merged_data.keys())
 
+    # Filter blocks based on targets if provided
+    if targets:
+        target_blocks = set(targets.split(","))
+        block_names = [block for block in block_names if block in target_blocks]
+
     # Define the order in which the resources should be created.
     resource_order = [
         "organizations",
diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py
@@ -117,13 +117,13 @@ def handle_overwrite(self, block, args, overwrite=False, destroy=False):
             if self.check_resource_exists(operation["name_key"], sp_args):
                 # if resource exists and overwrite is true, delete
                 if overwrite:
-                    logging.debug(
+                    logging.info(
                         f" The attempted {block} resource already exists."
                         " Overwriting.\n"
                     )
                     self.delete_resource(block, operation, sp_args)
                 elif destroy:
-                    logging.debug(f" Deleting the {block} resource.")
+                    logging.info(f" Deleting the {block} resource.")
                     self.delete_resource(block, operation, sp_args)
                 else:  # return an error if resource exists, overwrite=False
                     raise ResourceExistsError(
@@ -147,7 +147,8 @@ def _get_team_args(self, args):
 
         if not jsondata:
             json_method = getattr(self.sp, "-o json")
-            json_out = json_method("teams", "list", "-o", args["organization"])
+            with self.sp.suppress_output():
+                json_out = json_method("teams", "list", "-o", args["organization"])
             self.block_jsondata["teams"] = json_out
         else:
             json_out = jsondata
@@ -244,27 +245,30 @@ def _get_json_data(self, block, args, keys_to_get):
             # Fetch the data if it does not exist
             if block == "teams":
                 sp_args = self._get_values_from_cmd_args(args[0], keys_to_get)
-                self.cached_jsondata = json_method(
-                    block, "list", "-o", sp_args["organization"]
-                )
+                with self.sp.suppress_output():
+                    self.cached_jsondata = json_method(
+                        block, "list", "-o", sp_args["organization"]
+                    )
             elif block in Overwrite.generic_deletion or block in {
                 "participants",
                 "labels",
             }:
                 sp_args = self._get_values_from_cmd_args(args, keys_to_get)
-                self.cached_jsondata = json_method(
-                    block, "list", "-w", sp_args["workspace"]
-                )
-            elif block == "members" or block == "workspaces":  # TODO
+                with self.sp.suppress_output():
+                    self.cached_jsondata = json_method(
+                        block, "list", "-w", sp_args["workspace"]
+                    )
+            elif block == "members" or block == "workspaces":
                 sp_args = self._get_values_from_cmd_args(args, keys_to_get)
-                self.cached_jsondata = json_method(
-                    block, "list", "-o", sp_args["organization"]
-                )
+                with self.sp.suppress_output():
+                    self.cached_jsondata = json_method(
+                        block, "list", "-o", sp_args["organization"]
+                    )
             else:
                 sp_args = self._get_values_from_cmd_args(args, keys_to_get)
-                self.cached_jsondata = json_method(block, "list")
+                with self.sp.suppress_output():
+                    self.cached_jsondata = json_method(block, "list")
 
-        # Store this data in the block_jsondata dict for later use
         self.block_jsondata[block] = self.cached_jsondata
         return self.cached_jsondata, sp_args
 
diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py
@@ -12,15 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from contextlib import contextmanager
 import os
 import shlex
 import logging
 import subprocess
 import re
 import json
 
-logging.basicConfig(level=logging.DEBUG)
-
 
 class SeqeraPlatform:
     """
@@ -44,14 +43,16 @@ def __call__(self, *args, **kwargs):
             return self.tw_instance._tw_run(command, **kwargs)
 
     # Constructs a new SeqeraPlatform instance
-    def __init__(self, cli_args=None, dryrun=False, json=False):
+    def __init__(self, cli_args=None, dryrun=False, print_stdout=True, json=False):
         if cli_args and "--verbose" in cli_args:
             raise ValueError(
                 "--verbose is not supported as a CLI argument to seqerakit."
             )
         self.cli_args = cli_args or []
         self.dryrun = dryrun
+        self.print_stdout = print_stdout
         self.json = json
+        self._suppress_output = False
 
     def _construct_command(self, cmd, *args, **kwargs):
         command = ["tw"] + self.cli_args
@@ -68,8 +69,19 @@ def _construct_command(self, cmd, *args, **kwargs):
         if "params_file" in kwargs:
             command.append(f"--params-file={kwargs['params_file']}")
 
+        # Check for empty string arguments and handle them
+        self._check_empty_args(command)
+
         return self._check_env_vars(command)
 
+    def _check_empty_args(self, command):
+        for current_arg, next_arg in zip(command, command[1:]):
+            if isinstance(next_arg, str) and next_arg.strip() == "":
+                raise ValueError(
+                    f"Empty string argument found for parameter '{current_arg}'. "
+                    "Please provide a valid value or remove the argument."
+                )
+
     # Checks environment variables to see that they are set accordingly
     def _check_env_vars(self, command):
         full_cmd_parts = []
@@ -89,16 +101,23 @@ def _check_env_vars(self, command):
         return " ".join(full_cmd_parts)
 
     # Executes a 'tw' command in a subprocess and returns the output.
-    def _execute_command(self, full_cmd):
-        logging.debug(f" Running command: {full_cmd}")
+    def _execute_command(self, full_cmd, to_json=False, print_stdout=True):
+        logging.info(f" Running command: {full_cmd}")
         process = subprocess.Popen(
             full_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True
         )
         stdout, _ = process.communicate()
         stdout = stdout.decode("utf-8").strip()
 
+        should_print = (
+            print_stdout if print_stdout is not None else self.print_stdout
+        ) and not self._suppress_output
+
+        if should_print:
+            logging.info(f" Command output: {stdout}")
+
         if "ERROR: " in stdout or process.returncode != 0:
-            self._handle_command_errors(str(stdout))
+            self._handle_command_errors(stdout)
 
         if self.json:
             out = json.loads(stdout)
@@ -109,14 +128,7 @@ def _execute_command(self, full_cmd):
 
         return out
 
-    def _execute_info_command(self):
-        # Directly execute 'tw info' command
-        command = "tw info"
-        return self._execute_command(command)
-
     def _handle_command_errors(self, stdout):
-        logging.error(stdout)
-
         # Check for specific tw cli error patterns and raise custom exceptions
         if re.search(
             r"ERROR: .*already (exists|a participant)", stdout, flags=re.IGNORECASE
@@ -131,19 +143,33 @@ def _handle_command_errors(self, stdout):
             )
 
     def _tw_run(self, cmd, *args, **kwargs):
+        print_stdout = kwargs.pop("print_stdout", None)
         full_cmd = self._construct_command(cmd, *args, **kwargs)
         if not full_cmd or self.dryrun:
-            logging.debug(f"DRYRUN: Running command {full_cmd}")
+            logging.info(f"DRYRUN: Running command {full_cmd}")
             return None
-        result = self._execute_command(full_cmd)
-        return result
+        return self._execute_command(full_cmd, kwargs.get("to_json"), print_stdout)
+
+    @contextmanager
+    def suppress_output(self):
+        original_suppress = self._suppress_output
+        self._suppress_output = True
+        try:
+            yield
+        finally:
+            self._suppress_output = original_suppress
 
     # Allow any 'tw' subcommand to be called as a method.
     def __getattr__(self, cmd):
         if cmd == "info":
-            return self._execute_info_command
-        else:
-            return self.TwCommand(self, cmd.replace("_", "-"))
+            return lambda *args, **kwargs: self._tw_run(
+                ["info"], *args, **kwargs, print_stdout=False
+            )
+        if cmd == "-o json":
+            return lambda *args, **kwargs: self._tw_run(
+                ["-o", "json"] + list(args), **kwargs
+            )
+        return self.TwCommand(self, cmd.replace("_", "-"))
 
 
 class ResourceExistsError(Exception):
diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 from setuptools import find_packages, setup
 
-VERSION = "0.4.7"
+VERSION = "0.4.9"
 
 with open("README.md") as f:
     readme = f.read()
diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py
diff --git a/tests/unit/test_seqeraplatform.py b/tests/unit/test_seqeraplatform.py
