From 3727b174e9ad6e89cb72c42df448b246f0ea734d Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Tue, 5 Mar 2024 16:33:01 -0500 Subject: [PATCH 001/101] Add command to calculate coverage for each task and output --- src/wdlci/cli/__main__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index 2e4f37f..dc144ce 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -5,6 +5,9 @@ from wdlci.cli.submit import submit_handler from wdlci.cli.monitor import monitor_handler from wdlci.cli.cleanup import cleanup_handler +from wdlci.cli.detect_task_and_output_coverage import ( + detect_task_and_output_coverage_handler, +) from wdlci.utils.ordered_group import OrderedGroup remove_option = click.option( @@ -100,3 +103,10 @@ def cleanup(**kwargs): """Clean Workbench namespace of transient artifacts""" cleanup_handler(kwargs) + + +@main.command +def detect_task_and_output_coverage(**kwargs): + """Outputs how much coverage each task and output has, and which tasks/outputs have no associated tests""" + + detect_task_and_output_coverage_handler(kwargs) From ceeb069000cfdfaf5b63b9c51fcf1c8698cf365f Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 6 Mar 2024 13:24:37 -0500 Subject: [PATCH 002/101] Add initial version of script/CLI command to output task and output coverage --- .../cli/detect_task_and_output_coverage.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/wdlci/cli/detect_task_and_output_coverage.py diff --git a/src/wdlci/cli/detect_task_and_output_coverage.py b/src/wdlci/cli/detect_task_and_output_coverage.py new file mode 100644 index 0000000..1081070 --- /dev/null +++ b/src/wdlci/cli/detect_task_and_output_coverage.py @@ -0,0 +1,90 @@ +import sys +import WDL +from wdlci.config import Config +from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig +from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException + + +def detect_task_and_output_coverage_handler(kwargs): + try: + Config.load(kwargs) + config = Config.instance() + + tasks_without_tests = [] + outputs_without_tests = [] + + task_coverage = {} + output_coverage = {} + + for workflow, workflow_config in config.file.workflows.items(): + doc = WDL.load(workflow) + + for task in doc.tasks: + task_name = task.name + + # task_test_configs is a list of WorkflowTaskTestConfig objects, + # which contain inputs and output_tests as attributes + task_test_configs = workflow_config.tasks[task_name].tests + # TODO: this is actually telling us which WORKFLOWS don't have any tests, + # once I test with a larger wf, this may need to be adjusted + if not task_test_configs: + tasks_without_tests.append(task_name) + + task_coverage_count = 0 + + for task_test_config in task_test_configs: + # Create a dictionary of output tests + output_tests_dict = task_test_config.output_tests + # Also created a nested dictionary of output coverage, which will + # store a task name as the parent key + output_coverage[task_name] = {} + # For each key and value in a given output test dict + for output_key, output_value in output_tests_dict.items(): + # Ensure output value is a dict with the key test_tasks present + if ( + isinstance(output_value, dict) + and "test_tasks" in output_value + ): + # Create a list of test tasks for each output + output_tests_list = output_value["test_tasks"] + # Assign the number of test tasks to the key associated with a + # given output in a nested structure, where each task has a + # dictionary of outputs and their task counts + output_coverage[task_name][output_key] = len( + output_tests_list + ) + task_coverage_count += len(output_tests_list) + + if len(output_tests_list) == 0: + outputs_without_tests.append(output_key) + task_coverage[task_name] = task_coverage_count + + print("Task Coverage:") + for task_name, tests_count in task_coverage.items(): + print(f"\t{task_name}: {tests_count}\n") + + print("Output coverage:") + for task_name, output_tests_dict in output_coverage.items(): + print(f"\tTask name: {task_name}") + for output_name, tests_count in output_tests_dict.items(): + print( + f"\t\tOutput name: {output_name}\n\t\t\tnumber of tests: {tests_count}" + ) + + if tasks_without_tests and outputs_without_tests: + print( + f"\nWARNING: The following tasks have no tests: {', '.join(tasks_without_tests)}" + + f" and the following outputs have no tests: {', '.join(outputs_without_tests)}.\n" + ) + elif tasks_without_tests: + print( + f"\nWARNING: The following tasks have no tests: {', '.join(tasks_without_tests)}.\n" + ) + elif outputs_without_tests: + print( + f"\nWARNING: The following outputs have no tests: {', '.join(outputs_without_tests)}.\n" + ) + + except WdlTestCliExitException as e: + print(f"exiting with code {e.exit_code}, message: {e.message}") + sys.exit(e.exit_code) From 0b940efc9823dcc4df3762d43f4713444a15538d Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 6 Mar 2024 13:51:07 -0500 Subject: [PATCH 003/101] Make minor formatting adjustments --- src/wdlci/cli/detect_task_and_output_coverage.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/wdlci/cli/detect_task_and_output_coverage.py b/src/wdlci/cli/detect_task_and_output_coverage.py index 1081070..1078f45 100644 --- a/src/wdlci/cli/detect_task_and_output_coverage.py +++ b/src/wdlci/cli/detect_task_and_output_coverage.py @@ -25,8 +25,6 @@ def detect_task_and_output_coverage_handler(kwargs): # task_test_configs is a list of WorkflowTaskTestConfig objects, # which contain inputs and output_tests as attributes task_test_configs = workflow_config.tasks[task_name].tests - # TODO: this is actually telling us which WORKFLOWS don't have any tests, - # once I test with a larger wf, this may need to be adjusted if not task_test_configs: tasks_without_tests.append(task_name) @@ -73,16 +71,16 @@ def detect_task_and_output_coverage_handler(kwargs): if tasks_without_tests and outputs_without_tests: print( - f"\nWARNING: The following tasks have no tests: {', '.join(tasks_without_tests)}" - + f" and the following outputs have no tests: {', '.join(outputs_without_tests)}.\n" + f"\nWARNING: The following tasks have no tests:\n\n{', '.join(tasks_without_tests)}\n\n" + + f"Additionally, the following outputs have no tests:\n\n{', '.join(outputs_without_tests)}\n\n" ) elif tasks_without_tests: print( - f"\nWARNING: The following tasks have no tests: {', '.join(tasks_without_tests)}.\n" + f"\nWARNING: The following tasks have no tests:\n\n{', '.join(tasks_without_tests)}\n\n" ) elif outputs_without_tests: print( - f"\nWARNING: The following outputs have no tests: {', '.join(outputs_without_tests)}.\n" + f"\nWARNING: The following outputs have no tests:\n\n{', '.join(outputs_without_tests)}\n\n" ) except WdlTestCliExitException as e: From 4abf55a9d0cb2cfc190f99a1cf5ba0eb3e7cb3ce Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 6 Mar 2024 16:20:04 -0500 Subject: [PATCH 004/101] Make more minor formatting adjustments --- src/wdlci/cli/detect_task_and_output_coverage.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/wdlci/cli/detect_task_and_output_coverage.py b/src/wdlci/cli/detect_task_and_output_coverage.py index 1078f45..177695d 100644 --- a/src/wdlci/cli/detect_task_and_output_coverage.py +++ b/src/wdlci/cli/detect_task_and_output_coverage.py @@ -21,31 +21,26 @@ def detect_task_and_output_coverage_handler(kwargs): for task in doc.tasks: task_name = task.name - # task_test_configs is a list of WorkflowTaskTestConfig objects, # which contain inputs and output_tests as attributes task_test_configs = workflow_config.tasks[task_name].tests + if not task_test_configs: tasks_without_tests.append(task_name) task_coverage_count = 0 for task_test_config in task_test_configs: - # Create a dictionary of output tests output_tests_dict = task_test_config.output_tests - # Also created a nested dictionary of output coverage, which will - # store a task name as the parent key output_coverage[task_name] = {} - # For each key and value in a given output test dict + for output_key, output_value in output_tests_dict.items(): - # Ensure output value is a dict with the key test_tasks present if ( isinstance(output_value, dict) and "test_tasks" in output_value ): - # Create a list of test tasks for each output output_tests_list = output_value["test_tasks"] - # Assign the number of test tasks to the key associated with a + # Assign the length of test tasks to the key associated with a # given output in a nested structure, where each task has a # dictionary of outputs and their task counts output_coverage[task_name][output_key] = len( @@ -53,8 +48,9 @@ def detect_task_and_output_coverage_handler(kwargs): ) task_coverage_count += len(output_tests_list) - if len(output_tests_list) == 0: + if not output_tests_list: outputs_without_tests.append(output_key) + task_coverage[task_name] = task_coverage_count print("Task Coverage:") From 2425e705b88c791959d32a30c8fadf89a6ebdf89 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 6 Mar 2024 16:31:27 -0500 Subject: [PATCH 005/101] Remove check and append to list for tests with no tasks --- src/wdlci/cli/detect_changes.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/wdlci/cli/detect_changes.py b/src/wdlci/cli/detect_changes.py index b4cfa25..98aef26 100644 --- a/src/wdlci/cli/detect_changes.py +++ b/src/wdlci/cli/detect_changes.py @@ -24,9 +24,6 @@ def detect_changes_handler(kwargs): task_name = task.name task_digest = task.digest - if not workflow_config.tasks[task_name].tests: - tasks_without_tests.append(task_name) - if task_name in workflow_config.tasks: previous_digest = workflow_config.tasks[task_name].digest if ( @@ -40,7 +37,6 @@ def detect_changes_handler(kwargs): print(f"New task detected [{workflow} - {task_name}]") workflow_change = changeset.add_workflow_change(workflow) workflow_change.add_task_change(task_name) - print(f"The following tasks have no tests: {', '.join(tasks_without_tests)}") if len(changeset.workflow_changes) == 0: print("No new or modified tasks detected") From cf8a6ed0dbc40e6948155e2b1c2747e2aa0d69d2 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 6 Mar 2024 16:39:54 -0500 Subject: [PATCH 006/101] Adjust formatting of print statements --- src/wdlci/cli/detect_task_and_output_coverage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/wdlci/cli/detect_task_and_output_coverage.py b/src/wdlci/cli/detect_task_and_output_coverage.py index 177695d..7c67ed7 100644 --- a/src/wdlci/cli/detect_task_and_output_coverage.py +++ b/src/wdlci/cli/detect_task_and_output_coverage.py @@ -55,9 +55,9 @@ def detect_task_and_output_coverage_handler(kwargs): print("Task Coverage:") for task_name, tests_count in task_coverage.items(): - print(f"\t{task_name}: {tests_count}\n") + print(f"\t{task_name}: {tests_count}") - print("Output coverage:") + print("\nOutput coverage:") for task_name, output_tests_dict in output_coverage.items(): print(f"\tTask name: {task_name}") for output_name, tests_count in output_tests_dict.items(): @@ -67,16 +67,16 @@ def detect_task_and_output_coverage_handler(kwargs): if tasks_without_tests and outputs_without_tests: print( - f"\nWARNING: The following tasks have no tests:\n\n{', '.join(tasks_without_tests)}\n\n" - + f"Additionally, the following outputs have no tests:\n\n{', '.join(outputs_without_tests)}\n\n" + f"\nWARNING: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" + + f"Additionally, the following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" ) elif tasks_without_tests: print( - f"\nWARNING: The following tasks have no tests:\n\n{', '.join(tasks_without_tests)}\n\n" + f"\nWARNING: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" ) elif outputs_without_tests: print( - f"\nWARNING: The following outputs have no tests:\n\n{', '.join(outputs_without_tests)}\n\n" + f"\nWARNING: The following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" ) except WdlTestCliExitException as e: From 45f99e808eea4bc90d2a54f4390a2aab1c2ef9eb Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 7 Mar 2024 09:49:43 -0500 Subject: [PATCH 007/101] Remove creation of unused list --- src/wdlci/cli/detect_changes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/wdlci/cli/detect_changes.py b/src/wdlci/cli/detect_changes.py index 98aef26..2640675 100644 --- a/src/wdlci/cli/detect_changes.py +++ b/src/wdlci/cli/detect_changes.py @@ -15,8 +15,6 @@ def detect_changes_handler(kwargs): changeset = Changeset() - tasks_without_tests = [] - for workflow, workflow_config in config.file.workflows.items(): doc = WDL.load(workflow) From 013cdecf4af04627421c80e8d1da1bd034bdf38e Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 7 Mar 2024 10:44:04 -0500 Subject: [PATCH 008/101] Adjust style of warnings to match internal format --- src/wdlci/cli/detect_task_and_output_coverage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wdlci/cli/detect_task_and_output_coverage.py b/src/wdlci/cli/detect_task_and_output_coverage.py index 7c67ed7..8cfc315 100644 --- a/src/wdlci/cli/detect_task_and_output_coverage.py +++ b/src/wdlci/cli/detect_task_and_output_coverage.py @@ -67,16 +67,16 @@ def detect_task_and_output_coverage_handler(kwargs): if tasks_without_tests and outputs_without_tests: print( - f"\nWARNING: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" + f"[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" + f"Additionally, the following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" ) elif tasks_without_tests: print( - f"\nWARNING: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" + f"[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" ) elif outputs_without_tests: print( - f"\nWARNING: The following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" + f"[WARN]: The following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" ) except WdlTestCliExitException as e: From 0eea505ed7a12a36039f6ecc927befbdeecee593 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 7 Mar 2024 16:25:23 -0500 Subject: [PATCH 009/101] Initial commit for handling output key errors --- src/wdlci/utils/write_workflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index 2ec1c9b..f63a30d 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -3,6 +3,7 @@ from pathlib import Path from importlib.resources import files from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException +from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig def _order_structs(struct_typedefs): @@ -118,6 +119,10 @@ def write_workflow( f.write(f"\t\t{task_input}\n") f.write("\n") + ## TODO + # Issue: When removing an output from a task, if the wdl-ci.config.json file is not also updated to remove that output, the github action will fail with a KeyError when it tries to write the workflow for that changed task. + # Fix: We should check the config file and the outputs from the updated task and compare them, outputting a more useful error like "Expected output <> not found in task <>; has this output been removed?" and/or advice to remove this output from the wdl-ci.config.json. + for output_key in output_tests: test_output_type = _get_output_type(main_task_output_types, output_key) From bf59f2de1a3dc790cdf6ee8a5f6bb164d2304a16 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 8 Mar 2024 16:10:43 -0500 Subject: [PATCH 010/101] Add error message for when an output key is not found --- src/wdlci/utils/write_workflow.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index f63a30d..3d6a840 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -1,11 +1,12 @@ import WDL import subprocess +import re from pathlib import Path from importlib.resources import files from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException +from wdlci.config import Config from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig - def _order_structs(struct_typedefs): """ Order struct_typedefs such that structs that are dependencies of other structs are written first @@ -95,6 +96,21 @@ def write_workflow( struct_typedefs ([WDL.Env.Binding]): structs imported by the main workflow; these will be available to the test task custom_test_dir (str): Path to a directory containing test WDL tasks; this directory will be checked for test tasks first """ + + # Create an instance of the config and create a list of all the outputs + config = Config.instance() + all_outputs = [] + for workflow, workflow_config in config.file.workflows.items(): + doc = WDL.load(workflow) + for task in doc.tasks: + task_outputs = task.outputs + file_name_pattern = r"\b\w+\s+(\S+)\s+=" + for output in task_outputs: + output_str = str(output) + match = re.search(file_name_pattern, output_str) + if match: + all_outputs.append(match.group(1)) + wdl_version = main_task.effective_wdl_version main_task_output_types = {output.name: output.type for output in main_task.outputs} @@ -119,9 +135,16 @@ def write_workflow( f.write(f"\t\t{task_input}\n") f.write("\n") - ## TODO - # Issue: When removing an output from a task, if the wdl-ci.config.json file is not also updated to remove that output, the github action will fail with a KeyError when it tries to write the workflow for that changed task. - # Fix: We should check the config file and the outputs from the updated task and compare them, outputting a more useful error like "Expected output <> not found in task <>; has this output been removed?" and/or advice to remove this output from the wdl-ci.config.json. + filtered_output_tests = {key: output_tests[key] for key in output_tests if key in all_outputs} + + for output_key in output_tests.keys(): + if output_key not in filtered_output_tests.keys(): + raise WdlTestCliExitException( + f"Expected output {output_key} not found in task {main_task.name}; has this output been removed?\nIf so, you > + ) + # This is just a catch for now; can likely be removed once testing has been completed. + for output_key in filtered_output_tests.keys(): + print (f"{output_key} found in wdl-ci.config.json and {main_task.name}.") for output_key in output_tests: test_output_type = _get_output_type(main_task_output_types, output_key) From 89e896c13657920f6542b0d2c4397cdc62ce4c29 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 8 Mar 2024 16:11:47 -0500 Subject: [PATCH 011/101] Add error message for when an output key is not found --- src/wdlci/utils/write_workflow.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index 3d6a840..57157f9 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -7,6 +7,7 @@ from wdlci.config import Config from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig + def _order_structs(struct_typedefs): """ Order struct_typedefs such that structs that are dependencies of other structs are written first @@ -135,16 +136,19 @@ def write_workflow( f.write(f"\t\t{task_input}\n") f.write("\n") - filtered_output_tests = {key: output_tests[key] for key in output_tests if key in all_outputs} + filtered_output_tests = { + key: output_tests[key] for key in output_tests if key in all_outputs + } for output_key in output_tests.keys(): if output_key not in filtered_output_tests.keys(): raise WdlTestCliExitException( - f"Expected output {output_key} not found in task {main_task.name}; has this output been removed?\nIf so, you > + f"Expected output {output_key} not found in task {main_task.name}; has this output been removed?\nIf so, you ill need to remove this output from the wdl-ci.config.json before proceeding.", + 1, ) # This is just a catch for now; can likely be removed once testing has been completed. for output_key in filtered_output_tests.keys(): - print (f"{output_key} found in wdl-ci.config.json and {main_task.name}.") + print(f"{output_key} found in wdl-ci.config.json and {main_task.name}.") for output_key in output_tests: test_output_type = _get_output_type(main_task_output_types, output_key) From bdd8e82aa0ba037871812bd4309c98d728968efc Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 11 Mar 2024 11:16:49 -0400 Subject: [PATCH 012/101] Reorganize and simplify handling of missing outputs key error --- src/wdlci/utils/write_workflow.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index 57157f9..cca4364 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -101,16 +101,23 @@ def write_workflow( # Create an instance of the config and create a list of all the outputs config = Config.instance() all_outputs = [] + output_file_name_pattern = re.compile(r"\b\w+\s+(\S+)\s+=") + for workflow, workflow_config in config.file.workflows.items(): doc = WDL.load(workflow) for task in doc.tasks: task_outputs = task.outputs - file_name_pattern = r"\b\w+\s+(\S+)\s+=" - for output in task_outputs: - output_str = str(output) - match = re.search(file_name_pattern, output_str) - if match: - all_outputs.append(match.group(1)) + output_filenames = [ + match.group(1) + for output in task_outputs + if (match := output_file_name_pattern.search(str(output))) + ] + all_outputs.append(output_filenames) + + # Create a dictionary of outputs present in both the workflow and config file + filtered_output_tests = { + key: output_tests[key] for key in output_tests if key in all_outputs + } wdl_version = main_task.effective_wdl_version @@ -136,14 +143,10 @@ def write_workflow( f.write(f"\t\t{task_input}\n") f.write("\n") - filtered_output_tests = { - key: output_tests[key] for key in output_tests if key in all_outputs - } - for output_key in output_tests.keys(): if output_key not in filtered_output_tests.keys(): raise WdlTestCliExitException( - f"Expected output {output_key} not found in task {main_task.name}; has this output been removed?\nIf so, you ill need to remove this output from the wdl-ci.config.json before proceeding.", + f"Expected output {output_key} not found in task {main_task.name}; has this output been removed from the workflow?\nIf so, you will need to remove this output from the wdl-ci.config.json before proceeding.", 1, ) # This is just a catch for now; can likely be removed once testing has been completed. From 4ba438058f76290bf2029d800814dd9ba252214e Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 11 Mar 2024 12:50:02 -0400 Subject: [PATCH 013/101] Further reorganize and simplify handling of missing outputs key error Create a dictionary of absent outputs rather than present ones to simplify iterating over; adjust iterating over keys in the dictionary of missing outputs and corresponding exception --- src/wdlci/utils/write_workflow.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index cca4364..6623d45 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -98,25 +98,25 @@ def write_workflow( custom_test_dir (str): Path to a directory containing test WDL tasks; this directory will be checked for test tasks first """ - # Create an instance of the config and create a list of all the outputs + # Create an instance of the config and create a list of all the missing outputs config = Config.instance() all_outputs = [] output_file_name_pattern = re.compile(r"\b\w+\s+(\S+)\s+=") + missing_outputs_list = [] for workflow, workflow_config in config.file.workflows.items(): doc = WDL.load(workflow) for task in doc.tasks: task_outputs = task.outputs - output_filenames = [ + all_outputs = [ match.group(1) for output in task_outputs if (match := output_file_name_pattern.search(str(output))) ] - all_outputs.append(output_filenames) - # Create a dictionary of outputs present in both the workflow and config file - filtered_output_tests = { - key: output_tests[key] for key in output_tests if key in all_outputs + # Create a dictionary of outputs absent from the workflow but present in the config JSON + missing_outputs_dict = { + key: output_tests[key] for key in output_tests if key not in all_outputs } wdl_version = main_task.effective_wdl_version @@ -143,15 +143,14 @@ def write_workflow( f.write(f"\t\t{task_input}\n") f.write("\n") - for output_key in output_tests.keys(): - if output_key not in filtered_output_tests.keys(): - raise WdlTestCliExitException( - f"Expected output {output_key} not found in task {main_task.name}; has this output been removed from the workflow?\nIf so, you will need to remove this output from the wdl-ci.config.json before proceeding.", - 1, - ) - # This is just a catch for now; can likely be removed once testing has been completed. - for output_key in filtered_output_tests.keys(): - print(f"{output_key} found in wdl-ci.config.json and {main_task.name}.") + for output_key in missing_outputs_dict.keys(): + missing_outputs_list.append(output_key) + + if missing_outputs_list: + raise WdlTestCliExitException( + f"Expected output(s): [{', '.join(missing_outputs_list)}] not found in task: {main_task.name}; has this output been removed from the workflow?\nIf so, you will need to remove this output from the wdl-ci.config.json before proceeding.", + 1, + ) for output_key in output_tests: test_output_type = _get_output_type(main_task_output_types, output_key) From 76dda7747045dac22cf0c10f352804828dc91019 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Tue, 12 Mar 2024 16:25:43 -0400 Subject: [PATCH 014/101] Fix creation of list of outputs --- src/wdlci/utils/write_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index 6623d45..7169c24 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -108,11 +108,12 @@ def write_workflow( doc = WDL.load(workflow) for task in doc.tasks: task_outputs = task.outputs - all_outputs = [ + outputs = [ match.group(1) for output in task_outputs if (match := output_file_name_pattern.search(str(output))) ] + all_outputs.extend(outputs) # Create a dictionary of outputs absent from the workflow but present in the config JSON missing_outputs_dict = { From 6171a336b0a47b7eb283f40cdd3ba385121da86b Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 13 Mar 2024 10:35:39 -0400 Subject: [PATCH 015/101] Adjust style of warnings to match internal format --- src/wdlci/cli/detect_task_and_output_coverage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wdlci/cli/detect_task_and_output_coverage.py b/src/wdlci/cli/detect_task_and_output_coverage.py index 7c67ed7..752b01b 100644 --- a/src/wdlci/cli/detect_task_and_output_coverage.py +++ b/src/wdlci/cli/detect_task_and_output_coverage.py @@ -67,16 +67,16 @@ def detect_task_and_output_coverage_handler(kwargs): if tasks_without_tests and outputs_without_tests: print( - f"\nWARNING: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" + f"\n[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" + f"Additionally, the following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" ) elif tasks_without_tests: print( - f"\nWARNING: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" + f"\n[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" ) elif outputs_without_tests: print( - f"\nWARNING: The following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" + f"\n[WARN]: The following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" ) except WdlTestCliExitException as e: From 0639e9ae8756b5c25dc322ba70733d2f982fca43 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 14 Mar 2024 14:11:37 -0400 Subject: [PATCH 016/101] Test handling of output key error in submit script instead of write_workflow --- src/wdlci/cli/submit.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/wdlci/cli/submit.py b/src/wdlci/cli/submit.py index 379802a..b60ca84 100644 --- a/src/wdlci/cli/submit.py +++ b/src/wdlci/cli/submit.py @@ -3,6 +3,7 @@ import sys import itertools import WDL +import re from importlib.resources import files from wdlci.auth.refresh_token_auth import RefreshTokenAuth from wdlci.config import Config @@ -67,6 +68,10 @@ def submit_handler(kwargs): # register workflow(s) tasks_to_test = dict() + workflow_outputs = [] + missing_outputs_list = [] + output_file_name_pattern = re.compile(r"\b\w+\s+(\S+)\s+=") + for workflow_key in changeset.get_workflow_keys(): for task_key in changeset.get_tasks(workflow_key): doc = WDL.load(workflow_key) @@ -78,8 +83,28 @@ def submit_handler(kwargs): for test_index, test_input_set in enumerate(task.tests): doc_main_task = doc_tasks[task_key] + for output in doc_main_task.outputs: + match = output_file_name_pattern.search(str(output)) + if match: + workflow_outputs.extend([match.group(1)]) + output_tests = test_input_set.output_tests + missing_outputs_dict = { + key: output_tests[key] + for key in output_tests + if key not in workflow_outputs + } + + for output_key in missing_outputs_dict.keys(): + missing_outputs_list.append(output_key) + + if missing_outputs_list: + raise WdlTestCliExitException( + f"Expected output(s): [{', '.join(missing_outputs_list)}] not found in task: {doc_main_task.name}; has this output been removed from the workflow?\nIf so, you will need to remove this output from the wdl-ci.config.json before proceeding.", + 1, + ) + # Skip input sets with no test tasks defined for any output all_test_tasks = [ test_task From bd269ed2b8cd13c36829460946ed3f3143e71c84 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 14 Mar 2024 14:16:49 -0400 Subject: [PATCH 017/101] Add comments for clarity --- src/wdlci/cli/submit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wdlci/cli/submit.py b/src/wdlci/cli/submit.py index b60ca84..ba90596 100644 --- a/src/wdlci/cli/submit.py +++ b/src/wdlci/cli/submit.py @@ -82,14 +82,14 @@ def submit_handler(kwargs): # Register and create workflows for all tasks with tests for test_index, test_input_set in enumerate(task.tests): doc_main_task = doc_tasks[task_key] - + # Create list of workflow output names for output in doc_main_task.outputs: match = output_file_name_pattern.search(str(output)) if match: workflow_outputs.extend([match.group(1)]) output_tests = test_input_set.output_tests - + # Create dictionary of missing outputs missing_outputs_dict = { key: output_tests[key] for key in output_tests From 2efa231b024722a99e961eaef74995c350d7b07c Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 14 Mar 2024 16:09:25 -0400 Subject: [PATCH 018/101] Revert all commits that handled output key errors in write_workflow --- src/wdlci/utils/write_workflow.py | 34 ------------------------------- 1 file changed, 34 deletions(-) diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index cb2eafc..8730dbd 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -1,10 +1,8 @@ import WDL import subprocess -import re from pathlib import Path from importlib.resources import files from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException -from wdlci.config import Config from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig import linecache @@ -98,29 +96,6 @@ def write_workflow( struct_typedefs ([WDL.Env.Binding]): structs imported by the main workflow; these will be available to the test task custom_test_dir (str): Path to a directory containing test WDL tasks; this directory will be checked for test tasks first """ - - # Create an instance of the config and create a list of all the missing outputs - config = Config.instance() - all_outputs = [] - output_file_name_pattern = re.compile(r"\b\w+\s+(\S+)\s+=") - missing_outputs_list = [] - - for workflow, workflow_config in config.file.workflows.items(): - doc = WDL.load(workflow) - for task in doc.tasks: - task_outputs = task.outputs - outputs = [ - match.group(1) - for output in task_outputs - if (match := output_file_name_pattern.search(str(output))) - ] - all_outputs.extend(outputs) - - # Create a dictionary of outputs absent from the workflow but present in the config JSON - missing_outputs_dict = { - key: output_tests[key] for key in output_tests if key not in all_outputs - } - wdl_version = main_task.effective_wdl_version main_task_output_types = {output.name: output.type for output in main_task.outputs} @@ -145,15 +120,6 @@ def write_workflow( f.write(f"\t\t{task_input}\n") f.write("\n") - for output_key in missing_outputs_dict.keys(): - missing_outputs_list.append(output_key) - - if missing_outputs_list: - raise WdlTestCliExitException( - f"Expected output(s): [{', '.join(missing_outputs_list)}] not found in task: {main_task.name}; has this output been removed from the workflow?\nIf so, you will need to remove this output from the wdl-ci.config.json before proceeding.", - 1, - ) - for output_key in output_tests: test_output_type = _get_output_type(main_task_output_types, output_key) From 09d4094eaf66f6a3d5cebbf0bc21a5d4c0902bb9 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 27 Mar 2024 17:50:39 -0400 Subject: [PATCH 019/101] Add documentation surrounding wdl-ci coverage command --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 9a3ae53..8280d11 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,29 @@ If a configuration file already exists, this will add workflows or tasks that do `wdl-ci generate-config [--remove]` +## Calculate coverage + +Calculates percent coverage for each task and workflow. Task coverage is computed as the sum of task outputs with at least one test divided by the number of task outputs, while workflow coverage is computed as the sum of task outputs across all tasks with at least one test divided by the number of task outputs across all tasks. If there are any tasks or outputs with no tests, a warning is provided to the user, listing the relevant tasks/outputs. + +As long as an output is covered by one task, it is considered to be 100% covered, while if 3/4 outputs for a task have at least one test, that task has 75% test coverage. + +Example output: + +``` +workflowNameA coverage: 84% + taskA coverage: 75% + taskB coverage: 63% + taskC coverage: 100% + taskD coverage: 100% +workflowNameB coverage: 100% + taskA coverage: 100% + taskB coverage: 100% +... +Total coverage across outputs and tasks among all workflows: 89% +``` + +`wdl-ci coverage` + ## Lint workflows `wdl-ci lint` From b46ffaef7c10e11e7d17ebeab0d45abdb32eeeed Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 4 Apr 2024 14:52:06 -0400 Subject: [PATCH 020/101] Rename subcommand; remove old subcommand script --- src/wdlci/cli/__main__.py | 10 +-- .../cli/detect_task_and_output_coverage.py | 84 ------------------- 2 files changed, 4 insertions(+), 90 deletions(-) delete mode 100644 src/wdlci/cli/detect_task_and_output_coverage.py diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index dc144ce..e6c070f 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -5,9 +5,7 @@ from wdlci.cli.submit import submit_handler from wdlci.cli.monitor import monitor_handler from wdlci.cli.cleanup import cleanup_handler -from wdlci.cli.detect_task_and_output_coverage import ( - detect_task_and_output_coverage_handler, -) +from wdlci.cli.coverage import coverage_handler from wdlci.utils.ordered_group import OrderedGroup remove_option = click.option( @@ -106,7 +104,7 @@ def cleanup(**kwargs): @main.command -def detect_task_and_output_coverage(**kwargs): - """Outputs how much coverage each task and output has, and which tasks/outputs have no associated tests""" +def coverage(**kwargs): + """Outputs percent coverage for each task and output, and which tasks/outputs have no associated tests""" - detect_task_and_output_coverage_handler(kwargs) + coverage_handler(kwargs) diff --git a/src/wdlci/cli/detect_task_and_output_coverage.py b/src/wdlci/cli/detect_task_and_output_coverage.py deleted file mode 100644 index 752b01b..0000000 --- a/src/wdlci/cli/detect_task_and_output_coverage.py +++ /dev/null @@ -1,84 +0,0 @@ -import sys -import WDL -from wdlci.config import Config -from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig -from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException - - -def detect_task_and_output_coverage_handler(kwargs): - try: - Config.load(kwargs) - config = Config.instance() - - tasks_without_tests = [] - outputs_without_tests = [] - - task_coverage = {} - output_coverage = {} - - for workflow, workflow_config in config.file.workflows.items(): - doc = WDL.load(workflow) - - for task in doc.tasks: - task_name = task.name - # task_test_configs is a list of WorkflowTaskTestConfig objects, - # which contain inputs and output_tests as attributes - task_test_configs = workflow_config.tasks[task_name].tests - - if not task_test_configs: - tasks_without_tests.append(task_name) - - task_coverage_count = 0 - - for task_test_config in task_test_configs: - output_tests_dict = task_test_config.output_tests - output_coverage[task_name] = {} - - for output_key, output_value in output_tests_dict.items(): - if ( - isinstance(output_value, dict) - and "test_tasks" in output_value - ): - output_tests_list = output_value["test_tasks"] - # Assign the length of test tasks to the key associated with a - # given output in a nested structure, where each task has a - # dictionary of outputs and their task counts - output_coverage[task_name][output_key] = len( - output_tests_list - ) - task_coverage_count += len(output_tests_list) - - if not output_tests_list: - outputs_without_tests.append(output_key) - - task_coverage[task_name] = task_coverage_count - - print("Task Coverage:") - for task_name, tests_count in task_coverage.items(): - print(f"\t{task_name}: {tests_count}") - - print("\nOutput coverage:") - for task_name, output_tests_dict in output_coverage.items(): - print(f"\tTask name: {task_name}") - for output_name, tests_count in output_tests_dict.items(): - print( - f"\t\tOutput name: {output_name}\n\t\t\tnumber of tests: {tests_count}" - ) - - if tasks_without_tests and outputs_without_tests: - print( - f"\n[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" - + f"Additionally, the following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" - ) - elif tasks_without_tests: - print( - f"\n[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n" - ) - elif outputs_without_tests: - print( - f"\n[WARN]: The following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n" - ) - - except WdlTestCliExitException as e: - print(f"exiting with code {e.exit_code}, message: {e.message}") - sys.exit(e.exit_code) From f611b0940a33a7dc46bc4e55b9afa8296c4be3f6 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 4 Apr 2024 14:59:51 -0400 Subject: [PATCH 021/101] Add coverage subcommand to calculate and display test coverage Instead of relying on the config file to tell us what tasks/outputs to test, the WDL files in the directory tell us what needs testing and then the config file is checked for associated tests. Currently, there are some edge cases that need handling; 1. Compound outputs and 2. Workflows from the custom_tests directory are included in the coverage calculation, which is not correct. --- src/wdlci/cli/coverage.py | 113 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 src/wdlci/cli/coverage.py diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py new file mode 100644 index 0000000..f33c275 --- /dev/null +++ b/src/wdlci/cli/coverage.py @@ -0,0 +1,113 @@ +import WDL +import os +import sys +import re + +from wdlci.config import Config +from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig +from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException + +output_file_name_pattern = re.compile(r"\b\w+\?*\s+(\S+|\S+\[\S+\])\s+=") + + +def coverage_handler(kwargs): + try: + """ """ + # Load the config file + Config.load(kwargs) + # Get the config instance + config = Config.instance() + output_tests = {} + # Iterate over each workflow in the config file + for workflow_name, workflow_config in config.file.workflows.items(): + # Iterate over each task in the workflow + for task_name, task_config in workflow_config.tasks.items(): + # Iterate over each test in the task + for test_config in task_config.tests: + # Iterate over each output in the test + for output, test in test_config.output_tests.items(): + # Add the output to the dictionary if it is not already present + if output not in output_tests: + output_tests[output] = [] + output_tests[output].append(test["test_tasks"]) + + # Load all WDL files in the directory + cwd = os.getcwd() + wdl_files = [] + for root_path, subfolders, filenames in os.walk(cwd): + for filename in filenames: + if filename.endswith( + ".wdl" + ): # Need to make sure this does NOT look in custom_tests dir + wdl_files.append( + os.path.relpath(os.path.join(root_path, filename), cwd) + ) + + # Initialize counters + all_outputs = all_tests = workflow_outputs = workflow_tests = 0 + + # Iterate over each WDL file + for wdl_file in wdl_files: + # Strip everything except workflow name + wdl_filename = wdl_file.split("/")[-1] + # Load the WDL document + doc = WDL.load(wdl_file) + + # Calculate and print the workflow coverage + for task in doc.tasks: + task_outputs = len(task.outputs) + workflow_outputs += task_outputs + for output in task.outputs: + if output.name in output_tests.keys(): + workflow_tests += 1 + if not workflow_outputs: + print( + f"\nworkflow: {wdl_filename} has no outputs and thus cannot have any associated tests" + ) + if workflow_tests and workflow_outputs: + workflow_coverage = (workflow_tests / workflow_outputs) * 100 + print(f"\nworkflow: {wdl_filename}: {workflow_coverage:.2f}%") + elif workflow_outputs and not workflow_tests: + print( + f"[WARN]: workflow: {wdl_filename} has outputs but no associated tests\n" + ) + + # Iterate over each task in the WDL document + for task in doc.tasks: + task_outputs = [] + tested_outputs = [] + # Create a list of outputs that are present in the task/worfklow but not in the config JSON using the output_tests dictionary + missing_config_outputs = [ + output_file_name_pattern.search(str(output)).group(1) + for output in task.outputs + if output.name not in output_tests + ] + + # Count the number of outputs for the task + task_outputs = task.outputs + # total_outputs += task_outputs + all_outputs += len(task_outputs) + + # Check if there are tests for each output + for output in task.outputs: + if output.name in output_tests.keys(): + tested_outputs.append(output.name) + all_tests += 1 + # Check if task_outputs is non-zero + if task_outputs: + # Calculate and print the task coverage + task_coverage = (len(tested_outputs) / len(task_outputs)) * 100 + print(f"\ttask.{task.name}: {task_coverage:.2f}%") + if missing_config_outputs: + print( + f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_config_outputs} in task {task.name}" + ) + + # Calculate and print the total coverage + if all_outputs: + total_coverage = (all_tests / all_outputs) * 100 + print(f"\nTotal coverage: {total_coverage:.2f}%") + + except WdlTestCliExitException as e: + print(f"exiting with code {e.exit_code}, message: {e.message}") + sys.exit(e.exit_code) From 8d9d5ce8abccf79d460a2e2b115609af3e813769 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 5 Apr 2024 11:24:31 -0400 Subject: [PATCH 022/101] Adjust subcommand to handle compound outputs; clean up calculations --- src/wdlci/cli/coverage.py | 58 +++++++++++++++------------------------ 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index f33c275..779bd4d 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -7,7 +7,7 @@ from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException -output_file_name_pattern = re.compile(r"\b\w+\?*\s+(\S+|\S+\[\S+\])\s+=") +output_file_name_pattern = re.compile(r"(\b\w+\b)\s*=") def coverage_handler(kwargs): @@ -36,46 +36,29 @@ def coverage_handler(kwargs): wdl_files = [] for root_path, subfolders, filenames in os.walk(cwd): for filename in filenames: - if filename.endswith( - ".wdl" - ): # Need to make sure this does NOT look in custom_tests dir + if filename.endswith(".wdl"): wdl_files.append( os.path.relpath(os.path.join(root_path, filename), cwd) ) - # Initialize counters - all_outputs = all_tests = workflow_outputs = workflow_tests = 0 + # Initialize counters/lists for total coverage + all_outputs = 0 + all_tests = [] + # Create a set of keys for the output_tests dictionary for faster lookup + output_tests_keys = set(output_tests.keys()) # Iterate over each WDL file for wdl_file in wdl_files: + workflow_tests = [] + workflow_outputs = 0 # Strip everything except workflow name wdl_filename = wdl_file.split("/")[-1] # Load the WDL document doc = WDL.load(wdl_file) - # Calculate and print the workflow coverage - for task in doc.tasks: - task_outputs = len(task.outputs) - workflow_outputs += task_outputs - for output in task.outputs: - if output.name in output_tests.keys(): - workflow_tests += 1 - if not workflow_outputs: - print( - f"\nworkflow: {wdl_filename} has no outputs and thus cannot have any associated tests" - ) - if workflow_tests and workflow_outputs: - workflow_coverage = (workflow_tests / workflow_outputs) * 100 - print(f"\nworkflow: {wdl_filename}: {workflow_coverage:.2f}%") - elif workflow_outputs and not workflow_tests: - print( - f"[WARN]: workflow: {wdl_filename} has outputs but no associated tests\n" - ) - # Iterate over each task in the WDL document for task in doc.tasks: - task_outputs = [] - tested_outputs = [] + task_tests = [] # Create a list of outputs that are present in the task/worfklow but not in the config JSON using the output_tests dictionary missing_config_outputs = [ output_file_name_pattern.search(str(output)).group(1) @@ -84,28 +67,31 @@ def coverage_handler(kwargs): ] # Count the number of outputs for the task - task_outputs = task.outputs - # total_outputs += task_outputs - all_outputs += len(task_outputs) + all_outputs += len(task.outputs) + workflow_outputs += len(task.outputs) # Check if there are tests for each output for output in task.outputs: - if output.name in output_tests.keys(): - tested_outputs.append(output.name) - all_tests += 1 + if output.name in output_tests_keys: + task_tests.append(output.name) + workflow_tests.append(output.name) + all_tests.append(output.name) # Check if task_outputs is non-zero - if task_outputs: + if task.outputs: # Calculate and print the task coverage - task_coverage = (len(tested_outputs) / len(task_outputs)) * 100 + task_coverage = (len(task_tests) / len(task.outputs)) * 100 print(f"\ttask.{task.name}: {task_coverage:.2f}%") if missing_config_outputs: print( f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_config_outputs} in task {task.name}" ) + if workflow_tests and workflow_outputs: + workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 + print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%\n") # Calculate and print the total coverage if all_outputs: - total_coverage = (all_tests / all_outputs) * 100 + total_coverage = (len(all_tests) / all_outputs) * 100 print(f"\nTotal coverage: {total_coverage:.2f}%") except WdlTestCliExitException as e: From d8c0ad7e3c36f4f729e485c385b443b876317d67 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 5 Apr 2024 15:31:29 -0400 Subject: [PATCH 023/101] Adjust formatting/organization of task coverage output --- src/wdlci/cli/coverage.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 779bd4d..cc315fb 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -4,7 +4,6 @@ import re from wdlci.config import Config -from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException output_file_name_pattern = re.compile(r"(\b\w+\b)\s*=") @@ -41,9 +40,10 @@ def coverage_handler(kwargs): os.path.relpath(os.path.join(root_path, filename), cwd) ) - # Initialize counters/lists for total coverage + # Initialize counters/lists all_outputs = 0 all_tests = [] + untested_tasks = {} # Create a set of keys for the output_tests dictionary for faster lookup output_tests_keys = set(output_tests.keys()) @@ -76,21 +76,32 @@ def coverage_handler(kwargs): task_tests.append(output.name) workflow_tests.append(output.name) all_tests.append(output.name) - # Check if task_outputs is non-zero - if task.outputs: + # Print task coverage for tasks with outputs and tests + if task.outputs and task_tests: # Calculate and print the task coverage task_coverage = (len(task_tests) / len(task.outputs)) * 100 - print(f"\ttask.{task.name}: {task_coverage:.2f}%") - if missing_config_outputs: + print(f"task.{task.name}: {task_coverage:.2f}%") + # If there are outputs but no tests for the entire task, add the task to the untested_tasks list + elif task.outputs and not task_tests: + if wdl_filename not in untested_tasks: + untested_tasks[wdl_filename] = [] + untested_tasks[wdl_filename].append(task.name) + if missing_config_outputs and task_tests: print( f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_config_outputs} in task {task.name}" ) + # Print workflow coverage for tasks with outputs and tests if workflow_tests and workflow_outputs: workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 + print("-" * 75) print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%\n") - + # Warn the user about tasks that have no associated tests + for workflow, tasks in untested_tasks.items(): + print(f"For {workflow}, these tasks are untested:") + for task in tasks: + print(f"\ttask") # Calculate and print the total coverage - if all_outputs: + if all_tests and all_outputs: total_coverage = (len(all_tests) / all_outputs) * 100 print(f"\nTotal coverage: {total_coverage:.2f}%") From 5bb45db5c14951755f2ed19e84399785bdc63381 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 15 Nov 2024 15:35:01 -0500 Subject: [PATCH 024/101] Wrap task name properly --- src/wdlci/cli/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index cc315fb..5bae2e0 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -99,7 +99,7 @@ def coverage_handler(kwargs): for workflow, tasks in untested_tasks.items(): print(f"For {workflow}, these tasks are untested:") for task in tasks: - print(f"\ttask") + print(f"\t{task}") # Calculate and print the total coverage if all_tests and all_outputs: total_coverage = (len(all_tests) / all_outputs) * 100 From 2f0e8cc3ea3b8b7e394b533fadf8632145282e4d Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Mon, 18 Nov 2024 15:34:28 -0500 Subject: [PATCH 025/101] Clarify parameter precedence --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8280d11..b5ed802 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,7 @@ Test params can be used to avoid repeating paths and values for test inputs and - For parameters whose value is an object, object members can be accessed using `.`s: `"reference_index": ${reference.fasta.data_index}` - Global params will replace values for all engines - Engine params will replace values only when submitting to a particular engine; useful if for example input sets exist across multiple environments and are prefixed with different paths +- For parameters with the same key, `engine_params` take precedence over `global_params` - Objects and arrays can be used for parameters; if you are using a complex parameter as an input or output value, this parameter must be the only content of the value, e.g. `"my_input": "${complex_param}"`, not `"my_input": "gs://my_bucket/${complex_param}"` - The values of complex parameters can themselves use parameters, and will be substituted appropriately From 3ac87fcaf3e8da3a6da939d1cd58882c5e250980 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 21 Nov 2024 15:35:28 -0500 Subject: [PATCH 026/101] Improve coverage calculation login; add specific warning for optional outputs Handle case where output is present in config but test tasks is an empty list; notify user is all optional outputs are covered by a test; add TODO --- src/wdlci/cli/coverage.py | 48 +++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 5bae2e0..8f8fd43 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -11,16 +11,15 @@ def coverage_handler(kwargs): try: - """ """ # Load the config file Config.load(kwargs) # Get the config instance config = Config.instance() output_tests = {} # Iterate over each workflow in the config file - for workflow_name, workflow_config in config.file.workflows.items(): + for _, workflow_config in config.file.workflows.items(): # Iterate over each task in the workflow - for task_name, task_config in workflow_config.tasks.items(): + for _, task_config in workflow_config.tasks.items(): # Iterate over each test in the task for test_config in task_config.tests: # Iterate over each output in the test @@ -28,7 +27,9 @@ def coverage_handler(kwargs): # Add the output to the dictionary if it is not already present if output not in output_tests: output_tests[output] = [] - output_tests[output].append(test["test_tasks"]) + # Check if 'test_tasks' key exists in the test dictionary and is not empty and append to output_tests if present and not empty + if "test_tasks" in test and test["test_tasks"]: + output_tests[output].append(test["test_tasks"]) # Load all WDL files in the directory cwd = os.getcwd() @@ -44,6 +45,7 @@ def coverage_handler(kwargs): all_outputs = 0 all_tests = [] untested_tasks = {} + untested_optional_outputs = [] # Create a set of keys for the output_tests dictionary for faster lookup output_tests_keys = set(output_tests.keys()) @@ -60,10 +62,14 @@ def coverage_handler(kwargs): for task in doc.tasks: task_tests = [] # Create a list of outputs that are present in the task/worfklow but not in the config JSON using the output_tests dictionary - missing_config_outputs = [ + missing_outputs = [ output_file_name_pattern.search(str(output)).group(1) for output in task.outputs - if output.name not in output_tests + if output.name not in output_tests_keys + or ( + output.name in output_tests_keys + and not output_tests[output.name] + ) ] # Count the number of outputs for the task @@ -72,10 +78,22 @@ def coverage_handler(kwargs): # Check if there are tests for each output for output in task.outputs: - if output.name in output_tests_keys: + if ( + output.name in output_tests_keys + and output_tests[output.name] != [] + ): task_tests.append(output.name) workflow_tests.append(output.name) all_tests.append(output.name) + if ( + output.type.optional + and output.name not in output_tests_keys + or ( + output.name in output_tests_keys + and output_tests[output.name] == [] + ) + ): + untested_optional_outputs.append(output.name) # Print task coverage for tasks with outputs and tests if task.outputs and task_tests: # Calculate and print the task coverage @@ -86,20 +104,30 @@ def coverage_handler(kwargs): if wdl_filename not in untested_tasks: untested_tasks[wdl_filename] = [] untested_tasks[wdl_filename].append(task.name) - if missing_config_outputs and task_tests: + if missing_outputs and task_tests: print( - f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_config_outputs} in task {task.name}" + f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs} in task {task.name}" ) # Print workflow coverage for tasks with outputs and tests if workflow_tests and workflow_outputs: workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 - print("-" * 75) + print("-" * 150) print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%\n") + ## TODO: Implement cutoff check for workflow coverage (e..g, adding sub-command/arg/flag for threshold; output any workflows with <80% coverage (and maybe even block merge if below a certain threshold), --workflow_name and just show me this one for example) # Warn the user about tasks that have no associated tests + + print("-" * 150) for workflow, tasks in untested_tasks.items(): print(f"For {workflow}, these tasks are untested:") for task in tasks: print(f"\t{task}") + # Warn the user about optional outputs that are not tested + if untested_optional_outputs: + print( + f"\n[WARN]: These optional outputs are not tested: {untested_optional_outputs}" + ) + else: + print("\n✓ All optional outputs are tested") # Calculate and print the total coverage if all_tests and all_outputs: total_coverage = (len(all_tests) / all_outputs) * 100 From 8498965a00c2aacbf7a3978b58916f81a7351f69 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 21 Nov 2024 16:55:39 -0500 Subject: [PATCH 027/101] Add threshold and workflow name arguments to coverage command --- src/wdlci/cli/__main__.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index e6c070f..fb56940 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -35,6 +35,24 @@ help="Do not exit upon encountering a linting warning or error", ) +coverage_threshold = click.option( + "--coverage-threshold", + "-t", + type=float, + default=None, + show_default=True, + help="Maximum coverage percent threshold; any tasks or workflows with coverage above this threshold will not be displayed", +) + +workflow_name = click.option( + "--workflow-name", + "-w", + type=str, + default=None, + show_default=True, + help="Workflow name to filter coverage results", +) + @click.group(cls=OrderedGroup) @click.version_option( @@ -104,6 +122,8 @@ def cleanup(**kwargs): @main.command +@coverage_threshold +@workflow_name def coverage(**kwargs): """Outputs percent coverage for each task and output, and which tasks/outputs have no associated tests""" From 8e88ad11a58a7ee70577359d20f0c42b3ad624f5 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 21 Nov 2024 16:56:12 -0500 Subject: [PATCH 028/101] Implement usage of threshold and workflow name within coverage command --- src/wdlci/cli/coverage.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 8f8fd43..787536a 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -10,6 +10,10 @@ def coverage_handler(kwargs): + threshold = kwargs["coverage_threshold"] + workflow_name_filter = kwargs["workflow_name"] + print(f"Coverage threshold: ", threshold) + print(f"Workflow name filter: ", workflow_name_filter + "\n") try: # Load the config file Config.load(kwargs) @@ -46,6 +50,11 @@ def coverage_handler(kwargs): all_tests = [] untested_tasks = {} untested_optional_outputs = [] + # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter + tasks_below_threshold = False + workflows_below_threshold = False + workflow_found = False + # Create a set of keys for the output_tests dictionary for faster lookup output_tests_keys = set(output_tests.keys()) @@ -58,6 +67,11 @@ def coverage_handler(kwargs): # Load the WDL document doc = WDL.load(wdl_file) + # If workflow_name_filter is provided, skip all other workflows + if workflow_name_filter and workflow_name_filter not in wdl_filename: + continue + workflow_found = True + # Iterate over each task in the WDL document for task in doc.tasks: task_tests = [] @@ -98,7 +112,9 @@ def coverage_handler(kwargs): if task.outputs and task_tests: # Calculate and print the task coverage task_coverage = (len(task_tests) / len(task.outputs)) * 100 - print(f"task.{task.name}: {task_coverage:.2f}%") + if threshold is None or threshold and task_coverage < threshold: + tasks_below_threshold = True + print(f"task.{task.name}: {task_coverage:.2f}%") # If there are outputs but no tests for the entire task, add the task to the untested_tasks list elif task.outputs and not task_tests: if wdl_filename not in untested_tasks: @@ -111,12 +127,16 @@ def coverage_handler(kwargs): # Print workflow coverage for tasks with outputs and tests if workflow_tests and workflow_outputs: workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 - print("-" * 150) - print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%\n") - ## TODO: Implement cutoff check for workflow coverage (e..g, adding sub-command/arg/flag for threshold; output any workflows with <80% coverage (and maybe even block merge if below a certain threshold), --workflow_name and just show me this one for example) + if threshold is None or threshold and workflow_coverage < threshold: + print("-" * 150) + workflows_below_threshold = True + print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%") # Warn the user about tasks that have no associated tests - print("-" * 150) + # Inform the user if no workflows matched the filter + if workflow_name_filter and not workflow_found: + print(f"\nNo workflows found matching the filter: {workflow_name_filter}") + sys.exit(0) for workflow, tasks in untested_tasks.items(): print(f"For {workflow}, these tasks are untested:") for task in tasks: @@ -128,6 +148,13 @@ def coverage_handler(kwargs): ) else: print("\n✓ All optional outputs are tested") + + # Print a warning if any tasks or workflows are below the threshold + if not tasks_below_threshold: + print("\n✓ All tasks exceed the specified coverage threshold.") + if not workflows_below_threshold: + print("\n✓ All workflows exceed the specified coverage threshold.") + # Calculate and print the total coverage if all_tests and all_outputs: total_coverage = (len(all_tests) / all_outputs) * 100 From a29df08f0b175075d09f5b255b429afd17a6b221 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 21 Nov 2024 17:01:32 -0500 Subject: [PATCH 029/101] Add context to new features of wdl-ci coverage --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index b5ed802..a0f2160 100644 --- a/README.md +++ b/README.md @@ -483,6 +483,10 @@ Total coverage across outputs and tasks among all workflows: 89% `wdl-ci coverage` +`wdl-ci coverage` can be run with additional options to filter results and set thresholds. The `--workflow-name` option takes a float percent value and allows you to specify a workflow name to filter the results, showing only the coverage for that specific workflow. The `--coverage-threshold` option allows you to set a threshold percentage; only tasks and workflows with coverage below this threshold will be displayed. If no workflows match the specified filter, a message will be printed to inform the user. Additionally, if all optional outputs are tested, a confirmation message will be displayed. + +`wdl-ci coverage --workflow-name --coverage-threshold ` + ## Lint workflows `wdl-ci lint` From a998e8fd2d8accefe5b061e66b2473e4bb717cb1 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 22 Nov 2024 10:38:21 -0500 Subject: [PATCH 030/101] Clean up documentation --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a0f2160..e1609f1 100644 --- a/README.md +++ b/README.md @@ -462,7 +462,7 @@ If a configuration file already exists, this will add workflows or tasks that do ## Calculate coverage -Calculates percent coverage for each task and workflow. Task coverage is computed as the sum of task outputs with at least one test divided by the number of task outputs, while workflow coverage is computed as the sum of task outputs across all tasks with at least one test divided by the number of task outputs across all tasks. If there are any tasks or outputs with no tests, a warning is provided to the user, listing the relevant tasks/outputs. +Calculates percent coverage for each task and workflow. Task coverage is computed as the sum of task outputs with at least one test divided by the number of task outputs, while workflow coverage is computed as the sum of task outputs across all tasks with at least one test divided by the number of task outputs across all tasks within a given workflow. Total coverage is also computed, as the total number of outputs with tests defined divided by the number of total outputs across all workflows. If there are any tasks or outputs with no tests, a warning is provided to the user, listing the relevant outputs and associated tasks. As long as an output is covered by one task, it is considered to be 100% covered, while if 3/4 outputs for a task have at least one test, that task has 75% test coverage. @@ -481,9 +481,7 @@ workflowNameB coverage: 100% Total coverage across outputs and tasks among all workflows: 89% ``` -`wdl-ci coverage` - -`wdl-ci coverage` can be run with additional options to filter results and set thresholds. The `--workflow-name` option takes a float percent value and allows you to specify a workflow name to filter the results, showing only the coverage for that specific workflow. The `--coverage-threshold` option allows you to set a threshold percentage; only tasks and workflows with coverage below this threshold will be displayed. If no workflows match the specified filter, a message will be printed to inform the user. Additionally, if all optional outputs are tested, a confirmation message will be displayed. +`wdl-ci coverage` can be run with additional options to filter results and set thresholds. The `--workflow-name` option allows the user to specify a workflow name to filter the results, showing only the coverage for that specific workflow. The `--coverage-threshold` option takes a float percent value and allows the user to set a threshold percentage; only tasks and workflows with coverage below this threshold will be displayed. If no tasks/workflows match the specified filter, a message will be printed to inform the user. Additionally, if all optional outputs are tested, a confirmation message will be displayed. `wdl-ci coverage --workflow-name --coverage-threshold ` From 624b22ad0e96cc30af31accfa9a45ba00f3b2b8b Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 22 Nov 2024 13:19:38 -0500 Subject: [PATCH 031/101] Make formatting changes --- src/wdlci/cli/coverage.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 787536a..d005475 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -13,7 +13,10 @@ def coverage_handler(kwargs): threshold = kwargs["coverage_threshold"] workflow_name_filter = kwargs["workflow_name"] print(f"Coverage threshold: ", threshold) - print(f"Workflow name filter: ", workflow_name_filter + "\n") + if workflow_name_filter: + print(f"Workflow name filter: {workflow_name_filter}\n") + else: + print("Workflow name filter: None\n") try: # Load the config file Config.load(kwargs) @@ -131,12 +134,12 @@ def coverage_handler(kwargs): print("-" * 150) workflows_below_threshold = True print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%") - # Warn the user about tasks that have no associated tests - print("-" * 150) + print("-" * 150 + "\n") # Inform the user if no workflows matched the filter if workflow_name_filter and not workflow_found: print(f"\nNo workflows found matching the filter: {workflow_name_filter}") sys.exit(0) + # Warn the user about tasks that have no associated tests for workflow, tasks in untested_tasks.items(): print(f"For {workflow}, these tasks are untested:") for task in tasks: From b6a665e1863d64ee7e2d08839e1fdd153ee034d1 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 22 Nov 2024 14:00:08 -0500 Subject: [PATCH 032/101] Report to user any optional input not covered as part of wdl-ci tests --- src/wdlci/cli/coverage.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index d005475..52cd7c0 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -23,12 +23,17 @@ def coverage_handler(kwargs): # Get the config instance config = Config.instance() output_tests = {} + covered_inputs = set() # Iterate over each workflow in the config file for _, workflow_config in config.file.workflows.items(): # Iterate over each task in the workflow for _, task_config in workflow_config.tasks.items(): # Iterate over each test in the task for test_config in task_config.tests: + # Iterate over each input in the test + for input in test_config.inputs: + # Add the input to the set + covered_inputs.add(input) # Iterate over each output in the test for output, test in test_config.output_tests.items(): # Add the output to the dictionary if it is not already present @@ -52,6 +57,8 @@ def coverage_handler(kwargs): all_outputs = 0 all_tests = [] untested_tasks = {} + untested_optional_inputs = set() + # Use a set to avoid duplicate entries untested_optional_outputs = [] # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter tasks_below_threshold = False @@ -127,6 +134,11 @@ def coverage_handler(kwargs): print( f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs} in task {task.name}" ) + # Check if all optional inputs are covered + for input in task.inputs: + if input.type.optional and input.name not in covered_inputs: + untested_optional_inputs.add(input.name) + # Print workflow coverage for tasks with outputs and tests if workflow_tests and workflow_outputs: workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 @@ -151,7 +163,13 @@ def coverage_handler(kwargs): ) else: print("\n✓ All optional outputs are tested") - + # Warn the user about optional inputs that are not tested + if untested_optional_inputs: + print( + f"\n[WARN]: These optional inputs are not used in any tests: {untested_optional_inputs}" + ) + else: + print("\n✓ All optional inputs are tested") # Print a warning if any tasks or workflows are below the threshold if not tasks_below_threshold: print("\n✓ All tasks exceed the specified coverage threshold.") From c4da0401b78d7fa8e7f089f41df570cbf985c781 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 22 Nov 2024 14:13:20 -0500 Subject: [PATCH 033/101] Comment out print statement re: optional inputs; add relevant TODO --- src/wdlci/cli/coverage.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 52cd7c0..de4430c 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -163,13 +163,14 @@ def coverage_handler(kwargs): ) else: print("\n✓ All optional outputs are tested") - # Warn the user about optional inputs that are not tested - if untested_optional_inputs: - print( - f"\n[WARN]: These optional inputs are not used in any tests: {untested_optional_inputs}" - ) - else: - print("\n✓ All optional inputs are tested") + ## TODO: We don't really care if optional inputs are used or not; what we need to measure is if there is a test that covered running that task with the optional input and without it + # # Warn the user about optional inputs that are not tested + # if untested_optional_inputs: + # print( + # f"\n[WARN]: These optional inputs are not used in any tests: {untested_optional_inputs}" + # ) + # else: + # print("\n✓ All optional inputs are tested") # Print a warning if any tasks or workflows are below the threshold if not tasks_below_threshold: print("\n✓ All tasks exceed the specified coverage threshold.") From fb07f30bd8f6978aa37b34474093eb8731ff65ed Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 25 Nov 2024 08:22:51 -0500 Subject: [PATCH 034/101] Remove unused input checking code; adjust TODO --- src/wdlci/cli/coverage.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index de4430c..cafb6c2 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -23,17 +23,12 @@ def coverage_handler(kwargs): # Get the config instance config = Config.instance() output_tests = {} - covered_inputs = set() # Iterate over each workflow in the config file for _, workflow_config in config.file.workflows.items(): # Iterate over each task in the workflow for _, task_config in workflow_config.tasks.items(): # Iterate over each test in the task for test_config in task_config.tests: - # Iterate over each input in the test - for input in test_config.inputs: - # Add the input to the set - covered_inputs.add(input) # Iterate over each output in the test for output, test in test_config.output_tests.items(): # Add the output to the dictionary if it is not already present @@ -57,7 +52,6 @@ def coverage_handler(kwargs): all_outputs = 0 all_tests = [] untested_tasks = {} - untested_optional_inputs = set() # Use a set to avoid duplicate entries untested_optional_outputs = [] # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter @@ -134,10 +128,6 @@ def coverage_handler(kwargs): print( f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs} in task {task.name}" ) - # Check if all optional inputs are covered - for input in task.inputs: - if input.type.optional and input.name not in covered_inputs: - untested_optional_inputs.add(input.name) # Print workflow coverage for tasks with outputs and tests if workflow_tests and workflow_outputs: @@ -163,7 +153,7 @@ def coverage_handler(kwargs): ) else: print("\n✓ All optional outputs are tested") - ## TODO: We don't really care if optional inputs are used or not; what we need to measure is if there is a test that covered running that task with the optional input and without it + ## TODO: Measure is if there is a test that covered running that task with the optional input and without it # # Warn the user about optional inputs that are not tested # if untested_optional_inputs: # print( From 2f3248cf85dec0a14455b2964ee762ac322e6bcc Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Fri, 29 Nov 2024 11:16:05 -0500 Subject: [PATCH 035/101] Rename coverage_threshold to target_coverage to make it a bit clearer --- src/wdlci/cli/__main__.py | 8 ++++---- src/wdlci/cli/coverage.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index fb56940..240c4d5 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -35,13 +35,13 @@ help="Do not exit upon encountering a linting warning or error", ) -coverage_threshold = click.option( - "--coverage-threshold", +target_coverage = click.option( + "--target-coverage", "-t", type=float, default=None, show_default=True, - help="Maximum coverage percent threshold; any tasks or workflows with coverage above this threshold will not be displayed", + help="Target coverage (%); only output tasks or workflows with test coverage below this threshold", ) workflow_name = click.option( @@ -122,7 +122,7 @@ def cleanup(**kwargs): @main.command -@coverage_threshold +@target_coverage @workflow_name def coverage(**kwargs): """Outputs percent coverage for each task and output, and which tasks/outputs have no associated tests""" diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index cafb6c2..572d600 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -10,9 +10,9 @@ def coverage_handler(kwargs): - threshold = kwargs["coverage_threshold"] + threshold = kwargs["target_coverage"] workflow_name_filter = kwargs["workflow_name"] - print(f"Coverage threshold: ", threshold) + print(f"Target coverage threshold: ", threshold) if workflow_name_filter: print(f"Workflow name filter: {workflow_name_filter}\n") else: From 149eb63dd38f84e05b616a2aebcf0ea3324a2be4 Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Fri, 29 Nov 2024 11:44:06 -0500 Subject: [PATCH 036/101] Update docs for changed arg name --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 57168a4..f8a74ba 100644 --- a/README.md +++ b/README.md @@ -481,9 +481,9 @@ workflowNameB coverage: 100% Total coverage across outputs and tasks among all workflows: 89% ``` -`wdl-ci coverage` can be run with additional options to filter results and set thresholds. The `--workflow-name` option allows the user to specify a workflow name to filter the results, showing only the coverage for that specific workflow. The `--coverage-threshold` option takes a float percent value and allows the user to set a threshold percentage; only tasks and workflows with coverage below this threshold will be displayed. If no tasks/workflows match the specified filter, a message will be printed to inform the user. Additionally, if all optional outputs are tested, a confirmation message will be displayed. +`wdl-ci coverage` can be run with additional options to filter results and set thresholds. The `--workflow-name` option allows the user to specify a workflow name to filter the results, showing only the coverage for that specific workflow. The `--target-coverage` option takes a float percent value and allows the user to set a threshold percentage; only tasks and workflows with coverage below this threshold will be displayed. If no tasks/workflows match the specified filter, a message will be printed to inform the user. Additionally, if all optional outputs are tested, a confirmation message will be displayed. -`wdl-ci coverage --workflow-name --coverage-threshold ` +`wdl-ci coverage --workflow-name --target-coverage ` ## Lint workflows From 8b8b032cf27c1db0b39c77a980ce6344ecbb1965 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 5 Dec 2024 13:58:58 -0500 Subject: [PATCH 037/101] Remove comment and creation of set; add check for length to avoid false-y --- src/wdlci/cli/coverage.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 572d600..fe4de6d 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -52,16 +52,12 @@ def coverage_handler(kwargs): all_outputs = 0 all_tests = [] untested_tasks = {} - # Use a set to avoid duplicate entries untested_optional_outputs = [] # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter tasks_below_threshold = False workflows_below_threshold = False workflow_found = False - # Create a set of keys for the output_tests dictionary for faster lookup - output_tests_keys = set(output_tests.keys()) - # Iterate over each WDL file for wdl_file in wdl_files: workflow_tests = [] @@ -83,9 +79,9 @@ def coverage_handler(kwargs): missing_outputs = [ output_file_name_pattern.search(str(output)).group(1) for output in task.outputs - if output.name not in output_tests_keys + if output.name not in output_tests.keys() or ( - output.name in output_tests_keys + output.name in output_tests.keys() and not output_tests[output.name] ) ] @@ -97,7 +93,7 @@ def coverage_handler(kwargs): # Check if there are tests for each output for output in task.outputs: if ( - output.name in output_tests_keys + output.name in output_tests.keys() and output_tests[output.name] != [] ): task_tests.append(output.name) @@ -105,15 +101,15 @@ def coverage_handler(kwargs): all_tests.append(output.name) if ( output.type.optional - and output.name not in output_tests_keys + and output.name not in output_tests.keys() or ( - output.name in output_tests_keys + output.name in output_tests.keys() and output_tests[output.name] == [] ) ): untested_optional_outputs.append(output.name) # Print task coverage for tasks with outputs and tests - if task.outputs and task_tests: + if len(task.outputs) > 0 and len(task_tests) > 0: # Calculate and print the task coverage task_coverage = (len(task_tests) / len(task.outputs)) * 100 if threshold is None or threshold and task_coverage < threshold: From b24c357e9394fd1009842e675cf65da926774abe Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 5 Dec 2024 13:59:35 -0500 Subject: [PATCH 038/101] Clarify workflow file name is required --- src/wdlci/cli/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index 240c4d5..c5f6f0f 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -50,7 +50,7 @@ type=str, default=None, show_default=True, - help="Workflow name to filter coverage results", + help="Workflow file name to filter coverage results", ) From 50527446d95c8321c5ad7cca3f5ca0bee346a289 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 5 Dec 2024 14:10:28 -0500 Subject: [PATCH 039/101] Avoid more false-y traps; remove redundant checks if threshold is not None --- src/wdlci/cli/coverage.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index fe4de6d..6802cb3 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -112,29 +112,29 @@ def coverage_handler(kwargs): if len(task.outputs) > 0 and len(task_tests) > 0: # Calculate and print the task coverage task_coverage = (len(task_tests) / len(task.outputs)) * 100 - if threshold is None or threshold and task_coverage < threshold: + if threshold is None and task_coverage < threshold: tasks_below_threshold = True print(f"task.{task.name}: {task_coverage:.2f}%") # If there are outputs but no tests for the entire task, add the task to the untested_tasks list - elif task.outputs and not task_tests: + elif len(task.outputs) > 0 and not task_tests: if wdl_filename not in untested_tasks: untested_tasks[wdl_filename] = [] untested_tasks[wdl_filename].append(task.name) - if missing_outputs and task_tests: + if len(missing_outputs) > 0 and len(task_tests) > 0: print( f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs} in task {task.name}" ) # Print workflow coverage for tasks with outputs and tests - if workflow_tests and workflow_outputs: + if len(workflow_tests) > 0 and len(workflow_outputs) > 0: workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 - if threshold is None or threshold and workflow_coverage < threshold: + if threshold is None and workflow_coverage < threshold: print("-" * 150) workflows_below_threshold = True print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%") print("-" * 150 + "\n") # Inform the user if no workflows matched the filter - if workflow_name_filter and not workflow_found: + if workflow_name_filter is not None and not workflow_found: print(f"\nNo workflows found matching the filter: {workflow_name_filter}") sys.exit(0) # Warn the user about tasks that have no associated tests @@ -143,7 +143,7 @@ def coverage_handler(kwargs): for task in tasks: print(f"\t{task}") # Warn the user about optional outputs that are not tested - if untested_optional_outputs: + if len(untested_optional_outputs) > 0: print( f"\n[WARN]: These optional outputs are not tested: {untested_optional_outputs}" ) @@ -164,7 +164,7 @@ def coverage_handler(kwargs): print("\n✓ All workflows exceed the specified coverage threshold.") # Calculate and print the total coverage - if all_tests and all_outputs: + if len(all_tests) > 0 and len(all_outputs) > 0: total_coverage = (len(all_tests) / all_outputs) * 100 print(f"\nTotal coverage: {total_coverage:.2f}%") From 3ec8f62b3747c98680f4cb1785e2c886fedb856a Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 5 Dec 2024 14:14:29 -0500 Subject: [PATCH 040/101] Fix check for threshold > task_coverage or threshold not specified --- src/wdlci/cli/coverage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 6802cb3..a59e059 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -112,7 +112,7 @@ def coverage_handler(kwargs): if len(task.outputs) > 0 and len(task_tests) > 0: # Calculate and print the task coverage task_coverage = (len(task_tests) / len(task.outputs)) * 100 - if threshold is None and task_coverage < threshold: + if threshold is None or task_coverage < threshold: tasks_below_threshold = True print(f"task.{task.name}: {task_coverage:.2f}%") # If there are outputs but no tests for the entire task, add the task to the untested_tasks list @@ -128,7 +128,7 @@ def coverage_handler(kwargs): # Print workflow coverage for tasks with outputs and tests if len(workflow_tests) > 0 and len(workflow_outputs) > 0: workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 - if threshold is None and workflow_coverage < threshold: + if threshold is None or workflow_coverage < threshold: print("-" * 150) workflows_below_threshold = True print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%") From ac61953885773a63c915ec2daee657240fdb80f1 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 6 Dec 2024 08:15:15 -0500 Subject: [PATCH 041/101] Avoid more false-ys; add TODOs --- src/wdlci/cli/coverage.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index a59e059..fc24ee4 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -41,7 +41,7 @@ def coverage_handler(kwargs): # Load all WDL files in the directory cwd = os.getcwd() wdl_files = [] - for root_path, subfolders, filenames in os.walk(cwd): + for root_path, _, filenames in os.walk(cwd): for filename in filenames: if filename.endswith(".wdl"): wdl_files.append( @@ -92,19 +92,18 @@ def coverage_handler(kwargs): # Check if there are tests for each output for output in task.outputs: - if ( - output.name in output_tests.keys() - and output_tests[output.name] != [] + if output.name in output_tests.keys() and len( + output_tests[output.name] > 0 ): task_tests.append(output.name) workflow_tests.append(output.name) all_tests.append(output.name) - if ( + elif ( output.type.optional and output.name not in output_tests.keys() or ( output.name in output_tests.keys() - and output_tests[output.name] == [] + and len(output_tests[output.name]) == 0 ) ): untested_optional_outputs.append(output.name) @@ -116,7 +115,9 @@ def coverage_handler(kwargs): tasks_below_threshold = True print(f"task.{task.name}: {task_coverage:.2f}%") # If there are outputs but no tests for the entire task, add the task to the untested_tasks list - elif len(task.outputs) > 0 and not task_tests: + elif ( + len(task.outputs) > 0 and not task_tests + ): # Consider building this into the above so that we can incorporate untested tasks into the threshold check/statement returned to user if wdl_filename not in untested_tasks: untested_tasks[wdl_filename] = [] untested_tasks[wdl_filename].append(task.name) @@ -158,9 +159,11 @@ def coverage_handler(kwargs): # else: # print("\n✓ All optional inputs are tested") # Print a warning if any tasks or workflows are below the threshold - if not tasks_below_threshold: - print("\n✓ All tasks exceed the specified coverage threshold.") - if not workflows_below_threshold: + if tasks_below_threshold is False: + print( + "\n✓ All tasks with a non-zero number of tests exceed the specified coverage threshold." + ) ## TODO: Decide if we prefer here to specify in the print statement that untested tasks are not included, or add to the if statement to include untested tasks. expect to return if any tasks are completely untested - do see if any test has 0 coverage but also include an option to SKIP COMPLETELY UNTESTED. First step here is that untested tests DO NOT Meet the threshold --> include those in the output + if workflows_below_threshold is False: print("\n✓ All workflows exceed the specified coverage threshold.") # Calculate and print the total coverage From 5538a073f3304096aadbedd21839a8012cbf4f09 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 6 Dec 2024 17:37:54 -0500 Subject: [PATCH 042/101] Begin working through PR revisions; making holistic changes to test namespaced outputs that are linked to their respective workflows --- src/wdlci/cli/coverage.py | 277 +++++++++++++++++++++++--------------- 1 file changed, 170 insertions(+), 107 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index fc24ee4..85b3a01 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -1,12 +1,11 @@ import WDL import os import sys -import re from wdlci.config import Config from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException -output_file_name_pattern = re.compile(r"(\b\w+\b)\s*=") +# TODO: add structure of all dicts as comments for clarity once dev is complete e.g., # {wdl_file_name: [task_name]} def coverage_handler(kwargs): @@ -22,23 +21,60 @@ def coverage_handler(kwargs): Config.load(kwargs) # Get the config instance config = Config.instance() + + # Initialize dictionary of necessary variables to compute coverage; aims to mimic config file structure + coverage_state = { + "untested_tasks": {}, + "total_output_count": 0, + "all_tests_list": [], + } + # print(config.__dict__) + # print(config._file) + # print(config._file.__dict__.keys()) + # print(config._file.workflows.keys()) + # for workflow in config.file.workflows.keys(): + # print(f"Workflow: {workflow}") + # doc = WDL.load(workflow) + # task = doc.tasks[0] + # print(f"Task: {task.__dict__.keys()}") + # print(f"Task: {task.name}") + # output = task.outputs[0] + # print(output.__dict__.keys()) + # print(f"Output: {output.name}") + # tests = config.file.workflows[workflow].tasks[task.name].tests + # for test in tests: + # print(test) + + # raise SystemExit() + + #### Across all wdl files in the config, the set of outputs and their associated tasks #### + + # {worfklow_name: {output_name: [tests_associated_without_output]} output_tests = {} + # Iterate over each workflow in the config file - for _, workflow_config in config.file.workflows.items(): + for workflow_name, workflow_config in config.file.workflows.items(): # Iterate over each task in the workflow - for _, task_config in workflow_config.tasks.items(): - # Iterate over each test in the task + for task_name, task_config in workflow_config.tasks.items(): + # Iterate over each test in each task (test has two nested dicts - inputs and output_tests) for test_config in task_config.tests: - # Iterate over each output in the test + # Iterate over each output and associated tests for output, test in test_config.output_tests.items(): - # Add the output to the dictionary if it is not already present - if output not in output_tests: - output_tests[output] = [] + # Add the output to the dictionary for the current workflow if it is not already present + if workflow_name not in output_tests: + output_tests[workflow_name] = {} + if task_name not in output_tests[workflow_name]: + output_tests[workflow_name][task_name] = {} + if output not in output_tests[workflow_name][task_name]: + output_tests[workflow_name][task_name][output] = [] # Check if 'test_tasks' key exists in the test dictionary and is not empty and append to output_tests if present and not empty if "test_tasks" in test and test["test_tasks"]: - output_tests[output].append(test["test_tasks"]) + output_tests[workflow_name][task_name][output].append( + test["test_tasks"] + ) # Load all WDL files in the directory + ## TODO: Since this is used in generate_config, let's make it a function that both coverage and generate_config import and use to discover wdl files in a directory (a good place to store it is in utils). cwd = os.getcwd() wdl_files = [] for root_path, _, filenames in os.walk(cwd): @@ -47,11 +83,12 @@ def coverage_handler(kwargs): wdl_files.append( os.path.relpath(os.path.join(root_path, filename), cwd) ) - # Initialize counters/lists - all_outputs = 0 - all_tests = [] - untested_tasks = {} + # total_output_count = 0 + # all_tests = [] + # # For each wdl file, the set of tasks that do not have any tests associated + # # {wdl_file_name: [task_name]} + # untested_tasks = {} untested_optional_outputs = [] # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter tasks_below_threshold = False @@ -60,116 +97,142 @@ def coverage_handler(kwargs): # Iterate over each WDL file for wdl_file in wdl_files: - workflow_tests = [] - workflow_outputs = 0 + workflow_tests_list = [] + workflow_output_count = 0 # Strip everything except workflow name wdl_filename = wdl_file.split("/")[-1] # Load the WDL document doc = WDL.load(wdl_file) # If workflow_name_filter is provided, skip all other workflows - if workflow_name_filter and workflow_name_filter not in wdl_filename: + if ( + workflow_name_filter is not None + and workflow_name_filter not in wdl_filename + ): continue workflow_found = True # Iterate over each task in the WDL document for task in doc.tasks: - task_tests = [] - # Create a list of outputs that are present in the task/worfklow but not in the config JSON using the output_tests dictionary - missing_outputs = [ - output_file_name_pattern.search(str(output)).group(1) - for output in task.outputs - if output.name not in output_tests.keys() - or ( - output.name in output_tests.keys() - and not output_tests[output.name] - ) - ] - - # Count the number of outputs for the task - all_outputs += len(task.outputs) - workflow_outputs += len(task.outputs) - - # Check if there are tests for each output - for output in task.outputs: - if output.name in output_tests.keys() and len( - output_tests[output.name] > 0 - ): - task_tests.append(output.name) - workflow_tests.append(output.name) - all_tests.append(output.name) - elif ( - output.type.optional - and output.name not in output_tests.keys() - or ( - output.name in output_tests.keys() - and len(output_tests[output.name]) == 0 + try: + task_tests = config._file.workflows[wdl_file].tasks[task.name].tests + # print([test.__dict__ for test in task_tests]) + # we are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # TODO: consider that different behaviour may be desired (eg for tasks with optional inputs (and/or outputs?), we probably want to confirm that there are at least 2 tests for each output: one where the optional input is defined, one where it isn't + tested_outputs = list( + set( + [ + output_name + for test in task_tests + for output_name in test.output_tests.keys() + ] ) - ): - untested_optional_outputs.append(output.name) - # Print task coverage for tasks with outputs and tests - if len(task.outputs) > 0 and len(task_tests) > 0: - # Calculate and print the task coverage - task_coverage = (len(task_tests) / len(task.outputs)) * 100 - if threshold is None or task_coverage < threshold: - tasks_below_threshold = True - print(f"task.{task.name}: {task_coverage:.2f}%") - # If there are outputs but no tests for the entire task, add the task to the untested_tasks list - elif ( - len(task.outputs) > 0 and not task_tests - ): # Consider building this into the above so that we can incorporate untested tasks into the threshold check/statement returned to user - if wdl_filename not in untested_tasks: - untested_tasks[wdl_filename] = [] - untested_tasks[wdl_filename].append(task.name) - if len(missing_outputs) > 0 and len(task_tests) > 0: - print( - f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs} in task {task.name}" ) + all_task_outputs = [output.name for output in task.outputs] + missing_outputs = [ + output_name + for output_name in all_task_outputs + if output_name not in tested_outputs + ] + print(f"Tested outputs: {tested_outputs}") + print(f"All task outputs: {all_task_outputs}") + print(f"Missing outputs: {missing_outputs}") + except KeyError: + # Create a list of outputs that are present in the task/worfklow but not in the config JSON using the output_tests dictionary + outputs_present_in_workflow_absent_in_config = [ + output.name + for output in task.outputs + if output.name not in output_tests.keys() + or output.name in output_tests.keys() + and not output_tests[output.name] + ] + print(outputs_present_in_workflow_absent_in_config) + raise SystemExit + + ## TODO: haven't worked on the below yet - not to say the above is complete, but it's moving in the right direction + + # Count the number of outputs for the task + # coverage_state["total_output_count"] += len(task.outputs) + # workflow_output_count += len(task.outputs) - # Print workflow coverage for tasks with outputs and tests - if len(workflow_tests) > 0 and len(workflow_outputs) > 0: - workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 - if threshold is None or workflow_coverage < threshold: - print("-" * 150) - workflows_below_threshold = True - print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%") - print("-" * 150 + "\n") - # Inform the user if no workflows matched the filter - if workflow_name_filter is not None and not workflow_found: - print(f"\nNo workflows found matching the filter: {workflow_name_filter}") - sys.exit(0) - # Warn the user about tasks that have no associated tests - for workflow, tasks in untested_tasks.items(): - print(f"For {workflow}, these tasks are untested:") - for task in tasks: - print(f"\t{task}") - # Warn the user about optional outputs that are not tested - if len(untested_optional_outputs) > 0: - print( - f"\n[WARN]: These optional outputs are not tested: {untested_optional_outputs}" - ) - else: - print("\n✓ All optional outputs are tested") - ## TODO: Measure is if there is a test that covered running that task with the optional input and without it - # # Warn the user about optional inputs that are not tested - # if untested_optional_inputs: + # # Check if there are tests for each output + # for output in task.outputs: + # if output.name in output_tests.keys() and len(output_tests) > 0: + # task_tests.append(output.name) + # workflow_tests.append(output.name) + # all_tests.append(output.name) + # elif ( + # output.type.optional + # and output.name not in output_tests.keys() + # or ( + # output.name in output_tests.keys() + # and len(output_tests[output.name]) == 0 + # ) + # ): + # untested_optional_outputs.append(output.name) + # # Print task coverage for tasks with outputs and tests + # if len(task.outputs) > 0 and len(task_tests) > 0: + # # Calculate and print the task coverage + # task_coverage = (len(task_tests) / len(task.outputs)) * 100 + # if threshold is None or task_coverage < threshold: + # tasks_below_threshold = True + # print(f"task.{task.name}: {task_coverage:.2f}%") + # # If there are outputs but no tests for the entire task, add the task to the untested_tasks list + # elif ( + # len(task.outputs) > 0 and not task_tests + # ): # Consider building this into the above so that we can incorporate untested tasks into the threshold check/statement returned to user + # if wdl_filename not in untested_tasks: + # untested_tasks[wdl_filename] = [] + # untested_tasks[wdl_filename].append(task.name) + # if len(missing_outputs) > 0 and len(task_tests) > 0: + # print( + # f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs} in task {task.name}" + # ) + + # # Print workflow coverage for tasks with outputs and tests + # if len(workflow_tests) > 0 and workflow_outputs > 0: + # workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 + # if threshold is None or workflow_coverage < threshold: + # print("-" * 150) + # workflows_below_threshold = True + # print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%") + # print("-" * 150 + "\n") + # # Inform the user if no workflows matched the filter + # if workflow_name_filter is not None and not workflow_found: + # print(f"\nNo workflows found matching the filter: {workflow_name_filter}") + # sys.exit(0) + # # Warn the user about tasks that have no associated tests + # for workflow, tasks in untested_tasks.items(): + # print(f"For {workflow}, these tasks are untested:") + # for task in tasks: + # print(f"\t{task}") + # # Warn the user about optional outputs that are not tested + # if len(untested_optional_outputs) > 0: # print( - # f"\n[WARN]: These optional inputs are not used in any tests: {untested_optional_inputs}" + # f"\n[WARN]: These optional outputs are not tested: {untested_optional_outputs}" # ) # else: - # print("\n✓ All optional inputs are tested") - # Print a warning if any tasks or workflows are below the threshold - if tasks_below_threshold is False: - print( - "\n✓ All tasks with a non-zero number of tests exceed the specified coverage threshold." - ) ## TODO: Decide if we prefer here to specify in the print statement that untested tasks are not included, or add to the if statement to include untested tasks. expect to return if any tasks are completely untested - do see if any test has 0 coverage but also include an option to SKIP COMPLETELY UNTESTED. First step here is that untested tests DO NOT Meet the threshold --> include those in the output - if workflows_below_threshold is False: - print("\n✓ All workflows exceed the specified coverage threshold.") - - # Calculate and print the total coverage - if len(all_tests) > 0 and len(all_outputs) > 0: - total_coverage = (len(all_tests) / all_outputs) * 100 - print(f"\nTotal coverage: {total_coverage:.2f}%") + # print("\n✓ All optional outputs are tested") + # ## TODO: Measure is if there is a test that covered running that task with the optional input and without it + # # # Warn the user about optional inputs that are not tested + # # if untested_optional_inputs: + # # print( + # # f"\n[WARN]: These optional inputs are not used in any tests: {untested_optional_inputs}" + # # ) + # # else: + # # print("\n✓ All optional inputs are tested") + # # Print a warning if any tasks or workflows are below the threshold + # if tasks_below_threshold is False: + # print( + # "\n✓ All tasks with a non-zero number of tests exceed the specified coverage threshold." + # ) ## TODO: User would expect to return if any tasks are completely untested - do see if any test has 0 coverage but also include an option to SKIP COMPLETELY UNTESTED. First step here is that untested tests DO NOT Meet the threshold --> include those in the output; add option to skip seeing completely untested tasks + # if workflows_below_threshold is False: + # print("\n✓ All workflows exceed the specified coverage threshold.") + + # # Calculate and print the total coverage + # if len(all_tests) > 0 and total_output_count > 0: + # total_coverage = (len(all_tests) / total_output_count) * 100 + # print(f"\nTotal coverage: {total_coverage:.2f}%") except WdlTestCliExitException as e: print(f"exiting with code {e.exit_code}, message: {e.message}") From 262f0ea68a4e57d76fb269acee3a0a5d4967701f Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 9 Dec 2024 09:47:11 -0500 Subject: [PATCH 043/101] Add function for repeated code and import/implement --- src/wdlci/cli/coverage.py | 11 ++--------- src/wdlci/cli/generate_config.py | 10 ++-------- src/wdlci/utils/initialize_worklows_and_tasks.py | 13 +++++++++++++ 3 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 src/wdlci/utils/initialize_worklows_and_tasks.py diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 85b3a01..80c28f1 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -4,6 +4,7 @@ from wdlci.config import Config from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException +from wdlci.utils.initialize_worklows_and_tasks import find_wdl_files # TODO: add structure of all dicts as comments for clarity once dev is complete e.g., # {wdl_file_name: [task_name]} @@ -74,15 +75,7 @@ def coverage_handler(kwargs): ) # Load all WDL files in the directory - ## TODO: Since this is used in generate_config, let's make it a function that both coverage and generate_config import and use to discover wdl files in a directory (a good place to store it is in utils). - cwd = os.getcwd() - wdl_files = [] - for root_path, _, filenames in os.walk(cwd): - for filename in filenames: - if filename.endswith(".wdl"): - wdl_files.append( - os.path.relpath(os.path.join(root_path, filename), cwd) - ) + wdl_files = find_wdl_files() # Initialize counters/lists # total_output_count = 0 # all_tests = [] diff --git a/src/wdlci/cli/generate_config.py b/src/wdlci/cli/generate_config.py index 7dee502..db3ff2c 100644 --- a/src/wdlci/cli/generate_config.py +++ b/src/wdlci/cli/generate_config.py @@ -2,6 +2,7 @@ import WDL from wdlci.config.config_file import WorkflowConfig, WorkflowTaskConfig from wdlci.constants import CONFIG_JSON +from wdlci.utils.initialize_worklows_and_tasks import find_wdl_files import os.path @@ -19,14 +20,7 @@ def generate_config_handler(kwargs, update_task_digests=False): config = Config.instance() # Initialize workflows and tasks - cwd = os.getcwd() - wdl_files = [] - for root_path, subfolders, filenames in os.walk(cwd): - for filename in filenames: - if filename.endswith(".wdl"): - wdl_files.append( - os.path.relpath(os.path.join(root_path, filename), cwd) - ) + wdl_files = find_wdl_files() for workflow_path in wdl_files: if workflow_path not in config.file.workflows: diff --git a/src/wdlci/utils/initialize_worklows_and_tasks.py b/src/wdlci/utils/initialize_worklows_and_tasks.py new file mode 100644 index 0000000..6af5cdd --- /dev/null +++ b/src/wdlci/utils/initialize_worklows_and_tasks.py @@ -0,0 +1,13 @@ +import os.path + + +def find_wdl_files(): + cwd = os.getcwd() + wdl_files = [] + for root_path, _, filenames in os.walk(cwd): + for filename in filenames: + if filename.endswith(".wdl"): + wdl_files.append( + os.path.relpath(os.path.join(root_path, filename), cwd) + ) + return wdl_files From fbf26450715c6263d63671e9947cdf0f3f8c2d0d Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Tue, 10 Dec 2024 17:15:13 -0500 Subject: [PATCH 044/101] Clarify how to provide workflow name based on changes to coverage command --- src/wdlci/cli/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index c5f6f0f..3708bfd 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -50,7 +50,7 @@ type=str, default=None, show_default=True, - help="Workflow file name to filter coverage results", + help="Name of the workflow to filter coverage results (not file name)", ) From 7c8bc767a8bc83ca906df299f64dcc811a823afa Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Tue, 10 Dec 2024 17:17:01 -0500 Subject: [PATCH 045/101] Get command in a functional state based on PR revisions; UX changes and testing to follow --- src/wdlci/cli/coverage.py | 419 ++++++++++++++++++++++++-------------- 1 file changed, 271 insertions(+), 148 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 80c28f1..cec18e2 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -20,15 +20,8 @@ def coverage_handler(kwargs): try: # Load the config file Config.load(kwargs) - # Get the config instance config = Config.instance() - # Initialize dictionary of necessary variables to compute coverage; aims to mimic config file structure - coverage_state = { - "untested_tasks": {}, - "total_output_count": 0, - "all_tests_list": [], - } # print(config.__dict__) # print(config._file) # print(config._file.__dict__.keys()) @@ -48,10 +41,19 @@ def coverage_handler(kwargs): # raise SystemExit() - #### Across all wdl files in the config, the set of outputs and their associated tasks #### + # Initialize dictionary with necessary variables to help compute coverage + coverage_state = { + "untested_workflows": [], + "untested_tasks": {}, + "untested_outputs": {}, + "total_output_count": 0, + "all_tests_list": [], + } + + #### Across all workflows found in the config, the set of outputs and their associated tasks #### - # {worfklow_name: {output_name: [tests_associated_without_output]} - output_tests = {} + # {worfklow_name: {task_name: {output_name: [[tests_associated_without_output]]} + config_output_tests_dict = {} # Iterate over each workflow in the config file for workflow_name, workflow_config in config.file.workflows.items(): @@ -61,144 +63,244 @@ def coverage_handler(kwargs): for test_config in task_config.tests: # Iterate over each output and associated tests for output, test in test_config.output_tests.items(): - # Add the output to the dictionary for the current workflow if it is not already present - if workflow_name not in output_tests: - output_tests[workflow_name] = {} - if task_name not in output_tests[workflow_name]: - output_tests[workflow_name][task_name] = {} - if output not in output_tests[workflow_name][task_name]: - output_tests[workflow_name][task_name][output] = [] - # Check if 'test_tasks' key exists in the test dictionary and is not empty and append to output_tests if present and not empty + # Initialize the dictionary with the workflow name if it doesn't exist + if workflow_name not in config_output_tests_dict: + config_output_tests_dict[workflow_name] = {} + # Initialize the nested dictionary within the workflow with the task name if it doesn't exist + if task_name not in config_output_tests_dict[workflow_name]: + config_output_tests_dict[workflow_name][task_name] = {} + # Initialize nested dictionary within the task dictionary with the output name if it doesn't exist + if ( + output + not in config_output_tests_dict[workflow_name][task_name] + ): + config_output_tests_dict[workflow_name][task_name][ + output + ] = [] + # Check if 'test_tasks' key exists in the test dictionary and is not empty; if so, append the test_tasks to the output list if "test_tasks" in test and test["test_tasks"]: - output_tests[workflow_name][task_name][output].append( - test["test_tasks"] - ) + config_output_tests_dict[workflow_name][task_name][ + output + ].append(test["test_tasks"]) + + #### Now that we've constructed the dictionary of outputs and their associated tests using the config file, we can compare this to the WDL files #### # Load all WDL files in the directory wdl_files = find_wdl_files() - # Initialize counters/lists - # total_output_count = 0 - # all_tests = [] - # # For each wdl file, the set of tasks that do not have any tests associated - # # {wdl_file_name: [task_name]} - # untested_tasks = {} + ## TODO - do we need/want the below? untested_optional_outputs = [] # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter tasks_below_threshold = False - workflows_below_threshold = False workflow_found = False + workflows_below_threshold = False # Iterate over each WDL file for wdl_file in wdl_files: workflow_tests_list = [] workflow_output_count = 0 - # Strip everything except workflow name - wdl_filename = wdl_file.split("/")[-1] # Load the WDL document doc = WDL.load(wdl_file) + # print(doc.workflow) + ## if doc.workflow or len(doc.tasks) > 0: + # if len(doc.tasks) > 0: + # print(f"Tasks in {wdl_file}") + # print(task.__dict__ for task in doc.tasks) + # else: + # print(f"{wdl_file} has no tasks") + # Check if the WDL document has >0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for tasks >0 misses parent workflows that just import and call other tasks/workflows. TBD if we want to include this, but ultimately, if there are no tasks or a workflow at all, we skip the WDL file + if len(doc.tasks) > 0 or doc.workflow: + # If workflow_name_filter is provided, skip all other workflows + if doc.workflow and workflow_name_filter: + if workflow_name_filter not in doc.workflow.name: + continue + workflow_found = True - # If workflow_name_filter is provided, skip all other workflows - if ( - workflow_name_filter is not None - and workflow_name_filter not in wdl_filename - ): - continue - workflow_found = True - - # Iterate over each task in the WDL document - for task in doc.tasks: - try: - task_tests = config._file.workflows[wdl_file].tasks[task.name].tests - # print([test.__dict__ for test in task_tests]) - # we are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # TODO: consider that different behaviour may be desired (eg for tasks with optional inputs (and/or outputs?), we probably want to confirm that there are at least 2 tests for each output: one where the optional input is defined, one where it isn't - tested_outputs = list( - set( - [ - output_name - for test in task_tests - for output_name in test.output_tests.keys() - ] + # Iterate over each task in the WDL document + for task in doc.tasks: + # Set the workflow name as the workflow name in the WDL file if it exists, otherwise set it to the task name as this handles the case where there is no workflow block in the WDL file (i.e., it's a single task WDL file) + workflow_name = doc.workflow.name if doc.workflow else task.name + # Add to counters for total output count and workflow output count + coverage_state["total_output_count"] += len(task.outputs) + workflow_output_count += len(task.outputs) + try: + # Create a list of dictionaries for each set of task tests in our config file + task_tests = ( + config._file.workflows[wdl_file].tasks[task.name].tests ) - ) - all_task_outputs = [output.name for output in task.outputs] - missing_outputs = [ - output_name - for output_name in all_task_outputs - if output_name not in tested_outputs - ] - print(f"Tested outputs: {tested_outputs}") - print(f"All task outputs: {all_task_outputs}") - print(f"Missing outputs: {missing_outputs}") - except KeyError: - # Create a list of outputs that are present in the task/worfklow but not in the config JSON using the output_tests dictionary - outputs_present_in_workflow_absent_in_config = [ - output.name - for output in task.outputs - if output.name not in output_tests.keys() - or output.name in output_tests.keys() - and not output_tests[output.name] - ] - print(outputs_present_in_workflow_absent_in_config) - raise SystemExit - - ## TODO: haven't worked on the below yet - not to say the above is complete, but it's moving in the right direction - - # Count the number of outputs for the task - # coverage_state["total_output_count"] += len(task.outputs) - # workflow_output_count += len(task.outputs) - - # # Check if there are tests for each output - # for output in task.outputs: - # if output.name in output_tests.keys() and len(output_tests) > 0: - # task_tests.append(output.name) - # workflow_tests.append(output.name) - # all_tests.append(output.name) - # elif ( - # output.type.optional - # and output.name not in output_tests.keys() - # or ( - # output.name in output_tests.keys() - # and len(output_tests[output.name]) == 0 - # ) - # ): - # untested_optional_outputs.append(output.name) - # # Print task coverage for tasks with outputs and tests - # if len(task.outputs) > 0 and len(task_tests) > 0: - # # Calculate and print the task coverage - # task_coverage = (len(task_tests) / len(task.outputs)) * 100 - # if threshold is None or task_coverage < threshold: - # tasks_below_threshold = True - # print(f"task.{task.name}: {task_coverage:.2f}%") - # # If there are outputs but no tests for the entire task, add the task to the untested_tasks list + + # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # TODO: consider that different behaviour may be desired (eg for tasks with optional inputs (and/or outputs?), we probably want to confirm that there are at least 2 tests for each output: one where the optional input is defined, one where it isn't + + # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary; duplicates are removed + tested_outputs = list( + set( + [ + output_name + for test in task_tests + for output_name in test.output_tests.keys() + ] + ) + ) + # Create a list of all the outputs that are present in the tas + all_task_outputs = [output.name for output in task.outputs] + + # Create a list of outputs that are present in the task but not in the config file + missing_outputs = [ + output_name + for output_name in all_task_outputs + if output_name not in tested_outputs + ] + + # Add tested outputs to workflow_tests_list and all_tests_list + workflow_tests_list.extend(tested_outputs) + coverage_state["all_tests_list"].extend(tested_outputs) + + # Add missing outputs to the coverage_state dictionary under the structure {workflow_name: {task_name: {output_name: [missing_outputs]}}} + if workflow_name not in coverage_state["untested_outputs"]: + coverage_state["untested_outputs"][workflow_name] = {} + if ( + task.name + not in coverage_state["untested_outputs"][workflow_name] + ): + coverage_state["untested_outputs"][workflow_name][ + task.name + ] = missing_outputs + + # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + optional_inputs = [ + input.name for input in task.inputs if input.type.optional + ] + if len(optional_inputs) > 0: + outputs_missing_tests = [] + for output_name in all_task_outputs: + tests_with_optional_inputs = [ + test + for test in task_tests + if any( + optional_input in test.inputs + for optional_input in optional_inputs + ) + ] + tests_without_optional_inputs = [ + test + for test in task_tests + if all( + optional_input not in test.inputs + for optional_input in optional_inputs + ) + ] + if ( + len(tests_with_optional_inputs) < 1 + or len(tests_without_optional_inputs) < 1 + ): + outputs_missing_tests.append(output_name) + if len(outputs_missing_tests) > 0: + print( + f"\n\t[WARN]: Outputs {outputs_missing_tests} in task {task.name} do not have tests with and without optional inputs (optional inputs: {optional_inputs})." + ) + # print(f"{task.name}") + # print(f"Tested outputs: {tested_outputs}") + # print(f"All task outputs: {all_task_outputs}") + # print(f"Missing outputs: {missing_outputs}") + # Catch the case where tasks are completely absent from the config + except KeyError: + # Initialize workflow in coverage state[untested_tasks] dict if there is a workflow in the WDL file but no tests in the config file + if workflow_name not in coverage_state["untested_tasks"]: + coverage_state["untested_tasks"][workflow_name] = [] + # Create a list of tasks associated with the respective workflow that are not present in the config file + coverage_state["untested_tasks"][workflow_name].append( + task.name + ) + # If there are outputs, untested tasks, and tests for the task, calculate the task coverage + if ( + len(task.outputs) > 0 + and len(coverage_state["untested_outputs"]) > 0 + ): + if len(task_tests) > 0: + # Calculate and print the task coverage + task_coverage = ( + len(tested_outputs) / len(task.outputs) + ) * 100 + if threshold is None or task_coverage < threshold: + tasks_below_threshold = True + print(f"\ntask.{task.name}: {task_coverage:.2f}%") + + # Warn the user about outputs that are not tested + if workflow_name in coverage_state["untested_outputs"]: + if ( + task.name + in coverage_state["untested_outputs"][workflow_name] + ): + missing_outputs = coverage_state[ + "untested_outputs" + ][workflow_name][task.name] + if len(missing_outputs) > 0: + # Filter out tasks with no missing outputs + filtered_missing_outputs = { + k: v + for k, v in coverage_state[ + "untested_outputs" + ][workflow_name].items() + if v + } + if filtered_missing_outputs: + missing_outputs_list = [ + item + for sublist in filtered_missing_outputs.values() + for item in sublist + ] + print( + f"\n\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs_list} in task {task.name}" + ) + + # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list + # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows + if ( + workflow_output_count > 0 + and len(workflow_tests_list) > 0 + and doc.workflow + and doc.workflow.name + ): + workflow_coverage = ( + len(workflow_tests_list) / workflow_output_count + ) * 100 + if threshold is None or workflow_coverage < threshold: + workflows_below_threshold = True + print("-" * 150) + print( + f"Workflow: {doc.workflow.name}: {workflow_coverage:.2f}%" + ) + elif ( + workflow_output_count == 0 + or len(workflow_tests_list) == 0 + and doc.workflow + and doc.workflow.name + ): + if doc.workflow.name not in coverage_state["untested_workflows"]: + coverage_state["untested_workflows"].append(doc.workflow.name) + + # Inform the user if no workflows matched the filter + if workflow_name_filter and not workflow_found: + print(f"\nNo workflows found matching the filter: {workflow_name_filter}") + sys.exit(0) + + # Check if there are tests for each output + # for output in task.outputs: + # if output.name in output_tests.keys() and len(output_tests) > 0: + # task_tests.append(output.name) + # workflow_tests.append(output.name) + # all_tests.append(output.name) # elif ( - # len(task.outputs) > 0 and not task_tests - # ): # Consider building this into the above so that we can incorporate untested tasks into the threshold check/statement returned to user - # if wdl_filename not in untested_tasks: - # untested_tasks[wdl_filename] = [] - # untested_tasks[wdl_filename].append(task.name) - # if len(missing_outputs) > 0 and len(task_tests) > 0: - # print( - # f"\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs} in task {task.name}" + # output.type.optional + # and output.name not in output_tests.keys() + # or ( + # output.name in output_tests.keys() + # and len(output_tests[output.name]) == 0 # ) + # ): + # untested_optional_outputs.append(output.name) - # # Print workflow coverage for tasks with outputs and tests - # if len(workflow_tests) > 0 and workflow_outputs > 0: - # workflow_coverage = (len(workflow_tests) / workflow_outputs) * 100 - # if threshold is None or workflow_coverage < threshold: - # print("-" * 150) - # workflows_below_threshold = True - # print(f"workflow: {wdl_filename}: {workflow_coverage:.2f}%") - # print("-" * 150 + "\n") - # # Inform the user if no workflows matched the filter - # if workflow_name_filter is not None and not workflow_found: - # print(f"\nNo workflows found matching the filter: {workflow_name_filter}") - # sys.exit(0) - # # Warn the user about tasks that have no associated tests - # for workflow, tasks in untested_tasks.items(): - # print(f"For {workflow}, these tasks are untested:") - # for task in tasks: - # print(f"\t{task}") + # ) # # Warn the user about optional outputs that are not tested # if len(untested_optional_outputs) > 0: # print( @@ -206,26 +308,47 @@ def coverage_handler(kwargs): # ) # else: # print("\n✓ All optional outputs are tested") - # ## TODO: Measure is if there is a test that covered running that task with the optional input and without it - # # # Warn the user about optional inputs that are not tested - # # if untested_optional_inputs: - # # print( - # # f"\n[WARN]: These optional inputs are not used in any tests: {untested_optional_inputs}" - # # ) - # # else: - # # print("\n✓ All optional inputs are tested") - # # Print a warning if any tasks or workflows are below the threshold - # if tasks_below_threshold is False: - # print( - # "\n✓ All tasks with a non-zero number of tests exceed the specified coverage threshold." - # ) ## TODO: User would expect to return if any tasks are completely untested - do see if any test has 0 coverage but also include an option to SKIP COMPLETELY UNTESTED. First step here is that untested tests DO NOT Meet the threshold --> include those in the output; add option to skip seeing completely untested tasks - # if workflows_below_threshold is False: - # print("\n✓ All workflows exceed the specified coverage threshold.") - - # # Calculate and print the total coverage - # if len(all_tests) > 0 and total_output_count > 0: - # total_coverage = (len(all_tests) / total_output_count) * 100 - # print(f"\nTotal coverage: {total_coverage:.2f}%") + + if ( + tasks_below_threshold is False + and len(coverage_state["untested_tasks"]) == 0 + ): + print("\n✓ All tasks exceed the specified coverage threshold.") + elif len(coverage_state["untested_tasks"]) > 0: + print("\n[WARN]: The following tasks have no tests:") + for workflow, tasks in coverage_state["untested_tasks"].items(): + for task in tasks: + print(f"\t{workflow}.{task}") + + # Check if any workflows are below the threshold and there are no untested workflows; if so return to the user that all workflows exceed the threshold + if ( + workflows_below_threshold is False + and len(coverage_state["untested_workflows"]) == 0 + ): + print("\n✓ All workflows exceed the specified coverage threshold.") + + # If there are any workflows that are below the threshold, warn the user. Include a check for the workflow_name_filter to ensure that only the specified workflow is printed if a filter is provided + else: + if ( + workflow_name_filter in coverage_state["untested_workflows"] + or workflow_name_filter is None + and len(coverage_state["untested_workflows"]) > 0 + ): + print("\n[WARN]: The following workflows have no tests:") + for workflow in coverage_state["untested_workflows"]: + print(f"\t{workflow}") + + # Calculate and print the total coverage + if ( + len(coverage_state["all_tests_list"]) > 0 + and coverage_state["total_output_count"] > 0 + and not workflow_name_filter + ): + total_coverage = ( + len(coverage_state["all_tests_list"]) + / coverage_state["total_output_count"] + ) * 100 + print(f"\nTotal coverage: {total_coverage:.2f}%") except WdlTestCliExitException as e: print(f"exiting with code {e.exit_code}, message: {e.message}") From f1806291fe83805df0138370cd5fd788e120b1b2 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Tue, 10 Dec 2024 17:18:29 -0500 Subject: [PATCH 046/101] Remove some dev code and a TODO --- src/wdlci/cli/coverage.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index cec18e2..81697b9 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -22,25 +22,6 @@ def coverage_handler(kwargs): Config.load(kwargs) config = Config.instance() - # print(config.__dict__) - # print(config._file) - # print(config._file.__dict__.keys()) - # print(config._file.workflows.keys()) - # for workflow in config.file.workflows.keys(): - # print(f"Workflow: {workflow}") - # doc = WDL.load(workflow) - # task = doc.tasks[0] - # print(f"Task: {task.__dict__.keys()}") - # print(f"Task: {task.name}") - # output = task.outputs[0] - # print(output.__dict__.keys()) - # print(f"Output: {output.name}") - # tests = config.file.workflows[workflow].tasks[task.name].tests - # for test in tests: - # print(test) - - # raise SystemExit() - # Initialize dictionary with necessary variables to help compute coverage coverage_state = { "untested_workflows": [], @@ -100,13 +81,7 @@ def coverage_handler(kwargs): workflow_output_count = 0 # Load the WDL document doc = WDL.load(wdl_file) - # print(doc.workflow) - ## if doc.workflow or len(doc.tasks) > 0: - # if len(doc.tasks) > 0: - # print(f"Tasks in {wdl_file}") - # print(task.__dict__ for task in doc.tasks) - # else: - # print(f"{wdl_file} has no tasks") + # Check if the WDL document has >0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for tasks >0 misses parent workflows that just import and call other tasks/workflows. TBD if we want to include this, but ultimately, if there are no tasks or a workflow at all, we skip the WDL file if len(doc.tasks) > 0 or doc.workflow: # If workflow_name_filter is provided, skip all other workflows @@ -129,7 +104,6 @@ def coverage_handler(kwargs): ) # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # TODO: consider that different behaviour may be desired (eg for tasks with optional inputs (and/or outputs?), we probably want to confirm that there are at least 2 tests for each output: one where the optional input is defined, one where it isn't # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary; duplicates are removed tested_outputs = list( From 3647f99b7be4ec996abca7fe296c97857e83eda5 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 12 Dec 2024 15:54:17 -0500 Subject: [PATCH 047/101] Add TODOs for more flags for the coverage command --- src/wdlci/cli/__main__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index 3708bfd..4a56d1a 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -124,6 +124,12 @@ def cleanup(**kwargs): @main.command @target_coverage @workflow_name +# TODO: Add options for minimalist output; e.g., maybe hide warning output for: +# tests that don't have tests both excluding and including optional inputs +# list of outputs that are not tested for each task +# skipped workflows +# workflows that have outputs but no tests +# TODO: Add option to consider totally untested tasks/outputs as 0% coverage or ignore them def coverage(**kwargs): """Outputs percent coverage for each task and output, and which tasks/outputs have no associated tests""" From 2b88c4211db057e41010d60e4eb6ba20edc73784 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 12 Dec 2024 16:51:04 -0500 Subject: [PATCH 048/101] Add working version but still in dev --- src/wdlci/cli/coverage.py | 466 ++++++++++++++++++++++---------------- 1 file changed, 267 insertions(+), 199 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 81697b9..9f919f2 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -6,7 +6,22 @@ from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException from wdlci.utils.initialize_worklows_and_tasks import find_wdl_files -# TODO: add structure of all dicts as comments for clarity once dev is complete e.g., # {wdl_file_name: [task_name]} +# Initialize dictionary with necessary variables to compute coverage +coverage_summary = { + "untested_workflows": [], + # {workflow_name: [task_name]} + "untested_tasks": {}, + # {workflow_name: {task_name: [output_name]}} + "untested_outputs": {}, + # {workflow_name: {task_name: [output_name]}} + "untested_outputs_with_optional_inputs": {}, + ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; case where outputs in different workflows or tasks that share a name (e.g. vcf as an output from glnexus and deepvariant + # {workflow_name: {task_name: [output_name]}} + "tested_outputs_dict": {}, + "total_output_count": 0, + "all_tests_list": [], + "skipped_workflows": [], +} def coverage_handler(kwargs): @@ -17,59 +32,17 @@ def coverage_handler(kwargs): print(f"Workflow name filter: {workflow_name_filter}\n") else: print("Workflow name filter: None\n") + print("┍━━━━━━━━━━━━━┑") + print("│ Coverage │") + print("┕━━━━━━━━━━━━━┙") try: # Load the config file Config.load(kwargs) config = Config.instance() - # Initialize dictionary with necessary variables to help compute coverage - coverage_state = { - "untested_workflows": [], - "untested_tasks": {}, - "untested_outputs": {}, - "total_output_count": 0, - "all_tests_list": [], - } - - #### Across all workflows found in the config, the set of outputs and their associated tasks #### - - # {worfklow_name: {task_name: {output_name: [[tests_associated_without_output]]} - config_output_tests_dict = {} - - # Iterate over each workflow in the config file - for workflow_name, workflow_config in config.file.workflows.items(): - # Iterate over each task in the workflow - for task_name, task_config in workflow_config.tasks.items(): - # Iterate over each test in each task (test has two nested dicts - inputs and output_tests) - for test_config in task_config.tests: - # Iterate over each output and associated tests - for output, test in test_config.output_tests.items(): - # Initialize the dictionary with the workflow name if it doesn't exist - if workflow_name not in config_output_tests_dict: - config_output_tests_dict[workflow_name] = {} - # Initialize the nested dictionary within the workflow with the task name if it doesn't exist - if task_name not in config_output_tests_dict[workflow_name]: - config_output_tests_dict[workflow_name][task_name] = {} - # Initialize nested dictionary within the task dictionary with the output name if it doesn't exist - if ( - output - not in config_output_tests_dict[workflow_name][task_name] - ): - config_output_tests_dict[workflow_name][task_name][ - output - ] = [] - # Check if 'test_tasks' key exists in the test dictionary and is not empty; if so, append the test_tasks to the output list - if "test_tasks" in test and test["test_tasks"]: - config_output_tests_dict[workflow_name][task_name][ - output - ].append(test["test_tasks"]) - - #### Now that we've constructed the dictionary of outputs and their associated tests using the config file, we can compare this to the WDL files #### - # Load all WDL files in the directory wdl_files = find_wdl_files() - ## TODO - do we need/want the below? - untested_optional_outputs = [] + # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter tasks_below_threshold = False workflow_found = False @@ -79,43 +52,67 @@ def coverage_handler(kwargs): for wdl_file in wdl_files: workflow_tests_list = [] workflow_output_count = 0 + # Load the WDL document doc = WDL.load(wdl_file) - # Check if the WDL document has >0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for tasks >0 misses parent workflows that just import and call other tasks/workflows. TBD if we want to include this, but ultimately, if there are no tasks or a workflow at all, we skip the WDL file + # Handle the case where the WDL file is not in the configuration but is present in the directory + if wdl_file not in config._file.workflows: + coverage_summary["skipped_workflows"].append(wdl_file) + continue + + # Now that we know the WDL file is in the configuration, we can set the workflow name from the WDL.Tree.Document workflow attribute if it exists, otherwise we can grab the workflow name from the key from the configuration file as single task WDL files do not have a workflow attribute and some workflows have no tasks. This also helps organize the coverage output when we have a WDL file with >1 task but no workflow block (e.g., https://github.com/PacificBiosciences/wdl-common/blob/main/wdl/tasks/samtools.wdl), so that each task from the WDL file is grouped under the WDL file name regardless if it's defined as a workflow or not + workflow_name = ( + doc.workflow.name + if doc.workflow + else os.path.basename(config._file.workflows[wdl_file].key).replace( + ".wdl", "" + ) + ) + + # Check if the WDL document has > 0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for > 0 tasks misses parent workflows that just import and call other tasks/workflows. TBD if we want to include these 'parent' workflows, but ultimately, if there are no tasks or a workflow attribute, we skip the WDL file and print a warning if len(doc.tasks) > 0 or doc.workflow: # If workflow_name_filter is provided, skip all other workflows - if doc.workflow and workflow_name_filter: - if workflow_name_filter not in doc.workflow.name: - continue - workflow_found = True + if ( + workflow_name_filter is not None + and workflow_name_filter not in workflow_name + ): + continue + workflow_found = True # Iterate over each task in the WDL document for task in doc.tasks: - # Set the workflow name as the workflow name in the WDL file if it exists, otherwise set it to the task name as this handles the case where there is no workflow block in the WDL file (i.e., it's a single task WDL file) - workflow_name = doc.workflow.name if doc.workflow else task.name # Add to counters for total output count and workflow output count - coverage_state["total_output_count"] += len(task.outputs) + coverage_summary["total_output_count"] += len(task.outputs) workflow_output_count += len(task.outputs) + # Initialize a list of task test dictionaries + task_tests_list = [] try: # Create a list of dictionaries for each set of task tests in our config file - task_tests = ( + task_tests_list = ( config._file.workflows[wdl_file].tasks[task.name].tests ) # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary; duplicates are removed tested_outputs = list( set( [ output_name - for test in task_tests + for test in task_tests_list for output_name in test.output_tests.keys() ] ) ) - # Create a list of all the outputs that are present in the tas + + _update_coverage_summary( + "tested_outputs_dict", + workflow_name, + task.name, + output_names=tested_outputs, + ) + + # Create a list of all the outputs that are present in the task all_task_outputs = [output.name for output in task.outputs] # Create a list of outputs that are present in the task but not in the config file @@ -127,203 +124,274 @@ def coverage_handler(kwargs): # Add tested outputs to workflow_tests_list and all_tests_list workflow_tests_list.extend(tested_outputs) - coverage_state["all_tests_list"].extend(tested_outputs) - - # Add missing outputs to the coverage_state dictionary under the structure {workflow_name: {task_name: {output_name: [missing_outputs]}}} - if workflow_name not in coverage_state["untested_outputs"]: - coverage_state["untested_outputs"][workflow_name] = {} - if ( - task.name - not in coverage_state["untested_outputs"][workflow_name] - ): - coverage_state["untested_outputs"][workflow_name][ - task.name - ] = missing_outputs + coverage_summary["all_tests_list"].extend(tested_outputs) + + # Add missing outputs to the coverage_summary[untested_outputs] dictionary + _update_coverage_summary( + "untested_outputs", + workflow_name, + task.name, + output_names=missing_outputs, + ) + + # if workflow_name not in coverage_summary["untested_outputs"]: + # coverage_summary["untested_outputs"][workflow_name] = {} + # if ( + # task.name + # not in coverage_summary["untested_outputs"][workflow_name] + # ): + # coverage_summary["untested_outputs"][workflow_name][ + # task.name + # ] = missing_outputs # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it optional_inputs = [ input.name for input in task.inputs if input.type.optional ] + outputs_where_optional_inputs_not_dually_tested = [] if len(optional_inputs) > 0: - outputs_missing_tests = [] for output_name in all_task_outputs: - tests_with_optional_inputs = [ - test - for test in task_tests - if any( - optional_input in test.inputs - for optional_input in optional_inputs - ) - ] - tests_without_optional_inputs = [ - test - for test in task_tests - if all( - optional_input not in test.inputs - for optional_input in optional_inputs - ) - ] + ( + tests_with_optional_inputs, + tests_without_optional_inputs, + ) = _check_optional_inputs( + task_tests_list, optional_inputs + ) if ( len(tests_with_optional_inputs) < 1 or len(tests_without_optional_inputs) < 1 ): - outputs_missing_tests.append(output_name) - if len(outputs_missing_tests) > 0: - print( - f"\n\t[WARN]: Outputs {outputs_missing_tests} in task {task.name} do not have tests with and without optional inputs (optional inputs: {optional_inputs})." - ) - # print(f"{task.name}") - # print(f"Tested outputs: {tested_outputs}") - # print(f"All task outputs: {all_task_outputs}") - # print(f"Missing outputs: {missing_outputs}") + outputs_where_optional_inputs_not_dually_tested.append( + output_name + ) + _update_coverage_summary( + "untested_outputs_with_optional_inputs", + workflow_name, + task.name, + output_names=outputs_where_optional_inputs_not_dually_tested, + ) + # if (workflow_name not in coverage_summary["untested_outputs_with_optional_inputs"] + # ): + # coverage_summary["untested_outputs_with_optional_inputs"][workflow_name] = {} + # if (task.name not in coverage_summary["untested_outputs_with_optional_inputs"][workflow_name] + # ): + # coverage_summary["untested_outputs_with_optional_inputs"][workflow_name][task.name] = [] + # coverage_summary["untested_outputs_with_optional_inputs"][workflow_name][task.name].append(output_name) + # Catch the case where tasks are completely absent from the config except KeyError: # Initialize workflow in coverage state[untested_tasks] dict if there is a workflow in the WDL file but no tests in the config file - if workflow_name not in coverage_state["untested_tasks"]: - coverage_state["untested_tasks"][workflow_name] = [] - # Create a list of tasks associated with the respective workflow that are not present in the config file - coverage_state["untested_tasks"][workflow_name].append( - task.name + _update_coverage_summary( + "untested_tasks", workflow_name, task.name ) - # If there are outputs, untested tasks, and tests for the task, calculate the task coverage - if ( - len(task.outputs) > 0 - and len(coverage_state["untested_outputs"]) > 0 - ): - if len(task_tests) > 0: + # if workflow_name not in coverage_summary["untested_tasks"]: + # coverage_summary["untested_tasks"][workflow_name] = [] + # # Create a list of tasks associated with the respective workflow that are not present in the config file + # coverage_summary["untested_tasks"][workflow_name].append( + # task.name + # ) + + # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage + if len(task.outputs) > 0: + if len(task_tests_list) == 0: + _update_coverage_summary( + "untested_tasks", workflow_name, task.name + ) + # # Handle the case where the task is in the config but has no associated tests + # if workflow_name not in coverage_summary["untested_tasks"]: + # coverage_summary["untested_tasks"][workflow_name] = [] + # if ( + # task.name + # not in coverage_summary["untested_tasks"][workflow_name] + # ): + # coverage_summary["untested_tasks"][workflow_name].append( + # task.name + # ) + else: # Calculate and print the task coverage task_coverage = ( len(tested_outputs) / len(task.outputs) ) * 100 - if threshold is None or task_coverage < threshold: + if threshold is not None and task_coverage < threshold: tasks_below_threshold = True print(f"\ntask.{task.name}: {task_coverage:.2f}%") - - # Warn the user about outputs that are not tested - if workflow_name in coverage_state["untested_outputs"]: - if ( - task.name - in coverage_state["untested_outputs"][workflow_name] - ): - missing_outputs = coverage_state[ - "untested_outputs" - ][workflow_name][task.name] - if len(missing_outputs) > 0: - # Filter out tasks with no missing outputs - filtered_missing_outputs = { - k: v - for k, v in coverage_state[ - "untested_outputs" - ][workflow_name].items() - if v - } - if filtered_missing_outputs: - missing_outputs_list = [ - item - for sublist in filtered_missing_outputs.values() - for item in sublist - ] - print( - f"\n\t[WARN]: Missing tests in wdl-ci.config.json for {missing_outputs_list} in task {task.name}" - ) + else: + print(f"\ntask.{task.name}: {task_coverage:.2f}%") # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows - if ( - workflow_output_count > 0 - and len(workflow_tests_list) > 0 - and doc.workflow - and doc.workflow.name - ): + print(len(workflow_tests_list)) + if workflow_output_count > 0 and len(workflow_tests_list) > 0: workflow_coverage = ( len(workflow_tests_list) / workflow_output_count ) * 100 - if threshold is None or workflow_coverage < threshold: + if threshold is not None and workflow_coverage < threshold: workflows_below_threshold = True + # print("-" * 150) + print( + f"\n" + + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" + ) print("-" * 150) + else: + # print("-" * 150) print( - f"Workflow: {doc.workflow.name}: {workflow_coverage:.2f}%" + f"\n" + + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" ) + print("-" * 150) elif ( workflow_output_count == 0 or len(workflow_tests_list) == 0 - and doc.workflow - and doc.workflow.name + and workflow_name ): - if doc.workflow.name not in coverage_state["untested_workflows"]: - coverage_state["untested_workflows"].append(doc.workflow.name) + if workflow_name not in coverage_summary["untested_workflows"]: + coverage_summary["untested_workflows"].append(workflow_name) + + # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks + else: + coverage_summary["skipped_workflows"].append(wdl_file) + # Calculate and print the total coverage + if ( + len(coverage_summary["all_tests_list"]) > 0 + and coverage_summary["total_output_count"] > 0 + and not workflow_name_filter + ): + total_coverage = ( + len(coverage_summary["all_tests_list"]) + / coverage_summary["total_output_count"] + ) * 100 + print("\n" + f"\033[33mTotal coverage: {total_coverage:.2f}%\033[0m") - # Inform the user if no workflows matched the filter + # Inform the user if no workflows matched the filter and exit if workflow_name_filter and not workflow_found: print(f"\nNo workflows found matching the filter: {workflow_name_filter}") sys.exit(0) - # Check if there are tests for each output - # for output in task.outputs: - # if output.name in output_tests.keys() and len(output_tests) > 0: - # task_tests.append(output.name) - # workflow_tests.append(output.name) - # all_tests.append(output.name) - # elif ( - # output.type.optional - # and output.name not in output_tests.keys() - # or ( - # output.name in output_tests.keys() - # and len(output_tests[output.name]) == 0 - # ) - # ): - # untested_optional_outputs.append(output.name) - - # ) - # # Warn the user about optional outputs that are not tested - # if len(untested_optional_outputs) > 0: - # print( - # f"\n[WARN]: These optional outputs are not tested: {untested_optional_outputs}" - # ) - # else: - # print("\n✓ All optional outputs are tested") + # Sum the total number of untested outputs and untested outputs with optional inputs where there is not a test for the output with and without the optional input + total_untested_outputs = _sum_outputs(coverage_summary, "untested_outputs") + total_untested_outputs_with_optional_inputs = _sum_outputs( + coverage_summary, "untested_outputs_with_optional_inputs" + ) - if ( - tasks_below_threshold is False - and len(coverage_state["untested_tasks"]) == 0 + # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold + print("\n┍━━━━━━━━━━━━━┑") + print("│ Warning(s) │") + print("┕━━━━━━━━━━━━━┙") + if _check_threshold(tasks_below_threshold, total_untested_outputs, threshold): + print("\n✓ All outputs exceed the specified coverage threshold.") + if total_untested_outputs > 0: + print("\n" + "\033[31m[WARN]: The following outputs have no tests:\033[0m") + _print_untested_items(coverage_summary, "untested_outputs") + # for workflow, tasks in coverage_summary["untested_outputs"].items(): + # for task, outputs in tasks.items(): + # if len(outputs) > 0: + # print(f"\t{workflow}.{task}: {outputs}") + if total_untested_outputs_with_optional_inputs > 0: + # TODO: Would it be a requirement to report what input is optional here? + print( + "\n" + + "\033[31m[WARN]: The following outputs are not covered by tests that include and exclude optional inputs:\033[0m" + ) + _print_untested_items( + coverage_summary, "untested_outputs_with_optional_inputs" + ) + # for workflow, tasks in coverage_summary[ + # "untested_outputs_with_optional_inputs" + # ].items(): + # for task, outputs in tasks.items(): + # if len(outputs) > 0: + # print(f"\t{workflow}.{task}: {outputs}") + + # Warn the user if any workflows were skipped + if len(coverage_summary["skipped_workflows"]) > 0: + print( + "\n" + + "\033[31m[WARN]: The following workflows were skipped as they were not found in the wdl-ci.config.json but are present in the directory or they were present in the config JSON had no tasks or workflow blocks:\033[0m" + ) + for workflow in coverage_summary["skipped_workflows"]: + print(f"\t{workflow}") + + # Check if any tasks are below the threshold and there are no untested tasks; if so return to the user that all tasks exceed the threshold + if _check_threshold( + tasks_below_threshold, len(coverage_summary["untested_tasks"]), threshold ): print("\n✓ All tasks exceed the specified coverage threshold.") - elif len(coverage_state["untested_tasks"]) > 0: - print("\n[WARN]: The following tasks have no tests:") - for workflow, tasks in coverage_state["untested_tasks"].items(): + if len(coverage_summary["untested_tasks"]) > 0: + print("\n" + "\033[31m[WARN]: The following tasks have no tests:\033[0m") + for workflow, tasks in coverage_summary["untested_tasks"].items(): for task in tasks: print(f"\t{workflow}.{task}") # Check if any workflows are below the threshold and there are no untested workflows; if so return to the user that all workflows exceed the threshold - if ( - workflows_below_threshold is False - and len(coverage_state["untested_workflows"]) == 0 + if _check_threshold( + workflows_below_threshold, + len(coverage_summary["untested_workflows"]), + threshold, ): print("\n✓ All workflows exceed the specified coverage threshold.") # If there are any workflows that are below the threshold, warn the user. Include a check for the workflow_name_filter to ensure that only the specified workflow is printed if a filter is provided else: if ( - workflow_name_filter in coverage_state["untested_workflows"] + workflow_name_filter in coverage_summary["untested_workflows"] or workflow_name_filter is None - and len(coverage_state["untested_workflows"]) > 0 + and len(coverage_summary["untested_workflows"]) > 0 ): - print("\n[WARN]: The following workflows have no tests:") - for workflow in coverage_state["untested_workflows"]: + print( + "\n" + + "\033[31m[WARN]: The following workflows have outputs but no tests:\033[0m" + ) + for workflow in coverage_summary["untested_workflows"]: print(f"\t{workflow}") - # Calculate and print the total coverage - if ( - len(coverage_state["all_tests_list"]) > 0 - and coverage_state["total_output_count"] > 0 - and not workflow_name_filter - ): - total_coverage = ( - len(coverage_state["all_tests_list"]) - / coverage_state["total_output_count"] - ) * 100 - print(f"\nTotal coverage: {total_coverage:.2f}%") - except WdlTestCliExitException as e: print(f"exiting with code {e.exit_code}, message: {e.message}") sys.exit(e.exit_code) + + +# Helper functions +def _check_optional_inputs(task_tests_list, optional_inputs): + tests_with_optional_inputs = [ + test + for test in task_tests_list + if any(optional_input in test.inputs for optional_input in optional_inputs) + ] + tests_without_optional_inputs = [ + test + for test in task_tests_list + if all(optional_input not in test.inputs for optional_input in optional_inputs) + ] + return tests_with_optional_inputs, tests_without_optional_inputs + + +def _sum_outputs(coverage_summary, key): + return sum( + len(outputs) + for tasks in coverage_summary[key].values() + for outputs in tasks.values() + ) + + +def _update_coverage_summary(key, workflow_name, task_name, **kwargs): + output_names = kwargs.get("output_names", []) + if workflow_name not in coverage_summary[key]: + coverage_summary[key][workflow_name] = {} + if task_name not in coverage_summary[key][workflow_name]: + if len(output_names) > 0: + coverage_summary[key][workflow_name][task_name] = output_names + else: + coverage_summary[key][workflow_name][task_name] = [] + + +def _print_untested_items(coverage_summary, key): + for workflow, tasks in coverage_summary[key].items(): + for task, outputs in tasks.items(): + if len(outputs) > 0: + print(f"\t{workflow}.{task}: {outputs}") + + +def _check_threshold(below_threshold_flag, untested_count, threshold): + return ( + below_threshold_flag is False and untested_count == 0 and threshold is not None + ) From 12edb32ead0c07bcc802501ce20325c66eaed68e Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Thu, 12 Dec 2024 16:52:19 -0500 Subject: [PATCH 049/101] Remove redundant code that was refactored with helper functions --- src/wdlci/cli/coverage.py | 46 +++------------------------------------ 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 9f919f2..ce8d479 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -134,16 +134,6 @@ def coverage_handler(kwargs): output_names=missing_outputs, ) - # if workflow_name not in coverage_summary["untested_outputs"]: - # coverage_summary["untested_outputs"][workflow_name] = {} - # if ( - # task.name - # not in coverage_summary["untested_outputs"][workflow_name] - # ): - # coverage_summary["untested_outputs"][workflow_name][ - # task.name - # ] = missing_outputs - # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it optional_inputs = [ input.name for input in task.inputs if input.type.optional @@ -170,13 +160,6 @@ def coverage_handler(kwargs): task.name, output_names=outputs_where_optional_inputs_not_dually_tested, ) - # if (workflow_name not in coverage_summary["untested_outputs_with_optional_inputs"] - # ): - # coverage_summary["untested_outputs_with_optional_inputs"][workflow_name] = {} - # if (task.name not in coverage_summary["untested_outputs_with_optional_inputs"][workflow_name] - # ): - # coverage_summary["untested_outputs_with_optional_inputs"][workflow_name][task.name] = [] - # coverage_summary["untested_outputs_with_optional_inputs"][workflow_name][task.name].append(output_name) # Catch the case where tasks are completely absent from the config except KeyError: @@ -184,29 +167,14 @@ def coverage_handler(kwargs): _update_coverage_summary( "untested_tasks", workflow_name, task.name ) - # if workflow_name not in coverage_summary["untested_tasks"]: - # coverage_summary["untested_tasks"][workflow_name] = [] - # # Create a list of tasks associated with the respective workflow that are not present in the config file - # coverage_summary["untested_tasks"][workflow_name].append( - # task.name - # ) # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage if len(task.outputs) > 0: + # Handle the case where the task is in the config but has no associated tests if len(task_tests_list) == 0: _update_coverage_summary( "untested_tasks", workflow_name, task.name ) - # # Handle the case where the task is in the config but has no associated tests - # if workflow_name not in coverage_summary["untested_tasks"]: - # coverage_summary["untested_tasks"][workflow_name] = [] - # if ( - # task.name - # not in coverage_summary["untested_tasks"][workflow_name] - # ): - # coverage_summary["untested_tasks"][workflow_name].append( - # task.name - # ) else: # Calculate and print the task coverage task_coverage = ( @@ -280,13 +248,11 @@ def coverage_handler(kwargs): print("┕━━━━━━━━━━━━━┙") if _check_threshold(tasks_below_threshold, total_untested_outputs, threshold): print("\n✓ All outputs exceed the specified coverage threshold.") + if total_untested_outputs > 0: print("\n" + "\033[31m[WARN]: The following outputs have no tests:\033[0m") _print_untested_items(coverage_summary, "untested_outputs") - # for workflow, tasks in coverage_summary["untested_outputs"].items(): - # for task, outputs in tasks.items(): - # if len(outputs) > 0: - # print(f"\t{workflow}.{task}: {outputs}") + if total_untested_outputs_with_optional_inputs > 0: # TODO: Would it be a requirement to report what input is optional here? print( @@ -296,12 +262,6 @@ def coverage_handler(kwargs): _print_untested_items( coverage_summary, "untested_outputs_with_optional_inputs" ) - # for workflow, tasks in coverage_summary[ - # "untested_outputs_with_optional_inputs" - # ].items(): - # for task, outputs in tasks.items(): - # if len(outputs) > 0: - # print(f"\t{workflow}.{task}: {outputs}") # Warn the user if any workflows were skipped if len(coverage_summary["skipped_workflows"]) > 0: From a66bde7b1750460d471a851c00fd3d0553e14473 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 09:12:47 -0500 Subject: [PATCH 050/101] Remove unnecessary print --- src/wdlci/cli/coverage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index ce8d479..f6bbcc2 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -188,7 +188,6 @@ def coverage_handler(kwargs): # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows - print(len(workflow_tests_list)) if workflow_output_count > 0 and len(workflow_tests_list) > 0: workflow_coverage = ( len(workflow_tests_list) / workflow_output_count From e79a4adfd27199432aaea9458f204570811f9812 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 10:07:35 -0500 Subject: [PATCH 051/101] Handle case where test_tasks is an empty list; would have considered tested before this check --- src/wdlci/cli/coverage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index f6bbcc2..5d1d18a 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -100,7 +100,8 @@ def coverage_handler(kwargs): [ output_name for test in task_tests_list - for output_name in test.output_tests.keys() + for output_name, output_test in test.output_tests.items() + if len(output_test.get("test_tasks")) > 0 ] ) ) From 4ed57c1e160759732e0ad72267e8bd87b746f83b Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 11:17:15 -0500 Subject: [PATCH 052/101] Adjust comment wording --- src/wdlci/cli/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 5d1d18a..2b48960 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -15,7 +15,7 @@ "untested_outputs": {}, # {workflow_name: {task_name: [output_name]}} "untested_outputs_with_optional_inputs": {}, - ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; case where outputs in different workflows or tasks that share a name (e.g. vcf as an output from glnexus and deepvariant + ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name (e.g. vcf as an output from glnexus and deepvariant # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, From f1c3ee0675cd625cd765e4a369a87354a521f07d Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 11:20:24 -0500 Subject: [PATCH 053/101] Add .DS_STORE --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b827105..14a555b 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ dmypy.json .wdltest.changes.json .wdltest.submission.json + +# MAC +*.DS_STORE From 4237fd4e64ae6a23b56619ad68f9963400ea6366 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 11:20:35 -0500 Subject: [PATCH 054/101] Adjust TODO --- src/wdlci/cli/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 2b48960..5c5c123 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -15,7 +15,7 @@ "untested_outputs": {}, # {workflow_name: {task_name: [output_name]}} "untested_outputs_with_optional_inputs": {}, - ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name (e.g. vcf as an output from glnexus and deepvariant + ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name (e.g. vcf as an output from glnexus and deepvariant. I think I'd like to return coverage from the dict so I can reference output names and the tasks they belong to # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, From e243224693557b9730d5c071dd1397b2ba137674 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 11:20:53 -0500 Subject: [PATCH 055/101] Add tests for coverage handler -- in dev --- src/wdlci/tests/test_coverage_handler.py | 173 +++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 src/wdlci/tests/test_coverage_handler.py diff --git a/src/wdlci/tests/test_coverage_handler.py b/src/wdlci/tests/test_coverage_handler.py new file mode 100644 index 0000000..a7c0543 --- /dev/null +++ b/src/wdlci/tests/test_coverage_handler.py @@ -0,0 +1,173 @@ +import unittest +import os +import subprocess +import json + +from src.wdlci.cli.coverage import coverage_handler, coverage_summary +from src.wdlci.config import Config + + +class TestCoverageHandler(unittest.TestCase): + EXAMPLE_WDL_WORKFLOW = """version 1.0 + +struct Reference { + File fasta + String organism +} + +workflow call_variants { + input { + File bam + Reference ref + } + + call freebayes { + input: + bam=bam, + ref=ref + } + + output { + File vcf = freebayes.vcf + } +} + +task freebayes { + input { + File bam + Reference ref + Float? min_alternate_fraction + } + + String prefix = basename(bam, ".bam") + Float default_min_alternate_fraction = select_first([min_alternate_fraction, 0.2]) + + command <<< + freebayes -v '~{prefix}.vcf' -f ~{ref.fasta} \ + -F ~{default_min_alternate_fraction} \ + ~{bam} + >>> + + runtime { + docker: "quay.io/biocontainers/freebayes:1.3.2--py36hc088bd4_0" + } + + output { + File vcf = "${prefix}.vcf" + } +} +""" + + def setUp(self): + # Create the WDL files with different workflow names + wdl_workflow_1 = self.EXAMPLE_WDL_WORKFLOW.replace( + "call_variants", "call_variants_1" + ) + wdl_workflow_2 = self.EXAMPLE_WDL_WORKFLOW.replace( + "call_variants", "call_variants_2" + ) + + with open("test_call-variants_1.wdl", "w") as f: + f.write(wdl_workflow_1) + with open("test_call-variants_2.wdl", "w") as f: + f.write(wdl_workflow_2) + # Run the wdl-ci generate-config subcommand + subprocess.run( + ["wdl-ci", "generate-config"], check=True, stdout=subprocess.DEVNULL + ) + + def tearDown(self): + # Remove the WDL files + if os.path.exists("test_call-variants_1.wdl"): + os.remove("test_call-variants_1.wdl") + if os.path.exists("test_call-variants_2.wdl"): + os.remove("test_call-variants_2.wdl") + if os.path.exists("wdl-ci.config.json"): + os.remove("wdl-ci.config.json") + + # def reset_config(self): + # Config._cli_kwargs = None + # Config._instance = None + + def update_config_with_tests(self, wdl_1_tests, wdl_2_tests): + # Read the existing config file + with open("wdl-ci.config.json", "r") as f: + config = json.load(f) + + config["workflows"]["test_call-variants_1.wdl"]["tasks"]["freebayes"][ + "tests" + ] = wdl_1_tests + config["workflows"]["test_call-variants_2.wdl"]["tasks"]["freebayes"][ + "tests" + ] = wdl_2_tests + + # Write the updated config back to the file + with open("wdl-ci.config.json", "w") as f: + json.dump(config, f, indent=2) + + def reset_coverage_summary(self): + coverage_summary["untested_workflows"] = [] + coverage_summary["untested_tasks"] = {} + coverage_summary["untested_outputs"] = {} + coverage_summary["untested_outputs_with_optional_inputs"] = {} + coverage_summary["tested_outputs_dict"] = {} + coverage_summary["total_output_count"] = 0 + coverage_summary["all_tests_list"] = [] + coverage_summary["skipped_workflows"] = [] + + def test_identical_output_names(self): + # Reset the coverage_summary + self.reset_coverage_summary() + + # Update the "tests" list for specific workflows + test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + } + ] + self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) + + # Call the coverage_handler function + kwargs = {"target_coverage": None, "workflow_name": None} + coverage_handler(kwargs) + + # Assertions + self.assertIn("compare_float", coverage_summary["untested_workflows"]) + self.assertNotIn("call_variants_1", coverage_summary["untested_workflows"]) + self.assertNotIn("call_variants_2", coverage_summary["untested_workflows"]) + self.assertNotIn("freebayes", coverage_summary["untested_tasks"]) + self.assertIn( + "vcf", + coverage_summary["untested_outputs_with_optional_inputs"][ + "call_variants_1" + ]["freebayes"], + ) + self.assertIn( + "vcf", + coverage_summary["tested_outputs_dict"]["call_variants_1"]["freebayes"], + ) + self.assertIn( + "vcf", + coverage_summary["tested_outputs_dict"]["call_variants_2"]["freebayes"], + ) + + # def test_no_tasks_in_workflow(self): + # # Update the "tests" list for specific workflows + # test_cases = [] + # self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) + # # Call the coverage_handler function + # kwargs = {"target_coverage": None, "workflow_name": None} + # coverage_handler(kwargs) + + +if __name__ == "__main__": + unittest.main() From 8187b2bf8735491d29d67d58422f866ca81ac707 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 13:16:55 -0500 Subject: [PATCH 056/101] Improve naming conventions of coverage_summary keys; set boolean flags dynamically as part of code; adjust TODOs --- src/wdlci/cli/coverage.py | 70 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 5c5c123..dcdb8ea 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -8,19 +8,19 @@ # Initialize dictionary with necessary variables to compute coverage coverage_summary = { - "untested_workflows": [], + "untested_workflows_list": [], # {workflow_name: [task_name]} - "untested_tasks": {}, + "untested_tasks_dict": {}, # {workflow_name: {task_name: [output_name]}} - "untested_outputs": {}, + "untested_outputs_dict": {}, # {workflow_name: {task_name: [output_name]}} - "untested_outputs_with_optional_inputs": {}, - ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name (e.g. vcf as an output from glnexus and deepvariant. I think I'd like to return coverage from the dict so I can reference output names and the tasks they belong to + "untested_outputs_with_optional_inputs_dict": {}, + ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name -- might be preferable to return coverage from the dict so we can reference output names and the tasks they belong to. Currentlt, the tested_outputs_dict is updated but not used to calculate coverage. It might only be useful if we want to report the outputs that are tested for a given workflow or task. # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, "all_tests_list": [], - "skipped_workflows": [], + "skipped_workflows_list": [], } @@ -43,11 +43,6 @@ def coverage_handler(kwargs): # Load all WDL files in the directory wdl_files = find_wdl_files() - # Flags to track if any tasks/workflows are below the threshold and if any workflows match the filter - tasks_below_threshold = False - workflow_found = False - workflows_below_threshold = False - # Iterate over each WDL file for wdl_file in wdl_files: workflow_tests_list = [] @@ -58,7 +53,7 @@ def coverage_handler(kwargs): # Handle the case where the WDL file is not in the configuration but is present in the directory if wdl_file not in config._file.workflows: - coverage_summary["skipped_workflows"].append(wdl_file) + coverage_summary["skipped_workflows_list"].append(wdl_file) continue # Now that we know the WDL file is in the configuration, we can set the workflow name from the WDL.Tree.Document workflow attribute if it exists, otherwise we can grab the workflow name from the key from the configuration file as single task WDL files do not have a workflow attribute and some workflows have no tasks. This also helps organize the coverage output when we have a WDL file with >1 task but no workflow block (e.g., https://github.com/PacificBiosciences/wdl-common/blob/main/wdl/tasks/samtools.wdl), so that each task from the WDL file is grouped under the WDL file name regardless if it's defined as a workflow or not @@ -129,7 +124,7 @@ def coverage_handler(kwargs): # Add missing outputs to the coverage_summary[untested_outputs] dictionary _update_coverage_summary( - "untested_outputs", + "untested_outputs_dict", workflow_name, task.name, output_names=missing_outputs, @@ -156,7 +151,7 @@ def coverage_handler(kwargs): output_name ) _update_coverage_summary( - "untested_outputs_with_optional_inputs", + "untested_outputs_with_optional_inputs_dict", workflow_name, task.name, output_names=outputs_where_optional_inputs_not_dually_tested, @@ -166,7 +161,7 @@ def coverage_handler(kwargs): except KeyError: # Initialize workflow in coverage state[untested_tasks] dict if there is a workflow in the WDL file but no tests in the config file _update_coverage_summary( - "untested_tasks", workflow_name, task.name + "untested_tasks_dict", workflow_name, task.name ) # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage @@ -174,7 +169,7 @@ def coverage_handler(kwargs): # Handle the case where the task is in the config but has no associated tests if len(task_tests_list) == 0: _update_coverage_summary( - "untested_tasks", workflow_name, task.name + "untested_tasks_dict", workflow_name, task.name ) else: # Calculate and print the task coverage @@ -185,6 +180,7 @@ def coverage_handler(kwargs): tasks_below_threshold = True print(f"\ntask.{task.name}: {task_coverage:.2f}%") else: + tasks_below_threshold = False print(f"\ntask.{task.name}: {task_coverage:.2f}%") # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list @@ -195,14 +191,13 @@ def coverage_handler(kwargs): ) * 100 if threshold is not None and workflow_coverage < threshold: workflows_below_threshold = True - # print("-" * 150) print( f"\n" + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" ) print("-" * 150) else: - # print("-" * 150) + workflows_below_threshold = False print( f"\n" + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" @@ -213,12 +208,15 @@ def coverage_handler(kwargs): or len(workflow_tests_list) == 0 and workflow_name ): - if workflow_name not in coverage_summary["untested_workflows"]: - coverage_summary["untested_workflows"].append(workflow_name) + if workflow_name not in coverage_summary["untested_workflows_list"]: + coverage_summary["untested_workflows_list"].append( + workflow_name + ) # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks else: - coverage_summary["skipped_workflows"].append(wdl_file) + coverage_summary["skipped_workflows_list"].append(wdl_file) + workflow_found = False # Calculate and print the total coverage if ( len(coverage_summary["all_tests_list"]) > 0 @@ -237,9 +235,9 @@ def coverage_handler(kwargs): sys.exit(0) # Sum the total number of untested outputs and untested outputs with optional inputs where there is not a test for the output with and without the optional input - total_untested_outputs = _sum_outputs(coverage_summary, "untested_outputs") + total_untested_outputs = _sum_outputs(coverage_summary, "untested_outputs_dict") total_untested_outputs_with_optional_inputs = _sum_outputs( - coverage_summary, "untested_outputs_with_optional_inputs" + coverage_summary, "untested_outputs_with_optional_inputs_dict" ) # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold @@ -251,42 +249,44 @@ def coverage_handler(kwargs): if total_untested_outputs > 0: print("\n" + "\033[31m[WARN]: The following outputs have no tests:\033[0m") - _print_untested_items(coverage_summary, "untested_outputs") + _print_untested_items(coverage_summary, "untested_outputs_dict") if total_untested_outputs_with_optional_inputs > 0: - # TODO: Would it be a requirement to report what input is optional here? + # TODO: Would it be a requirement to report the specific input that is optional here? print( "\n" + "\033[31m[WARN]: The following outputs are not covered by tests that include and exclude optional inputs:\033[0m" ) _print_untested_items( - coverage_summary, "untested_outputs_with_optional_inputs" + coverage_summary, "untested_outputs_with_optional_inputs_dict" ) # Warn the user if any workflows were skipped - if len(coverage_summary["skipped_workflows"]) > 0: + if len(coverage_summary["skipped_workflows_list"]) > 0: print( "\n" + "\033[31m[WARN]: The following workflows were skipped as they were not found in the wdl-ci.config.json but are present in the directory or they were present in the config JSON had no tasks or workflow blocks:\033[0m" ) - for workflow in coverage_summary["skipped_workflows"]: + for workflow in coverage_summary["skipped_workflows_list"]: print(f"\t{workflow}") # Check if any tasks are below the threshold and there are no untested tasks; if so return to the user that all tasks exceed the threshold if _check_threshold( - tasks_below_threshold, len(coverage_summary["untested_tasks"]), threshold + tasks_below_threshold, + len(coverage_summary["untested_tasks_dict"]), + threshold, ): print("\n✓ All tasks exceed the specified coverage threshold.") - if len(coverage_summary["untested_tasks"]) > 0: + if len(coverage_summary["untested_tasks_dict"]) > 0: print("\n" + "\033[31m[WARN]: The following tasks have no tests:\033[0m") - for workflow, tasks in coverage_summary["untested_tasks"].items(): + for workflow, tasks in coverage_summary["untested_tasks_dict"].items(): for task in tasks: print(f"\t{workflow}.{task}") # Check if any workflows are below the threshold and there are no untested workflows; if so return to the user that all workflows exceed the threshold if _check_threshold( workflows_below_threshold, - len(coverage_summary["untested_workflows"]), + len(coverage_summary["untested_workflows_list"]), threshold, ): print("\n✓ All workflows exceed the specified coverage threshold.") @@ -294,15 +294,15 @@ def coverage_handler(kwargs): # If there are any workflows that are below the threshold, warn the user. Include a check for the workflow_name_filter to ensure that only the specified workflow is printed if a filter is provided else: if ( - workflow_name_filter in coverage_summary["untested_workflows"] + workflow_name_filter in coverage_summary["untested_workflows_list"] or workflow_name_filter is None - and len(coverage_summary["untested_workflows"]) > 0 + and len(coverage_summary["untested_workflows_list"]) > 0 ): print( "\n" + "\033[31m[WARN]: The following workflows have outputs but no tests:\033[0m" ) - for workflow in coverage_summary["untested_workflows"]: + for workflow in coverage_summary["untested_workflows_list"]: print(f"\t{workflow}") except WdlTestCliExitException as e: From 086c75aee51c30227f7830eb375ce41908322f7c Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 13:37:23 -0500 Subject: [PATCH 057/101] Add additional handling of boolean checks --- src/wdlci/cli/coverage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index dcdb8ea..1b9f66d 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -182,6 +182,8 @@ def coverage_handler(kwargs): else: tasks_below_threshold = False print(f"\ntask.{task.name}: {task_coverage:.2f}%") + else: + tasks_below_threshold = False # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows @@ -208,6 +210,7 @@ def coverage_handler(kwargs): or len(workflow_tests_list) == 0 and workflow_name ): + workflows_below_threshold = False if workflow_name not in coverage_summary["untested_workflows_list"]: coverage_summary["untested_workflows_list"].append( workflow_name From 3cebce8c6e91eb5704b2fd6244ff8f80b3421383 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 13:39:21 -0500 Subject: [PATCH 058/101] Added extra test; rename coverage summary keys --- src/wdlci/tests/test_coverage_handler.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/wdlci/tests/test_coverage_handler.py b/src/wdlci/tests/test_coverage_handler.py index a7c0543..cc4d6bf 100644 --- a/src/wdlci/tests/test_coverage_handler.py +++ b/src/wdlci/tests/test_coverage_handler.py @@ -4,7 +4,6 @@ import json from src.wdlci.cli.coverage import coverage_handler, coverage_summary -from src.wdlci.config import Config class TestCoverageHandler(unittest.TestCase): @@ -141,13 +140,13 @@ def test_identical_output_names(self): coverage_handler(kwargs) # Assertions - self.assertIn("compare_float", coverage_summary["untested_workflows"]) - self.assertNotIn("call_variants_1", coverage_summary["untested_workflows"]) - self.assertNotIn("call_variants_2", coverage_summary["untested_workflows"]) - self.assertNotIn("freebayes", coverage_summary["untested_tasks"]) + self.assertIn("compare_float", coverage_summary["untested_workflows_list"]) + self.assertNotIn("call_variants_1", coverage_summary["untested_workflows_list"]) + self.assertNotIn("call_variants_2", coverage_summary["untested_workflows_list"]) + self.assertNotIn("freebayes", coverage_summary["untested_tasks_dict"]) self.assertIn( "vcf", - coverage_summary["untested_outputs_with_optional_inputs"][ + coverage_summary["untested_outputs_with_optional_inputs_dict"][ "call_variants_1" ]["freebayes"], ) @@ -161,12 +160,19 @@ def test_identical_output_names(self): ) # def test_no_tasks_in_workflow(self): + # self.reset_coverage_summary() # # Update the "tests" list for specific workflows # test_cases = [] # self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) # # Call the coverage_handler function # kwargs = {"target_coverage": None, "workflow_name": None} # coverage_handler(kwargs) + # # Assertions + # self.assertNotEqual(len(coverage_summary["untested_outputs_dict"]), 0) + # self.assertEqual( + # len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), 2 + # ) + # self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) if __name__ == "__main__": From c6e774cdbcafa88f359195a8a90775995e73e3d6 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:23:38 -0500 Subject: [PATCH 059/101] Improve prints; return tested outputs for each {workflow.task} --- src/wdlci/cli/coverage.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 1b9f66d..0018eb8 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -234,7 +234,9 @@ def coverage_handler(kwargs): # Inform the user if no workflows matched the filter and exit if workflow_name_filter and not workflow_found: - print(f"\nNo workflows found matching the filter: {workflow_name_filter}") + print( + f"\nNo workflows found matching the filter: [{workflow_name_filter}] or the workflow you searched for has no tasks or workflow attribute" + ) sys.exit(0) # Sum the total number of untested outputs and untested outputs with optional inputs where there is not a test for the output with and without the optional input @@ -242,6 +244,11 @@ def coverage_handler(kwargs): total_untested_outputs_with_optional_inputs = _sum_outputs( coverage_summary, "untested_outputs_with_optional_inputs_dict" ) + # TODO: This might be a little overkill + total_tested_outputs = _sum_outputs(coverage_summary, "tested_outputs_dict") + if total_tested_outputs > 0: + print("\n The following outputs are tested:") + _print_untested_items(coverage_summary, "tested_outputs_dict") # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold print("\n┍━━━━━━━━━━━━━┑") @@ -258,21 +265,12 @@ def coverage_handler(kwargs): # TODO: Would it be a requirement to report the specific input that is optional here? print( "\n" - + "\033[31m[WARN]: The following outputs are not covered by tests that include and exclude optional inputs:\033[0m" + + "\033[31m[WARN]: The following outputs are part of a task that contains an optional input and are not covered by at least two tests; they should be covered for both cases where the optional input is and is not provided:\033[0m" ) _print_untested_items( coverage_summary, "untested_outputs_with_optional_inputs_dict" ) - # Warn the user if any workflows were skipped - if len(coverage_summary["skipped_workflows_list"]) > 0: - print( - "\n" - + "\033[31m[WARN]: The following workflows were skipped as they were not found in the wdl-ci.config.json but are present in the directory or they were present in the config JSON had no tasks or workflow blocks:\033[0m" - ) - for workflow in coverage_summary["skipped_workflows_list"]: - print(f"\t{workflow}") - # Check if any tasks are below the threshold and there are no untested tasks; if so return to the user that all tasks exceed the threshold if _check_threshold( tasks_below_threshold, @@ -308,6 +306,15 @@ def coverage_handler(kwargs): for workflow in coverage_summary["untested_workflows_list"]: print(f"\t{workflow}") + # Warn the user if any workflows were skipped + if len(coverage_summary["skipped_workflows_list"]) > 0: + print( + "\n" + + "\033[31m[WARN]: The following workflows were skipped as they were not found in the wdl-ci.config.json but are present in the directory, or they were present in the config JSON had no tasks or workflow blocks:\033[0m" + ) + for workflow in coverage_summary["skipped_workflows_list"]: + print(f"\t{workflow}") + except WdlTestCliExitException as e: print(f"exiting with code {e.exit_code}, message: {e.message}") sys.exit(e.exit_code) From 36b712b6096f48fda23ceb0c62926896cda550a6 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:30:43 -0500 Subject: [PATCH 060/101] Suppress ResourceWarning; add additional tests to add; comment out second test to avoid config load error I need to find a way to avoid the message: exiting with code 1, message: Cannot load Config, already loaded when calling coverage_handler >1 times --- src/wdlci/tests/test_coverage_handler.py | 236 +++++++++++++---------- 1 file changed, 131 insertions(+), 105 deletions(-) diff --git a/src/wdlci/tests/test_coverage_handler.py b/src/wdlci/tests/test_coverage_handler.py index cc4d6bf..465adcf 100644 --- a/src/wdlci/tests/test_coverage_handler.py +++ b/src/wdlci/tests/test_coverage_handler.py @@ -2,62 +2,65 @@ import os import subprocess import json +import warnings from src.wdlci.cli.coverage import coverage_handler, coverage_summary class TestCoverageHandler(unittest.TestCase): - EXAMPLE_WDL_WORKFLOW = """version 1.0 - -struct Reference { - File fasta - String organism -} - -workflow call_variants { - input { - File bam - Reference ref - } - - call freebayes { - input: - bam=bam, - ref=ref - } - - output { - File vcf = freebayes.vcf - } -} - -task freebayes { - input { - File bam - Reference ref - Float? min_alternate_fraction - } - - String prefix = basename(bam, ".bam") - Float default_min_alternate_fraction = select_first([min_alternate_fraction, 0.2]) - - command <<< - freebayes -v '~{prefix}.vcf' -f ~{ref.fasta} \ - -F ~{default_min_alternate_fraction} \ - ~{bam} - >>> - - runtime { - docker: "quay.io/biocontainers/freebayes:1.3.2--py36hc088bd4_0" - } - - output { - File vcf = "${prefix}.vcf" - } -} + EXAMPLE_WDL_WORKFLOW = """ + version 1.0 + + struct Reference { + File fasta + String organism + } + + workflow call_variants { + input { + File bam + Reference ref + } + + call freebayes { + input: + bam=bam, + ref=ref + } + + output { + File vcf = freebayes.vcf + } + } + + task freebayes { + input { + File bam + Reference ref + Float? min_alternate_fraction + } + + String prefix = basename(bam, ".bam") + Float default_min_alternate_fraction = select_first([min_alternate_fraction, 0.2]) + + command <<< + freebayes -v '~{prefix}.vcf' -f ~{ref.fasta} \ + -F ~{default_min_alternate_fraction} \ + ~{bam} + >>> + + runtime { + docker: "quay.io/biocontainers/freebayes:1.3.2--py36hc088bd4_0" + } + + output { + File vcf = "${prefix}.vcf" + } + } """ def setUp(self): + # Suppress the ResourceWarning complaining about the config_file json.load not being closed # Create the WDL files with different workflow names wdl_workflow_1 = self.EXAMPLE_WDL_WORKFLOW.replace( "call_variants", "call_variants_1" @@ -84,10 +87,6 @@ def tearDown(self): if os.path.exists("wdl-ci.config.json"): os.remove("wdl-ci.config.json") - # def reset_config(self): - # Config._cli_kwargs = None - # Config._instance = None - def update_config_with_tests(self, wdl_1_tests, wdl_2_tests): # Read the existing config file with open("wdl-ci.config.json", "r") as f: @@ -117,62 +116,89 @@ def reset_coverage_summary(self): def test_identical_output_names(self): # Reset the coverage_summary self.reset_coverage_summary() - - # Update the "tests" list for specific workflows - test_cases = [ - { - "inputs": { - "bam": "test.bam", - "ref": {"fasta": "test.fasta", "organism": "test_organism"}, - }, - "output_tests": { - "vcf": { - "value": "test.vcf", - "test_tasks": ["compare_file_basename"], - } - }, - } - ] - self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) - - # Call the coverage_handler function - kwargs = {"target_coverage": None, "workflow_name": None} - coverage_handler(kwargs) - - # Assertions - self.assertIn("compare_float", coverage_summary["untested_workflows_list"]) - self.assertNotIn("call_variants_1", coverage_summary["untested_workflows_list"]) - self.assertNotIn("call_variants_2", coverage_summary["untested_workflows_list"]) - self.assertNotIn("freebayes", coverage_summary["untested_tasks_dict"]) - self.assertIn( - "vcf", - coverage_summary["untested_outputs_with_optional_inputs_dict"][ - "call_variants_1" - ]["freebayes"], - ) - self.assertIn( - "vcf", - coverage_summary["tested_outputs_dict"]["call_variants_1"]["freebayes"], - ) - self.assertIn( - "vcf", - coverage_summary["tested_outputs_dict"]["call_variants_2"]["freebayes"], - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + + # Update the "tests" list for specific workflows + test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + } + ] + self.update_config_with_tests( + wdl_1_tests=test_cases, wdl_2_tests=test_cases + ) + + # Call the coverage_handler function + kwargs = {"target_coverage": None, "workflow_name": None} + coverage_handler(kwargs) + + # Assert both workflows are not in the untested workflows list + self.assertNotIn( + "call_variants_1", coverage_summary["untested_workflows_list"] + ) + self.assertNotIn( + "call_variants_2", coverage_summary["untested_workflows_list"] + ) + # Assert that the corresponding task is not in the untested tasks dictionary + self.assertNotIn("freebayes", coverage_summary["untested_tasks_dict"]) + # Assert that "vcf" is found in both sets of {workflow: {task: [tested_outputs]}} in the parent untested_outputs_with_optional_inputs_dict + self.assertIn( + "vcf", + coverage_summary["untested_outputs_with_optional_inputs_dict"][ + "call_variants_1" + ]["freebayes"], + ) + # Assert that "vcf" is found in both sets of {workflow: {task: [tested_outputs]}} in the parent tested_output_dict + self.assertIn( + "vcf", + coverage_summary["tested_outputs_dict"]["call_variants_1"]["freebayes"], + ) + self.assertIn( + "vcf", + coverage_summary["tested_outputs_dict"]["call_variants_2"]["freebayes"], + ) # def test_no_tasks_in_workflow(self): # self.reset_coverage_summary() - # # Update the "tests" list for specific workflows - # test_cases = [] - # self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) - # # Call the coverage_handler function - # kwargs = {"target_coverage": None, "workflow_name": None} - # coverage_handler(kwargs) - # # Assertions - # self.assertNotEqual(len(coverage_summary["untested_outputs_dict"]), 0) - # self.assertEqual( - # len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), 2 - # ) - # self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) + # # Suppress the ResourceWarning complaining about the config_file json.load not being closed + # with warnings.catch_warnings(): + # warnings.simplefilter("ignore", ResourceWarning) + # # Update the "tests" list for specific workflows + # test_cases = [] + # self.update_config_with_tests( + # wdl_1_tests=test_cases, wdl_2_tests=test_cases + # ) + # # Call the coverage_handler function + # kwargs = { + # "target_coverage": None, + # "workflow_name": None, + # } + # coverage_handler(kwargs) + # # Assertions + # self.assertNotEqual(len(coverage_summary["untested_outputs_dict"]), 0) + # self.assertEqual( + # len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), 2 + # ) + # self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) + + #### Additional tests I'd like to add #### + # Providing a value to command to --target-coverage where all outputs/tasks/workflows are above the thresold + # Providing a value to --target-coverage where only some outputs/tasks/workflows are above the threshold + # Providing a value to --target-coverage where none of the above are above the threshold + # Providing any valid workflow name to --workflow-name + # Providing an invalid workflow name to --workflow-name + # Providing a valid workflow name to --workflow name where the workflow exists, but has no tasks + # Providing an extremely large wdl-ci.config.json if __name__ == "__main__": From 7ff888ecc989d551b8c99addbdf59b6efa44d871 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:31:51 -0500 Subject: [PATCH 061/101] Move tests dir --- .../tests => tests}/test_coverage_handler.py | 0 tests/test_coverage_handler_no_tests.py | 144 ++++++++++++++++++ 2 files changed, 144 insertions(+) rename {src/wdlci/tests => tests}/test_coverage_handler.py (100%) create mode 100644 tests/test_coverage_handler_no_tests.py diff --git a/src/wdlci/tests/test_coverage_handler.py b/tests/test_coverage_handler.py similarity index 100% rename from src/wdlci/tests/test_coverage_handler.py rename to tests/test_coverage_handler.py diff --git a/tests/test_coverage_handler_no_tests.py b/tests/test_coverage_handler_no_tests.py new file mode 100644 index 0000000..3835aa6 --- /dev/null +++ b/tests/test_coverage_handler_no_tests.py @@ -0,0 +1,144 @@ +import unittest +import os +import subprocess +import json +import warnings + +from src.wdlci.cli.coverage import coverage_handler, coverage_summary + + +class TestCoverageHandler(unittest.TestCase): + EXAMPLE_WDL_WORKFLOW = """ + version 1.0 + + struct Reference { + File fasta + String organism + } + + workflow call_variants { + input { + File bam + Reference ref + } + + call freebayes { + input: + bam=bam, + ref=ref + } + + output { + File vcf = freebayes.vcf + } + } + + task freebayes { + input { + File bam + Reference ref + Float? min_alternate_fraction + } + + String prefix = basename(bam, ".bam") + Float default_min_alternate_fraction = select_first([min_alternate_fraction, 0.2]) + + command <<< + freebayes -v '~{prefix}.vcf' -f ~{ref.fasta} \ + -F ~{default_min_alternate_fraction} \ + ~{bam} + >>> + + runtime { + docker: "quay.io/biocontainers/freebayes:1.3.2--py36hc088bd4_0" + } + + output { + File vcf = "${prefix}.vcf" + } + } +""" + + def setUp(self): + # Suppress the ResourceWarning complaining about the config_file json.load not being closed + # Create the WDL files with different workflow names + wdl_workflow_1 = self.EXAMPLE_WDL_WORKFLOW.replace( + "call_variants", "call_variants_1" + ) + wdl_workflow_2 = self.EXAMPLE_WDL_WORKFLOW.replace( + "call_variants", "call_variants_2" + ) + + with open("test_call-variants_1.wdl", "w") as f: + f.write(wdl_workflow_1) + with open("test_call-variants_2.wdl", "w") as f: + f.write(wdl_workflow_2) + # Run the wdl-ci generate-config subcommand + subprocess.run( + ["wdl-ci", "generate-config"], check=True, stdout=subprocess.DEVNULL + ) + + def tearDown(self): + # Remove the WDL files + if os.path.exists("test_call-variants_1.wdl"): + os.remove("test_call-variants_1.wdl") + if os.path.exists("test_call-variants_2.wdl"): + os.remove("test_call-variants_2.wdl") + if os.path.exists("wdl-ci.config.json"): + os.remove("wdl-ci.config.json") + + def update_config_with_tests(self, wdl_1_tests, wdl_2_tests): + # Read the existing config file + with open("wdl-ci.config.json", "r") as f: + config = json.load(f) + + config["workflows"]["test_call-variants_1.wdl"]["tasks"]["freebayes"][ + "tests" + ] = wdl_1_tests + config["workflows"]["test_call-variants_2.wdl"]["tasks"]["freebayes"][ + "tests" + ] = wdl_2_tests + + # Write the updated config back to the file + with open("wdl-ci.config.json", "w") as f: + json.dump(config, f, indent=2) + + def reset_coverage_summary(self): + coverage_summary["untested_workflows"] = [] + coverage_summary["untested_tasks"] = {} + coverage_summary["untested_outputs"] = {} + coverage_summary["untested_outputs_with_optional_inputs"] = {} + coverage_summary["tested_outputs_dict"] = {} + coverage_summary["total_output_count"] = 0 + coverage_summary["all_tests_list"] = [] + coverage_summary["skipped_workflows"] = [] + + def test_no_tasks_in_workflow(self): + self.reset_coverage_summary() + # Suppress the ResourceWarning complaining about the config_file json.load not being closed + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + # Update the "tests" list for specific workflows + test_cases = [] + self.update_config_with_tests( + wdl_1_tests=test_cases, wdl_2_tests=test_cases + ) + # Call the coverage_handler function + kwargs = { + "target_coverage": None, + "workflow_name": None, + "instance": None, + "initialize": False, + } + coverage_handler(kwargs) + # Assertions + self.assertNotEqual(len(coverage_summary["untested_outputs_dict"]), 0) + self.assertEqual( + len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), + 2, + ) + self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) + + +if __name__ == "__main__": + unittest.main() From 9f8339401fa9141981b7f006bddad0bf95dbe3af Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:32:43 -0500 Subject: [PATCH 062/101] Fix typo --- src/wdlci/cli/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 0018eb8..35bb42a 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -15,7 +15,7 @@ "untested_outputs_dict": {}, # {workflow_name: {task_name: [output_name]}} "untested_outputs_with_optional_inputs_dict": {}, - ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name -- might be preferable to return coverage from the dict so we can reference output names and the tasks they belong to. Currentlt, the tested_outputs_dict is updated but not used to calculate coverage. It might only be useful if we want to report the outputs that are tested for a given workflow or task. + ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name -- might be preferable to return coverage from the dict so we can reference output names and the tasks they belong to. Currently, the tested_outputs_dict is updated but not used to calculate coverage. It might only be useful if we want to report the outputs that are tested for a given workflow or task. # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, From 32c214dfb9672264c25efe9ef2e6716687da7cd9 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:37:13 -0500 Subject: [PATCH 063/101] Remove unused file with single test --- tests/test_coverage_handler_no_tests.py | 144 ------------------------ 1 file changed, 144 deletions(-) delete mode 100644 tests/test_coverage_handler_no_tests.py diff --git a/tests/test_coverage_handler_no_tests.py b/tests/test_coverage_handler_no_tests.py deleted file mode 100644 index 3835aa6..0000000 --- a/tests/test_coverage_handler_no_tests.py +++ /dev/null @@ -1,144 +0,0 @@ -import unittest -import os -import subprocess -import json -import warnings - -from src.wdlci.cli.coverage import coverage_handler, coverage_summary - - -class TestCoverageHandler(unittest.TestCase): - EXAMPLE_WDL_WORKFLOW = """ - version 1.0 - - struct Reference { - File fasta - String organism - } - - workflow call_variants { - input { - File bam - Reference ref - } - - call freebayes { - input: - bam=bam, - ref=ref - } - - output { - File vcf = freebayes.vcf - } - } - - task freebayes { - input { - File bam - Reference ref - Float? min_alternate_fraction - } - - String prefix = basename(bam, ".bam") - Float default_min_alternate_fraction = select_first([min_alternate_fraction, 0.2]) - - command <<< - freebayes -v '~{prefix}.vcf' -f ~{ref.fasta} \ - -F ~{default_min_alternate_fraction} \ - ~{bam} - >>> - - runtime { - docker: "quay.io/biocontainers/freebayes:1.3.2--py36hc088bd4_0" - } - - output { - File vcf = "${prefix}.vcf" - } - } -""" - - def setUp(self): - # Suppress the ResourceWarning complaining about the config_file json.load not being closed - # Create the WDL files with different workflow names - wdl_workflow_1 = self.EXAMPLE_WDL_WORKFLOW.replace( - "call_variants", "call_variants_1" - ) - wdl_workflow_2 = self.EXAMPLE_WDL_WORKFLOW.replace( - "call_variants", "call_variants_2" - ) - - with open("test_call-variants_1.wdl", "w") as f: - f.write(wdl_workflow_1) - with open("test_call-variants_2.wdl", "w") as f: - f.write(wdl_workflow_2) - # Run the wdl-ci generate-config subcommand - subprocess.run( - ["wdl-ci", "generate-config"], check=True, stdout=subprocess.DEVNULL - ) - - def tearDown(self): - # Remove the WDL files - if os.path.exists("test_call-variants_1.wdl"): - os.remove("test_call-variants_1.wdl") - if os.path.exists("test_call-variants_2.wdl"): - os.remove("test_call-variants_2.wdl") - if os.path.exists("wdl-ci.config.json"): - os.remove("wdl-ci.config.json") - - def update_config_with_tests(self, wdl_1_tests, wdl_2_tests): - # Read the existing config file - with open("wdl-ci.config.json", "r") as f: - config = json.load(f) - - config["workflows"]["test_call-variants_1.wdl"]["tasks"]["freebayes"][ - "tests" - ] = wdl_1_tests - config["workflows"]["test_call-variants_2.wdl"]["tasks"]["freebayes"][ - "tests" - ] = wdl_2_tests - - # Write the updated config back to the file - with open("wdl-ci.config.json", "w") as f: - json.dump(config, f, indent=2) - - def reset_coverage_summary(self): - coverage_summary["untested_workflows"] = [] - coverage_summary["untested_tasks"] = {} - coverage_summary["untested_outputs"] = {} - coverage_summary["untested_outputs_with_optional_inputs"] = {} - coverage_summary["tested_outputs_dict"] = {} - coverage_summary["total_output_count"] = 0 - coverage_summary["all_tests_list"] = [] - coverage_summary["skipped_workflows"] = [] - - def test_no_tasks_in_workflow(self): - self.reset_coverage_summary() - # Suppress the ResourceWarning complaining about the config_file json.load not being closed - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ResourceWarning) - # Update the "tests" list for specific workflows - test_cases = [] - self.update_config_with_tests( - wdl_1_tests=test_cases, wdl_2_tests=test_cases - ) - # Call the coverage_handler function - kwargs = { - "target_coverage": None, - "workflow_name": None, - "instance": None, - "initialize": False, - } - coverage_handler(kwargs) - # Assertions - self.assertNotEqual(len(coverage_summary["untested_outputs_dict"]), 0) - self.assertEqual( - len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), - 2, - ) - self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) - - -if __name__ == "__main__": - unittest.main() From f7166e00ee6297354215ec423e2a14890672dee2 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:40:06 -0500 Subject: [PATCH 064/101] Add more tests to add --- tests/test_coverage_handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index 465adcf..7029e91 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -192,6 +192,8 @@ def test_identical_output_names(self): # self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) #### Additional tests I'd like to add #### + # Test workflows with >1 tasks + # Test WDL files with >1 tasks but no 'workflow' block # Providing a value to command to --target-coverage where all outputs/tasks/workflows are above the thresold # Providing a value to --target-coverage where only some outputs/tasks/workflows are above the threshold # Providing a value to --target-coverage where none of the above are above the threshold From 3a2100ee385a55577d5810d37502d10215dfe1d8 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:41:23 -0500 Subject: [PATCH 065/101] Comment our warning suppression so I can handle it later --- tests/test_coverage_handler.py | 99 ++++++++++++++++------------------ 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index 7029e91..c175134 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -60,7 +60,6 @@ class TestCoverageHandler(unittest.TestCase): """ def setUp(self): - # Suppress the ResourceWarning complaining about the config_file json.load not being closed # Create the WDL files with different workflow names wdl_workflow_1 = self.EXAMPLE_WDL_WORKFLOW.replace( "call_variants", "call_variants_1" @@ -116,57 +115,53 @@ def reset_coverage_summary(self): def test_identical_output_names(self): # Reset the coverage_summary self.reset_coverage_summary() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ResourceWarning) - - # Update the "tests" list for specific workflows - test_cases = [ - { - "inputs": { - "bam": "test.bam", - "ref": {"fasta": "test.fasta", "organism": "test_organism"}, - }, - "output_tests": { - "vcf": { - "value": "test.vcf", - "test_tasks": ["compare_file_basename"], - } - }, - } - ] - self.update_config_with_tests( - wdl_1_tests=test_cases, wdl_2_tests=test_cases - ) - - # Call the coverage_handler function - kwargs = {"target_coverage": None, "workflow_name": None} - coverage_handler(kwargs) - - # Assert both workflows are not in the untested workflows list - self.assertNotIn( - "call_variants_1", coverage_summary["untested_workflows_list"] - ) - self.assertNotIn( - "call_variants_2", coverage_summary["untested_workflows_list"] - ) - # Assert that the corresponding task is not in the untested tasks dictionary - self.assertNotIn("freebayes", coverage_summary["untested_tasks_dict"]) - # Assert that "vcf" is found in both sets of {workflow: {task: [tested_outputs]}} in the parent untested_outputs_with_optional_inputs_dict - self.assertIn( - "vcf", - coverage_summary["untested_outputs_with_optional_inputs_dict"][ - "call_variants_1" - ]["freebayes"], - ) - # Assert that "vcf" is found in both sets of {workflow: {task: [tested_outputs]}} in the parent tested_output_dict - self.assertIn( - "vcf", - coverage_summary["tested_outputs_dict"]["call_variants_1"]["freebayes"], - ) - self.assertIn( - "vcf", - coverage_summary["tested_outputs_dict"]["call_variants_2"]["freebayes"], - ) + + # Suppress the ResourceWarning complaining about the config_file json.load not being closed + # with warnings.catch_warnings(): + # warnings.simplefilter("ignore", ResourceWarning) + + # Update the "tests" list for specific workflows + test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + } + ] + self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) + + # Call the coverage_handler function + kwargs = {"target_coverage": None, "workflow_name": None} + coverage_handler(kwargs) + + # Assert both workflows are not in the untested workflows list + self.assertNotIn("call_variants_1", coverage_summary["untested_workflows_list"]) + self.assertNotIn("call_variants_2", coverage_summary["untested_workflows_list"]) + # Assert that the corresponding task is not in the untested tasks dictionary + self.assertNotIn("freebayes", coverage_summary["untested_tasks_dict"]) + # Assert that "vcf" is found in both sets of {workflow: {task: [tested_outputs]}} in the parent untested_outputs_with_optional_inputs_dict + self.assertIn( + "vcf", + coverage_summary["untested_outputs_with_optional_inputs_dict"][ + "call_variants_1" + ]["freebayes"], + ) + # Assert that "vcf" is found in both sets of {workflow: {task: [tested_outputs]}} in the parent tested_output_dict + self.assertIn( + "vcf", + coverage_summary["tested_outputs_dict"]["call_variants_1"]["freebayes"], + ) + self.assertIn( + "vcf", + coverage_summary["tested_outputs_dict"]["call_variants_2"]["freebayes"], + ) # def test_no_tasks_in_workflow(self): # self.reset_coverage_summary() From 9156cdcfdac5d8e097ed582c2f85f61bc6d9bdb8 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:42:53 -0500 Subject: [PATCH 066/101] Adjust name of coverage_summary dict keys --- tests/test_coverage_handler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index c175134..583a54e 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -103,14 +103,14 @@ def update_config_with_tests(self, wdl_1_tests, wdl_2_tests): json.dump(config, f, indent=2) def reset_coverage_summary(self): - coverage_summary["untested_workflows"] = [] - coverage_summary["untested_tasks"] = {} - coverage_summary["untested_outputs"] = {} - coverage_summary["untested_outputs_with_optional_inputs"] = {} + coverage_summary["untested_workflows_list"] = [] + coverage_summary["untested_tasks_dict"] = {} + coverage_summary["untested_outputs_dict"] = {} + coverage_summary["untested_outputs_with_optional_inputs_dict"] = {} coverage_summary["tested_outputs_dict"] = {} coverage_summary["total_output_count"] = 0 coverage_summary["all_tests_list"] = [] - coverage_summary["skipped_workflows"] = [] + coverage_summary["skipped_workflows_list"] = [] def test_identical_output_names(self): # Reset the coverage_summary From ba1469347ab4fc3c04a2d616ccbf84f4e4417baa Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:47:51 -0500 Subject: [PATCH 067/101] Add threshold to test --- tests/test_coverage_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index 583a54e..f82ceaf 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -138,7 +138,7 @@ def test_identical_output_names(self): self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) # Call the coverage_handler function - kwargs = {"target_coverage": None, "workflow_name": None} + kwargs = {"target_coverage": 50, "workflow_name": None} coverage_handler(kwargs) # Assert both workflows are not in the untested workflows list From 4c589e20c52f94937f76c7d05c41a40cbcb7d17e Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 16:57:18 -0500 Subject: [PATCH 068/101] rename test --- src/wdlci/cli/coverage.py | 2 +- tests/test_coverage_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 35bb42a..092b361 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -67,7 +67,7 @@ def coverage_handler(kwargs): # Check if the WDL document has > 0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for > 0 tasks misses parent workflows that just import and call other tasks/workflows. TBD if we want to include these 'parent' workflows, but ultimately, if there are no tasks or a workflow attribute, we skip the WDL file and print a warning if len(doc.tasks) > 0 or doc.workflow: - # If workflow_name_filter is provided, skip all other workflows + # If workflow_name_filter is provided and the target workflow is not the current workflow, skip the current workflow if ( workflow_name_filter is not None and workflow_name_filter not in workflow_name diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index f82ceaf..39870ad 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -112,7 +112,7 @@ def reset_coverage_summary(self): coverage_summary["all_tests_list"] = [] coverage_summary["skipped_workflows_list"] = [] - def test_identical_output_names(self): + def test_identical_output_names_with_threshold(self): # Reset the coverage_summary self.reset_coverage_summary() From a8833d4255e1cd01c7eca7675c940fc0ee743af3 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 17:09:18 -0500 Subject: [PATCH 069/101] Rename key --- tests/test_coverage_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index 39870ad..73d8bf7 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -109,7 +109,7 @@ def reset_coverage_summary(self): coverage_summary["untested_outputs_with_optional_inputs_dict"] = {} coverage_summary["tested_outputs_dict"] = {} coverage_summary["total_output_count"] = 0 - coverage_summary["all_tests_list"] = [] + coverage_summary["all_outputs_list"] = [] coverage_summary["skipped_workflows_list"] = [] def test_identical_output_names_with_threshold(self): From 660f8be3690074208de1051ad56e03acedef7f4b Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 17:11:57 -0500 Subject: [PATCH 070/101] Improve naming conventions --- src/wdlci/cli/coverage.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 092b361..8f6ce66 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -19,7 +19,7 @@ # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, - "all_tests_list": [], + "all_outputs_list": [], "skipped_workflows_list": [], } @@ -45,7 +45,8 @@ def coverage_handler(kwargs): # Iterate over each WDL file for wdl_file in wdl_files: - workflow_tests_list = [] + # TODO: Integrate these with coverage_summary + workflow_tested_outputs_list = [] workflow_output_count = 0 # Load the WDL document @@ -118,9 +119,9 @@ def coverage_handler(kwargs): if output_name not in tested_outputs ] - # Add tested outputs to workflow_tests_list and all_tests_list - workflow_tests_list.extend(tested_outputs) - coverage_summary["all_tests_list"].extend(tested_outputs) + # Add tested outputs to workflow_tested_outputs_list and all_outputs_list + workflow_tested_outputs_list.extend(tested_outputs) + coverage_summary["all_outputs_list"].extend(tested_outputs) # Add missing outputs to the coverage_summary[untested_outputs] dictionary _update_coverage_summary( @@ -187,9 +188,9 @@ def coverage_handler(kwargs): # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows - if workflow_output_count > 0 and len(workflow_tests_list) > 0: + if workflow_output_count > 0 and len(workflow_tested_outputs_list) > 0: workflow_coverage = ( - len(workflow_tests_list) / workflow_output_count + len(workflow_tested_outputs_list) / workflow_output_count ) * 100 if threshold is not None and workflow_coverage < threshold: workflows_below_threshold = True @@ -207,7 +208,7 @@ def coverage_handler(kwargs): print("-" * 150) elif ( workflow_output_count == 0 - or len(workflow_tests_list) == 0 + or len(workflow_tested_outputs_list) == 0 and workflow_name ): workflows_below_threshold = False @@ -222,12 +223,12 @@ def coverage_handler(kwargs): workflow_found = False # Calculate and print the total coverage if ( - len(coverage_summary["all_tests_list"]) > 0 + len(coverage_summary["all_outputs_list"]) > 0 and coverage_summary["total_output_count"] > 0 and not workflow_name_filter ): total_coverage = ( - len(coverage_summary["all_tests_list"]) + len(coverage_summary["all_outputs_list"]) / coverage_summary["total_output_count"] ) * 100 print("\n" + f"\033[33mTotal coverage: {total_coverage:.2f}%\033[0m") From 987a03c3e3d923cb894f604034b6955f7652dc5a Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 17:23:14 -0500 Subject: [PATCH 071/101] Add explicit tests for any remaining instances of if --- src/wdlci/cli/coverage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 8f6ce66..db734b3 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -28,7 +28,7 @@ def coverage_handler(kwargs): threshold = kwargs["target_coverage"] workflow_name_filter = kwargs["workflow_name"] print(f"Target coverage threshold: ", threshold) - if workflow_name_filter: + if workflow_name_filter is not None: print(f"Workflow name filter: {workflow_name_filter}\n") else: print("Workflow name filter: None\n") @@ -60,14 +60,14 @@ def coverage_handler(kwargs): # Now that we know the WDL file is in the configuration, we can set the workflow name from the WDL.Tree.Document workflow attribute if it exists, otherwise we can grab the workflow name from the key from the configuration file as single task WDL files do not have a workflow attribute and some workflows have no tasks. This also helps organize the coverage output when we have a WDL file with >1 task but no workflow block (e.g., https://github.com/PacificBiosciences/wdl-common/blob/main/wdl/tasks/samtools.wdl), so that each task from the WDL file is grouped under the WDL file name regardless if it's defined as a workflow or not workflow_name = ( doc.workflow.name - if doc.workflow + if doc.workflow is not None else os.path.basename(config._file.workflows[wdl_file].key).replace( ".wdl", "" ) ) # Check if the WDL document has > 0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for > 0 tasks misses parent workflows that just import and call other tasks/workflows. TBD if we want to include these 'parent' workflows, but ultimately, if there are no tasks or a workflow attribute, we skip the WDL file and print a warning - if len(doc.tasks) > 0 or doc.workflow: + if len(doc.tasks) > 0 or doc.workflow is not None: # If workflow_name_filter is provided and the target workflow is not the current workflow, skip the current workflow if ( workflow_name_filter is not None @@ -234,7 +234,7 @@ def coverage_handler(kwargs): print("\n" + f"\033[33mTotal coverage: {total_coverage:.2f}%\033[0m") # Inform the user if no workflows matched the filter and exit - if workflow_name_filter and not workflow_found: + if workflow_name_filter is not None and not workflow_found: print( f"\nNo workflows found matching the filter: [{workflow_name_filter}] or the workflow you searched for has no tasks or workflow attribute" ) From 074bde5ff6ed21d7a963ee521b9fc0d8b6ffe2fc Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 13 Dec 2024 18:13:01 -0500 Subject: [PATCH 072/101] Add clarity in some comments; add single TODO --- src/wdlci/cli/coverage.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index db734b3..6cc5da6 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -57,7 +57,7 @@ def coverage_handler(kwargs): coverage_summary["skipped_workflows_list"].append(wdl_file) continue - # Now that we know the WDL file is in the configuration, we can set the workflow name from the WDL.Tree.Document workflow attribute if it exists, otherwise we can grab the workflow name from the key from the configuration file as single task WDL files do not have a workflow attribute and some workflows have no tasks. This also helps organize the coverage output when we have a WDL file with >1 task but no workflow block (e.g., https://github.com/PacificBiosciences/wdl-common/blob/main/wdl/tasks/samtools.wdl), so that each task from the WDL file is grouped under the WDL file name regardless if it's defined as a workflow or not + # Now that we know the WDL file is in the configuration, we can set the workflow name from the WDL.Tree.Document workflow attribute if it exists. Otherwise we can grab the workflow name from the key from the configuration file as single task WDL files do not have a workflow attribute and some workflows have no tasks. This also helps organize the coverage output when we have a WDL file with >1 task but no workflow block (e.g., https://github.com/PacificBiosciences/wdl-common/blob/main/wdl/tasks/samtools.wdl), so that each task from the WDL file is grouped under the WDL file name regardless if it's defined as a workflow or not workflow_name = ( doc.workflow.name if doc.workflow is not None @@ -74,6 +74,7 @@ def coverage_handler(kwargs): and workflow_name_filter not in workflow_name ): continue + # Initialize the workflow found flag to True if the prior condition is not met workflow_found = True # Iterate over each task in the WDL document @@ -90,18 +91,19 @@ def coverage_handler(kwargs): ) # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary; duplicates are removed + # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary tested_outputs = list( set( [ output_name for test in task_tests_list for output_name, output_test in test.output_tests.items() + # Handle the case where test_tasks exists but is an empty list if len(output_test.get("test_tasks")) > 0 ] ) ) - + # Update coverage_summary with tested outputs for each task _update_coverage_summary( "tested_outputs_dict", workflow_name, @@ -160,7 +162,7 @@ def coverage_handler(kwargs): # Catch the case where tasks are completely absent from the config except KeyError: - # Initialize workflow in coverage state[untested_tasks] dict if there is a workflow in the WDL file but no tests in the config file + # Initialize workflow.task in coverage state[untested_tasks] dict if there is a workflow in the WDL file but no tests in the config file _update_coverage_summary( "untested_tasks_dict", workflow_name, task.name ) @@ -217,7 +219,7 @@ def coverage_handler(kwargs): workflow_name ) - # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks + # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks and set the workflow_found flag to False else: coverage_summary["skipped_workflows_list"].append(wdl_file) workflow_found = False @@ -258,10 +260,12 @@ def coverage_handler(kwargs): if _check_threshold(tasks_below_threshold, total_untested_outputs, threshold): print("\n✓ All outputs exceed the specified coverage threshold.") + # Warn the user if any outputs have no tests if total_untested_outputs > 0: print("\n" + "\033[31m[WARN]: The following outputs have no tests:\033[0m") _print_untested_items(coverage_summary, "untested_outputs_dict") + # Warn the user if any outputs are part of a task that contains an optional input and are not covered by at least two tests (one for each case where the optional input is and is not provided) if total_untested_outputs_with_optional_inputs > 0: # TODO: Would it be a requirement to report the specific input that is optional here? print( @@ -272,20 +276,23 @@ def coverage_handler(kwargs): coverage_summary, "untested_outputs_with_optional_inputs_dict" ) - # Check if any tasks are below the threshold and there are no untested tasks; if so return to the user that all tasks exceed the threshold + # Check if any tasks are below the threshold and there are no untested tasks; if so, return to the user that all tasks exceed the threshold if _check_threshold( tasks_below_threshold, len(coverage_summary["untested_tasks_dict"]), threshold, ): print("\n✓ All tasks exceed the specified coverage threshold.") + + # Warn the user if any tasks have no tests if len(coverage_summary["untested_tasks_dict"]) > 0: + ## TODO: confirm if I need to add the same workflow_name_filter check as below - requires testing print("\n" + "\033[31m[WARN]: The following tasks have no tests:\033[0m") for workflow, tasks in coverage_summary["untested_tasks_dict"].items(): for task in tasks: print(f"\t{workflow}.{task}") - # Check if any workflows are below the threshold and there are no untested workflows; if so return to the user that all workflows exceed the threshold + # Check if any workflows are below the threshold and there are no untested workflows; if so, return to the user that all workflows exceed the threshold if _check_threshold( workflows_below_threshold, len(coverage_summary["untested_workflows_list"]), From 0a0ec335137b9ae3355b3dbd21c0b2d1a9034a6e Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Sat, 14 Dec 2024 08:08:19 -0500 Subject: [PATCH 073/101] Avoid adding to untested_outputs_dict if there are no missing outputs --- src/wdlci/cli/coverage.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 6cc5da6..e556899 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -125,13 +125,14 @@ def coverage_handler(kwargs): workflow_tested_outputs_list.extend(tested_outputs) coverage_summary["all_outputs_list"].extend(tested_outputs) - # Add missing outputs to the coverage_summary[untested_outputs] dictionary - _update_coverage_summary( - "untested_outputs_dict", - workflow_name, - task.name, - output_names=missing_outputs, - ) + # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + if len(missing_outputs) > 0: + _update_coverage_summary( + "untested_outputs_dict", + workflow_name, + task.name, + output_names=missing_outputs, + ) # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it optional_inputs = [ From d6c82f403799ecf826f06a4232b921527640123e Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Sat, 14 Dec 2024 08:15:59 -0500 Subject: [PATCH 074/101] Avoid adding to tested_outputs_dict if there are no tested outputs --- src/wdlci/cli/coverage.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index e556899..33bcef1 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -104,12 +104,13 @@ def coverage_handler(kwargs): ) ) # Update coverage_summary with tested outputs for each task - _update_coverage_summary( - "tested_outputs_dict", - workflow_name, - task.name, - output_names=tested_outputs, - ) + if len(tested_outputs) > 0: + _update_coverage_summary( + "tested_outputs_dict", + workflow_name, + task.name, + output_names=tested_outputs, + ) # Create a list of all the outputs that are present in the task all_task_outputs = [output.name for output in task.outputs] From 351a3ade30245756e4a6c4811e60cecbce902640 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Sat, 14 Dec 2024 10:30:58 -0500 Subject: [PATCH 075/101] Add a class method to reset the config and reset it at the end of the coverage script to facilitate testing --- src/wdlci/cli/coverage.py | 1 + src/wdlci/config/__init__.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 33bcef1..8c77dd8 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -324,6 +324,7 @@ def coverage_handler(kwargs): ) for workflow in coverage_summary["skipped_workflows_list"]: print(f"\t{workflow}") + config.reset() except WdlTestCliExitException as e: print(f"exiting with code {e.exit_code}, message: {e.message}") diff --git a/src/wdlci/config/__init__.py b/src/wdlci/config/__init__.py index 5a5136a..17f8974 100644 --- a/src/wdlci/config/__init__.py +++ b/src/wdlci/config/__init__.py @@ -37,6 +37,11 @@ def instance(cls): raise WdlTestCliExitException("Config not loaded, use load() first") return cls._instance + @classmethod + def reset(cls): + cls._cli_kwargs = None + cls._instance = None + def write(self): with open(CONFIG_JSON, "w") as f: json.dump(self.file, f, cls=ConfigEncoder, indent=2) From 9e5a19bed3e5e84949f737a830d1fb37ceebc659 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Sat, 14 Dec 2024 10:32:04 -0500 Subject: [PATCH 076/101] Redirect stdout to hide coverage output; suppress warning as part of setUp; add more TODO tests --- tests/test_coverage_handler.py | 54 ++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index 73d8bf7..ea0b0c4 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -3,6 +3,7 @@ import subprocess import json import warnings +import sys from src.wdlci.cli.coverage import coverage_handler, coverage_summary @@ -60,6 +61,11 @@ class TestCoverageHandler(unittest.TestCase): """ def setUp(self): + # Redirect stdout to hide the output of the coverage command and just see the test results + self._original_stdout = sys.stdout + sys.stdout = open(os.devnull, "w") + # Suppress the ResourceWarning complaining about the config_file json.load not being closed + warnings.simplefilter("ignore", ResourceWarning) # Create the WDL files with different workflow names wdl_workflow_1 = self.EXAMPLE_WDL_WORKFLOW.replace( "call_variants", "call_variants_1" @@ -85,6 +91,8 @@ def tearDown(self): os.remove("test_call-variants_2.wdl") if os.path.exists("wdl-ci.config.json"): os.remove("wdl-ci.config.json") + sys.stdout.close() + sys.stdout = self._original_stdout def update_config_with_tests(self, wdl_1_tests, wdl_2_tests): # Read the existing config file @@ -116,10 +124,6 @@ def test_identical_output_names_with_threshold(self): # Reset the coverage_summary self.reset_coverage_summary() - # Suppress the ResourceWarning complaining about the config_file json.load not being closed - # with warnings.catch_warnings(): - # warnings.simplefilter("ignore", ResourceWarning) - # Update the "tests" list for specific workflows test_cases = [ { @@ -163,28 +167,24 @@ def test_identical_output_names_with_threshold(self): coverage_summary["tested_outputs_dict"]["call_variants_2"]["freebayes"], ) - # def test_no_tasks_in_workflow(self): - # self.reset_coverage_summary() - # # Suppress the ResourceWarning complaining about the config_file json.load not being closed - # with warnings.catch_warnings(): - # warnings.simplefilter("ignore", ResourceWarning) - # # Update the "tests" list for specific workflows - # test_cases = [] - # self.update_config_with_tests( - # wdl_1_tests=test_cases, wdl_2_tests=test_cases - # ) - # # Call the coverage_handler function - # kwargs = { - # "target_coverage": None, - # "workflow_name": None, - # } - # coverage_handler(kwargs) - # # Assertions - # self.assertNotEqual(len(coverage_summary["untested_outputs_dict"]), 0) - # self.assertEqual( - # len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), 2 - # ) - # self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) + def test_no_tasks_in_workflow(self): + self.reset_coverage_summary() + # Update the "tests" list for specific workflows + test_cases = [] + self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) + # Call the coverage_handler function + kwargs = { + "target_coverage": None, + "workflow_name": None, + } + coverage_handler(kwargs) + # Assertions + self.assertGreaterEqual(len(coverage_summary["untested_outputs_dict"]), 2) + self.assertEqual( + len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), 2 + ) + self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) + self.assertGreaterEqual(len(coverage_summary["untested_workflows_list"]), 2) #### Additional tests I'd like to add #### # Test workflows with >1 tasks @@ -196,6 +196,8 @@ def test_identical_output_names_with_threshold(self): # Providing an invalid workflow name to --workflow-name # Providing a valid workflow name to --workflow name where the workflow exists, but has no tasks # Providing an extremely large wdl-ci.config.json + # Case where some tasks with optional inputs have outputs dually tested but others do not + # Case where no tasks are tested at all if __name__ == "__main__": From 586ce4dd782569b8ed66ca0b5d4d9e3e781fa11e Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Sat, 14 Dec 2024 12:03:35 -0500 Subject: [PATCH 077/101] Set tasks below threshold to false if there is the workflow is skipped based on workflow_name filter --- src/wdlci/cli/coverage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 8c77dd8..8a82b02 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -73,6 +73,7 @@ def coverage_handler(kwargs): workflow_name_filter is not None and workflow_name_filter not in workflow_name ): + # tasks_below_threshold = False continue # Initialize the workflow found flag to True if the prior condition is not met workflow_found = True From 98e2c31a8c88cfa3fad54a982ce335599ff7c466 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Sat, 14 Dec 2024 12:04:41 -0500 Subject: [PATCH 078/101] Add more tests; still in dev --- tests/test_coverage_handler.py | 153 ++++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 22 deletions(-) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index ea0b0c4..615bf32 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -21,16 +21,23 @@ class TestCoverageHandler(unittest.TestCase): input { File bam Reference ref + String input_str } - call freebayes { + call freebayes as freebayes_1 { input: bam=bam, ref=ref } + call hello_world { + input: + input_str = input_str + } + output { - File vcf = freebayes.vcf + File vcf = freebayes_1.vcf + File greeting = hello_world.greeting } } @@ -58,6 +65,24 @@ class TestCoverageHandler(unittest.TestCase): File vcf = "${prefix}.vcf" } } + + task hello_world { + input { + String input_str + } + + command <<< + echo ~{input_str} > output_str.txt + >>> + + runtime { + docker: "ubuntu:xenial" + } + + output { + File greeting = "output_str.txt" + } + } """ def setUp(self): @@ -93,18 +118,14 @@ def tearDown(self): os.remove("wdl-ci.config.json") sys.stdout.close() sys.stdout = self._original_stdout + self.reset_coverage_summary() - def update_config_with_tests(self, wdl_1_tests, wdl_2_tests): + def update_config_with_tests(self, workflow_name, task_name, wdl_tests): # Read the existing config file with open("wdl-ci.config.json", "r") as f: config = json.load(f) - config["workflows"]["test_call-variants_1.wdl"]["tasks"]["freebayes"][ - "tests" - ] = wdl_1_tests - config["workflows"]["test_call-variants_2.wdl"]["tasks"]["freebayes"][ - "tests" - ] = wdl_2_tests + config["workflows"][workflow_name]["tasks"][task_name]["tests"] = wdl_tests # Write the updated config back to the file with open("wdl-ci.config.json", "w") as f: @@ -121,9 +142,6 @@ def reset_coverage_summary(self): coverage_summary["skipped_workflows_list"] = [] def test_identical_output_names_with_threshold(self): - # Reset the coverage_summary - self.reset_coverage_summary() - # Update the "tests" list for specific workflows test_cases = [ { @@ -139,7 +157,16 @@ def test_identical_output_names_with_threshold(self): }, } ] - self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) + self.update_config_with_tests( + workflow_name="test_call-variants_1.wdl", + task_name="freebayes", + wdl_tests=test_cases, + ) + self.update_config_with_tests( + workflow_name="test_call-variants_2.wdl", + task_name="freebayes", + wdl_tests=test_cases, + ) # Call the coverage_handler function kwargs = {"target_coverage": 50, "workflow_name": None} @@ -157,6 +184,12 @@ def test_identical_output_names_with_threshold(self): "call_variants_1" ]["freebayes"], ) + self.assertIn( + "vcf", + coverage_summary["untested_outputs_with_optional_inputs_dict"][ + "call_variants_2" + ]["freebayes"], + ) # Assert that "vcf" is found in both sets of {workflow: {task: [tested_outputs]}} in the parent tested_output_dict self.assertIn( "vcf", @@ -167,37 +200,113 @@ def test_identical_output_names_with_threshold(self): coverage_summary["tested_outputs_dict"]["call_variants_2"]["freebayes"], ) + # Case where no tasks are tested at all def test_no_tasks_in_workflow(self): - self.reset_coverage_summary() # Update the "tests" list for specific workflows test_cases = [] - self.update_config_with_tests(wdl_1_tests=test_cases, wdl_2_tests=test_cases) + self.update_config_with_tests( + workflow_name="test_call-variants_1.wdl", + task_name="freebayes", + wdl_tests=test_cases, + ) + self.update_config_with_tests( + workflow_name="test_call-variants_2.wdl", + task_name="freebayes", + wdl_tests=test_cases, + ) # Call the coverage_handler function kwargs = { "target_coverage": None, "workflow_name": None, } coverage_handler(kwargs) + # Assertions - self.assertGreaterEqual(len(coverage_summary["untested_outputs_dict"]), 2) + + # Assert all four outputs are untested + self.assertEqual( + sum( + len(tasks) + for tasks in coverage_summary["untested_outputs_dict"].values() + ), + 4, + ) + + # Assert both outputs with optional inputs are not dually tested + self.assertEqual( + sum( + len(tasks) + for tasks in coverage_summary[ + "untested_outputs_with_optional_inputs_dict" + ].values() + ), + 2, + ) + # Assert all four tasks are untested self.assertEqual( - len(coverage_summary["untested_outputs_with_optional_inputs_dict"]), 2 + sum( + len(tasks) for tasks in coverage_summary["untested_tasks_dict"].values() + ), + 4, ) - self.assertEqual(len(coverage_summary["untested_tasks_dict"]), 2) + # Assert both workflows are untested self.assertGreaterEqual(len(coverage_summary["untested_workflows_list"]), 2) + # Providing a valid workflow name to --workflow name where the workflow exists, but has no tests + def test_valid_workflow_name_with_no_tasks(self): + test_cases = [ + # { + # "inputs": {"input_str": "hello_world"}, + # "output_tests": { + # "greeting": { + # "value": "hello world", + # "test_tasks": ["compare_string"], + # } + # }, + # } + ] + # Update the workflow we are NOT filtering for + self.update_config_with_tests( + workflow_name="test_call-variants_1.wdl", + task_name="hello_world", + wdl_tests=test_cases, + ) + kwargs = {"target_coverage": None, "workflow_name": "call_variants_2"} + coverage_handler(kwargs) + + # Assertions + + # Assert one workflow is untested (in reality both are, but we are filtering for one) + self.assertEqual(len(coverage_summary["untested_workflows_list"]), 1) + + # Assert both tasks in the workflow we are filtering for are untested + self.assertEqual( + sum( + len(tasks) for tasks in coverage_summary["untested_tasks_dict"].values() + ), + 2, + ) + + # Providing an invalid workflow name to --workflow-name + ## TODO: this doesn't work as the config doesn't get reset when the workflow exits with the No workflows found method -- adjust config reset to support + # def test_invalid_workflow_name(self): + # kwargs = {"target_coverage": None, "workflow_name": "nonexistent_workflow"} + + # coverage_handler(kwargs) + # output = sys.stdout + + # # Assert that the output contains the expected message + # expected_message = f"No workflows found matching the filter: [{kwargs['workflow_name']}] or the workflow you searched for has no tasks or workflow attribute" + # self.assertIn(expected_message, output) + #### Additional tests I'd like to add #### # Test workflows with >1 tasks # Test WDL files with >1 tasks but no 'workflow' block # Providing a value to command to --target-coverage where all outputs/tasks/workflows are above the thresold # Providing a value to --target-coverage where only some outputs/tasks/workflows are above the threshold # Providing a value to --target-coverage where none of the above are above the threshold - # Providing any valid workflow name to --workflow-name - # Providing an invalid workflow name to --workflow-name - # Providing a valid workflow name to --workflow name where the workflow exists, but has no tasks # Providing an extremely large wdl-ci.config.json # Case where some tasks with optional inputs have outputs dually tested but others do not - # Case where no tasks are tested at all if __name__ == "__main__": From b271da1c97df4488ca33599c1eec7f3be0541a15 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Sun, 15 Dec 2024 20:46:29 -0500 Subject: [PATCH 079/101] Set tasks_below_threshold to false if workflows are skipped based on name; calculate task coverage using dict instead of list; add TODO --- src/wdlci/cli/coverage.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 8a82b02..033d346 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -73,7 +73,7 @@ def coverage_handler(kwargs): workflow_name_filter is not None and workflow_name_filter not in workflow_name ): - # tasks_below_threshold = False + tasks_below_threshold = False continue # Initialize the workflow found flag to True if the prior condition is not met workflow_found = True @@ -99,7 +99,7 @@ def coverage_handler(kwargs): output_name for test in task_tests_list for output_name, output_test in test.output_tests.items() - # Handle the case where test_tasks exists but is an empty list + # Handle the case where test_tasks exists but is an empty list if len(output_test.get("test_tasks")) > 0 ] ) @@ -180,7 +180,12 @@ def coverage_handler(kwargs): else: # Calculate and print the task coverage task_coverage = ( - len(tested_outputs) / len(task.outputs) + len( + coverage_summary["tested_outputs_dict"][ + workflow_name + ][task.name] + ) + / len(task.outputs) ) * 100 if threshold is not None and task_coverage < threshold: tasks_below_threshold = True @@ -195,7 +200,9 @@ def coverage_handler(kwargs): # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows if workflow_output_count > 0 and len(workflow_tested_outputs_list) > 0: workflow_coverage = ( - len(workflow_tested_outputs_list) / workflow_output_count + ## TODO: Think about adding this to the coverage_summary dict/calculating as above, after figuring out if there's any added utility + len(workflow_tested_outputs_list) + / workflow_output_count ) * 100 if threshold is not None and workflow_coverage < threshold: workflows_below_threshold = True From 2a59feb8b8d8c6b6f9e9bdec06e9d3274e7b9a5a Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 07:04:06 -0500 Subject: [PATCH 080/101] Set workflow_found via else explicitly --- src/wdlci/cli/coverage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 033d346..5fec205 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -75,8 +75,9 @@ def coverage_handler(kwargs): ): tasks_below_threshold = False continue - # Initialize the workflow found flag to True if the prior condition is not met - workflow_found = True + else: + # Initialize the workflow found flag to True if the prior condition is not met + workflow_found = True # Iterate over each task in the WDL document for task in doc.tasks: From 47c97df3bb9b4fcb4c35ee83b3b0d8e5f7fbd021 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 07:14:35 -0500 Subject: [PATCH 081/101] Remove TODO --- src/wdlci/cli/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 5fec205..0218af7 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -15,7 +15,7 @@ "untested_outputs_dict": {}, # {workflow_name: {task_name: [output_name]}} "untested_outputs_with_optional_inputs_dict": {}, - ## TODO: TBD if the tested_outputs nested dict is necessary - maybe some nuance with the calculations I'm missing right now; specifically case where outputs in different workflows or tasks that share a name -- might be preferable to return coverage from the dict so we can reference output names and the tasks they belong to. Currently, the tested_outputs_dict is updated but not used to calculate coverage. It might only be useful if we want to report the outputs that are tested for a given workflow or task. + ## TODO: TBD if the tested_outputs nested dict is necessary # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, From 9a667b9f5e365d76c12dc7de48de682243b3e831 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 08:38:31 -0500 Subject: [PATCH 082/101] Add three threshold tests; add test for optional inputs --- tests/test_coverage_handler.py | 253 +++++++++++++++++++++++++++++---- 1 file changed, 228 insertions(+), 25 deletions(-) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index 615bf32..26acdcf 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -4,6 +4,8 @@ import json import warnings import sys +from io import StringIO +from unittest.mock import patch from src.wdlci.cli.coverage import coverage_handler, coverage_summary @@ -81,6 +83,7 @@ class TestCoverageHandler(unittest.TestCase): output { File greeting = "output_str.txt" + File unused_greeting = "unused_output_str.txt" } } """ @@ -142,7 +145,6 @@ def reset_coverage_summary(self): coverage_summary["skipped_workflows_list"] = [] def test_identical_output_names_with_threshold(self): - # Update the "tests" list for specific workflows test_cases = [ { "inputs": { @@ -221,8 +223,6 @@ def test_no_tasks_in_workflow(self): } coverage_handler(kwargs) - # Assertions - # Assert all four outputs are untested self.assertEqual( sum( @@ -254,17 +254,7 @@ def test_no_tasks_in_workflow(self): # Providing a valid workflow name to --workflow name where the workflow exists, but has no tests def test_valid_workflow_name_with_no_tasks(self): - test_cases = [ - # { - # "inputs": {"input_str": "hello_world"}, - # "output_tests": { - # "greeting": { - # "value": "hello world", - # "test_tasks": ["compare_string"], - # } - # }, - # } - ] + test_cases = [] # Update the workflow we are NOT filtering for self.update_config_with_tests( workflow_name="test_call-variants_1.wdl", @@ -274,8 +264,6 @@ def test_valid_workflow_name_with_no_tasks(self): kwargs = {"target_coverage": None, "workflow_name": "call_variants_2"} coverage_handler(kwargs) - # Assertions - # Assert one workflow is untested (in reality both are, but we are filtering for one) self.assertEqual(len(coverage_summary["untested_workflows_list"]), 1) @@ -287,8 +275,231 @@ def test_valid_workflow_name_with_no_tasks(self): 2, ) + # Case where some tasks with optional inputs have outputs dually tested but others do not + def test_handling_of_optional_inputs(self): + dually_tested_optional_input_test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + "min_alternate_fraction": 0.5, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + }, + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + }, + ] + + untested_optionals_test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + "min_alternate_fraction": 0.5, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + } + ] + + self.update_config_with_tests( + workflow_name="test_call-variants_1.wdl", + task_name="freebayes", + wdl_tests=dually_tested_optional_input_test_cases, + ) + self.update_config_with_tests( + workflow_name="test_call-variants_2.wdl", + task_name="freebayes", + wdl_tests=untested_optionals_test_cases, + ) + kwargs = {"target_coverage": None, "workflow_name": None} + coverage_handler(kwargs) + + self.assertEqual( + sum( + len(outputs) + for outputs in coverage_summary[ + "untested_outputs_with_optional_inputs_dict" + ].values() + ), + 1, + ) + self.assertEqual( + list(coverage_summary["untested_outputs_with_optional_inputs_dict"].keys()), + ["call_variants_2"], + ) + + # Threshold testing cases + # Test case where workflow exceeds target coverage but outputs and tasks do not + def test_workflow_exceed_threshold(self): + workflow_threshold_pass_test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + } + ] + + self.update_config_with_tests( + workflow_name="test_call-variants_1.wdl", + task_name="freebayes", + wdl_tests=workflow_threshold_pass_test_cases, + ) + with patch("sys.stdout", new=StringIO()) as fake_out: + kwargs = {"target_coverage": 30, "workflow_name": "call_variants_1"} + coverage_handler(kwargs) + coverage_handler_stdout = fake_out.getvalue() + + expected_workflow_pass_message = ( + "All workflows exceed the specified coverage threshold" + ) + self.assertIn(expected_workflow_pass_message, coverage_handler_stdout) + + # Test case where workflow and tasks exceeds target coverage but outputs not + def test_workflow_and_tasks_exceed_threshold(self): + freebayes_workflow_and_task_threshold_pass_test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + } + ] + hello_world_workflow_and_task_threshold_pass_test_cases = [ + { + "inputs": {"input_str": "test"}, + "output_tests": { + "greeting": { + "value": "output_str_test.txt", + "test_tasks": ["compare_file_basename"], + } + }, + }, + ] + + self.update_config_with_tests( + workflow_name="test_call-variants_2.wdl", + task_name="freebayes", + wdl_tests=freebayes_workflow_and_task_threshold_pass_test_cases, + ) + self.update_config_with_tests( + workflow_name="test_call-variants_2.wdl", + task_name="hello_world", + wdl_tests=hello_world_workflow_and_task_threshold_pass_test_cases, + ) + with patch("sys.stdout", new=StringIO()) as fake_out: + kwargs = {"target_coverage": 45, "workflow_name": "call_variants_2"} + coverage_handler(kwargs) + coverage_handler_stdout = fake_out.getvalue() + + expected_workflow_pass_message = ( + "All workflows exceed the specified coverage threshold" + ) + self.assertIn(expected_workflow_pass_message, coverage_handler_stdout) + expected_task_pass_message = "All tasks exceed the specified coverage threshold" + self.assertIn(expected_task_pass_message, coverage_handler_stdout) + + # Test case where workflow, tasks, and outputs exceeds target coverage + def test_workflow_and_tasks_and_outputs_exceed_threshold(self): + freebayes_workflow_and_task_threshold_pass_test_cases = [ + { + "inputs": { + "bam": "test.bam", + "ref": {"fasta": "test.fasta", "organism": "test_organism"}, + }, + "output_tests": { + "vcf": { + "value": "test.vcf", + "test_tasks": ["compare_file_basename"], + } + }, + } + ] + hello_world_workflow_and_task_threshold_pass_test_cases = [ + { + "inputs": {"input_str": "test"}, + "output_tests": { + "greeting": { + "value": "output_str_test.txt", + "test_tasks": ["compare_file_basename"], + } + }, + }, + { + "inputs": {"input_str": "test"}, + "output_tests": { + "unused_greeting": { + "value": "output_str_test.txt", + "test_tasks": ["compare_file_basename"], + } + }, + }, + ] + + self.update_config_with_tests( + workflow_name="test_call-variants_2.wdl", + task_name="freebayes", + wdl_tests=freebayes_workflow_and_task_threshold_pass_test_cases, + ) + self.update_config_with_tests( + workflow_name="test_call-variants_2.wdl", + task_name="hello_world", + wdl_tests=hello_world_workflow_and_task_threshold_pass_test_cases, + ) + with patch("sys.stdout", new=StringIO()) as fake_out: + kwargs = {"target_coverage": 45, "workflow_name": "call_variants_2"} + coverage_handler(kwargs) + coverage_handler_stdout = fake_out.getvalue() + + expected_workflow_pass_message = ( + "All workflows exceed the specified coverage threshold" + ) + self.assertIn(expected_workflow_pass_message, coverage_handler_stdout) + expected_task_pass_message = "All tasks exceed the specified coverage threshold" + self.assertIn(expected_task_pass_message, coverage_handler_stdout) + expected_output_pass_message = ( + "All outputs exceed the specified coverage threshold" + ) + self.assertIn(expected_output_pass_message, coverage_handler_stdout) + + +if __name__ == "__main__": + unittest.main() + # Providing an invalid workflow name to --workflow-name - ## TODO: this doesn't work as the config doesn't get reset when the workflow exits with the No workflows found method -- adjust config reset to support + # # TODO: this doesn't work as the config doesn't get reset when the workflow exits with the No workflows found method -- adjust config reset to support # def test_invalid_workflow_name(self): # kwargs = {"target_coverage": None, "workflow_name": "nonexistent_workflow"} @@ -302,12 +513,4 @@ def test_valid_workflow_name_with_no_tasks(self): #### Additional tests I'd like to add #### # Test workflows with >1 tasks # Test WDL files with >1 tasks but no 'workflow' block - # Providing a value to command to --target-coverage where all outputs/tasks/workflows are above the thresold - # Providing a value to --target-coverage where only some outputs/tasks/workflows are above the threshold - # Providing a value to --target-coverage where none of the above are above the threshold # Providing an extremely large wdl-ci.config.json - # Case where some tasks with optional inputs have outputs dually tested but others do not - - -if __name__ == "__main__": - unittest.main() From db6c3fdbe5a5b44ae8b2fd7780d52da95843daad Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 09:29:34 -0500 Subject: [PATCH 083/101] Clarify TODO --- src/wdlci/cli/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 0218af7..e311af9 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -45,7 +45,7 @@ def coverage_handler(kwargs): # Iterate over each WDL file for wdl_file in wdl_files: - # TODO: Integrate these with coverage_summary + # TODO: think about integrating these with coverage_summary workflow_tested_outputs_list = [] workflow_output_count = 0 From a727fbf49b7f47c8fe743a3f93eeee3d09c77ddd Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 09:54:33 -0500 Subject: [PATCH 084/101] Remove TODO; improve naming of all tests list; handle case where no outputs to compute coverage for --- src/wdlci/cli/coverage.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index e311af9..f2eff3a 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -15,11 +15,10 @@ "untested_outputs_dict": {}, # {workflow_name: {task_name: [output_name]}} "untested_outputs_with_optional_inputs_dict": {}, - ## TODO: TBD if the tested_outputs nested dict is necessary # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, - "all_outputs_list": [], + "all_output_tests_list": [], "skipped_workflows_list": [], } @@ -124,9 +123,9 @@ def coverage_handler(kwargs): if output_name not in tested_outputs ] - # Add tested outputs to workflow_tested_outputs_list and all_outputs_list + # Add tested outputs to workflow_tested_outputs_list and all_output_tests_list workflow_tested_outputs_list.extend(tested_outputs) - coverage_summary["all_outputs_list"].extend(tested_outputs) + coverage_summary["all_output_tests_list"].extend(tested_outputs) # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs if len(missing_outputs) > 0: @@ -236,15 +235,17 @@ def coverage_handler(kwargs): workflow_found = False # Calculate and print the total coverage if ( - len(coverage_summary["all_outputs_list"]) > 0 + len(coverage_summary["all_output_tests_list"]) > 0 and coverage_summary["total_output_count"] > 0 and not workflow_name_filter ): total_coverage = ( - len(coverage_summary["all_outputs_list"]) + len(coverage_summary["all_output_tests_list"]) / coverage_summary["total_output_count"] ) * 100 print("\n" + f"\033[33mTotal coverage: {total_coverage:.2f}%\033[0m") + else: + print("There are no outputs to compute coverage for.") # Inform the user if no workflows matched the filter and exit if workflow_name_filter is not None and not workflow_found: @@ -264,10 +265,10 @@ def coverage_handler(kwargs): print("\n The following outputs are tested:") _print_untested_items(coverage_summary, "tested_outputs_dict") - # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold print("\n┍━━━━━━━━━━━━━┑") print("│ Warning(s) │") print("┕━━━━━━━━━━━━━┙") + # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold if _check_threshold(tasks_below_threshold, total_untested_outputs, threshold): print("\n✓ All outputs exceed the specified coverage threshold.") From 1645d98f92ab239a840f2cbcc019fd0ba6422101 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 09:56:06 -0500 Subject: [PATCH 085/101] Remove another TODO --- src/wdlci/cli/coverage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index f2eff3a..afc1ed9 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -298,7 +298,6 @@ def coverage_handler(kwargs): # Warn the user if any tasks have no tests if len(coverage_summary["untested_tasks_dict"]) > 0: - ## TODO: confirm if I need to add the same workflow_name_filter check as below - requires testing print("\n" + "\033[31m[WARN]: The following tasks have no tests:\033[0m") for workflow, tasks in coverage_summary["untested_tasks_dict"].items(): for task in tasks: From 59e65cc3cacb8cc9e254be5a96c5f898288a71ee Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 17:27:12 -0500 Subject: [PATCH 086/101] Rename for clarity --- src/wdlci/cli/coverage.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index afc1ed9..d059a86 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -18,7 +18,7 @@ # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, "total_output_count": 0, - "all_output_tests_list": [], + "all_tested_outputs_list": [], "skipped_workflows_list": [], } @@ -123,9 +123,11 @@ def coverage_handler(kwargs): if output_name not in tested_outputs ] - # Add tested outputs to workflow_tested_outputs_list and all_output_tests_list + # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list workflow_tested_outputs_list.extend(tested_outputs) - coverage_summary["all_output_tests_list"].extend(tested_outputs) + coverage_summary["all_tested_outputs_list"].extend( + tested_outputs + ) # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs if len(missing_outputs) > 0: @@ -235,12 +237,12 @@ def coverage_handler(kwargs): workflow_found = False # Calculate and print the total coverage if ( - len(coverage_summary["all_output_tests_list"]) > 0 + len(coverage_summary["all_tested_outputs_list"]) > 0 and coverage_summary["total_output_count"] > 0 and not workflow_name_filter ): total_coverage = ( - len(coverage_summary["all_output_tests_list"]) + len(coverage_summary["all_tested_outputs_list"]) / coverage_summary["total_output_count"] ) * 100 print("\n" + f"\033[33mTotal coverage: {total_coverage:.2f}%\033[0m") From b1f68b96edb4cde5258f15486503e9cf4d192cc4 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Mon, 16 Dec 2024 17:33:46 -0500 Subject: [PATCH 087/101] Simplify workflow_name_filter returned to user; remove check for filter that resulted in confusing output Previously, workflow_name_filter was being checked as part of the total coverage being returned; this was confusing as if it was set, a print message would tell the user there was no outputs to compute coverage for. It makes more sense to return coverage at all levels (task, workflow, and total) provided there are outputs to test --- src/wdlci/cli/coverage.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index d059a86..73a3eb5 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -27,10 +27,8 @@ def coverage_handler(kwargs): threshold = kwargs["target_coverage"] workflow_name_filter = kwargs["workflow_name"] print(f"Target coverage threshold: ", threshold) - if workflow_name_filter is not None: - print(f"Workflow name filter: {workflow_name_filter}\n") - else: - print("Workflow name filter: None\n") + print(f"Workflow name filter: {workflow_name_filter}\n") + print("┍━━━━━━━━━━━━━┑") print("│ Coverage │") print("┕━━━━━━━━━━━━━┙") @@ -239,7 +237,6 @@ def coverage_handler(kwargs): if ( len(coverage_summary["all_tested_outputs_list"]) > 0 and coverage_summary["total_output_count"] > 0 - and not workflow_name_filter ): total_coverage = ( len(coverage_summary["all_tested_outputs_list"]) From b2e46a966e983a96106c395c9caf4c22e8792a08 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Tue, 17 Dec 2024 08:03:54 -0500 Subject: [PATCH 088/101] Add test todo --- tests/test_coverage_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_coverage_handler.py b/tests/test_coverage_handler.py index 26acdcf..cafa7ee 100644 --- a/tests/test_coverage_handler.py +++ b/tests/test_coverage_handler.py @@ -514,3 +514,4 @@ def test_workflow_and_tasks_and_outputs_exceed_threshold(self): # Test workflows with >1 tasks # Test WDL files with >1 tasks but no 'workflow' block # Providing an extremely large wdl-ci.config.json + # Test case where a WDL file is not in the configuration but is present in the directory and make sure it's appended to the skipped workflows list From 3385477f2fb9b83524aa2d541c81fb4832301196 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Tue, 17 Dec 2024 09:41:30 -0500 Subject: [PATCH 089/101] Remove comment; explicitly check workflow name; rename function for clarity --- src/wdlci/cli/coverage.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 73a3eb5..d93eb0a 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -165,7 +165,6 @@ def coverage_handler(kwargs): # Catch the case where tasks are completely absent from the config except KeyError: - # Initialize workflow.task in coverage state[untested_tasks] dict if there is a workflow in the WDL file but no tests in the config file _update_coverage_summary( "untested_tasks_dict", workflow_name, task.name ) @@ -221,7 +220,7 @@ def coverage_handler(kwargs): elif ( workflow_output_count == 0 or len(workflow_tested_outputs_list) == 0 - and workflow_name + and workflow_name is not None ): workflows_below_threshold = False if workflow_name not in coverage_summary["untested_workflows_list"]: @@ -262,7 +261,7 @@ def coverage_handler(kwargs): total_tested_outputs = _sum_outputs(coverage_summary, "tested_outputs_dict") if total_tested_outputs > 0: print("\n The following outputs are tested:") - _print_untested_items(coverage_summary, "tested_outputs_dict") + _print_coverage_items(coverage_summary, "tested_outputs_dict") print("\n┍━━━━━━━━━━━━━┑") print("│ Warning(s) │") @@ -274,7 +273,7 @@ def coverage_handler(kwargs): # Warn the user if any outputs have no tests if total_untested_outputs > 0: print("\n" + "\033[31m[WARN]: The following outputs have no tests:\033[0m") - _print_untested_items(coverage_summary, "untested_outputs_dict") + _print_coverage_items(coverage_summary, "untested_outputs_dict") # Warn the user if any outputs are part of a task that contains an optional input and are not covered by at least two tests (one for each case where the optional input is and is not provided) if total_untested_outputs_with_optional_inputs > 0: @@ -283,7 +282,7 @@ def coverage_handler(kwargs): "\n" + "\033[31m[WARN]: The following outputs are part of a task that contains an optional input and are not covered by at least two tests; they should be covered for both cases where the optional input is and is not provided:\033[0m" ) - _print_untested_items( + _print_coverage_items( coverage_summary, "untested_outputs_with_optional_inputs_dict" ) @@ -373,7 +372,7 @@ def _update_coverage_summary(key, workflow_name, task_name, **kwargs): coverage_summary[key][workflow_name][task_name] = [] -def _print_untested_items(coverage_summary, key): +def _print_coverage_items(coverage_summary, key): for workflow, tasks in coverage_summary[key].items(): for task, outputs in tasks.items(): if len(outputs) > 0: From b21590a4e35234cc51a272107d7749bbcda2a0ca Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 18 Dec 2024 09:51:12 -0500 Subject: [PATCH 090/101] Move reset of config into a finally block --- src/wdlci/cli/coverage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index d93eb0a..ffd3746 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -331,12 +331,15 @@ def coverage_handler(kwargs): ) for workflow in coverage_summary["skipped_workflows_list"]: print(f"\t{workflow}") - config.reset() except WdlTestCliExitException as e: print(f"exiting with code {e.exit_code}, message: {e.message}") sys.exit(e.exit_code) + # Reset config regardless of try and except outcome + finally: + config.reset() + # Helper functions def _check_optional_inputs(task_tests_list, optional_inputs): From 87b6ae9d8300bdbefc6c86e7b2d4ffb0478eeb13 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 18 Dec 2024 13:59:30 -0500 Subject: [PATCH 091/101] Adjust if to elif --- src/wdlci/cli/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index ffd3746..0f612c7 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -271,7 +271,7 @@ def coverage_handler(kwargs): print("\n✓ All outputs exceed the specified coverage threshold.") # Warn the user if any outputs have no tests - if total_untested_outputs > 0: + elif total_untested_outputs > 0: print("\n" + "\033[31m[WARN]: The following outputs have no tests:\033[0m") _print_coverage_items(coverage_summary, "untested_outputs_dict") From 7396c865f9b35b10ec9f26a02898ad2f31bc7e16 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 18 Dec 2024 14:00:41 -0500 Subject: [PATCH 092/101] Add todo --- src/wdlci/cli/coverage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 0f612c7..12defdb 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -287,6 +287,7 @@ def coverage_handler(kwargs): ) # Check if any tasks are below the threshold and there are no untested tasks; if so, return to the user that all tasks exceed the threshold + ## TODO: Not working as intended - needs fix if _check_threshold( tasks_below_threshold, len(coverage_summary["untested_tasks_dict"]), From b391d6e39e1084827d6bf7b27536358486a5301d Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Wed, 18 Dec 2024 14:17:37 -0500 Subject: [PATCH 093/101] Simplify checking for missing outputs --- src/wdlci/cli/submit.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/wdlci/cli/submit.py b/src/wdlci/cli/submit.py index af656c8..7bca484 100644 --- a/src/wdlci/cli/submit.py +++ b/src/wdlci/cli/submit.py @@ -3,7 +3,6 @@ import sys import itertools import WDL -import re from importlib.resources import files from wdlci.auth.refresh_token_auth import RefreshTokenAuth from wdlci.config import Config @@ -70,7 +69,6 @@ def submit_handler(kwargs): tasks_to_test = dict() workflow_outputs = [] missing_outputs_list = [] - output_file_name_pattern = re.compile(r"\b\w+\s+(\S+)\s+=") for workflow_key in changeset.get_workflow_keys(): for task_key in changeset.get_tasks(workflow_key): @@ -84,22 +82,15 @@ def submit_handler(kwargs): doc_main_task = doc_tasks[task_key] # Create list of workflow output names for output in doc_main_task.outputs: - match = output_file_name_pattern.search(str(output)) - if match: - workflow_outputs.extend([match.group(1)]) + workflow_outputs.append(output.name) output_tests = test_input_set.output_tests - # Create dictionary of missing outputs - missing_outputs_dict = { - key: output_tests[key] - for key in output_tests - if key not in workflow_outputs - } - for output_key in missing_outputs_dict.keys(): - missing_outputs_list.append(output_key) + missing_outputs_list = [ + key for key in output_tests if key not in workflow_outputs + ] - if missing_outputs_list: + if len(missing_outputs_list) > 0: raise WdlTestCliExitException( f"Expected output(s): [{', '.join(missing_outputs_list)}] not found in task: {doc_main_task.name}; has this output been removed from the workflow?\nIf so, you will need to remove this output from the wdl-ci.config.json before proceeding.", 1, @@ -196,9 +187,9 @@ def submit_handler(kwargs): output_test_val_hydrated = HydrateParams.hydrate( source_params, output_test_val, update_key=False ) - output_tests_hydrated[ - output_test_key_hydrated - ] = output_test_val_hydrated + output_tests_hydrated[output_test_key_hydrated] = ( + output_test_val_hydrated + ) workflow_id = submission_state.workflows[test_key]._workflow_id From ac983365dba0c96b39876d4bf2873d3209e3a526 Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Wed, 18 Dec 2024 15:19:52 -0500 Subject: [PATCH 094/101] Allow multiple workflow filters to be passed --- src/wdlci/cli/__main__.py | 5 +-- src/wdlci/cli/coverage.py | 82 +++++++++++++++++++-------------------- 2 files changed, 41 insertions(+), 46 deletions(-) diff --git a/src/wdlci/cli/__main__.py b/src/wdlci/cli/__main__.py index 4a56d1a..1388456 100644 --- a/src/wdlci/cli/__main__.py +++ b/src/wdlci/cli/__main__.py @@ -47,10 +47,9 @@ workflow_name = click.option( "--workflow-name", "-w", - type=str, - default=None, + multiple=True, show_default=True, - help="Name of the workflow to filter coverage results (not file name)", + help="Set of workflows to filter by; should be the full path to this workflow (same as the key in the config file)", ) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 12defdb..3907b52 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -25,9 +25,13 @@ def coverage_handler(kwargs): threshold = kwargs["target_coverage"] - workflow_name_filter = kwargs["workflow_name"] + workflow_name_filters = kwargs["workflow_name"] print(f"Target coverage threshold: ", threshold) - print(f"Workflow name filter: {workflow_name_filter}\n") + print(f"Workflow name filters: {workflow_name_filters}\n") + + # TODO come back to this; compute on the fly at the end? + tasks_below_threshold = False + workflows_below_threshold = False print("┍━━━━━━━━━━━━━┑") print("│ Coverage │") @@ -40,41 +44,46 @@ def coverage_handler(kwargs): # Load all WDL files in the directory wdl_files = find_wdl_files() + # TODO filter wdl_ files to only include those in workflow_name_filter + wdl_files_filtered = [] + if len(workflow_name_filters) > 0: + for workflow_name in workflow_name_filters: + print(f"This is the workflow name: {workflow_name}") + if workflow_name in wdl_files: + wdl_files_filtered.append(workflow_name) + else: + raise WdlTestCliExitException( + f"No workflows found matching the filter: [{workflow_name}]. Possible workflow options are:\n{wdl_files}", + 1, + ) + + else: + wdl_files_filtered = [workflow_name for workflow_name in wdl_files] + # Iterate over each WDL file - for wdl_file in wdl_files: + for workflow_name in wdl_files_filtered: # TODO: think about integrating these with coverage_summary workflow_tested_outputs_list = [] workflow_output_count = 0 # Load the WDL document - doc = WDL.load(wdl_file) + doc = WDL.load(workflow_name) # Handle the case where the WDL file is not in the configuration but is present in the directory - if wdl_file not in config._file.workflows: - coverage_summary["skipped_workflows_list"].append(wdl_file) + # TODO consider whether if we are filtering on a workflow to gather this information or not + if workflow_name not in config._file.workflows: + coverage_summary["skipped_workflows_list"].append(workflow_name) continue - # Now that we know the WDL file is in the configuration, we can set the workflow name from the WDL.Tree.Document workflow attribute if it exists. Otherwise we can grab the workflow name from the key from the configuration file as single task WDL files do not have a workflow attribute and some workflows have no tasks. This also helps organize the coverage output when we have a WDL file with >1 task but no workflow block (e.g., https://github.com/PacificBiosciences/wdl-common/blob/main/wdl/tasks/samtools.wdl), so that each task from the WDL file is grouped under the WDL file name regardless if it's defined as a workflow or not - workflow_name = ( - doc.workflow.name - if doc.workflow is not None - else os.path.basename(config._file.workflows[wdl_file].key).replace( - ".wdl", "" - ) - ) - # Check if the WDL document has > 0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for > 0 tasks misses parent workflows that just import and call other tasks/workflows. TBD if we want to include these 'parent' workflows, but ultimately, if there are no tasks or a workflow attribute, we skip the WDL file and print a warning if len(doc.tasks) > 0 or doc.workflow is not None: - # If workflow_name_filter is provided and the target workflow is not the current workflow, skip the current workflow - if ( - workflow_name_filter is not None - and workflow_name_filter not in workflow_name - ): - tasks_below_threshold = False - continue - else: - # Initialize the workflow found flag to True if the prior condition is not met - workflow_found = True + # raise SystemExit(workflow_name_filters) + # If workflow_name_filters is provided and the target workflow is not the current workflow, skip the current workflow + # if ( + # workflow_name_filters is not None + # and workflow_name_filters not in workflow_name + # ): + # tasks_below_threshold = False # Iterate over each task in the WDL document for task in doc.tasks: @@ -86,7 +95,7 @@ def coverage_handler(kwargs): try: # Create a list of dictionaries for each set of task tests in our config file task_tests_list = ( - config._file.workflows[wdl_file].tasks[task.name].tests + config._file.workflows[workflow_name].tasks[task.name].tests ) # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested @@ -228,10 +237,9 @@ def coverage_handler(kwargs): workflow_name ) - # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks and set the workflow_found flag to False + # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks else: - coverage_summary["skipped_workflows_list"].append(wdl_file) - workflow_found = False + coverage_summary["skipped_workflows_list"].append(workflow_name) # Calculate and print the total coverage if ( len(coverage_summary["all_tested_outputs_list"]) > 0 @@ -245,13 +253,6 @@ def coverage_handler(kwargs): else: print("There are no outputs to compute coverage for.") - # Inform the user if no workflows matched the filter and exit - if workflow_name_filter is not None and not workflow_found: - print( - f"\nNo workflows found matching the filter: [{workflow_name_filter}] or the workflow you searched for has no tasks or workflow attribute" - ) - sys.exit(0) - # Sum the total number of untested outputs and untested outputs with optional inputs where there is not a test for the output with and without the optional input total_untested_outputs = _sum_outputs(coverage_summary, "untested_outputs_dict") total_untested_outputs_with_optional_inputs = _sum_outputs( @@ -271,7 +272,7 @@ def coverage_handler(kwargs): print("\n✓ All outputs exceed the specified coverage threshold.") # Warn the user if any outputs have no tests - elif total_untested_outputs > 0: + if total_untested_outputs > 0: print("\n" + "\033[31m[WARN]: The following outputs have no tests:\033[0m") _print_coverage_items(coverage_summary, "untested_outputs_dict") @@ -287,7 +288,6 @@ def coverage_handler(kwargs): ) # Check if any tasks are below the threshold and there are no untested tasks; if so, return to the user that all tasks exceed the threshold - ## TODO: Not working as intended - needs fix if _check_threshold( tasks_below_threshold, len(coverage_summary["untested_tasks_dict"]), @@ -312,11 +312,7 @@ def coverage_handler(kwargs): # If there are any workflows that are below the threshold, warn the user. Include a check for the workflow_name_filter to ensure that only the specified workflow is printed if a filter is provided else: - if ( - workflow_name_filter in coverage_summary["untested_workflows_list"] - or workflow_name_filter is None - and len(coverage_summary["untested_workflows_list"]) > 0 - ): + if len(coverage_summary["untested_workflows_list"]) > 0: print( "\n" + "\033[31m[WARN]: The following workflows have outputs but no tests:\033[0m" From 7def2dafce0b3efd77f10fe3f831d92633e1efe5 Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Wed, 18 Dec 2024 16:10:17 -0500 Subject: [PATCH 095/101] Move coverage summary inside function; change strategy for detecting untested optional outputs --- src/wdlci/cli/coverage.py | 114 ++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 3907b52..ad4fb68 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -6,24 +6,24 @@ from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException from wdlci.utils.initialize_worklows_and_tasks import find_wdl_files -# Initialize dictionary with necessary variables to compute coverage -coverage_summary = { - "untested_workflows_list": [], - # {workflow_name: [task_name]} - "untested_tasks_dict": {}, - # {workflow_name: {task_name: [output_name]}} - "untested_outputs_dict": {}, - # {workflow_name: {task_name: [output_name]}} - "untested_outputs_with_optional_inputs_dict": {}, - # {workflow_name: {task_name: [output_name]}} - "tested_outputs_dict": {}, - "total_output_count": 0, - "all_tested_outputs_list": [], - "skipped_workflows_list": [], -} - def coverage_handler(kwargs): + # Initialize dictionary with necessary variables to compute coverage + coverage_summary = { + "untested_workflows_list": [], + # {workflow_name: [task_name]} + "untested_tasks_dict": {}, + # {workflow_name: {task_name: [output_name]}} + "untested_outputs_dict": {}, + # {workflow_name: {task_name: [output_name]}} + "untested_outputs_with_optional_inputs_dict": {}, + # {workflow_name: {task_name: [output_name]}} + "tested_outputs_dict": {}, + "total_output_count": 0, + "all_tested_outputs_list": [], + "skipped_workflows_list": [], + } + threshold = kwargs["target_coverage"] workflow_name_filters = kwargs["workflow_name"] print(f"Target coverage threshold: ", threshold) @@ -44,11 +44,9 @@ def coverage_handler(kwargs): # Load all WDL files in the directory wdl_files = find_wdl_files() - # TODO filter wdl_ files to only include those in workflow_name_filter wdl_files_filtered = [] if len(workflow_name_filters) > 0: for workflow_name in workflow_name_filters: - print(f"This is the workflow name: {workflow_name}") if workflow_name in wdl_files: wdl_files_filtered.append(workflow_name) else: @@ -56,7 +54,6 @@ def coverage_handler(kwargs): f"No workflows found matching the filter: [{workflow_name}]. Possible workflow options are:\n{wdl_files}", 1, ) - else: wdl_files_filtered = [workflow_name for workflow_name in wdl_files] @@ -77,14 +74,6 @@ def coverage_handler(kwargs): # Check if the WDL document has > 0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for > 0 tasks misses parent workflows that just import and call other tasks/workflows. TBD if we want to include these 'parent' workflows, but ultimately, if there are no tasks or a workflow attribute, we skip the WDL file and print a warning if len(doc.tasks) > 0 or doc.workflow is not None: - # raise SystemExit(workflow_name_filters) - # If workflow_name_filters is provided and the target workflow is not the current workflow, skip the current workflow - # if ( - # workflow_name_filters is not None - # and workflow_name_filters not in workflow_name - # ): - # tasks_below_threshold = False - # Iterate over each task in the WDL document for task in doc.tasks: # Add to counters for total output count and workflow output count @@ -111,9 +100,11 @@ def coverage_handler(kwargs): ] ) ) + # Update coverage_summary with tested outputs for each task if len(tested_outputs) > 0: - _update_coverage_summary( + coverage_summary = _update_coverage_summary( + coverage_summary, "tested_outputs_dict", workflow_name, task.name, @@ -138,7 +129,8 @@ def coverage_handler(kwargs): # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs if len(missing_outputs) > 0: - _update_coverage_summary( + coverage_summary = _update_coverage_summary( + coverage_summary, "untested_outputs_dict", workflow_name, task.name, @@ -149,41 +141,52 @@ def coverage_handler(kwargs): optional_inputs = [ input.name for input in task.inputs if input.type.optional ] - outputs_where_optional_inputs_not_dually_tested = [] - if len(optional_inputs) > 0: - for output_name in all_task_outputs: - ( - tests_with_optional_inputs, - tests_without_optional_inputs, - ) = _check_optional_inputs( - task_tests_list, optional_inputs + optional_inputs_not_dually_tested = [] + for optional_input in optional_inputs: + test_exists_with_optional_set = False + test_exists_with_optional_not_set = False + for task_test in task_tests_list: + if optional_input in task_test.inputs: + test_exists_with_optional_set = True + else: + test_exists_with_optional_not_set = True + + if ( + test_exists_with_optional_set + and test_exists_with_optional_not_set + ): + print("nice") + else: + optional_inputs_not_dually_tested.append( + [output.name for output in task.outputs] ) - if ( - len(tests_with_optional_inputs) < 1 - or len(tests_without_optional_inputs) < 1 - ): - outputs_where_optional_inputs_not_dually_tested.append( - output_name - ) - _update_coverage_summary( - "untested_outputs_with_optional_inputs_dict", - workflow_name, - task.name, - output_names=outputs_where_optional_inputs_not_dually_tested, - ) + + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_outputs_with_optional_inputs_dict", + workflow_name, + task.name, + output_names=optional_inputs_not_dually_tested, + ) # Catch the case where tasks are completely absent from the config except KeyError: - _update_coverage_summary( - "untested_tasks_dict", workflow_name, task.name + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_tasks_dict", + workflow_name, + task.name, ) # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage if len(task.outputs) > 0: # Handle the case where the task is in the config but has no associated tests if len(task_tests_list) == 0: - _update_coverage_summary( - "untested_tasks_dict", workflow_name, task.name + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_tasks_dict", + workflow_name, + task.name, ) else: # Calculate and print the task coverage @@ -361,7 +364,7 @@ def _sum_outputs(coverage_summary, key): ) -def _update_coverage_summary(key, workflow_name, task_name, **kwargs): +def _update_coverage_summary(coverage_summary, key, workflow_name, task_name, **kwargs): output_names = kwargs.get("output_names", []) if workflow_name not in coverage_summary[key]: coverage_summary[key][workflow_name] = {} @@ -370,6 +373,7 @@ def _update_coverage_summary(key, workflow_name, task_name, **kwargs): coverage_summary[key][workflow_name][task_name] = output_names else: coverage_summary[key][workflow_name][task_name] = [] + return coverage_summary def _print_coverage_items(coverage_summary, key): From db8950c0da9ccee7d4b04bf70aaabd56f8118daf Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Wed, 18 Dec 2024 16:57:51 -0500 Subject: [PATCH 096/101] Moved final output messages around --- src/wdlci/cli/coverage.py | 224 ++++++++++++++++---------------------- 1 file changed, 94 insertions(+), 130 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index ad4fb68..022058d 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -59,7 +59,6 @@ def coverage_handler(kwargs): # Iterate over each WDL file for workflow_name in wdl_files_filtered: - # TODO: think about integrating these with coverage_summary workflow_tested_outputs_list = [] workflow_output_count = 0 @@ -67,7 +66,6 @@ def coverage_handler(kwargs): doc = WDL.load(workflow_name) # Handle the case where the WDL file is not in the configuration but is present in the directory - # TODO consider whether if we are filtering on a workflow to gather this information or not if workflow_name not in config._file.workflows: coverage_summary["skipped_workflows_list"].append(workflow_name) continue @@ -81,103 +79,90 @@ def coverage_handler(kwargs): workflow_output_count += len(task.outputs) # Initialize a list of task test dictionaries task_tests_list = [] - try: - # Create a list of dictionaries for each set of task tests in our config file - task_tests_list = ( - config._file.workflows[workflow_name].tasks[task.name].tests - ) - - # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary - tested_outputs = list( - set( - [ - output_name - for test in task_tests_list - for output_name, output_test in test.output_tests.items() - # Handle the case where test_tasks exists but is an empty list - if len(output_test.get("test_tasks")) > 0 - ] - ) - ) - - # Update coverage_summary with tested outputs for each task - if len(tested_outputs) > 0: - coverage_summary = _update_coverage_summary( - coverage_summary, - "tested_outputs_dict", - workflow_name, - task.name, - output_names=tested_outputs, - ) - # Create a list of all the outputs that are present in the task - all_task_outputs = [output.name for output in task.outputs] - - # Create a list of outputs that are present in the task but not in the config file - missing_outputs = [ - output_name - for output_name in all_task_outputs - if output_name not in tested_outputs - ] + # Create a list of dictionaries for each set of task tests in our config file + task_tests_list = ( + config._file.workflows[workflow_name].tasks[task.name].tests + ) - # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list - workflow_tested_outputs_list.extend(tested_outputs) - coverage_summary["all_tested_outputs_list"].extend( - tested_outputs + # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary + tested_outputs = list( + set( + [ + output_name + for test in task_tests_list + for output_name, output_test in test.output_tests.items() + # Handle the case where test_tasks exists but is an empty list + if len(output_test.get("test_tasks")) > 0 + ] ) + ) - # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs - if len(missing_outputs) > 0: - coverage_summary = _update_coverage_summary( - coverage_summary, - "untested_outputs_dict", - workflow_name, - task.name, - output_names=missing_outputs, - ) - - # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it - optional_inputs = [ - input.name for input in task.inputs if input.type.optional - ] - optional_inputs_not_dually_tested = [] - for optional_input in optional_inputs: - test_exists_with_optional_set = False - test_exists_with_optional_not_set = False - for task_test in task_tests_list: - if optional_input in task_test.inputs: - test_exists_with_optional_set = True - else: - test_exists_with_optional_not_set = True - - if ( - test_exists_with_optional_set - and test_exists_with_optional_not_set - ): - print("nice") - else: - optional_inputs_not_dually_tested.append( - [output.name for output in task.outputs] - ) - + # Update coverage_summary with tested outputs for each task + if len(tested_outputs) > 0: coverage_summary = _update_coverage_summary( coverage_summary, - "untested_outputs_with_optional_inputs_dict", + "tested_outputs_dict", workflow_name, task.name, - output_names=optional_inputs_not_dually_tested, + output_names=tested_outputs, ) - # Catch the case where tasks are completely absent from the config - except KeyError: + # Create a list of all the outputs that are present in the task + all_task_outputs = [output.name for output in task.outputs] + + # Create a list of outputs that are present in the task but not in the config file + missing_outputs = [ + output_name + for output_name in all_task_outputs + if output_name not in tested_outputs + ] + + # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list + workflow_tested_outputs_list.extend(tested_outputs) + coverage_summary["all_tested_outputs_list"].extend(tested_outputs) + + # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + if len(missing_outputs) > 0: coverage_summary = _update_coverage_summary( coverage_summary, - "untested_tasks_dict", + "untested_outputs_dict", workflow_name, task.name, + output_names=missing_outputs, ) + # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + optional_inputs = [ + input.name for input in task.inputs if input.type.optional + ] + optional_inputs_not_dually_tested = [] + for optional_input in optional_inputs: + test_exists_with_optional_set = False + test_exists_with_optional_not_set = False + for task_test in task_tests_list: + if optional_input in task_test.inputs: + test_exists_with_optional_set = True + else: + test_exists_with_optional_not_set = True + + if not ( + test_exists_with_optional_set + and test_exists_with_optional_not_set + ): + optional_inputs_not_dually_tested.extend( + [output.name for output in task.outputs] + ) + + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_outputs_with_optional_inputs_dict", + workflow_name, + task.name, + output_names=list(set(optional_inputs_not_dually_tested)), + ) + # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage if len(task.outputs) > 0: # Handle the case where the task is in the config but has no associated tests @@ -202,47 +187,30 @@ def coverage_handler(kwargs): tasks_below_threshold = True print(f"\ntask.{task.name}: {task_coverage:.2f}%") else: - tasks_below_threshold = False print(f"\ntask.{task.name}: {task_coverage:.2f}%") - else: - tasks_below_threshold = False # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows if workflow_output_count > 0 and len(workflow_tested_outputs_list) > 0: workflow_coverage = ( - ## TODO: Think about adding this to the coverage_summary dict/calculating as above, after figuring out if there's any added utility - len(workflow_tested_outputs_list) - / workflow_output_count + len(workflow_tested_outputs_list) / workflow_output_count ) * 100 if threshold is not None and workflow_coverage < threshold: workflows_below_threshold = True - print( - f"\n" - + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" - ) - print("-" * 150) - else: - workflows_below_threshold = False - print( - f"\n" - + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" - ) - print("-" * 150) + print( + f"\n" + + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" + ) + print("-" * 150) elif ( - workflow_output_count == 0 - or len(workflow_tested_outputs_list) == 0 - and workflow_name is not None + workflow_output_count == 0 or len(workflow_tested_outputs_list) == 0 ): - workflows_below_threshold = False - if workflow_name not in coverage_summary["untested_workflows_list"]: - coverage_summary["untested_workflows_list"].append( - workflow_name - ) + coverage_summary["untested_workflows_list"].append(workflow_name) # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks else: coverage_summary["skipped_workflows_list"].append(workflow_name) + # Calculate and print the total coverage if ( len(coverage_summary["all_tested_outputs_list"]) > 0 @@ -261,18 +229,25 @@ def coverage_handler(kwargs): total_untested_outputs_with_optional_inputs = _sum_outputs( coverage_summary, "untested_outputs_with_optional_inputs_dict" ) - # TODO: This might be a little overkill total_tested_outputs = _sum_outputs(coverage_summary, "tested_outputs_dict") if total_tested_outputs > 0: print("\n The following outputs are tested:") _print_coverage_items(coverage_summary, "tested_outputs_dict") - print("\n┍━━━━━━━━━━━━━┑") - print("│ Warning(s) │") - print("┕━━━━━━━━━━━━━┙") # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold + ## TODO: rephrase / assess if needed if _check_threshold(tasks_below_threshold, total_untested_outputs, threshold): - print("\n✓ All outputs exceed the specified coverage threshold.") + print("\n✓ All outputs are tested.") + + if ( + total_untested_outputs > 0 + or total_untested_outputs_with_optional_inputs > 0 + or len(coverage_summary["untested_tasks_dict"]) > 0 + or len(coverage_summary["untested_workflows_list"]) > 0 + ): + print("┍━━━━━━━━━━━━━┑") + print("│ Warning(s) │") + print("┕━━━━━━━━━━━━━┙") # Warn the user if any outputs have no tests if total_untested_outputs > 0: @@ -281,7 +256,6 @@ def coverage_handler(kwargs): # Warn the user if any outputs are part of a task that contains an optional input and are not covered by at least two tests (one for each case where the optional input is and is not provided) if total_untested_outputs_with_optional_inputs > 0: - # TODO: Would it be a requirement to report the specific input that is optional here? print( "\n" + "\033[31m[WARN]: The following outputs are part of a task that contains an optional input and are not covered by at least two tests; they should be covered for both cases where the optional input is and is not provided:\033[0m" @@ -327,7 +301,7 @@ def coverage_handler(kwargs): if len(coverage_summary["skipped_workflows_list"]) > 0: print( "\n" - + "\033[31m[WARN]: The following workflows were skipped as they were not found in the wdl-ci.config.json but are present in the directory, or they were present in the config JSON had no tasks or workflow blocks:\033[0m" + + "\033[31m[WARN]: The following workflows were skipped as they were not found in the wdl-ci.config.json but are present in the directory, or they were present in the config JSON had no task blocks:\033[0m" ) for workflow in coverage_summary["skipped_workflows_list"]: print(f"\t{workflow}") @@ -342,21 +316,11 @@ def coverage_handler(kwargs): # Helper functions -def _check_optional_inputs(task_tests_list, optional_inputs): - tests_with_optional_inputs = [ - test - for test in task_tests_list - if any(optional_input in test.inputs for optional_input in optional_inputs) - ] - tests_without_optional_inputs = [ - test - for test in task_tests_list - if all(optional_input not in test.inputs for optional_input in optional_inputs) - ] - return tests_with_optional_inputs, tests_without_optional_inputs - - def _sum_outputs(coverage_summary, key): + """ + Returns: + (int): The number of outputs for the given category + """ return sum( len(outputs) for tasks in coverage_summary[key].values() From 6d492701cbec529183fe012e653d65880410857b Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Fri, 20 Dec 2024 11:21:02 -0500 Subject: [PATCH 097/101] Add skipped workflows to check prior to printing warnings --- src/wdlci/cli/coverage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 022058d..ddc0b04 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -244,6 +244,7 @@ def coverage_handler(kwargs): or total_untested_outputs_with_optional_inputs > 0 or len(coverage_summary["untested_tasks_dict"]) > 0 or len(coverage_summary["untested_workflows_list"]) > 0 + or len(coverage_summary["skipped_workflows_list"]) ): print("┍━━━━━━━━━━━━━┑") print("│ Warning(s) │") From 2103dac2ea8c2bfe91f57e3dba06604dfea43fc5 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 8 Jan 2025 11:25:28 -0500 Subject: [PATCH 098/101] Adjust total and workflow output counts to lists; write function to update coverage for imports and tasks --- src/wdlci/cli/coverage.py | 587 +++++++++++++++++++++++++++++--------- 1 file changed, 446 insertions(+), 141 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index ddc0b04..6bd5400 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -19,7 +19,7 @@ def coverage_handler(kwargs): "untested_outputs_with_optional_inputs_dict": {}, # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, - "total_output_count": 0, + "all_outputs_list": [], "all_tested_outputs_list": [], "skipped_workflows_list": [], } @@ -60,165 +60,357 @@ def coverage_handler(kwargs): # Iterate over each WDL file for workflow_name in wdl_files_filtered: workflow_tested_outputs_list = [] - workflow_output_count = 0 - - # Load the WDL document - doc = WDL.load(workflow_name) + workflow_outputs_list = [] # Handle the case where the WDL file is not in the configuration but is present in the directory if workflow_name not in config._file.workflows: coverage_summary["skipped_workflows_list"].append(workflow_name) continue - # Check if the WDL document has > 0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for > 0 tasks misses parent workflows that just import and call other tasks/workflows. TBD if we want to include these 'parent' workflows, but ultimately, if there are no tasks or a workflow attribute, we skip the WDL file and print a warning - if len(doc.tasks) > 0 or doc.workflow is not None: + # Load the WDL document + doc = WDL.load(workflow_name) + + ## TODO: Ensure we handle the case where there are tasks AND imports, case with >1 imports, case with nested imports + + # If the WDL document has imports + ## TODO: this is only recursing a single level, e.g., family importing upstream will not lead to all outputs from upstream being reported, but tertiary will work as it only has a single level of imports + if len(doc.imports) > 0: + # Populate list of workflow imports + for wdl_import in doc.imports: + print(f"{wdl_import.namespace} is imported") + ## Need to write a function to do all the calculations below recursively based on nested imports + print(wdl_import.doc.imports) + # print(f"workflow with imports: {wdl_import.pos.uri}") + # print(f"imported workflows from above: {wdl_import.uri}") + # This handles the case where the import has tasks or imported workflows that have no tasks and only import as well (e.g., family.wdl imports upstream.wdl which also just imports other tasks/workflows). It will exclude imported workflows that don't have a workflow attribute or tasks (e.g., structs) + if ( + len(wdl_import.doc.tasks) > 0 + or wdl_import.doc.workflow is not None + ): + # Iterate over each import's set of tasks and extend outputs + for imported_task in wdl_import.doc.tasks: + # Use the imports URI to populate the task_tests_list properly + ## TODO: definitely not ideal handling as it assumes there is a parent workflows directory -- find a better approach + import_workflow_name = "workflows/" + wdl_import.uri.strip( + "../*" + ) + workflow_outputs_list.extend( + [output.name for output in imported_task.outputs] + ) + coverage_summary["all_outputs_list"].extend( + [output.name for output in imported_task.outputs] + ) + + coverage_summary = _update_coverage( + coverage_summary, + config, + import_workflow_name, + imported_task, + workflow_tested_outputs_list, + threshold, + ) + + # # Initialize a list of task test dictionaries + # task_tests_list = [] + + # # Create a list of dictionaries for each set of task tests in our config file + # task_tests_list = ( + # config._file.workflows[import_workflow_name] + # .tasks[imported_task.name] + # .tests + # ) + + # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary + # tested_outputs = list( + # set( + # [ + # output_name + # for test in task_tests_list + # for output_name, output_test in test.output_tests.items() + # # Handle the case where test_tasks exists but is an empty list + # if len(output_test.get("test_tasks")) > 0 + # ] + # ) + # ) + # # Update coverage_summary with tested outputs for each task + # if len(tested_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "tested_outputs_dict", + # workflow_name, + # imported_task.name, + # output_names=tested_outputs, + # ) + # # Create a list of all the outputs that are present in the task + # all_task_outputs = [ + # output.name for output in imported_task.outputs + # ] + + # # Create a list of outputs that are present in the task but not in the config file + # missing_outputs = [ + # output_name + # for output_name in all_task_outputs + # if output_name not in tested_outputs + # ] + + # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list + # workflow_tested_outputs_list.extend(tested_outputs) + # coverage_summary["all_tested_outputs_list"].extend( + # tested_outputs + # ) + + # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + # if len(missing_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_dict", + # workflow_name, + # imported_task.name, + # output_names=missing_outputs, + # ) + + # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + # optional_inputs = [ + # input.name + # for input in imported_task.inputs + # if input.type.optional + # ] + # optional_inputs_not_dually_tested = [] + # for optional_input in optional_inputs: + # test_exists_with_optional_set = False + # test_exists_with_optional_not_set = False + # for task_test in task_tests_list: + # if optional_input in task_test.inputs: + # test_exists_with_optional_set = True + # else: + # test_exists_with_optional_not_set = True + + # if not ( + # test_exists_with_optional_set + # and test_exists_with_optional_not_set + # ): + # optional_inputs_not_dually_tested.extend( + # [ + # output.name + # for output in imported_task.outputs + # ] + # ) + + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_with_optional_inputs_dict", + # workflow_name, + # imported_task.name, + # output_names=list( + # set(optional_inputs_not_dually_tested) + # ), + # ) + # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage + # if len(imported_task.outputs) > 0: + # # Handle the case where the task is in the config but has no associated tests + # if len(task_tests_list) == 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_tasks_dict", + # workflow_name, + # imported_task.name, + # ) + # else: + # # Calculate and print the task coverage + # task_coverage = ( + # len( + # coverage_summary["tested_outputs_dict"][ + # workflow_name + # ][imported_task.name] + # ) + # / len(imported_task.outputs) + # ) * 100 + # if ( + # threshold is not None + # and task_coverage < threshold + # ): + # tasks_below_threshold = True + # print( + # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" + # ) + # else: + # print( + # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" + # ) + # Check if the WDL document has tasks + if len(doc.tasks) > 0: # Iterate over each task in the WDL document for task in doc.tasks: - # Add to counters for total output count and workflow output count - coverage_summary["total_output_count"] += len(task.outputs) - workflow_output_count += len(task.outputs) - # Initialize a list of task test dictionaries - task_tests_list = [] - - # Create a list of dictionaries for each set of task tests in our config file - task_tests_list = ( - config._file.workflows[workflow_name].tasks[task.name].tests + # Check if the task name is present in any workflow dictionary and skip if so + task_present = any( + task.name in workflow_dict.keys() + for workflow_dict in coverage_summary[ + "tested_outputs_dict" + ].values() ) - # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary - tested_outputs = list( - set( - [ - output_name - for test in task_tests_list - for output_name, output_test in test.output_tests.items() - # Handle the case where test_tasks exists but is an empty list - if len(output_test.get("test_tasks")) > 0 - ] + ## TODO: below needs testing to make sure works as intended when scaling up config -- this is not working perfectly + if task_present: + print(f"{task.name} already in dict; is being skipped") + continue + else: + # Add outputs to all_outputs_list and add to workflow output count + coverage_summary["all_outputs_list"].extend( + [output.name for output in task.outputs] ) - ) - - # Update coverage_summary with tested outputs for each task - if len(tested_outputs) > 0: - coverage_summary = _update_coverage_summary( - coverage_summary, - "tested_outputs_dict", - workflow_name, - task.name, - output_names=tested_outputs, + workflow_outputs_list.extend( + [output.name for output in task.outputs] ) - # Create a list of all the outputs that are present in the task - all_task_outputs = [output.name for output in task.outputs] - - # Create a list of outputs that are present in the task but not in the config file - missing_outputs = [ - output_name - for output_name in all_task_outputs - if output_name not in tested_outputs - ] - - # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list - workflow_tested_outputs_list.extend(tested_outputs) - coverage_summary["all_tested_outputs_list"].extend(tested_outputs) - - # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs - if len(missing_outputs) > 0: - coverage_summary = _update_coverage_summary( + _update_coverage( coverage_summary, - "untested_outputs_dict", + config, workflow_name, - task.name, - output_names=missing_outputs, + task, + workflow_tested_outputs_list, + threshold, ) - - # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it - optional_inputs = [ - input.name for input in task.inputs if input.type.optional - ] - optional_inputs_not_dually_tested = [] - for optional_input in optional_inputs: - test_exists_with_optional_set = False - test_exists_with_optional_not_set = False - for task_test in task_tests_list: - if optional_input in task_test.inputs: - test_exists_with_optional_set = True - else: - test_exists_with_optional_not_set = True - - if not ( - test_exists_with_optional_set - and test_exists_with_optional_not_set - ): - optional_inputs_not_dually_tested.extend( - [output.name for output in task.outputs] - ) - - coverage_summary = _update_coverage_summary( - coverage_summary, - "untested_outputs_with_optional_inputs_dict", - workflow_name, - task.name, - output_names=list(set(optional_inputs_not_dually_tested)), - ) - - # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage - if len(task.outputs) > 0: - # Handle the case where the task is in the config but has no associated tests - if len(task_tests_list) == 0: - coverage_summary = _update_coverage_summary( - coverage_summary, - "untested_tasks_dict", - workflow_name, - task.name, - ) - else: - # Calculate and print the task coverage - task_coverage = ( - len( - coverage_summary["tested_outputs_dict"][ - workflow_name - ][task.name] - ) - / len(task.outputs) - ) * 100 - if threshold is not None and task_coverage < threshold: - tasks_below_threshold = True - print(f"\ntask.{task.name}: {task_coverage:.2f}%") - else: - print(f"\ntask.{task.name}: {task_coverage:.2f}%") - - # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list - # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows - if workflow_output_count > 0 and len(workflow_tested_outputs_list) > 0: - workflow_coverage = ( - len(workflow_tested_outputs_list) / workflow_output_count - ) * 100 - if threshold is not None and workflow_coverage < threshold: - workflows_below_threshold = True - print( - f"\n" - + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" - ) - print("-" * 150) - elif ( - workflow_output_count == 0 or len(workflow_tested_outputs_list) == 0 - ): - coverage_summary["untested_workflows_list"].append(workflow_name) - - # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks - else: - coverage_summary["skipped_workflows_list"].append(workflow_name) - + # # Initialize a list of task test dictionaries + # task_tests_list = [] + + # # Create a list of dictionaries for each set of task tests in our config file + # task_tests_list = ( + # config._file.workflows[workflow_name].tasks[task.name].tests + # ) + + # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary + # tested_outputs = list( + # set( + # [ + # output_name + # for test in task_tests_list + # for output_name, output_test in test.output_tests.items() + # # Handle the case where test_tasks exists but is an empty list + # if len(output_test.get("test_tasks")) > 0 + # ] + # ) + # ) + + # # Update coverage_summary with tested outputs for each task + # if len(tested_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "tested_outputs_dict", + # workflow_name, + # task.name, + # output_names=tested_outputs, + # ) + + # # Create a list of all the outputs that are present in the task + # all_task_outputs = [output.name for output in task.outputs] + + # # Create a list of outputs that are present in the task but not in the config file + # missing_outputs = [ + # output_name + # for output_name in all_task_outputs + # if output_name not in tested_outputs + # ] + + # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list + # workflow_tested_outputs_list.extend(tested_outputs) + # coverage_summary["all_tested_outputs_list"].extend( + # tested_outputs + # ) + + # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + # if len(missing_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_dict", + # workflow_name, + # task.name, + # output_names=missing_outputs, + # ) + + # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + # optional_inputs = [ + # input.name for input in task.inputs if input.type.optional + # ] + # optional_inputs_not_dually_tested = [] + # for optional_input in optional_inputs: + # test_exists_with_optional_set = False + # test_exists_with_optional_not_set = False + # for task_test in task_tests_list: + # if optional_input in task_test.inputs: + # test_exists_with_optional_set = True + # else: + # test_exists_with_optional_not_set = True + + # if not ( + # test_exists_with_optional_set + # and test_exists_with_optional_not_set + # ): + # optional_inputs_not_dually_tested.extend( + # [output.name for output in task.outputs] + # ) + + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_with_optional_inputs_dict", + # workflow_name, + # task.name, + # output_names=list(set(optional_inputs_not_dually_tested)), + # ) + + # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage + # if len(task.outputs) > 0: + # # Handle the case where the task is in the config but has no associated tests + # if len(task_tests_list) == 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_tasks_dict", + # workflow_name, + # task.name, + # ) + # else: + # # Calculate and print the task coverage + # task_coverage = ( + # len( + # coverage_summary["tested_outputs_dict"][ + # workflow_name + # ][task.name] + # ) + # / len(task.outputs) + # ) * 100 + # if threshold is not None and task_coverage < threshold: + # tasks_below_threshold = True + # print(f"\ntask.{task.name}: {task_coverage:.2f}%") + # else: + # print(f"\ntask.{task.name}: {task_coverage:.2f}%") + + # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list + # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows + if len(workflow_outputs_list) > 0 and len(workflow_tested_outputs_list) > 0: + workflow_coverage = ( + len(workflow_tested_outputs_list) / len(workflow_outputs_list) + ) * 100 + if threshold is not None and workflow_coverage < threshold: + workflows_below_threshold = True + print( + f"\n" + + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" + ) + print("-" * 150) + elif ( + len(workflow_outputs_list) == 0 + or len(workflow_tested_outputs_list) == 0 + ): + coverage_summary["untested_workflows_list"].append(workflow_name) # Calculate and print the total coverage if ( len(coverage_summary["all_tested_outputs_list"]) > 0 - and coverage_summary["total_output_count"] > 0 + and len(coverage_summary["all_outputs_list"]) > 0 ): total_coverage = ( len(coverage_summary["all_tested_outputs_list"]) - / coverage_summary["total_output_count"] + / len(coverage_summary["all_outputs_list"]) ) * 100 print("\n" + f"\033[33mTotal coverage: {total_coverage:.2f}%\033[0m") else: @@ -235,8 +427,13 @@ def coverage_handler(kwargs): _print_coverage_items(coverage_summary, "tested_outputs_dict") # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold - ## TODO: rephrase / assess if needed - if _check_threshold(tasks_below_threshold, total_untested_outputs, threshold): + + ## TODO: the below doesn't get printed because the output from the task that is imported is double counted in total tested outputs but not in all outputs list - can either double count in both or adjust the way that tested outputs are constructed in general -- TBD. + ## TODO: confirm numbers look correct once other todos are addressed + print(total_tested_outputs) + print(len(coverage_summary["all_outputs_list"])) + + if total_tested_outputs == len(coverage_summary["all_outputs_list"]): print("\n✓ All outputs are tested.") if ( @@ -295,6 +492,7 @@ def coverage_handler(kwargs): "\n" + "\033[31m[WARN]: The following workflows have outputs but no tests:\033[0m" ) + ## TODO: Need to make sure imports aren't caught here for workflow in coverage_summary["untested_workflows_list"]: print(f"\t{workflow}") @@ -311,7 +509,7 @@ def coverage_handler(kwargs): print(f"exiting with code {e.exit_code}, message: {e.message}") sys.exit(e.exit_code) - # Reset config regardless of try and except outcome + # Reset config regardless of try and except outcome - only required for running unit tests finally: config.reset() @@ -352,3 +550,110 @@ def _check_threshold(below_threshold_flag, untested_count, threshold): return ( below_threshold_flag is False and untested_count == 0 and threshold is not None ) + + +def _update_coverage( + coverage_summary, + config, + workflow_name, + task, + workflow_tested_outputs_list, + threshold, +): + # Initialize a list of task test dictionaries + task_tests_list = [] + + # Create a list of dictionaries for each set of task tests in our config file + task_tests_list = config._file.workflows[workflow_name].tasks[task.name].tests + # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary + tested_outputs = list( + set( + [ + output_name + for test in task_tests_list + for output_name, output_test in test.output_tests.items() + # Handle the case where test_tasks exists but is an empty list + if len(output_test.get("test_tasks")) > 0 + ] + ) + ) + # Update coverage_summary with tested outputs for each task + if len(tested_outputs) > 0: + coverage_summary = _update_coverage_summary( + coverage_summary, + "tested_outputs_dict", + workflow_name, + task.name, + output_names=tested_outputs, + ) + # Create a list of all the outputs that are present in the task + all_task_outputs = [output.name for output in task.outputs] + + # Create a list of outputs that are present in the task but not in the config file + missing_outputs = [ + output_name + for output_name in all_task_outputs + if output_name not in tested_outputs + ] + + # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list + workflow_tested_outputs_list.extend(tested_outputs) + coverage_summary["all_tested_outputs_list"].extend(tested_outputs) + + # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + if len(missing_outputs) > 0: + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_outputs_dict", + workflow_name, + task.name, + output_names=missing_outputs, + ) + + # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + optional_inputs = [input.name for input in task.inputs if input.type.optional] + optional_inputs_not_dually_tested = [] + for optional_input in optional_inputs: + test_exists_with_optional_set = False + test_exists_with_optional_not_set = False + for task_test in task_tests_list: + if optional_input in task_test.inputs: + test_exists_with_optional_set = True + else: + test_exists_with_optional_not_set = True + + if not (test_exists_with_optional_set and test_exists_with_optional_not_set): + optional_inputs_not_dually_tested.extend( + [output.name for output in task.outputs] + ) + + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_outputs_with_optional_inputs_dict", + workflow_name, + task.name, + output_names=list(set(optional_inputs_not_dually_tested)), + ) + # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage + if len(task.outputs) > 0: + # Handle the case where the task is in the config but has no associated tests + if len(task_tests_list) == 0: + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_tasks_dict", + workflow_name, + task.name, + ) + else: + # Calculate and print the task coverage + task_coverage = ( + len(coverage_summary["tested_outputs_dict"][workflow_name][task.name]) + / len(task.outputs) + ) * 100 + if threshold is not None and task_coverage < threshold: + tasks_below_threshold = True + print(f"\ntask.{task.name}: {task_coverage:.2f}%") + else: + print(f"\ntask.{task.name}: {task_coverage:.2f}%") + return coverage_summary From f4d8b57f0cd81f4d2328076a5144b0d27a7822e0 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 8 Jan 2025 16:03:03 -0500 Subject: [PATCH 099/101] Remove redundant code; begin trying to handle nested imports - in dev --- src/wdlci/cli/coverage.py | 306 ++++++++------------------------------ 1 file changed, 59 insertions(+), 247 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 6bd5400..3654b59 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -75,13 +75,16 @@ def coverage_handler(kwargs): # If the WDL document has imports ## TODO: this is only recursing a single level, e.g., family importing upstream will not lead to all outputs from upstream being reported, but tertiary will work as it only has a single level of imports if len(doc.imports) > 0: - # Populate list of workflow imports for wdl_import in doc.imports: - print(f"{wdl_import.namespace} is imported") - ## Need to write a function to do all the calculations below recursively based on nested imports - print(wdl_import.doc.imports) - # print(f"workflow with imports: {wdl_import.pos.uri}") - # print(f"imported workflows from above: {wdl_import.uri}") + # _process_imports( + # wdl_import, + # coverage_summary, + # config, + # workflow_outputs_list, + # workflow_tested_outputs_list, + # threshold, + # ) + # This handles the case where the import has tasks or imported workflows that have no tasks and only import as well (e.g., family.wdl imports upstream.wdl which also just imports other tasks/workflows). It will exclude imported workflows that don't have a workflow attribute or tasks (e.g., structs) if ( len(wdl_import.doc.tasks) > 0 @@ -110,134 +113,6 @@ def coverage_handler(kwargs): threshold, ) - # # Initialize a list of task test dictionaries - # task_tests_list = [] - - # # Create a list of dictionaries for each set of task tests in our config file - # task_tests_list = ( - # config._file.workflows[import_workflow_name] - # .tasks[imported_task.name] - # .tests - # ) - - # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary - # tested_outputs = list( - # set( - # [ - # output_name - # for test in task_tests_list - # for output_name, output_test in test.output_tests.items() - # # Handle the case where test_tasks exists but is an empty list - # if len(output_test.get("test_tasks")) > 0 - # ] - # ) - # ) - # # Update coverage_summary with tested outputs for each task - # if len(tested_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "tested_outputs_dict", - # workflow_name, - # imported_task.name, - # output_names=tested_outputs, - # ) - # # Create a list of all the outputs that are present in the task - # all_task_outputs = [ - # output.name for output in imported_task.outputs - # ] - - # # Create a list of outputs that are present in the task but not in the config file - # missing_outputs = [ - # output_name - # for output_name in all_task_outputs - # if output_name not in tested_outputs - # ] - - # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list - # workflow_tested_outputs_list.extend(tested_outputs) - # coverage_summary["all_tested_outputs_list"].extend( - # tested_outputs - # ) - - # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs - # if len(missing_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_dict", - # workflow_name, - # imported_task.name, - # output_names=missing_outputs, - # ) - - # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it - # optional_inputs = [ - # input.name - # for input in imported_task.inputs - # if input.type.optional - # ] - # optional_inputs_not_dually_tested = [] - # for optional_input in optional_inputs: - # test_exists_with_optional_set = False - # test_exists_with_optional_not_set = False - # for task_test in task_tests_list: - # if optional_input in task_test.inputs: - # test_exists_with_optional_set = True - # else: - # test_exists_with_optional_not_set = True - - # if not ( - # test_exists_with_optional_set - # and test_exists_with_optional_not_set - # ): - # optional_inputs_not_dually_tested.extend( - # [ - # output.name - # for output in imported_task.outputs - # ] - # ) - - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_with_optional_inputs_dict", - # workflow_name, - # imported_task.name, - # output_names=list( - # set(optional_inputs_not_dually_tested) - # ), - # ) - # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage - # if len(imported_task.outputs) > 0: - # # Handle the case where the task is in the config but has no associated tests - # if len(task_tests_list) == 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_tasks_dict", - # workflow_name, - # imported_task.name, - # ) - # else: - # # Calculate and print the task coverage - # task_coverage = ( - # len( - # coverage_summary["tested_outputs_dict"][ - # workflow_name - # ][imported_task.name] - # ) - # / len(imported_task.outputs) - # ) * 100 - # if ( - # threshold is not None - # and task_coverage < threshold - # ): - # tasks_below_threshold = True - # print( - # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" - # ) - # else: - # print( - # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" - # ) # Check if the WDL document has tasks if len(doc.tasks) > 0: # Iterate over each task in the WDL document @@ -271,119 +146,6 @@ def coverage_handler(kwargs): workflow_tested_outputs_list, threshold, ) - # # Initialize a list of task test dictionaries - # task_tests_list = [] - - # # Create a list of dictionaries for each set of task tests in our config file - # task_tests_list = ( - # config._file.workflows[workflow_name].tasks[task.name].tests - # ) - - # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary - # tested_outputs = list( - # set( - # [ - # output_name - # for test in task_tests_list - # for output_name, output_test in test.output_tests.items() - # # Handle the case where test_tasks exists but is an empty list - # if len(output_test.get("test_tasks")) > 0 - # ] - # ) - # ) - - # # Update coverage_summary with tested outputs for each task - # if len(tested_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "tested_outputs_dict", - # workflow_name, - # task.name, - # output_names=tested_outputs, - # ) - - # # Create a list of all the outputs that are present in the task - # all_task_outputs = [output.name for output in task.outputs] - - # # Create a list of outputs that are present in the task but not in the config file - # missing_outputs = [ - # output_name - # for output_name in all_task_outputs - # if output_name not in tested_outputs - # ] - - # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list - # workflow_tested_outputs_list.extend(tested_outputs) - # coverage_summary["all_tested_outputs_list"].extend( - # tested_outputs - # ) - - # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs - # if len(missing_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_dict", - # workflow_name, - # task.name, - # output_names=missing_outputs, - # ) - - # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it - # optional_inputs = [ - # input.name for input in task.inputs if input.type.optional - # ] - # optional_inputs_not_dually_tested = [] - # for optional_input in optional_inputs: - # test_exists_with_optional_set = False - # test_exists_with_optional_not_set = False - # for task_test in task_tests_list: - # if optional_input in task_test.inputs: - # test_exists_with_optional_set = True - # else: - # test_exists_with_optional_not_set = True - - # if not ( - # test_exists_with_optional_set - # and test_exists_with_optional_not_set - # ): - # optional_inputs_not_dually_tested.extend( - # [output.name for output in task.outputs] - # ) - - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_with_optional_inputs_dict", - # workflow_name, - # task.name, - # output_names=list(set(optional_inputs_not_dually_tested)), - # ) - - # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage - # if len(task.outputs) > 0: - # # Handle the case where the task is in the config but has no associated tests - # if len(task_tests_list) == 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_tasks_dict", - # workflow_name, - # task.name, - # ) - # else: - # # Calculate and print the task coverage - # task_coverage = ( - # len( - # coverage_summary["tested_outputs_dict"][ - # workflow_name - # ][task.name] - # ) - # / len(task.outputs) - # ) * 100 - # if threshold is not None and task_coverage < threshold: - # tasks_below_threshold = True - # print(f"\ntask.{task.name}: {task_coverage:.2f}%") - # else: - # print(f"\ntask.{task.name}: {task_coverage:.2f}%") # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows @@ -515,6 +277,9 @@ def coverage_handler(kwargs): # Helper functions +## TODO: Add docstrings and improve naming of functions that update coverage summary or combine + + def _sum_outputs(coverage_summary, key): """ Returns: @@ -657,3 +422,50 @@ def _update_coverage( else: print(f"\ntask.{task.name}: {task_coverage:.2f}%") return coverage_summary + + +def _process_imports( + wdl_import, + coverage_summary, + config, + workflow_outputs_list, + workflow_tested_outputs_list, + threshold, +): + print(f"{wdl_import.namespace} is imported") + + # Handle nested imports + if len(wdl_import.doc.imports) > 0: + for nested_import in wdl_import.doc.imports: + _process_imports( + nested_import, + coverage_summary, + config, + workflow_outputs_list, + workflow_tested_outputs_list, + threshold, + ) + + # This handles the case where the import has tasks or imported workflows that have no tasks and only import as well (e.g., family.wdl imports upstream.wdl which also just imports other tasks/workflows). It will exclude imported workflows that don't have a workflow attribute or tasks (e.g., structs) + if len(wdl_import.doc.tasks) > 0 or wdl_import.doc.workflow is not None: + # Iterate over each import's set of tasks and extend outputs + for imported_task in wdl_import.doc.tasks: + # Use the imports URI to populate the task_tests_list properly + ## TODO: definitely not ideal handling as it assumes there is a parent workflows directory -- find a better approach + import_workflow_name = "workflows/" + wdl_import.uri.strip("../*") + workflow_outputs_list.extend( + [output.name for output in imported_task.outputs] + ) + coverage_summary["all_outputs_list"].extend( + [output.name for output in imported_task.outputs] + ) + + coverage_summary = _update_coverage( + coverage_summary, + config, + import_workflow_name, + imported_task, + workflow_tested_outputs_list, + threshold, + ) + return coverage_summary From 8c650d015adc4875119349020220b0f7af18b37d Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 8 Jan 2025 16:07:10 -0500 Subject: [PATCH 100/101] Revert "Remove redundant code; begin trying to handle nested imports - in dev" This reverts commit f4d8b57f0cd81f4d2328076a5144b0d27a7822e0. --- src/wdlci/cli/coverage.py | 306 ++++++++++++++++++++++++++++++-------- 1 file changed, 247 insertions(+), 59 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 3654b59..6bd5400 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -75,16 +75,13 @@ def coverage_handler(kwargs): # If the WDL document has imports ## TODO: this is only recursing a single level, e.g., family importing upstream will not lead to all outputs from upstream being reported, but tertiary will work as it only has a single level of imports if len(doc.imports) > 0: + # Populate list of workflow imports for wdl_import in doc.imports: - # _process_imports( - # wdl_import, - # coverage_summary, - # config, - # workflow_outputs_list, - # workflow_tested_outputs_list, - # threshold, - # ) - + print(f"{wdl_import.namespace} is imported") + ## Need to write a function to do all the calculations below recursively based on nested imports + print(wdl_import.doc.imports) + # print(f"workflow with imports: {wdl_import.pos.uri}") + # print(f"imported workflows from above: {wdl_import.uri}") # This handles the case where the import has tasks or imported workflows that have no tasks and only import as well (e.g., family.wdl imports upstream.wdl which also just imports other tasks/workflows). It will exclude imported workflows that don't have a workflow attribute or tasks (e.g., structs) if ( len(wdl_import.doc.tasks) > 0 @@ -113,6 +110,134 @@ def coverage_handler(kwargs): threshold, ) + # # Initialize a list of task test dictionaries + # task_tests_list = [] + + # # Create a list of dictionaries for each set of task tests in our config file + # task_tests_list = ( + # config._file.workflows[import_workflow_name] + # .tasks[imported_task.name] + # .tests + # ) + + # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary + # tested_outputs = list( + # set( + # [ + # output_name + # for test in task_tests_list + # for output_name, output_test in test.output_tests.items() + # # Handle the case where test_tasks exists but is an empty list + # if len(output_test.get("test_tasks")) > 0 + # ] + # ) + # ) + # # Update coverage_summary with tested outputs for each task + # if len(tested_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "tested_outputs_dict", + # workflow_name, + # imported_task.name, + # output_names=tested_outputs, + # ) + # # Create a list of all the outputs that are present in the task + # all_task_outputs = [ + # output.name for output in imported_task.outputs + # ] + + # # Create a list of outputs that are present in the task but not in the config file + # missing_outputs = [ + # output_name + # for output_name in all_task_outputs + # if output_name not in tested_outputs + # ] + + # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list + # workflow_tested_outputs_list.extend(tested_outputs) + # coverage_summary["all_tested_outputs_list"].extend( + # tested_outputs + # ) + + # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + # if len(missing_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_dict", + # workflow_name, + # imported_task.name, + # output_names=missing_outputs, + # ) + + # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + # optional_inputs = [ + # input.name + # for input in imported_task.inputs + # if input.type.optional + # ] + # optional_inputs_not_dually_tested = [] + # for optional_input in optional_inputs: + # test_exists_with_optional_set = False + # test_exists_with_optional_not_set = False + # for task_test in task_tests_list: + # if optional_input in task_test.inputs: + # test_exists_with_optional_set = True + # else: + # test_exists_with_optional_not_set = True + + # if not ( + # test_exists_with_optional_set + # and test_exists_with_optional_not_set + # ): + # optional_inputs_not_dually_tested.extend( + # [ + # output.name + # for output in imported_task.outputs + # ] + # ) + + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_with_optional_inputs_dict", + # workflow_name, + # imported_task.name, + # output_names=list( + # set(optional_inputs_not_dually_tested) + # ), + # ) + # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage + # if len(imported_task.outputs) > 0: + # # Handle the case where the task is in the config but has no associated tests + # if len(task_tests_list) == 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_tasks_dict", + # workflow_name, + # imported_task.name, + # ) + # else: + # # Calculate and print the task coverage + # task_coverage = ( + # len( + # coverage_summary["tested_outputs_dict"][ + # workflow_name + # ][imported_task.name] + # ) + # / len(imported_task.outputs) + # ) * 100 + # if ( + # threshold is not None + # and task_coverage < threshold + # ): + # tasks_below_threshold = True + # print( + # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" + # ) + # else: + # print( + # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" + # ) # Check if the WDL document has tasks if len(doc.tasks) > 0: # Iterate over each task in the WDL document @@ -146,6 +271,119 @@ def coverage_handler(kwargs): workflow_tested_outputs_list, threshold, ) + # # Initialize a list of task test dictionaries + # task_tests_list = [] + + # # Create a list of dictionaries for each set of task tests in our config file + # task_tests_list = ( + # config._file.workflows[workflow_name].tasks[task.name].tests + # ) + + # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary + # tested_outputs = list( + # set( + # [ + # output_name + # for test in task_tests_list + # for output_name, output_test in test.output_tests.items() + # # Handle the case where test_tasks exists but is an empty list + # if len(output_test.get("test_tasks")) > 0 + # ] + # ) + # ) + + # # Update coverage_summary with tested outputs for each task + # if len(tested_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "tested_outputs_dict", + # workflow_name, + # task.name, + # output_names=tested_outputs, + # ) + + # # Create a list of all the outputs that are present in the task + # all_task_outputs = [output.name for output in task.outputs] + + # # Create a list of outputs that are present in the task but not in the config file + # missing_outputs = [ + # output_name + # for output_name in all_task_outputs + # if output_name not in tested_outputs + # ] + + # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list + # workflow_tested_outputs_list.extend(tested_outputs) + # coverage_summary["all_tested_outputs_list"].extend( + # tested_outputs + # ) + + # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + # if len(missing_outputs) > 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_dict", + # workflow_name, + # task.name, + # output_names=missing_outputs, + # ) + + # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + # optional_inputs = [ + # input.name for input in task.inputs if input.type.optional + # ] + # optional_inputs_not_dually_tested = [] + # for optional_input in optional_inputs: + # test_exists_with_optional_set = False + # test_exists_with_optional_not_set = False + # for task_test in task_tests_list: + # if optional_input in task_test.inputs: + # test_exists_with_optional_set = True + # else: + # test_exists_with_optional_not_set = True + + # if not ( + # test_exists_with_optional_set + # and test_exists_with_optional_not_set + # ): + # optional_inputs_not_dually_tested.extend( + # [output.name for output in task.outputs] + # ) + + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_outputs_with_optional_inputs_dict", + # workflow_name, + # task.name, + # output_names=list(set(optional_inputs_not_dually_tested)), + # ) + + # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage + # if len(task.outputs) > 0: + # # Handle the case where the task is in the config but has no associated tests + # if len(task_tests_list) == 0: + # coverage_summary = _update_coverage_summary( + # coverage_summary, + # "untested_tasks_dict", + # workflow_name, + # task.name, + # ) + # else: + # # Calculate and print the task coverage + # task_coverage = ( + # len( + # coverage_summary["tested_outputs_dict"][ + # workflow_name + # ][task.name] + # ) + # / len(task.outputs) + # ) * 100 + # if threshold is not None and task_coverage < threshold: + # tasks_below_threshold = True + # print(f"\ntask.{task.name}: {task_coverage:.2f}%") + # else: + # print(f"\ntask.{task.name}: {task_coverage:.2f}%") # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows @@ -277,9 +515,6 @@ def coverage_handler(kwargs): # Helper functions -## TODO: Add docstrings and improve naming of functions that update coverage summary or combine - - def _sum_outputs(coverage_summary, key): """ Returns: @@ -422,50 +657,3 @@ def _update_coverage( else: print(f"\ntask.{task.name}: {task_coverage:.2f}%") return coverage_summary - - -def _process_imports( - wdl_import, - coverage_summary, - config, - workflow_outputs_list, - workflow_tested_outputs_list, - threshold, -): - print(f"{wdl_import.namespace} is imported") - - # Handle nested imports - if len(wdl_import.doc.imports) > 0: - for nested_import in wdl_import.doc.imports: - _process_imports( - nested_import, - coverage_summary, - config, - workflow_outputs_list, - workflow_tested_outputs_list, - threshold, - ) - - # This handles the case where the import has tasks or imported workflows that have no tasks and only import as well (e.g., family.wdl imports upstream.wdl which also just imports other tasks/workflows). It will exclude imported workflows that don't have a workflow attribute or tasks (e.g., structs) - if len(wdl_import.doc.tasks) > 0 or wdl_import.doc.workflow is not None: - # Iterate over each import's set of tasks and extend outputs - for imported_task in wdl_import.doc.tasks: - # Use the imports URI to populate the task_tests_list properly - ## TODO: definitely not ideal handling as it assumes there is a parent workflows directory -- find a better approach - import_workflow_name = "workflows/" + wdl_import.uri.strip("../*") - workflow_outputs_list.extend( - [output.name for output in imported_task.outputs] - ) - coverage_summary["all_outputs_list"].extend( - [output.name for output in imported_task.outputs] - ) - - coverage_summary = _update_coverage( - coverage_summary, - config, - import_workflow_name, - imported_task, - workflow_tested_outputs_list, - threshold, - ) - return coverage_summary From e369b38bf6330e98e8c87a5a954dc19e084a6a43 Mon Sep 17 00:00:00 2001 From: geneticsjesse Date: Wed, 8 Jan 2025 16:07:20 -0500 Subject: [PATCH 101/101] Revert "Adjust total and workflow output counts to lists; write function to update coverage for imports and tasks" This reverts commit 2103dac2ea8c2bfe91f57e3dba06604dfea43fc5. --- src/wdlci/cli/coverage.py | 587 +++++++++----------------------------- 1 file changed, 141 insertions(+), 446 deletions(-) diff --git a/src/wdlci/cli/coverage.py b/src/wdlci/cli/coverage.py index 6bd5400..ddc0b04 100644 --- a/src/wdlci/cli/coverage.py +++ b/src/wdlci/cli/coverage.py @@ -19,7 +19,7 @@ def coverage_handler(kwargs): "untested_outputs_with_optional_inputs_dict": {}, # {workflow_name: {task_name: [output_name]}} "tested_outputs_dict": {}, - "all_outputs_list": [], + "total_output_count": 0, "all_tested_outputs_list": [], "skipped_workflows_list": [], } @@ -60,357 +60,165 @@ def coverage_handler(kwargs): # Iterate over each WDL file for workflow_name in wdl_files_filtered: workflow_tested_outputs_list = [] - workflow_outputs_list = [] + workflow_output_count = 0 + + # Load the WDL document + doc = WDL.load(workflow_name) # Handle the case where the WDL file is not in the configuration but is present in the directory if workflow_name not in config._file.workflows: coverage_summary["skipped_workflows_list"].append(workflow_name) continue - # Load the WDL document - doc = WDL.load(workflow_name) - - ## TODO: Ensure we handle the case where there are tasks AND imports, case with >1 imports, case with nested imports - - # If the WDL document has imports - ## TODO: this is only recursing a single level, e.g., family importing upstream will not lead to all outputs from upstream being reported, but tertiary will work as it only has a single level of imports - if len(doc.imports) > 0: - # Populate list of workflow imports - for wdl_import in doc.imports: - print(f"{wdl_import.namespace} is imported") - ## Need to write a function to do all the calculations below recursively based on nested imports - print(wdl_import.doc.imports) - # print(f"workflow with imports: {wdl_import.pos.uri}") - # print(f"imported workflows from above: {wdl_import.uri}") - # This handles the case where the import has tasks or imported workflows that have no tasks and only import as well (e.g., family.wdl imports upstream.wdl which also just imports other tasks/workflows). It will exclude imported workflows that don't have a workflow attribute or tasks (e.g., structs) - if ( - len(wdl_import.doc.tasks) > 0 - or wdl_import.doc.workflow is not None - ): - # Iterate over each import's set of tasks and extend outputs - for imported_task in wdl_import.doc.tasks: - # Use the imports URI to populate the task_tests_list properly - ## TODO: definitely not ideal handling as it assumes there is a parent workflows directory -- find a better approach - import_workflow_name = "workflows/" + wdl_import.uri.strip( - "../*" - ) - workflow_outputs_list.extend( - [output.name for output in imported_task.outputs] - ) - coverage_summary["all_outputs_list"].extend( - [output.name for output in imported_task.outputs] - ) - - coverage_summary = _update_coverage( - coverage_summary, - config, - import_workflow_name, - imported_task, - workflow_tested_outputs_list, - threshold, - ) - - # # Initialize a list of task test dictionaries - # task_tests_list = [] - - # # Create a list of dictionaries for each set of task tests in our config file - # task_tests_list = ( - # config._file.workflows[import_workflow_name] - # .tasks[imported_task.name] - # .tests - # ) - - # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary - # tested_outputs = list( - # set( - # [ - # output_name - # for test in task_tests_list - # for output_name, output_test in test.output_tests.items() - # # Handle the case where test_tasks exists but is an empty list - # if len(output_test.get("test_tasks")) > 0 - # ] - # ) - # ) - # # Update coverage_summary with tested outputs for each task - # if len(tested_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "tested_outputs_dict", - # workflow_name, - # imported_task.name, - # output_names=tested_outputs, - # ) - # # Create a list of all the outputs that are present in the task - # all_task_outputs = [ - # output.name for output in imported_task.outputs - # ] - - # # Create a list of outputs that are present in the task but not in the config file - # missing_outputs = [ - # output_name - # for output_name in all_task_outputs - # if output_name not in tested_outputs - # ] - - # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list - # workflow_tested_outputs_list.extend(tested_outputs) - # coverage_summary["all_tested_outputs_list"].extend( - # tested_outputs - # ) - - # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs - # if len(missing_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_dict", - # workflow_name, - # imported_task.name, - # output_names=missing_outputs, - # ) - - # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it - # optional_inputs = [ - # input.name - # for input in imported_task.inputs - # if input.type.optional - # ] - # optional_inputs_not_dually_tested = [] - # for optional_input in optional_inputs: - # test_exists_with_optional_set = False - # test_exists_with_optional_not_set = False - # for task_test in task_tests_list: - # if optional_input in task_test.inputs: - # test_exists_with_optional_set = True - # else: - # test_exists_with_optional_not_set = True - - # if not ( - # test_exists_with_optional_set - # and test_exists_with_optional_not_set - # ): - # optional_inputs_not_dually_tested.extend( - # [ - # output.name - # for output in imported_task.outputs - # ] - # ) - - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_with_optional_inputs_dict", - # workflow_name, - # imported_task.name, - # output_names=list( - # set(optional_inputs_not_dually_tested) - # ), - # ) - # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage - # if len(imported_task.outputs) > 0: - # # Handle the case where the task is in the config but has no associated tests - # if len(task_tests_list) == 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_tasks_dict", - # workflow_name, - # imported_task.name, - # ) - # else: - # # Calculate and print the task coverage - # task_coverage = ( - # len( - # coverage_summary["tested_outputs_dict"][ - # workflow_name - # ][imported_task.name] - # ) - # / len(imported_task.outputs) - # ) * 100 - # if ( - # threshold is not None - # and task_coverage < threshold - # ): - # tasks_below_threshold = True - # print( - # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" - # ) - # else: - # print( - # f"\ntask.{imported_task.name}: {task_coverage:.2f}%" - # ) - # Check if the WDL document has tasks - if len(doc.tasks) > 0: + # Check if the WDL document has > 0 tasks or a workflow attribute exists; structs might be part of the config and do not have tasks nor do they have outputs to test. Additionally, just checking for > 0 tasks misses parent workflows that just import and call other tasks/workflows. TBD if we want to include these 'parent' workflows, but ultimately, if there are no tasks or a workflow attribute, we skip the WDL file and print a warning + if len(doc.tasks) > 0 or doc.workflow is not None: # Iterate over each task in the WDL document for task in doc.tasks: - # Check if the task name is present in any workflow dictionary and skip if so - task_present = any( - task.name in workflow_dict.keys() - for workflow_dict in coverage_summary[ - "tested_outputs_dict" - ].values() + # Add to counters for total output count and workflow output count + coverage_summary["total_output_count"] += len(task.outputs) + workflow_output_count += len(task.outputs) + # Initialize a list of task test dictionaries + task_tests_list = [] + + # Create a list of dictionaries for each set of task tests in our config file + task_tests_list = ( + config._file.workflows[workflow_name].tasks[task.name].tests ) - ## TODO: below needs testing to make sure works as intended when scaling up config -- this is not working perfectly - if task_present: - print(f"{task.name} already in dict; is being skipped") - continue - else: - # Add outputs to all_outputs_list and add to workflow output count - coverage_summary["all_outputs_list"].extend( - [output.name for output in task.outputs] + # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested + # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary + tested_outputs = list( + set( + [ + output_name + for test in task_tests_list + for output_name, output_test in test.output_tests.items() + # Handle the case where test_tasks exists but is an empty list + if len(output_test.get("test_tasks")) > 0 + ] ) - workflow_outputs_list.extend( - [output.name for output in task.outputs] + ) + + # Update coverage_summary with tested outputs for each task + if len(tested_outputs) > 0: + coverage_summary = _update_coverage_summary( + coverage_summary, + "tested_outputs_dict", + workflow_name, + task.name, + output_names=tested_outputs, ) - _update_coverage( + # Create a list of all the outputs that are present in the task + all_task_outputs = [output.name for output in task.outputs] + + # Create a list of outputs that are present in the task but not in the config file + missing_outputs = [ + output_name + for output_name in all_task_outputs + if output_name not in tested_outputs + ] + + # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list + workflow_tested_outputs_list.extend(tested_outputs) + coverage_summary["all_tested_outputs_list"].extend(tested_outputs) + + # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs + if len(missing_outputs) > 0: + coverage_summary = _update_coverage_summary( coverage_summary, - config, + "untested_outputs_dict", workflow_name, - task, - workflow_tested_outputs_list, - threshold, + task.name, + output_names=missing_outputs, ) - # # Initialize a list of task test dictionaries - # task_tests_list = [] - - # # Create a list of dictionaries for each set of task tests in our config file - # task_tests_list = ( - # config._file.workflows[workflow_name].tasks[task.name].tests - # ) - - # # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary - # tested_outputs = list( - # set( - # [ - # output_name - # for test in task_tests_list - # for output_name, output_test in test.output_tests.items() - # # Handle the case where test_tasks exists but is an empty list - # if len(output_test.get("test_tasks")) > 0 - # ] - # ) - # ) - - # # Update coverage_summary with tested outputs for each task - # if len(tested_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "tested_outputs_dict", - # workflow_name, - # task.name, - # output_names=tested_outputs, - # ) - - # # Create a list of all the outputs that are present in the task - # all_task_outputs = [output.name for output in task.outputs] - - # # Create a list of outputs that are present in the task but not in the config file - # missing_outputs = [ - # output_name - # for output_name in all_task_outputs - # if output_name not in tested_outputs - # ] - - # # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list - # workflow_tested_outputs_list.extend(tested_outputs) - # coverage_summary["all_tested_outputs_list"].extend( - # tested_outputs - # ) - - # # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs - # if len(missing_outputs) > 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_dict", - # workflow_name, - # task.name, - # output_names=missing_outputs, - # ) - - # # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it - # optional_inputs = [ - # input.name for input in task.inputs if input.type.optional - # ] - # optional_inputs_not_dually_tested = [] - # for optional_input in optional_inputs: - # test_exists_with_optional_set = False - # test_exists_with_optional_not_set = False - # for task_test in task_tests_list: - # if optional_input in task_test.inputs: - # test_exists_with_optional_set = True - # else: - # test_exists_with_optional_not_set = True - - # if not ( - # test_exists_with_optional_set - # and test_exists_with_optional_not_set - # ): - # optional_inputs_not_dually_tested.extend( - # [output.name for output in task.outputs] - # ) - - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_outputs_with_optional_inputs_dict", - # workflow_name, - # task.name, - # output_names=list(set(optional_inputs_not_dually_tested)), - # ) - - # # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage - # if len(task.outputs) > 0: - # # Handle the case where the task is in the config but has no associated tests - # if len(task_tests_list) == 0: - # coverage_summary = _update_coverage_summary( - # coverage_summary, - # "untested_tasks_dict", - # workflow_name, - # task.name, - # ) - # else: - # # Calculate and print the task coverage - # task_coverage = ( - # len( - # coverage_summary["tested_outputs_dict"][ - # workflow_name - # ][task.name] - # ) - # / len(task.outputs) - # ) * 100 - # if threshold is not None and task_coverage < threshold: - # tasks_below_threshold = True - # print(f"\ntask.{task.name}: {task_coverage:.2f}%") - # else: - # print(f"\ntask.{task.name}: {task_coverage:.2f}%") - - # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list - # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows - if len(workflow_outputs_list) > 0 and len(workflow_tested_outputs_list) > 0: - workflow_coverage = ( - len(workflow_tested_outputs_list) / len(workflow_outputs_list) - ) * 100 - if threshold is not None and workflow_coverage < threshold: - workflows_below_threshold = True - print( - f"\n" - + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" - ) - print("-" * 150) - elif ( - len(workflow_outputs_list) == 0 - or len(workflow_tested_outputs_list) == 0 - ): - coverage_summary["untested_workflows_list"].append(workflow_name) + + # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it + optional_inputs = [ + input.name for input in task.inputs if input.type.optional + ] + optional_inputs_not_dually_tested = [] + for optional_input in optional_inputs: + test_exists_with_optional_set = False + test_exists_with_optional_not_set = False + for task_test in task_tests_list: + if optional_input in task_test.inputs: + test_exists_with_optional_set = True + else: + test_exists_with_optional_not_set = True + + if not ( + test_exists_with_optional_set + and test_exists_with_optional_not_set + ): + optional_inputs_not_dually_tested.extend( + [output.name for output in task.outputs] + ) + + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_outputs_with_optional_inputs_dict", + workflow_name, + task.name, + output_names=list(set(optional_inputs_not_dually_tested)), + ) + + # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage + if len(task.outputs) > 0: + # Handle the case where the task is in the config but has no associated tests + if len(task_tests_list) == 0: + coverage_summary = _update_coverage_summary( + coverage_summary, + "untested_tasks_dict", + workflow_name, + task.name, + ) + else: + # Calculate and print the task coverage + task_coverage = ( + len( + coverage_summary["tested_outputs_dict"][ + workflow_name + ][task.name] + ) + / len(task.outputs) + ) * 100 + if threshold is not None and task_coverage < threshold: + tasks_below_threshold = True + print(f"\ntask.{task.name}: {task_coverage:.2f}%") + else: + print(f"\ntask.{task.name}: {task_coverage:.2f}%") + + # Calculate workflow coverage; only calculate if there are outputs and tests for the workflow. If there are no outputs or tests but there is a workflow block and name, add the workflow to the untested_workflows list + # Need to make sure there is a valid workflow and that the workflow has a name; avoids trying to calculate coverage for struct workflows + if workflow_output_count > 0 and len(workflow_tested_outputs_list) > 0: + workflow_coverage = ( + len(workflow_tested_outputs_list) / workflow_output_count + ) * 100 + if threshold is not None and workflow_coverage < threshold: + workflows_below_threshold = True + print( + f"\n" + + f"\033[34mWorkflow: {workflow_name}: {workflow_coverage:.2f}%\033[0m" + ) + print("-" * 150) + elif ( + workflow_output_count == 0 or len(workflow_tested_outputs_list) == 0 + ): + coverage_summary["untested_workflows_list"].append(workflow_name) + + # Append the workflow to the skipped_workflows list if there are no tasks or workflow blocks + else: + coverage_summary["skipped_workflows_list"].append(workflow_name) + # Calculate and print the total coverage if ( len(coverage_summary["all_tested_outputs_list"]) > 0 - and len(coverage_summary["all_outputs_list"]) > 0 + and coverage_summary["total_output_count"] > 0 ): total_coverage = ( len(coverage_summary["all_tested_outputs_list"]) - / len(coverage_summary["all_outputs_list"]) + / coverage_summary["total_output_count"] ) * 100 print("\n" + f"\033[33mTotal coverage: {total_coverage:.2f}%\033[0m") else: @@ -427,13 +235,8 @@ def coverage_handler(kwargs): _print_coverage_items(coverage_summary, "tested_outputs_dict") # Check if any outputs are below the threshold and there are no untested outputs; if so return to the user that all outputs exceed the threshold - - ## TODO: the below doesn't get printed because the output from the task that is imported is double counted in total tested outputs but not in all outputs list - can either double count in both or adjust the way that tested outputs are constructed in general -- TBD. - ## TODO: confirm numbers look correct once other todos are addressed - print(total_tested_outputs) - print(len(coverage_summary["all_outputs_list"])) - - if total_tested_outputs == len(coverage_summary["all_outputs_list"]): + ## TODO: rephrase / assess if needed + if _check_threshold(tasks_below_threshold, total_untested_outputs, threshold): print("\n✓ All outputs are tested.") if ( @@ -492,7 +295,6 @@ def coverage_handler(kwargs): "\n" + "\033[31m[WARN]: The following workflows have outputs but no tests:\033[0m" ) - ## TODO: Need to make sure imports aren't caught here for workflow in coverage_summary["untested_workflows_list"]: print(f"\t{workflow}") @@ -509,7 +311,7 @@ def coverage_handler(kwargs): print(f"exiting with code {e.exit_code}, message: {e.message}") sys.exit(e.exit_code) - # Reset config regardless of try and except outcome - only required for running unit tests + # Reset config regardless of try and except outcome finally: config.reset() @@ -550,110 +352,3 @@ def _check_threshold(below_threshold_flag, untested_count, threshold): return ( below_threshold_flag is False and untested_count == 0 and threshold is not None ) - - -def _update_coverage( - coverage_summary, - config, - workflow_name, - task, - workflow_tested_outputs_list, - threshold, -): - # Initialize a list of task test dictionaries - task_tests_list = [] - - # Create a list of dictionaries for each set of task tests in our config file - task_tests_list = config._file.workflows[workflow_name].tasks[task.name].tests - # We are reducing the set of tested_outputs (output names) across input sets for the same task, ie if the same output is tested multiple times with different inputs, we'll count it as tested - # Create a list of all the outputs that are tested in the config file and found in the task output_tests dictionary - tested_outputs = list( - set( - [ - output_name - for test in task_tests_list - for output_name, output_test in test.output_tests.items() - # Handle the case where test_tasks exists but is an empty list - if len(output_test.get("test_tasks")) > 0 - ] - ) - ) - # Update coverage_summary with tested outputs for each task - if len(tested_outputs) > 0: - coverage_summary = _update_coverage_summary( - coverage_summary, - "tested_outputs_dict", - workflow_name, - task.name, - output_names=tested_outputs, - ) - # Create a list of all the outputs that are present in the task - all_task_outputs = [output.name for output in task.outputs] - - # Create a list of outputs that are present in the task but not in the config file - missing_outputs = [ - output_name - for output_name in all_task_outputs - if output_name not in tested_outputs - ] - - # Add tested outputs to workflow_tested_outputs_list and all_tested_outputs_list - workflow_tested_outputs_list.extend(tested_outputs) - coverage_summary["all_tested_outputs_list"].extend(tested_outputs) - - # Add missing outputs to the coverage_summary[untested_outputs] dictionary if there are any missing outputs - if len(missing_outputs) > 0: - coverage_summary = _update_coverage_summary( - coverage_summary, - "untested_outputs_dict", - workflow_name, - task.name, - output_names=missing_outputs, - ) - - # Check for optional inputs and check if there is a test that covers running that task with the optional input and without it - optional_inputs = [input.name for input in task.inputs if input.type.optional] - optional_inputs_not_dually_tested = [] - for optional_input in optional_inputs: - test_exists_with_optional_set = False - test_exists_with_optional_not_set = False - for task_test in task_tests_list: - if optional_input in task_test.inputs: - test_exists_with_optional_set = True - else: - test_exists_with_optional_not_set = True - - if not (test_exists_with_optional_set and test_exists_with_optional_not_set): - optional_inputs_not_dually_tested.extend( - [output.name for output in task.outputs] - ) - - coverage_summary = _update_coverage_summary( - coverage_summary, - "untested_outputs_with_optional_inputs_dict", - workflow_name, - task.name, - output_names=list(set(optional_inputs_not_dually_tested)), - ) - # If there are outputs but no tests, add the task to the untested_tasks list. If there are outputs and tests, calculate the task coverage - if len(task.outputs) > 0: - # Handle the case where the task is in the config but has no associated tests - if len(task_tests_list) == 0: - coverage_summary = _update_coverage_summary( - coverage_summary, - "untested_tasks_dict", - workflow_name, - task.name, - ) - else: - # Calculate and print the task coverage - task_coverage = ( - len(coverage_summary["tested_outputs_dict"][workflow_name][task.name]) - / len(task.outputs) - ) * 100 - if threshold is not None and task_coverage < threshold: - tasks_below_threshold = True - print(f"\ntask.{task.name}: {task_coverage:.2f}%") - else: - print(f"\ntask.{task.name}: {task_coverage:.2f}%") - return coverage_summary