Include outputs from tasks in scatters and conditionals when a workfl…

…ow has no output section (#5095)

* Stack up scatters and conditionals to find tasks and fix #5094

* Use conformance tests with correct failure expectation

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

src/toil/test/wdl/wdltoil_test.py

@@ -46,7 +46,7 @@ def tearDown(self) -> None:
 
 
 WDL_CONFORMANCE_TEST_REPO = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
-WDL_CONFORMANCE_TEST_COMMIT = "01401a46bc0e60240fb2b69af4b978d0a5bd8fc8"
+WDL_CONFORMANCE_TEST_COMMIT = "2d617b703a33791f75f30a9db43c3740a499cd89"
 # These tests are known to require things not implemented by
 # Toil and will not be run in CI.
 WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL= [

src/toil/wdl/wdltoil.py

@@ -3386,10 +3386,23 @@ def run(self, file_store: AbstractFileStore) -> WDLBindings:
                 # So get all task outputs and return that
                 # First get all task output names
                 output_set = set()
-                for call in self._workflow.body:
-                    if isinstance(call, WDL.Tree.Call):
-                        for type_binding in call.effective_outputs:
+                # We need to recurse down through scatters and conditionals to find all the task names.
+                # The output variable names won't involve the scatters or conditionals as components.
+                stack = list(self._workflow.body)
+                while stack != []:
+                    node = stack.pop()
+                    if isinstance(node, WDL.Tree.Call):
+                        # For calls, promote all output names to workflow output names
+                        # TODO: Does effective_outputs already have the right
+                        # stuff for calls to workflows that themselves lack
+                        # output sections? If so, can't we just use that for
+                        # *this* workflow?
+                        for type_binding in node.effective_outputs:
                             output_set.add(type_binding.name)
+                    elif isinstance(node, WDL.Tree.Scatter) or isinstance(node, WDL.Tree.Conditional):
+                        # For scatters and conditionals, recurse looking for calls.
+                        for subnode in node.body:
+                            stack.append(subnode)
                 # Collect all bindings that are task outputs
                 output_bindings: WDL.Env.Bindings[WDL.Value.Base] = WDL.Env.Bindings()
                 for binding in unwrap(self._bindings):