Merge pull request #847 from nipype/lazy-val-setting-bugfix

Set provided non-lazy input vals when constructing workflows, if they are not set in the cached workflow

Author: tclose

docs/source/explanation/conditional-lazy.rst

@@ -13,7 +13,7 @@ Lazy fields
 
 Pydra workflows are constructed by the assignment of "lazy field" placeholders from
 the outputs of upstream nodes to the inputs of downstream nodes. These placeholders,
-which are instances of the :class:`pydra.engine.specs.LazyField` class, are replaced
+which are instances of the :class:`pydra.engine.lazy.LazyField` class, are replaced
 by the actual values they represent when the workflow is run.
 
 

docs/source/reference/api.rst

@@ -25,10 +25,10 @@ Workflows
    :undoc-members:
    :show-inheritance:
 
-Specification classes
----------------------
+Engine classes
+--------------
 
-.. automodule:: pydra.engine.specs
+.. automodule:: pydra.engine
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/tutorial/1-getting-started.ipynb

@@ -151,7 +151,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from pydra.tasks.mrtrix3.v3_0 import MrGrid\n",
+    "from pydra.tasks.mrtrix3.v3_1 import MrGrid\n",
     "\n",
     "# Instantiate the task, \"splitting\" over all NIfTI files in the test directory\n",
     "# by splitting the \"input\" input field over all files in the directory\n",
@@ -236,7 +236,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from pydra.tasks.mrtrix3.v3_0 import MrGrid\n",
+    "from pydra.tasks.mrtrix3.v3_1 import MrGrid\n",
     "\n",
     "if (\n",
     "    __name__ == \"__main__\"\n",

docs/source/tutorial/2-advanced-execution.ipynb

@@ -191,7 +191,7 @@
     "from pprint import pprint\n",
     "from fileformats.medimage import Nifti1\n",
     "from pydra.engine.submitter import Submitter\n",
-    "from pydra.tasks.mrtrix3.v3_0 import MrGrid\n",
+    "from pydra.tasks.mrtrix3.v3_1 import MrGrid\n",
     "\n",
     "# Make a temporary directory\n",
     "test_dir = Path(tempfile.mkdtemp())\n",
@@ -323,7 +323,7 @@
    "outputs": [],
    "source": [
     "import tempfile\n",
-    "from pydra.tasks.mrtrix3.v3_0 import MrGrid\n",
+    "from pydra.tasks.mrtrix3.v3_1 import MrGrid\n",
     "from pydra.environments import docker\n",
     "\n",
     "test_dir = tempfile.mkdtemp()\n",

docs/source/tutorial/6-workflow.ipynb

@@ -481,10 +481,10 @@
     "from pathlib import Path\n",
     "import numpy as np\n",
     "from fileformats.medimage import Nifti1\n",
-    "import fileformats.medimage_mrtrix3 as mrtrix3\n",
+    "import fileformats.vendor.mrtrix3.medimage as mrtrix3\n",
     "from pydra.environments import docker\n",
     "from pydra.compose import workflow, python\n",
-    "from pydra.tasks.mrtrix3.v3_0 import MrConvert, MrThreshold\n",
+    "from pydra.tasks.mrtrix3.v3_1 import MrConvert, MrThreshold\n",
     "\n",
     "MRTRIX2NUMPY_DTYPES = {\n",
     "    \"Int8\": np.dtype(\"i1\"),\n",

pydra/compose/tests/test_workflow_fields.py

@@ -266,9 +266,10 @@ def MyTestShellWorkflow(
 
     # check to see that the cache is used if we provide a concrete value for one of the
     # lazy fields
+    wf2.name = "tagged"
     workflow_spec.input_video = video.Mp4.mock("input.mp4")
     wf3 = Workflow.construct(workflow_spec)
-    assert wf3 is wf2
+    assert wf3.name == "tagged"
     assert list(workflow_cache) == [key_set]
     assert len(workflow_cache[key_set]) == 2
 

pydra/engine/workflow.py

@@ -1,7 +1,7 @@
 import logging
 import inspect
 import typing as ty
-from copy import copy
+from copy import copy, deepcopy
 from collections import defaultdict
 from typing import Self
 import attrs
@@ -115,7 +115,12 @@ def construct(
                     }
                     subset_hash = hash_function(subset_vals, cache=hash_cache)
                     if subset_hash in key_set_cache:
-                        return key_set_cache[subset_hash]
+                        wf = deepcopy(key_set_cache[subset_hash])
+                        for key in non_lazy_keys - key_set:
+                            # Set any additional non-lazy inputs that were not in the
+                            # cached workflow.
+                            setattr(wf.inputs, key, non_lazy_vals[key])
+                        return wf
 
         # Initialise the outputs of the workflow
         outputs = task.Outputs(

pyproject.toml

@@ -9,6 +9,7 @@ readme = "README.rst"
 requires-python = ">=3.11"
 dependencies = [
     "attrs >=24.2.0",
+    "click >=8.2.0",
     "cloudpickle >=3.0.0",
     "etelemetry >=0.3.1",
     "filelock >=3.0.0",
@@ -39,9 +40,11 @@ dynamic = ["version"]
 [project.optional-dependencies]
 dev = ["black", "pre-commit", "pydra[test]", "matplotlib"]
 doc = [
-    "fileformats-extras >= v0.15.0a6",
+    "fileformats-extras[application,image] >= v0.15.0a6",
     "fileformats-medimage >= v0.10.0a2",
     "fileformats-medimage-extras >= v0.10.0a2",
+    "fileformats-vendor-mrtrix3 >=3.1.0a5",
+    "fileformats-vendor-mrtrix3-extras >=3.1.0a5",
     "furo>=2022.2.14.1",
     "ipython",
     "ipykernel",
@@ -57,7 +60,7 @@ doc = [
     "packaging",
     "pandas",
     "pandoc",
-    "pydra-tasks-mrtrix3 >=3.1.0a1",
+    "pydra-tasks-mrtrix3 >=3.1.0a5",
     "scipy",
     "sphinx",
     "sphinx-argparse",
@@ -75,17 +78,19 @@ test = [
     "pytest-rerunfailures >=15.0",
     "pytest-timeout >=2.3",
     "pympler >=1.1",
-    "fileformats-extras >=0.15.0a7",
+    "fileformats-extras[application,image] >=0.15.0a7",
     "numpy >=1.26",
     "pyld >=2.0",
     "psutil >=5.9.0",
     "python-dateutil >=2.8.2",
     "tornado >=6.1",
 ]
 tutorial = [
-    "fileformats-extras >= v0.15.0a6",
+    "fileformats-extras[application,image] >= v0.15.0a6",
     "fileformats-medimage >= v0.10.0a2",
     "fileformats-medimage-extras >= v0.10.0a2",
+    "fileformats-vendor-mrtrix3 >=3.1.0a5",
+    "fileformats-vendor-mrtrix3-extras >=3.1.0a5",
     "jupyter",
     "jupyter_contrib_nbextensions",
     "jupytext",
@@ -100,7 +105,7 @@ tutorial = [
     "openneuro-py",
     "pandas",
     "psutil",
-    "pydra-tasks-mrtrix3 >=3.1.0a1",
+    "pydra-tasks-mrtrix3 >=3.1.0a5",
     "scipy",
     "sh",
 ]