diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 0b061392..836c43c5 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -19,9 +19,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-import glob
import itertools
-import os.path
import tempfile
import unittest
@@ -34,12 +32,14 @@
import lsst.utils
import lsst.utils.tests
+from lsst.resources import ResourcePath
+
class PipelineDefintionsTestSuite(lsst.utils.tests.TestCase):
"""Tests of the self-consistency of our pipeline definitions.
"""
def setUp(self):
- self.path = os.path.join(lsst.utils.getPackageDir("ap_pipe"), "pipelines")
+ self.path = ResourcePath("eups://ap_pipe/pipelines/", forceDirectory=True)
# Each pipeline file should have a subset that represents it in
# higher-level pipelines.
self.synonyms = {"ApPipe.yaml": "apPipe",
@@ -55,30 +55,38 @@ def test_graph_build(self):
"""Test that each pipeline definition file can be
used to build a graph.
"""
- files = glob.glob(os.path.join(self.path, "**", "*.yaml"))
+ files = ResourcePath.findFileResources([self.path], file_filter=r".*\.yaml$")
for file in files:
- if "QuickTemplate" in file:
+ print("GRAPH BUILD FILE: ", file)
+ if "QuickTemplate" in file.path:
# Our QuickTemplate definition cannot be tested here because it
# depends on drp_tasks, which we cannot make a dependency here.
continue
- if "PromptTemplate" in file:
+ if "PromptTemplate" in file.path:
# Our PromptTemplate definition cannot be tested here because it
# depends on drp_tasks, which we cannot make a dependency here.
continue
- with self.subTest(file):
+ with self.subTest(file=str(file)):
pipeline = lsst.pipe.base.Pipeline.from_uri(file)
pipeline.addConfigOverride("parameters", "apdb_config", "some/file/path.yaml")
# If this fails, it will produce a useful error message.
pipeline.to_graph()
def test_datasets(self):
- files = glob.glob(os.path.join(self.path, "_ingredients", "*.yaml"))
+ files = [
+ f for f in ResourcePath.findFileResources(
+ [self.path.join("_ingredients", forceDirectory=True)], file_filter=r".*\.yaml$"
+ )
+ # Validation currently broken for injection pipelines.
+ # TODO: DM-54077
+ if "injection/" not in f.path
+ ]
for file in files:
- if "QuickTemplate" in file:
+ if "QuickTemplate" in file.path:
# Our QuickTemplate definition cannot be tested here because it
# depends on drp_tasks, which we cannot make a dependency here.
continue
- with self.subTest(file):
+ with self.subTest(file=str(file)):
expected_inputs = {
# ISR
"raw", "camera", "crosstalk", "crosstalkSources", "bias", "dark", "flat", "ptc",
@@ -91,7 +99,7 @@ def test_datasets(self):
"skyMap", "gaia_dr3_20230707", "gaia_dr2_20200414", "ps1_pv3_3pi_20170110",
"template_coadd", "pretrainedModelPackage", "dia_source_apdb"
}
- if "WithFakes" in file:
+ if "WithFakes" in file.path:
expected_inputs.add("injection_catalog")
tester = PipelineStepTester(
filename=file,
@@ -115,38 +123,48 @@ def test_whole_subset(self):
"""Test that each pipeline's synonymous subset includes all tasks,
including those imported from other files.
"""
- files = glob.glob(os.path.join(self.path, "**", "*.yaml"))
+ files = [
+ f for f in ResourcePath.findFileResources([self.path], file_filter=r".*\.yaml$")
+ # Validation currently broken for injection pipelines.
+ # TODO: DM-54077
+ if "injection/" not in f.path
+ ]
for file in files:
- if "QuickTemplate" in file:
+ if "QuickTemplate" in file.path:
# Our QuickTemplate definition cannot be tested here because it
# depends on drp_tasks, which we cannot make a dependency here.
continue
- elif "ApdbDeduplication" in file:
+ elif "ApdbDeduplication" in file.path:
# The task to export catalogs from the APDB and re-run
# association is not intended to be part of Prompt Processing
# or batch AP pipeline runs.
continue
- elif "PromptTemplate" in file:
+ elif "PromptTemplate" in file.path:
# Our PromptTemplate definition cannot be tested here because it
# depends on drp_tasks, which we cannot make a dependency here.
continue
- with self.subTest(file):
+ with self.subTest(file=str(file)):
pipeline = lsst.pipe.base.Pipeline.from_uri(file)
- subset = self.synonyms[os.path.basename(file)]
- self.assertEqual(pipeline.subsets[subset], set(pipeline.task_labels),
- msg=f"These tasks are missing from subset '{subset}'.")
+ subset = self.synonyms.get(file.basename(), "")
+ self.assertEqual(pipeline.subsets.get(subset, ""), set(pipeline.task_labels),
+ msg=f"These tasks are missing from subset '{subset}'")
def test_ap_pipe_subsets(self):
"""Test the unique subsets of ApPipe.
"""
- files = glob.glob(os.path.join(self.path, "**", "ApPipe*.yaml"))
+ files = [
+ f for f in ResourcePath.findFileResources([self.path], file_filter=r"^ApPipe.*\.yaml$")
+ # Validation currently broken for injection pipelines.
+ # TODO: DM-54077
+ if "injection/" not in f.path
+ ]
required_subsets = {"preload", "prompt", "afterburner"}
# getRegionTimeFromVisit is part of no subset besides apPipe. This is a
# very deliberate exception; see RFC-997.
no_subset_wanted = {"getRegionTimeFromVisit"}
for file in files:
- with self.subTest(file):
+ with self.subTest(file=str(file)):
pipeline = lsst.pipe.base.Pipeline.from_uri(file)
# Do all steps exist?
self.assertGreaterEqual(pipeline.subsets.keys(), required_subsets,
@@ -171,12 +189,14 @@ def test_inherited_subsets(self):
Note that this does not check inheritance *within* `_ingredients`!
"""
- files = [f for f in glob.glob(os.path.join(self.path, "**", "*.yaml"))
- if "_ingredients" not in f]
+ files = [
+ f for f in ResourcePath.findFileResources([self.path], file_filter=r".*\.yaml$")
+ if "_ingredients" not in f.path
+ ]
for file in files:
- with self.subTest(file):
- generic = os.path.join(self.path, "_ingredients", os.path.basename(file))
- if not os.path.exists(generic):
+ with self.subTest(file=str(file)):
+ generic = self.path.join("_ingredients/", forceDirectory=True).join(file.basename())
+ if not generic.exists():
continue
special_subsets = lsst.pipe.base.Pipeline.from_uri(file).subsets.keys()
generic_subsets = lsst.pipe.base.Pipeline.from_uri(generic).subsets.keys()