add entrypoint hooks and function to add config sources

src/noob/config.py

@@ -1,3 +1,5 @@
+import warnings
+from importlib.metadata import entry_points
 from pathlib import Path
 from typing import Literal
 
@@ -11,9 +13,15 @@
     YamlConfigSettingsSource,
 )
 
+from noob.exceptions import EntrypointImportWarning
+
 _default_userdir = Path().home() / ".config" / "noob"
 _dirs = PlatformDirs("noob", "noob")
 LOG_LEVELS = Literal["DEBUG", "INFO", "WARNING", "ERROR"]
+_extra_sources = []
+"""Extra sources for tube configs added by `add_sources`"""
+_entrypoint_sources: list[Path] | None = None
+"""Sources added by entrypoint functions. Initially `None`, populated on first load of a config"""
 
 
 class LogConfig(BaseModel):
@@ -126,4 +134,65 @@ def settings_customise_sources(
         )
 
 
+def add_config_source(path: Path) -> None:
+    """
+    Add a directory as a source of tube configs when searching by tube id
+    """
+    global _extra_sources
+    path = Path(path)
+    _extra_sources.append(path)
+
+
+def get_extra_sources() -> list[Path]:
+    """
+    Get the extra sources added by :func:`.add_config_source`
+
+    (avoid importing the private module-level collection anywhere else,
+    as it makes mutation weird and unpredictable)
+    """
+    global _extra_sources
+    return _extra_sources
+
+
+def get_entrypoint_sources() -> list[Path]:
+    """
+    Get additional config sources added by entrypoint functions.
+
+    Packages that ship noob tubes can make those tubes available by adding an
+    entrypoint function with a signature ``() -> list[Path]`` to their pyproject.toml
+    like:
+
+        [project.entry-points."noob.add_sources"]
+        tubes = "my_package.something:add_sources"
+
+    References:
+        https://setuptools.pypa.io/en/latest/userguide/entry_point.html
+    """
+    global _entrypoint_sources
+    if _entrypoint_sources is None:
+        _entrypoint_sources = []
+        for ext in entry_points(group="noob.add_sources"):
+            try:
+                add_sources_fn = ext.load()
+            except (ImportError, AttributeError):
+                warnings.warn(
+                    f"Config source entrypoint {ext.name}, {ext.value} "
+                    f"could not be imported, or the function could not be found. Ignoring",
+                    EntrypointImportWarning,
+                    stacklevel=1,
+                )
+                continue
+            try:
+                _entrypoint_sources.extend([Path(p) for p in add_sources_fn()])
+            except Exception as e:
+                # bare exception is fine here - we're calling external code and can't know.
+                warnings.warn(
+                    f"Config source entrypoint {ext.name}, {ext.value} "
+                    f"threw an error, or returned an invalid list of paths, ignoring.\n{str(e)}",
+                    EntrypointImportWarning,
+                    stacklevel=1,
+                )
+    return _entrypoint_sources
+
+
 config = Config()

src/noob/exceptions.py

@@ -16,6 +16,10 @@ class ConfigError(NoobError):
     """Base config error type"""
 
 
+class ConfigWarning(NoobWarning):
+    """Base config warning type"""
+
+
 class SchedulerError(NoobError):
     """Base error in the scheduler"""
 
@@ -51,6 +55,10 @@ class ConfigMismatchError(ConfigError, ValueError):
     """
 
 
+class EntrypointImportWarning(ConfigWarning, ImportWarning):
+    """Some problem with a configuration entypoint, usually when importing"""
+
+
 class AlreadyRunningError(RunnerError, RuntimeError):
     """
     A tube is already running!

src/noob/testing/entrypoint.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+ENTRYPOINT_PATH = Path("/tmp/notreal/path")
+
+
+def some_entrypoint_fn() -> list[Path]:
+    """See test_config and config.get_entrypoint_sources"""
+    return [ENTRYPOINT_PATH]

src/noob/yaml.py

@@ -199,9 +199,9 @@ def config_sources(cls: type[Self]) -> list[Path]:
         Directories to search for config files, in order of priority
         such that earlier sources are preferred over later sources.
         """
-        from noob.config import Config
+        from noob.config import Config, get_entrypoint_sources, get_extra_sources
 
-        return [Config().config_dir]
+        return [Config().config_dir, *get_extra_sources(), *get_entrypoint_sources()]
 
     def _dump_data(self, **kwargs: Any) -> dict:
         """Ensure that header is prepended to model data"""

tests/conftest.py

@@ -11,13 +11,19 @@
 
 @pytest.fixture(scope="session", autouse=True)
 def patch_config_source(monkeypatch_session: MonkeyPatch) -> None:
+    """Patch config sources so we don't accidentally use any user pipelines during testing."""
+    from noob.config import add_config_source
     from noob.yaml import ConfigYAMLMixin
 
     current_sources = ConfigYAMLMixin.config_sources()
+    original_method = ConfigYAMLMixin.config_sources
+
+    for pth in (CONFIG_DIR, SPECIAL_DIR, PIPELINE_DIR):
+        add_config_source(pth)
 
     def _config_sources(cls: type[ConfigYAMLMixin]) -> list[Path]:
-        nonlocal current_sources
-        return [CONFIG_DIR, PIPELINE_DIR, SPECIAL_DIR, *current_sources]
+        nonlocal current_sources, original_method
+        return [p for p in original_method() if p not in current_sources]
 
     monkeypatch_session.setattr(ConfigYAMLMixin, "config_sources", classmethod(_config_sources))
 

tests/test_config.py

@@ -1,10 +1,10 @@
-import os
 import random
+from importlib import metadata
 from pathlib import Path
 
-import yaml
-
-from noob.config import Config
+from noob.config import Config, add_config_source
+from noob.testing.entrypoint import ENTRYPOINT_PATH
+from noob.yaml import ConfigYAMLMixin
 
 
 def test_config(tmp_path):
@@ -68,3 +68,36 @@ def test_config_sources_overrides(set_env, set_dotenv, set_pyproject, set_local_
     set_env({"logs": {"file_n": 5}})
     assert Config().logs.file_n == 5
     assert Config(**{"logs": {"file_n": 6}}).logs.file_n == 6
+
+
+def test_add_config_source(tmp_path):
+    """
+    Adding config source via the function hook adds it to the `config_sources`
+    """
+    assert tmp_path not in ConfigYAMLMixin.config_sources()
+    add_config_source(tmp_path)
+    assert tmp_path in ConfigYAMLMixin.config_sources()
+
+
+def test_entrypoint_config_source(tmp_path, monkeypatch):
+    """
+    Packages that specify an entrypoint function to add config sources are added to `config_sources`
+
+    References:
+        https://stackoverflow.com/a/79386262/13113166
+    """
+    from noob import config
+
+    config._entrypoint_sources = None
+    ep = metadata.EntryPoint(
+        name="test", group="noob.add_sources", value="noob.testing.entrypoint:some_entrypoint_fn"
+    )
+
+    def _entry_points(**params: dict) -> metadata.EntryPoints:
+        if not params.get("group", False) == "noob.add_sources":
+            raise ValueError("We should be trying to get the noob.add_sources group!")
+        return metadata.EntryPoints([ep])
+
+    monkeypatch.setattr(config, "entry_points", _entry_points)
+
+    assert ENTRYPOINT_PATH in ConfigYAMLMixin.config_sources()