From dcc9798906d89706591779fd786fb0a8be931382 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Thu, 11 Dec 2025 00:01:43 -0800 Subject: [PATCH 1/2] add entrypoint hooks and function to add config sources --- src/noob/config.py | 58 ++++++++++++++++++++++++++++++++++++++++++ src/noob/exceptions.py | 8 ++++++ src/noob/yaml.py | 4 +-- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/noob/config.py b/src/noob/config.py index a50b6e8..4322387 100644 --- a/src/noob/config.py +++ b/src/noob/config.py @@ -1,3 +1,5 @@ +import warnings +from importlib.metadata import entry_points from pathlib import Path from typing import Literal @@ -11,9 +13,15 @@ YamlConfigSettingsSource, ) +from noob.exceptions import EntrypointImportWarning + _default_userdir = Path().home() / ".config" / "noob" _dirs = PlatformDirs("noob", "noob") LOG_LEVELS = Literal["DEBUG", "INFO", "WARNING", "ERROR"] +_extra_sources = [] +"""Extra sources for tube configs added by `add_sources`""" +_entrypoint_sources: list[Path] | None = None +"""Sources added by entrypoint functions. Initially `None`, populated on first load of a config""" class LogConfig(BaseModel): @@ -126,4 +134,54 @@ def settings_customise_sources( ) +def add_config_source(path: Path) -> None: + """ + Add a directory as a source of tube configs when searching by tube id + """ + global _extra_sources + path = Path(path) + _extra_sources.append(path) + + +def get_entrypoint_sources() -> list[Path]: + """ + Get additional config sources added by entrypoint functions. + + Packages that ship noob tubes can make those tubes available by adding an + entrypoint function with a signature ``() -> list[Path]`` to their pyproject.toml + like: + + [project.entry-points."noob.add_sources"] + tubes = "my_package.something:add_sources" + + References: + https://setuptools.pypa.io/en/latest/userguide/entry_point.html + """ + global _entrypoint_sources + if _entrypoint_sources is None: + _entrypoint_sources = [] + for ext in entry_points(group="noob.add_sources"): + try: + add_sources_fn = ext.load() + except (ImportError, AttributeError): + warnings.warn( + f"Config source entrypoint {ext.name}, {ext.value} " + f"could not be imported, or the function could not be found. Ignoring", + EntrypointImportWarning, + stacklevel=1, + ) + continue + try: + _entrypoint_sources.extend([Path(p) for p in add_sources_fn()]) + except Exception as e: + # bare exception is fine here - we're calling external code and can't know. + warnings.warn( + f"Config source entrypoint {ext.name}, {ext.value} " + f"threw an error, or returned an invalid list of paths, ignoring.\n{str(e)}", + EntrypointImportWarning, + stacklevel=1, + ) + return _entrypoint_sources + + config = Config() diff --git a/src/noob/exceptions.py b/src/noob/exceptions.py index c3bb44c..a6de7da 100644 --- a/src/noob/exceptions.py +++ b/src/noob/exceptions.py @@ -16,6 +16,10 @@ class ConfigError(NoobError): """Base config error type""" +class ConfigWarning(NoobWarning): + """Base config warning type""" + + class SchedulerError(NoobError): """Base error in the scheduler""" @@ -51,6 +55,10 @@ class ConfigMismatchError(ConfigError, ValueError): """ +class EntrypointImportWarning(ConfigWarning, ImportWarning): + """Some problem with a configuration entypoint, usually when importing""" + + class AlreadyRunningError(RunnerError, RuntimeError): """ A tube is already running! diff --git a/src/noob/yaml.py b/src/noob/yaml.py index e95dfb2..45236c9 100644 --- a/src/noob/yaml.py +++ b/src/noob/yaml.py @@ -199,9 +199,9 @@ def config_sources(cls: type[Self]) -> list[Path]: Directories to search for config files, in order of priority such that earlier sources are preferred over later sources. """ - from noob.config import Config + from noob.config import Config, _extra_sources, get_entrypoint_sources - return [Config().config_dir] + return [Config().config_dir, *_extra_sources, *get_entrypoint_sources()] def _dump_data(self, **kwargs: Any) -> dict: """Ensure that header is prepended to model data""" From 1f45e40c117fd3d606833472f8842a5b959e68fb Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Thu, 11 Dec 2025 18:44:39 -0800 Subject: [PATCH 2/2] tests for config sources --- src/noob/config.py | 11 +++++++++ src/noob/testing/entrypoint.py | 8 +++++++ src/noob/yaml.py | 4 ++-- tests/conftest.py | 10 +++++++-- tests/test_config.py | 41 ++++++++++++++++++++++++++++++---- 5 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 src/noob/testing/entrypoint.py diff --git a/src/noob/config.py b/src/noob/config.py index 4322387..85d479e 100644 --- a/src/noob/config.py +++ b/src/noob/config.py @@ -143,6 +143,17 @@ def add_config_source(path: Path) -> None: _extra_sources.append(path) +def get_extra_sources() -> list[Path]: + """ + Get the extra sources added by :func:`.add_config_source` + + (avoid importing the private module-level collection anywhere else, + as it makes mutation weird and unpredictable) + """ + global _extra_sources + return _extra_sources + + def get_entrypoint_sources() -> list[Path]: """ Get additional config sources added by entrypoint functions. diff --git a/src/noob/testing/entrypoint.py b/src/noob/testing/entrypoint.py new file mode 100644 index 0000000..025f7ab --- /dev/null +++ b/src/noob/testing/entrypoint.py @@ -0,0 +1,8 @@ +from pathlib import Path + +ENTRYPOINT_PATH = Path("/tmp/notreal/path") + + +def some_entrypoint_fn() -> list[Path]: + """See test_config and config.get_entrypoint_sources""" + return [ENTRYPOINT_PATH] diff --git a/src/noob/yaml.py b/src/noob/yaml.py index 45236c9..45528d4 100644 --- a/src/noob/yaml.py +++ b/src/noob/yaml.py @@ -199,9 +199,9 @@ def config_sources(cls: type[Self]) -> list[Path]: Directories to search for config files, in order of priority such that earlier sources are preferred over later sources. """ - from noob.config import Config, _extra_sources, get_entrypoint_sources + from noob.config import Config, get_entrypoint_sources, get_extra_sources - return [Config().config_dir, *_extra_sources, *get_entrypoint_sources()] + return [Config().config_dir, *get_extra_sources(), *get_entrypoint_sources()] def _dump_data(self, **kwargs: Any) -> dict: """Ensure that header is prepended to model data""" diff --git a/tests/conftest.py b/tests/conftest.py index f148848..8a0f652 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,13 +11,19 @@ @pytest.fixture(scope="session", autouse=True) def patch_config_source(monkeypatch_session: MonkeyPatch) -> None: + """Patch config sources so we don't accidentally use any user pipelines during testing.""" + from noob.config import add_config_source from noob.yaml import ConfigYAMLMixin current_sources = ConfigYAMLMixin.config_sources() + original_method = ConfigYAMLMixin.config_sources + + for pth in (CONFIG_DIR, SPECIAL_DIR, PIPELINE_DIR): + add_config_source(pth) def _config_sources(cls: type[ConfigYAMLMixin]) -> list[Path]: - nonlocal current_sources - return [CONFIG_DIR, PIPELINE_DIR, SPECIAL_DIR, *current_sources] + nonlocal current_sources, original_method + return [p for p in original_method() if p not in current_sources] monkeypatch_session.setattr(ConfigYAMLMixin, "config_sources", classmethod(_config_sources)) diff --git a/tests/test_config.py b/tests/test_config.py index 8918b93..cdd92fe 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,10 +1,10 @@ -import os import random +from importlib import metadata from pathlib import Path -import yaml - -from noob.config import Config +from noob.config import Config, add_config_source +from noob.testing.entrypoint import ENTRYPOINT_PATH +from noob.yaml import ConfigYAMLMixin def test_config(tmp_path): @@ -68,3 +68,36 @@ def test_config_sources_overrides(set_env, set_dotenv, set_pyproject, set_local_ set_env({"logs": {"file_n": 5}}) assert Config().logs.file_n == 5 assert Config(**{"logs": {"file_n": 6}}).logs.file_n == 6 + + +def test_add_config_source(tmp_path): + """ + Adding config source via the function hook adds it to the `config_sources` + """ + assert tmp_path not in ConfigYAMLMixin.config_sources() + add_config_source(tmp_path) + assert tmp_path in ConfigYAMLMixin.config_sources() + + +def test_entrypoint_config_source(tmp_path, monkeypatch): + """ + Packages that specify an entrypoint function to add config sources are added to `config_sources` + + References: + https://stackoverflow.com/a/79386262/13113166 + """ + from noob import config + + config._entrypoint_sources = None + ep = metadata.EntryPoint( + name="test", group="noob.add_sources", value="noob.testing.entrypoint:some_entrypoint_fn" + ) + + def _entry_points(**params: dict) -> metadata.EntryPoints: + if not params.get("group", False) == "noob.add_sources": + raise ValueError("We should be trying to get the noob.add_sources group!") + return metadata.EntryPoints([ep]) + + monkeypatch.setattr(config, "entry_points", _entry_points) + + assert ENTRYPOINT_PATH in ConfigYAMLMixin.config_sources()