Add YamlConfigSettingsSource for loading app config from yaml file

Author: tdg5

requirements.txt

@@ -72,6 +72,7 @@ SecretStorage==3.3.3
 toml==0.10.2
 tomli==2.0.1
 twine==4.0.2
+types-PyYAML==6.0.12.12
 typing_extensions==4.9.0
 urllib3==2.1.0
 virtualenv==20.25.0

service_oriented/application/config/base_config.py

@@ -1,10 +1,6 @@
-from typing import Any, Tuple, Type
+from typing import Any, Optional
 
-from pydantic_settings import (
-    BaseSettings,
-    PydanticBaseSettingsSource,
-    SettingsConfigDict,
-)
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from service_oriented.deployment_environment import DeploymentEnvironment
 

service_oriented/application/config/yaml_config_settings_source.py

@@ -0,0 +1,64 @@
+import os
+from typing import Any, Dict, List, Optional, Tuple, Type
+
+import yaml
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource
+
+
+class YamlConfigSettingsSource(PydanticBaseSettingsSource):
+    @classmethod
+    def get_yaml_config_path_from_sources(
+        cls,
+        sources: List[PydanticBaseSettingsSource],
+    ) -> Optional[str]:
+        for source in sources:
+            config_from_source = source()
+            yaml_config_path = config_from_source.get("yaml_config_path")
+            if yaml_config_path:
+                return str(yaml_config_path)
+        return None
+
+    @classmethod
+    def from_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        sources: List[PydanticBaseSettingsSource],
+    ) -> "YamlConfigSettingsSource":
+        yaml_config_path = YamlConfigSettingsSource.get_yaml_config_path_from_sources(
+            sources=sources,
+        )
+
+        return YamlConfigSettingsSource(
+            settings_cls=settings_cls,
+            yaml_config_path=yaml_config_path,
+        )
+
+    def __init__(
+        self,
+        settings_cls: Type[BaseSettings],
+        yaml_config_path: Optional[str] = None,
+    ):
+        super().__init__(settings_cls)
+
+        self._yaml_config: Dict[str, Any] = self._load_yaml_config(yaml_config_path)
+
+    def _load_yaml_config(self, yaml_config_path: Optional[str]) -> Dict[str, Any]:
+        if not yaml_config_path or not os.path.exists(yaml_config_path):
+            return {}
+
+        with open(yaml_config_path, mode="r", encoding="utf-8") as stream:
+            yaml_config = yaml.safe_load(stream)
+
+        return yaml_config if isinstance(yaml_config, dict) else {}
+
+    def get_field_value(
+        self,
+        field: FieldInfo,
+        field_name: str,
+    ) -> Tuple[Any, str, bool]:
+        # We're not required to do anything here. Implement return to make mypy happy.
+        return None, "", False
+
+    def __call__(self) -> Dict[str, Any]:
+        return self._yaml_config

service_oriented_test/application/config/yaml_config_settings_source_test.py

@@ -0,0 +1,94 @@
+"""
+Tests for YamlConfigSettingsSource integration with pydantic.
+"""
+
+import os
+import textwrap
+from tempfile import NamedTemporaryFile
+from typing import Any, Dict, Optional, Tuple, Type
+
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource
+
+from service_oriented.application.config.yaml_config_settings_source import (
+    YamlConfigSettingsSource,
+)
+
+
+YAML_TEMPLATE = """
+a_str: '{a_str}'
+an_int: {an_int}
+a_dict:
+    a_str: '{a_str}'
+    an_int: {an_int}
+"""
+
+
+class YamlConfigTestConfig(BaseSettings):
+    """
+    Test configuration class that is configured to load config from yaml.
+    """
+
+    a_str: str
+    an_int: int
+    a_dict: Dict[str, Any]
+    yaml_config_path: Optional[str] = None
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        yaml_config_settings = YamlConfigSettingsSource.from_sources(
+            settings_cls=settings_cls,
+            sources=[
+                init_settings,
+                env_settings,
+            ],
+        )
+        return (
+            # Include a built-in settings source for extra validation
+            init_settings,
+            yaml_config_settings,
+        )
+
+
+def test_yaml_config_settings_source_loads_init_yaml_config_as_expected() -> None:
+    """
+    Test the the yaml config path can be taken from the init settings and that
+    the config is loaded from yaml as expected.
+    """
+    a_str: str = "A man a plan, a canal, panama"
+    an_int: int = 5
+    yaml = YAML_TEMPLATE.format(a_str=a_str, an_int=an_int)
+    with NamedTemporaryFile(mode="w+") as temp_file:
+        temp_file.write(textwrap.dedent(yaml))
+        temp_file.flush()
+        config = YamlConfigTestConfig(yaml_config_path=temp_file.name)
+        assert a_str == config.a_str
+        assert an_int == config.an_int
+        assert a_str == config.a_dict["a_str"]
+        assert an_int == config.a_dict["an_int"]
+
+
+def test_yaml_config_settings_source_loads_config_yaml_config_as_expected() -> None:
+    """
+    Test that the yaml config path can come from other settings sources if no
+    yaml config path is given as an argument to init and the config is loaded
+    as expected.
+    """
+    a_str: str = "taco cat"
+    an_int: int = 42
+    yaml = YAML_TEMPLATE.format(a_str=a_str, an_int=an_int)
+    with NamedTemporaryFile(mode="w+") as temp_file:
+        temp_file.write(textwrap.dedent(yaml))
+        temp_file.flush()
+        os.environ["YAML_CONFIG_PATH"] = temp_file.name
+        config = YamlConfigTestConfig()
+        assert a_str == config.a_str
+        assert an_int == config.an_int
+        assert a_str == config.a_dict["a_str"]
+        assert an_int == config.a_dict["an_int"]

setup.py

@@ -10,6 +10,7 @@
     VERSION = f.read().strip()
 
 _dependencies = [
+    "PyYaml~=6.0.1",
     "pydantic~=2.5.3",
     "pydantic-settings~=2.1.0",
 ]

whitelist.txt

@@ -1,3 +1,4 @@
 dirname
+dotenv
 exinfo
 pydantic