From 6f1c3a12190aeb4644b69f693ce3eee23956457c Mon Sep 17 00:00:00 2001 From: Dan Cardin Date: Thu, 6 Jan 2022 13:44:18 -0500 Subject: [PATCH] feat: Add ability to compose multiple interpolation in a single value. --- README.md | 71 ++++++++++++++++++++++------------------- pyproject.toml | 4 +-- src/configly/process.py | 21 ++++++++---- tests/test_process.py | 6 ++++ 4 files changed, 61 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index f364903..df619e3 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,12 @@ ```yaml # config.yml foo: - bar: <% ENV[REQUIRED] %> - baz: <% ENV[OPTIONAL, true] %> + bar: <% ENV[REQUIRED] %> + baz: <% ENV[OPTIONAL, true] %> list_of_stuff: - - fun<% ENV[NICE, dament] %>al - - fun<% ENV[AGH, er] %>al + - fun<% ENV[NICE, dament] %>al + - fun<% ENV[AGH, er] %>al + - more/<% ENV[THAN, er] %>/one/<% ENV[interpolation, er] %>! ``` ```python @@ -78,29 +79,29 @@ import configparser And this is all assuming that everyone is loading configuration at the outermost entrypoint! The two worst possible outcomes in configuration are: -* You are loading configuration lazily and/or deeply within your application, such that it +- You are loading configuration lazily and/or deeply within your application, such that it hits a critical failure after having seemingly successfully started up. -* There is not a singular location at which you can go to see all configuration your app might +- There is not a singular location at which you can go to see all configuration your app might possibly be reading from. - ## The pitch `Configly` asserts configuration should: -* Be centralized - * One should be able to look at one file to see all (env vars, files, etc) which must exist for the + +- Be centralized + - One should be able to look at one file to see all (env vars, files, etc) which must exist for the application to function. -* Be comprehensive - * One should not find configuration being loaded secretly elsewhere -* Be declarative/static - * code-execution (e.g. the class above) in the definition of the config inevitably makes it +- Be comprehensive + - One should not find configuration being loaded secretly elsewhere +- Be declarative/static + - code-execution (e.g. the class above) in the definition of the config inevitably makes it hard to interpret, as the config becomes more complex. -* Be namespacable - * One should not have to prepend `foo_` namespaces to all `foo` related config names -* Be loaded, once, at app startup - * (At least the _definition_ of the configuration you're loading) -* (Ideally) have structured output - * If something is an `int`, ideally it would be read as an int. +- Be namespacable + - One should not have to prepend `foo_` namespaces to all `foo` related config names +- Be loaded, once, at app startup + - (At least the _definition_ of the configuration you're loading) +- (Ideally) have structured output + - If something is an `int`, ideally it would be read as an int. To that end, the `configly.Config` class exposes a series of classmethods from which your config can be loaded. It's largely unimportant what the input format is, but we started with formats @@ -118,24 +119,28 @@ Given an input `config.yml` file: ```yaml # config.yml foo: - bar: <% ENV[REQUIRED] %> - baz: <% ENV[OPTIONAL, true] %> + bar: <% ENV[REQUIRED] %> + baz: <% ENV[OPTIONAL, true] %> list_of_stuff: - - fun<% ENV[NICE, dament] %>al - - fun<% ENV[AGH, er] %>al + - fun<% ENV[NICE, dament] %>al + - fun<% ENV[AGH, er] %>al + - more/<% ENV[THAN, er] %>/one/<% ENV[interpolation, er] %>! ``` -A couple of things jump out: -* Most importantly, whatever the configuration value is, it's intreted as a literal value in the - format of the file which loads it. I.E. loading `"true"` from the evironment in a yaml file - will yield a python `True`. Ditto `"1"`, or `"null"`. -* Each `<% ... %>` section indicates a variable -* `ENV` is an "interpolator" which knows how to obtain environment variables -* `[VAR]` Will raise an error if that piece of config is not found -* `[VAR, true]` Will `VAR` to the value after the comma -* The interpolation can be a sub-portion of a key (`fun<% ENV[NICE, dament] %>al` interpolates - to "fundamental"). Another example being `'<% ENV[X, 3] %>'` interpolates to `'1'` instead of `1` +A number of things are exemplified in the example above: + +- Each `<% ... %>` section indicates an interpolated value, the interpolation can + be a fragment of the overall value, and multiple values can be interpolated + within a single value. + +- `ENV` is an "interpolator" which knows how to obtain environment variables + +- `[VAR]` Will raise an error if that piece of config is not found, whereas + `[VAR, true]` will default `VAR` to the value after the comma +- Whatever the final value is, it's interpreted as a literal value in the + format of the file which loads it. I.E. `true` -> python `True`, `1` -> + python `1`, and `null` -> python `None`. Now that you've loaded the above configuration: diff --git a/pyproject.toml b/pyproject.toml index d325781..b3740f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [tool.poetry] name = "configly" -version = "0.2.2" +version = "0.3.0" description = "" -authors = [] +authors = ["Dan Cardin ", "Omar Khan "] license = "MIT" keywords = [ "config", "yaml", "toml", "env" ] repository = "https://github.com/schireson/configly" diff --git a/src/configly/process.py b/src/configly/process.py index 950c3ed..e2831e3 100644 --- a/src/configly/process.py +++ b/src/configly/process.py @@ -4,6 +4,8 @@ from configly.registry import registry from configly.utilities import quote_string +INTERPOLATION_REGEX = re.compile(r"(.*)\<%\s*(\w+)\[([\w.]+)(?:,\s*(.+))?\]\s*%>(.*)") + def post_process(loader, value, registry=registry): if isinstance(value, Mapping): @@ -19,11 +21,17 @@ def post_process(loader, value, registry=registry): return result else: - if not isinstance(value, str): - return value + # Repeatedly evaluate the string until there are no interpolation blocks + while True: + # The post-interpolation of previous values might coerce them into + # concrete values, on which no further processing is necessary. + if not isinstance(value, str): + break + + match = re.match(INTERPOLATION_REGEX, value) + if not match: + break - match = re.match(r"(.*)\<%\s*(\w+)\[([\w.]+)(?:,\s*(.+))?\]\s*%>(.*)", value) - if match: groups = match.groups() pre, interpolation_type, var_name, default, post = groups @@ -49,7 +57,8 @@ def post_process(loader, value, registry=registry): result = quote_string(result) if getattr(interpolator, "yaml_safe", True): - return loader.load_value(result) + value = loader.load_value(result) + else: + value = result - return result return value diff --git a/tests/test_process.py b/tests/test_process.py index 17ab439..d17c121 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -135,3 +135,9 @@ def test_object_values(self): config = Config(post_process(yaml, input_)) assert type(config.foo) == Config assert config.foo.to_dict() == {"hello": "world"} + + @patch("os.environ", new={"foo": "one", "bar": 'two', 'baz': 'three'}) + def test_multiple_matches(self): + input_ = {"foo": "<% ENV[foo] %>+<% ENV[bar] %>=<% ENV[baz] %>"} + config = Config(post_process(yaml, input_)) + assert config.to_dict() == {"foo": "one+two=three"}