From bf0da889f923170102fc662fdd49f3daf02863d4 Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Thu, 16 May 2024 15:49:58 +0000 Subject: [PATCH] ## [2.0.0-dev.1] - 2024-05-16 ### Added - Hashicorp Vault integration --- CHANGELOG.md | 9 +- fidelius/__init__.py | 2 +- fidelius/gateway/_abstract.py | 26 +++++- fidelius/gateway/vault/__init__.py | 2 + fidelius/gateway/vault/_client.py | 73 ++++++++++++++++ fidelius/gateway/vault/_std.py | 2 + fidelius/gateway/vault/_structs.py | 44 ++++++++++ fidelius/gateway/vault/_vaultadmin.py | 66 +++++++++++++++ fidelius/gateway/vault/_vaultrepo.py | 66 +++++++++++++++ fidelius/utils/__init__.py | 1 + fidelius/utils/_selfresdc.py | 117 ++++++++++++++++++++++++++ pyproject.toml | 5 +- requirements.txt | 1 + 13 files changed, 407 insertions(+), 7 deletions(-) create mode 100644 fidelius/gateway/vault/__init__.py create mode 100644 fidelius/gateway/vault/_client.py create mode 100644 fidelius/gateway/vault/_std.py create mode 100644 fidelius/gateway/vault/_structs.py create mode 100644 fidelius/gateway/vault/_vaultadmin.py create mode 100644 fidelius/gateway/vault/_vaultrepo.py create mode 100644 fidelius/utils/__init__.py create mode 100644 fidelius/utils/_selfresdc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7472871..df8fdb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.0.0-beta.1] - 2024-04-11 +## [2.0.0-dev.1] - 2024-05-16 + +### Added + +- Hashicorp Vault integration + + +## [1.0.0] - 2024-04-11 ### Added diff --git a/fidelius/__init__.py b/fidelius/__init__.py index 99c8bf5..9777236 100644 --- a/fidelius/__init__.py +++ b/fidelius/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.0.0' +__version__ = '2.0.0-dev.1' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/fidelius/gateway/_abstract.py b/fidelius/gateway/_abstract.py index 4a62576..3222e41 100644 --- a/fidelius/gateway/_abstract.py +++ b/fidelius/gateway/_abstract.py @@ -2,6 +2,9 @@ '_BaseFideliusRepo', '_BaseFideliusAdminRepo', ] + +import re + from .interface import * from fidelius.structs import * @@ -49,9 +52,9 @@ def make_shared_path(self, folder: str, env: Optional[str] = None) -> str: """The full path to group shared parameters/secrets. """ return self._SHARED_PATH_FORMAT.format(group=self.app_props.group, - env=env or self.app_props.env, - folder=folder, - name='{name}') + env=env or self.app_props.env, + folder=folder, + name='{name}') def get_expression_string(self, name: str, folder: Optional[str] = None) -> str: """Return a Fidelius expression string (e.g. to use in configuration @@ -166,6 +169,23 @@ def replace(self, string: str, no_default: bool = False) -> str: return self.get(m.group('name'), m.group('folder'), no_default=no_default) or '' return string + def set_app_path_format(self, new_format: str): + self._APP_PATH_FORMAT = new_format + + def set_shared_path_format(self, new_format: str): + self._SHARED_PATH_FORMAT = new_format + + def set_app_expression_format(self, new_format: str): + self._EXPRESSION_APP_FORMAT = new_format + + def set_shared_expression_format(self, new_format: str): + self._EXPRESSION_SHARED_FORMAT = new_format + + def set_expression_pattern(self, new_format: Union[str, re.Pattern]): + if isinstance(new_format, str): + new_format = re.compile(new_format) + self._EXPRESSION_PATTERN = new_format + class _BaseFideliusAdminRepo(_BaseFideliusRepo, IFideliusAdminRepo, abc.ABC): """Covers a lot of admin basic functionality common across most storage back-ends. diff --git a/fidelius/gateway/vault/__init__.py b/fidelius/gateway/vault/__init__.py new file mode 100644 index 0000000..ec70c6d --- /dev/null +++ b/fidelius/gateway/vault/__init__.py @@ -0,0 +1,2 @@ +from ._vaultrepo import * +from ._vaultadmin import * diff --git a/fidelius/gateway/vault/_client.py b/fidelius/gateway/vault/_client.py new file mode 100644 index 0000000..e666f3e --- /dev/null +++ b/fidelius/gateway/vault/_client.py @@ -0,0 +1,73 @@ +__all__ = [ + 'VaultGateway', +] +from fidelius.structs import * +from ._structs import * +import hvac + + +import logging +log = logging.getLogger(__file__) + + +class VaultGateway: + def __init__(self, url: str, token: str, verify: bool = True, timeout: int = 30, namespace: Optional[str] = None): + self._client = hvac.Client(url=url, token=token, verify=verify, timeout=timeout, namespace=namespace) + self._keyvals: Dict[str, Dict[str, str]] = {} # self._keyvals[path][key] = val + + def flush_cache(self): + self._keyvals = {} + + def _read_secret(self, path: str) -> VaultResponse: + res_dict = self._client.secrets.kv.read_secret(path=path) + return VaultResponse.from_dict(res_dict) + + def _load_path(self, path: str): + if path not in self._keyvals: + self._keyvals[path] = {} + res = self._read_secret(path) + if res.data and isinstance(res.data.data, dict): + self._keyvals[path] = res.data.data + else: + log.error(f'The data for requested path was not a dict or doesnt exist! {path=}, {res=}') + + def get_secret_param(self, path: str, key: str) -> Optional[str]: + self._load_path(path) + return self._keyvals[path].get(key, None) + + def _force_path_update(self, path: str): + # First, clear this path from the cache! + if path in self._keyvals: + del self._keyvals[path] + # Then, load the path so we're up to date! + self._load_path(path) + + def create_secret_param(self, path: str, key: str, value: str): + self._force_path_update(path) + old_data = self._keyvals[path] + if key in old_data: + raise FideliusParameterAlreadyExists(f'parameter already exists: {path}/{key}') + old_data[key] = value + self._client.secrets.kv.create_or_update_secret(path=path, secret=old_data) + self._force_path_update(path) + + def set_metadata(self, path: str, metadata: Dict[str, str]): + self._client.secrets.kv.update_metadata(path=path, custom_metadata=metadata) + + def update_secret_param(self, path: str, key: str, value: str): + self._force_path_update(path) + old_data = self._keyvals[path] + if key not in old_data: + raise FideliusParameterNotFound(f'parameter not found: {path}/{key}') + old_data[key] = value + self._client.secrets.kv.create_or_update_secret(path=path, secret=old_data) + self._force_path_update(path) + + def delete_secret_param(self, path: str, key: str): + self._force_path_update(path) + old_data = self._keyvals[path] + if key not in old_data: + raise FideliusParameterNotFound(f'parameter not found: {path}/{key}') + del old_data[key] + self._client.secrets.kv.create_or_update_secret(path=path, secret=old_data) + self._force_path_update(path) diff --git a/fidelius/gateway/vault/_std.py b/fidelius/gateway/vault/_std.py new file mode 100644 index 0000000..3ca784a --- /dev/null +++ b/fidelius/gateway/vault/_std.py @@ -0,0 +1,2 @@ +from ._vaultrepo import VaultKeyValRepo as FideliusRepo +from ._vaultadmin import VaultKeyValAdmin as FideliusAdmin diff --git a/fidelius/gateway/vault/_structs.py b/fidelius/gateway/vault/_structs.py new file mode 100644 index 0000000..97bdeaa --- /dev/null +++ b/fidelius/gateway/vault/_structs.py @@ -0,0 +1,44 @@ +__all__ = [ + 'VaultResponse', + 'VaultResponseData', + 'VaultResponseMetadata', +] + +from ccptools.structs import * +from fidelius.utils import SelfResolvingFromDictDataclass + + +@dataclasses.dataclass +class VaultResponseMetadata(SelfResolvingFromDictDataclass): + created_time: Optional[Datetime] = None + custom_metadata: Any = None + deletion_time: Optional[Datetime] = None + destroyed: bool = False + version: Optional[int] = None + + +@dataclasses.dataclass +class VaultResponseData(SelfResolvingFromDictDataclass): + data: Dict[str, str] = dataclasses.field(default_factory=dict) + metadata: Optional[VaultResponseMetadata] = None + + +@dataclasses.dataclass +class VaultResponse(SelfResolvingFromDictDataclass): + request_id: str = '' + lease_id: Optional[str] = '' + renewable: bool = False + lease_duration: Optional[int] = None + data: Optional[VaultResponseData] = None + wrap_info: Optional[Any] = None + warnings: Optional[Any] = None + auth: Optional[Any] = None + mount_type: Optional[str] = None + + def get_keyval(self, key: str) -> Optional[str]: + if isinstance(self.data, VaultResponseData): + if isinstance(self.data.data, dict): + return self.data.data.get(key, None) + return None + + diff --git a/fidelius/gateway/vault/_vaultadmin.py b/fidelius/gateway/vault/_vaultadmin.py new file mode 100644 index 0000000..8a526a9 --- /dev/null +++ b/fidelius/gateway/vault/_vaultadmin.py @@ -0,0 +1,66 @@ +__all__ = [ + 'VaultKeyValAdmin', +] + +from fidelius.structs import * +from fidelius.gateway._abstract import * +from ._vaultrepo import * + +import logging +log = logging.getLogger(__name__) + + +class VaultKeyValAdmin(_BaseFideliusAdminRepo, VaultKeyValRepo): + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + log.debug('VaultKeyValAdmin.__init__') + super().__init__(app_props, tags, **kwargs) + + def create_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + self._gw.create_secret_param(path=self._nameless_path(env=env), key=name, value=value) + self._gw.set_metadata(path=self._nameless_path(env=env), metadata=self.tags.to_dict()) + return self.get_full_path(name, env=env), self.get_expression_string(name) + + def update_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + self._gw.update_secret_param(path=self._nameless_path(env=env), key=name, value=value) + return self.get_full_path(name, env=env), self.get_expression_string(name) + + def delete_param(self, name: str, env: Optional[str] = None): + self._gw.delete_secret_param(path=self._nameless_path(env=env), key=name) + + def create_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + self._gw.create_secret_param(path=self._nameless_path(folder=folder, env=env), key=name, value=value) + self._gw.set_metadata(path=self._nameless_path(folder=folder, env=env), metadata=self.tags.to_dict()) + return self.get_full_path(name, folder=folder, env=env), self.get_expression_string(name, folder=folder) + + def update_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + self._gw.update_secret_param(path=self._nameless_path(folder=folder, env=env), key=name, value=value) + return self.get_full_path(name, folder=folder, env=env), self.get_expression_string(name, folder=folder) + + def delete_shared_param(self, name: str, folder: str, env: Optional[str] = None): + self._gw.delete_secret_param(path=self._nameless_path(env=env, folder=folder), key=name) + + def create_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self.create_param(name=name, value=value, description=description, env=env) + + def update_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self.update_param(name=name, value=value, description=description, env=env) + + def delete_secret(self, name: str, env: Optional[str] = None): + self.delete_param(name=name, env=env) + + def create_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self.create_shared_param(name=name, folder=folder, value=value, description=description, env=env) + + def update_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self.update_shared_param(name=name, folder=folder, value=value, description=description, env=env) + + def delete_shared_secret(self, name: str, folder: str, env: Optional[str] = None): + self.delete_shared_param(name=name, folder=folder, env=env) diff --git a/fidelius/gateway/vault/_vaultrepo.py b/fidelius/gateway/vault/_vaultrepo.py new file mode 100644 index 0000000..404daab --- /dev/null +++ b/fidelius/gateway/vault/_vaultrepo.py @@ -0,0 +1,66 @@ +__all__ = [ + 'VaultKeyValRepo', +] + +from fidelius.structs import * +from fidelius.gateway._abstract import * +from ._client import * + +import os + +import logging +log = logging.getLogger(__name__) + + +class VaultKeyValRepo(_BaseFideliusRepo): + def __init__(self, app_props: FideliusAppProps, + vault_url: Optional[str] = None, + vault_token: Optional[str] = None, + + verify: Union[bool, str] = True, + timeout_sec: int = 30, + flush_cache_every_time: bool = False, + **kwargs): + """Fidelius Admin Repo that uses Hashicorp's Vault and its Secrets Key/Value store as a backend + + VAULT_ADDR + + VAULT_TOKEN + VAULT_CACERT + VAULT_CAPATH + VAULT_CLIENT_CERT + VAULT_CLIENT_KEY + + :param app_props: The current application properties. + + ... + + :param flush_cache_every_time: Optional flat that'll flush the entire + cache before every operation if set to + True and is just intended for testing + purposes. + """ + super().__init__(app_props, **kwargs) + self._flush_cache_every_time = flush_cache_every_time + + self._vault_url = vault_url or os.environ.get('FIDELIUS_VAULT_ADDR', '') or os.environ.get('VAULT_ADDR', '') + if not self._vault_url: + raise EnvironmentError('Fidelius VaultKeyValRepo requires the base API URL address for Vault when initialising or in the FIDELIUS_VAULT_ADDR or VAULT_ADDR environment variables') + + self._vault_token = vault_token or os.environ.get('FIDELIUS_VAULT_TOKEN', '') or os.environ.get('VAULT_TOKEN', '') + if not self._vault_token: + raise EnvironmentError('Fidelius VaultKeyValRepo requires a vault token to access Vault when initialising or in the FIDELIUS_VAULT_ADDR or VAULT_ADDR environment variables') + + self._verify = verify + self._timeout_sec = timeout_sec + + self._gw = VaultGateway(url=self._vault_url, token=self._vault_token, verify=self._verify, timeout=self._timeout_sec) + + def _nameless_path(self, folder: Optional[str] = None, env: Optional[str] = None) -> str: + return self.get_full_path(name='', folder=folder, env=env)[:-1] + + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + return self._gw.get_secret_param(self._nameless_path(env=env), name) + + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + return self._gw.get_secret_param(self._nameless_path(folder=folder, env=env), name) diff --git a/fidelius/utils/__init__.py b/fidelius/utils/__init__.py new file mode 100644 index 0000000..28759ac --- /dev/null +++ b/fidelius/utils/__init__.py @@ -0,0 +1 @@ +from ._selfresdc import * diff --git a/fidelius/utils/_selfresdc.py b/fidelius/utils/_selfresdc.py new file mode 100644 index 0000000..798e233 --- /dev/null +++ b/fidelius/utils/_selfresdc.py @@ -0,0 +1,117 @@ +__all__ = [ + 'SelfResolvingFromDictDataclass', +] +from ccptools.structs import * +from ccptools import dtu +from ccptools.tpu import strimp +import dataclasses +from typing import _GenericAlias, _SpecialForm # noqa + +_T_ANNOTATION = Union[str, type, _GenericAlias, _SpecialForm, ForwardRef] + + +def _get_annotation_type(annotation: _T_ANNOTATION, globalns=None, localns=None) -> Type: + # TODO(thordurm@ccpgames.com>) 2024-05-16: Lists/Sets/Dics of Classes?!? + if isinstance(annotation, _SpecialForm) and annotation is Any: + return Any + + if isinstance(annotation, _GenericAlias): + if annotation.__origin__ is Union: + real_annotations = [a for a in annotation.__args__ if a is not None.__class__] # noqa + if len(real_annotations) == 1: + annotation = real_annotations[0] + else: + raise TypeError(f'cant get type for multiple annotations: {real_annotations}') + elif isinstance(annotation.__origin__, type): + annotation = annotation.__origin__ + + return _get_annotation_type(annotation, globalns=globalns, localns=localns) + + if isinstance(annotation, str): + # TODO(thordurm@ccpgames.com>) 2024-05-16: Check globals/locals?!? + if '.' in annotation: + return strimp.get_class(annotation, reraise=True) + else: + annotation = ForwardRef(annotation) + + if isinstance(annotation, ForwardRef): + annotation = annotation._evaluate(globalns, localns) # noqa + + if isinstance(annotation, type): + return annotation + + raise TypeError(f'cant find type of annotation: {annotation}') + + +@dataclasses.dataclass +class SelfResolvingFromDictDataclass: + extra_attr: dataclasses.InitVar[Optional[Dict[str, Any]]] = None + _extra_attr: Dict[str, Any] = dataclasses.field(default_factory=dict, init=False) + + def __post_init__(self, extra_attr: Dict[str, Any] = None): + if extra_attr: + self._extra_attr = extra_attr + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> 'SelfResolvingFromDictDataclass': + fd = {f.name: f for f in dataclasses.fields(cls)} + kwargs = {} + extras = {} + for k, v in d.items(): + if k in fd: # Field is in class + if v is None: + kwargs[k] = v + continue + + field = fd.get(k) + field_type = None + if field.type: + field_type = _get_annotation_type(field.type, globalns=globals(), localns=locals()) + + if not field_type or field_type is Any: # No type annotation! + kwargs[k] = v + continue + + if isinstance(v, field_type): # Types are the same! + kwargs[k] = v + continue + + if field_type is Datetime: + kwargs[k] = dtu.any_to_datetime(v) + continue + + if field_type is Time: + kwargs[k] = dtu.any_to_datetime(v).time() + continue + + if field_type is Date: + kwargs[k] = dtu.any_to_datetime(v).date() + continue + + if isinstance(v, dict): + if dataclasses.is_dataclass(field_type): + if hasattr(field_type, 'from_dict'): + kwargs[k] = field_type.from_dict(v) + else: + kwargs[k] = field_type(**v) + else: + kwargs[k] = field_type(**v) + continue + + raise ValueError(f'No idea how to handle this: {k=}, {v=}, {field_type=}') + + else: + extras[k] = v + + return cls(**kwargs, extra_attr=extras) # noqa + + def __getattr__(self, item): + if item in self._extra_attr: + return self._extra_attr[item] + return EmptyDict + + def __setattr__(self, key, value): + if key in {f.name for f in dataclasses.fields(self)}: + super().__setattr__(key, value) + else: + self._extra_attr[key] = value diff --git a/pyproject.toml b/pyproject.toml index ac234f0..d2eb40e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ authors = [ { name = "Thordur Matthiasson", email = "thordurm@ccpgames.com" }, { name = "Kristin Fjola Tomasdottir", email = "kristinf@ccpgames.com" } ] -keywords = [ "parameter store", "aws", "secrets", "tools", "ccp", "utils" ] +keywords = [ "parameter store", "aws", "secrets", "tools", "ccp", "utils", "hashicorp vault" ] classifiers = [ "Development Status :: 4 - Beta", @@ -35,7 +35,8 @@ classifiers = [ ] dependencies = [ "ccptools >=1.1, <2", - "boto3 >=1.20, <2" + "boto3 >=1.20, <2", + "havc >= 2.2, <3" ] [project.urls] diff --git a/requirements.txt b/requirements.txt index 1269059..448106b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ ccptools >=1.1, <2 boto3 >=1.20, <2 +hvac >= 2.2, <3 \ No newline at end of file