From 87aa131d867868b806e8ff383247ecd25443a864 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 5 Dec 2024 08:30:18 -0500 Subject: [PATCH] Move inject_credential from awx --- .flake8 | 14 + dependencies/direct/py.in | 2 + .../interfaces/_temporary_private_api.py | 234 ++++++++++++++++- tests/_temporary_private_api_test.py | 244 +++++++++++++++++- 4 files changed, 491 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index 96e87c66..1a8b1daf 100644 --- a/.flake8 +++ b/.flake8 @@ -110,6 +110,20 @@ per-file-ignores = # additionally test docstrings don't need param lists (DAR, DCO020): tests/**.py: DAR, DCO020, S101, S105, S108, S404, S603, WPS202, WPS210, WPS430, WPS436, WPS441, WPS442, WPS450 + src/awx_plugins/interfaces/_temporary_private_api.py: ANN001,ANN201,B950,C901,CCR001,D103,E800,LN001,LN002,Q003,WPS110,WPS111,WPS118,WPS125,WPS204,WPS210,WPS211,WPS213,WPS221,WPS226,WPS231,WPS232,WPS319,WPS323,WPS336,WPS337,WPS361,WPS421,WPS429,WPS430,WPS431,WPS436,WPS442,WPS503,WPS507,WPS516 + + + + + + + + + + + + + # Count the number of occurrences of each error/warning code and print a report: statistics = true diff --git a/dependencies/direct/py.in b/dependencies/direct/py.in index ed38bc6a..ea28d2fe 100644 --- a/dependencies/direct/py.in +++ b/dependencies/direct/py.in @@ -4,7 +4,9 @@ covdefaults coverage # accessed directly from tox coverage-enable-subprocess hypothesis +jinja2 pytest pytest-cov pytest-mock pytest-xdist +pyyaml diff --git a/src/awx_plugins/interfaces/_temporary_private_api.py b/src/awx_plugins/interfaces/_temporary_private_api.py index 058b9084..34e833d9 100644 --- a/src/awx_plugins/interfaces/_temporary_private_api.py +++ b/src/awx_plugins/interfaces/_temporary_private_api.py @@ -4,14 +4,83 @@ The hope is that it will be refactored into something more standardized. """ +import os +import re +import stat +import tempfile from collections.abc import Callable +from jinja2 import sandbox +from yaml import safe_dump as yaml_safe_dump + +from ._temporary_private_container_api import get_incontainer_path from ._temporary_private_credential_api import ( # noqa: WPS436 Credential as Credential, GenericOptionalPrimitiveType, ) +HIDDEN_PASSWORD = '*' * 10 +SENSITIVE_ENV_VAR_NAMES = 'API|TOKEN|KEY|SECRET|PASS' + +HIDDEN_PASSWORD_RE = re.compile(SENSITIVE_ENV_VAR_NAMES, re.I) +HIDDEN_URL_PASSWORD_RE = re.compile('^.*?://[^:]+:(.*?)@.*?$') + +ENV_BLOCKLIST = frozenset( + ( + 'VIRTUAL_ENV', + 'PATH', + 'PYTHONPATH', + 'JOB_ID', + 'INVENTORY_ID', + 'INVENTORY_SOURCE_ID', + 'INVENTORY_UPDATE_ID', + 'AD_HOC_COMMAND_ID', + 'REST_API_URL', + 'REST_API_TOKEN', + 'MAX_EVENT_RES', + 'CALLBACK_QUEUE', + 'CALLBACK_CONNECTION', + 'CACHE', + 'JOB_CALLBACK_DEBUG', + 'INVENTORY_HOSTVARS', + 'AWX_HOST', + 'PROJECT_REVISION', + 'SUPERVISOR_CONFIG_PATH', + ) +) + +def build_safe_env( + env: dict[str, GenericOptionalPrimitiveType], +) -> dict[str, GenericOptionalPrimitiveType]: + """Obscure potentially sensitive env values. + + Given a set of environment variables, execute a set of heuristics to + obscure potentially sensitive environment values. + + :param env: Existing environment variables + :returns: Sanitized environment variables. + """ + safe_env = dict(env) + for env_k, env_val in safe_env.items(): + is_special = ( + env_k == 'AWS_ACCESS_KEY_ID' + or ( + env_k.startswith('ANSIBLE_') + and not env_k.startswith('ANSIBLE_NET') + and not env_k.startswith('ANSIBLE_GALAXY_SERVER') + ) + ) + if is_special: + continue + elif HIDDEN_PASSWORD_RE.search(env_k): + safe_env[env_k] = HIDDEN_PASSWORD + elif isinstance(env_val, str) and HIDDEN_URL_PASSWORD_RE.match(env_val): + safe_env[env_k] = HIDDEN_URL_PASSWORD_RE.sub( + HIDDEN_PASSWORD, env_val, + ) + return safe_env + try: # pylint: disable-next=unused-import from awx.main.models.credential import ( # noqa: WPS433 @@ -21,7 +90,7 @@ from dataclasses import dataclass # noqa: WPS433 @dataclass(frozen=True) - class ManagedCredentialType: # type: ignore[no-redef] # noqa: WPS440 + class ManagedCredentialType: """Managed credential type stub.""" namespace: str @@ -33,7 +102,7 @@ class ManagedCredentialType: # type: ignore[no-redef] # noqa: WPS440 kind: str """Plugin category.""" - inputs: dict[str, list[dict[str, bool | str] | str]] + inputs: dict[str, list[dict[str, str | bool]]] """UI input fields schema.""" injectors: dict[str, dict[str, str]] | None = None @@ -50,5 +119,166 @@ class ManagedCredentialType: # type: ignore[no-redef] # noqa: WPS440 ] | None = None """Function to call as an alternative to the templated injection.""" + @property + def secret_fields(self: 'ManagedCredentialType') -> list[str]: + return [ + str(field['id']) + for field in self.inputs.get('fields', []) + if field.get('secret', False) is True + ] + + def inject_credential( + self: 'ManagedCredentialType', + credential: Credential, + env: dict[str, GenericOptionalPrimitiveType], + safe_env: dict[str, GenericOptionalPrimitiveType], + args: list[GenericOptionalPrimitiveType], + private_data_dir: str, + ) -> None: + """Inject credential data. + + Inject credential data into the environment variables and + arguments passed to `ansible-playbook` + + :param credential: a :class:`awx.main.models.Credential` instance + :param env: a dictionary of environment variables used in + the `ansible-playbook` call. This method adds + additional environment variables based on + custom `env` injectors defined on this + CredentialType. + :param safe_env: a dictionary of environment variables stored + in the database for the job run + (`UnifiedJob.job_env`); secret values should + be stripped + :param args: a list of arguments passed to + `ansible-playbook` in the style of + `subprocess.call(args)`. This method appends + additional arguments based on custom + `extra_vars` injectors defined on this + CredentialType. + :param private_data_dir: a temporary directory to store files generated + by `file` injectors (like config files or key + files) + """ + if not self.injectors: + if self.managed and self.custom_injectors: + injected_env: dict[str, GenericOptionalPrimitiveType] = {} + self.custom_injectors( + credential, injected_env, private_data_dir, + ) + env.update(injected_env) + safe_env.update(build_safe_env(injected_env)) + return + + class TowerNamespace: + """Dummy class.""" + + tower_namespace = TowerNamespace() + + # maintain a normal namespace for building the ansible-playbook + # arguments (env and args) + namespace: dict[str, TowerNamespace | GenericOptionalPrimitiveType] = { + 'tower': tower_namespace, + } + + # maintain a sanitized namespace for building the DB-stored arguments + # (safe_env) + safe_namespace: dict[str, TowerNamespace | GenericOptionalPrimitiveType] = { + 'tower': tower_namespace, } + + # build a normal namespace with secret values decrypted (for + # ansible-playbook) and a safe namespace with secret values hidden (for + # DB storage) + for field_name in credential.get_input_keys(): + value = credential.get_input(field_name) + + if type(value) is bool: + # boolean values can't be secret/encrypted/external + safe_namespace[field_name] = value + namespace[field_name] = value + continue + + if field_name in self.secret_fields: + safe_namespace[field_name] = HIDDEN_PASSWORD + elif value: + safe_namespace[field_name] = value + if value: + namespace[field_name] = value + + for field in self.inputs.get('fields', []): + field_id = str(field['id']) + # default missing boolean fields to False + if field['type'] == 'boolean' and field_id not in credential.get_input_keys(): + namespace[field_id] = False + safe_namespace[field_id] = False + # make sure private keys end with a \n + if field.get('format') == 'ssh_private_key': + if field_id in namespace and not str(namespace[field_id]).endswith( + '\n', + ): + namespace[field_id] = str(namespace[field_id]) + '\n' + + file_tmpls = self.injectors.get('file', {}) + # If any file templates are provided, render the files and update the + # special `tower` template namespace so the filename can be + # referenced in other injectors + + sandbox_env = sandbox.ImmutableSandboxedEnvironment() # type: ignore[misc] + + for file_label, file_tmpl in file_tmpls.items(): + data: str = sandbox_env.from_string(file_tmpl).render(**namespace) # type: ignore[misc] + env_dir = os.path.join(private_data_dir, 'env') + _, path = tempfile.mkstemp(dir=env_dir) + with open(path, 'w') as f: + f.write(data) + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + container_path = get_incontainer_path(path, private_data_dir) + + # determine if filename indicates single file or many + if file_label.find('.') == -1: + tower_namespace.filename = container_path + else: + if not hasattr(tower_namespace, 'filename'): + tower_namespace.filename = TowerNamespace() + file_label = file_label.split('.')[1] + setattr(tower_namespace.filename, file_label, container_path) + + for env_var, tmpl in self.injectors.get('env', {}).items(): + if env_var in ENV_BLOCKLIST: + continue + env[env_var] = sandbox_env.from_string(tmpl).render(**namespace) + safe_env[env_var] = sandbox_env.from_string( + tmpl, + ).render(**safe_namespace) + + if 'INVENTORY_UPDATE_ID' not in env: + # awx-manage inventory_update does not support extra_vars via -e + def build_extra_vars(node: dict[str, str | list[str]] | list[str] | str) -> dict[str, str] | list[str] | str: + if isinstance(node, dict): + return { + build_extra_vars(k): build_extra_vars(v) for k, + v in node.items() + } + elif isinstance(node, list): + return [build_extra_vars(x) for x in node] + else: + return sandbox_env.from_string(node).render(**namespace) + + def build_extra_vars_file(vars, private_dir: str) -> str: + handle, path = tempfile.mkstemp( + dir=os.path.join(private_dir, 'env'), + ) + f = os.fdopen(handle, 'w') + f.write(yaml_safe_dump(vars)) + f.close() + os.chmod(path, stat.S_IRUSR) + return path + + extra_vars = build_extra_vars(self.injectors.get('extra_vars', {})) + if extra_vars: + path = build_extra_vars_file(extra_vars, private_data_dir) + container_path = get_incontainer_path(path, private_data_dir) + args.extend(['-e', '@%s' % container_path]) + __all__ = () # noqa: WPS410 diff --git a/tests/_temporary_private_api_test.py b/tests/_temporary_private_api_test.py index 2b6aab89..0df63f8c 100644 --- a/tests/_temporary_private_api_test.py +++ b/tests/_temporary_private_api_test.py @@ -1,8 +1,250 @@ """Tests for the temporarily hosted private helpers.""" -from awx_plugins.interfaces._temporary_private_api import ManagedCredentialType +import os +import jinja2 +import pytest +import shutil +import tempfile + +from pathlib import Path, PurePath + +import yaml +from awx_plugins.interfaces._temporary_private_api import HIDDEN_PASSWORD, ManagedCredentialType +from awx_plugins.interfaces._temporary_private_credential_api import Credential + +from awx_plugins.interfaces._temporary_private_container_api import CONTAINER_ROOT + + +def to_host_path(path, private_data_dir): + """Given a path inside of the EE container, this gives the absolute path + on the host machine within the private_data_dir + """ + if not os.path.isabs(private_data_dir): + raise RuntimeError('The private_data_dir path must be absolute') + if CONTAINER_ROOT != path and Path(CONTAINER_ROOT) not in Path(path).resolve().parents: + raise RuntimeError(f'Cannot convert path {path} unless it is a subdir of {CONTAINER_ROOT}') + return path.replace(CONTAINER_ROOT, private_data_dir, 1) + +def read_extra_vars(private_data_dir: str, args: list[str]) -> dict[str, str]: + fname = to_host_path(args[1][1:], private_data_dir) + with open(fname, 'r') as f: + return yaml.safe_load(f) + +def assert_dict_subset(subset, full_dict): + """ + Recursively asserts that `subset` is a subset of `full_dict`. + """ + for key, value in subset.items(): + assert key in full_dict, f"Key '{key}' not found in full_dict" + if isinstance(value, dict): + assert isinstance(full_dict[key], dict), f"Key '{key}' is not a dictionary in full_dict" + assert_dict_subset(value, full_dict[key]) + else: + assert value == full_dict[key], f"Value mismatch for key '{key}': {value} != {full_dict[key]}" + +@pytest.fixture +def private_data_dir(): + private_data = tempfile.mkdtemp(prefix='awx_') + for subfolder in ('inventory', 'env'): + runner_subfolder = os.path.join(private_data, subfolder) + if not os.path.exists(runner_subfolder): + os.mkdir(runner_subfolder) + yield private_data + shutil.rmtree(private_data, True) def test_managed_credential_type_instantiation() -> None: """Check that managed credential type can be instantiated.""" assert ManagedCredentialType('', '', '', {}) + + +def test_managed_credential_type_inject_cred() -> None: + """Check basic env var injection.""" + cred_type = ManagedCredentialType( + namespace='animal', + name='dog', + kind='companion', + managed=False, + inputs={ + 'fields': [ + { + 'id': 'my_pet_name', + 'label': 'My pet name', + 'type': 'string', + }, + ], + }, + injectors={'env': {'PET_NAME': '{{my_pet_name}}'}}, + ) + cred = Credential( + inputs={ + 'my_pet_name': 'birdie', + }, + ) + + env = {} + cred_type.inject_credential(cred, env, {}, [], '') + + assert env['PET_NAME'] == 'birdie' + +def test_custom_environment_injectors_with_jinja_syntax_error(private_data_dir): + cred_type = ManagedCredentialType( + kind='cloud', + name='SomeCloud', + namespace='foo', + managed=False, + inputs={'fields': [{'id': 'api_token', 'label': 'API Token', 'type': 'string'}]}, + injectors={'env': {'MY_CLOUD_API_TOKEN': '{{api_token.foo()}}'}}, + ) + credential = Credential(inputs={'api_token': 'ABC123'}) + + with pytest.raises(jinja2.exceptions.UndefinedError): + cred_type.inject_credential(credential, {}, {}, [], private_data_dir) + +def test_custom_environment_injectors_with_reserved_env_var(private_data_dir): + cred_type = ManagedCredentialType( + kind='cloud', + name='SomeCloud', + namespace='foo', + managed=False, + inputs={'fields': [{'id': 'api_token', 'label': 'API Token', 'type': 'string'}]}, + injectors={'env': {'JOB_ID': 'reserved'}}, + ) + credential = Credential(inputs={'api_token': 'ABC123'}) + + env = {} + cred_type.inject_credential(credential, env, {}, [], private_data_dir) + + assert 'JOB_ID' not in env + +def test_custom_environment_injectors_with_secret_field(private_data_dir): + cred_type = ManagedCredentialType( + kind='cloud', + name='SomeCloud', + namespace='foo', + managed=False, + inputs={'fields': [{'id': 'password', 'label': 'Password', 'type': 'string', 'secret': True}]}, + injectors={'env': {'MY_CLOUD_PRIVATE_VAR': '{{password}}'}}, + ) + credential = Credential(inputs={'password': 'SUPER-SECRET-123'}) + + env = {} + safe_env = {} + cred_type.inject_credential(credential, env, safe_env, [], private_data_dir) + + assert env['MY_CLOUD_PRIVATE_VAR'] == 'SUPER-SECRET-123' + assert 'SUPER-SECRET-123' not in safe_env.values() + assert safe_env['MY_CLOUD_PRIVATE_VAR'] == HIDDEN_PASSWORD + +@pytest.mark.parametrize( + ('inputs', 'injectors', 'cred_inputs', 'expected_extra_vars'), + ( + pytest.param( + {'fields': [{'id': 'api_token', 'label': 'API Token', 'type': 'string'}]}, + {'extra_vars': {'api_token': '{{api_token}}'}}, + {'api_token': 'ABC123'}, + {'api_token': 'ABC123'}, + id='happy-path', + ), + pytest.param ( + {'fields': [{'id': 'turbo_button', 'label': 'Turbo Button', 'type': 'boolean'}]}, + {'extra_vars': {'turbo_button': '{{turbo_button}}'}}, + {'turbo_button': True}, + {'turbo_button': "True"}, + id='boolean', + ), + pytest.param( + {'fields': [{'id': 'host', 'label': 'Host', 'type': 'string'}]}, + {'extra_vars': {'auth': {'host': '{{host}}'}}}, + {'host': 'example.com'}, + {'auth': {'host': 'example.com'}}, + id='nested', + ), + pytest.param( + {'fields': [{'id': 'environment', 'label': 'Environment', 'type': 'string'}, {'id': 'host', 'label': 'Host', 'type': 'string'}]}, + {'extra_vars': {'{{environment}}_auth': {'host': '{{host}}'}}}, + {'environment': 'test', 'host': 'example.com'}, + {'test_auth': {'host': 'example.com'}}, + id='templated-key', + ), + pytest.param( + {'fields': [{'id': 'turbo_button', 'label': 'Turbo Button', 'type': 'boolean'}]}, + {'extra_vars': {'turbo_button': '{% if turbo_button %}FAST!{% else %}SLOW!{% endif %}'}}, + {'turbo_button': True}, + {'turbo_button': 'FAST!'}, + id='templated-bool', + ) + ), +) +def test_custom_environment_injectors_with_extra_vars(private_data_dir, inputs, injectors, cred_inputs, expected_extra_vars): + cred_type = ManagedCredentialType( + kind='cloud', + name='SomeCloud', + namespace='foo', + managed=False, + inputs=inputs, + injectors=injectors, + ) + credential = Credential(inputs=cred_inputs) + + args = [] + cred_type.inject_credential(credential, {}, {}, args, private_data_dir) + + extra_vars = read_extra_vars(private_data_dir, args) + + assert_dict_subset(expected_extra_vars, extra_vars) + +@pytest.mark.parametrize( + ('inputs', 'injectors', 'cred_inputs', 'expected_file_content'), + ( + pytest.param( + {'fields': [{'id': 'api_token', 'label': 'API Token', 'type': 'string'}]}, + {'file': {'template': '[mycloud]\n{{api_token}}'}, 'env': {'MY_CLOUD_INI_FILE': '{{tower.filename}}'}}, + {'api_token': 'ABC123'}, + { + 'MY_CLOUD_INI_FILE': '[mycloud]\nABC123', + }, + id='ini-file', + ), + pytest.param( + {'fields': []}, + {'file': {'template': 'Iñtërnâtiônàlizætiøn'}, 'env': {'MY_CLOUD_INI_FILE': '{{tower.filename}}'}}, + {}, + { + 'MY_CLOUD_INI_FILE': 'Iñtërnâtiônàlizætiøn', + }, + id='unicode', + ), + pytest.param( + {'fields': [{'id': 'cert', 'label': 'Certificate', 'type': 'string'}, {'id': 'key', 'label': 'Key', 'type': 'string'}]}, + { + 'file': {'template.cert': '[mycert]\n{{cert}}', 'template.key': '[mykey]\n{{key}}'}, + 'env': {'MY_CERT_INI_FILE': '{{tower.filename.cert}}', 'MY_KEY_INI_FILE': '{{tower.filename.key}}'}, + }, + {'cert': 'CERT123', 'key': 'KEY123'}, + { + 'MY_CERT_INI_FILE': '[mycert]\nCERT123', + 'MY_KEY_INI_FILE': '[mykey]\nKEY123', + }, + id='multiple-files', + ) + ), +) +def test_custom_environment_injectors_with_file(private_data_dir, inputs, injectors, cred_inputs, expected_file_content): + cred_type = ManagedCredentialType( + kind='cloud', + name='SomeCloud', + namespace='foo', + managed=False, + inputs=inputs, + injectors=injectors, + ) + credential = Credential(inputs=cred_inputs) + + env = {} + cred_type.inject_credential(credential, env, {}, [], private_data_dir) + + for env_fname_key, expected_content in expected_file_content.items(): + path = to_host_path(env[env_fname_key], private_data_dir) + with open(path, 'r') as f: + assert f.read() == expected_content