diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 477b7b85..737d7238 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ on: branches: [ master ] pull_request: branches: [ master ] + workflow_dispatch: jobs: build: diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fd776cd..dec3b0fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # BigFlow changelog +## Version 1.6.0 + +### Fixed + +* Enabled vault endpoint TLS certificate verification by default for `bf build` and `bf deploy` commands. This fixes the MITM attack vulnerability. Kudos to Konstantin Weddige for reporting. + +### Breaking changes + +* Default vault endpoint TLS certificate verification for `bf build` and `bf deploy` may fail in some environments. Use `-vev`/`--vault-endpoint-verify` option to disable or provide path to custom trusted certificates or CA certificates. Disabling makes execution vulnerable for MITM attacks and is discouraged - do it only when justified and in trusted environments. See [https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification](https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification) for details. + ## Version 1.5.4 ### Changed diff --git a/bigflow/_version.py b/bigflow/_version.py index c24ed73b..8640030c 100644 --- a/bigflow/_version.py +++ b/bigflow/_version.py @@ -1 +1 @@ -__version__ = '1.5.4' +__version__ = '1.6.0.dev1' diff --git a/bigflow/build/operate.py b/bigflow/build/operate.py index 482bf112..68517319 100644 --- a/bigflow/build/operate.py +++ b/bigflow/build/operate.py @@ -95,6 +95,7 @@ def _build_docker_image( auth_method=cache_params.auth_method or bigflow.deploy.AuthorizationType.LOCAL_ACCOUNT, vault_endpoint=cache_params.vault_endpoint, vault_secret=cache_params.vault_secret, + vault_endpoint_verify=cache_params.vault_endpoint_verify ) for image in (cache_params.cache_from_image or []): @@ -120,6 +121,7 @@ class BuildImageCacheParams: vault_secret: str | None = None cache_from_version: list[str] | None = None cache_from_image: list[str] | None = None + vault_endpoint_verify: str | bool | None = None def build_image( diff --git a/bigflow/cli.py b/bigflow/cli.py index 3a834a2e..3ad15e16 100644 --- a/bigflow/cli.py +++ b/bigflow/cli.py @@ -13,8 +13,7 @@ from importlib import import_module from pathlib import Path from types import ModuleType -from typing import Tuple, Iterator -from typing import Optional +from typing import Tuple, Iterator, Optional import fnmatch import bigflow as bf @@ -384,6 +383,13 @@ def _add_parsers_common_arguments(parser): def _add_auth_parsers_arguments(parser): + class VaultEndpointVerifyAction(argparse.Action): + def __call__(self, parser, args, values, option_string=None): + if values in ['true', 'false']: + setattr(args, self.dest, values == 'true') + else: + setattr(args, self.dest, str(values)) + parser.add_argument('-a', '--auth-method', type=bigflow.deploy.AuthorizationType, default='local_account', @@ -399,6 +405,17 @@ def _add_auth_parsers_arguments(parser): 'Required if auth-method is vault. ' 'If not set, will be read from deployment_config.py.' ) + parser.add_argument('-vev', '--vault-endpoint-verify', + type=str, + action=VaultEndpointVerifyAction, + help='Can be "true", "false", a path to certificate PEM file or a path to ' + 'directory with PEM files (see the link for details). ' + 'Enables/disables vault endpoint TLS certificate verification. Enabled by default. ' + 'Disabling makes execution vulnerable for MITM attacks - do it only when justified and in trusted environments. ' + 'For details see: https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification', + dest='vault_endpoint_verify', + default=True + ) parser.add_argument('-vs', '--vault-secret', type=str, help='Vault secret token. ' @@ -514,7 +531,7 @@ def _resolve_vault_endpoint(args): def _resolve_property(args, property_name, ignore_value_error=False): try: cli_atr = getattr(args, property_name) - if cli_atr: + if cli_atr or cli_atr is False: return cli_atr else: config = import_deployment_config(_resolve_deployment_config_path(args), property_name) @@ -533,6 +550,7 @@ def _cli_deploy_dags(args): clear_dags_folder=args.clear_dags_folder, auth_method=args.auth_method, vault_endpoint=_resolve_vault_endpoint(args), + vault_endpoint_verify=_resolve_property(args, 'vault_endpoint_verify', ignore_value_error=True), vault_secret=vault_secret, project_id=_resolve_property(args, 'gcp_project_id') ) @@ -543,6 +561,7 @@ def _cli_deploy_image(args): docker_repository = _resolve_property(args, 'docker_repository') vault_secret = _resolve_property(args, 'vault_secret', ignore_value_error=True) vault_endpoint = _resolve_vault_endpoint(args) + vault_endpoint_verify = _resolve_property(args, 'vault_endpoint_verify', ignore_value_error=True) image_tar_path = args.image_tar_path if args.image_tar_path else find_image_file() bigflow.deploy.deploy_docker_image( @@ -550,6 +569,7 @@ def _cli_deploy_image(args): auth_method=args.auth_method, docker_repository=docker_repository, vault_endpoint=vault_endpoint, + vault_endpoint_verify=vault_endpoint_verify, vault_secret=vault_secret, ) @@ -579,12 +599,14 @@ def _grab_image_cache_params(args): logger.debug("Image caching is requested - create build image cache params obj") vault_secret = _resolve_property(args, 'vault_secret', ignore_value_error=True) vault_endpoint = _resolve_vault_endpoint(args) + vault_endpoint_verify = _resolve_property(args, 'vault_endpoint_verify', ignore_value_error=True) return bigflow.build.operate.BuildImageCacheParams( auth_method=args.auth_method, vault_endpoint=vault_endpoint, vault_secret=vault_secret, cache_from_version=args.cache_from_version, cache_from_image=args.cache_from_image, + vault_endpoint_verify=vault_endpoint_verify ) else: logger.debug("No caching is requested - so just disable it completly") diff --git a/bigflow/deploy.py b/bigflow/deploy.py index 41fbcd6c..be0ba8b5 100644 --- a/bigflow/deploy.py +++ b/bigflow/deploy.py @@ -42,6 +42,7 @@ def deploy_docker_image( docker_repository: str, auth_method: AuthorizationType = AuthorizationType.LOCAL_ACCOUNT, vault_endpoint: T.Optional[str] = None, + vault_endpoint_verify: str | bool | None = None, vault_secret: T.Optional[str] = None, ) -> str: if image_tar_path.endswith(".toml"): @@ -53,6 +54,7 @@ def deploy_docker_image( docker_repository, auth_method, vault_endpoint, + vault_endpoint_verify, vault_secret, ) @@ -62,6 +64,7 @@ def _deploy_docker_image_from_local_repo( docker_repository: str, auth_method: AuthorizationType = AuthorizationType.LOCAL_ACCOUNT, vault_endpoint: str | None = None, + vault_endpoint_verify: str | bool | None = None, vault_secret: str | None = None, ) -> str: @@ -81,6 +84,7 @@ def _deploy_docker_image_from_local_repo( docker_repository=docker_repository, auth_method=auth_method, vault_endpoint=vault_endpoint, + vault_endpoint_verify=vault_endpoint_verify, vault_secret=vault_secret, image_id=image_id, build_ver=build_ver, @@ -92,6 +96,7 @@ def _deploy_docker_image_from_fs( docker_repository: str, auth_method: AuthorizationType = AuthorizationType.LOCAL_ACCOUNT, vault_endpoint: str | None = None, + vault_endpoint_verify: str | bool | None = None, vault_secret: str | None = None, ) -> str: build_ver = bf_commons.decode_version_number_from_file_name(Path(image_tar_path)) @@ -105,6 +110,7 @@ def _deploy_docker_image_from_fs( image_id=image_id, auth_method=auth_method, vault_endpoint=vault_endpoint, + vault_endpoint_verify=vault_endpoint_verify, vault_secret=vault_secret, ) finally: @@ -118,6 +124,7 @@ def _deploy_image_loaded_to_local_registry( image_id: str, auth_method: AuthorizationType, vault_endpoint: str | None = None, + vault_endpoint_verify: str | bool | None = None, vault_secret: str | None = None, ) -> str: docker_image = docker_repository + ":" + build_ver @@ -125,7 +132,7 @@ def _deploy_image_loaded_to_local_registry( tag_image(image_id, docker_repository, "latest") logger.info("Deploying docker image tag=%s auth_method=%s", docker_image, auth_method) - authenticate_to_registry(auth_method, vault_endpoint, vault_secret) + authenticate_to_registry(auth_method, vault_endpoint, vault_secret, vault_endpoint_verify) bf_commons.run_process(['docker', 'push', docker_image]) bf_commons.run_process(['docker', 'push', docker_image_latest]) @@ -136,13 +143,14 @@ def authenticate_to_registry( auth_method: AuthorizationType, vault_endpoint: T.Optional[str] = None, vault_secret: T.Optional[str] = None, + vault_endpoint_verify: str | bool | None = None, ): logger.info("Authenticating to registry with auth_method=%s", auth_method) if auth_method == AuthorizationType.LOCAL_ACCOUNT: bf_commons.run_process(['gcloud', 'auth', 'configure-docker']) elif auth_method == AuthorizationType.VAULT: - oauthtoken = get_vault_token(vault_endpoint, vault_secret) + oauthtoken = get_vault_token(vault_endpoint, vault_secret, vault_endpoint_verify) bf_commons.run_process( ['docker', 'login', '-u', 'oauth2accesstoken', '--password-stdin', 'https://eu.gcr.io'], input=oauthtoken, @@ -156,9 +164,10 @@ def check_images_exist( auth_method: AuthorizationType, vault_endpoint: T.Optional[str] = None, vault_secret: T.Optional[str] = None, + vault_endpoint_verify: str | bool | None = None ): logger.info("Checking if images used in DAGs exist in the registry") - authenticate_to_registry(auth_method, vault_endpoint, vault_secret) + authenticate_to_registry(auth_method, vault_endpoint, vault_secret, vault_endpoint_verify) missing_images = set() for image in images: found_images = bf_commons.run_process(['docker', 'manifest', 'inspect', image], check=False, verbose=False) @@ -189,6 +198,7 @@ def deploy_dags_folder( clear_dags_folder: bool = False, auth_method: AuthorizationType = AuthorizationType.LOCAL_ACCOUNT, vault_endpoint: T.Optional[str] = None, + vault_endpoint_verify: str | bool | None = None, vault_secret: T.Optional[str] = None, gs_client: T.Optional[storage.Client] = None, ) -> str: @@ -196,12 +206,13 @@ def deploy_dags_folder( if images: check_images_exist(auth_method=auth_method, vault_endpoint=vault_endpoint, + vault_endpoint_verify=vault_endpoint_verify, vault_secret=vault_secret, images=images) logger.info("Deploying DAGs folder, auth_method=%s, clear_dags_folder=%s, dags_dir=%s", auth_method, clear_dags_folder, dags_dir) - client = gs_client or create_storage_client(auth_method, project_id, vault_endpoint, vault_secret) + client = gs_client or create_storage_client(auth_method, project_id, vault_endpoint, vault_secret, vault_endpoint_verify) bucket = client.bucket(dags_bucket) if clear_dags_folder: @@ -246,30 +257,31 @@ def create_storage_client( project_id: str, vault_endpoint: str, vault_secret: str, + vault_endpoint_verify: str | bool | None = None ) -> storage.Client: if auth_method == AuthorizationType.LOCAL_ACCOUNT: return storage.Client(project=project_id) elif auth_method == AuthorizationType.VAULT: - oauthtoken = get_vault_token(vault_endpoint, vault_secret) + oauthtoken = get_vault_token(vault_endpoint, vault_secret, vault_endpoint_verify) return storage.Client(project=project_id, credentials=credentials.Credentials(oauthtoken)) else: raise ValueError(f"unsupported auth_method: {auth_method!r}") -def get_vault_token(vault_endpoint: str, vault_secret: str) -> str: +def get_vault_token(vault_endpoint: str, vault_secret: str, vault_endpoint_verify: str | bool | None = True) -> str: if not vault_endpoint: raise ValueError('vault_endpoint is required') if not vault_secret: raise ValueError('vault_secret is required') headers = {'X-Vault-Token': vault_secret} - response = requests.get(vault_endpoint, headers=headers, verify=False) + response = requests.get(vault_endpoint, headers=headers, verify=vault_endpoint_verify) + logger.info("get oauth token from %s status_code=%s", vault_endpoint, response.status_code) if response.status_code != 200: logger.info(response.text) raise ValueError( 'Could not get vault token, response code: {}'.format( response.status_code)) - logger.info("get oauth token from %s status_code=%s", vault_endpoint, response.status_code) return response.json()['data']['token'] diff --git a/requirements.in b/requirements.in index fb443a3e..00c83bdf 100644 --- a/requirements.in +++ b/requirements.in @@ -12,4 +12,5 @@ bs4 pytest pytest-html pytest-cov -pytest-github-actions-annotate-failures \ No newline at end of file +pytest-github-actions-annotate-failures +parameterized \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1a5061b4..a0db9ddf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ +# *** autogenerated: don't edit *** +# $source-hash: sha256:758f72d93f463f46b9823c68917482b898b517bf66baaa0dc72b6b6dd534aab7 +# $source-file: requirements.in # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: -# -# pip-compile requirements.in -# +# run 'bigflow build-requirements requirements.in' to update this file + apache-beam[gcp]==2.36.0 # via -r requirements/dataflow_extras.txt attrs==22.1.0 @@ -205,6 +205,8 @@ packaging==21.3 # pytest pandas==1.3.5 # via -r requirements/bigquery_extras.txt +parameterized==0.8.1 + # via -r requirements.in pep517==0.13.0 # via build pexpect==4.8.0 diff --git a/test/cli/test_cli.py b/test/cli/test_cli.py index 9671f69c..d51ffd6f 100644 --- a/test/cli/test_cli.py +++ b/test/cli/test_cli.py @@ -1,6 +1,7 @@ from unittest import mock import shutil import freezegun +from parameterized import parameterized from bigflow.build.operate import BuildImageCacheParams from bigflow.deploy import AuthorizationType @@ -291,13 +292,16 @@ def test_should_call_cli_deploy_dags_command__with_defaults_and_with_implicit_de cli(['deploy-dags']) # then - deploy_dags_folder_mock.assert_called_with(auth_method=AuthorizationType.LOCAL_ACCOUNT, - clear_dags_folder=False, - dags_bucket='my-dags-bucket', - dags_dir=self._expected_default_dags_dir(), - project_id='my-gcp-project-id', - vault_endpoint=None, - vault_secret='secret') + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.LOCAL_ACCOUNT, + clear_dags_folder=False, + dags_bucket='my-dags-bucket', + dags_dir=self._expected_default_dags_dir(), + project_id='my-gcp-project-id', + vault_endpoint=None, + vault_secret='secret', + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_dags_folder') def test_should_call_cli_deploy_dags_command_for_different_environments(self, deploy_dags_folder_mock): @@ -325,37 +329,46 @@ def test_should_call_cli_deploy_dags_command_for_different_environments(self, de cli(['deploy-dags']) # then - deploy_dags_folder_mock.assert_called_with(auth_method=AuthorizationType.LOCAL_ACCOUNT, - clear_dags_folder=False, - dags_bucket='my-dags-dev-bucket', - dags_dir=self._expected_default_dags_dir(), - project_id='my-gcp-dev-project-id', - vault_endpoint=None, - vault_secret='secret-dev') + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.LOCAL_ACCOUNT, + clear_dags_folder=False, + dags_bucket='my-dags-dev-bucket', + dags_dir=self._expected_default_dags_dir(), + project_id='my-gcp-dev-project-id', + vault_endpoint=None, + vault_secret='secret-dev', + vault_endpoint_verify=True + ) # when cli(['deploy-dags', '--config', 'dev']) # then - deploy_dags_folder_mock.assert_called_with(auth_method=AuthorizationType.LOCAL_ACCOUNT, - clear_dags_folder=False, - dags_bucket='my-dags-dev-bucket', - dags_dir=self._expected_default_dags_dir(), - project_id='my-gcp-dev-project-id', - vault_endpoint=None, - vault_secret='secret-dev') + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.LOCAL_ACCOUNT, + clear_dags_folder=False, + dags_bucket='my-dags-dev-bucket', + dags_dir=self._expected_default_dags_dir(), + project_id='my-gcp-dev-project-id', + vault_endpoint=None, + vault_secret='secret-dev', + vault_endpoint_verify=True + ) # when cli(['deploy-dags', '--config', 'prod']) # then - deploy_dags_folder_mock.assert_called_with(auth_method=AuthorizationType.LOCAL_ACCOUNT, - clear_dags_folder=False, - dags_bucket='my-dags-prod-bucket', - dags_dir=self._expected_default_dags_dir(), - project_id='my-gcp-prod-project-id', - vault_endpoint=None, - vault_secret='secret-prod') + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.LOCAL_ACCOUNT, + clear_dags_folder=False, + dags_bucket='my-dags-prod-bucket', + dags_dir=self._expected_default_dags_dir(), + project_id='my-gcp-prod-project-id', + vault_endpoint=None, + vault_secret='secret-prod', + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_dags_folder') def test_should_call_cli_deploy_dags_command__when_parameters_are_given_by_explicit_deployment_config_file(self, @@ -382,13 +395,16 @@ def test_should_call_cli_deploy_dags_command__when_parameters_are_given_by_expli ]) # then - deploy_dags_folder_mock.assert_called_with(auth_method=AuthorizationType.VAULT, - clear_dags_folder=False, - dags_bucket='my-another-dags-bucket', - dags_dir='/tmp/my-dags-dir', - project_id='my-another-gcp-project-id', - vault_endpoint='my-another-vault-endpoint', - vault_secret='secrett') + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.VAULT, + clear_dags_folder=False, + dags_bucket='my-another-dags-bucket', + dags_dir='/tmp/my-dags-dir', + project_id='my-another-gcp-project-id', + vault_endpoint='my-another-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_dags_folder') def test_should_call_cli_deploy_dags_command__when_all_parameters_are_given_by_cli_arguments(self, @@ -405,13 +421,16 @@ def test_should_call_cli_deploy_dags_command__when_all_parameters_are_given_by_c ]) # then - deploy_dags_folder_mock.assert_called_with(auth_method=AuthorizationType.VAULT, - clear_dags_folder=True, - dags_bucket='my-dags-bucket', - dags_dir='/tmp/my-dags-dir', - project_id='my-gcp-project-id', - vault_endpoint='my-vault-endpoint', - vault_secret='secrett') + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.VAULT, + clear_dags_folder=True, + dags_bucket='my-dags-bucket', + dags_dir='/tmp/my-dags-dir', + project_id='my-gcp-project-id', + vault_endpoint='my-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_docker_image') def test_should_call_cli_deploy_image_command__with_defaults_and_with_implicit_deployment_config_file(self, @@ -431,11 +450,14 @@ def test_should_call_cli_deploy_image_command__with_defaults_and_with_implicit_d cli(['deploy-image', '--image-tar-path', 'image-0.0.2.tar']) # then - deploy_docker_image_mock.assert_called_with(auth_method=AuthorizationType.LOCAL_ACCOUNT, - docker_repository='my-docker--repository', - image_tar_path='image-0.0.2.tar', - vault_endpoint=None, - vault_secret=None) + deploy_docker_image_mock.assert_called_with( + auth_method=AuthorizationType.LOCAL_ACCOUNT, + docker_repository='my-docker--repository', + image_tar_path='image-0.0.2.tar', + vault_endpoint=None, + vault_secret=None, + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_docker_image') def test_should_call_cli_deploy_image_command__with_explicit_deployment_config_file(self, deploy_docker_image_mock): @@ -460,11 +482,14 @@ def test_should_call_cli_deploy_image_command__with_explicit_deployment_config_f ]) # then - deploy_docker_image_mock.assert_called_with(auth_method=AuthorizationType.VAULT, - docker_repository='my-another-docker-repository', - image_tar_path='image-0.0.3.tar', - vault_endpoint='my-another-vault-endpoint', - vault_secret='secrett') + deploy_docker_image_mock.assert_called_with( + auth_method=AuthorizationType.VAULT, + docker_repository='my-another-docker-repository', + image_tar_path='image-0.0.3.tar', + vault_endpoint='my-another-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_docker_image') def test_should_call_cli_deploy_image_command__when_all_parameters_are_given_by_cli_arguments_and_image_is_loaded_from_tar( @@ -479,11 +504,14 @@ def test_should_call_cli_deploy_image_command__when_all_parameters_are_given_by_ ]) # then - deploy_docker_image_mock.assert_called_with(auth_method=AuthorizationType.VAULT, - docker_repository='my-docker-repository', - image_tar_path='image-0.0.1.tar', - vault_endpoint='my-vault-endpoint', - vault_secret='secrett') + deploy_docker_image_mock.assert_called_with( + auth_method=AuthorizationType.VAULT, + docker_repository='my-docker-repository', + image_tar_path='image-0.0.1.tar', + vault_endpoint='my-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_docker_image') def test_should_find_tar_in_image_directory(self, deploy_docker_image_mock): @@ -501,11 +529,14 @@ def test_should_find_tar_in_image_directory(self, deploy_docker_image_mock): ]) # then - deploy_docker_image_mock.assert_called_with(auth_method=AuthorizationType.VAULT, - docker_repository='my-docker-repository', - image_tar_path='.image/image-123.tar', - vault_endpoint='my-vault-endpoint', - vault_secret='secrett') + deploy_docker_image_mock.assert_called_with( + auth_method=AuthorizationType.VAULT, + docker_repository='my-docker-repository', + image_tar_path='.image/image-123.tar', + vault_endpoint='my-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=True + ) @mock.patch('bigflow.deploy.deploy_docker_image') def test_should_find_toml_ref_in_image_directory(self, deploy_docker_image_mock): @@ -529,6 +560,7 @@ def test_should_find_toml_ref_in_image_directory(self, deploy_docker_image_mock) image_tar_path='.image/imageinfo-123.toml', vault_endpoint='my-vault-endpoint', vault_secret='secrett', + vault_endpoint_verify=True, ) @mock.patch('bigflow.deploy.deploy_dags_folder') @@ -552,19 +584,72 @@ def test_should_call_both_deploy_methods_with_deploy_command(self, deploy_docker cli(['deploy', '-i', 'my-images/image-version']) # then - deploy_dags_folder_mock.assert_called_with(auth_method=AuthorizationType.LOCAL_ACCOUNT, - clear_dags_folder=False, - dags_bucket='my-dags-bucket', - dags_dir=self._expected_default_dags_dir(), - project_id='my-gcp-project-id', - vault_endpoint=None, - vault_secret=None) + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.LOCAL_ACCOUNT, + clear_dags_folder=False, + dags_bucket='my-dags-bucket', + dags_dir=self._expected_default_dags_dir(), + project_id='my-gcp-project-id', + vault_endpoint=None, + vault_secret=None, + vault_endpoint_verify=True + ) + + deploy_docker_image_mock.assert_called_with( + auth_method=AuthorizationType.LOCAL_ACCOUNT, + docker_repository='my-docker--repository', + image_tar_path='my-images/image-version', + vault_endpoint=None, + vault_secret=None, + vault_endpoint_verify=True + ) + + @parameterized.expand([ + ['true', True], + ['false', False], + ['certificate/path', 'certificate/path'], + ]) + @mock.patch('bigflow.deploy.deploy_dags_folder') + @mock.patch('bigflow.deploy.deploy_docker_image') + def test_should_use_provided_vault_endpoint_verify_value_when_deploy( + self, verify, expected_verify, deploy_docker_image_mock, deploy_dags_folder_mock): + # given + shutil.rmtree(Path.cwd() / ".image", ignore_errors=True) + self._touch_file('imageinfo-123.toml', '', '.image') + + # when + cli(['deploy', + '--docker-repository', 'my-docker-repository', + '--vault-endpoint', 'my-vault-endpoint', + '--auth-method', 'vault', + '--vault-secret', 'secrett', + '--dags-bucket', 'my-dags-bucket', + '--dags-dir', '/tmp/my-dags-dir', + '--gcp-project-id', 'my-gcp-project-id', + '--clear-dags-folder', + '--vault-endpoint-verify', verify + ]) + + # then + deploy_dags_folder_mock.assert_called_with( + auth_method=AuthorizationType.VAULT, + clear_dags_folder=True, + dags_bucket='my-dags-bucket', + dags_dir='/tmp/my-dags-dir', + project_id='my-gcp-project-id', + vault_endpoint='my-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=expected_verify + ) - deploy_docker_image_mock.assert_called_with(auth_method=AuthorizationType.LOCAL_ACCOUNT, - docker_repository='my-docker--repository', - image_tar_path='my-images/image-version', - vault_endpoint=None, - vault_secret=None) + deploy_docker_image_mock.assert_called_with( + auth_method=AuthorizationType.VAULT, + docker_repository='my-docker-repository', + image_tar_path='.image/imageinfo-123.toml', + vault_endpoint='my-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=expected_verify, + ) @mock.patch('bigflow.cli._cli_build_dags') def test_should_call_cli_build_dags_command(self, _cli_build_dags_mock): @@ -664,27 +749,6 @@ def test_should_call_cli_build_image_command_with_tar(self): cli_build_mock.assert_called_once() self.assertEqual(cli_build_mock.call_args[0][0].export_image_tar, True) - @mock.patch('bigflow.cli._cli_build_image') - def test_should_call_cli_build_image_command_without_tar(self, _cli_build_image_mock): - # when - cli(['build-image', '--no-export-image-tar']) - - # then - _cli_build_image_mock.assert_called_with( - Namespace( - auth_method=AuthorizationType.LOCAL_ACCOUNT, - cache_from_image=None, - cache_from_version=None, - config=None, - deployment_config_path=None, - export_image_tar=False, - operation='build-image', - vault_endpoint=None, - vault_secret=None, - verbose=False, - ) - ) - def test_should_call_cli_build_image_with_cached_from_image(self): # given @@ -712,6 +776,7 @@ def test_should_call_cli_build_image_with_cached_from_image(self): vault_secret='secrett', cache_from_image=['xyz.org/foo:bar', 'xyz.org/foo:baz'], cache_from_version=None, + vault_endpoint_verify=True, )) def test_should_call_cli_build_image_with_cached_from_version(self): @@ -741,6 +806,63 @@ def test_should_call_cli_build_image_with_cached_from_version(self): vault_secret='secrett', cache_from_image=None, cache_from_version=['bar', 'baz'], + vault_endpoint_verify=True, + )) + + def test_should_call_cli_build_image_from_cache_with_vault_endpoint_verify_by_default(self): + # given + self.addMock(mock.patch('bigflow.build.spec.read_project_spec')) + build_image_mock = self.addMock(mock.patch('bigflow.build.operate.build_image')) + + # when + cli([ + 'build-image', + '--vault-endpoint', 'my-vault-endpoint', + '--auth-method', 'vault', + '--vault-secret', 'secrett', + '--cache-from-image', 'xyz.org/foo:bar', + ]) + + # then + build_image_mock.assert_called_once() + _, kwrgs = build_image_mock.call_args + self.assertEqual(kwrgs['cache_params'], BuildImageCacheParams( + auth_method=AuthorizationType.VAULT, + vault_endpoint='my-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=True, + cache_from_image=['xyz.org/foo:bar'], + )) + + @parameterized.expand([ + ['true', True], + ['false', False], + ['certificate/path', 'certificate/path'], + ]) + def test_should_call_cli_build_image_from_cache_with_vault_endpoint_verify(self, verify, expected_verify): + # given + self.addMock(mock.patch('bigflow.build.spec.read_project_spec')) + build_image_mock = self.addMock(mock.patch('bigflow.build.operate.build_image')) + + # when + cli([ + 'build-image', + '--vault-endpoint', 'my-vault-endpoint', + '--auth-method', 'vault', + '--vault-secret', 'secrett', + '--cache-from-image', 'xyz.org/foo:bar', + '--vault-endpoint-verify', verify, + ]) + + # then + build_image_mock.assert_called_once() + _, kwrgs = build_image_mock.call_args + self.assertEqual(kwrgs['cache_params'], BuildImageCacheParams( + auth_method=AuthorizationType.VAULT, + vault_endpoint='my-vault-endpoint', + vault_secret='secrett', + vault_endpoint_verify=expected_verify, + cache_from_image=['xyz.org/foo:bar'], )) @mock.patch('bigflow.build.operate.build_project') @@ -789,6 +911,7 @@ def test_should_call_cli_build_command(self, _cli_build_mock): verbose=False, workflow=None, config=None, + vault_endpoint_verify=True, ) # then diff --git a/test/test_deploy.py b/test/test_deploy.py index db85fa33..282846e2 100644 --- a/test/test_deploy.py +++ b/test/test_deploy.py @@ -1,8 +1,8 @@ import os from unittest import mock from pathlib import Path -import unittest +import requests import responses from bigflow.build.operate import create_image_version_file @@ -18,6 +18,11 @@ from test.mixins import TempCwdMixin, BaseTestCase +TEST_VAULT_ENDPOINT = 'https://example.com/v1/gcp/token' +TEST_VAULT_SECRET = 'secret' +VAULT_TOKEN_HEADER = 'X-Vault-Token' +TEST_VAULT_GET_HEADERS = {VAULT_TOKEN_HEADER: TEST_VAULT_SECRET} + class DeployTestCase(TempCwdMixin, BaseTestCase): @@ -93,32 +98,82 @@ def blobs(name): @responses.activate def test_should_retrieve_token_from_vault(self): # given - vault_endpoint = 'https://example.com/v1/gcp/token' - responses.add(responses.GET, vault_endpoint, status=200, + responses.add(responses.GET, TEST_VAULT_ENDPOINT, status=200, json={'data': {'token': 'token_value'}}) # when - token = get_vault_token(vault_endpoint, 'secret') + token = get_vault_token(TEST_VAULT_ENDPOINT, TEST_VAULT_SECRET) # then self.assertEqual(token, 'token_value') self.assertEqual(len(responses.calls), 1) - self.assertEqual(responses.calls[0].request.url, 'https://example.com/v1/gcp/token') - self.assertEqual(responses.calls[0].request.headers['X-Vault-Token'], 'secret') + self.assertEqual(responses.calls[0].request.url, TEST_VAULT_ENDPOINT) + self.assertEqual(responses.calls[0].request.headers[VAULT_TOKEN_HEADER], TEST_VAULT_SECRET) + + @responses.activate + @mock.patch('requests.get', wraps=requests.get) + def test_should_retrieve_token_from_vault_verifying_endpoint_by_default(self, requests_get_mock): + # given + responses.add(responses.GET, TEST_VAULT_ENDPOINT, status=200, + json={'data': {'token': 'token_value'}}) + + # when + token = get_vault_token(TEST_VAULT_ENDPOINT, TEST_VAULT_SECRET) + + # then + requests_get_mock.assert_called_with(TEST_VAULT_ENDPOINT, headers=TEST_VAULT_GET_HEADERS, verify=True) + + @responses.activate + @mock.patch('requests.get', wraps=requests.get) + def test_should_retrieve_token_from_vault_verifying_endpoint(self, requests_get_mock): + # given + responses.add(responses.GET, TEST_VAULT_ENDPOINT, status=200, + json={'data': {'token': 'token_value'}}) + + # when + token = get_vault_token(TEST_VAULT_ENDPOINT, TEST_VAULT_SECRET, vault_endpoint_verify=True) + + # then + requests_get_mock.assert_called_with(TEST_VAULT_ENDPOINT, headers=TEST_VAULT_GET_HEADERS, verify=True) + + @responses.activate + @mock.patch('requests.get', wraps=requests.get) + def test_should_retrieve_token_from_vault_without_endpoint_verification(self, requests_get_mock): + # given + responses.add(responses.GET, TEST_VAULT_ENDPOINT, status=200, + json={'data': {'token': 'token_value'}}) + + # when + token = get_vault_token(TEST_VAULT_ENDPOINT, TEST_VAULT_SECRET, vault_endpoint_verify=False) + + # then + requests_get_mock.assert_called_with(TEST_VAULT_ENDPOINT, headers=TEST_VAULT_GET_HEADERS, verify=False) + + @responses.activate + @mock.patch('requests.get', wraps=requests.get) + def test_should_retrieve_token_from_vault_with_trusted_cert_path(self, requests_get_mock): + # given + responses.add(responses.GET, TEST_VAULT_ENDPOINT, status=200, + json={'data': {'token': 'token_value'}}) + + # when + token = get_vault_token(TEST_VAULT_ENDPOINT, TEST_VAULT_SECRET, vault_endpoint_verify='/path/to/trusted/certificate.pem') + + # then + requests_get_mock.assert_called_with(TEST_VAULT_ENDPOINT, headers=TEST_VAULT_GET_HEADERS, verify='/path/to/trusted/certificate.pem') @responses.activate def test_should_raise_value_error_if_vault_problem_occurred_during_fetching_token(self): # given - responses.add(responses.GET, 'https://example.com/v1/gcp/token', status=503) - vault_endpoint = 'https://example.com/v1/gcp/token' + responses.add(responses.GET, TEST_VAULT_ENDPOINT, status=503) # then with self.assertRaises(ValueError): # when - get_vault_token(vault_endpoint, 'secret') + get_vault_token(TEST_VAULT_ENDPOINT, TEST_VAULT_SECRET) self.assertEqual(len(responses.calls), 1) - self.assertEqual(responses.calls[0].request.url, 'https://example.com/v1/gcp/token') - self.assertEqual(responses.calls[0].request.headers['X-Vault-Token'], 'secret') + self.assertEqual(responses.calls[0].request.url, TEST_VAULT_ENDPOINT) + self.assertEqual(responses.calls[0].request.headers[VAULT_TOKEN_HEADER], TEST_VAULT_SECRET) @mock.patch('bigflow.commons.decode_version_number_from_file_name') @mock.patch('bigflow.deploy.load_image_from_tar') @@ -146,6 +201,7 @@ def test_should_remove_image_from_local_registry_after_deploy(self, docker_repository='docker_repository', image_id='image_id', vault_endpoint=None, + vault_endpoint_verify=None, vault_secret=None, ) remove_docker_image_from_local_registry.assert_called_with('docker_repository:version123') @@ -220,7 +276,6 @@ def test_should_not_upload_dags_if_image_is_missing(self, authenticate_to_regist gs_client.bucket.assert_not_called() upload_dags_folder.assert_not_called() - def test_deploy_image_pushes_tags(self): # given authenticate_to_registry_mock = self.addMock(mock.patch('bigflow.deploy.authenticate_to_registry')) @@ -232,13 +287,13 @@ def test_deploy_image_pushes_tags(self): docker_repository="docker_repository", image_id="image123", auth_method=AuthorizationType.VAULT, - vault_endpoint="vault_endpoint", - vault_secret="vault_secret", + vault_endpoint=TEST_VAULT_ENDPOINT, + vault_secret=TEST_VAULT_SECRET, ) # then authenticate_to_registry_mock.assert_called_once_with( - AuthorizationType.VAULT, "vault_endpoint", "vault_secret") + AuthorizationType.VAULT, TEST_VAULT_ENDPOINT, TEST_VAULT_SECRET, None) run_process_mock.assert_has_calls([ mock.call(["docker", "tag", "image123", "docker_repository:latest"]),