From c8404ea7fbed8d1c0c4de8fee56311baf4917ce7 Mon Sep 17 00:00:00 2001 From: Tom Maddox Date: Fri, 14 Feb 2014 13:28:10 +0000 Subject: [PATCH 1/4] Added gonzo instance tag injection into stack templates --- gonzo/backends/__init__.py | 59 +++++++++++++++++++++++---- tests/scripts/stack/test_ownership.py | 4 +- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/gonzo/backends/__init__.py b/gonzo/backends/__init__.py index ac8c567..090ef6c 100644 --- a/gonzo/backends/__init__.py +++ b/gonzo/backends/__init__.py @@ -29,6 +29,19 @@ def get_next_hostname(env_type): return name +def get_default_instance_tags_dict(environment, server_type, owner=None): + """ Generate a tag dictionary suitable for identifying instance ownership + and for identifying instances for gonzo releases. + """ + default_tags = { + 'environment': environment, + 'server_type': server_type, + } + if owner: + default_tags['owner'] = owner + return default_tags + + def launch_instance(env_type, size=None, user_data_uri=None, user_data_params=None, security_groups=None, extra_tags=None, owner=None): @@ -63,13 +76,8 @@ def launch_instance(env_type, size=None, key_name = config.CLOUD['PUBLIC_KEY_NAME'] tags = extra_tags or {} - tags.update({ - 'environment': environment, - 'server_type': server_type, - }) - - if owner: - tags['owner'] = owner + tags.update( + get_default_instance_tags_dict(environment, server_type, owner)) security_groups = add_default_security_groups(server_type, security_groups) for security_group in security_groups: @@ -130,6 +138,10 @@ def generate_stack_template(stack_type, stack_name, if owner: template_dict = insert_stack_owner_output(template_dict, owner) + template_dict = insert_stack_instance_tags(template_dict, + stack_name, + owner) + return json.dumps(template_dict) @@ -145,6 +157,39 @@ def insert_stack_owner_output(template_dict, owner): return template_dict +def insert_stack_instance_tags(template_dict, environment, owner): + """ Updates tags of each instance resource to include gonzo defaults + """ + resource_type = get_current_cloud().stack_class.instance_type + + resources = template_dict.get('Resources', {}) + for resource_name, resource_details in resources.items(): + if resource_details['Type'] != resource_type: + # We only care about tagging instances + continue + + # Ensure proper tag structure exists + if 'Properties' not in resource_details: + resource_details['Properties'] = {} + if 'Tags' not in resource_details['Properties']: + resource_details['Properties']['Tags'] = [] + + existing_tags = resource_details['Properties']['Tags'] + default_tags = get_default_instance_tags_dict( + environment, resource_name, owner) + + # Remove potential duplicates + for existing_tag in existing_tags: + if existing_tag['Key'] in default_tags.keys(): + existing_tags.remove(existing_tag) + + # Add defaults + for key, value in default_tags.iteritems(): + existing_tags.append({'Key': key, 'Value': value}) + + return template_dict + + def launch_stack(stack_name, template_uri, template_params, owner=None): """ Launch stacks """ diff --git a/tests/scripts/stack/test_ownership.py b/tests/scripts/stack/test_ownership.py index 1024c37..7a0f082 100644 --- a/tests/scripts/stack/test_ownership.py +++ b/tests/scripts/stack/test_ownership.py @@ -9,7 +9,9 @@ def test_ownership(get_parsed_doc): template = """ { "Resources" : { - "existing_resource": {} + "existing_resource": { + "Type": "moot" + } }, "Outputs" : { From fb798c7a885b30d8090cb92f663abf791ad3a748 Mon Sep 17 00:00:00 2001 From: Tom Maddox Date: Fri, 14 Feb 2014 13:28:32 +0000 Subject: [PATCH 2/4] Stack instance tagging tests --- tests/scripts/stack/test_tagging.py | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/scripts/stack/test_tagging.py diff --git a/tests/scripts/stack/test_tagging.py b/tests/scripts/stack/test_tagging.py new file mode 100644 index 0000000..d82696a --- /dev/null +++ b/tests/scripts/stack/test_tagging.py @@ -0,0 +1,69 @@ +import json +from mock import patch, Mock +from gonzo.backends import generate_stack_template + + +@patch("gonzo.backends.get_default_instance_tags_dict") +@patch("gonzo.backends.get_parsed_document") +def test_instance_tagging(get_parsed_doc, get_tags_dict, cloud_fixture): + tags_dict = { + "environment": "overwritten", + "extra": "value", + } + get_tags_dict.return_value = tags_dict + + (get_cloud, get_hostname, create_security_group) = cloud_fixture + cloud = get_cloud.return_value + cloud.stack_class = Mock(instance_type="instance-type") + + template = """ +{ + "Resources" : { + "decoy-resource": { + "Type": "not-interesting-type", + "Properties": {} + }, + "no-other-tags": { + "Type": "instance-type", + "Properties": {} + }, + "existing-tags": { + "Type": "instance-type", + "Properties": { + "Tags": [ + { + "Key": "decoy", + "Value": "leave me be" + }, + { + "Key": "environment", + "Value": "overwrite me" + } + ] + } + } + } +}""" + get_parsed_doc.return_value = template + + template = generate_stack_template(None, None, "template_uri", None, + owner="test-user") + template_dict = json.loads(template) + + resources = template_dict.get("Resources", None) + + assert resources is not None + + decoy_resource = resources.get("decoy-resource", None) + assert decoy_resource is not None + assert decoy_resource['Properties'] == {} + + no_other_tags_resource = resources.get("no-other-tags", None) + no_other_tags_tags = no_other_tags_resource['Properties']['Tags'] + for key, value in tags_dict.items(): + assert {'Key': key, 'Value': value} in no_other_tags_tags + + existing_tags_resource = resources.get("existing-tags", None) + existing_tags_tags = existing_tags_resource['Properties']['Tags'] + for key, value in tags_dict.items(): + assert {'Key': key, 'Value': value} in existing_tags_tags From 6755bb3dbdfe675126450fab8c911a4ab5cc8661 Mon Sep 17 00:00:00 2001 From: Tom Maddox Date: Fri, 14 Feb 2014 14:33:00 +0000 Subject: [PATCH 3/4] Some refactoring --- gonzo/backends/__init__.py | 111 ++------------------------ gonzo/backends/instance_utils.py | 29 +++++++ gonzo/backends/template_utils.py | 74 +++++++++++++++++ gonzo/scripts/instance/launch.py | 3 +- tests/scripts/stack/test_launch.py | 2 +- tests/scripts/stack/test_ownership.py | 7 +- tests/scripts/stack/test_tagging.py | 17 ++-- 7 files changed, 124 insertions(+), 119 deletions(-) create mode 100644 gonzo/backends/instance_utils.py create mode 100644 gonzo/backends/template_utils.py diff --git a/gonzo/backends/__init__.py b/gonzo/backends/__init__.py index 090ef6c..b10cd06 100644 --- a/gonzo/backends/__init__.py +++ b/gonzo/backends/__init__.py @@ -1,7 +1,9 @@ -import json from gonzo.aws.route53 import Route53 from gonzo.config import config_proxy as config from gonzo.helpers.document_loader import get_parsed_document +from gonzo.backends.instance_utils import add_default_security_groups +from gonzo.backends.template_utils import (generate_stack_template, + get_default_instance_tags_dict) def get_current_cloud(): @@ -29,19 +31,6 @@ def get_next_hostname(env_type): return name -def get_default_instance_tags_dict(environment, server_type, owner=None): - """ Generate a tag dictionary suitable for identifying instance ownership - and for identifying instances for gonzo releases. - """ - default_tags = { - 'environment': environment, - 'server_type': server_type, - } - if owner: - default_tags['owner'] = owner - return default_tags - - def launch_instance(env_type, size=None, user_data_uri=None, user_data_params=None, security_groups=None, extra_tags=None, owner=None): @@ -95,111 +84,21 @@ def launch_instance(env_type, size=None, user_data=user_data, tags=tags) -def add_default_security_groups(server_type, additional_security_groups=None): - # Set defaults - security_groups = [server_type, 'gonzo'] - - # Add argument passed groups - if additional_security_groups is not None: - security_groups += additional_security_groups - - # Remove Duplicates - security_groups = list(set(security_groups)) - - return security_groups - - def create_if_not_exist_security_group(group_name): cloud = get_current_cloud() if not cloud.security_group_exists(group_name): cloud.create_security_group(group_name) -def configure_instance(instance): - instance.create_dns_entry() - - -def generate_stack_template(stack_type, stack_name, - template_uri, template_params, - owner=None): - template_uri = config.get_namespaced_cloud_config_value( - 'ORCHESTRATION_TEMPLATE_URIS', stack_type, override=template_uri) - if template_uri is None: - raise ValueError('A template must be specified by argument or ' - 'in config') - - template = get_parsed_document(stack_name, template_uri, - 'ORCHESTRATION_TEMPLATE_PARAMS', - template_params) - - # Parse as json for validation and for injecting gonzo defaults - template_dict = json.loads(template) - - if owner: - template_dict = insert_stack_owner_output(template_dict, owner) - - template_dict = insert_stack_instance_tags(template_dict, - stack_name, - owner) - - return json.dumps(template_dict) - - -def insert_stack_owner_output(template_dict, owner): - """ Adds a stack output to a template with key "owner" """ - template_outputs = template_dict.get('Outputs', {}) - template_outputs['owner'] = { - 'Value': owner, - 'Description': "This stack's launcher (Managed by Gonzo)" - } - template_dict.update({'Outputs': template_outputs}) - - return template_dict - - -def insert_stack_instance_tags(template_dict, environment, owner): - """ Updates tags of each instance resource to include gonzo defaults - """ - resource_type = get_current_cloud().stack_class.instance_type - - resources = template_dict.get('Resources', {}) - for resource_name, resource_details in resources.items(): - if resource_details['Type'] != resource_type: - # We only care about tagging instances - continue - - # Ensure proper tag structure exists - if 'Properties' not in resource_details: - resource_details['Properties'] = {} - if 'Tags' not in resource_details['Properties']: - resource_details['Properties']['Tags'] = [] - - existing_tags = resource_details['Properties']['Tags'] - default_tags = get_default_instance_tags_dict( - environment, resource_name, owner) - - # Remove potential duplicates - for existing_tag in existing_tags: - if existing_tag['Key'] in default_tags.keys(): - existing_tags.remove(existing_tag) - - # Add defaults - for key, value in default_tags.iteritems(): - existing_tags.append({'Key': key, 'Value': value}) - - return template_dict - - def launch_stack(stack_name, template_uri, template_params, owner=None): """ Launch stacks """ unique_stack_name = get_next_hostname(stack_name) - + cloud = get_current_cloud() template = generate_stack_template(stack_name, unique_stack_name, template_uri, template_params, + cloud.stack_class.instance_type, owner) - - cloud = get_current_cloud() return cloud.launch_stack(unique_stack_name, template) diff --git a/gonzo/backends/instance_utils.py b/gonzo/backends/instance_utils.py new file mode 100644 index 0000000..09c7fd9 --- /dev/null +++ b/gonzo/backends/instance_utils.py @@ -0,0 +1,29 @@ +def get_default_instance_tags_dict(environment, server_type, owner=None): + """ Generate a tag dictionary suitable for identifying instance ownership + and for identifying instances for gonzo releases. + """ + default_tags = { + 'environment': environment, + 'server_type': server_type, + } + if owner: + default_tags['owner'] = owner + return default_tags + + +def add_default_security_groups(server_type, additional_security_groups=None): + # Set defaults + security_groups = [server_type, 'gonzo'] + + # Add argument passed groups + if additional_security_groups is not None: + security_groups += additional_security_groups + + # Remove Duplicates + security_groups = list(set(security_groups)) + + return security_groups + + +def configure_instance(instance): + instance.create_dns_entry() diff --git a/gonzo/backends/template_utils.py b/gonzo/backends/template_utils.py new file mode 100644 index 0000000..aed05f5 --- /dev/null +++ b/gonzo/backends/template_utils.py @@ -0,0 +1,74 @@ +import json + +from gonzo.config import config_proxy as config +from gonzo.helpers.document_loader import get_parsed_document +from gonzo.backends.instance_utils import get_default_instance_tags_dict + + +def generate_stack_template(stack_type, stack_name, + template_uri, template_params, + instance_resource_type, owner=None): + template_uri = config.get_namespaced_cloud_config_value( + 'ORCHESTRATION_TEMPLATE_URIS', stack_type, override=template_uri) + if template_uri is None: + raise ValueError('A template must be specified by argument or ' + 'in config') + + template = get_parsed_document(stack_name, template_uri, + 'ORCHESTRATION_TEMPLATE_PARAMS', + template_params) + + # Parse as json for validation and for injecting gonzo defaults + template_dict = json.loads(template) + + if owner: + template_dict = insert_stack_owner_output(template_dict, owner) + + template_dict = insert_stack_instance_tags( + template_dict, instance_resource_type, stack_name, owner) + + return json.dumps(template_dict) + + +def insert_stack_owner_output(template_dict, owner): + """ Adds a stack output to a template with key "owner" """ + template_outputs = template_dict.get('Outputs', {}) + template_outputs['owner'] = { + 'Value': owner, + 'Description': "This stack's launcher (Managed by Gonzo)" + } + template_dict.update({'Outputs': template_outputs}) + + return template_dict + + +def insert_stack_instance_tags(template_dict, instance_resource_type, + environment, owner): + """ Updates tags of each instance resource to include gonzo defaults + """ + resources = template_dict.get('Resources', {}) + for resource_name, resource_details in resources.items(): + if resource_details['Type'] != instance_resource_type: + # We only care about tagging instances + continue + + # Ensure proper tag structure exists + if 'Properties' not in resource_details: + resource_details['Properties'] = {} + if 'Tags' not in resource_details['Properties']: + resource_details['Properties']['Tags'] = [] + + existing_tags = resource_details['Properties']['Tags'] + default_tags = get_default_instance_tags_dict( + environment, resource_name, owner) + + # Remove potential duplicates + for existing_tag in existing_tags: + if existing_tag['Key'] in default_tags.keys(): + existing_tags.remove(existing_tag) + + # Add defaults + for key, value in default_tags.iteritems(): + existing_tags.append({'Key': key, 'Value': value}) + + return template_dict diff --git a/gonzo/scripts/instance/launch.py b/gonzo/scripts/instance/launch.py index 60aa6e1..c0c6416 100755 --- a/gonzo/scripts/instance/launch.py +++ b/gonzo/scripts/instance/launch.py @@ -7,8 +7,9 @@ import sys from time import sleep -from gonzo.backends import launch_instance, configure_instance +from gonzo.backends import launch_instance from gonzo.exceptions import CommandError, DataError +from gonzo.backends.instance_utils import configure_instance from gonzo.scripts.utils import colorize from gonzo.utils import abort, csv_dict, csv_list diff --git a/tests/scripts/stack/test_launch.py b/tests/scripts/stack/test_launch.py index fff3435..0fbce4b 100644 --- a/tests/scripts/stack/test_launch.py +++ b/tests/scripts/stack/test_launch.py @@ -7,7 +7,7 @@ @patch('gonzo.scripts.stack.launch.print_stack') @patch('gonzo.scripts.stack.launch.wait_for_stack_complete') -@patch('gonzo.backends.get_parsed_document') +@patch('gonzo.backends.template_utils.get_parsed_document') def test_launch(get_parsed_doc, wait_for_stack_complete, print_stack, cloud_fixture, minimum_config_fixture): (get_cloud, get_hostname, create_security_group) = cloud_fixture diff --git a/tests/scripts/stack/test_ownership.py b/tests/scripts/stack/test_ownership.py index 7a0f082..883a165 100644 --- a/tests/scripts/stack/test_ownership.py +++ b/tests/scripts/stack/test_ownership.py @@ -1,9 +1,11 @@ import json + from mock import patch -from gonzo.backends import generate_stack_template + +from gonzo.backends.template_utils import generate_stack_template -@patch("gonzo.backends.get_parsed_document") +@patch("gonzo.backends.template_utils.get_parsed_document") def test_ownership(get_parsed_doc): template = """ @@ -24,6 +26,7 @@ def test_ownership(get_parsed_doc): get_parsed_doc.return_value = template template = generate_stack_template(None, None, "template_uri", None, + instance_resource_type="", owner="test-user") template_dict = json.loads(template) diff --git a/tests/scripts/stack/test_tagging.py b/tests/scripts/stack/test_tagging.py index d82696a..5696f45 100644 --- a/tests/scripts/stack/test_tagging.py +++ b/tests/scripts/stack/test_tagging.py @@ -1,21 +1,19 @@ import json -from mock import patch, Mock -from gonzo.backends import generate_stack_template +from mock import patch -@patch("gonzo.backends.get_default_instance_tags_dict") -@patch("gonzo.backends.get_parsed_document") -def test_instance_tagging(get_parsed_doc, get_tags_dict, cloud_fixture): +from gonzo.backends.template_utils import generate_stack_template + + +@patch("gonzo.backends.template_utils.get_default_instance_tags_dict") +@patch("gonzo.backends.template_utils.get_parsed_document") +def test_instance_tagging(get_parsed_doc, get_tags_dict): tags_dict = { "environment": "overwritten", "extra": "value", } get_tags_dict.return_value = tags_dict - (get_cloud, get_hostname, create_security_group) = cloud_fixture - cloud = get_cloud.return_value - cloud.stack_class = Mock(instance_type="instance-type") - template = """ { "Resources" : { @@ -47,6 +45,7 @@ def test_instance_tagging(get_parsed_doc, get_tags_dict, cloud_fixture): get_parsed_doc.return_value = template template = generate_stack_template(None, None, "template_uri", None, + instance_resource_type="instance-type", owner="test-user") template_dict = json.loads(template) From a57f45f2684e96c587693585caa9d5789972eb0d Mon Sep 17 00:00:00 2001 From: Tom Maddox Date: Fri, 14 Feb 2014 14:34:31 +0000 Subject: [PATCH 4/4] Indentation error --- gonzo/backends/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gonzo/backends/__init__.py b/gonzo/backends/__init__.py index b10cd06..c756359 100644 --- a/gonzo/backends/__init__.py +++ b/gonzo/backends/__init__.py @@ -3,7 +3,7 @@ from gonzo.helpers.document_loader import get_parsed_document from gonzo.backends.instance_utils import add_default_security_groups from gonzo.backends.template_utils import (generate_stack_template, - get_default_instance_tags_dict) + get_default_instance_tags_dict) def get_current_cloud():